diff --git a/README.Ubuntu.bash b/README.Ubuntu.bash index 2a383b9d8d31..543af4a6536d 100755 --- a/README.Ubuntu.bash +++ b/README.Ubuntu.bash @@ -43,6 +43,7 @@ apt-get install -y \ python3-psycopg2 \ python3-psutil \ python3-yaml \ + python3-transitions \ zlib1g-dev tee -a /etc/sysctl.conf << EOF diff --git a/gpMgmt/bin/Makefile b/gpMgmt/bin/Makefile index c05171049828..9d2dbc1d7065 100644 --- a/gpMgmt/bin/Makefile +++ b/gpMgmt/bin/Makefile @@ -7,7 +7,7 @@ ifneq "$(wildcard $(top_builddir)/src/Makefile.global)" "" include $(top_builddir)/src/Makefile.global endif -SUBDIRS = stream gpcheckcat_modules gpconfig_modules gpssh_modules gppylib lib +SUBDIRS = stream gpcheckcat_modules gpconfig_modules gpssh_modules gppylib lib gprebalance_modules SUBDIRS += ifaddrs $(recurse) @@ -16,7 +16,8 @@ PROGRAMS= analyzedb gpactivatestandby gpaddmirrors gpcheckcat gpcheckperf \ gpcheckresgroupimpl gpconfig gpdeletesystem gpexpand gpinitstandby \ gpinitsystem gpload gpload.py gplogfilter gpmovemirrors \ gprecoverseg gpreload gpsync gpsd gpssh gpssh-exkeys gpstart \ - gpstate gpstop minirepro gpmemwatcher gpmemreport gpcheckresgroupv2impl + gpstate gpstop minirepro gpmemwatcher gpmemreport gpcheckresgroupv2impl \ + gprebalance ggrebalance installdirs: $(MKDIR_P) '$(DESTDIR)$(bindir)/lib' @@ -109,7 +110,8 @@ unitdevel: @echo "Running pure unit tests..." PYTHONPATH=$(SERVER_SRC):$(SERVER_SBIN):$(PYTHONPATH):$(PYTHONSRC_INSTALL_PYTHON_PATH):$(SRC)/ext:$(SBIN_DIR):$(LIB_DIR):$(PYLIB_DIR)/mock-1.0.1 \ python3 -m unittest discover --verbose -s $(SRC)/gppylib -p "test_unit*.py" - + PYTHONPATH=$(SERVER_SRC):$(SERVER_SBIN):$(PYTHONPATH):$(PYTHONSRC_INSTALL_PYTHON_PATH):$(SRC)/ext:$(SBIN_DIR):$(LIB_DIR):$(PYLIB_DIR)/mock-1.0.1 \ + python3 -m unittest discover --verbose -s $(SRC)/gprebalance_modules -t $(SRC) .PHONY: installcheck-bash installcheck-bash: diff --git a/gpMgmt/bin/ggrebalance b/gpMgmt/bin/ggrebalance new file mode 100755 index 000000000000..05837ee8ec5c --- /dev/null +++ b/gpMgmt/bin/ggrebalance @@ -0,0 +1,365 @@ +#!/usr/bin/env python3 + +from gppylib.mainUtils import getProgramName + +from psycopg2 import DatabaseError +try: + from gppylib.commands.unix import * + from gppylib.commands.gp import * + from gppylib.gpparseopts import OptParser, OptChecker + from gppylib.gplog import * + from gppylib.gparray import GpArray + from gppylib.db import dbconn + from gppylib.userinput import * + from gppylib.system.environment import GpCoordinatorEnvironment + from gppylib.system import configurationInterface, configurationImplGpdb + from gprebalance_modules.shrink import GGShrink, DBNAME + from gprebalance_modules.ggrebalance_main_sm import GGRebalanceMainSM + from gprebalance_modules.planner import * + from gppylib.programs.clsRecoverSegment_triples import get_segments_with_running_basebackup +except ImportError as e: + sys.exit('ERROR: Cannot import modules. Please check that you have sourced greenplum_path.sh. Detail: ' + str(e)) + + +_description = (""" + TBD. Curretnly only minimum shrink implementation is implemented. +""") + +_help = [""" +TBD +"""] + +_usage = """ + TBD +""" + +EXECNAME = os.path.split(__file__)[-1] +MAX_BATCH_SIZE = 128 +MAX_PARALLEL_WORKERS = 96 +MAX_SEGMENTS = 4096 + +def parseargs(): + parser = OptParser(option_class=OptChecker, + description=' '.join(_description.split()), + version='%prog version $Revision$') + parser.setHelp(_help) + parser.set_usage('%prog ' + _usage) + parser.remove_option('-h') + + parser.add_option('-x', '--target-segment-count', type='int', dest='target_segment_count', metavar='', + help='Target segments number of the whole cluster') + parser.add_option('-B', '--batch-size', type='int', default=4, dest='batch_size', metavar='', + help='Max worker number in WorkerPool instance. Valid values are 1-%d.' % MAX_BATCH_SIZE) + parser.add_option('-n', '--parallel', type="int", default=4, dest="parallel", metavar='', + help='Max number of workers at a time. Valid values are 1-%d.' % MAX_PARALLEL_WORKERS) + parser.add_option('-c', '--clean', action='store_true', dest="clean_required", + help='remove the rebalance schema.') + parser.add_option('-r', '--rollback', action='store_true', dest="rollback_required", + help='performs rebalance rollback if possible.') + parser.add_option('-h', '-?', '--help', action='help', + help='show this help message and exit.') + parser.add_option('-y', '--non-interactive-mode', dest="interactive", action='store_false', default=True, + help="non-interactive mode, do not require user input for confirmations, use defaults.") + parser.add_option('-H', '--target-hosts', dest='target_hosts', metavar='', + help='Set of hostnames where rebalance will distribute segments to') + parser.add_option('-A', '--add-hosts', dest='add_hosts', metavar='', + help='Hosts extending the configuration') + parser.add_option('-R', '--remove-hosts', dest='remove_hosts', metavar='', + help='Decomissioned hosts') + parser.add_option('-m', '--mirror-mode', dest='mirror_mode', metavar='', + help='Desirable mirroring strategy after rebalance.') + parser.add_option('--skip-rebalance', dest='skip_rebalance', metavar='', action='store_true', default=False, + help='Perform shrink or expand only.') + parser.add_option('-p', '--show-plan', dest='show_plan', metavar='', action='store_true', default=False, + help='Show rebalance plan') + parser.add_option('-d', '--target-datadirs', dest='target_datadirs', metavar='', + help='segment datadirs on new hosts') + parser.add_option('--target-hosts-file', dest='target_hosts_file', metavar='', + help='Target hosts list file. One host per line') + parser.add_option('--target-datadirs-file', dest='target_datadirs_file', metavar='', + help='Target segment data directories passed as file. One datadir per line') + parser.add_option('--add-hosts-file', dest='add_hosts_file', metavar='', + help='Hosts that user wants to add to a cluster. One host per line') + parser.add_option('--remove-hosts-file', dest='remove_hosts_file', metavar='', + help='Hosts that user wants to remove from a cluster. One host per line') + parser.add_option('--skip-resource-estimation', dest='skip_resource_estimation', metavar='', + action='store_true', default=False, help='Skip resource estimation (storage)') + parser.add_option('-l', '--log-dir', dest='logfile_directory', metavar='', + help='The directory to write the log files. The default location is ~/gpAdminLogs.') + parser.add_option('-a', '--analyze', dest="analyze", action='store_true', default=False, + help='Run ANALYZE after rebalance tables redistribution.' + + 'Ignored if no shrink or expand had been performed.') + parser.add_option('--hba-hostnames', dest="hba_hostnames", action='store_true', default=False, + help='Use host names when updating the pg_hba.conf file.') + parser.add_option('--replay-lag', dest="replay_lag", type="float", + help='Replay lag(in GBs) allowed on mirror when rebalancing the segments.') + parser.add_option('-v', '--verbose', action='store_true', default=False, + help='debug output.') + parser.add_option('--inplace-swap-roles', dest="inplace_swap_roles", action='store_true', default=False, + help='Allows to place primary and mirror at the same host during rebalance.') + + # Parse the command line arguments + options, args = parser.parse_args() + return options, args, parser + + +def validate_options(options, args, parser): + if len(args) > 0: + logger.error(f'Unknown argument {args[0]}') + parser.exit(1) + + try: + options.coordinator_data_directory = get_coordinatordatadir() + options.gphome = get_gphome() + except GpError as msg: + logger.error(msg) + parser.exit(1) + + if not os.path.exists(options.coordinator_data_directory): + logger.error('Coordinator data directory %s does not exist.' % + options.coordinator_data_directory) + parser.exit(1) + + if options.batch_size < 1 or options.batch_size > MAX_BATCH_SIZE: + logger.error( + 'Invalid argument. batch-size value must be >= 1 and <= %d' % MAX_BATCH_SIZE) + parser.exit(1) + + if options.parallel < 1 or options.parallel > MAX_PARALLEL_WORKERS: + logger.error( + 'Invalid argument. parallel value must be >= 1 and <= %d' % MAX_PARALLEL_WORKERS) + parser.exit(1) + + if options.target_segment_count and ( + options.target_segment_count < 1 + or options.target_segment_count > MAX_SEGMENTS): + logger.error( + 'Invalid value of --target-segment-count. It must be >= 1 and <= %d' % MAX_SEGMENTS) + parser.exit(1) + + incompatibility_rules = [ + # (condition_option, incompatible_options) + ('target_hosts', ['add_hosts', 'add_hosts_file', 'remove_hosts', 'remove_hosts_file', 'target_hosts_file']), + ('target_hosts_file', ['add_hosts', 'add_hosts_file', 'remove_hosts', 'remove_hosts_file']), + ('add_hosts', ['add_hosts_file', 'remove_hosts', 'remove_hosts_file']), + ('add_hosts_file', ['remove_hosts', 'remove_hosts_file']), + ('remove_hosts', ['remove_hosts_file']), + + ('target_datadirs', ['target_datadirs_file']), + + ('skip_rebalance', ['mirror_mode', 'inplace_swap_roles']), + + ('clean_required', [ + 'target_hosts', 'target_hosts_file', 'add_hosts', 'add_hosts_file', + 'remove_hosts', 'remove_hosts_file', 'target_datadirs', 'target_datadirs_file', + 'target_segment_count', 'mirror_mode', 'skip_rebalance', 'show_plan', + 'skip_resource_estimation', 'analyze', 'replay_lag', 'hba_hostnames', 'inplace_swap_roles' + ]), + + ('rollback_required', [ + 'target_hosts', 'target_hosts_file', 'add_hosts', 'add_hosts_file', + 'remove_hosts', 'remove_hosts_file', 'target_datadirs', 'target_datadirs_file', + 'target_segment_count', 'mirror_mode', 'skip_rebalance', 'show_plan', + 'skip_resource_estimation', 'duration', 'end', 'inplace_swap_roles' + ]), + ] + + def option_is_set(opt_name): + """Check if option is set""" + if not hasattr(options, opt_name): + return False + value = getattr(options, opt_name) + return value is not None and value != False + + for rule in incompatibility_rules: + condition_opt, incompatible_opts = rule + if option_is_set(condition_opt): + # Check if any incompatible option is set + for incomp_opt in incompatible_opts: + if option_is_set(incomp_opt): + error_msg = \ + f"Can't use together options '--{condition_opt.replace('_', '-')}' and '--{incomp_opt.replace('_', '-')}'" + logger.error(error_msg) + parser.exit(1) + + if (option_is_set('target_hosts') or option_is_set('add_hosts')) and not option_is_set('target_datadirs'): + logger.error("--target-datadirs option is required when using --target-hosts or --add-hosts") + parser.exit(1) + + if option_is_set('mirror_mode') and options.mirror_mode not in ('grouped', 'spread'): + logger.error('Mirroring strategy %s is not supported' % options.mirror_mode) + parser.exit(1) + + return options, args + + +def check_running_gputils(dburl: dbconn.DbURL, coordinator_data_directory: str): + """ + Checks if there are any running instances of + gprebalance/gpbackup/gpexpand/gpshrink/gpresize + """ + for util in ('ggrebalance', 'gprebalance', 'gpexpand', 'gpshrink', 'gpresize'): + try: + with open(f'{coordinator_data_directory}/{util}.pid', 'r') as fp: + pid = int(fp.readline().strip()) + if check_pid(pid): + logger.error(f'{util} is already running.') + sys.exit(1) + except IOError: + pass + + gpexpand_status_file = 'gpexpand.status' + if os.path.exists(os.path.join(coordinator_data_directory, gpexpand_status_file)): + logger.error(f'{gpexpand_status_file} file exists. Assuming gpexpand is already running.') + sys.exit(1) + + with closing(dbconn.connect(dburl, encoding='UTF8')) as conn: + cursor = dbconn.query(conn, "SELECT oid FROM pg_namespace WHERE nspname='gpexpand'") + if cursor.rowcount > 0: + logger.error( + '''gpexpand schema exists. Assuming gpexpand is already running.''') + sys.exit(1) + + cursor = dbconn.query(conn, '''SELECT datid + FROM pg_stat_activity + WHERE application_name LIKE 'gpbackup%' OR + application_name LIKE 'gprestore%' ''') + if cursor.rowcount > 0: + logger.error( + '''gpbackup/gprestore utility is already running.''') + sys.exit(1) + + if is_gprecoverseg_running(): + logger.error( + '''gprecoverseg is already running.''') + sys.exit(1) + + segments_with_running_basebackup = get_segments_with_running_basebackup() + if len(segments_with_running_basebackup) > 0: + logger.error( + f'''Segments {segments_with_running_basebackup} have running pg_basebackup.''') + sys.exit(1) + + +def create_pid_file(coordinator_data_directory: str): + """ + Creates ggrebalance pid file + """ + with open(coordinator_data_directory + '/ggrebalance.pid', 'w') as fp: + fp.write(str(os.getpid())) + + +def remove_pid_file(coordinator_data_directory: str): + """ + Removes gprebalance pid file + """ + try: + os.unlink(coordinator_data_directory + '/ggrebalance.pid') + except IOError: + pass + +gg_main_rebalance_SM = None +cmd_recoverseg = None + +def sig_handler(sig, arg): + if gg_main_rebalance_SM is not None: + gg_main_rebalance_SM.shutdown() + else: + if cmd_recoverseg != None: + cmd_recoverseg.cancel() + sys.exit(1) + +def check_down_segments(logger: Any, options: Any, dburl: dbconn.DbURL): + # check if some of the segments are down - it can happen if we're recovering from an + # interrupted state, and such segments are left by the interrupted gprecoverseg + conn = dbconn.connect(dburl, encoding='UTF8', allowSystemTableMods=True) + dbconn.execSQL(conn, "SELECT gp_request_fts_probe_scan()") + cnt_primaries_down = int(dbconn.queryRow(conn, f"SELECT COUNT(1) FROM gp_segment_configuration WHERE role = 'p' AND status = 'd'")[0]) + cnt_mirrors_down = int(dbconn.queryRow(conn, f"SELECT COUNT(1) FROM gp_segment_configuration WHERE role = 'm' AND status = 'd'")[0]) + conn.close() + + if cnt_primaries_down != 0: + raise Exception('Detected some primary segments are down, please recover manually') + + if cnt_mirrors_down != 0: + logger.info("Some mirrors are down, trying to recover them, it may take some time...") + recoverseg_options = "-a -F" + if options.logfile_directory is not None: + recoverseg_options = recoverseg_options + f' -l "{str(options.logfile_directory)}"' + global cmd_recoverseg + try: + cmd_recoverseg = GpRecoverSeg("Running gprecoverseg", options=recoverseg_options) + cmd_recoverseg.run(validateAfter=True) + except Exception as e: + logger.error(str(e)) + error_msg = f"Failed to execute 'gprecoverseg {recoverseg_options}'" + raise Exception(error_msg) + finally: + cmd_recoverseg = None + +def main(options, args, parser): + conn = None + try: + signal.signal(signal.SIGTERM, sig_handler) + signal.signal(signal.SIGHUP, sig_handler) + signal.signal(signal.SIGINT, sig_handler) + + logger = get_default_logger() + setup_tool_logging(EXECNAME, getLocalHostname(), getUserName(), options.logfile_directory) + + options, args = validate_options(options, args, parser) + + if options.verbose: + enable_verbose_logging() + + gpenv = GpCoordinatorEnvironment( + options.coordinator_data_directory, True) + + # can we remove it? + configurationInterface.registerConfigurationProvider( + configurationImplGpdb.GpConfigurationProviderUsingGpdbCatalog()) + + configurationInterface.getConfigurationProvider( + ).initializeProvider(gpenv.getCoordinatorPort()) + + dburl = dbconn.DbURL(dbname=DBNAME, port=gpenv.getCoordinatorPort()) + + check_running_gputils(dburl, options.coordinator_data_directory) + + create_pid_file(options.coordinator_data_directory) + + gparray_dump_filename = options.coordinator_data_directory + '/gparraydump' + + check_down_segments(logger, options, dburl) + + logger.info('Init gparray from catalog') + try: + gparray = GpArray.initFromCatalog(dburl, utility=True) + except DatabaseError as ex: + logger.error("Failed to connect to database. Make sure the" + " Greengage instance you wish to expand is running" + " and that your environment is correct, then rerun" + " gprebalance" + ' '.join(sys.argv[1:])) + sys.exit(1) + except ConnectionError as ex: + logger.error(f"{str(ex)}") + sys.exit(1) + + conn = dbconn.connect(dburl, encoding='UTF8', allowSystemTableMods=True) + + global gg_main_rebalance_SM + gg_main_rebalance_SM = GGRebalanceMainSM(conn, logger, dburl, options, gpenv, gparray, gparray_dump_filename) + gg_main_rebalance_SM.run() + + sys.exit(0) + except Exception as e: + logger.error(f'ggrebalance failed: {e} \n\nExiting...') + sys.exit(1) + finally: + if conn != None: + conn.close() + remove_pid_file(get_coordinatordatadir()) + +if __name__ == '__main__': + options, args, parser = parseargs() + main(options, args, parser) diff --git a/gpMgmt/bin/gpmovemirrors b/gpMgmt/bin/gpmovemirrors index 11e459469206..463a7d692c99 100755 --- a/gpMgmt/bin/gpmovemirrors +++ b/gpMgmt/bin/gpmovemirrors @@ -400,7 +400,7 @@ try: recoversegOptions = "-i " + newConfig.inputFile + " -B " + str(options.batch_size) + \ " -b " + str(options.segment_batch_size) + " -v -a -d " + options.coordinator_data_directory if options.logfile_directory != None: - recoversegOptions = recoversegOptions + " -l " + str(options.logfile_directory) + recoversegOptions = recoversegOptions + f' -l "{str(options.logfile_directory)}"' logger.info('About to run gprecoverseg with options: ' + recoversegOptions) cmd = GpRecoverSeg("Running gprecoverseg", options=recoversegOptions) cmd.run(validateAfter=True) diff --git a/gpMgmt/bin/gppylib/Makefile b/gpMgmt/bin/gppylib/Makefile index 04972f5345fe..cfe4ef59f9fe 100644 --- a/gpMgmt/bin/gppylib/Makefile +++ b/gpMgmt/bin/gppylib/Makefile @@ -10,6 +10,7 @@ $(recurse) PROGRAMS= gparray.py gpunit unit2 DATA= __init__.py \ + fault_injection.py \ datetimeutils.py \ gp_era.py \ gpcatalog.py \ diff --git a/gpMgmt/bin/gppylib/commands/gp.py b/gpMgmt/bin/gppylib/commands/gp.py index 034e694f2af2..f6b7ed70b42e 100644 --- a/gpMgmt/bin/gppylib/commands/gp.py +++ b/gpMgmt/bin/gppylib/commands/gp.py @@ -453,7 +453,7 @@ def __init__(self,name,directory,ctxt=LOCAL,remoteHost=None): def is_shutdown(self): for key, value in self.results.split_stdout(): if key == 'Database cluster state': - return value.strip() == 'shut down' + return value.strip() == 'shut down' or value.strip() == 'shut down in recovery' return False @staticmethod @@ -1688,6 +1688,20 @@ def __init__(self, name, options = "", ctxt = LOCAL, remoteHost = None): cmdStr = "$GPHOME/bin/gprecoverseg %s" % (options) Command.__init__(self,name,cmdStr,ctxt,remoteHost) +class GpMoveMirrors(Command): + """ + This command will execute the gpmovemirror utility + """ + + def __init__(self, name, options = "", ctxt = LOCAL, remoteHost = None): + self.name = name + self.options = options + self.ctxt = ctxt + self.remoteHost = remoteHost + + cmdStr = "$GPHOME/bin/gpmovemirrors %s" % (options) + Command.__init__(self,name,cmdStr,ctxt,remoteHost) + class IfAddrs: @staticmethod def list_addrs(hostname=None, include_loopback=False): diff --git a/gpMgmt/bin/gppylib/commands/unix.py b/gpMgmt/bin/gppylib/commands/unix.py index d78e4b63db67..9ce764a3c0bf 100644 --- a/gpMgmt/bin/gppylib/commands/unix.py +++ b/gpMgmt/bin/gppylib/commands/unix.py @@ -691,6 +691,29 @@ def get_hostname(self): raise Exception('Command not yet executed') return self.results.stdout.strip() +# --------------port is not busy-------------------- +class PortIsAvailable(Command): + def __init__(self, name, port, ctxt=REMOTE, remoteHost=None): + self.port = port + # Check if port is listening: return code 0 if is in use, + # 1 if is available + cmdStr = ( + f"(command -v ss >/dev/null 2>&1 && " + f"ss -tuln | grep -q ':{port} ') && " + f"echo 'IN_USE' || echo 'AVAILABLE'" + ) + Command.__init__(self, name, cmdStr, ctxt, remoteHost) + + def is_port_available(self) -> bool: + """ + Check if port is available based on command results + """ + if not self.results: + return False + + output = self.results.stdout.strip() + + return output == 'AVAILABLE' # --------------tcp port is active ----------------------- class PgPortIsActive(Command): diff --git a/gpMgmt/bin/gppylib/fault_injection.py b/gpMgmt/bin/gppylib/fault_injection.py new file mode 100755 index 000000000000..388097634575 --- /dev/null +++ b/gpMgmt/bin/gppylib/fault_injection.py @@ -0,0 +1,41 @@ +#!/usr/bin/env python3 + +import os +import threading +import time +import signal + +GPMGMT_FAULT_POINT = 'GPMGMT_FAULT_POINT' +GPMGMT_FAULT_DELAY_MS = 'GPMGMT_FAULT_DELAY_MS' +GPMGMT_FAULT_TYPE = 'GPMGMT_FAULT_TYPE' +GPMGMT_FAULT_FILE_FLAG = 'GPMGMT_FAULT_FILE_FLAG' + +GPMGMT_FAULT_TYPE_SYSPEND = 'suspend' + +def inject_fault(fault_point): + if GPMGMT_FAULT_POINT in os.environ and fault_point == os.environ[GPMGMT_FAULT_POINT]: + if GPMGMT_FAULT_TYPE in os.environ and os.environ[GPMGMT_FAULT_TYPE] == GPMGMT_FAULT_TYPE_SYSPEND: + while GPMGMT_FAULT_FILE_FLAG in os.environ and os.path.exists(os.environ[GPMGMT_FAULT_FILE_FLAG]): + time.sleep(0.1) + return + if GPMGMT_FAULT_DELAY_MS in os.environ and int(os.environ[GPMGMT_FAULT_DELAY_MS]) > 0: + delay_ms = int(os.environ[GPMGMT_FAULT_DELAY_MS]) + + def raise_exception(delay: int): + time.sleep(delay / 1000) + print('Fault Injection %s' % os.environ[GPMGMT_FAULT_POINT]) + os.kill(os.getpid(), signal.SIGINT) + + thread = threading.Thread(target=raise_exception, kwargs={"delay": delay_ms}) + thread.daemon = True + thread.start() + else: + raise Exception('Fault Injection %s' % os.environ[GPMGMT_FAULT_POINT]) + +# decorator for test purposes +def wrap_state_func_with_faults(func): + def func_with_faults(*args): + inject_fault(f'{func.__name__}_begin') + func(*args) + inject_fault(f'{func.__name__}_end') + return func_with_faults diff --git a/gpMgmt/bin/gprebalance b/gpMgmt/bin/gprebalance new file mode 100755 index 000000000000..1e03b3b7d4f5 --- /dev/null +++ b/gpMgmt/bin/gprebalance @@ -0,0 +1,475 @@ +#!/usr/bin/env python3 + +from gppylib.mainUtils import getProgramName + +import psycopg2 +import yaml +import datetime +from psycopg2 import DatabaseError, OperationalError +try: + from gppylib.commands.unix import * + from gppylib.commands.gp import * + from gppylib.gparray import GpArray, Segment + from gppylib.gpparseopts import OptParser, OptChecker + from gppylib.gplog import * + from gppylib.db import dbconn + from gppylib.userinput import * + from gppylib.system.environment import GpCoordinatorEnvironment + from gppylib.system import configurationInterface, configurationImplGpdb + from gppylib.programs.clsRecoverSegment_triples import get_segments_with_running_basebackup + from gppylib.parseutils import line_reader, check_values, canonicalize_address + from gppylib.operations.update_pg_hba_on_segments import update_pg_hba_on_segments + from gprebalance_modules.rebalance import GPRebalance, MirrorStrategy + from gprebalance_modules.rebalance_validator import ClusterValidator, StateValidationError +except ImportError as e: + sys.exit('ERROR: Cannot import modules. Please check that you have sourced greenplum_path.sh. Detail: ' + str(e)) + + +DBNAME = 'postgres' + +_description = (""" +Rebalances the existing segments configuration for getting +optimal performance of whole cluster. +""") + +_help = [""" +[-m spread|grouped] +Mode is the desirable mirroring strategy after rebalance. Makes sense only when mirroring +is enabled for the cluster. Default value is "grouped". +[-f ] +The hosts configuration file (YAML format) defines the target hosts on which +the rebalance procedure will distribute segments from existing configuration. +Existing info can be generated from gp_segment_configuration through -g option. +hosts: +- hostname: # Machine hostname + address:
# Network address for connections + primary_datadirs: # Directories for primary segments + - /datadir1 + - /datadir1 + mirror_datadirs: # Directories for mirror segments + - /mdatadir1 + - /mdatadir2 +You can include empty hosts in the configuration or shrink the existing number +of hosts, the utility will try to balance the segments strictly across hosts +from the file. +"""] + +_usage = """ + gprebalance -g + + gprebalance [-m ] [-c] [-f ] [-s] [-v] + + gprebalance [-d duration[hh][:mm[:ss]] | [-e 'YYYY-MM-DD hh:mm:ss']] + + gprebalance -r + + gprebalance -c + + gprebalance -p + + gprebalance -? | -h | --help | --verbose | -v +""" + +EXECNAME = os.path.split(__file__)[-1] +MAX_BATCH_SIZE = 128 +MAX_PARALLEL_WORKERS = 96 + + +def parseargs(): + parser = OptParser(option_class=OptChecker, + description=' '.join(_description.split()), + version='%prog version $Revision$') + parser.setHelp(_help) + parser.set_usage('%prog ' + _usage) + parser.remove_option('-h') + + parser.add_option('-m', '--mirror-mode', dest='mirroring', + help='desirable mirroring strategy') + parser.add_option('-c', '--clean', action='store_true', + help='remove the rebalance schema.') + parser.add_option('-f', '--target-hosts', metavar='', dest='filename', + help='yaml containing target hosts configuration') + parser.add_option('-r', '--rollback', action='store_true', + help='bring the cluster back to the state before rebalance') + parser.add_option('-d', '--duration', type='duration', metavar='[h][:m[:s]]', + help='duration from beginning to end.') + parser.add_option('-e', '--end', type='datetime', metavar='datetime', + help="ending date and time in the format 'YYYY-MM-DD hh:mm:ss'.") + parser.add_option('-n', '--parallel', type="int", default=4, metavar="", + help='number of workerks performing segment movements at a time. Valid values are 1-%d.' % MAX_PARALLEL_WORKERS) + parser.add_option('--allow-mirrorless', dest='allow_mirrorless', action='store_true', + help='Allow to rebalance a cluster without mirrors', default=False) + parser.add_option('-g', '--gen-hosts', action='store_true', dest='genconf', + help='dump cluster hosts configuration in yaml format') + parser.add_option('-s', '--silent', action='store_true', default=False, + help='Do not prompt for confirmation to proceed on warnings') + parser.add_option('-p', '--show-plan', dest='show_plan', action='store_true', default=False, + help='show rebalance plan') + parser.add_option('-v', '--verbose', action='store_true', + help='debug output.') + parser.add_option('-h', '-?', '--help', action='help', + help='show this help message and exit.') + parser.add_option('-B', '--batch-size', type='int', default=16, metavar="", + help='Rebalance configuration batch size. Valid values are 1-%d' % MAX_BATCH_SIZE) + parser.add_option('--usage', action="briefhelp") + parser.add_option('-S', '--simple-progress', action='store_true', + help='show simple progress.') + parser.add_option('', '--hba-hostnames', action='store_true', default=False, dest='hba_hostnames', + help='use hostnames instead of CIDR in pg_hba.conf') + parser.add_option('--allow-intermediate-mixture', dest='allow_mixture', action='store_true', + help='Allow primary and mirror from one pair to be at the same host during balancing', default=False) + + parser.set_defaults(verbose=False) + + # Parse the command line arguments + (options, args) = parser.parse_args() + return options, args, parser + + +def validate_options(options, args, parser): + if len(args) > 0: + logger.error(f'Unknown argument {args[0]}') + parser.exit(1) + + if options.mirroring and options.mirroring not in ('grouped', 'spread'): + logger.error('Mirroring strategy %s is not supported' % + options.mirroring) + parser.print_help() + parser.exit(1) + + try: + options.coordinator_data_directory = get_coordinatordatadir() + options.gphome = get_gphome() + except GpError as msg: + logger.error(msg) + parser.exit(1) + + if not os.path.exists(options.coordinator_data_directory): + logger.error('Coordinator data directory %s does not exist.' % + options.coordinator_data_directory) + parser.exit(1) + + if options.genconf: + for arg in sys.argv[1:]: + if arg not in ('-g', '--gen-hosts'): + logger.error('-g or --gen-hosts flag must be used alone') + parser.print_usage() + parser.exit(1) + + if options.parallel > MAX_PARALLEL_WORKERS or options.parallel < 1: + logger.error( + 'Invalid argument. parallel value must be >= 1 and <= %d' % MAX_PARALLEL_WORKERS) + parser.print_help() + parser.exit(1) + + if options.end: + if not isinstance(options.end, datetime.datetime): + options.end = datetime.datetime.combine( + options.end, datetime.time(0)) + if options.end < datetime.datetime.now(): + logger.error('End time occurs in the past') + parser.print_help() + parser.exit(1) + if options.duration: + if options.end > datetime.datetime.now() + options.duration: + options.end = datetime.datetime.now() + options.duration + elif options.duration: + options.end = datetime.datetime.now() + options.duration + + if options.rollback and options.clean: + rollbackOpt = "--rollback" if "--rollback" in sys.argv else "-r" + cleanOpt = "--clean" if "--clean" in sys.argv else "-c" + logger.error("%s and %s options cannot be specified together." % + (rollbackOpt, cleanOpt)) + parser.exit(1) + + return options, args + + +def check_running_gputils(dburl: dbconn.DbURL, coordinator_data_directory: str): + """ + Checks if there are any running instances of + gprebalance/gpbackup/gpexpand/gpshrink/gpresize + """ + for util in ('gprebalance', 'gpexpand', 'gpshrink', 'gpresize'): + try: + with open(f'{coordinator_data_directory}/{util}.pid', 'r') as fp: + pid = int(fp.readline().strip()) + if check_pid(pid): + logger.error(f'{util} is already running.') + sys.exit(1) + except IOError: + pass + + with closing(dbconn.connect(dburl, encoding='UTF8')) as conn: + cursor = dbconn.query(conn, '''SELECT datid + FROM pg_stat_activity + WHERE application_name LIKE 'gpbackup%' OR + application_name LIKE 'gprestore%' ''') + if cursor.rowcount > 0: + logger.error( + '''gpbackup/gprestore utility is already running.''') + sys.exit(1) + + if is_gprecoverseg_running(): + logger.error( + '''gprecoverseg is already running.''') + sys.exit(1) + + +def create_pid_file(coordinator_data_directory: str): + """ + Creates gprebalance pid file + """ + with open(coordinator_data_directory + '/gprebalance.pid', 'w') as fp: + fp.write(str(os.getpid())) + + +def remove_pid_file(coordinator_data_directory: str): + """ + Removes gprebalance pid file + """ + try: + os.unlink(coordinator_data_directory + '/gprebalance.pid') + except IOError: + pass + + +def dump_hosts_info(hosts): + """ + Converts info about hosts to YAML format + """ + yaml_entries = [] + for (hostname, address), val in hosts.items(): + host_entry = { + 'hostname': hostname, + 'address': address, + 'primary_datadirs': list(val.primary_datadirs), + 'mirror_datadirs': list(val.mirror_datadirs), + } + yaml_entries.append(host_entry) + + yaml_entries.sort(key=lambda x: x['hostname']) + config = {'hosts': yaml_entries} + config_yaml = yaml.dump(config, default_flow_style=False, sort_keys=False) + outputfile = os.getcwd() + "/gprebalance_hosts.yaml" + with open('gprebalance_hosts.yaml', 'w') as fp: + fp.write(config_yaml) + logger.info('Generating hosts config in %s' % outputfile) + + +def possibility_checks(gpreb: GPRebalance): + logger.info('Validation of rebalance possibility...') + + hosts_list = gpreb.current_hosts + + startup_validator = ClusterValidator( + hosts_list, + gpreb.target_hosts, + gpreb.original_gparray.getSegmentsAsLoadedFromDb(), + gpreb.original_gparray.get_mirroring_enabled(), + gpreb.target_strategy) + try: + startup_validator.validate_segment_status() + except StateValidationError as e: + if not gpreb.options.silent and not ask_yesno('', " %s Are you sure you want " + "to continue with this gprebalance session?" % str(e), "N"): + logger.error('User Aborted. Exiting...') + sys.exit(1) + elif gpreb.options.silent: + logger.error(f'{str(e)} Exiting...') + sys.exit(1) + if gpreb.unpreferred_segments: + logger.error( + "Current role does not match preferred role for several segments. " + "Call gprecoverseg -r to restore the original roles") + sys.exit(1) + if not gpreb.original_gparray.get_mirroring_enabled() and not gpreb.options.allow_mirrorless: + if not gpreb.options.silent and not ask_yesno('', + "Mirroring is disabled. \n" + "During rebalance the whole cluster may " + "be not available. \n" + "Are you sure you want " + "to continue with this gprebalance session?", "N"): + logger.error('User Aborted. Exiting...') + sys.exit(1) + elif gpreb.options.silent: + logger.error('Mirroring is disabled. Exiting...') + sys.exit(1) + + if gpreb.original_gparray.get_mirroring_enabled() and startup_validator.mirror_strategy == MirrorStrategy.MIRRORLESS: + if not gpreb.options.silent: + mirror_type = ask_string( + "\nYou haven't specified desirable mirroring strategy. Spread mirroring places\n" + "a given hosts mirrored segments each on a separate host. You must be \n" + "using more hosts than the number of segments per host for spread mirroring. \n" + "Grouped mirroring places all of a given hosts segments on a single \n" + "mirrored host. You must be using at least 2 hosts for grouped strategy.\n\n", + "What type of mirroring strategy would you like?", + 'grouped', ['spread', 'grouped']) + else: + mirror_type = 'grouped' + strat = MirrorStrategy.SPREAD if mirror_type == 'spread' else MirrorStrategy.GROUPED + gpreb.setMirroringStrategy(strat) + startup_validator.mirror_strategy = strat + + already_bal, _ = startup_validator.validate_existing_configuration() + if already_bal: + # case of hosts shrinkage, expansion + if len(startup_validator.existing_hosts & + startup_validator.target_hosts) < len(startup_validator.existing_hosts) or \ + len(startup_validator.target_hosts) > len(startup_validator.existing_hosts): + try: + startup_validator.prevalidate_segment_distribution() + startup_validator.prevalidate_mirror_strategy() + except StateValidationError: + logger.error( + 'Cluster is already balanced. Cannot additionally rebalance to newly added hosts') + sys.exit(1) + else: + logger.info('Cluster is already balanced') + sys.exit(0) + else: + startup_validator.prevalidate_segment_distribution() + startup_validator.prevalidate_mirror_strategy() + + +gprebalance = None + + +def sig_handler(sig, arg): + if gprebalance is not None: + gprebalance.shutdown() + if gprebalance.executor is None: + signal.signal(signal.SIGTERM, signal.SIG_DFL) + signal.signal(signal.SIGHUP, signal.SIG_DFL) + signal.signal(signal.SIGINT, signal.SIG_DFL) + os.kill(os.getpid(), sig) + + +def main(options, args, parser): + try: + signal.signal(signal.SIGTERM, sig_handler) + signal.signal(signal.SIGHUP, sig_handler) + signal.signal(signal.SIGINT, sig_handler) + + logger = get_default_logger() + setup_tool_logging(EXECNAME, getLocalHostname(), getUserName()) + + options, args = validate_options(options, args, parser) + + if options.verbose: + enable_verbose_logging() + + gpEnv = GpCoordinatorEnvironment( + options.coordinator_data_directory, True) + configurationInterface.registerConfigurationProvider( + configurationImplGpdb.GpConfigurationProviderUsingGpdbCatalog()) + configurationInterface.getConfigurationProvider( + ).initializeProvider(gpEnv.getCoordinatorPort()) + + dburl = dbconn.DbURL(dbname=DBNAME, port=gpEnv.getCoordinatorPort()) + + try: + # check parallel execution of other cluster utilities + check_running_gputils(dburl, options.coordinator_data_directory) + gparray = GpArray.initFromCatalog(dburl, utility=True) + except DatabaseError as ex: + logger.error("Failed to connect to database. Make sure the" + " Greenplum instance you wish to expand is running" + " and that your environment is correct, then rerun" + " gprebalance" + ' '.join(sys.argv[1:])) + sys.exit(1) + except ConnectionError as ex: + logger.error(f"{str(ex)}") + sys.exit(1) + + # create pid file + create_pid_file(options.coordinator_data_directory) + + global gprebalance + gprebalance = GPRebalance(logger, gparray, dburl, options, gpEnv) + + # dump existing hosts configuration + if options.genconf: + dump_hosts_info(gprebalance.getHostsFromGpArray()) + sys.exit(0) + + gprebalance_file_status = gprebalance.get_state_from_file() + + if options.clean: + gprebalance.remove_status_file() + gprebalance.cleanup_directory() + logger.info('Cleanup Finished. exiting...') + sys.exit(0) + + from gprebalance_modules.rebalance_status import RebalanceStatus # nopep8 + + if gprebalance_file_status == RebalanceStatus.DONE and not options.rollback: + logger.info('Rebalance has already completed') + logger.info( + 'If you want to rebalance again, run gprebalance -c to perform cleanup') + elif options.rollback and gprebalance_file_status == RebalanceStatus.ROLLBACK_DONE: + logger.info('Rollback has already completed') + logger.info( + 'If you want to rebalance again, run gprebalance -c to perform cleanup') + elif gprebalance_file_status is None or gprebalance_file_status == RebalanceStatus.UNINITIALIZED: + if options.rollback: + logger.error("Cannot perform rollback operation. Rebalance session status files are absent.") + sys.exit(1) + gprebalance.statusManager.create_status_file() + possibility_checks(gprebalance) + logger.info('Validation passed. Preparing rebalance...') + gprebalance.statusManager.set_status(RebalanceStatus.VALIDATED, os.path.abspath( + options.filename) if options.filename else None) + plan = gprebalance.createPlan() + gprebalance.save_plan(plan) + if options.show_plan: + logger.info(f"Resulting plan: \n{str(plan)}") + # should be removed when state machine will be ready + gprebalance.statusManager.set_status(RebalanceStatus.UNINITIALIZED) + sys.exit(0) + gprebalance.execute_plan(plan) + elif gprebalance.statusManager.conf_dir: + plan = gprebalance.load_plan(gprebalance.statusManager.conf_dir, False) + if options.show_plan: + logger.info(f"Resulting plan: \n{str(plan)}") + sys.exit(0) + if gprebalance_file_status == RebalanceStatus.PREPARED: + gprebalance.execute_plan(plan) + sys.exit(0) + elif gprebalance_file_status == RebalanceStatus.ROLLBACK_FAILED: + logger.info('Previous rollback session was interrupted due to an error. Trying to resume...') + plan = gprebalance.load_plan(gprebalance.statusManager.conf_dir, True) + gprebalance.resume(plan) + sys.exit(0) + elif gprebalance_file_status == RebalanceStatus.FAILED: + if not options.rollback: + logger.info('Previous rebalance session was interrupted due to an error. Trying to resume...') + gprebalance.resume(plan) + sys.exit(0) + elif gprebalance_file_status == RebalanceStatus.STOPPED: + if not options.rollback: + logger.info('Previous rebalance session was interrupted due to an timeout or user signal. Trying to resume...') + gprebalance.resume(plan) + sys.exit(0) + if options.rollback: + gprebalance.rollback(plan) + sys.exit(0) + except Exception as e: + if options and options.verbose: + logger.exception('gprebalance failed. exiting...') + else: + logger.error(f'gprebalance failed: {e} \n\nExiting...') + sys.exit(1) + except KeyboardInterrupt: + sys.exit('\nUser Interrupted') + finally: + if gprebalance: + gprebalance.shutdown() + remove_pid_file(get_coordinatordatadir()) + + +if __name__ == '__main__': + options, args, parser = parseargs() + main(options, args, parser) diff --git a/gpMgmt/bin/gprebalance_modules/Makefile b/gpMgmt/bin/gprebalance_modules/Makefile new file mode 100644 index 000000000000..b9b823bb02f3 --- /dev/null +++ b/gpMgmt/bin/gprebalance_modules/Makefile @@ -0,0 +1,31 @@ +# gpMgmt/bin/gprebalance_modules/Makefile + +top_builddir = ../../.. +include $(top_builddir)/src/Makefile.global + +PROGRAMS= __init__.py rebalance_validator.py \ +rebalance.py \ +rebalance_plan.py \ +rebalance_executor.py \ +rebalance_status.py \ +shrink.py planner.py \ +rebalance_commons.py \ +rebalance_schema.py \ +solver.py ggrebalance_sm.py \ +ggrebalance_main_sm.py rebalance_step.py + +installdirs: + $(MKDIR_P) '$(DESTDIR)$(bindir)/gprebalance_modules' + +install: installdirs + for file in $(PROGRAMS); do \ + $(INSTALL_SCRIPT) $$file '$(DESTDIR)$(bindir)/gprebalance_modules/'$$file ; \ + done + +uninstall: + for file in $(PROGRAMS); do \ + rm -f '$(DESTDIR)$(bindir)/gprebalance_modules/'$$file ; \ + done + +clean distclean: + rm -f *.pyc diff --git a/gpMgmt/bin/gprebalance_modules/__init__.py b/gpMgmt/bin/gprebalance_modules/__init__.py new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/gpMgmt/bin/gprebalance_modules/ggrebalance_main_sm.py b/gpMgmt/bin/gprebalance_modules/ggrebalance_main_sm.py new file mode 100755 index 000000000000..a1d347d27899 --- /dev/null +++ b/gpMgmt/bin/gprebalance_modules/ggrebalance_main_sm.py @@ -0,0 +1,300 @@ +#!/usr/bin/env python3 + +from transitions import Machine + +try: + from gppylib.commands.unix import * + from gppylib.commands.gp import * + from gppylib.gplog import * + from gppylib.system.environment import * + from gprebalance_modules.planner import * + from gprebalance_modules.rebalance_schema import RebalanceSchema + from gppylib.fault_injection import * + from gprebalance_modules.shrink import GGShrink + from gprebalance_modules.ggrebalance_sm import RebalanceSM +except ImportError as e: + sys.exit('ERROR: Cannot import modules. Please check that you have sourced greenplum_path.sh. Detail: ' + str(e)) + +class GGRebalanceMainSM: + + states_not_logged = [ + 'STATE_START', + 'STATE_OPTIONS_VALIDATION', + 'STATE_CLEANUP', + 'STATE_ROLLBACK', + 'STATE_PLANNING_STARTED', + 'STATE_PLANNING_DONE', + 'STATE_CHECK_PREVIOUS_RUN', + 'STATE_END', + 'STATE_ERROR' + ] + + states_logged = [ + 'STATE_SETUP_SCHEMA_STARTED', + 'STATE_SETUP_SCHEMA_DONE', + 'STATE_EXECUTOR_STARTED', + 'STATE_EXECUTOR_DONE', + 'STATE_SHRINK_STARTED', + 'STATE_SHRINK_DONE', + 'STATE_REBALANCE_STARTED', + 'STATE_REBALANCE_DONE', + ] + + transitions = [ + { + 'trigger': 'start', + 'source': 'STATE_START', + 'dest': 'STATE_OPTIONS_VALIDATION' + }, + { + 'trigger': 'move_to_STATE_CLEANUP', + 'source': 'STATE_OPTIONS_VALIDATION', + 'dest': 'STATE_CLEANUP' + }, + { + 'trigger': 'move_to_STATE_ROLLBACK', + 'source': 'STATE_OPTIONS_VALIDATION', + 'dest': 'STATE_ROLLBACK' + }, + { + 'trigger': 'move_to_STATE_PLANNING_STARTED', + 'source': 'STATE_OPTIONS_VALIDATION', + 'dest': 'STATE_PLANNING_STARTED' + }, + { + 'trigger': 'move_to_STATE_PLANNING_DONE', + 'source': 'STATE_PLANNING_STARTED', + 'dest': 'STATE_PLANNING_DONE' + }, + { + 'trigger': 'move_to_STATE_CHECK_PREVIOUS_RUN', + 'source': 'STATE_PLANNING_DONE', + 'dest': 'STATE_CHECK_PREVIOUS_RUN' + }, + { + 'trigger': 'move_to_STATE_SETUP_SCHEMA_STARTED', + 'source': 'STATE_CHECK_PREVIOUS_RUN', + 'dest': 'STATE_SETUP_SCHEMA_STARTED' + }, + { + 'trigger': 'move_to_STATE_SETUP_SCHEMA_DONE', + 'source': 'STATE_SETUP_SCHEMA_STARTED', + 'dest': 'STATE_SETUP_SCHEMA_DONE' + }, + { + 'trigger': 'move_to_STATE_EXECUTOR_STARTED', + 'source': ['STATE_SETUP_SCHEMA_DONE', 'STATE_CHECK_PREVIOUS_RUN'], + 'dest': 'STATE_EXECUTOR_STARTED' + }, + { + 'trigger': 'move_to_STATE_SHRINK_STARTED', + 'source': 'STATE_EXECUTOR_STARTED', + 'dest': 'STATE_SHRINK_STARTED' + }, + { + 'trigger': 'move_to_STATE_SHRINK_DONE', + 'source': 'STATE_SHRINK_STARTED', + 'dest': 'STATE_SHRINK_DONE' + }, + { + 'trigger': 'move_to_STATE_REBALANCE_STARTED', + 'source': ['STATE_EXECUTOR_STARTED', 'STATE_SHRINK_DONE'], + 'dest': 'STATE_REBALANCE_STARTED' + }, + { + 'trigger': 'move_to_STATE_REBALANCE_DONE', + 'source': 'STATE_REBALANCE_STARTED', + 'dest': 'STATE_REBALANCE_DONE' + }, + { + 'trigger': 'move_to_STATE_EXECUTOR_DONE', + 'source': ['STATE_EXECUTOR_STARTED', 'STATE_SHRINK_DONE', 'STATE_REBALANCE_DONE'], + 'dest': 'STATE_EXECUTOR_DONE' + }, + { + 'trigger': 'move_to_STATE_END', + 'source': ['STATE_EXECUTOR_DONE', 'STATE_CHECK_PREVIOUS_RUN', 'STATE_CLEANUP', 'STATE_ROLLBACK'], + 'dest': 'STATE_END' + }, + { + 'trigger': 'move_to_STATE_ERROR', + 'source': '*', + 'dest': 'STATE_ERROR' + } + ] + + def __init__(self, conn: dbconn.Connection, logger: Any, dburl: dbconn.DbURL, options: Any, gpEnv: GpCoordinatorEnvironment, gpArray: gparray.GpArray, gpArrayDumpFilename: str): + self.logger = logger + self.dburl = dburl + self.options = options + self.gparray = gpArray + self.conn = conn + + self.rebalance_schema = RebalanceSchema(self.conn) + + self.machine = Machine(model = self, + queued=True, + states = self.states_not_logged + self.states_logged, + transitions = self.transitions, + initial = 'STATE_START', + before_state_change = 'on_every_state') + + self.gg_shrink = GGShrink(self.conn, self.rebalance_schema, self.logger, self.options, gpEnv, self.gparray, gpArrayDumpFilename) + self.gg_rebalance = RebalanceSM(self.conn, self.rebalance_schema, self.logger, self.options, self.gparray) + + self.plan = None + self.main_state_from_prev_run = self.rebalance_schema.getMainStateFromPreviousRun() + + def on_every_state(self) -> None: + if self.state in self.states_logged: + self.rebalance_schema.storeMainState(self.state) + + def run(self) -> None: + self.trigger('start') + + def shutdown(self) -> None: + need_exit = True + + if self.gg_shrink is not None: + self.gg_shrink.shutdown() + need_exit = False + + if self.gg_rebalance is not None: + self.gg_rebalance.shutdown() + need_exit = False + + if need_exit: + sys.exit(1) + + # state callbacks start here + + @wrap_state_func_with_faults + def on_enter_STATE_OPTIONS_VALIDATION(self) -> None: + if self.options.clean_required: + self.trigger('move_to_STATE_CLEANUP') + elif self.options.rollback_required: + self.trigger('move_to_STATE_ROLLBACK') + else: + self.trigger('move_to_STATE_PLANNING_STARTED') + + @wrap_state_func_with_faults + def on_enter_STATE_CLEANUP(self) -> None: + if not self.rebalance_schema.schemaExists(): + self.logger.info(f"Rebalance schema doesn't exist. Cleanup is not required.") + else: + prev_run_was_complete = (self.main_state_from_prev_run == 'STATE_EXECUTOR_DONE' or + self.main_state_from_prev_run == 'STATE_ROLLBACK') + self.gg_shrink.cleanup(prev_run_was_complete) + self.rebalance_schema.dropSchema() + self.logger.info('Cleanup is complete') + self.trigger('move_to_STATE_END') + + @wrap_state_func_with_faults + def on_enter_STATE_ROLLBACK(self) -> None: + self.plan = self.rebalance_schema.retrieveSavedPlan() + self.gg_shrink.rollback(self.plan) + self.trigger('move_to_STATE_END') + + @wrap_state_func_with_faults + def on_enter_STATE_PLANNING_STARTED(self) -> None: + if self.options.target_segment_count != None: + self.plan = Planner(self.logger, self.dburl, self.gparray, self.options).plan() + + if self.options.target_segment_count != None and self.options.show_plan: + self.logger.info(f"Final plan:\n{self.plan}") + + self.trigger('move_to_STATE_PLANNING_DONE') + + @wrap_state_func_with_faults + def on_enter_STATE_PLANNING_DONE(self) -> None: + self.trigger('move_to_STATE_CHECK_PREVIOUS_RUN') + + @wrap_state_func_with_faults + def on_enter_STATE_CHECK_PREVIOUS_RUN(self) -> None: + if not self.rebalance_schema.schemaExists(): + if self.plan == None: + self.logger.error("Rebalance schema doesn't exists and no shrink plan is supplied. Please specify shrink plan.") + self.trigger('move_to_STATE_ERROR') + return + if self.gparray.get_segment_count() < self.plan.target_segment_count: + logger.error('Target segment count (%s) > current segment count (%s).\n' + 'Currently only shrink is supported (target segment count < current segment count).' + % (self.plan.target_segment_count, self.gparray.get_segment_count())) + self.trigger('move_to_STATE_ERROR') + return + self.trigger('move_to_STATE_SETUP_SCHEMA_STARTED') + else: + # Schema already exists from the previous run. + # In this case we already have a plan saved in the schema, + # and we'll continue (or rollback) according to it. + # Or, if everything is complete, just exit. + if self.main_state_from_prev_run == 'STATE_EXECUTOR_DONE': + self.logger.info('Previous run was completed successfully. Please execute cleanup before a new run.') + return + + if self.plan != None: + self.logger.error("Can't start a new operation, because the previous one was interrupted. " + "Please try to launch again without a plan to continue from the interrupted state, " + "or use '--rollback' or '--cleanup' options.") + self.trigger('move_to_STATE_ERROR') + return + + self.plan = self.rebalance_schema.retrieveSavedPlan() + + self.trigger('move_to_STATE_EXECUTOR_STARTED') + + @wrap_state_func_with_faults + def on_enter_STATE_SETUP_SCHEMA_STARTED(self) -> None: + # Create schema and status tables. + # It will also save plan in order to use it for recovering after interruption + self.rebalance_schema.createSchema(self.plan) + self.trigger('move_to_STATE_SETUP_SCHEMA_DONE') + + @wrap_state_func_with_faults + def on_enter_STATE_SETUP_SCHEMA_DONE(self) -> None: + self.logger.info(f'Created "{self.rebalance_schema.getSchemaName()}" schema') + self.trigger('move_to_STATE_EXECUTOR_STARTED') + + @wrap_state_func_with_faults + def on_enter_STATE_EXECUTOR_STARTED(self) -> None: + if isinstance(self.plan, ShrinkPlan): + shrink_state_from_prev_run = self.rebalance_schema.getShrinkStateFromPreviousRun() + if not self.gg_shrink.state_is_final(shrink_state_from_prev_run): + self.trigger('move_to_STATE_SHRINK_STARTED') + return + self.trigger('move_to_STATE_REBALANCE_STARTED') + + @wrap_state_func_with_faults + def on_enter_STATE_EXECUTOR_DONE(self) -> None: + self.trigger('move_to_STATE_END') + + @wrap_state_func_with_faults + def on_enter_STATE_SHRINK_STARTED(self) -> None: + self.gg_shrink.run(self.plan) + self.trigger('move_to_STATE_SHRINK_DONE') + + @wrap_state_func_with_faults + def on_enter_STATE_SHRINK_DONE(self) -> None: + self.trigger('move_to_STATE_REBALANCE_STARTED') + + @wrap_state_func_with_faults + def on_enter_STATE_REBALANCE_STARTED(self) -> None: + if self.plan is not None and self.plan.getMoves() is not None: + self.gg_rebalance.run(self.plan) + self.logger.info('Rebalance is complete') + + self.trigger('move_to_STATE_REBALANCE_DONE') + + @wrap_state_func_with_faults + def on_enter_STATE_REBALANCE_DONE(self) -> None: + self.trigger('move_to_STATE_EXECUTOR_DONE') + + @wrap_state_func_with_faults + def on_enter_STATE_END(self) -> None: + pass + + @wrap_state_func_with_faults + def on_enter_STATE_ERROR(self) -> None: + raise Exception('Main SM entered STATE_ERROR') + + # state callbacks end here diff --git a/gpMgmt/bin/gprebalance_modules/ggrebalance_sm.py b/gpMgmt/bin/gprebalance_modules/ggrebalance_sm.py new file mode 100755 index 000000000000..cbb2bc2a7c4d --- /dev/null +++ b/gpMgmt/bin/gprebalance_modules/ggrebalance_sm.py @@ -0,0 +1,521 @@ +#!/usr/bin/env python3 + +from transitions import Machine +from enum import Enum + +try: + from gppylib.commands.unix import * + from gppylib.commands.gp import * + from gppylib.gplog import * + from gppylib.commands.gp import GpMoveMirrors + from gppylib.system.environment import * + from gprebalance_modules.planner import * + from gprebalance_modules.rebalance_schema import RebalanceSchema, STATE_NOT_DEFINED + from gprebalance_modules.rebalance_step import * + from gppylib.fault_injection import * +except ImportError as e: + sys.exit('ERROR: Cannot import modules. Please check that you have sourced greenplum_path.sh. Detail: ' + str(e)) + +class RebalanceSM: + + states_not_logged = [ + 'STATE_REBALANCE_INIT', + 'STATE_CHECK_PREVIOUS_RUN', + 'STATE_ERROR' + ] + + states_main_rebalance_flow = [ + 'STATE_REBALANCE_STARTED', + 'STATE_REBALANCE_PREPARE_MOVES_STARTED', + 'STATE_REBALANCE_PREPARE_MOVES_DONE', + 'STATE_REBALANCE_EXECUTION_STARTED', + 'STATE_REBALANCE_MOVES_SUCCEEDED', + 'STATE_REBALANCE_EXECUTION_AWAITING_SWITCHOVER_APPROVE_STARTED', + 'STATE_REBALANCE_EXECUTION_AWAITING_SWITCHOVER_APPROVE_DONE', + 'STATE_REBALANCE_EXECUTION_DONE', + 'STATE_REBALANCE_DONE' + ] + + transitions = [ + { + 'trigger': 'start', + 'source': 'STATE_REBALANCE_INIT', + 'dest': 'STATE_CHECK_PREVIOUS_RUN' + }, + { + 'trigger': 'move_to_STATE_REBALANCE_STARTED', + 'source': 'STATE_CHECK_PREVIOUS_RUN', + 'dest': 'STATE_REBALANCE_STARTED' + }, + { + 'trigger': 'move_to_STATE_REBALANCE_PREPARE_MOVES_STARTED', + 'source': 'STATE_REBALANCE_STARTED', + 'dest': 'STATE_REBALANCE_PREPARE_MOVES_STARTED' + }, + { + 'trigger': 'move_to_STATE_REBALANCE_PREPARE_MOVES_DONE', + 'source': 'STATE_REBALANCE_PREPARE_MOVES_STARTED', + 'dest': 'STATE_REBALANCE_PREPARE_MOVES_DONE' + }, + { + 'trigger': 'move_to_STATE_REBALANCE_EXECUTION_STARTED', + 'source': ['STATE_REBALANCE_PREPARE_MOVES_DONE', 'STATE_REBALANCE_MOVES_SUCCEEDED', 'STATE_REBALANCE_EXECUTION_AWAITING_SWITCHOVER_APPROVE_DONE'], + 'dest': 'STATE_REBALANCE_EXECUTION_STARTED' + }, + { + 'trigger': 'move_to_STATE_REBALANCE_MOVES_SUCCEEDED', + 'source': 'STATE_REBALANCE_EXECUTION_STARTED', + 'dest': 'STATE_REBALANCE_MOVES_SUCCEEDED' + }, + { + 'trigger': 'move_to_STATE_REBALANCE_EXECUTION_AWAITING_SWITCHOVER_APPROVE_STARTED', + 'source': 'STATE_REBALANCE_EXECUTION_STARTED', + 'dest': 'STATE_REBALANCE_EXECUTION_AWAITING_SWITCHOVER_APPROVE_STARTED' + }, + { + 'trigger': 'move_to_STATE_REBALANCE_EXECUTION_AWAITING_SWITCHOVER_APPROVE_DONE', + 'source': 'STATE_REBALANCE_EXECUTION_AWAITING_SWITCHOVER_APPROVE_STARTED', + 'dest': 'STATE_REBALANCE_EXECUTION_AWAITING_SWITCHOVER_APPROVE_DONE' + }, + { + 'trigger': 'move_to_STATE_REBALANCE_EXECUTION_DONE', + 'source': 'STATE_REBALANCE_EXECUTION_STARTED', + 'dest': 'STATE_REBALANCE_EXECUTION_DONE' + }, + { + 'trigger': 'move_to_STATE_REBALANCE_DONE', + 'source': 'STATE_REBALANCE_EXECUTION_DONE', + 'dest': 'STATE_REBALANCE_DONE' + }, + { + 'trigger': 'move_to_STATE_ERROR', + 'source': '*', + 'dest': 'STATE_ERROR' + } + ] + + class RoleSwapDirection(Enum): + PRIMARY_TO_MIRROR = 1 + MIRROR_TO_PRIMARY = 2 + + def __init__(self, conn: dbconn.Connection, schema: RebalanceSchema, logger: Any, options: Any, gpArray: gparray.GpArray): + self.logger = logger + self.options = options + self.shutdown_requested = False + self.gparray = gpArray + self.conn = conn + self.rebalance_schema = schema + self.cmd = None + + self.machine = Machine(model = self, + queued=True, + states = self.states_main_rebalance_flow + self.states_not_logged, + transitions = self.transitions, + initial = 'STATE_REBALANCE_INIT', + before_state_change = 'on_every_state') + + def on_every_state(self) -> None: + if self.shutdown_requested: + self.logger.info('Rebalance was interrupted') + raise Exception('Rebalance was interrupted') + + if self.state in self.states_main_rebalance_flow: + self.rebalance_schema.storeRebalanceState(self.state) + + def run(self, plan: Plan) -> None: + self.rebalance_plan = plan + if not self.rebalance_plan.getMoves(): + return + + self.moves_primaries = [] + self.moves_mirrors = [] + for move in self.rebalance_plan.getMoves(): + if move.seg.isSegmentPrimary() : + self.moves_primaries.append(move) + else: + self.moves_mirrors.append(move) + + self.primary_segments_to_move = [move.seg for move in self.moves_primaries] + + self.trigger('start') + + def process_moves(self, moves: List[LogicalMove]): + if len(moves) == 0: + return + + filename = self.create_config_file(moves) + gpmovemirrors_options = f'-a -i {filename}' + + if self.options.parallel is not None: + batch_size = self.options.parallel + # gpmovemirrors has its own limitation for batch size, + # need to consider it here. + if batch_size > MAX_COORDINATOR_NUM_WORKERS: + batch_size = MAX_COORDINATOR_NUM_WORKERS + gpmovemirrors_options += f' -B {batch_size}' + if self.options.hba_hostnames: + gpmovemirrors_options = gpmovemirrors_options + ' --hba-hostnames' + if self.options.logfile_directory is not None: + gpmovemirrors_options = gpmovemirrors_options + f' -l "{str(self.options.logfile_directory)}"' + try: + self.cmd = GpMoveMirrors("Running gpmovemirrors", options=gpmovemirrors_options) + self.cmd.run(validateAfter=True) + except Exception as e: + logger.error(str(e)) + error_msg = f"Failed to execute 'gpmovemirrors {gpmovemirrors_options}'" + raise Exception(error_msg) + finally: + self.cmd = None + + if os.path.exists(filename): + os.remove(filename) + + def execute_role_swaps(self, segments_to_move: List[Segment], direction: RoleSwapDirection): + """Execute multiple role swaps in single gprecoverseg -r call""" + + assert (len(segments_to_move) > 0) + + segids = [segment.getSegmentContentId() for segment in segments_to_move] + dbids = [segment.getSegmentDbId() for segment in segments_to_move] + + seg_list = ', '.join(str(seg) for seg in segids) + dbid_list = ', '.join(str(dbid) for dbid in dbids) + + dbconn.execSQL(self.conn, "BEGIN") + + # check the current status of 'preferred_role' and 'role' for all requested dbids + # in order to recover properly from the previous interrupted run (if any) + + cnt_preferred_role_p = \ + int(dbconn.queryRow(self.conn, + f"SELECT COUNT(1) FROM gp_segment_configuration WHERE preferred_role = 'p' AND dbid IN ({dbid_list})")[0]) + cnt_role_p = \ + int(dbconn.queryRow(self.conn, + f"SELECT COUNT(1) FROM gp_segment_configuration WHERE role = 'p' AND dbid IN ({dbid_list})")[0]) + cnt_preferred_role_m = \ + int(dbconn.queryRow(self.conn, + f"SELECT COUNT(1) FROM gp_segment_configuration WHERE preferred_role = 'm' AND dbid IN ({dbid_list})")[0]) + cnt_role_m = \ + int(dbconn.queryRow(self.conn, + f"SELECT COUNT(1) FROM gp_segment_configuration WHERE role = 'm' AND dbid IN ({dbid_list})")[0]) + + # if some have 'preferred_role'='p' and some have 'preferred_role'='m' - shouldn't happen, error out, needs to be resolved manually. + # also some sanity check that there are no other values in catalog except 'm' and 'p' for 'preferred_role'. + if ((cnt_preferred_role_p > 0 and cnt_preferred_role_m > 0) or + (cnt_preferred_role_p + cnt_preferred_role_m != len(segids))): + raise Exception("Error in catalog configuration: " + f"for dbid list ({dbid_list}) " + f"{cnt_preferred_role_p} have 'p' preferred role, and " + f"{cnt_preferred_role_m} have 'm' preferred role") + + is_catalog_update_required = False + is_gprecoverseg_required = False + + if direction == self.RoleSwapDirection.PRIMARY_TO_MIRROR: + # if all have 'preferred_role'='p' - it is our first run, need to update catalog and launch gprecoverseg + if cnt_preferred_role_p == len(segids): + is_catalog_update_required = True + is_gprecoverseg_required = True + else: + # if all have 'preferred_role'='m' and not all have 'role'='m' - previous gprecoverseg was interrupted, need to launch it again + if cnt_role_m != cnt_preferred_role_m: + is_gprecoverseg_required = True + # if all have 'preferred_role'='m' and 'role'='m' - we've done everything on previous interrupted run, nothing to do + else: + # moving back in MIRROR_TO_PRIMARY direction + # if all have 'preferred_role'='m' - it is our first run, need to update catalog and launch gprecoverseg + if cnt_preferred_role_m == len(segids): + is_catalog_update_required = True + is_gprecoverseg_required = True + else: + # if all have 'preferred_role'='p' and not all have 'role'='p' - previous gprecoverseg was interrupted, need to launch it again + if cnt_role_p != cnt_preferred_role_p: + is_gprecoverseg_required = True + # if all have 'preferred_role'='p' and 'role'='p' - we've done everything on previous interrupted run, nothing to do + + if is_catalog_update_required: + dbconn.execSQL(self.conn, "UPDATE gp_segment_configuration SET preferred_role = 't' WHERE " + f"content IN ({seg_list}) AND preferred_role = 'm'") + dbconn.execSQL(self.conn, "UPDATE gp_segment_configuration SET preferred_role = 'm' WHERE " + f"content IN ({seg_list}) AND preferred_role = 'p'") + dbconn.execSQL(self.conn, "UPDATE gp_segment_configuration SET preferred_role = 'p' WHERE " + f"content IN ({seg_list}) AND preferred_role = 't'") + + dbconn.execSQL(self.conn, "COMMIT") + + if direction == self.RoleSwapDirection.PRIMARY_TO_MIRROR: + inject_fault('FAULT_BEFORE_GPRECOVERSEG_PRIMARY_TO_MIRROR') + else: + inject_fault('FAULT_BEFORE_GPRECOVERSEG_MIRROR_TO_PRIMARY') + + if is_gprecoverseg_required: + recoverseg_options = "-r -a" + + if self.options.parallel is not None: + batch_size = self.options.parallel + # gprecoverseg has its own limitation for batch size, + # need to consider it here. + if batch_size > MAX_COORDINATOR_NUM_WORKERS: + batch_size = MAX_COORDINATOR_NUM_WORKERS + recoverseg_options += f' -B {batch_size}' + if self.options.replay_lag is not None: + recoverseg_options = recoverseg_options + f' --replay-lag {self.options.replay_lag}' + if self.options.logfile_directory is not None: + recoverseg_options = recoverseg_options + f' -l "{str(self.options.logfile_directory)}"' + try: + self.cmd = GpRecoverSeg("Running gprecoverseg", options=recoverseg_options) + self.cmd.run(validateAfter=True) + except Exception as e: + logger.error(str(e)) + error_msg = f"Failed to execute 'gprecoverseg {recoverseg_options}'" + raise Exception(error_msg) + finally: + self.cmd = None + + def lookup_seg(self, seg: Segment) -> bool: + """ Look up the segment gpdb by address, port, and dataDirectory """ + for db in self.gparray.getDbList(): + if (seg.getSegmentHostName() == db.getSegmentHostName() and + seg.getSegmentPort() == db.getSegmentPort() and + seg.getSegmentDataDirectory() == db.getSegmentDataDirectory()): + return True + return False + + def create_config_file(self, moves: List[LogicalMove]) -> str: + filename = f'/tmp/ggrebalance_move_config_pid{os.getpid()}' + with open(filename, 'w') as fp: + for move in moves: + segment_current_info = move.seg + is_swap_phase3 = (move.swap_metadata is not None and move.swap_metadata.get('phase') == 3) + if not is_swap_phase3 and not self.lookup_seg(segment_current_info): + self.logger.info(f'Skip segment for gpmovemirrors: {str(segment_current_info)}') + continue + cfg_line = f'{segment_current_info.getSegmentHostName()}|{segment_current_info.getSegmentPort()}|{segment_current_info.getSegmentDataDirectory()} ' + cfg_line += f'{move.dstHost.hostname}|{move.target_port}|{move.target_datadir}\n' + fp.write(cfg_line) + return filename + + def shutdown(self) -> None: + self.shutdown_requested = True + if self.cmd != None: + self.cmd.cancel() + + def state_is_final(self, state: str) -> bool: + return state == self.states_main_rebalance_flow[-1] + + def get_state_after_interrupt(self, prev_state) -> str: + if (prev_state == 'STATE_REBALANCE_EXECUTION_STARTED' or + prev_state == 'STATE_REBALANCE_MOVES_SUCCEEDED' or + prev_state == 'STATE_REBALANCE_EXECUTION_AWAITING_SWITCHOVER_APPROVE_DONE'): + return 'STATE_REBALANCE_EXECUTION_STARTED' + + prev_idx = self.states_main_rebalance_flow.index(prev_state) + + return self.states_main_rebalance_flow[prev_idx + 1] + + def reset_in_progress_execution_steps(self) -> None: + in_progress_steps = self.rebalance_schema.getExecutionSteps([RebalanceStep.Status.IN_PROGRESS]) + for step in in_progress_steps: + step.setStatus(RebalanceStep.Status.PLANNED) + self.rebalance_schema.updateExecutionStep(step) + + @staticmethod + def convert_moves_to_rebalance_steps(moves: List[LogicalMove]) -> List[RebalanceStep]: + # In the loop below we create a list of rebalance execution steps from the plan's list of moves. + # Mirror's movemenet is simply translated into single step. + # But primary's movement is translated into 3 steps: + # 1st is switchover with the mirror; + # 2nd is movement itself; + # 3rd is the back switchover. + # If there are several consequent primary movements, we assume that they can be done in parallel, + # and, to allow batch processing for them, we re-order their steps to combine together + # the respective switchover and movement steps. + rebalance_steps = [] + + batch_mirror_steps = [] + batch_primary_steps = [[],[],[]] + + def fill_rebalance_steps(): + nonlocal rebalance_steps + nonlocal batch_mirror_steps + nonlocal batch_primary_steps + assert not (len(batch_mirror_steps) > 0 and len(batch_primary_steps[0]) > 0) + if len(batch_mirror_steps) > 0: + rebalance_steps = rebalance_steps + batch_mirror_steps + if len(batch_primary_steps[0]) > 0: + flattened_batch_primary_steps = [step for sublist in batch_primary_steps for step in sublist] + rebalance_steps = rebalance_steps + flattened_batch_primary_steps + # clear the batches + batch_mirror_steps = [] + batch_primary_steps = [[],[],[]] + + prev_move = None + for move in moves: + # if the move type switched, fill rebalance_steps with the content of + # previously gathered batches + if prev_move != None and prev_move.seg.getSegmentRole() != move.seg.getSegmentRole(): + fill_rebalance_steps() + prev_move = move + # add steps to the current batch + if move.seg.isSegmentMirror(): + batch_mirror_steps.append(RebalanceStepMoveMirror(move)) + else: + batch_primary_steps[0].append(RebalanceStepSwitchoverToMirror(move)) + batch_primary_steps[1].append(RebalanceStepMoveMirror(move)) + batch_primary_steps[2].append(RebalanceStepSwitchoverToPrimary(move)) + fill_rebalance_steps() # fill what's left + + return rebalance_steps + + # state callbacks start here + + @wrap_state_func_with_faults + def on_enter_STATE_CHECK_PREVIOUS_RUN(self) -> None: + state_from_prev_run = self.rebalance_schema.getRebalanceStateFromPreviousRun() + + if state_from_prev_run == STATE_NOT_DEFINED: + self.trigger('move_to_STATE_REBALANCE_STARTED') + elif self.state_is_final(state_from_prev_run): + self.logger.info('Cluster is already rebalanced...') + else: + self.logger.info('Continue interrupted rebalance operation...') + self.logger.info(f"Previous run stopped after state '{state_from_prev_run}', trying to continue from the next state...") + try: + next_state = self.get_state_after_interrupt(state_from_prev_run) + except: + self.logger.error("Can't determine next state. Try to execute cleanup.") + self.trigger('move_to_STATE_ERROR') + return + # use auto to_«state» method to recover + self.trigger(f'to_{next_state}') + + @wrap_state_func_with_faults + def on_enter_STATE_REBALANCE_STARTED(self) -> None: + self.trigger('move_to_STATE_REBALANCE_PREPARE_MOVES_STARTED') + + @wrap_state_func_with_faults + def on_enter_STATE_REBALANCE_PREPARE_MOVES_STARTED(self) -> None: + if not self.rebalance_plan.getMoves(): + raise Exception('Rebalance executor was launched with a plan without segment movements') + + rebalance_steps = self.convert_moves_to_rebalance_steps(self.rebalance_plan.getMoves()) + + self.logger.info('Saving following rebalance execution steps:') + + move_order = 0 + for step in rebalance_steps: + step.setMoveOrder(move_order) + move_order += 1 + self.logger.info(str(step)) + + self.rebalance_schema.saveExecutionSteps(rebalance_steps) + + self.logger.info('Saved rebalance execution steps') + + self.trigger('move_to_STATE_REBALANCE_PREPARE_MOVES_DONE') + + @wrap_state_func_with_faults + def on_enter_STATE_REBALANCE_PREPARE_MOVES_DONE(self) -> None: + self.trigger('move_to_STATE_REBALANCE_EXECUTION_STARTED') + + @wrap_state_func_with_faults + def on_enter_STATE_REBALANCE_EXECUTION_STARTED(self) -> None: + + if self.rebalance_schema.allExecutionStepsAreDone(): + self.trigger('move_to_STATE_REBALANCE_EXECUTION_DONE') + return + + # In normal execution we shouldn't have IN_PROGRESS steps at this moment. + # If they are presented, it means they are left from previous interrupted run. + # Bring them back to PLANNED state, so we can try to process them again. + self.reset_in_progress_execution_steps() + + steps_to_execute = self.rebalance_schema.getExecutionSteps([RebalanceStep.Status.PLANNED, RebalanceStep.Status.APPROVE_REQUIRED]) + + if len(steps_to_execute) > 0: + + if steps_to_execute[0].getStatus() == RebalanceStep.Status.APPROVE_REQUIRED: + self.trigger('move_to_STATE_REBALANCE_EXECUTION_AWAITING_SWITCHOVER_APPROVE_STARTED') + return + + current_batch = [] + for step in steps_to_execute: + # fill current batch until we found a step that requires approve + # (it will be handled the next time we enter this state), + if (step.getStatus() == RebalanceStep.Status.APPROVE_REQUIRED or + # or till the type of RebalanceStep changes. + (len(current_batch) > 0 and (type(current_batch[0]) is not type(step)))): + break + + step.setStatus(RebalanceStep.Status.IN_PROGRESS) + self.rebalance_schema.updateExecutionStep(step) + current_batch.append(step) + + if isinstance(current_batch[0], RebalanceStepMoveMirror): + self.logger.info('Rebalance - start moving segments:') + moves = [step.getMove() for step in current_batch] + for move in moves: + self.logger.info(str(move)) + self.process_moves(moves) + self.logger.info('Rebalance - end moving segments') + else: + direction = self.RoleSwapDirection.PRIMARY_TO_MIRROR + if not isinstance(current_batch[0], RebalanceStepSwitchoverToMirror): + direction = self.RoleSwapDirection.MIRROR_TO_PRIMARY + segments = [step.getMove().seg for step in current_batch] + self.logger.info(f'Rebalance - start role swap {str(direction)} for segments:') + for segment in segments: + self.logger.info(str(segment)) + self.execute_role_swaps(segments, direction) + self.logger.info('Rebalance - end role swap') + + # TODO: check the errored segments here, once we implement rollback for the rebalance. + # For now if some error happened, the entire tool will halt its work, so if we reached this point + # just mark all steps as done. + for step in steps_to_execute: + if step.getStatus() == RebalanceStep.Status.IN_PROGRESS: + step.setStatus(RebalanceStep.Status.DONE) + self.rebalance_schema.updateExecutionStep(step) + + self.trigger('move_to_STATE_REBALANCE_MOVES_SUCCEEDED') + + @wrap_state_func_with_faults + def on_enter_STATE_REBALANCE_MOVES_SUCCEEDED(self) -> None: + self.trigger('move_to_STATE_REBALANCE_EXECUTION_STARTED') + + @wrap_state_func_with_faults + def on_enter_STATE_REBALANCE_EXECUTION_DONE(self) -> None: + self.trigger('move_to_STATE_REBALANCE_DONE') + + @wrap_state_func_with_faults + def on_enter_STATE_REBALANCE_EXECUTION_AWAITING_SWITCHOVER_APPROVE_STARTED(self) -> None: + + # Approve all consequent steps that require approval + steps = self.rebalance_schema.getExecutionSteps([RebalanceStep.Status.PLANNED, RebalanceStep.Status.APPROVE_REQUIRED]) + + assert len(steps) > 0 + + for step in steps: + if step.getStatus() != RebalanceStep.Status.APPROVE_REQUIRED: + break + # TODO: we'll need to add logic here to get approval from the user in the interactive mode, + # once we start implementing the interactive mode. + # In non-interactive mode we assume that the switchover is always approved. + step.setStatus(RebalanceStep.Status.PLANNED) + self.rebalance_schema.updateExecutionStep(step) + + self.trigger('move_to_STATE_REBALANCE_EXECUTION_AWAITING_SWITCHOVER_APPROVE_DONE') + + @wrap_state_func_with_faults + def on_enter_STATE_REBALANCE_EXECUTION_AWAITING_SWITCHOVER_APPROVE_DONE(self) -> None: + self.trigger('move_to_STATE_REBALANCE_EXECUTION_STARTED') + + @wrap_state_func_with_faults + def on_enter_STATE_REBALANCE_DONE(self) -> None: + pass + + @wrap_state_func_with_faults + def on_enter_STATE_ERROR(self) -> None: + raise Exception('Rebalance execution entered STATE_ERROR') + + # state callbacks end here diff --git a/gpMgmt/bin/gprebalance_modules/planner.py b/gpMgmt/bin/gprebalance_modules/planner.py new file mode 100755 index 000000000000..095cc61e09b2 --- /dev/null +++ b/gpMgmt/bin/gprebalance_modules/planner.py @@ -0,0 +1,1704 @@ +#!/usr/bin/env python3 + +from collections import defaultdict +import copy +import os +import pickle +from typing import Any, Tuple, List, Dict +from contextlib import closing +from dataclasses import dataclass, field +from copy import deepcopy +import math + +from gppylib.db import dbconn +import gppylib.gparray as gparray +from gprebalance_modules.rebalance_commons import * +from gprebalance_modules.solver import GreedySolver, HostId, SolverConfig +from gppylib.commands.unix import PortIsAvailable + +class PlanningError(Exception): + pass + +GROUPED = 'grouped' +SPREAD = 'spread' + +HostMapping = Dict[Host, HostId] + +@dataclass +class LogicalMove: + """ + Move operation assumed by rebalance + + Attributes: + seg: segment info + dst: destination host + target_datadir: target data directory + target_port: target port + segment_size: segment volume including tablespaces + swap_metadata: defines the order of swap moves + """ + seg: Segment + srcHost: Host + dstHost: Host + target_datadir: str + target_port: int + segment_size: Optional[SegmentSize] = None + swap_metadata: Optional[Dict[str, Any]] = None + + def __str__(self): + """Pretty print logical move""" + + # Source information + src_host = self.seg.getSegmentHostName() + src_datadir = self.seg.getSegmentDataDirectory() + src_port = self.seg.getSegmentPort() + + # Destination information + dst_host = self.dstHost.hostname + dst_datadir = self.target_datadir + dst_port = self.target_port + + # Size information (if available) + size_str = "" + if self.segment_size: + size_str = f" [{self.segment_size}]" + + return ( + f"Move Segment(content={self.seg.getSegmentContentId()}, dbid={self.seg.getSegmentDbId()}, " + f"role={self.seg.role}){size_str}\n" + f" From: {src_host}:{src_port} → {src_datadir}\n" + f" To: {dst_host}:{dst_port} → {dst_datadir}" + ) + + +class Plan: + def __init__(self): + self.target_segment_count = 0 + self.moves = None + + def getTargetSegmentCount(self) -> int: + return self.target_segment_count + + def setTargetSegmentCount(self, target_segment_count: int) -> None: + self.target_segment_count = target_segment_count + + def getMoves(self)-> List[LogicalMove]: + return self.moves + + def setMoves(self, moves: List[LogicalMove]): + self.moves = moves + + def serializePlan(self) -> bytes: + return pickle.dumps(self) + + def __str__(self): + lines = [] + + lines.append(f"\n{'BALANCE MOVES':-^80}") + if self.moves: + lines.append(f"Total moves planned: {len(self.moves)}") + lines.append("") + + for idx, move in enumerate(self.moves, 1): + lines.append(f" [{idx}] {move}") + if idx < len(self.moves): + lines.append("") + else: + lines.append(" No data migration needed.") + + + return "\n".join(lines) + +def deserializePlan(input: bytes) -> Plan: + return pickle.loads(input) + +class ShrinkPlan(Plan): + """ + Shrink plan + """ + def __init__(self, shrinked_segs: List[gparray.SegmentPair]): + Plan.__init__(self) + self.shrinked_segments = shrinked_segs + + def __str__(self): + """ + Pretty print shrink plan with segments to remove and moves to execute + """ + lines = [] + lines.append("=" * 80) + lines.append("SHRINK PLAN".center(80)) + lines.append("=" * 80) + + # Target segment count + lines.append(f"\nTarget Segment Count: {self.target_segment_count}") + + # Shrinked segments section + lines.append(f"\n{'SEGMENTS TO REMOVE':-^80}") + if self.shrinked_segments: + lines.append(f"Total segments to shrink: {len(self.shrinked_segments)}") + lines.append("") + + for idx, seg_pair in enumerate(self.shrinked_segments, 1): + lines.append(f" [{idx}] Segment Pair:") + + # Primary segment + if seg_pair.primaryDB: + lines.append(f" Primary:") + lines.append(f" Content: {seg_pair.primaryDB.content}") + lines.append(f" DbId: {seg_pair.primaryDB.dbid}") + lines.append(f" Host: {seg_pair.primaryDB.hostname}") + lines.append(f" Datadir: {seg_pair.primaryDB.datadir}") + lines.append(f" Port: {seg_pair.primaryDB.port}") + + # Mirror segment + if seg_pair.mirrorDB: + lines.append(f" Mirror:") + lines.append(f" Content: {seg_pair.mirrorDB.content}") + lines.append(f" DbId: {seg_pair.mirrorDB.dbid}") + lines.append(f" Host: {seg_pair.mirrorDB.hostname}") + lines.append(f" Datadir: {seg_pair.mirrorDB.datadir}") + lines.append(f" Port: {seg_pair.mirrorDB.port}") + + if idx < len(self.shrinked_segments): + lines.append("") + else: + lines.append(" No segments to remove.") + + # Moves section + lines.append(super().__str__()) + + lines.append("\n" + "=" * 80) + + return "\n".join(lines) + +class ConfigurationEncoder: + + @staticmethod + def encode_configuration(gparray: gparray.GpArray, + target_hosts: List[Host], + strategy: str, + resolver: HostResolver = HostResolver()) -> Tuple[SolverConfig, HostMapping]: + """ + The rebalance solvers work with abstract input segment configuration. + Each host must have an id in [0, n-1] interval, where n is a size + of full hosts set, including decommissioned and new hosts. We encode + segment placement as a vecor (h0, h1, ..., hj, ... hk), k - number of segments, + hj - host id, j - segment contentid. E.x. H0 = {p0, p1, p2, m3, m4, m5} + H1= {p3, p4, p5. m0, m1, m2}. We want to decommission H1 and add 2 new hosts H2 + and H3. Thus, the configuration will be encoded as: + primaries (0, 0, 0, 3, 3, 3) mirrors (3, 3, 3, 0, 0, 0). Host id equal 3 corresponds + to H1, since we want the decommissioned hosts be out of interval [0, nt-1] nt <= n, + where nt - number of target hosts, which the rebalance should be performed on. + The initial host load can encoded as (6, 0, 0, 6). n = 4, nt = 3. + H0 H2 H3 H1 + After rebalance we want to get the configuration such that the load (4, 4, 4, 0) + and mirroring strategy are achieved. + """ + host_mapping = {} + sorted_hosts = sorted(target_hosts, key=lambda h: h.status) + for i, h in enumerate(sorted_hosts): + host_mapping[h] = i + primary_plcmnt = [0] * gparray.get_primary_count() + mirror_plcmnt = [0] * gparray.get_primary_count() + for pair in gparray.segmentPairs: + primary_plcmnt[pair.primaryDB.content] = host_mapping[Host(hostname=pair.primaryDB.hostname, + address=resolver.get_address(pair.primaryDB.hostname))] + mirror_plcmnt[pair.mirrorDB.content] = host_mapping[Host(hostname=pair.mirrorDB.hostname, + address=resolver.get_address(pair.mirrorDB.hostname))] + n_initial = len(target_hosts) + n_target = sum([1 for h in target_hosts if h.status != HostStatus.DECOMMISSIONED]) + conf = SolverConfig(gparray.get_primary_count(), + n_target, + n_initial, + primary_plcmnt, + mirror_plcmnt, + strategy) + return (conf, host_mapping) + +class PortAllocator: + """ + Manages port allocation for segment moves + + Tracks existing ports and allocates new ones following the existing pattern + """ + + def __init__(self, gparray: gparray.GpArray, logger: Any, + verify_ports: bool = False): + """ + Initialize port allocator with existing segment information + + Args: + gparray: Current gp_segment_configuration + """ + # Map: hostname -> set of ports currently in use + self.existing_ports_by_host: Dict[str, Set[int]] = defaultdict(set) + + # Map: hostname -> (primary_base_port, mirror_base_port) + self.base_ports_by_host: Dict[str, Tuple[int, int]] = {} + + # Map: hostname -> set of ports planned to be used (for moves) + self.planned_ports_by_host: Dict[str, Set[int]] = defaultdict(set) + + # Map: hostname -> (set of primary ports, set of mirror ports) + self.existing_ports_by_role: Dict[str, Tuple[Set[int], Set[int]]] = defaultdict( + lambda: (set(), set()) + ) + + self.logger = logger + self.verify_ports = verify_ports + + # Initialize from existing segments + self._initialize_from_array(gparray) + + def _initialize_from_array(self, gparray: gparray.GpArray): + """ + Build initial port usage maps from gparray + Tracks separate port patterns for primaries and mirrors + """ + # First pass: collect all ports by role + for seg in gparray.getSegDbList(): + hostname = seg.getSegmentHostName() + port = seg.getSegmentPort() + + self.existing_ports_by_host[hostname].add(port) + + # Track ports by role (primary vs mirror) + primary_ports, mirror_ports = self.existing_ports_by_role[hostname] + if seg.isSegmentPrimary(): + primary_ports.add(port) + else: + mirror_ports.add(port) + + # Second pass: determine base ports for each role + for hostname in self.existing_ports_by_role.keys(): + primary_ports, mirror_ports = self.existing_ports_by_role[hostname] + + primary_base = min(primary_ports) if primary_ports else None + mirror_base = min(mirror_ports) if mirror_ports else None + + if primary_base is not None or mirror_base is not None: + self.base_ports_by_host[hostname] = (primary_base, mirror_base) + + def allocate_port(self, + host: Host, + current_port: int, + is_mirror: bool) -> int: + """ + Allocate a port for a segment being moved to target host + + Args: + host: Target host + current_port: Current port of segment on source host + is_mirror: True if segment is a mirror, False if primary + host_status: Status of target host (NEW or ACTIVE) + + Returns: + Port number to use on target host + """ + if host.status == HostStatus.NEW: + return self._allocate_port_new_host(host, current_port, is_mirror) + else: + return self._allocate_port_existing_host(host, current_port, is_mirror) + + def _allocate_port_new_host(self, host: Host, current_port: int, is_mirror: bool) -> int: + """ + Allocate port on a new host + + For new hosts, first primary/mirror establishes base port for that role. + Subsequent segments of same role follow the established pattern. + """ + primary_base, mirror_base = self.base_ports_by_host.get(host.hostname, (None, None)) + + # If this is the first segment of this role on new host + if is_mirror and mirror_base is None: + # Establish mirror base port + port = self._verify_and_allocate_port(host, current_port) + self.base_ports_by_host[host.hostname] = (primary_base, port) + self.planned_ports_by_host[host.hostname].add(port) + # Track by role + _, mirror_ports = self.existing_ports_by_role[host.hostname] + mirror_ports.add(port) + return port + elif not is_mirror and primary_base is None: + # Establish primary base port + port = self._verify_and_allocate_port(host, current_port) + self.base_ports_by_host[host.hostname] = (port, mirror_base) + self.planned_ports_by_host[host.hostname].add(port) + # Track by role + primary_ports, _ = self.existing_ports_by_role[host.hostname] + primary_ports.add(port) + return port + + # Base port for this role already established, find next available + return self._find_next_available_port(host, current_port, is_mirror) + + def _allocate_port_existing_host(self, host: Host, current_port: int, is_mirror: bool) -> int: + """ + Allocate port on an existing host + + Try to use segment's current port if available, otherwise find next free port + following the pattern for this role (primary/mirror) + """ + # Try to use the segment's current port if it's free + if self._is_port_available(host.hostname, current_port): + if self.verify_ports: + if not self._check_port_on_host(host, current_port): + self.logger.info(f"Port {current_port} on {host.hostname} " + "appears in use, finding alternative") + return self._find_next_available_port(host, current_port, is_mirror) + self.planned_ports_by_host[host.hostname].add(current_port) + primary_ports, mirror_ports = self.existing_ports_by_role[host.hostname] + if is_mirror: + mirror_ports.add(current_port) + else: + primary_ports.add(current_port) + return current_port + + # Port conflict - find next available port for this role + return self._find_next_available_port(host, current_port, is_mirror) + + def _is_port_available(self, hostname: str, port: int) -> bool: + """ + Check if port is available (not used by existing or planned segments) + """ + return (port not in self.existing_ports_by_host[hostname] and + port not in self.planned_ports_by_host[hostname]) + + def _check_port_on_host(self, host: Host, port: int) -> bool: + """ + Verify port is actually available on the target host + """ + if not self.verify_ports: + return True + + try: + cmd = PortIsAvailable( + name=f'check port {port} on {host.hostname}', + port=port, + ctxt=REMOTE, + remoteHost=host.address + ) + cmd.run() + + is_available = cmd.is_port_available() + + if self.logger: + status = "available" if is_available else "in use" + self.logger.debug(f"Port {port} on {host.hostname}: {status}") + + return is_available + + except Exception as e: + self.logger.warning(f"Failed to verify port {port} on {host.hostname}: {e}. Assuming available.") + # On error, assume available + return True + + def _verify_and_allocate_port(self, host: Host, preferred_port: int) -> int: + """ + Verify port is available on host and allocate it + + If verification fails, find next available port. + """ + if self._is_port_available(host.hostname, preferred_port): + if self.verify_ports and not self._check_port_on_host(host, preferred_port): + # Preferred port is actually in use, find alternative + self.logger.info(f"Preferred port {preferred_port} on {host.hostname} " + "is in use, searching for alternative") + self.existing_ports_by_host[host.hostname].add(preferred_port) + return self._find_verified_port(host, preferred_port + 1) + return preferred_port + + # Port not available in tracking, search from next + return self._find_verified_port(host, preferred_port + 1) + + def _find_verified_port(self, host: Host, start_port: int) -> int: + """ + Find an available port starting from start_port, with optional verification + """ + candidate_port = start_port + max_attempts = 1000 + + for _ in range(max_attempts): + if self._is_port_available(host.hostname, candidate_port): + # Port available in tracking + if self.verify_ports: + # Verify on actual host + if self._check_port_on_host(host, candidate_port): + return candidate_port + else: + # Port in use on host, mark and continue + self.existing_ports_by_host[host.hostname].add(candidate_port) + candidate_port += 1 + continue + else: + # No verification, trust tracking + return candidate_port + + candidate_port += 1 + + raise PlanningError( + f"Cannot find available port on host {host.hostname} " + f"after {max_attempts} attempts starting from {start_port}" + ) + + def _find_next_available_port(self, host: Host, preferred_port: int, is_mirror: bool) -> int: + """ + Find next available port following the pattern for this role + """ + # Get base port for this role on this host + primary_base, mirror_base = self.base_ports_by_host.get(host.hostname, (None, None)) + + if is_mirror: + base_port = mirror_base if mirror_base is not None else preferred_port + else: + base_port = primary_base if primary_base is not None else preferred_port + + # If no base port established yet, use preferred and establish it + if is_mirror and mirror_base is None: + self.base_ports_by_host[host.hostname] = (primary_base, base_port) + elif not is_mirror and primary_base is None: + self.base_ports_by_host[host.hostname] = (base_port, mirror_base) + + # Search for available port starting from base + candidate_port = self._find_verified_port(host, base_port) + + self.planned_ports_by_host[host.hostname].add(candidate_port) + + # Track by role + primary_ports, mirror_ports = self.existing_ports_by_role[host.hostname] + if is_mirror: + mirror_ports.add(candidate_port) + else: + primary_ports.add(candidate_port) + + return candidate_port + +class Planner: + """ + Performs the main planning procedure. Decides the operations to be performed + on given configuration. + """ + def __init__(self, logger: Any, dburl: dbconn.DbURL, gpArray: gparray.GpArray, options: Any): + self.dburl = dburl + self.gparray = gpArray + self.options = options + self.logger = logger + self.virtual_gparray = deepcopy(self.gparray) + + if not self.virtual_gparray.hasMirrors: + raise ValidationError("Cluster has mirroring disabled. Can't proceed with rebalance") + + if self.options.target_hosts_file: + self.options.target_hosts = get_hosts_from_file(self.options.target_hosts_file, "target-hosts-file") + if self.options.add_hosts_file: + self.options.add_hosts = get_hosts_from_file(self.options.add_hosts_file, "add-hosts-file") + if self.options.remove_hosts_file: + self.options.remove_hosts = get_hosts_from_file(self.options.remove_hosts_file, "remove-hosts-file") + + self.validate_options() + + self.dir_template_p, self.dir_template_m = self.get_datadirs() + + self.resolver = HostResolver() + target, add, remove = self.resolve_hosts() + + self.target_hosts, self.host_set_changed = Planner.get_target_hosts(self.virtual_gparray, + self.dir_template_p, + self.dir_template_m, + target, add, remove, + self.resolver + ) + + def validate_options(self): + # Provide basic validation of hostnames and addresses. + validate_hosts_basic(self.options.target_hosts, "target-hosts") + validate_hosts_basic(self.options.add_hosts, "add-hosts") + validate_hosts_basic(self.options.remove_hosts, "remove-hosts") + + def resolve_hosts(self) -> Tuple[List[str], List[str], List[str]]: + """ + Returns: + List of hostnames from target_hosts, add_hosts, + remove_hosts options + """ + existing_hosts = set() + for seg in self.gparray.getSegDbList(): + existing_hosts.add(seg.getSegmentHostName()) + + existing_hostname_list = list(existing_hosts) + target_hostname_list = [] + add_hostname_list = [] + remove_hostname_list = [] + + def _parse_and_resolve_hosts( + hosts_input: str, + option_name: str, + check_exists: bool = False, + check_not_exists: bool = False) -> List[str]: + """ + Parse comma-separated host list, resolve IPs to hostnames, and validate + + Returns: + List of resolved hostnames + """ + hosts_list = [h.strip() for h in hosts_input.split(',')] + resolved_hosts = [] + + for host in hosts_list: + # Resolve IP to hostname if needed + if is_ip_address(host): + hostname = self.resolver.resolve_ip(host) + if not hostname: + raise ValidationError(f"{option_name}: Cannot resolve IP {host}") + else: + hostname = host + self.resolver.resolve_hostname(hostname) + + # Check existence constraints + if check_exists or check_not_exists: + matching_host = self.resolver.find_matching_hostname(hostname, existing_hostname_list) + + if check_not_exists and matching_host: + raise ValidationError( + f"{option_name}: Host '{host}' already exists in cluster " + f"as '{matching_host}'" + ) + + if check_exists and not matching_host: + raise ValidationError( + f"{option_name}: Host '{host}' does not exist in cluster" + ) + + resolved_hosts.append(hostname) + + return resolved_hosts + + if self.options.target_hosts: + target_hostname_list = _parse_and_resolve_hosts( + self.options.target_hosts, '--target-hosts') + + if self.options.add_hosts: + add_hostname_list = _parse_and_resolve_hosts( + self.options.add_hosts, '--add-hosts', check_not_exists=True) + + if self.options.remove_hosts: + remove_hostname_list = _parse_and_resolve_hosts( + self.options.remove_hosts, '--remove-hosts', check_exists=True) + + # Resolve the rest of hostnames + for hostname in existing_hostname_list: + self.resolver.resolve_hostname(hostname) + + return target_hostname_list, add_hostname_list, remove_hostname_list + + def plan(self) -> Plan: + plan = Plan() + + self.validate_segment_status() + if self.options.target_segment_count < self.gparray.get_segment_count(): + plan = self.plan_shrink() + + elif self.options.target_segment_count > self.gparray.get_segment_count(): + raise PlanningError("Expand is not supported yet") + + if self.options.skip_rebalance: + self.logger.warning("Skipping rebalance") + return plan + + rebalance_moves = self.form_moves() + + if rebalance_moves is None: + self.logger.info("Cluster is already balanced, no segment moves will be held.") + + plan.setMoves(rebalance_moves) + plan.setTargetSegmentCount(self.options.target_segment_count) + return plan + + def validate_segment_status(self): + if len([1 for pair in self.gparray.segmentPairs\ + if not pair.primaryDB.valid or not pair.mirrorDB.valid]): + raise ValidationError("Some segments in 'down' status. ggrebalance can't proceed further") + + + def plan_shrink(self) -> ShrinkPlan: + self.logger.info("Planning shrink") + shrinkedSegs = [] + for seg_pair in self.gparray.getSegmentList(): + primary_seg = seg_pair.primaryDB + if primary_seg.getSegmentContentId() >= self.options.target_segment_count: + shrinkedSegs.append(seg_pair) + self.remove_segpair_from_array(seg_pair) + plan = ShrinkPlan(shrinkedSegs) + plan.setTargetSegmentCount(self.options.target_segment_count) + return plan + + def remove_segpair_from_array(self, segPair: gparray.SegmentPair): + """ + Removes the segment pair from gparray. + """ + self.virtual_gparray.numPrimarySegments -= 1 + self.virtual_gparray.segmentPairs = list(filter(lambda x: x.primaryDB.dbid != segPair.primaryDB.dbid, + self.virtual_gparray.segmentPairs)) + segs_list = self.virtual_gparray.getSegmentsAsLoadedFromDb() + if segs_list: + segs_list = list(filter(lambda x: x.dbid != segPair.primaryDB.dbid, + segs_list)) + if segPair.mirrorDB: + segs_list = list(filter(lambda x: x.dbid != segPair.mirrorDB.dbid, + segs_list)) + self.virtual_gparray.setSegmentsAsLoadedFromDb(segs_list) + + @staticmethod + def get_target_hosts(array: gparray.GpArray, + primary_template: str = DEFAULT_PRIMARY_TEMPLATE, + mirror_template: str = DEFAULT_MIRROR_TEMPLATE, + target_hostname_list: List[str] = [], + add_hostname_list: List[str] = [], + remove_hostname_list: List[str] = [], + resolver: HostResolver = HostResolver()) -> Tuple[List[Host], bool]: + """ + Form set of hosts where we need to rebalance segments on + Returns: + - List of Host objects with: + * For existing hosts: templates + existing datadirs filled + * For new hosts: only templates, existing datadirs empty + - Boolean indicating if host set changed + """ + hosts = {} + for seg in array.segmentPairs: + if seg.primaryDB.content >= 0: + primary_host = seg.primaryDB.hostname + mirror_host = seg.mirrorDB.hostname + if primary_host not in hosts: + hosts[primary_host] = Host(hostname=primary_host, + address=resolver.get_address(seg.primaryDB.hostname), + datadir_info=DatadirInfo(primary_template, mirror_template), + status = HostStatus.ACTIVE) + if mirror_host not in hosts: + hosts[mirror_host] = Host(hostname=mirror_host, + address=resolver.get_address(seg.mirrorDB.hostname), + datadir_info=DatadirInfo(primary_template, mirror_template), + status = HostStatus.ACTIVE) + for pair in array.segmentPairs: + primary = pair.primaryDB + mirror = pair.mirrorDB + primary_base = TemplateParser.extract_parent_directory(primary.datadir) + hosts[primary.hostname].datadir_info.existing_primary_datadirs.add(primary_base) + if mirror: + mirror_base = TemplateParser.extract_parent_directory(mirror.datadir) + hosts[mirror.hostname].datadir_info.existing_mirror_datadirs.add(mirror_base) + + host_set_changed = False + if target_hostname_list: + for host in hosts.keys(): + if host not in target_hostname_list: + hosts[host].status = HostStatus.DECOMMISSIONED + host_set_changed = True + for host in target_hostname_list: + if host not in hosts: + hosts[host] = Host(hostname=host, + address=resolver.get_address(host), + datadir_info=DatadirInfo(primary_template, mirror_template), + status = HostStatus.NEW) + host_set_changed = True + + if add_hostname_list: + for host in add_hostname_list: + if host not in hosts: + hosts[host] = Host(hostname=host, + address=resolver.get_address(host), + datadir_info=DatadirInfo(primary_template, mirror_template), + status = HostStatus.NEW) + host_set_changed = True + + if remove_hostname_list: + for host in hosts.keys(): + if host in remove_hostname_list: + hosts[host].status = HostStatus.DECOMMISSIONED + host_set_changed = True + + return list(hosts.values()), host_set_changed + + def get_datadirs(self) -> Tuple[str, str]: + """ + Get datadir templates from options or use defaults + """ + if self.options.target_datadirs: + return TemplateParser.parse_datadirs_input(self.options.target_datadirs) + elif self.options.target_datadirs_file: + return TemplateParser.parse_datadirs_file(self.options.target_datadirs_file) + else: + return DEFAULT_PRIMARY_TEMPLATE, DEFAULT_MIRROR_TEMPLATE + + def form_moves(self) -> List[LogicalMove]: + self.logger.info("Validation of rebalance possibility") + + for pair in self.virtual_gparray.segmentPairs: + prim = pair.primaryDB + mir = pair.mirrorDB + if prim.role != prim.preferred_role and mir.role != mir.preferred_role: + raise ValidationError("Current role does not match preferred role for several segments.") + + total_primaries = self.virtual_gparray.get_primary_count() + total_hosts = len([h for h in self.target_hosts\ + if h.status == HostStatus.NEW or h.status == HostStatus.ACTIVE]) + if total_hosts < 2: + raise ValidationError("Cannot perform rebalance at 1 host") + if total_primaries % total_hosts != 0: + raise ValidationError(f"Cannot evenly distribute {total_primaries}" + f" segments across {total_hosts} hosts.") + + expected_per_host = total_primaries // total_hosts + strat = self.options.mirror_mode + + if not strat: + strat = GROUPED + + if strat == SPREAD and expected_per_host > total_hosts - 1: + raise ValidationError("Cannot provide spread mirroring. Specify other " + "mirroring strategy via -m option") + + if not self.host_set_changed\ + and self.already_balanced(expected_per_host): + return None + + # dry movements are planned here + config, host_mapping = ConfigurationEncoder.encode_configuration(self.virtual_gparray, + self.target_hosts, + strat, + self.resolver) + id_to_host = {v: k for k, v in host_mapping.items()} + self.logger.info("Planning rebalance moves. Can take up to 60s.") + planning_seed_value = int.from_bytes(os.urandom(16) , 'big') + self.logger.info(f"Running randomized plan improvement with seed:{planning_seed_value}") + solution, _ = GreedySolver(config, seed=planning_seed_value).solve() + + self.logger.debug(f"Solution: {solution}") + self.logger.debug(f"Hosts: {id_to_host}") + + port_allocator = PortAllocator(self.virtual_gparray, + self.logger, + not self.options.skip_resource_estimation) + final_moves = [] + moves = [] + for seg in self.virtual_gparray.getSegDbList(): + is_mirror = seg.isSegmentMirror() + if is_mirror: + hostId = config.initial_mirror_mapping[seg.content] + final_hostId = solution[seg.content][1] + else: + hostId = config.initial_primary_mapping[seg.content] + final_hostId = solution[seg.content][0] + + if hostId != final_hostId: + #segment is moved + source_host = id_to_host[hostId] + target_host = id_to_host[final_hostId] + target_datadir = self._get_datadir_for_segment(target_host, seg.content, is_mirror) + + # Allocate port for this segment on target host + target_port = port_allocator.allocate_port( + host=target_host, + current_port=seg.getSegmentPort(), + is_mirror=is_mirror) + + moves.append(LogicalMove(seg, source_host, target_host, target_datadir, target_port)) + + + if len(moves) == 0: + return None + + resource_estimator = None + + if not self.options.skip_resource_estimation: + try: + resource_estimator = ResourceEstimator( + self.logger, + self.dburl, + self.virtual_gparray, + batch_size=getattr(self.options, 'batch_size', 16) + ) + resource_estimator.estimate_and_validate_moves(moves) + except ResourceError as e: + raise PlanningError(f"Resource validation failed: {e}") + else: + self.logger.warning("Skipping resource estimation") + # Detect swaps + swap_pairs = None + if not self.options.inplace_swap_roles: + swap_pairs = self.detect_swap_pairs(moves) + if swap_pairs: + self.logger.info(f"Detected {len(swap_pairs)} primary-mirror pairs which just swap hosts") + final_moves = self.handle_swaps(swap_pairs, moves, port_allocator, resource_estimator) + else: + final_moves = moves + if resource_estimator: + self.logger.info( + f"Estimated total data to move: " + f"{resource_estimator.total_gb(final_moves):.2f} GB" + ) + + return final_moves + + def already_balanced(self, load: int) -> bool: + primaries_by_host = defaultdict(int) + mirrors_by_host = defaultdict(int) + for pair in self.virtual_gparray.segmentPairs: + primaries_by_host[pair.primaryDB.hostname] += 1 + if pair.mirrorDB: + mirrors_by_host[pair.mirrorDB.hostname] += 1 + # This is mirroring strategy violation + if pair.mirrorDB.hostname == pair.primaryDB.hostname: + return False + for n in primaries_by_host.values(): + if n != load: + return False + for n in mirrors_by_host.values(): + if n != load: + return False + return True + + def _get_datadir_for_segment(self, host: Host, content_id: int, is_mirror: bool) -> str: + """ + Determine datadir for a segment on target host + + Logic: + - Always use template for either new/exising hosts + """ + datadir_info = host.datadir_info + + assert(host.status == HostStatus.NEW or host.status == HostStatus.ACTIVE) + # use template + template = datadir_info.mirror_template if is_mirror else datadir_info.primary_template + return TemplateParser.instantiate_template(template, host.hostname, content_id) + + def detect_swap_pairs(self, moves: List[LogicalMove]) -> List[Tuple[LogicalMove, LogicalMove]]: + """ + Detect primary-mirror swap scenarios + + A swap occurs when: + - Primary of content N moves from host A to host B + - Mirror of content N moves from host B to host A + + Returns: + List of (primary_move, mirror_move) tuples for detected swaps + """ + # Index moves by content + moves_by_content = defaultdict(dict) + for move in moves: + content_id = move.seg.getSegmentContentId() + if move.seg.isSegmentPrimary(): + moves_by_content[content_id]['primary'] = move + else: + moves_by_content[content_id]['mirror'] = move + + swap_pairs = [] + for content_id, seg_moves in moves_by_content.items(): + # Check if both primary and mirror are moved + if 'primary' not in seg_moves or 'mirror' not in seg_moves: + continue + + prim_move = seg_moves['primary'] + mir_move = seg_moves['mirror'] + + # Check if they're swapping hosts + prim_src = prim_move.seg.getSegmentHostName() + prim_dst = prim_move.dstHost.hostname + mir_src = mir_move.seg.getSegmentHostName() + mir_dst = mir_move.dstHost.hostname + + if (prim_src == mir_dst and + prim_dst == mir_src): + swap_pairs.append((prim_move, mir_move)) + self.logger.debug( + f"Detected swap for content {content_id}: " + f"{prim_src} <-> {prim_dst}" + ) + + return swap_pairs + + def select_intermediate_host(self, + primary_move: LogicalMove, + mirror_move: LogicalMove, + used_intermediate_hosts: Dict[str, int], + resource_estimator: Optional['ResourceEstimator'] = None + ) -> Host: + """ + Select an intermediate host for swap operation with basic space validation + + Naive criteria (since the executor runs all mirror moves before role swap): + 1. Not involved in this swap (not source or dest for either segment) + 2. Has sufficient space for the mirror PLUS other planned moves to that host + 3. Prefer hosts with fewer intermediate segments already assigned + + Args: + primary_move: Primary segment move + mirror_move: Mirror segment move + used_intermediate_hosts: Dict tracking hostname -> count of intermediates assigned + resource_estimator: Optional ResourceEstimator for space validation + + Returns: + Selected intermediate host + """ + content_id = primary_move.seg.getSegmentContentId() + + # Collect excluded hostnames + excluded_hosts = { + primary_move.dstHost.hostname, + mirror_move.dstHost.hostname + } + + # Find candidate hosts + candidates = [] + for host in self.target_hosts: + if host.status not in [HostStatus.ACTIVE, HostStatus.NEW]: + continue + + # Check if excluded + if host.hostname in excluded_hosts: + continue + + candidates.append(host) + + if not candidates: + raise PlanningError( + f"No intermediate host available for swap of content {content_id}. " + f"Need at least 3 hosts to perform swaps safely. Or you may allow " + f"primary-mirror coexistence by --inplace-swap-roles." + ) + + # Get mirror size for space check + mirror_size_kb = 0 + if mirror_move.segment_size: + mirror_size_kb = int(mirror_move.segment_size.total_size_kb * (1 + DISK_SPACE_SAFETY_MARGIN)) + + # Score each candidate + scored_candidates = [] + + for host in candidates: + # Check space if resource estimation is enabled + has_space = True + available_space_kb = float('inf') + filesystems = 'unknown' + + if resource_estimator: + try: + intermediate_datadir = self._get_datadir_for_segment( + host, content_id, is_mirror=True) + + has_space, available_space_kb, filesystems = \ + resource_estimator.check_space( + hostname=host.hostname, + host_address=host.address, + data_directory=intermediate_datadir, + required_size=mirror_move.segment_size + ) + + if not has_space: + self.logger.debug( + f"Host {host.hostname}: insufficient space insufficient space on {filesystems}\n" + f" Directory: {intermediate_datadir}\n" + f" (available: {available_space_kb / 1024 / 1024:.2f} GB, " + f" required: {mirror_size_kb / 1024 / 1024:.2f} GB)" + ) + continue + else: + self.logger.debug( + f"Host {host.hostname}: viable candidate\n" + f" Directory: {intermediate_datadir}\n" + f" Filesystem: {filesystems}\n" + f" Available: {available_space_kb / 1024 / 1024:.2f} GB\n" + f" Required: {mirror_size_kb / 1024 / 1024:.2f} GB" + ) + except Exception as e: + self.logger.error(f"Could not check space on {host.hostname}: {e}") + has_space = False + + if not has_space: + continue + + # Calculate score + # Factors: available space, intermediate usage + if mirror_size_kb <= 0: + space_score = 1.0 + else: + ratio = available_space_kb / mirror_size_kb + # sigmoid. more sensitive around ratio=1 + space_score = 1.0 / (1.0 + math.exp(-3.0 * (ratio - 1.0))) + + # Penalize hosts already used as intermediate + intermediate_count = used_intermediate_hosts.get(host.hostname, 0) + intermediate_score = math.exp(-1.0 * intermediate_count) + + # make score lie in [0, 1] + score = space_score * 0.7 + intermediate_score * 0.3 + + scored_candidates.append((host, score, available_space_kb, filesystems)) + + if not scored_candidates: + raise PlanningError( + f"No suitable intermediate host found for swap of content {content_id}. " + f"Need at least 3 hosts with sufficient space to perform swaps safely." + ) + + # Sort by score (higher is better) + scored_candidates.sort(key=lambda x: x[1], reverse=True) + + selected_host, selected_score, selected_space, selected_fs = scored_candidates[0] + + # Track usage + used_intermediate_hosts[selected_host.hostname] = \ + used_intermediate_hosts.get(selected_host.hostname, 0) + 1 + + self.logger.info( + f"Selected intermediate host {selected_host.hostname} for content {content_id}\n" + f" Score: {selected_score:.1f}\n" + f" Filesystems: {selected_fs}\n" + f" Available: {selected_space / 1024 / 1024:.2f} GB\n" + f" Required: {mirror_size_kb / 1024 / 1024:.2f} GB") + + if resource_estimator: + resource_estimator.reserve_space( + hostname=host.hostname, + host_address=host.address, + data_directory=self._get_datadir_for_segment(host, content_id, is_mirror=True), + dbid=mirror_move.seg.dbid, + required_size=mirror_move.segment_size + ) + + return selected_host + + def decompose_swap(self, + primary_move: LogicalMove, + mirror_move: LogicalMove, + intermediate_host: Host, + port_allocator: PortAllocator) -> Tuple[LogicalMove, LogicalMove, LogicalMove]: + """ + Decompose a swap into 3 moves through intermediate host + + Phase 1: Mirror -> Intermediate host + Phase 2: Primary -> Mirror's original host (direct move) + Phase 3: Mirror (from intermediate) -> Primary's original host + + Returns: + (phase1_move, phase2_move, phase3_move) + """ + content_id = primary_move.seg.getSegmentContentId() + + # Phase 1: Move mirror to intermediate host + intermediate_datadir = self._get_datadir_for_segment( + intermediate_host, + content_id, + is_mirror=True + ) + intermediate_port = port_allocator.allocate_port( + host=intermediate_host, + current_port=mirror_move.seg.getSegmentPort(), + is_mirror=True + ) + + phase1_move = LogicalMove( + seg=mirror_move.seg, + srcHost=mirror_move.srcHost, + dstHost=intermediate_host, + target_datadir=intermediate_datadir, + target_port=intermediate_port, + segment_size=mirror_move.segment_size, + swap_metadata={ + 'phase': 1, + 'content_id': content_id, + 'intermediate_host': intermediate_host.hostname + } + ) + + # Phase 2: Primary move (direct to final location) + # This keeps the original primary move unchanged + phase2_move = primary_move + phase2_move.swap_metadata = {'phase': 2, + 'content_id': content_id} + + # Phase 3: Move mirror from intermediate to final location (primary's original host) + final_datadir = mirror_move.target_datadir + final_port = port_allocator.allocate_port( + host=mirror_move.dstHost, + current_port=mirror_move.seg.getSegmentPort(), + is_mirror=True + ) + + # Create a copy of mirror segment with updated location (now on intermediate) + intermediate_seg = copy.deepcopy(mirror_move.seg) + intermediate_seg.hostname = intermediate_host.hostname + intermediate_seg.address = intermediate_host.address + intermediate_seg.datadir = intermediate_datadir + intermediate_seg.port = intermediate_port + + phase3_move = LogicalMove( + seg=intermediate_seg, + srcHost=intermediate_host, + dstHost=mirror_move.dstHost, # Final location (primary's original host) + target_datadir=final_datadir, + target_port=final_port, + segment_size=mirror_move.segment_size, + swap_metadata={ + 'phase': 3, + 'content_id': content_id, + 'intermediate_host': intermediate_host.hostname + } + ) + + return phase1_move, phase2_move, phase3_move + + def handle_swaps(self, + swap_pairs: List[Tuple[LogicalMove, LogicalMove]], + moves: List[LogicalMove], + port_allocator: PortAllocator, + resource_estimator: 'ResourceEstimator' + ) -> List[LogicalMove]: + swap_move_ids = set() + # Track intermediate host usage for better distribution + used_intermediate_hosts = {} + # Decompose swaps with intermediate host selection + swap_phase1_moves = [] + swap_phase2_moves = [] + swap_phase3_moves = [] + + for prim_move, mir_move in swap_pairs: + swap_move_ids.add(prim_move.seg.getSegmentDbId()) + swap_move_ids.add(mir_move.seg.getSegmentDbId()) + try: + # Select intermediate host + intermediate_host = self.select_intermediate_host( + prim_move, + mir_move, + used_intermediate_hosts, + resource_estimator # Pass the estimator with cached filesystem data + ) + # Decompose into 3 phases + phase1, phase2, phase3 = self.decompose_swap( + prim_move, + mir_move, + intermediate_host, + port_allocator + ) + swap_phase1_moves.append(phase1) + swap_phase2_moves.append(phase2) + swap_phase3_moves.append(phase3) + except Exception as e: + raise PlanningError(f"Failed to plan swap for content {prim_move.seg.getSegmentContentId()}: {e}") + + # Collect non-swap moves + non_swap_mirror_moves = [] + non_swap_primary_moves = [] + + for move in moves: + if move.seg.getSegmentDbId() not in swap_move_ids: + if move.seg.isSegmentPrimary(): + non_swap_primary_moves.append(move) + else: + non_swap_mirror_moves.append(move) + + moves_with_swap = [] + # Batch 1: All initial mirror moves (regular + phase1) + moves_with_swap.extend(non_swap_mirror_moves) + moves_with_swap.extend(swap_phase1_moves) + + # Batch 2: All primary moves (regular + phase2) + moves_with_swap.extend(non_swap_primary_moves) + moves_with_swap.extend(swap_phase2_moves) + + # Batch 3: Phase 3 mirrors (must execute last) + moves_with_swap.extend(swap_phase3_moves) + + return moves_with_swap + +@dataclass +class FilesystemRequirement: + """ + Tracks space requirements and allocations for a single filesystem on a host. + """ + space_info: DiskSpaceInfo + required_kb: int = 0 + # dbids and datadirs, which contribute to this filesystem + dbid_datadirs: Dict[int, Tuple[str, int]] = field(default_factory=dict) + # dict(dbid, dict(tablespace_base, size_kb)) contributing to this filesystem + dbid_tablespaces: Dict[int, Dict[str, int]] = field(default_factory=dict) + # tablespace_base/dbid for error msgs + tablespace_paths: Set[str] = field(default_factory=set) + + def add_datadir(self, dbid: int, directory: str, size_kb: int) -> None: + """ + Add a datadir contribution. + """ + request = int(size_kb * (1 + DISK_SPACE_SAFETY_MARGIN)) + self.dbid_datadirs[dbid] = (directory, request) + self.required_kb += request + + def add_tablespace(self, dbid: int, tbl_path: str, tbl_base: str, size_kb: int) -> None: + """ + Add a tablespace contribution. + """ + request = int(size_kb * (1 + DISK_SPACE_SAFETY_MARGIN)) + self.tablespace_paths.add(tbl_path) + if dbid not in self.dbid_tablespaces: + self.dbid_tablespaces[dbid] = defaultdict(int) + self.dbid_tablespaces[dbid][tbl_base] += request + self.required_kb += request + + def remove_datadir(self, dbid: int) -> None: + if dbid in self.dbid_datadirs: + item = self.dbid_datadirs.pop(dbid) + self.required_kb -= item[1] + + def remove_tablespaces(self, dbid: int) -> None: + if dbid in self.dbid_tablespaces: + tblsps = self.dbid_tablespaces.pop(dbid) + for _, size in tblsps.items(): + self.required_kb -= size + + @property + def available_kb(self) -> int: + return self.space_info.available_kb + + @property + def datadir_paths(self) -> Set[str]: + """for error reporting""" + paths = set() + for _, item in self.dbid_datadirs.items(): + paths.add(item[0]) + return paths + + @property + def unique_segment_count(self) -> int: + return len(self.dbid_datadirs) + \ + len(set(self.dbid_tablespaces.keys()) - set(self.dbid_datadirs.keys())) + +class ResourceEstimator: + """ + Estimates and validates resource requirements for segment moves + + This class handles move-specific resource estimation logic, + using DiskSpaceChecker for invoking remote disk operations. + """ + + def __init__(self, logger: Any, dburl: dbconn.DbURL, gparray: gparray.GpArray, batch_size: int = 16): + """ + Initialize resource estimator + + Args: + logger: Logger instance + dburl: Database url + gparray: Current GpArray configuration + batch_size: Number of parallel operations + """ + self.logger = logger + self.dburl = dburl + self.gparray = gparray + self.disk_checker = DiskSpaceChecker(logger, batch_size) + # Cache filesystem allocation data after validation + # Maps (hostname, filesystem) -> {required_kb, available_kb, datadir_moves, ...} + self.filesystem_allocations: Dict[Tuple[str, str], FilesystemRequirement] = {} + # Cache space_info_by_host for reuse + self.space_info_by_host: Dict[str, Dict[str, DiskSpaceInfo]] = {} + + def estimate_and_validate_moves(self, moves: List['LogicalMove']) -> None: + """ + Estimate resource requirements and validate moves can be performed + + This method modifies the moves in-place, setting the segment_size attribute. + + Args: + moves: List of LogicalMove objects to validate + """ + if not moves: + return + + self.logger.info(f"Estimating resource requirements for {len(moves)} segment moves...") + # Step 1: Estimate segment sizes + self._estimate_segment_sizes(moves) + self.logger.info("Validating available disk space on target hosts...") + # Step 2: Validate available space on target hosts + self._validate_and_build_allocations(moves) + + def total_gb(self, moves: List[LogicalMove]) -> int: + total_size_kb = sum(move.segment_size.total_size_kb for move in moves if move.segment_size) + return total_size_kb / 1024 / 1024 + + def _estimate_segment_sizes(self, moves: List[LogicalMove]) -> None: + """ + Estimate sizes for all segments being moved + + Populates the segment_size attribute on each LogicalMove + """ + tablespace_map = self._get_tablespace_locations([m.seg.dbid for m in moves]) + self._estimate_datadir_sizes(moves) + if tablespace_map: + self._estimate_tablespace_sizes(moves, tablespace_map) + + def _estimate_datadir_sizes(self, moves: List[LogicalMove]) -> None: + moves_by_host: Dict[str, List[LogicalMove]] = defaultdict(list) + for move in moves: + moves_by_host[move.srcHost.address].append(move) + + for host_addr, host_moves in moves_by_host.items(): + dirs = [m.seg.datadir for m in host_moves] + try: + disk_usage = self.disk_checker.get_disk_usage(host_addr, dirs) + for move in host_moves: + size_kb = disk_usage.get(move.seg.datadir, 0) + move.segment_size = SegmentSize(datadir_size_kb=size_kb) + except Exception as e: + raise ResourceError(f"Cannot estimate segment sizes on host {host_addr}: {e}") + + def _get_tablespace_locations(self, dbids: List[int]) -> Dict[int, List[str]]: + """ + Query database for tablespace locations for given segments + + Returns: + Dict mapping dbid to list of tablespace paths + """ + + if not dbids: + return {} + + oid_subq = """ + (SELECT * + FROM ( + SELECT oid FROM pg_tablespace + WHERE spcname NOT IN ('pg_default', 'pg_global') + ) AS _q1, + LATERAL gp_tablespace_location(_q1.oid) + ) AS t + """ + + segment_dbids = ','.join(f'({dbid})' for dbid in dbids) + + tablespace_location_sql = f""" + SELECT c.dbid, t.tblspc_loc||'/'||c.dbid AS tblspc_loc + FROM {oid_subq} + JOIN gp_segment_configuration AS c + ON t.gp_segment_id = c.content + WHERE c.dbid IN (VALUES {segment_dbids}) + """ + + try: + with closing(dbconn.connect(self.dburl, encoding='UTF8')) as conn: + cursor = dbconn.query(conn, tablespace_location_sql) + tablespaces = defaultdict(list) + + for dbid, loc in cursor: + tablespaces[dbid].append(loc) + + return tablespaces + except Exception as e: + raise ResourceError(f"Failed to query tablespace locations: {e}") + + def _estimate_tablespace_sizes(self, + moves: List['LogicalMove'], + tablespace_map: Dict[int, List[str]]) -> None: + """ + Estimate tablespace sizes and add to segment_size + + Modifies move.segment_size in-place + """ + if not tablespace_map: + return + + # Group tablespace directories by host + tblspace_by_host = defaultdict(list) + for move in moves: + for tbl_dir in tablespace_map.get(move.seg.dbid, []): + tblspace_by_host[move.srcHost.address].append((move.seg.dbid, tbl_dir)) + + # Process each host + for host_addr, host_tablespaces in tblspace_by_host.items(): + # Get unique directories + dirs = list(set(tblspace_dir for _, tblspace_dir in host_tablespaces)) + + try: + disk_usage = self.disk_checker.get_disk_usage(host_addr, dirs) + + # Aggregate by segment + for dbid, tblspace_dir in host_tablespaces: + size_kb = disk_usage.get(tblspace_dir, 0) + + # Find all moves for this segment and add tablespace size + for move in moves: + if move.seg.dbid == dbid and move.segment_size: + if move.segment_size.tablespace_usage is None: + move.segment_size.tablespace_usage = {} + move.segment_size.tablespace_usage[tblspace_dir] = size_kb + + except Exception as e: + self.logger.warning(f"Failed to get tablespace disk usage for host {host_addr}: {e}") + # Continue without tablespace sizes + + def _validate_and_build_allocations(self, moves: List[LogicalMove]) -> None: + """ + Check that every target filesystem has enough free space. + + Builds filesystem_allocations as a side effect so that + reserve_space() can later account for committed space. + Raises ResourceError listing all filesystems with insufficient space. + """ + self._fetch_target_space_info(moves) + self._build_filesystem_allocations(moves) + issues = self._find_space_issues() + + if issues: + details = ''.join(self._format_space_issue(i) for i in issues) + raise ResourceError( + f"Insufficient disk space for rebalance operation:\n{details}" + f"\nNote: Estimates include {int(DISK_SPACE_SAFETY_MARGIN * 100)}% safety margin" + ) + self.logger.info("Disk space validation completed successfully") + + def _fetch_target_space_info(self, moves: List[LogicalMove]) -> None: + """ + Query free space for every target directory referenced by the moves. + """ + dirs_by_host: Dict[str, Set[str]] = defaultdict(set) + for move in moves: + if not move.segment_size: + continue + dirs_by_host[move.dstHost.address].add(move.target_datadir) + for tbl_path in (move.segment_size.tablespace_usage or {}): + dirs_by_host[move.dstHost.address].add(os.path.dirname(tbl_path)) + + try: + self.space_info_by_host = self.disk_checker.check_batch_available_space( + {host: list(dirs) for host, dirs in dirs_by_host.items()} + ) + except Exception as e: + raise ResourceError(f"Failed to check available disk space: {e}") + + def _build_filesystem_allocations(self, moves: List[LogicalMove]) -> None: + """ + Populate self.filesystem_allocations from the planned moves. + + Each move contributes its datadir size to the filesystem that hosts + target_datadir, and each tablespace path to the filesystem that hosts + that tablespace. + """ + self.filesystem_allocations = {} + self.logger.debug("Aggregating space requirements by filesystem...") + + for move in moves: + if not move.segment_size: + continue + + hostname = move.dstHost.hostname + host_address = move.dstHost.address + + # datadir + datadir_fs = self._require_space_info(hostname, host_address, move.target_datadir) + fs_req = self._get_or_create_fs_requirement(hostname, datadir_fs) + fs_req.add_datadir(move.seg.dbid, move.target_datadir, move.segment_size.datadir_size_kb) + + # tablespace + for tbl_path, size_kb in (move.segment_size.tablespace_usage or {}).items(): + tbl_base = os.path.dirname(tbl_path) + tbl_fs = self._require_space_info(hostname, host_address, tbl_base) + fs_req = self._get_or_create_fs_requirement(hostname, tbl_fs) + fs_req.add_tablespace(move.seg.dbid, tbl_path, tbl_base, size_kb) + + def _find_space_issues(self) -> List[Dict]: + issues = [] + for (hostname, filesystem), fs_req in self.filesystem_allocations.items(): + required_gb = fs_req.required_kb / 1024 / 1024 + available_gb = fs_req.available_kb / 1024 / 1024 + + self.logger.debug( + f"Filesystem {filesystem} on {hostname}: " + f"required {required_gb:.2f} GB, available {available_gb:.2f} GB ") + + if fs_req.available_kb < fs_req.required_kb: + issues.append({ + 'hostname': hostname, + 'filesystem': filesystem, + 'target_dirs': sorted(fs_req.datadir_paths | fs_req.tablespace_paths), + 'num_datadirs': len(fs_req.datadir_paths), + 'num_tablespaces': len(fs_req.tablespace_paths), + 'num_segments': fs_req.unique_segment_count, + 'required_gb': required_gb, + 'available_gb': available_gb, + }) + return issues + + def _get_or_create_fs_requirement(self, hostname: str, space_info: DiskSpaceInfo) -> FilesystemRequirement: + fs_key = (hostname, space_info.filesystem) + if fs_key not in self.filesystem_allocations: + self.filesystem_allocations[fs_key] = FilesystemRequirement(space_info=space_info) + return self.filesystem_allocations[fs_key] + + def _require_space_info(self, hostname: str, host_address: str, directory: str) -> DiskSpaceInfo: + """ + Return DiskSpaceInfo for directory, raising ResourceError if not found. + """ + info = self._get_or_fetch_space_info(hostname, host_address, directory) + if not info: + raise ResourceError(f"No disk space information for {hostname}:{directory}") + return info + + def _get_or_fetch_space_info(self, hostname: str, host_address: str, directory: str) -> Optional[DiskSpaceInfo]: + """ + Return cached DiskSpaceInfo for directory (or its parent). + On a cache miss, query the host and cache the result. + """ + # Check cache + host_cache = self.space_info_by_host.get(host_address, {}) + info = host_cache.get(directory) or host_cache.get( + TemplateParser.extract_parent_directory(directory) + ) + if info: + return info + + # Cache miss - query the host + try: + fetched = self.disk_checker.check_batch_available_space( + {host_address: [directory]} + ) + info = fetched.get(host_address, {}).get(directory) + if info: + self.space_info_by_host.setdefault(host_address, {})[directory] = info + return info + except Exception as e: + self.logger.warning(f"Could not check space for {hostname}:{directory}: {e}") + return None + + def _reserve_space(self, + fs_key: Tuple[str, str], + space_info: DiskSpaceInfo, + directory: str, + required_kb: int, + dbid: int, + is_tablespace: bool) -> None: + hostname, filesystem = fs_key + if fs_key not in self.filesystem_allocations: + self.filesystem_allocations[fs_key] = FilesystemRequirement(space_info=space_info) + if is_tablespace: + self.filesystem_allocations[fs_key].add_tablespace(dbid, directory + '/' + str(dbid), directory, required_kb) + else: + self.filesystem_allocations[fs_key].add_datadir(dbid, directory, required_kb) + total_allocated = self.filesystem_allocations[fs_key].required_kb + self.logger.debug( + f"Reserved {required_kb / 1024 / 1024:.2f} GB on {hostname}:{filesystem} - " + f"total allocated: {total_allocated / 1024 / 1024:.2f} GB" + ) + + def reserve_space(self, + hostname: str, + host_address: str, + data_directory: str, + dbid: int, + required_size: SegmentSize) -> None: + + dirs_to_reserve = [(data_directory, required_size.datadir_size_kb)] + tablespace_dirs = {os.path.dirname(dir) for dir in (required_size.tablespace_usage or {})} + for tbl_dir in tablespace_dirs: + # Sum all tablespace paths that live under this dir + tbl_size_kb = sum(size_kb + for tbl_path, size_kb in required_size.tablespace_usage.items() + if os.path.dirname(tbl_path) == tbl_dir) + dirs_to_reserve.append((tbl_dir, tbl_size_kb)) + for directory, size_kb in dirs_to_reserve: + space_info = self._get_or_fetch_space_info(hostname, host_address, directory) + self._reserve_space( + fs_key=(hostname, space_info.filesystem), + space_info=space_info, + directory=directory, + required_kb=size_kb, + dbid=dbid, + is_tablespace=(directory in tablespace_dirs) + ) + + def _format_space_issue(self, issue: Dict) -> str: + """ + Format a space issue for error reporting + """ + # Build directory breakdown + dir_info = [] + if issue.get('num_datadirs', 0) > 0: + dir_info.append(f"{issue['num_datadirs']} datadir(s)") + if issue.get('num_tablespaces', 0) > 0: + dir_info.append(f"{issue['num_tablespaces']} tablespace(s)") + + dir_breakdown = ', '.join(dir_info) if dir_info else 'unknown' + + return ( + f"\n Host: {issue['hostname']}\n" + f" Filesystem: {issue['filesystem']}\n" + f" Directories: {dir_breakdown}\n" + f" Paths: {', '.join(issue['target_dirs'])}\n" + f" Segments affected: {issue['num_segments']}\n" + f" Required: {issue['required_gb']:.2f} GB\n" + f" Available: {issue['available_gb']:.2f} GB\n" + ) + + def get_allocated_space_for_filesystem(self, hostname: str, filesystem: str) -> int: + """ + Get the space already allocated to a specific filesystem + + Returns: + Allocated space in KB, or 0 if no allocations found + """ + fs_key = (hostname, filesystem) + if fs_key in self.filesystem_allocations: + return self.filesystem_allocations[fs_key]['required_kb'] + return 0 + + def check_space(self, + hostname: str, + host_address: str, + data_directory: str, + required_size: SegmentSize) -> Tuple[bool, int, str]: + """ + Check if a data and tablespaces directories have enough space. + + Args: + hostname: Target hostname + host_address: Target host address + data_directory: Target directory path + requied_size: Required space in SegmentSize + + Returns: + (has_space, min_available_kb, comma_separated_filesystems) + """ + dirs_to_check = [(data_directory, required_size.datadir_size_kb)] + tablespace_dirs = {os.path.dirname(dir) for dir in (required_size.tablespace_usage or {})} + for tbl_dir in tablespace_dirs: + # Sum all tablespace paths that live under this dir + tbl_size_kb = sum(size_kb + for tbl_path, size_kb in required_size.tablespace_usage.items() + if os.path.dirname(tbl_path) == tbl_dir) + dirs_to_check.append((tbl_dir, tbl_size_kb)) + + # Resolve each directory to a filesystem and accumulate per-filesystem + # requirements. + fs_requirements: Dict[str, int] = defaultdict(int) + fs_space_info: Dict[str, DiskSpaceInfo] = {} + + for directory, size_kb in dirs_to_check: + space_info = self._get_or_fetch_space_info(hostname, host_address, directory) + if not space_info: + return False, 0, 'unknown' + fs = space_info.filesystem + fs_requirements[fs] += int(size_kb * (1 + DISK_SPACE_SAFETY_MARGIN)) + fs_space_info[fs] = space_info + + fs_available: Dict[str, int] = {} + for fs, required_kb in fs_requirements.items(): + fs_key = (hostname, fs) + already_allocated_kb = self.filesystem_allocations[fs_key].required_kb \ + if fs_key in self.filesystem_allocations else 0 + available_kb = fs_space_info[fs].available_kb - already_allocated_kb + fs_available[fs] = available_kb + + if available_kb < required_kb: + self.logger.debug( + f"Host {hostname}: insufficient space on {fs}\n" + f" Available: {available_kb / 1024 / 1024:.2f} GB\n" + f" Required: {required_kb / 1024 / 1024:.2f} GB" + ) + return False, min(fs_available.values()), ', '.join(fs_requirements) + + return True, min(fs_available.values()), ', '.join(fs_requirements) diff --git a/gpMgmt/bin/gprebalance_modules/rebalance.py b/gpMgmt/bin/gprebalance_modules/rebalance.py new file mode 100644 index 000000000000..a7b2f2b93ba5 --- /dev/null +++ b/gpMgmt/bin/gprebalance_modules/rebalance.py @@ -0,0 +1,325 @@ +from collections import defaultdict +from datetime import datetime +import yaml +import os +from dataclasses import dataclass +from typing import List, Set, Dict, NamedTuple +from enum import Enum +from gppylib.db import dbconn +from gppylib.gparray import GpArray, Segment, MODE_NOT_SYNC, STATUS_DOWN +from gppylib.commands.base import * +from gppylib.commands.gp import * + + +class SegmentId(NamedTuple): + dbid: int + contentid: int + + +@dataclass +class SegmentSize(): + source_data_dir_usage: int + source_tablespace_usage: Dict[str, int] + + +class HostStatus(Enum): + ACTIVE = "active" + DECOMMISSIONING = "decommissioning" + NEW = "new" + REPLACEMENT = "replacement" + + +class Host: + def __init__(self, hostname: str, address: str, primary_datadirs: Set[str] = set(), + mirror_datadirs: Set[str] = set(), primary_segments: Set[SegmentId] = set(), + mirror_segments: Set[SegmentId] = set(), status: HostStatus = HostStatus.ACTIVE): + self.hostname = hostname + self.address = address + # set of strings + self.primary_datadirs = primary_datadirs + self.mirror_datadirs = mirror_datadirs + # set of SegmentId(dbid, content id) + self.primary_segments = primary_segments + self.mirror_segments = mirror_segments + self.replacement_for = None + self.status = status + + def __eq__(self, other): + return self.hostname == other.hostname and \ + self.address == other.address + + def __hash__(self): + return hash((self.hostname, self.address)) + + def __len__(self): + return len(self.primary_segments) + + def is_overloaded(self, target_load: int) -> bool: + return len(self.primary_segments) > target_load + + def is_underloaded(self, target_load: int) -> bool: + return len(self.primary_segments) < target_load + + def add_primary(self, seg): + self.primary_segments.add(seg) + + def remove_primary(self, seg): + self.primary_segments.discard(seg) + + def add_mirror(self, seg): + self.mirror_segments.add(seg) + + def remove_mirror(self, seg): + self.mirror_segments.discard(seg) + + def __str__(self): + mirrors = [] + prims = [] + for m in self.mirror_segments: + mirrors.append(f"M{m.contentid}") + for p in self.primary_segments: + prims.append(f"S{p.contentid}") + + ps = " ".join(prims) + ms = " ".join(mirrors) + return f"H_{self.hostname}:[{ps} {ms}]" + + +ClusterState = Dict[tuple[str, str], Host] + + +class MirrorStrategy(Enum): + MIRRORLESS = "none" + GROUPED = "grouped" + SPREAD = "spread" + + +from gprebalance_modules.rebalance_plan import ClusterBalancer, Move, Plan # nopep8 +from gprebalance_modules.rebalance_executor import RebalanceExecutor, CONF_DIR # nopep8 +from gprebalance_modules.rebalance_status import StatusManager, RebalanceStatus # nopep8 + + +class GPRebalance: + def __init__(self, logger, gparray, dburl, options, gpEnv): + self.logger = logger + self.dburl = dburl + self.options = options + self.original_gparray = gparray + self.conn = dbconn.connect( + self.dburl, encoding='UTF8', allowSystemTableMods=True) + if options.mirroring == 'spread': + self.target_strategy = MirrorStrategy.SPREAD + elif options.mirroring == 'grouped': + self.target_strategy = MirrorStrategy.GROUPED + else: + self.target_strategy = MirrorStrategy.MIRRORLESS + + self.current_conf = self.getHostsFromGpArray() + self.current_hosts = set(list(self.current_conf.values())) + self.target_hosts = self.current_hosts + if options.filename: + with open(options.filename, 'r') as fp: + hosts = {} + config = yaml.safe_load(fp) + for host_config in config['hosts']: + key = (host_config['hostname'], host_config['address']) + file_host = Host(hostname=host_config['hostname'], + address=host_config['address'], + primary_datadirs=set( + host_config['primary_datadirs']), + primary_segments=set(), + mirror_datadirs=set( + host_config['mirror_datadirs']), + mirror_segments=set()) + hosts[key] = file_host + if key in self.current_conf: + self.current_conf[key].primary_datadirs |= file_host.primary_datadirs + self.current_conf[key].mirror_datadirs |= file_host.mirror_datadirs + # User can explicitly mark the host to be replacement for existing one + if "replace" in host_config: + rep = tuple(host_config['replace'].split(', ')) + if rep in self.current_conf: + hosts[key].replacement_for = rep + hosts[key].status = HostStatus.REPLACEMENT + self.current_conf[rep].status = HostStatus.DECOMMISSIONING + else: + raise ValueError(f"\'replace\' field of host {host_config['hostname']} contains" + "does not belong to current configuration ") + + self.target_hosts = set(list(hosts.values())) + if len(self.current_hosts & self.target_hosts) < len(self.current_hosts): + for k, h in self.current_conf.items(): + if k not in hosts: + h.status = HostStatus.DECOMMISSIONING + + self.unpreferred_segments = self.getSegmentsUnpreferredRole() + self.segmentMap = {SegmentId( + seg.dbid, seg.content): seg for seg in self.original_gparray.getSegmentsAsLoadedFromDb()} + self.statusManager = StatusManager(self.options, self.logger, self.original_gparray, self.conn, gpEnv) + + self.executor = None + + def getSegmentsUnpreferredRole(self) -> List[tuple[Segment, Segment]]: + segs = [] + for pair in self.original_gparray.segmentPairs: + prim = pair.primaryDB + mir = pair.mirrorDB + if prim.role != prim.preferred_role and mir.role != mir.preferred_role: + segs.append((prim, mir)) + return segs + + def setMirroringStrategy(self, strategy: MirrorStrategy): + self.target_strategy = strategy + + def getHostsFromGpArray(self) -> ClusterState: + hosts = {} + for seg in self.original_gparray.getSegmentsAsLoadedFromDb(): + if seg.content >= 0: + hosts[(seg.hostname, seg.address)] = Host( + hostname=seg.hostname, address=seg.address, primary_datadirs=set(), primary_segments=set(), mirror_datadirs=set(), mirror_segments=set()) + for pair in self.original_gparray.segmentPairs: + primary = pair.primaryDB + mirror = pair.mirrorDB + key_pr = (primary.hostname, primary.address) + hosts[key_pr].primary_datadirs.add( + os.path.dirname(primary.datadir)) + hosts[key_pr].primary_segments.add( + SegmentId(primary.dbid, primary.content)) + if mirror: + key_mr = (mirror.hostname, mirror.address) + hosts[key_mr].mirror_datadirs.add( + os.path.dirname(mirror.datadir)) + hosts[key_mr].mirror_segments.add( + SegmentId(mirror.dbid, mirror.content)) + return hosts + + def getNewHosts(self) -> tuple[Set, Set]: + new_hosts = self.target_hosts.difference( + self.current_hosts & self.target_hosts) + replacement_hosts = set() + other = set() + for h in new_hosts: + if h.status == HostStatus.REPLACEMENT: + replacement_hosts.add(h) + else: + h.status = HostStatus.NEW + other.add(h) + return other if len(other) > 0 else None, replacement_hosts if len(replacement_hosts) > 0 else None + + def createPlan(self) -> Plan: + primaries_count = self.original_gparray.get_primary_count() + total_hosts = len(self.target_hosts) + assert (primaries_count % total_hosts == 0) + self.logger.info('Planning rebalance...') + + new_hosts, replacement_hosts = self.getNewHosts() + target_load = primaries_count // total_hosts + original_segments_map = {SegmentId( + seg.dbid, seg.content): seg for seg in self.original_gparray.getSegmentsAsLoadedFromDb()} + + balancer = ClusterBalancer(self.current_conf, (new_hosts, replacement_hosts), + original_segments_map, target_load, self.target_strategy) + + return balancer.getPlan(balancer.balance()) + + def save_plan(self, plan: Plan): + # pickle the plan in conf directory + datadir = self.options.coordinator_data_directory + CONF_DIR + os.makedirs(datadir, exist_ok=True) + plan.save_to_file(datadir, "plan") + + def load_plan(self, datadir, rollback): + filename = datadir + if rollback: + filename += "/rollback_plan.pkl" + else: + filename += "/plan.pkl" + if not os.path.exists(filename): + raise FileNotFoundError(f"No pickle file found at {filename}") + with open(filename, 'rb') as f: + plan = pickle.load(f) + return plan + + def execute_plan(self, plan: Plan): + self.executor = RebalanceExecutor(plan, + self.original_gparray, + self.segmentMap, + plan.in_conf, + self.logger, + self.statusManager, + self.conn, + self.dburl, + self.options) + self.executor.execute_moves() + + def get_state_from_file(self): + """Returns expansion state from status file""" + status = self.statusManager.get_current_status()[0] + if status: + return RebalanceStatus(status) + return None + + def cleanup_directory(self): + self.logger.info('Dropping rebalance directory') + datadir = self.options.coordinator_data_directory + CONF_DIR + cmd = RemoveDirectory("Dropping rebalance directory", datadir) + cmd.run(validateAfter=True) + + def remove_status_file(self): + self.logger.info('Dropping status file') + if self.statusManager: + self.statusManager.remove_all() + + + def rollback(self, plan:Plan): + new_plan = Plan() + cb = ClusterBalancer(self.current_conf, (set(), set()), + self.segmentMap, 0, MirrorStrategy.MIRRORLESS) + new_moves, rls = cb.get_moves_between_states(self.current_conf, plan.in_conf) + for new_move in new_moves: + seg = plan.segmentMap[new_move.segid] + if self.segmentMap[new_move.segid].role == seg.role: + new_move.target_datadir = seg.datadir + new_move.target_port = seg.port + else: + for id, pair_seg in plan.segmentMap.items(): + if new_move.segid.contentid == id.contentid and new_move.segid.dbid != id.dbid: + assert(pair_seg.role == self.segmentMap[new_move.segid].role) + new_move.target_datadir = pair_seg.datadir + new_move.target_port = pair_seg.port + break + for segid, current_seg in self.segmentMap.items(): + target_seg = plan.segmentMap[segid] + if (current_seg.hostname, current_seg.address) == (target_seg.hostname, target_seg.address) \ + and current_seg.role == target_seg.role and current_seg.datadir != target_seg.datadir: + # Are there any cases besides swap when previously rebalance moved segment + # to the same host but to different dir? + assert(current_seg.role == 'm') + move = Move(segid, self.current_conf[(current_seg.hostname, current_seg.address)], + self.current_conf[(current_seg.hostname, current_seg.address)], True, + target_seg.datadir, target_seg.port) + new_moves.append(move) + + new_plan.moves = new_moves + new_plan.segmentMap = self.segmentMap + new_plan.in_conf = self.current_conf + new_plan.out_conf = plan.in_conf + + self.executor = RebalanceExecutor(new_plan, self.original_gparray, self.segmentMap, + self.current_conf, self.logger, self.statusManager, + self.conn, self.dburl, self.options) + self.executor.execute_moves() + + def resume(self, plan): + """TODO: implement proper state handling and provide + possibility to perform rebalance after fails""" + raise NotImplementedError('Resuming operation is not implemented. Call gprebalance -c and rerun') + + def shutdown(self): + self.logger.info('Shutting down gprebalance...') + if self.executor: + self.executor.shutdown() + self.executor = None + if self.conn: + self.conn.close() + self.conn = None diff --git a/gpMgmt/bin/gprebalance_modules/rebalance_commons.py b/gpMgmt/bin/gprebalance_modules/rebalance_commons.py new file mode 100755 index 000000000000..98429b0f7703 --- /dev/null +++ b/gpMgmt/bin/gprebalance_modules/rebalance_commons.py @@ -0,0 +1,742 @@ +#!/usr/bin/env python3 + +import base64 +from dataclasses import dataclass +import ipaddress +import os +import pickle +import re +import socket +from typing import Any, Dict, List, Set, Optional, Tuple +from enum import IntEnum +from gppylib.gparray import Segment, GpArray +from gppylib.commands.base import REMOTE, WorkerPool +from gppylib.commands.unix import Hostname, DiskFree, DiskUsage +from gppylib.operations.validate_disk_space import FileSystem + +class ValidationError(Exception): + pass + +class ResourceError(Exception): + pass + +DEFAULT_PRIMARY_TEMPLATE = '/data1/primary/gpseg{content}' +DEFAULT_MIRROR_TEMPLATE = '/data1/mirror/gpseg{content}' + +class HostStatus(IntEnum): + ACTIVE = 1 + NEW = 2 + DECOMMISSIONED = 3 + +@dataclass +class DatadirInfo: + """ + Stores both template and actual datadirs for a host + """ + primary_template: str + mirror_template: str + # Actual paths from existing segment + existing_primary_datadirs: Set[str] + existing_mirror_datadirs: Set[str] + + def __init__(self, primary_template: str, mirror_template: str): + self.primary_template = primary_template + self.mirror_template = mirror_template + self.existing_primary_datadirs = set() + self.existing_mirror_datadirs = set() + +@dataclass +class Host: + """ + Segment host representaion + + Attributes: + hostname: hostname from gp_segment_configuration + address: address from gp_segment_configuration + primary_datadirs: set of datadirs which primary catalogs belong to + mirror_datadirs: set of datadirs which mirror catalogs belong to + status: intendend host usage + """ + hostname: str + address: str + datadir_info: DatadirInfo = None + status: HostStatus = None + + def __hash__(self): + return hash((self.hostname, self.address)) + + def __eq__(self, other): + if not isinstance(other, Host): + return NotImplemented + return self.hostname == other.hostname and self.address == other.address + + def __str__(self): + pass + +DISK_SPACE_SAFETY_MARGIN = 0.10 + +@dataclass +class SegmentSize: + """ + Storage size of segment instance with all tablespace info + + Attributes: + datadir_size_kb: Size of main datadir in KB + tablespace_usage: Dict mapping tablespace paths to their sizes in KB + total_size_kb: Total size including tablespaces + """ + datadir_size_kb: int + tablespace_usage: Optional[Dict[str, int]] = None + + @property + def total_size_kb(self) -> int: + """Calculate total size including tablespaces""" + total = self.datadir_size_kb + if self.tablespace_usage: + total += sum(self.tablespace_usage.values()) + return total + + def __str__(self): + """Human-readable size string""" + size_mb = self.total_size_kb / 1024 + if size_mb < 1024: + return f"{size_mb:.2f} MB" + else: + size_gb = size_mb / 1024 + return f"{size_gb:.2f} GB" + + def __repr__(self): + return f"SegmentSize(datadir={self.datadir_size_kb}KB, tablespaces={self.tablespace_usage})" + +class TemplateParser: + """ + Handles parsing and validation of directory templates + """ + + VALID_PLACEHOLDERS = {'hostname', 'content'} + PLACEHOLDER_PATTERN = r'\{(\w+)\}' + + @classmethod + def parse_datadirs_input(cls, input_str: str) -> Tuple[str, str]: + """ + Parse --target-datadirs input and return (primary_template, mirror_template) + + Handles: + - "/data/primary/gpseg{content}, /data/mirror/gpseg{content}" -> as is + - "/data/primary, /data/mirror" -> adds gpseg{content} + - "/data/primary/{hostname}, /data/mirror/{hostname}" -> adds gpseg{content} + - '"/data/primary", "/data/mirror"' -> removes quotes + - --target-datadirs="/dir1, /dir2" -> handles shell quoting + """ + # Split by comma, respecting quotes + parts = cls._split_respecting_quotes(input_str) + + if len(parts) != 2: + raise ValidationError( + '--target-datadirs should have format: ' + '"/data/primary/gpseg{content}, /data/mirror/gpseg{content}". ' + 'Available templated parameters: {hostname}, {content}' + ) + + # Clean and normalize each part + primary_template = cls._normalize_template(parts[0]) + mirror_template = cls._normalize_template(parts[1]) + + # Validate templates + cls._validate_templates(primary_template, mirror_template) + + return primary_template, mirror_template + + @classmethod + def _split_respecting_quotes(cls, input_str: str) -> list: + """ + Split input string by comma, respect quoted sections. + + Handles cases like: + - "/dir1, /dir2" -> ["/dir1", "/dir2"] + - '"/di,r1/", "/dir2"' -> ["/di,r1/", "/dir2"] + """ + # Manual parsing: split by comma while respecting quotes + parts = [] + current_part = [] + in_double_quotes = False + in_single_quotes = False + i = 0 + + while i < len(input_str): + char = input_str[i] + + # Track quote state + if char == '"' and not in_single_quotes: + in_double_quotes = not in_double_quotes + current_part.append(char) + elif char == "'" and not in_double_quotes: + in_single_quotes = not in_single_quotes + current_part.append(char) + elif char == ',' and not in_double_quotes and not in_single_quotes: + # Found unquoted comma - this is our delimiter + parts.append(''.join(current_part).strip()) + current_part = [] + else: + current_part.append(char) + + i += 1 + + # Add the last part + if current_part: + parts.append(''.join(current_part).strip()) + + # Clean up: remove empty parts + parts = [p for p in parts if p] + + return parts + + @classmethod + def _normalize_template(cls, path: str) -> str: + """ + Clean path from quotes and normalize template + - If it contains placeholders, validate and return as-is + - If it doesn't contain {content} placeholders, append gpseg{content} + """ + + path = path.strip() + if len(path) >= 2: + if (path[0] == '"' and path[-1] == '"') or (path[0] == "'" and path[-1] == "'"): + path = path[1:-1].strip() + + if not path: + raise ValidationError('Directory path cannot be empty') + + if not path.startswith('/'): + raise ValidationError( + f'Directory path must be absolute: {path}' + ) + + placeholders = re.findall(cls.PLACEHOLDER_PATTERN, path) + + # Validate placeholders + for placeholder in placeholders: + if placeholder not in cls.VALID_PLACEHOLDERS: + raise ValidationError( + f'Invalid placeholder {{{placeholder}}}. ' + f'Valid placeholders are: {", ".join("{" + p + "}" for p in cls.VALID_PLACEHOLDERS)}' + ) + + # If no placeholders, add default gpseg{content} + if not placeholders or ('content' not in placeholders): + # Remove trailing slash if present + path = path.rstrip('/') + return f'{path}/gpseg{{content}}' + + return path + + @classmethod + def _validate_templates(cls, primary_template: str, mirror_template: str) -> None: + """ + Validate primary and mirror templates for common issues + """ + # Check if templates are identical + if primary_template == mirror_template: + raise ValidationError( + 'Primary and mirror templates cannot be identical. ' + f'Both are: {primary_template}' + ) + + # Validate both contain {content} placeholder + # (should always be true after normalization, but double-check) + if '{content}' not in primary_template: + raise ValidationError( + f'Primary template must contain {{content}} placeholder: {primary_template}' + ) + + if '{content}' not in mirror_template: + raise ValidationError( + f'Mirror template must contain {{content}} placeholder: {mirror_template}' + ) + + @classmethod + def parse_datadirs_file(cls, filepath: str) -> Tuple[str, str]: + """ + Parse --target-datadirs-file + Expected format (2 lines): + /data/primary/gpseg{content} + /data/mirror/gpseg{content} + Available templated parameters: {hostname}, {content} + """ + + with open(filepath, 'r') as f: + lines = [line.strip() for line in f.readlines() if line.strip()] + + if len(lines) != 2: + raise ValidationError( + f'File {filepath} should contain exactly 2 lines: ' + 'primary template and mirror template' + ) + + primary_template = cls._normalize_template(lines[0]) + mirror_template = cls._normalize_template(lines[1]) + + cls._validate_templates(primary_template, mirror_template) + + return primary_template, mirror_template + + @staticmethod + def extract_parent_directory(datadir: str) -> str: + """ + Extract parent directory from an actual segment datadir + + This handles arbitrary naming conventions by just getting the parent. + + Examples: + /data/primary/gpseg0 -> /data/primary + + Returns: + Parent directory path (without trailing slash) + """ + return os.path.dirname(datadir.rstrip('/')) + + @staticmethod + def instantiate_template(template: str, hostname: str = None, content: int = None) -> str: + """ + Instantiate a template with actual values + """ + result = template + if hostname is not None: + result = result.replace('{hostname}', hostname) + if content is not None: + result = result.replace('{content}', str(content)) + return result + +def is_ip_address(ip_str: str): + try: + ipaddress.ip_address(ip_str) + return True + except ValueError: + return False + +class HostResolver: + """ + Utility class to resolve and match hostnames with IP addresses + Supports multiple IPs per hostname and maintains bidirectional mappings + """ + def __init__(self): + # hostname -> set of IP addresses + self._hostname_to_ips: Dict[str, Set[str]] = {} + # IP address -> hostname + self._ip_to_hostname: Dict[str, str] = {} + + def get_address(self, hostname: str) -> str: + """ + Get primary IP address for hostname + Returns first IP address or hostname if not resolved + """ + ips = self._hostname_to_ips.get(hostname, set()) + if ips: + # Return first IP (sorted for consistency) + return sorted(ips)[0] + return hostname + + def get_all_addresses(self, hostname: str) -> Set[str]: + """ + Get all IP addresses associated with hostname + """ + return self._hostname_to_ips.get(hostname, set()) + + def get_hostname(self, ip: str) -> str: + """ + Get hostname for IP address + Returns hostname or IP if not resolved + """ + return self._ip_to_hostname.get(ip, ip) + + def resolve_hostname(self, hostname: str) -> Optional[str]: + """ + Resolve hostname to IP addresses + Caches all IP addresses associated with hostname + + Returns: + Primary IP address (first one found), or None if resolution fails + """ + # Already resolved + if hostname in self._hostname_to_ips: + return self.get_address(hostname) + + try: + # Get all addresses for this hostname (IPv4 and IPv6) + addr_info = socket.getaddrinfo( + hostname, + None, + socket.AF_UNSPEC, + socket.SOCK_STREAM + ) + + ips = set() + for info in addr_info: + ip = info[4][0] + # For IPv6 + if ':' in ip: + ip = ip.split('%')[0] + ips.add(ip) + + if ips: + # Store hostname -> IPs mapping + self._hostname_to_ips[hostname] = ips + + # Store reverse mappings (IP -> hostname) + for ip in ips: + self._ip_to_hostname[ip] = hostname + + # Return primary IP + return sorted(ips)[0] + else: + return None + + except (socket.gaierror, socket.error) as e: + return None + + def resolve_ip(self, ip_str: str) -> Optional[str]: + """ + Reverse resolve IP to hostname using remote command + + Returns: + Hostname or None if resolution fails + """ + # Already resolved + if ip_str in self._ip_to_hostname: + return self._ip_to_hostname[ip_str] + + try: + # Validate it's a valid IP first + ipaddress.ip_address(ip_str) + + # Get hostname from remote host + cmd = Hostname('hostname', ctxt=REMOTE, remoteHost=ip_str) + cmd.run() + + if not cmd.was_successful(): + return None + + hostname = cmd.get_hostname() + + if hostname: + # Store reverse mapping + self._ip_to_hostname[ip_str] = hostname + + # Also store forward mapping + if hostname not in self._hostname_to_ips: + self._hostname_to_ips[hostname] = set() + self._hostname_to_ips[hostname].add(ip_str) + + return hostname + else: + return None + + except (ValueError, Exception) as e: + return None + + def hosts_match(self, host1: str, host2: str) -> bool: + """ + Check if two hosts match (considering hostname/IP resolution) + Handles cases where hosts are specified as hostname or IP + + Returns: + True if hosts represent the same machine + """ + # Direct match + if host1 == host2: + return True + + host1_normalized = host1.split('%')[0] if ':' in host1 else host1 + host2_normalized = host2.split('%')[0] if ':' in host2 else host2 + + if host1_normalized == host2_normalized: + return True + + # Check if both are IPs + is_ip1 = is_ip_address(host1) + is_ip2 = is_ip_address(host2) + + if is_ip1 and is_ip2: + # Both are IPs - they don't match if not equal + return False + + # One or both are hostnames - resolve and compare + if is_ip1 and not is_ip2: + # host1 is IP, host2 is hostname + # Resolve host2 to get its IPs + ips_of_host2 = self._hostname_to_ips.get(host2) + if not ips_of_host2: + # Try to resolve + self.resolve_hostname(host2) + ips_of_host2 = self._hostname_to_ips.get(host2) + + if ips_of_host2: + return host1 in ips_of_host2 + + if not is_ip1 and is_ip2: + # host1 is hostname, host2 is IP + # Resolve host1 to get its IPs + ips_of_host1 = self._hostname_to_ips.get(host1) + if not ips_of_host1: + # Try to resolve + self.resolve_hostname(host1) + ips_of_host1 = self._hostname_to_ips.get(host1) + + if ips_of_host1: + return host2 in ips_of_host1 + + if not is_ip1 and not is_ip2: + # Both are hostnames + # Resolve both and check if they share any IPs + ips1 = self._hostname_to_ips.get(host1) + if not ips1: + self.resolve_hostname(host1) + ips1 = self._hostname_to_ips.get(host1, set()) + + ips2 = self._hostname_to_ips.get(host2) + if not ips2: + self.resolve_hostname(host2) + ips2 = self._hostname_to_ips.get(host2, set()) + + # Check if they share any IP addresses + if ips1 and ips2: + return bool(ips1 & ips2) + + return False + + def find_matching_hostname(self, target_host: str, existing_hosts: List[str]) -> Optional[str]: + """ + Find if target_host matches any existing host + + Returns: + The matching existing host name, or None if no match + """ + for existing_host in existing_hosts: + if self.hosts_match(target_host, existing_host): + return existing_host + return None + +def validate_hostname(hostname:str): + if len(hostname) > 255: + raise ValidationError(f"Hostname '{hostname}' exceeds maximum length of 255 characters") + + if not re.match(r'^[a-zA-Z0-9._-]+$', hostname): + raise ValidationError(f"Hostname '{hostname}' contains invalid characters. " + "Only ASCII letters, digits, hyphen, underscore, and dot are allowed") + +def validate_hosts_basic(hosts: str, option_name: str): + + if not hosts: + return + + target_hosts = list(map(str.strip, hosts.split(','))) + + # Remove empty strings + target_hosts = [h for h in target_hosts if h] + if not target_hosts: + raise ValidationError(f" --{option_name}: No valid hosts provided") + + seen_hosts = set() + has_ip = False + has_hostname = False + for host in target_hosts: + # Check for duplicates + if host in seen_hosts: + raise ValidationError(f" --{option_name}: Duplicate host '{host}' found") + seen_hosts.add(host) + + if is_ip_address(host): + has_ip = True + continue + has_hostname = True + validate_hostname(host) + if has_ip and has_hostname: + raise ValidationError(f" --{option_name} must not contain IP adress and hostname simultaniously") + +def get_hosts_from_file(file, option_name) -> str: + hosts = [] + with open(file, 'r') as fp: + i = 0 + for line in fp: + if i >= 1000: + raise ValidationError(f" --{option_name} contains more than 1000 hosts") + hostname = line.strip() + if hostname != '': + hosts.append(line.strip()) + i += 1 + if len(hosts) == 0: + raise Exception(f"Empty '{file}' file") + return ", ".join(hosts) + +@dataclass +class SegmentId: + """Identifier for a segment""" + dbid: int + content: int + + def __hash__(self): + return hash((self.dbid, self.content)) + + def __eq__(self, other): + if not isinstance(other, SegmentId): + return NotImplemented + return self.dbid == other.dbid and self.content == other.content + +@dataclass +class DiskSpaceInfo: + """ + Disk space information for a filesystem + + Attributes: + filesystem: Filesystem name/device + available_kb: Available disk space in KB + directory: Directory that was checked + """ + filesystem: str + available_kb: int + directory: str + + @property + def available_mb(self) -> float: + return self.available_kb / 1024 + + @property + def available_gb(self) -> float: + return self.available_mb / 1024 + + def __str__(self): + return f"Available: {self.available_gb:.2f} GB on {self.filesystem}" + +class DiskSpaceChecker: + """ + Utility for checking disk space on local and remote hosts + """ + + def __init__(self, logger: Any, batch_size: int = 16): + """ + Initialize disk space checker + + Args: + logger: Logger instance + batch_size: Number of parallel operations + """ + self.logger = logger + self.batch_size = batch_size + + def get_disk_usage(self, hostaddr: str, directories: List[str]) -> Dict[str, int]: + """ + Get the disk usage for the given set of directories on the targeted host + + Args: + hostaddr: Host address (sometimes can be hostname) to check + directories: List of directories to check + + Returns: + Dictionary mapping directories to disk usage in KB + """ + dirs_disk_usage = {} + + if not directories: + return dirs_disk_usage + + pool = WorkerPool(numWorkers=min(len(directories), self.batch_size)) + try: + for directory in directories: + cmd = DiskUsage('check segment disk space used', + directory, ctxt=REMOTE, remoteHostAddr=hostaddr) + pool.addCommand(cmd) + pool.join() + finally: + pool.haltWork() + pool.joinWorkers() + + for cmd in pool.getCompletedItems(): + if not cmd.was_successful(): + raise Exception(f"Unable to check disk usage on segment: {cmd.get_results().stderr}") + + dirs_disk_usage[cmd.directory] = cmd.kbytes_used() + + return dirs_disk_usage + + def get_available_space(self, hostaddr: str, directories: List[str]) -> Dict[str, DiskSpaceInfo]: + """ + Get available disk space information for directories on remote host + + Uses DiskFree command which runs calculate_disk_free.py script. + This handles the case where directories don't exist yet by walking + up the path until it finds an existing directory. + + Args: + hostaddr: Host address to check + directories: List of directories/paths to check + + Returns: + Dictionary mapping directory to DiskSpaceInfo + """ + if not directories: + return {} + + filesystems = self._get_filesystems(hostaddr, directories) + + # Build result mapping + result = {} + for fs in filesystems: + # Each FileSystem has a list of directories it applies to + for directory in fs.directories: + result[directory] = DiskSpaceInfo( + filesystem=fs.name, + available_kb=fs.disk_free, + directory=directory + ) + + return result + + def _get_filesystems(self, hostaddr: str, directories: List[str]) -> List[FileSystem]: + """ + Get filesystem information for directories on target host + + Args: + hostaddr: Host address + directories: List of directories + + Returns: + List of FileSystem objects + + """ + filesystems = [] + + cmd = DiskFree(hostaddr, directories) + + cmd.run() + + if not cmd.was_successful(): + raise Exception(f"Failed to check disk free on target segment: {cmd.get_results().stderr}") + + # Decode the pickled result + filesystems = pickle.loads( + base64.urlsafe_b64decode(cmd.get_results().stdout)) + + return filesystems + + def check_batch_available_space(self, + directories_by_host: Dict[str, List[str]]) -> Dict[str, Dict[str, DiskSpaceInfo]]: + """ + Check available space for multiple directories across multiple hosts + + Args: + directories_by_host: Dict mapping host address to list of directories + + Returns: + Dict mapping host address to dict of (directory -> DiskSpaceInfo) + """ + results = {} + + for hostaddr, directories in directories_by_host.items(): + try: + space_info = self.get_available_space(hostaddr, directories) + results[hostaddr] = space_info + except Exception as e: + self.logger.error(f"Failed to check available space on {hostaddr}: {e}") + raise + + return results diff --git a/gpMgmt/bin/gprebalance_modules/rebalance_executor.py b/gpMgmt/bin/gprebalance_modules/rebalance_executor.py new file mode 100644 index 000000000000..a36c2f738d3b --- /dev/null +++ b/gpMgmt/bin/gprebalance_modules/rebalance_executor.py @@ -0,0 +1,1092 @@ +import base64 +import fcntl +import multiprocessing +import time +from collections import defaultdict +import pickle +from typing import List, Dict, Optional, Set, Tuple +from gprebalance_modules.rebalance_plan import Move, Plan # nopep8 +from gprebalance_modules.rebalance_status import StatusManager, RebalanceStatus, MoveStatus +from gprebalance_modules.rebalance import ClusterState, SegmentId, SegmentSize, Host +from gppylib.gparray import GpArray, Segment +from gppylib.db import dbconn +from gppylib.commands.base import * +from gppylib.commands.gp import * +from gppylib.commands.unix import DiskFree, DiskUsage, RemoveDirectory, getLocalHostname, getUserName +from gppylib.operations.validate_disk_space import FileSystem +from gppylib.parseutils import * +from gppylib.programs.clsRecoverSegment import GpRecoverSegmentProgram +from gppylib.system import configurationInterface, configurationImplGpdb, fileSystemInterface, \ + fileSystemImplOs, osInterface, osImplNative, faultProberInterface, faultProberImplGpdb +from gppylib.userinput import * + +MAX_BATCH_SIZE = 128 +FILENAME = "/move_" +CONF_DIR = "/rebalance" +DEFAULT_PRIMARY_PREF = "/data/primary" +DEFAULT_MIRROR_PREF = "/data/mirror" +GPRECOVERSEG_DIR = 'gpAdminLogs/rebalance' + +begining_timestamp = None +segment_prefix = "gpseg" + +class InsufficientDiskSpaceError(Exception): + pass + + +class NoValidDataDirectories(Exception): + pass + + +class RecoveryProcess: + @staticmethod + def run_recovery(cmd_args: list, result_queue: multiprocessing.Queue, log_file: str): + try: + #prevent signal propagation from parent + os.setpgrp() + signal.signal(signal.SIGINT, signal.SIG_IGN) + + log_fd = os.open(log_file, os.O_WRONLY | os.O_CREAT | os.O_APPEND) + flags = fcntl.fcntl(log_fd, fcntl.F_GETFL) + fcntl.fcntl(log_fd, fcntl.F_SETFL, flags | os.O_NONBLOCK) + os.dup2(log_fd, 1) # stdout + os.dup2(log_fd, 2) # stderr + os.close(log_fd) + + import gppylib.gplog as gplog + if gplog._LOGGER: + for handler in gplog._LOGGER.handlers[:]: + handler.close() + gplog._LOGGER.removeHandler(handler) + gplog._LOGGER = None + + gplog._FILENAME = None + gplog._DEFAULT_FORMATTER = None + gplog._LITERAL_FORMATTER = None + gplog._SOUT_HANDLER = None + gplog._FILE_HANDLER = None + + # Register all necessary interfaces to run a gprecoverseg + # in a separate process + configurationInterface.registerConfigurationProvider( + configurationImplGpdb.GpConfigurationProviderUsingGpdbCatalog()) + fileSystemInterface.registerFileSystemProvider( + fileSystemImplOs.GpFileSystemProviderUsingOs()) + osInterface.registerOsProvider( + osImplNative.GpOsProviderUsingNative()) + faultProberInterface.registerFaultProber( + faultProberImplGpdb.GpFaultProberImplGpdb()) + + local_parser = GpRecoverSegmentProgram.createParser() + local_options, args = local_parser.parse_args(cmd_args) + + gplog.setup_tool_logging("gprecoverseg", getLocalHostname(), + getUserName(), + logdir=local_options.logfileDirectory) + + # Create and run the program + cmd = GpRecoverSegmentProgram.createProgram(local_options, args) + cmd.run() + + except SystemExit as e: + error_msg = None + if e.code != 0: + error_msg = f"Gprecoverseg failed with exit code: {e.code}. See the log in {log_file}" + result_queue.put({ + "status": "FAILED" if e.code != 0 else "SUCCESS", + "error": error_msg + }) + except Exception as e: + error_msg = f"Error in gprecoverseg process: {str(e)}" + result_queue.put({ + "status": "FAILED", + "error": error_msg + }) + finally: + cmd.cleanup() + + +class SingleMoveCommand(SQLCommand): + def __init__(self, name: str, step_details, logger, statusManager: StatusManager): + self.status_manager = statusManager + self.logger = logger + (self.segment, self.move, self.segmentSize, + self.conf_dir, self.needs_switch, self.options) = step_details + + self.move_error = False + + SQLCommand.__init__(self, name) + + def write_gprecoverseg_config(self): + filename = self.conf_dir + FILENAME + "dbid" + str(self.segment.dbid) + with open(filename, 'w') as fp: + line = (f"{canonicalize_address(self.segment.address)}|" + f"{self.segment.port}|{self.segment.datadir} " + f"{canonicalize_address(self.move.dstHost.address)}|" + f"{self.move.target_port}|" + f"{self.move.target_datadir}") + self.logger.info( + "About to run gprecoverseg for mirror move " + f"(dbid = {self.segment.dbid}, content = {self.segment.content}) {line}") + fp.write(line) + return filename + + def run(self, validateAfter=False): + status_conn = None + try: + + segsize = self.segmentSize.source_data_dir_usage + if self.segmentSize.source_tablespace_usage: + segsize += sum(self.segmentSize.source_tablespace_usage.values()) + self.status_manager.update_move_status([self.segment.dbid], MoveStatus.IN_PROGRESS) + + filename = self.write_gprecoverseg_config() + + strtime=datetime.datetime.now().strftime('%Y%m%d_%H%M%S') + + # in order to run gprecoverseg processes separately and avoid any races + # for resources gprecoverseg generates, we create separate log directories. + log_dir = f"{os.path.join(os.environ.get('HOME', '.'),GPRECOVERSEG_DIR)}/gprecoverseg_dbid{self.segment.dbid}_{strtime}" + mkdirCmd = MakeDirectory("rebalance log dir", log_dir) + mkdirCmd.run(validateAfter=True) + + mkdirCmd = MakeDirectory("rebalance log dir", log_dir, ctxt=REMOTE, remoteHost=self.move.srcHost.hostname) + mkdirCmd.run(validateAfter=True) + mkdirCmd = MakeDirectory("rebalance log dir", log_dir, ctxt=REMOTE, remoteHost=self.move.dstHost.hostname) + mkdirCmd.run(validateAfter=True) + + log_file = f"{self.conf_dir}/gprecoverseg_dbid{self.segment.dbid}_{strtime}" + # Prepare command arguments + cmd_args = [ + '-i', filename, + '-B', '1', + '-v', '-a', + '-l', log_dir + ] + if self.options.hba_hostnames: + cmd_args.append('--hba-hostnames') + + result_queue = multiprocessing.Queue() + recovery_process = multiprocessing.Process( + target=RecoveryProcess.run_recovery, + args=(cmd_args, result_queue, log_file) + ) + recovery_process.start() + result = None + while recovery_process.is_alive(): + # Check if result is available without blocking + try: + if not result_queue.empty(): + result = result_queue.get(block=False) + recovery_process.join() + except Exception: + pass + time.sleep(10) + + if not result_queue.empty(): + result = result_queue.get(block=False) + + if result is None: + exit_code = recovery_process.exitcode + if exit_code < 0: + self.logger.error( + f"Could not perform mirror dbid={self.segment.dbid} " + f"move with content {self.segment.content} due to " + f"recoverseg error: Process terminated by signal {-exit_code}\n" + f"Check the gprecoverseg log file {log_file}, fix any problems, and re-run" + ) + else: + self.logger.error( + f"Could not perform mirror dbid={self.segment.dbid} " + f"move with content {self.segment.content} due to " + f"recoverseg error: Process exited with code {exit_code}\n" + f"Check the gprecoverseg log file {log_file}, fix any problems, and re-run") + self.status_manager.update_move_status([self.segment.dbid], MoveStatus.FAILED) + self.move_error = True + return + + if result["status"] == "FAILED": + self.logger.error( + f"Could not perform mirror dbid={self.segment.dbid} " + f"move with content {self.segment.content} due to " + f"recoverseg error: {result['error']}\n" + f"Check the gprecoverseg log file {log_file}, fix any problems, and re-run" + ) + self.status_manager.update_move_status([self.segment.dbid], MoveStatus.FAILED) + + self.move_error = True + return + if self.needs_switch: + self.status_manager.update_move_status([self.segment.dbid], MoveStatus.AWAITS_SWITCH) + else: + self.status_manager.update_move_status([self.segment.dbid], MoveStatus.COMPLETED) + + + self.logger.info("Removing old segment's datadir (dbidi = %d): %s", + self.segment.dbid, self.segment.datadir) + cmd = RemoveDirectory("remove old mirror segment directories", self.segment.datadir, + ctxt=REMOTE, remoteHost=self.segment.address) + cmd.run(validateAfter=True) + if self.segmentSize.source_tablespace_usage: + for tblspdir in self.segmentSize.source_tablespace_usage: + cmd = RemoveDirectory("remove old mirror segment directories", tblspdir, + ctxt=REMOTE, remoteHost=self.segment.address) + self.logger.info("Removing old segment's tablespace datadir (dbidi = %d): %s", + self.segment.dbid, tblspdir) + cmd.run(validateAfter=True) + + except Exception as ex: + self.logger.error(ex.__str__().strip()) + self.move_error = True + return + + + +class FilesystemSpace: + def __init__(self, filesystem: FileSystem, directories: Set[str]): + self.filesystem = filesystem + self.directories = directories # dirs on this filesystem + self.available_space = filesystem.disk_free + self.planned_usage = 0 + + def reserve_space(self, size: int): + self.planned_usage += size + + def can_accommodate(self, size: int) -> bool: + return (self.available_space - self.planned_usage) >= size + + +def _target_filesystems(addr: str, directories: List[str], batch_size) -> List[FileSystem]: + filesystems = [] # list of FileSystem() + pool = WorkerPool(numWorkers=min(len(directories), batch_size)) + try: + cmd = DiskFree(addr, directories) + pool.addCommand(cmd) + pool.join() + finally: + pool.haltWork() + pool.joinWorkers() + for cmd in pool.getCompletedItems(): + if not cmd.was_successful(): + raise Exception("Failed to check disk free on target segment: {}" .format( + cmd.get_results().stderr)) + filesystems = pickle.loads( + base64.urlsafe_b64decode(cmd.get_results().stdout)) + return filesystems + + +class HostResources: + def __init__(self, host: Host, ports: tuple[Set[int], Set[int]]): + self.host_address = host.address + self.primary_datadirs = host.primary_datadirs + self.mirror_datadirs = host.mirror_datadirs + self.used_primary_ports, self.used_mirror_ports = ports + self.filesystem_spaces: List[FilesystemSpace] = [] + self.hostname = host.hostname + + # Initialize filesystem tracking for all directories + all_dirs = host.primary_datadirs.union(host.mirror_datadirs) + self._init_filesystem_spaces(all_dirs) + self.base_port = self._determine_base_port() + + def _init_filesystem_spaces(self, directories: Set[str]): + """Initialize filesystem space tracking for all directories""" + filesystems = _target_filesystems( + self.host_address, list(directories), MAX_BATCH_SIZE) + + # Group directories by filesystem + for fs in filesystems: + dirs_on_fs = {d for d in directories if d in fs.directories} + self.filesystem_spaces.append(FilesystemSpace(fs, dirs_on_fs)) + + def accommodate_segment(self, segment_size: SegmentSize, target_datadir: str): + """Check if segment can be accommodated considering all its space requirements""" + + # Find filesystem for main datadir and add space requirement + datadir_fs = self._get_filesystem_for_dir(target_datadir) + if not datadir_fs: + raise Exception(f"Host {self.hostname} does not have any valid primary " + f"datadirs for segment") + if datadir_fs.can_accommodate(segment_size.source_data_dir_usage): + datadir_fs.reserve_space(segment_size.source_data_dir_usage) + + # Add tablespace requirements to respective filesystems + if segment_size.source_tablespace_usage: + for tblspc_dir, usage in segment_size.source_tablespace_usage.items(): + tblspc_fs = self._get_filesystem_for_dir(tblspc_dir) + if not tblspc_fs: + raise Exception(f"Host {self.hostname} does not have any valid primary " + f"datadirs for segment") + if tblspc_fs.can_accommodate(usage): + tblspc_fs.reserve_space(usage) + + def _get_filesystem_for_dir(self, directory: str) -> Optional[FilesystemSpace]: + """Find FilesystemSpace object containing given directory""" + # First check existing filesystem mappings + for fs_space in self.filesystem_spaces: + if directory in fs_space.directories: + return fs_space + + # If not found, fetch filesystem info for this directory + filesystems = _target_filesystems( + self.host_address, [directory], MAX_BATCH_SIZE) + + if not filesystems: + return None + + # Check if the filesystem already exists in our list + fs = filesystems[0] + for existing_fs in self.filesystem_spaces: + if existing_fs.filesystem.name == fs.name: + existing_fs.directories.add(directory) + return existing_fs + + # If not, create new FilesystemSpace + new_fs_space = FilesystemSpace(fs, {directory}) + self.filesystem_spaces.append(new_fs_space) + return new_fs_space + + def _determine_base_port(self) -> int: + """Determine base port from existing port assignments""" + all_ports = self.used_primary_ports | self.used_mirror_ports + + # Find the most common base port + port_bases = defaultdict(int) + for port in all_ports: + # For each port, calculate what base port it might correspond to + # assuming port = base + (content * 2) [+ 1 for mirrors] + for content in range(0, 128): # reasonable content_id range + if port % 2 == 0: # primary + possible_base = port - (content * 2) + else: # mirror + possible_base = port - (content * 2) - 1 + + if possible_base > 0: + port_bases[possible_base] += 1 + + if not port_bases: + return 7000 + + # Return the most frequently occurring base port + return max(port_bases.items(), key=lambda x: x[1])[0] + + def can_accommodate_port(self, is_mirror: bool, content_id: int) -> Optional[int]: + """ + Find available port for segment using existing base port pattern + Returns suitable port number or None if no port available + """ + used_ports = self.used_primary_ports | self.used_mirror_ports + + # Calculate port based on content_id and base port + port = self.base_port + (content_id * 2) + if is_mirror: + port += 1 + + if port not in used_ports: + return port + + # If standard port not available, try finding next available port + # maintaining the same even/odd pattern + start_port = max(used_ports) + 2 if used_ports else self.base_port + if start_port % 2 != (0 if not is_mirror else 1): + start_port += 1 + + current_port = start_port + while current_port < 65536: # Max TCP port + if current_port not in used_ports: + return current_port + current_port += 2 + + return None + + def reserve_port(self, port: int, is_mirror: bool): + """Reserve port for segment""" + if is_mirror: + self.used_mirror_ports.add(port) + else: + self.used_primary_ports.add(port) + + +class RebalanceExecutor: + def __init__(self, + plan: Plan, + original_array: GpArray, + segmentMap: Dict[SegmentId, Segment], + cluster_state: ClusterState, + logger, + statusManager: StatusManager, + conn: dbconn.Connection, + dburl: dbconn.DbURL, + options, + ): + self.moves = plan.moves + self.plan = plan + self.logger = logger + self.gparr = original_array + self.cluster_state = cluster_state + self.segmentMap = segmentMap + self.conn = conn + self.statusManager = statusManager + self.options = options + self.dburl = dburl + segids = [] + for m in plan.moves: + segids.append(m.segid) + self.segmentSizes = self.estimateSegmentSizes(segids) + self.resources = self.initializeHostResources(plan.moves) if not options.rollback else None + self.queue = None + self.shutdown_requested = False + + self.define_datadir_prefix() + + def define_datadir_prefix(self): + first_source_dir = None + for _, segment in self.segmentMap.items(): + if segment.content >= 0: + first_source_dir = segment.datadir + break + + basename = os.path.basename(first_source_dir) + global segment_prefix + segment_prefix = ''.join(c for c in basename if not c.isdigit()) + + def initializeHostResources(self, moves: List[Move]): + def datadir_validator(input_value, default, *args): + if not input_value and not default: + return None + elif not input_value or input_value == '': + input_value = default + if not input_value or input_value.find(' ') != -1 or input_value == '': + return None + else: + return input_value + resources = {} + for m in moves: + prim_ports = set() + mir_ports = set() + if m.dstHost not in resources: + for psid in self.cluster_state[(m.dstHost.hostname, m.dstHost.address)].primary_segments: + prim_ports.add(self.segmentMap[psid].port) + for msid in self.cluster_state[(m.dstHost.hostname, m.dstHost.address)].mirror_segments: + mir_ports.add(self.segmentMap[msid].port) + if len(m.dstHost.primary_datadirs) == 0: + prirmary_prefix = DEFAULT_PRIMARY_PREF + if not self.options.silent: + prirmary_prefix = ask_input(f"\nThe segment (dbid={m.segid.dbid}, content={m.segid.contentid}) " + f"is about to be moved to host {m.dstHost.hostname}, but no primary datadirs " + "are specified for the host.", "Enter the primary datadir prefix",f" (default={DEFAULT_PRIMARY_PREF})", + DEFAULT_PRIMARY_PREF, datadir_validator, None) + m.dstHost.primary_datadirs.add(prirmary_prefix.strip()) + if self.gparr.hasMirrors and len(m.dstHost.mirror_datadirs) == 0: + mirror_prefix = DEFAULT_MIRROR_PREF + if not self.options.silent: + mirror_prefix = ask_input(f"\nThe segment (dbid={m.segid.dbid}, content={m.segid.contentid}) " + f"is about to be moved to host {m.dstHost.hostname}, but no mirror datadirs " + "are specified for the host.", "Enter the mirror datadir prefix",f" (default={DEFAULT_MIRROR_PREF})", + DEFAULT_MIRROR_PREF, datadir_validator, None) + m.dstHost.mirror_datadirs.add(mirror_prefix.strip()) + + resources[m.dstHost] = HostResources( + m.dstHost, (prim_ports, mir_ports)) + return resources + + def _disk_usage(self, hostaddr: str, dirs: List[str]) -> Dict[str, int]: + """ + Get the Disk usage for the given set of directories to the targeted host + input: hostaddr , host from which the disk usage is fetched + input: dirs, list of directories to fetch the details + output: dictionary containing directories with it's disk usage stats in kb(kilo byte) + """ + dirs_disk_usage = {} # map of directories to disk usage + + if len(dirs) <= 0: + return dirs_disk_usage + + pool = WorkerPool(numWorkers=min(len(dirs), self.options.batch_size)) + try: + for directory in dirs: + cmd = DiskUsage('check source segments disk space used', + directory, ctxt=REMOTE, remoteHostAddr=hostaddr) + pool.addCommand(cmd) + pool.join() + finally: + pool.haltWork() + pool.joinWorkers() + + for cmd in pool.getCompletedItems(): + if not cmd.was_successful(): + raise Exception("Unable to check disk usage on source segment: {}" .format( + cmd.get_results().stderr)) + + dirs_disk_usage[cmd.directory] = cmd.kbytes_used() + + return dirs_disk_usage + + def estimateSegmentSizes(self, seglist: List[SegmentId]) -> Dict[SegmentId, SegmentSize]: + if not seglist: + return {} + oid_subq = """ (SELECT * + FROM ( + SELECT oid FROM pg_tablespace + WHERE spcname NOT IN ('pg_default', 'pg_global') + ) AS _q1, + LATERAL gp_tablespace_location(_q1.oid) + ) AS t """ + segment_dbids = ','.join(f'({seg.dbid})' for seg in seglist) + tablespace_location_sql = """ + SELECT c.dbid, c.content, t.tblspc_loc||'/'||c.dbid tblspc_loc + FROM {oid_subq} + JOIN gp_segment_configuration AS c + ON t.gp_segment_id = c.content WHERE c.dbid in (VALUES {segment_ids_str}) + """ .format(oid_subq=oid_subq, segment_ids_str=segment_dbids) + cursor = dbconn.query(self.conn, tablespace_location_sql) + tablespaces = defaultdict(list) + for dbid, content, loc in cursor: + tablespaces[SegmentId(dbid, content)].append(loc) + + segmentSizes = {} + for segid in seglist: + sourceSeg = self.segmentMap[segid] + source_data_dir_usage = self._disk_usage( + sourceSeg.address, [sourceSeg.datadir]) + segmentSizes[segid] = SegmentSize( + source_data_dir_usage[sourceSeg.datadir], None) + for segid, tblspace_dirs in tablespaces.items(): + sourceSeg = self.segmentMap[segid] + source_tblsps_usage = self._disk_usage( + sourceSeg.address, tblspace_dirs) + segmentSizes[segid].source_tablespace_usage = source_tblsps_usage + + return segmentSizes + + def _prepare_swaps(self, swaps: List[Tuple[Move, Move]]): + """ + Choose the target directory for swap case: + 1. primary is moved to mirror dir in its own host + 2. mirror is moved to primary dir in its own host + 3. role switching takes place + """ + for primary_move, mirror_move in swaps: + #important notice. srcHost and dstHost are different + # objects even if they are describing the same host. + # This happens due to code in get_moves_between_states(state1, state2) + # srcHost and dstHost are taken from state1 and state2 correspondingly + primary_host = mirror_move.dstHost + mirror_host = primary_move.dstHost + + primary_id = primary_move.segid + + if self.options.rollback: + mirror_move.dstHost = mirror_host + primary_move.dstHost = primary_host + primary_move.target_datadir, mirror_move.target_datadir = mirror_move.target_datadir, primary_move.target_datadir + primary_move.target_port, mirror_move.target_port = mirror_move.target_port, primary_move.target_port + continue + # define datadir + for datadir in primary_host.mirror_datadirs: + try: + self.resources[primary_host].accommodate_segment( + self.segmentSizes[primary_id], datadir) + primary_move.target_datadir = datadir + f"/{segment_prefix}{primary_move.segid.contentid}" + break + except: + continue + if primary_move.target_datadir == None: + raise NoValidDataDirectories(f"Host {primary_host.hostname} does not have any valid mirror " + f"datadirs for segment {primary_move.segid}. None of the " + f"{primary_host.mirror_datadirs} either exists or has " + "enough free space for segment movement") + primary_move.dstHost = primary_host + for datadir in mirror_host.primary_datadirs: + try: + self.resources[mirror_host].accommodate_segment( + self.segmentSizes[primary_id], datadir) + mirror_move.target_datadir = datadir + f"/{segment_prefix}{mirror_move.segid.contentid}" + break + except: + continue + if mirror_move.target_datadir == None: + raise NoValidDataDirectories(f"Host {mirror_host.hostname} does not have any valid primary " + f"datadirs for segment {mirror_move.segid}. None of the " + f"{mirror_host.primary_datadirs} either exists or has " + "enough free space for segment movement") + mirror_move.dstHost = mirror_host + + primary_move.target_port = self.resources[primary_host].can_accommodate_port( + True, primary_id.contentid) + if not primary_move.target_port: + raise Exception("Cannot accomodate port") + self.resources[primary_host].reserve_port(primary_move.target_port, True) + mirror_move.target_port = self.resources[mirror_host].can_accommodate_port( + False, primary_id.contentid) + if not mirror_move.target_port: + raise Exception("Cannot accomodate port") + self.resources[mirror_host].reserve_port(mirror_move.target_port,False) + + def _prepare_pms(self, primary_mirrors: List[Tuple[Move, Move]]): + """ + Choose the target directory for primary-mirror move case: + 1. mirror is moved to primary dir in primary's target host + 2. role switching takes place + 2. primary is moved to mirror dir in mirror's target host + """ + for primary_move, mirror_move in primary_mirrors: + primary_host = primary_move.dstHost + mirror_host = mirror_move.dstHost + + primary_id = primary_move.segid + mirror_id = mirror_move.segid + + if self.options.rollback: + primary_move.dstHost = mirror_host + mirror_move.dstHost = primary_host + primary_move.target_datadir, mirror_move.target_datadir = mirror_move.target_datadir, primary_move.target_datadir + primary_move.target_port, mirror_move.target_port = mirror_move.target_port, primary_move.target_port + continue + # define datadir + for datadir in mirror_host.mirror_datadirs: + try: + self.resources[primary_host].accommodate_segment( + self.segmentSizes[primary_id], datadir) + primary_move.target_datadir = datadir + f"/{segment_prefix}{primary_move.segid.contentid}" + break + except Exception as e: + self.logger.error(str(e)) + continue + primary_move.dstHost = mirror_host + if primary_move.target_datadir == None: + raise NoValidDataDirectories(f"Host {mirror_host.hostname} does not have any valid mirror " + f"datadirs for segment {primary_move.segid}. None of the " + f"{mirror_host.mirror_datadirs} either exists or has " + "enough free space for segment movement") + for datadir in primary_host.primary_datadirs: + try: + self.resources[mirror_host].accommodate_segment( + self.segmentSizes[primary_id], datadir) + mirror_move.target_datadir = datadir + f"/{segment_prefix}{mirror_move.segid.contentid}" + break + except: + continue + if mirror_move.target_datadir == None: + raise NoValidDataDirectories(f"Host {primary_host.hostname} does not have any valid primary " + f"datadirs for segment {mirror_move.segid}. None of the " + f"{primary_host.primary_datadirs} either exists or has " + "enough free space for segment movement") + mirror_move.dstHost = primary_host + + primary_move.target_port = self.resources[mirror_host].can_accommodate_port( + True, primary_id.contentid) + if not primary_move.target_port: + raise Exception("Cannot accomodate port") + self.resources[mirror_host].reserve_port(primary_move.target_port,True) + mirror_move.target_port = self.resources[primary_host].can_accommodate_port( + False, primary_id.contentid) + if not mirror_move.target_port: + raise Exception("Cannot accomodate port") + self.resources[primary_host].reserve_port(mirror_move.target_port,False) + + def _prepare_ps(self, primaries: List[Move]): + """ + Choose the target directory for primary-only move case: + 1. role switch takes place + 2. primary is moved to target dir + 3. role switch + """ + if self.options.rollback: + return + + for primary_move in primaries: + primary_host = primary_move.dstHost + + primary_id = primary_move.segid + # define datadir + for datadir in primary_host.primary_datadirs: + try: + self.resources[primary_host].accommodate_segment( + self.segmentSizes[primary_id], datadir) + primary_move.target_datadir = datadir + f"/{segment_prefix}{primary_move.segid.contentid}" + break + except: + continue + if primary_move.target_datadir == None: + raise NoValidDataDirectories(f"Host {primary_host.hostname} does not have any valid primary " + f"datadirs for segment {primary_move.segid}. None of the " + f"{primary_host.primary_datadirs} either exists or has " + "enough free space for segment movement") + + primary_move.target_port = self.resources[primary_host].can_accommodate_port( + True, primary_id.contentid) + if not primary_move.target_port: + raise Exception("Cannot accomodate port") + self.resources[primary_host].reserve_port(primary_move.target_port ,True) + + def _prepare_ms(self, mirrors: List[Move]): + """ + Choose the target directory for mirror-only move case: + 1. mirror is moved to mirror dir in mirror's target host + """ + if self.options.rollback: + return + for mirror_move in mirrors: + mirror_host = mirror_move.dstHost + + mirror_id = mirror_move.segid + # define datadir + for datadir in mirror_host.mirror_datadirs: + try: + self.resources[mirror_host].accommodate_segment( + self.segmentSizes[mirror_id], datadir) + mirror_move.target_datadir = datadir + f"/{segment_prefix}{mirror_move.segid.contentid}" + break + except: + continue + if mirror_move.target_datadir == None: + raise NoValidDataDirectories(f"Host {mirror_host.hostname} does not have any valid mirror " + f"datadirs for segment {mirror_move.segid}. None of the " + f"{mirror_host.mirror_datadirs} either exists or has " + "enough free space for segment movement") + + mirror_move.target_port = self.resources[mirror_host].can_accommodate_port( + True, mirror_id.contentid) + if not mirror_move.target_port: + raise Exception("Cannot accomodate port") + self.resources[mirror_host].reserve_port(mirror_move.target_port ,True) + + + def _classify_moves(self) -> Tuple[List[Tuple[Move, Move]], List[Tuple[Move, Move]], List[Move], List[Move]]: + """ + Classify moves into: + - pure_swaps: pairs of moves where primary and mirror just switch places + - primary_moves_with_mirrors: pairs of moves where we move both primary and mirror + - primary_moves: independent primary moves + - mirror_moves: independent mirror moves + """ + # Group moves by contentid + moves_by_content = defaultdict(list) + for move in self.moves: + moves_by_content[move.segid.contentid].append(move) + + primary_moves_with_mirrors = [] + primary_moves = [] + mirror_moves = [] + pure_swaps = [] + + for contentid, moves in moves_by_content.items(): + if len(moves) == 2: + primary_move = next( + (m for m in moves if not m.is_mirror), None) + mirror_move = next((m for m in moves if m.is_mirror), None) + + if (primary_move and mirror_move and + primary_move.srcHost == mirror_move.dstHost and + primary_move.dstHost == mirror_move.srcHost): + pure_swaps.append((primary_move, mirror_move)) + elif primary_move and mirror_move: + # This is a primary move with corresponding mirror move + primary_moves_with_mirrors.append( + (primary_move, mirror_move)) + + elif len(moves) == 1: + move = moves[0] + if move.is_mirror: + mirror_moves.append(move) + else: + primary_moves.append(move) + + return pure_swaps, primary_moves_with_mirrors, primary_moves, mirror_moves + + def _create_move_sequence(self) -> Tuple[List[List[Move]], Set[Move], Set[Move]]: + """Create full sequence of moves""" + sequences = [] + current_batch = [] + + pure_swaps, primary_mirrors, primaries, mirrors = self._classify_moves() + self._prepare_swaps(pure_swaps) + self._prepare_pms(primary_mirrors) + self._prepare_ps(primaries) + self._prepare_ms(mirrors) + + former_switches = set() + latter_switches = set() + + current_batch = [] + for mirror_move in mirrors: + if len(current_batch) >= self.options.batch_size: + sequences.append(current_batch) + current_batch = [] + current_batch.append(mirror_move) + + # Pure swaps: move mirrors to primary dirs + for primary_move, mirror_move in pure_swaps: + if len(current_batch) >= self.options.batch_size: + sequences.append(current_batch) + current_batch = [] + current_batch.append(mirror_move) + + # Primary-mirror pairs: move mirrors to primary's target + for primary_move, mirror_move in primary_mirrors: + if len(current_batch) >= self.options.batch_size: + sequences.append(current_batch) + current_batch = [] + current_batch.append(mirror_move) + + if current_batch: + sequences.append(current_batch) + + # First switch point - affects: + # - Pure swaps + # - Primary-mirror pairs + # - First switch for primary-only moves + segments_for_switch1 = [] + segments_for_switch1.extend([pm[0].segid for pm in pure_swaps]) + segments_for_switch1.extend([pm[0].segid for pm in primary_mirrors]) + # First switch for primaries + segments_for_switch1.extend([pm.segid for pm in primaries]) + + if segments_for_switch1: + sequences.append(['SWITCH', segments_for_switch1]) + for seg in segments_for_switch1: + former_switches.add(seg.contentid) + # Phase 3: Post-first-switch moves + current_batch = [] + + # Pure swaps: move ex-primaries to mirror dirs + for primary_move, _ in pure_swaps: + if len(current_batch) >= self.options.batch_size: + sequences.append(current_batch) + current_batch = [] + current_batch.append(primary_move) + + # Primary-mirror pairs: move ex-primaries to target mirror dirs + for primary_move, _ in primary_mirrors: + if len(current_batch) >= self.options.batch_size: + sequences.append(current_batch) + current_batch = [] + current_batch.append(primary_move) + + # Primary-only: move ex-primaries to target primary dirs + for primary_move in primaries: + if len(current_batch) >= self.options.batch_size: + sequences.append(current_batch) + current_batch = [] + current_batch.append(primary_move) + + if current_batch: + sequences.append(current_batch) + + # Second switch point - affects: + # - Second switch for primary-only moves + if primaries: + sequences.append(['SWITCH', [pm.segid for pm in primaries]]) + for seg in primaries: + latter_switches.add(seg.segid.contentid) + + return sequences, former_switches, latter_switches + + def execute_moves(self, firstRun=True): + """Main execution method""" + try: + if not firstRun: + raise NotImplementedError( + "rebalance rerun is not implemented properly yet") + + move_sequences, former_switches, latter_switches = self._create_move_sequence() + + global begining_timestamp + begining_timestamp = datetime.datetime.now() + + conf_dir = self.options.coordinator_data_directory + CONF_DIR + + #record moves in status file + detail_list = [] + for seq_no, seq in enumerate(move_sequences): + if not isinstance(seq[0], str): + for move in seq: + needs_switch = False + if move.segid.contentid in former_switches or move.segid.contentid in latter_switches: + needs_switch = True + if move.segid.contentid in former_switches and move.segid.contentid in latter_switches: + needs_switch = False + detail_list.append((seq_no, self.segmentMap[move.segid], + move, + self.segmentSizes[move.segid].source_data_dir_usage, + needs_switch)) + self.statusManager.record_moves_batch(detail_list) + + hosts = set() + for move in self.moves: + hosts.add(move.srcHost.hostname) + hosts.add(move.dstHost.hostname) + + if firstRun: + # in order to run gprecoverseg processes separately and avoid any races + # for resources gprecoverseg generates (progress file), we create + # separate log directories. + mkdirCmd = MakeDirectory("rebalance log dir", GPRECOVERSEG_DIR) + mkdirCmd.run(validateAfter=True) + for host in hosts: + mkdirCmd = MakeDirectory("rebalance log dir", GPRECOVERSEG_DIR, ctxt=REMOTE, remoteHost=host) + mkdirCmd.run(validateAfter=True) + + if self.options.rollback: + self.statusManager.set_status(RebalanceStatus.ROLLBACK_PREPARED, conf_dir) + self.plan.save_to_file(conf_dir, "rollback_plan") + else: + self.statusManager.set_status(RebalanceStatus.PREPARED, conf_dir) + self.plan.save_to_file(conf_dir, "plan") + + self.queue = WorkerPool(self.options.parallel) + + stopTime = None + stoppedEarly = False + had_error = False + if self.options.end: + stopTime = self.options.end + if self.options.rollback: + self.statusManager.set_status(RebalanceStatus.ROLLBACK_IN_PROGRESS) + else: + self.statusManager.set_status(RebalanceStatus.IN_PROGRESS) + for sequence in move_sequences: + if self.shutdown_requested: + break + if isinstance(sequence[0], str) and sequence[0] == 'SWITCH': + + while not self.queue.isDone(): + if stopTime and datetime.datetime.now() >= stopTime: + stoppedEarly = True + break + time.sleep(5) + + for moveCommand in self.queue.getCompletedItems(): + if moveCommand.move_error: + had_error = True + break + + if stoppedEarly or had_error: + break + + if self.shutdown_requested: + break + + self.statusManager.set_status( + RebalanceStatus.AWAITS_SWITCH) + self.logger.info( + f"Executing role swaps for {len(sequence[1])} segments") + try: + self._execute_role_swaps(sequence[1]) + for segid in sequence[1]: + if segid in former_switches and segid not in latter_switches: + self.statusManager.update_move_status(sequence[1], MoveStatus.COMPLETED) + break + except Exception as e: + had_error = True + self.logger.error(f"Could not execute role swaps:{str(e)}") + break + self.statusManager.set_status( + RebalanceStatus.IN_PROGRESS) + else: + for move in sequence: + if self.shutdown_requested: + break + segid = move.segid + needs_switch = False + if segid.contentid in former_switches or segid.contentid in latter_switches: + needs_switch = True + step_details = (self.segmentMap[segid], + move, + self.segmentSizes[segid], + conf_dir, + needs_switch, + self.options + ) + + cmd = SingleMoveCommand( + "name", step_details, self.logger, self.statusManager) + self.queue.addCommand(cmd) + + while not self.queue.isDone(): + if stopTime and datetime.datetime.now() >= stopTime: + stoppedEarly = True + break + time.sleep(5) + if stoppedEarly: + self.logger.info("Execution timeout is reached. Waiting the existing jobs to finish " + "and stopping rebalance.") + break + for moveCommand in self.queue.getCompletedItems(): + if moveCommand.move_error: + had_error = True + break + + if self.queue: + self.queue.haltWork() + self.queue.joinWorkers() + self.queue = None + + if stoppedEarly or self.shutdown_requested: + self.statusManager.set_status(RebalanceStatus.STOPPED) + if not self.shutdown_requested: + self.logger.info("Rebalance stopped due to timeout") + elif had_error: + if self.options.rollback: + self.statusManager.set_status( + RebalanceStatus.ROLLBACK_FAILED) + else: + self.statusManager.set_status( + RebalanceStatus.FAILED) + raise Exception("execution encountered movement erorrs") + else: + if self.options.rollback: + self.statusManager.set_status(RebalanceStatus.ROLLBACK_DONE) + else: + self.statusManager.set_status(RebalanceStatus.DONE) + + rmdirCmd = RemoveDirectory("remove recoverseg log dir", f"{os.path.join(os.environ.get('HOME', '.'),GPRECOVERSEG_DIR)}") + rmdirCmd.run() + for host in hosts: + rmdirCmd = RemoveDirectory("remove recoverseg log dir", f"{os.path.join(os.environ.get('HOME', '.'),GPRECOVERSEG_DIR)}", ctxt=REMOTE, remoteHost=host) + rmdirCmd.run() + + + except Exception as e: + if self.options.rollback: + self.statusManager.set_status(RebalanceStatus.ROLLBACK_FAILED) + else: + self.statusManager.set_status(RebalanceStatus.FAILED) + raise + + def _execute_role_swaps(self, segids: List[SegmentId]): + """Execute multiple role swaps in single gprecoverseg -r call""" + if not segids: + return + try: + with self.conn.cursor() as cur: + cur.execute("BEGIN") + cur.execute("SET allow_system_table_mods=1;") + data = tuple([segid.contentid for segid in segids]) + cur.execute("UPDATE gp_segment_configuration SET preferred_role = 't' WHERE " + "content IN %s AND preferred_role = 'm'", (data,)) + cur.execute("UPDATE gp_segment_configuration SET preferred_role = 'm' WHERE " + "content IN %s AND preferred_role = 'p'", (data,)) + cur.execute("UPDATE gp_segment_configuration SET preferred_role = 'p' WHERE " + "content IN %s AND preferred_role = 't'", (data,)) + cur.execute("COMMIT") + except Exception as e: + raise Exception('could not execute SQL : %s' % str(e)) + + recoversegOptions = f"-r -a -l {os.path.join(os.environ.get('HOME', '.'),GPRECOVERSEG_DIR)}" + if self.options.hba_hostnames: + recoversegOptions += " --hba-hostnames" + cmd = GpRecoverSeg("Running gprecverseg", options=recoversegOptions) + cmd.run(validateAfter=True) + + + def shutdown(self): + # the execution shutdown assumes finishing + # current jobs in queue + if self.queue: + self.logger.info("Shutdown requested, will complete current jobs...") + self.shutdown_requested = True + self.queue.haltWork() + self.queue.joinWorkers() + self.queue = None diff --git a/gpMgmt/bin/gprebalance_modules/rebalance_plan.py b/gpMgmt/bin/gprebalance_modules/rebalance_plan.py new file mode 100644 index 000000000000..51caf218e1da --- /dev/null +++ b/gpMgmt/bin/gprebalance_modules/rebalance_plan.py @@ -0,0 +1,337 @@ +import copy +import os +import pickle +import random +import math +from typing import List, Set, Dict +from dataclasses import dataclass +from collections import defaultdict +from gprebalance_modules.rebalance import Host, HostStatus, SegmentId, ClusterState, MirrorStrategy, Segment + + +@dataclass +class Move: + segid: SegmentId + srcHost: Host + dstHost: Host + is_mirror: bool + target_datadir: str + target_port: int + + +class Plan: + def __init__(self): + self.in_conf = None + self.moves = [] + self.out_conf = None + self.segmentMap = None + self.roles = [] + + def __str__(self): + finalStr = "" + for i in range(len(self.moves)): + m = self.moves[i] + if self.roles[m.segid] == 'p': + finalStr += f"{i} :S{m.segid.contentid} :" + else: + finalStr += f"{i} :M{m.segid.contentid} :" + finalStr += f"H{m.srcHost.hostname} -> H{m.dstHost.hostname}\n" + return finalStr + + def save_to_file(self, directory: str, filename: str) -> str: + file_path = os.path.join(directory, f"{filename}.pkl") + + # Pickle and save the plan + with open(file_path, 'wb') as f: + pickle.dump(self, f) + + +class NoValidMovesError(Exception): + def __init__(self, msg="No valid targets for selected segment"): + super().__init__(msg) + + +class ClusterBalancer(): + def __init__(self, initial_conf: ClusterState, + new_hosts: tuple[Set[Host], Set[Host]], + segmentMap: Dict[SegmentId, Segment], + target_load: int, + strat: MirrorStrategy + ): + self.in_conf = initial_conf + self.new_hosts, self.replacement_hosts = new_hosts + self.initialSegmentMap = segmentMap + self.target_load = target_load + self.target_strategy = strat + self.initial_mirror_mapping = {} + for k, v in segmentMap.items(): + if v.isSegmentMirror(): + self.initial_mirror_mapping[k.contentid] = v + + def _get_working_conf(self) -> ClusterState: + """Get host-centric representation of cluster configuration""" + working_conf = copy.deepcopy(self.in_conf) + if self.new_hosts: + for h in self.new_hosts: + assert (h.status == HostStatus.NEW) + working_conf[(h.hostname, h.address)] = copy.deepcopy(h) + if self.replacement_hosts: + for h in self.replacement_hosts: + assert (h.status == HostStatus.REPLACEMENT) + working_conf[(h.hostname, h.address)] = copy.deepcopy(h) + assert ( + working_conf[h.replacement_for].status == HostStatus.DECOMMISSIONING) + return working_conf + + def _getCurrentMirrorMapping(self, state: ClusterState) -> Dict[int, tuple[str, str]]: + mirror_mapping = {} + for k, v in state.items(): + for mirid in v.mirror_segments: + mirror_mapping[mirid.contentid] = (v.hostname, v.address) + return mirror_mapping + + def _getCurrentSegmentMapping(self, state: ClusterState) -> Dict[int, tuple[str, str]]: + segment_mapping = {} + for k, v in state.items(): + for segid in v.primary_segments: + segment_mapping[segid.contentid] = (v.hostname, v.address) + return segment_mapping + + def _satisfies_strategy(self, state: ClusterState) -> bool: + mirror_mapping = self._getCurrentMirrorMapping(state) + if self.target_strategy == MirrorStrategy.GROUPED: + for _, host in state.items(): + if host.status == HostStatus.DECOMMISSIONING: + continue + mirror_hosts = set() + for s in host.primary_segments: + mirror_hosts.add(mirror_mapping[s.contentid]) + if len(mirror_hosts) > 1: + return False + elif len(mirror_hosts & set([(host.hostname, host.address)])) > 0: + return False + elif len(host.mirror_segments) == 0: + return False + elif self.target_strategy == MirrorStrategy.SPREAD: + for _, host in state.items(): + if host.status == HostStatus.DECOMMISSIONING: + continue + mirror_hosts_counts: Dict[Host, int] = defaultdict(int) + for s in host.primary_segments: + mirror_hosts_counts[mirror_mapping[s.contentid]] += 1 + for t_h, count in mirror_hosts_counts.items(): + if t_h == (host.hostname, host.address): + return False + elif count > 1: + return False + return True + + def _check_balance(self, state: ClusterState) -> bool: + for _, host in state.items(): + load = len(host.primary_segments) + deviation = abs(load - self.target_load) + if host.status == HostStatus.DECOMMISSIONING and (host.primary_segments or host.mirror_segments): + return False + elif host.status != HostStatus.DECOMMISSIONING and deviation != 0: + return False + return True + + def _move_from_decomissioning(self, state: ClusterState): + if self.replacement_hosts: + for rep in self.replacement_hosts: + # Host from which we must move all segments + decom_host_id = rep.replacement_for + replacement_host_id = (rep.hostname, rep.address) + assert (state[decom_host_id].status == + HostStatus.DECOMMISSIONING) + primaries = state[decom_host_id].primary_segments + mirrors = state[decom_host_id].mirror_segments + for segid in primaries: + state[replacement_host_id].add_primary(segid) + for segid in mirrors: + state[replacement_host_id].add_mirror(segid) + state[decom_host_id].primary_segments = set() + state[decom_host_id].mirror_segments = set() + + def _check_constraints(self, state: ClusterState): + return self._check_balance(state) and self._satisfies_strategy(state) + + def balance(self) -> ClusterState: + working_conf = self._get_working_conf() + # First, check if there are hosts explicitly marked for replacement + if self.replacement_hosts: + self._move_from_decomissioning(working_conf) + if self._check_constraints(working_conf): + return working_conf + + conf = self.rough_balance(working_conf) + assert (self._check_constraints(conf)) + return conf + + def _is_move_valid(self, segmentid: SegmentId, target_host: Host, state: ClusterState) -> bool: + segment_locations = self._getCurrentSegmentMapping(state) + + current_host = segment_locations[segmentid.contentid] + + if current_host == (target_host.hostname, target_host.address): + return False + + # Check target host status + if target_host.status == HostStatus.DECOMMISSIONING: + return False + + # For primary moves, only check capacity + if state[current_host].status != HostStatus.DECOMMISSIONING: + return len(target_host.primary_segments) + 1 <= self.target_load + + return True + + def rough_balance(self, working_conf: ClusterState) -> ClusterState: + + def calculate_surplus_deficit(): + loads = {(h.hostname, h.address): len(h.primary_segments) + for _, h in working_conf.items() + if h.status != HostStatus.DECOMMISSIONING} + surplus = {} + deficit = {} + for host_id, load in loads.items(): + surplus[host_id] = max(0, load - self.target_load) + deficit[host_id] = max(0, self.target_load - load) + return surplus, deficit + # Phase 1: Redistribute Primary Segments + while True: + # Priority 1: Clear decommissioning hosts + decom_hosts = [h for _, h in working_conf.items() + if h.status == HostStatus.DECOMMISSIONING] + + moved = False + # Move from decommissioning hosts first + for source in decom_hosts: + if not source.primary_segments: + continue + + potential_targets = sorted( + [h for _, h in working_conf.items() + if h.status not in (HostStatus.DECOMMISSIONING,)], + key=lambda h: len(h.primary_segments) + ) + + for segment in sorted(list(source.primary_segments), key=lambda x: x.contentid, reverse=True): + for target in potential_targets: + if self._is_move_valid(segment, target, working_conf): + source.remove_primary(segment) + target.add_primary(segment) + moved = True + break + + surplus, deficit = calculate_surplus_deficit() + if sum(surplus.values()) == 0: + break + # If no decom moves, balance regular hosts + if not moved: + surplus_hosts = [(s, h) for h, s in surplus.items() if s > 0] + deficit_hosts = [(d, h) for h, d in deficit.items() if d > 0] + surplus_hosts.sort(reverse=True) + deficit_hosts.sort(reverse=True) + + for _, source_id in surplus_hosts: + source = working_conf[source_id] + for _, target_id in deficit_hosts: + target = working_conf[target_id] + + for segment in sorted(list(source.primary_segments), key=lambda x: x.contentid, reverse=True): + if self._is_move_valid(segment, target, working_conf): + source.remove_primary(segment) + target.add_primary(segment) + moved = True + break + if moved: + break + if moved: + break + + if not moved: + break + + # Phase 2: Deterministic Mirror Placement + for _, host in working_conf.items(): + host.mirror_segments = set() + + active_hosts = [(h.hostname, h.address) for _, h in working_conf.items() + if h.status != HostStatus.DECOMMISSIONING] + active_hosts.sort() # ensure consistent ordering + + if self.target_strategy == MirrorStrategy.GROUPED: + # For each host, place all its primaries' mirrors on the next host + for i, host_id in enumerate(active_hosts): + next_host_id = active_hosts[(i + 1) % len(active_hosts)] + primaries = working_conf[host_id].primary_segments + for segment in primaries: + mirror = self.initial_mirror_mapping[segment.contentid] + working_conf[next_host_id].add_mirror( + SegmentId(mirror.dbid, mirror.content)) + + elif self.target_strategy == MirrorStrategy.SPREAD: + # For each host, distribute mirrors across other hosts + for host_id in active_hosts: + other_hosts = [h for h in active_hosts if h != host_id] + primaries = list(working_conf[host_id].primary_segments) + + # Distribute mirrors evenly across other hosts + for i, segment in enumerate(primaries): + mirror_host = other_hosts[i % len(other_hosts)] + mirror = self.initial_mirror_mapping[segment.contentid] + working_conf[mirror_host].add_mirror( + SegmentId(mirror.dbid, mirror.content)) + + return working_conf + + def getPlan(self, finalState: ClusterState) -> Plan: + in_conf = self._get_working_conf() + moves, roles = self.get_moves_between_states(in_conf, finalState) + plan = Plan() + plan.moves = moves + plan.in_conf = in_conf + plan.out_conf = finalState + plan.segmentMap = self.initialSegmentMap + plan.roles = roles + return plan + + def get_moves_between_states(self, state1: ClusterState, state2: ClusterState): + moves = [] + roles = {} + primary_map1 = self._getCurrentSegmentMapping(state1) + mirror_map1 = self._getCurrentMirrorMapping(state1) + primary_map2 = self._getCurrentSegmentMapping(state2) + mirror_map2 = self._getCurrentMirrorMapping(state2) + + # Find primary segment moves + for contentid, target_location in primary_map2.items(): + current_location = primary_map1.get(contentid) + if current_location and current_location != target_location: + #important notice. srcHost and dstHost are different + # objects even if they are describing the same host. + # srcHost and dstHost are taken from state1 and state2 correspondingly + source_host = state1[current_location] + target_host = state2[target_location] + segid = next(seg for seg in source_host.primary_segments + if seg.contentid == contentid) + moves.append( + Move(segid, source_host, target_host, False, None, None)) + roles[segid] = 'p' + + # Find mirror segment moves + for contentid, target_location in mirror_map2.items(): + current_location = mirror_map1.get(contentid) + if current_location and current_location != target_location: + source_host = state1[current_location] + target_host = state2[target_location] + segid = next(seg for seg in source_host.mirror_segments + if seg.contentid == contentid) + moves.append( + Move(segid, source_host, target_host, True, None, None)) + roles[segid] = 'm' + + moves.sort(key=lambda m: 1 if m.is_mirror else 0) + + return moves, roles diff --git a/gpMgmt/bin/gprebalance_modules/rebalance_schema.py b/gpMgmt/bin/gprebalance_modules/rebalance_schema.py new file mode 100644 index 000000000000..a7bdbdf35814 --- /dev/null +++ b/gpMgmt/bin/gprebalance_modules/rebalance_schema.py @@ -0,0 +1,192 @@ +#!/usr/bin/env python3 + +from psycopg2.extensions import cursor +from gppylib.db import dbconn +from typing import List +from gprebalance_modules.planner import Plan, deserializePlan +from gprebalance_modules.rebalance_step import * + +STATE_NOT_DEFINED = 'not defined' + +def get_table_distr_segment_count(conn: dbconn.Connection, schema_name: str, table_name: str) -> int: + row = dbconn.queryRow(conn, + f'''SELECT p.numsegments + FROM pg_class c JOIN pg_namespace n ON c.relnamespace = n.oid + JOIN gp_distribution_policy p ON c.oid = p.localoid + WHERE n.nspname='{schema_name}' AND c.relname='{table_name}';''') + return int(row[0]) + +class RebalanceSchema: + STATE_CATEGORY_SHRINK = 'SHRINK' + STATE_CATEGORY_REBALANCE = 'REBALANCE' + STATE_CATEGORY_MAIN = 'MAIN' + + def __init__(self, conn: dbconn.Connection): + self.schema_name = 'ggrebalance' + self.rebalance_status = 'rebalance_status' + self.table_rebalance_status_detail = 'table_rebalance_status_detail' + self.saved_plan = 'saved_plan' + self.segment_move_steps = 'segment_move_steps' + self.conn = conn + + def createSchema(self, plan: Plan) -> None: + dbconn.execSQL(self.conn, 'BEGIN') + dbconn.execSQL(self.conn, f'CREATE SCHEMA {self.schema_name}') + dbconn.execSQL(self.conn, + f'''CREATE TABLE {self.schema_name}.{self.rebalance_status} + (state TEXT, state_category TEXT, updated TIMESTAMP WITH TIME ZONE) + DISTRIBUTED REPLICATED''') + dbconn.execSQL(self.conn, + f'''CREATE TABLE {self.schema_name}.{self.table_rebalance_status_detail} + (db_name TEXT, schema_name TEXT, rel_name TEXT, status TEXT, + CONSTRAINT unique_fqn UNIQUE (db_name, schema_name, rel_name)) + DISTRIBUTED REPLICATED''') + dbconn.execSQL(self.conn, + f'''CREATE TABLE {self.schema_name}.{self.saved_plan} + (plan BYTEA) + DISTRIBUTED REPLICATED''') + + self.savePlan(plan) + + dbconn.execSQL(self.conn, 'COMMIT') + + def dropSchema(self) -> None: + dbconn.execSQL(self.conn, f'DROP SCHEMA {self.schema_name} CASCADE') + + def getSchemaName(self) -> str: + return self.schema_name + + def savePlan(self, plan: Plan) -> None: + dbconn.execSQL(self.conn, + f'''INSERT INTO {self.schema_name}.{self.saved_plan} + VALUES ('\\x{plan.serializePlan().hex()}')''') + + def retrieveSavedPlan(self) -> Plan: + if not self.schemaExists(): + return None + + row = dbconn.queryRow(self.conn, f'SELECT count(1) FROM {self.schema_name}.{self.saved_plan}') + plan_count = int(row[0]) + if plan_count > 1: + raise Exception(f'Number of saved plans ({plan_count}) is > 1') + if plan_count == 0: + return None + + row = dbconn.queryRow(self.conn, f'SELECT plan FROM {self.schema_name}.{self.saved_plan} LIMIT 1') + return deserializePlan(row[0]) + + def schemaExists(self) -> bool: + row = dbconn.queryRow(self.conn, f"SELECT COUNT(1) FROM pg_namespace WHERE nspname = '{self.schema_name}'") + return int(row[0]) == 1 + + def getStateFromPreviousRun(self, state_category: str) -> str: + if self.schemaExists(): + cursor = dbconn.query(self.conn, f"SELECT state FROM {self.schema_name}.{self.rebalance_status} WHERE state_category = '{state_category}' ORDER BY updated DESC LIMIT 1") + if cursor.rowcount > 0: + return str(cursor.fetchone()[0]) + return STATE_NOT_DEFINED + + def getShrinkStateFromPreviousRun(self) -> str: + return self.getStateFromPreviousRun(self.STATE_CATEGORY_SHRINK) + + def getRebalanceStateFromPreviousRun(self) -> str: + return self.getStateFromPreviousRun(self.STATE_CATEGORY_REBALANCE) + + def getMainStateFromPreviousRun(self) -> str: + return self.getStateFromPreviousRun(self.STATE_CATEGORY_MAIN) + + def rebalanceSchema(self, target_segment_count: int) -> None: + # Before rebalancing check if the tables are already rebalanced + # (in case we re-enter after interruption that happened after COMMIT but before new state) + if get_table_distr_segment_count(self.conn, self.schema_name, self.rebalance_status) > target_segment_count: + dbconn.execSQL(self.conn, + f'''ALTER TABLE "{self.schema_name}"."{self.rebalance_status}" + REBALANCE {target_segment_count}''') + + if get_table_distr_segment_count(self.conn, self.schema_name, self.table_rebalance_status_detail) > target_segment_count: + dbconn.execSQL(self.conn, + f'''ALTER TABLE "{self.schema_name}"."{self.table_rebalance_status_detail}" + REBALANCE {target_segment_count}''') + + if get_table_distr_segment_count(self.conn, self.schema_name, self.saved_plan) > target_segment_count: + dbconn.execSQL(self.conn, + f'''ALTER TABLE "{self.schema_name}"."{self.saved_plan}" + REBALANCE {target_segment_count}''') + + def storeState(self, state: str, state_category: str) -> None: + if self.schemaExists(): + dbconn.execSQL(self.conn, + f'''INSERT INTO {self.schema_name}.{self.rebalance_status} + VALUES ('{state}', '{state_category}', NOW())''') + + def storeShrinkState(self, state: str) -> None: + self.storeState(state, self.STATE_CATEGORY_SHRINK) + + def storeRebalanceState(self, state: str) -> None: + self.storeState(state, self.STATE_CATEGORY_REBALANCE) + + def storeMainState(self, state: str) -> None: + self.storeState(state, self.STATE_CATEGORY_MAIN) + + def clearTablesToRebalanceWithStatus(self, status: str) -> None: + dbconn.execSQL(self.conn, + f'''DELETE FROM {self.schema_name}.{self.table_rebalance_status_detail} + WHERE (status = '{status}')''') + + def addTableToRebalance(self, db: str, schema_name: str, rel_name: str, status: str) -> None: + dbconn.execSQL(self.conn, + f'''INSERT INTO {self.schema_name}.{self.table_rebalance_status_detail} + VALUES ('{db}', '{schema_name}', '{rel_name}', '{status}')''') + + def setStatusForTableToRebalance(self, db: str, schema_name: str, rel_name: str, status: str) -> None: + dbconn.execSQL(self.conn, + f'''UPDATE {self.schema_name}.{self.table_rebalance_status_detail} SET status = '{status}' + WHERE db_name = '{db}' AND schema_name = '{schema_name}' AND rel_name = '{rel_name}';''') + + def getTablesToRebalanceWithStatus(self, status: str) -> cursor: + return dbconn.query(self.conn, f"""SELECT db_name, schema_name, rel_name FROM + {self.schema_name}.{self.table_rebalance_status_detail} WHERE status = '{status}'""") + + def saveExecutionSteps(self, steps: List[RebalanceStep]) -> None: + dbconn.execSQL(self.conn, 'BEGIN') + + dbconn.execSQL(self.conn, f'DROP TABLE IF EXISTS {self.schema_name}.{self.segment_move_steps}') + + dbconn.execSQL(self.conn, + f'''CREATE TABLE {self.schema_name}.{self.segment_move_steps} + (move_order INT NOT NULL UNIQUE, status TEXT, step BYTEA) + DISTRIBUTED REPLICATED''') + + for step in steps: + dbconn.execSQL(self.conn, + f'''INSERT INTO {self.schema_name}.{self.segment_move_steps} + VALUES ({step.getMoveOrder()}, '{step.getStatus().name}', '\\x{step.serializeStep().hex()}')''') + + dbconn.execSQL(self.conn, 'COMMIT') + + def updateExecutionStep(self, step: RebalanceStep) -> None: + dbconn.execSQL(self.conn, + f'''UPDATE {self.schema_name}.{self.segment_move_steps} + SET status='{step.getStatus().name}', step='\\x{step.serializeStep().hex()}' WHERE move_order = {step.getMoveOrder()}''') + + def allExecutionStepsAreDone(self) -> bool: + row = dbconn.queryRow(self.conn, + f"SELECT count(1) FROM {self.schema_name}.{self.segment_move_steps} WHERE status <> '{RebalanceStep.Status.DONE.name}'") + not_done_count = int(row[0]) + return not_done_count == 0 + + def getExecutionSteps(self, status_filter: List[RebalanceStep.Status]) -> List[RebalanceStep]: + result = [] + + filter = "" + if len(status_filter) > 0: + status_list = ', '.join("'" + status.name + "'" for status in status_filter) + filter = f" WHERE status IN ({status_list})" + + cursor = dbconn.query(self.conn, + f'SELECT step FROM {self.schema_name}.{self.segment_move_steps} {filter} ORDER BY move_order') + for row in cursor: + result.append(deserializeStep(row[0])) + + return result + diff --git a/gpMgmt/bin/gprebalance_modules/rebalance_status.py b/gpMgmt/bin/gprebalance_modules/rebalance_status.py new file mode 100644 index 000000000000..02b1c984e0b3 --- /dev/null +++ b/gpMgmt/bin/gprebalance_modules/rebalance_status.py @@ -0,0 +1,681 @@ +import atexit +import copy +from datetime import datetime +import json +import os +from enum import Enum +import pickle +import threading +import time +from typing import Dict, List, Tuple, Any, Optional + +from gprebalance_modules.rebalance_plan import Move +from gprebalance_modules.rebalance import SegmentId, Segment +import gppylib.operations.update_pg_hba_on_segments as hba_upd +from gppylib.operations.detect_unreachable_hosts import get_unreachable_segment_hosts, update_unreachable_flag_for_segments + +from gppylib.commands import pg +from gppylib.commands.base import Command, REMOTE +from gppylib.userinput import * +from gppylib.db import dbconn +from gppylib.system import configurationInterface as configInterface +from gppylib.parseutils import * + +class MoveStatus(Enum): + PENDING = "PENDING" + IN_PROGRESS = "IN_PROGRESS" + AWAITS_SWITCH = "AWAITS_SWITCH" + COMPLETED = "COMPLETED" + FAILED = "FAILED" + STOPPED = "STOPPED" + REVERTED = "REVERTED" + ASK_ROLLBACK_OR_SKIP = "ASK_USER" + SKIPPED = "NEEDS_MANUAL_RECOVERY" + +class RebalanceStatus(Enum): + UNKNOWN = "UNKNOWN" + UNINITIALIZED = "UNINITIALIZED" + VALIDATED = "VALIDATED" + PRE_EXECUTE_FAILED= "PRE_EXECUTE_NOT_PASSED" + PREPARED = "EXECUTION_PREPARED" + IN_PROGRESS = "EXECUTION_IN_PROGRESS" + AWAITS_SWITCH_APPROVE = "EXECUTION_AWAITING_SWITCHOVER_APPROVE" + AWAITS_SWITCH = "EXECUTION_AWAITING_SWITCHOVER" + STEPS_FAILED = "EXECUTION_STEPS_FAILED" + FAILED = "EXECUTION_FAILED" + DONE = "EXECUTION_DONE" + AWAITS_ROLLBACK_APPROVE = "EXECUTION_AWAITING_ROLLBACK_APPROVE" + STOPPED = "EXECUTION_STOPPED" + ROLLBACK_PREPARED = "EXECUTION_ROLLBACK_PREPARED" + ROLLBACK_IN_PROGRESS = "EXECUTION_ROLLBACK_PREPARED" + ROLLBACK_DONE = "EXECUTION_ROLLBACK_DONE" + ROLLBACK_FAILED = "EXECUTION_ROLLBACK_FAILED" + +class InvalidStatusError(Exception): + pass + +def load_plan(datadir, rollback): + filename = datadir + if rollback: + filename += "/rollback_plan.pkl" + else: + filename += "/plan.pkl" + if not os.path.exists(filename): + raise FileNotFoundError(f"No pickle file found at {filename}") + with open(filename, 'rb') as f: + plan = pickle.load(f) + return plan + +wait_timeout_s = 5 + +class StatusManager: + # Class variable for the singleton instance + _instance = None + _instance_lock = threading.Lock() + + @classmethod + def get_instance(cls, options=None, logger=None): + """ + Get or create the singleton instance of StatusManager. + + Args: + options: Configuration options (required first time) + logger: Logger object (required first time) + + Returns: + The singleton StatusManager instance + """ + with cls._instance_lock: + if cls._instance is None: + if options is None or logger is None: + raise ValueError("options and logger required for initial instance creation") + cls._instance = cls(options, logger) + return cls._instance + + def __init__(self, options, logger, gparray, conn, gpEnv): + self.options = options + self.logger = logger + self._status_filename = self.options.coordinator_data_directory + '/gprebalance.status.json' + self.initial_gparray= gparray + self._flush_interval = 2.0 + self._last_flush_time = 0 + self._dirty = False + self._lock = threading.RLock() + self.conn = conn + self.gpenv = gpEnv + + # Initialize the status data structure + self.status_data = { + "current_status": None, + "rebalance_dir" :None, + "hosts_file" : None, + "status_history": [], + "moves": [], + "last_modified": datetime.now().isoformat() + } + + self.conf_dir = None + self.target_hosts_filename = None + + # Load existing status file if it exists + if os.path.exists(self._status_filename): + self._read_status_file() + self.analyze_gprecoverseg_states() + + + atexit.register(self._ensure_flush) + self._lock_timeout = 10 # seconds to wait for lock acquisition + self._max_retries = 3 # maximum number of retries for operations + self._retry_delay = 0.5 # seconds between retries + + self._stop_flush_thread = False + self._flush_thread = threading.Thread(target=self._auto_flush_worker, daemon=True) + self._flush_thread.start() + + def _auto_flush_worker(self): + """Background thread that periodically flushes dirty status to disk""" + while not self._stop_flush_thread: + if self._dirty and time.time() - self._last_flush_time >= self._flush_interval: + try: + self._flush_to_disk() + except Exception as e: + self.logger.error(f"Auto-flush failed: {str(e)}") + time.sleep(0.5) # Check every half second + + def _ensure_flush(self): + """Ensure any pending changes are flushed to disk""" + if self._dirty: + try: + self._flush_to_disk() + except Exception as e: + self.logger.error(f"Final flush failed: {str(e)}") + + def _read_status_file(self): + """Load status data from disk once""" + self.logger.debug(f"Loading status file: {self._status_filename}") + try: + with open(self._status_filename, 'r') as fp: + data = json.load(fp) + + # Basic validation + required_keys = ["current_status", "status_history", "moves"] + for key in required_keys: + if key not in data: + raise InvalidStatusError(f"Status file missing required key: {key}") + + # Store the data + self.status_data = data + self.conf_dir = self.status_data["rebalance_dir"] + + self._dirty = False + self._last_flush_time = time.time() + + except json.JSONDecodeError as e: + self.logger.error(f"Invalid JSON in status file: {str(e)}") + raise InvalidStatusError(f"Status file contains invalid JSON: {str(e)}") + except Exception as e: + self.logger.error(f"Failed to load status file: {str(e)}") + raise + + def _flush_to_disk(self): + """Write current status data to disk with file locking""" + with self._lock: + if not self._dirty: + return + + self.logger.debug("Flushing status data to disk") + # Update last modified timestamp + self.status_data["last_modified"] = datetime.now().isoformat() + # Create temporary file + temp_file = f"{self._status_filename}.tmp" + try: + # Write to temp file first + with open(temp_file, 'w') as fp: + json.dump(self.status_data, fp, indent=2) + fp.flush() + os.fsync(fp.fileno()) + + # Atomic rename for safer file updates + os.rename(temp_file, self._status_filename) + + self._dirty = False + self._last_flush_time = time.time() + self.logger.debug("Status data flushed successfully") + except Exception as e: + self.logger.error(f"Failed to flush status data: {str(e)}") + # Clean up temp file if it exists + if os.path.exists(temp_file): + try: + os.unlink(temp_file) + except: + pass + raise + + def create_status_file(self): + """Creates a new status file with initial state""" + with self._lock: + self.status_data = { + "current_status": "UNINITIALIZED", + "rebalance_dir" : None, + "hosts_file" : None, + "status_history": [ + { + "status": "UNINITIALIZED", + "timestamp": datetime.now().isoformat(), + "info": None + } + ], + "moves": [], + "last_modified": datetime.now().isoformat() + } + self._dirty = True + self._flush_to_disk() # Immediate flush for initial creation + + def get_current_status(self) -> Tuple[str, str]: + """Gets the current status""" + with self._lock: + if not self.status_data["status_history"]: + return (None, None) + + last_status = self.status_data["status_history"][-1] + return (last_status["status"], last_status["info"]) + + def set_status(self, statusEnum: RebalanceStatus, status_info=None): + """Set overall rebalance status""" + status = statusEnum.value + with self._lock: + status_entry = { + "status": status, + "timestamp": datetime.now().isoformat(), + "info": status_info + } + + self.status_data["current_status"] = status + self.status_data["status_history"].append(status_entry) + + # Update conf_dir or target_hosts_filename if applicable + if statusEnum == RebalanceStatus.PREPARED: + self.status_data["rebalance_dir"] = status_info + elif statusEnum == RebalanceStatus.VALIDATED: + self.status_data["hosts_file"] = status_info + + self._dirty = True + + # Consider immediate flush for important status changes + if statusEnum in [RebalanceStatus.FAILED, RebalanceStatus.PRE_EXECUTE_FAILED, RebalanceStatus.STEPS_FAILED, RebalanceStatus.ROLLBACK_FAILED]: + self._flush_to_disk() + + def record_moves_batch(self, moves_data): + + with self._lock: + for seq_no, segment, move, size, needs_switch in moves_data: + move_entry = { + "dbid": segment.dbid, + "content": segment.content, + "role": segment.role, + "role_after_switch": ('p' if move.is_mirror else 'm') if needs_switch else segment.role, + "status": MoveStatus.PENDING.value, + "source_kbytes": size, + "seq_id": seq_no, + } + + self.status_data["moves"].append(move_entry) + + if moves_data: + self._dirty = True + self._flush_to_disk() + + def update_move_status(self, dbids: List[int], status: MoveStatus): + """Update status for specified moves""" + with self._lock: + updated = False + for move in self.status_data["moves"]: + if move["dbid"] in dbids: + move["status"] = status.value + updated = True + + if updated: + self._dirty = True + + # Consider immediate flush for completion status + if status in [MoveStatus.COMPLETED, MoveStatus.FAILED]: + self._flush_to_disk() + + def get_moves_by_status(self, status: MoveStatus) -> List[Dict[str, Any]]: + """Get all moves with the specified status""" + with self._lock: + return [move.copy() for move in self.status_data["moves"] + if move["status"] == status.value] + + def get_move_by_dbid(self, dbid: int) -> Optional[Dict[str, Any]]: + """Get a specific move by dbid""" + with self._lock: + for move in self.status_data["moves"]: + if move["dbid"] == dbid: + return move.copy() + return None + + def flush(self): + """Manually flush status data to disk""" + self._flush_to_disk() + + def remove_all(self): + """Remove the status file and reset state""" + with self._lock: + self._stop_flush_thread = True + if self._flush_thread.is_alive(): + self._flush_thread.join(timeout=2.0) + + if os.path.exists(self._status_filename): + try: + os.unlink(self._status_filename) + except Exception as e: + self.logger.error(f"Failed to remove status file: {str(e)}") + + # Reset internal state + self.status_data = { + "current_status": None, + "status_history": [], + "moves": [], + "last_modified": datetime.now().isoformat() + } + self._dirty = False + self.conf_dir = None + self.target_hosts_filename = None + + def analyze_gprecoverseg_states(self): + if not self.conf_dir: + return + is_rollback = self.options.rollback + plan = load_plan(self.conf_dir, is_rollback) + failed_moves = [] + in_progress_moves = [] + not_started_moves = [] + completed_moves = [] + for move in self.status_data.get("moves", []): + move_status = move["status"] + if move_status == "FAILED": + failed_moves.append(move) + elif move_status == "IN_PROGRESS": + in_progress_moves.append(move) + elif move_status == "PENDING": + not_started_moves.append(move) + elif move_status == "COMPLETED": + completed_moves.append(move) + if failed_moves or in_progress_moves: + self.logger.info(f"Found {len(failed_moves)} failed moves and {len(in_progress_moves)} in-progress moves") + moves_to_analyze = failed_moves + in_progress_moves + self._analyze_failed_segments(moves_to_analyze, plan) + moves_to_recover = [] + for move in moves_to_analyze: + if move["status"] == "ASK_USER": + if ask_yesno('', f"Failed mirror move (dbid={move['dbid']}, content={move['content']}) cannot be rerun" + f" properly. Do you want to try to rollback it to original state?", "N"): + moves_to_recover.append(move) + else: + move["status"] = MoveStatus.SKIPPED.value + self._try_restore_config(moves_to_recover, plan) + self._dirty = True + self._flush_to_disk() + + for move in moves_to_analyze: + if move["status"] == MoveStatus.SKIPPED.value: + self.logger.info(f"Couldn't revert the move (dbid={move['dbid']}, content={move['content']})." + " Please, restore the required state and re-run") + + + def _analyze_failed_segments(self, moves, plan): + + self.logger.info(f"Analyzing {len(moves)} moves for state recovery") + segmentMap = {SegmentId( + seg.dbid, seg.content): seg for seg in self.initial_gparray.getSegmentsAsLoadedFromDb()} + hosts = set(self.initial_gparray.get_hostlist(includeCoordinator=False)) + unreachable_hosts = get_unreachable_segment_hosts(hosts, self.options.parallel) + update_unreachable_flag_for_segments(self.initial_gparray, unreachable_hosts) + plan_moves = {move.segid.dbid: move for move in plan.moves} + for move in moves: + segid = SegmentId(move["dbid"], move["content"]) + original_segment = plan.segmentMap[segid] + current_segment = segmentMap[segid] + + peer = None + is_mirror = False + for pair in self.initial_gparray.segmentPairs: + if pair.primaryDB.dbid == segid.dbid: + peer = pair.mirrorDB + break + elif pair.mirrorDB.dbid == segid.dbid: + peer = pair.primaryDB + is_mirror = True + break + + if is_mirror: + self.logger.info(f"Analyzing failed mirror move (dbid={original_segment.dbid}, content={original_segment.content}) " + f"to host {plan_moves[original_segment.dbid].dstHost.hostname}, directory {plan_moves[original_segment.dbid].target_datadir}") + if original_segment.hostname == current_segment.hostname and\ + original_segment.address == current_segment.address and\ + original_segment.dbid == current_segment.dbid and\ + (original_segment.role == current_segment.role or move["role_after_switch"] == current_segment.role) and\ + original_segment.datadir == current_segment.datadir and\ + peer.valid and not current_segment.valid: + # case 1 gprecoverseg could only have updated pg_hba_conf + # of corresponding primary before failure + # check that current_segment is running + if current_segment.unreachable: + move["status"] = MoveStatus.FAILED.value + continue + commandStr = f"pg_ctl status -D {current_segment.datadir}" + cmd_status = Command("pid", commandStr, REMOTE, current_segment.hostname) + cmd_status.run() + if cmd_status.get_return_code() != 0: + move["status"] = MoveStatus.ASK_ROLLBACK_OR_SKIP.value + continue + + #restore primary's pg_hba_conf + entries = hba_upd.create_entries(peer.hostname, original_segment.hostname, self.options.hba_hostnames) + updatecmd = hba_upd.SegUpdateHba(entries, peer.datadir, REMOTE, peer.hostname) + updatecmd.run() + if updatecmd.get_return_code() != 0: + move["status"] = MoveStatus.ASK_ROLLBACK_OR_SKIP.value + continue + host_port_tuple = (peer.hostname, peer.port) + pg.kill_existing_walsenders_on_primary([host_port_tuple]) + time.sleep(wait_timeout_s) + cmd_status = Command("pid", commandStr, REMOTE, current_segment.hostname) + cmd_status.run() + if cmd_status.get_return_code() != 0: + move["status"] = MoveStatus.ASK_ROLLBACK_OR_SKIP.value + continue + move["status"] = MoveStatus.ASK_ROLLBACK_OR_SKIP.value + continue + # case 2 : gp_segment_configuration has been updated by gprecoverseg + # from buildMirrorSegments.py:295 and we need to define whether + # basebackup was launched, completed or interrupred + elif original_segment.dbid == current_segment.dbid and\ + (original_segment.role == current_segment.role or\ + # case when seg is moved after 1st switch + move["role_after_switch"] == current_segment.role or + #primary only move. 2 switches, case when seg is moved after 1st switch + move["role_after_switch"] == move["role"]) and\ + (original_segment.hostname != current_segment.hostname or\ + original_segment.address != current_segment.address or\ + original_segment.datadir != current_segment.datadir): + + if current_segment.valid and current_segment.mode == 's': + move["status"] = MoveStatus.ASK_ROLLBACK_OR_SKIP.value + continue + + # check whether target directory exists + cmd = Command( + name="check_directory", + cmdStr=f"test -d {current_segment.datadir} && echo 'exists' || echo 'not_exists'", + ctxt=REMOTE, + remoteHost=current_segment.hostname) + cmd.run() + if not cmd.was_successful() or cmd.get_stdout().strip() != 'exists': + self.logger.info(f"Failed to check directory on {current_segment.hostname}") + move["status"] = MoveStatus.ASK_ROLLBACK_OR_SKIP.value + continue + + #check the port update + cmd = Command( + name="get_postgresql_conf_port", + cmdStr=f"grep -E '^port\\s*=' {current_segment.datadir}/postgresql.conf | sed -E 's/^port\\s*=\\s*([0-9]+).*/\\1/'", + ctxt=REMOTE, + remoteHost=current_segment.hostname + ) + cmd.run() + if not cmd.was_successful(): + self.logger.info(f"Failed to get port from postgresql.conf on {current_segment.hostname}: {cmd.get_stderr()}") + move["status"] = MoveStatus.ASK_ROLLBACK_OR_SKIP.value + continue + output = cmd.get_stdout().strip() + if not output or not output.isdigit(): + move["status"] = MoveStatus.ASK_ROLLBACK_OR_SKIP.value + continue + datadir_port = int(output) + if datadir_port != current_segment.port: + self.logger.info(f"Port mismatch in postgresql.conf: expected {current_segment.port}, got {datadir_port}") + move["status"] = MoveStatus.ASK_ROLLBACK_OR_SKIP.value + continue + + #check if server is running + process_running = self._check_postgres_process(current_segment.hostname, current_segment.datadir) + if not process_running: + + is_started = self._wait_for_pg_startup(current_segment.hostname, current_segment.datadir, timeout_seconds=20) + if not is_started: + attemp_start = self._attempt_start_mirror(current_segment) + if not attemp_start and not self._wait_for_pg_startup(current_segment.hostname, current_segment.datadir, timeout_seconds=20): + self.logger.info("Failed to start the mirror") + move["status"] = MoveStatus.ASK_ROLLBACK_OR_SKIP.value + continue + if not self._wait_for_config_update(current_segment.dbid, 60): + move["status"] = MoveStatus.ASK_ROLLBACK_OR_SKIP.value + continue + elif not self._wait_for_config_update(current_segment.dbid, 60): + move["status"] = MoveStatus.ASK_ROLLBACK_OR_SKIP.value + continue + + move["status"] = MoveStatus.COMPLETED.value + + + def _check_postgres_process(self, host, data_dir): + try: + # First check with pg_ctl status + pg_ctl_cmd = Command( + name="check_pg_ctl_status", + cmdStr=f"pg_ctl status -D {data_dir}", + ctxt=REMOTE, + remoteHost=host + ) + pg_ctl_cmd.run() + + # pg_ctl status returns 0 if server is running + if pg_ctl_cmd.get_return_code() == 0: + return True + + # If pg_ctl status failed, check for actual process + # Get postmaster.pid contents if it exists + pid_cmd = Command( + name="check_postmaster_pid", + cmdStr=f"test -f {data_dir}/postmaster.pid && cat {data_dir}/postmaster.pid | head -1", + ctxt=REMOTE, + remoteHost=host + ) + pid_cmd.run() + + if not pid_cmd.was_successful() or not pid_cmd.get_stdout().strip(): + return False + + pid = pid_cmd.get_stdout().strip() + if not pid.isdigit(): + return False + + # Check if process with that PID exists and is postgres + process_cmd = Command( + name="check_postgres_process", + cmdStr=f"ps -p {pid} -o cmd= | grep postgres", + ctxt=REMOTE, + remoteHost=host + ) + process_cmd.run() + + return process_cmd.was_successful() and process_cmd.get_stdout().strip() + + except Exception as e: + self.logger.error(f"Error checking postgres process on {host}: {str(e)}") + return False + + def _wait_for_pg_startup(self, host, data_dir, timeout_seconds=120): + + self.logger.info(f"Waiting for PostgreSQL to start on {host}:{data_dir}") + + start_time = time.time() + check_interval = 5 # Check every 5 seconds + + while time.time() - start_time < timeout_seconds: + if self._check_postgres_process(host, data_dir): + elapsed = int(time.time() - start_time) + self.logger.info(f"PostgreSQL started on {host}:{data_dir} after {elapsed} seconds") + return True + + # Wait before next check + time.sleep(check_interval) + elapsed = int(time.time() - start_time) + + self.logger.info(f"Timeout waiting for PostgreSQL to start on {host}:{data_dir}") + return False + + def _wait_for_config_update(self, dbid, timeout_seconds=120): + + start_time = time.time() + check_interval = 5 + + conf_sql = """ + SELECT c.mode, c.status + FROM gp_segment_configuration AS c WHERE c.dbid = {sdbid} + """ .format(sdbid=dbid) + while time.time() - start_time < timeout_seconds: + cursor = dbconn.query(self.conn, conf_sql) + for mode, status in cursor: + if mode == 's' and status == 'u': + return True + + # Wait before next check + time.sleep(check_interval) + elapsed = int(time.time() - start_time) + + return False + + def _try_restore_config(self, moves_to_recover, plan): + from gppylib.commands.gp import GpRecoverSeg + gparray = copy.deepcopy(self.initial_gparray) + def write_gprecoverseg_config(segment, conf_dir): + filename = self.conf_dir + "_revert_" + "dbid" + str(segment.dbid) + with open(filename, 'w') as fp: + line = (f"{canonicalize_address(segment.address)}|" + f"{segment.port}|{segment.datadir}") + self.logger.info( + "About to run gprecoverseg for recovering mirror " + f"(dbid = {segment.dbid}, content = {segment.content}) {line}") + fp.write(line) + return filename + for move in moves_to_recover: + segid = SegmentId(move["dbid"], move["content"]) + original_segment = plan.segmentMap[segid] + dbids = {} + seg = None + peer = None + for pair in gparray.segmentPairs: + if pair.mirrorDB.dbid == segid.dbid: + pair.mirrorDB = copy.copy(original_segment) + pair.mirrorDB.status = 'd' + pair.mirrorDB.mode = 'n' + dbids[pair.mirrorDB.dbid] = True + pair.mirrorDB.role = 'm' + seg=pair.mirrorDB + peer = pair.primaryDB + configInterface.getConfigurationProvider().updateSystemConfig( + gparray, + "gprebalance: segment config for resync", + dbIdToForceMirrorRemoveAdd=dbids, + useUtilityMode=False, + allowPrimary=False + ) + #host_port_tuple = (peer.hostname, peer.port) + #pg.kill_existing_walsenders_on_primary([host_port_tuple]) + filename = write_gprecoverseg_config(seg, self.conf_dir) + + recoversegOptions = f"-a -l {os.path.join(os.environ.get('HOME', '.'),'gpAdminLogs/rebalance')} "\ + f"-i {filename}" + if self.options.hba_hostnames: + recoversegOptions += " --hba-hostnames" + cmd = GpRecoverSeg("Running gprecoverseg", options=recoversegOptions) + cmd.run() + recordStatus = MoveStatus.REVERTED.value + if not cmd.was_successful(): + self.logger.info(f"Failed to rollback mirror moves (dbid = {seg.dbid}, content = {seg.content}): {cmd.get_stderr()}, skipping...") + recordStatus = MoveStatus.SKIPPED.value + move["status"] = recordStatus + os.unlink(filename) + + def _attempt_start_mirror(self, segment): + from gppylib.commands.gp import SegmentStart + from gppylib.gp_era import read_era + seg = Segment(None, None, None, None, None, None, None, None, + segment.port, segment.datadir) + cmd = cmd = SegmentStart( + name="Attemping to start segment with dbid %s:" % (str(segment.dbid)) + , gpdb=seg + , numContentsInCluster=0 + , era=read_era(self.gpenv.getCoordinatorDataDir(), logger=self.logger) + , mirrormode="mirror" + , utilityMode=False + , ctxt=REMOTE + , remoteHost=segment.hostname) + self.logger.info(str(cmd)) + cmd.run() + if not cmd.was_successful(): + return False + return True \ No newline at end of file diff --git a/gpMgmt/bin/gprebalance_modules/rebalance_step.py b/gpMgmt/bin/gprebalance_modules/rebalance_step.py new file mode 100755 index 000000000000..39bc176d1616 --- /dev/null +++ b/gpMgmt/bin/gprebalance_modules/rebalance_step.py @@ -0,0 +1,77 @@ +#!/usr/bin/env python3 + +from enum import Enum +import pickle +from gprebalance_modules.planner import * + +class RebalanceStep: + class Status(Enum): + APPROVE_REQUIRED = 1 + PLANNED = 2 + IN_PROGRESS = 3 + ERROR = 4 + ROLLBACK_PLANNED = 5 + ROLLED_BACK = 6 + CANCELLED = 7 + DONE = 8 + + def __init__(self, move: LogicalMove): + self.move_order = -1 + self.move = move + self.status = self.Status.PLANNED + + def __str__(self): + return ( + f"Rebalance step with move_order: {self.getMoveOrder()}, status: {self.getStatus()}" + ) + + def getMoveOrder(self): + return self.move_order + + def setMoveOrder(self, move_order: int): + self.move_order = move_order + + def getStatus(self): + return self.status + + def getMove(self): + return self.move + + def setStatus(self, status: Status): + self.status = status + + def serializeStep(self) -> bytes: + return pickle.dumps(self) + +class RebalanceStepMoveMirror(RebalanceStep): + def __init__(self, move: LogicalMove): + super().__init__(move) + + def __str__(self): + return ( + f"{super().__str__()}, type: RebalanceStepMoveMirror:\n" + f"{str(self.move)}" + ) + +class RebalanceStepSwitchoverToMirror(RebalanceStep): + def __init__(self, move: LogicalMove): + super().__init__(move) + self.status = self.Status.APPROVE_REQUIRED + + def __str__(self): + return ( + f"{super().__str__()}, type: RebalanceStepSwitchoverToMirror, DBID {str(self.move.seg.getSegmentDbId())}" + ) + +class RebalanceStepSwitchoverToPrimary(RebalanceStep): + def __init__(self, move: LogicalMove): + super().__init__(move) + self.status = self.Status.APPROVE_REQUIRED + + def __str__(self): + return ( + f"{super().__str__()}, type: RebalanceStepSwitchoverToPrimary, DBID {str(self.move.seg.getSegmentDbId())}" + ) + +def deserializeStep(input: bytes) -> RebalanceStep: + return pickle.loads(input) diff --git a/gpMgmt/bin/gprebalance_modules/rebalance_validator.py b/gpMgmt/bin/gprebalance_modules/rebalance_validator.py new file mode 100644 index 000000000000..8c10fb142737 --- /dev/null +++ b/gpMgmt/bin/gprebalance_modules/rebalance_validator.py @@ -0,0 +1,78 @@ +from typing import List, Set +from gprebalance_modules.rebalance import Host, MirrorStrategy, dbconn, GpArray, Segment, MODE_NOT_SYNC, STATUS_DOWN + + +class StateValidationError(Exception): + pass + + +class ClusterValidator: + def __init__(self, existing_hosts: Set[Host], target_hosts: Set[Host], segarray: List[Segment], has_mirrors: bool, mirror_strategy: MirrorStrategy): + self.mirror_strategy = mirror_strategy + self.existing_hosts = existing_hosts + self.target_hosts = target_hosts + self.segarray = segarray + self.has_mirrors = has_mirrors + + def validate_segment_status(self): + inv = [seg.content for seg in self.segarray if not seg.valid] + if len(inv) > 0: + raise StateValidationError( + f"The {[c for c in inv]} segments are down") + + def validate_existing_configuration(self) -> tuple[bool, MirrorStrategy]: + arr = GpArray(self.segarray) + total_primaries = arr.get_primary_count() + total_hosts = len(self.existing_hosts) + expected_primaries = total_primaries // total_hosts + strat = None + + if arr.guessIsSpreadMirror(): + strat = MirrorStrategy.SPREAD + elif arr.hasMirrors: + strat = MirrorStrategy.GROUPED + for host in self.existing_hosts: + if set([s.contentid for s in host.primary_segments]) & set([s.contentid for s in host.mirror_segments]): + strat = None + break + + for host in self.existing_hosts: + if len(host.primary_segments) != expected_primaries: + return False, strat + + return True, strat + + def prevalidate_segment_distribution(self): + """ + Validate whether segments can be uniformly distributed across target hosts + """ + total_primary_segments = sum(len(h.primary_segments) + for h in self.existing_hosts) + total_hosts = len(self.target_hosts) + + if total_primary_segments % total_hosts != 0: + raise StateValidationError( + f"Cannot evenly distribute {total_primary_segments} segments across {total_hosts} hosts." + ) + + def prevalidate_mirror_strategy(self): + """ + Validate whether the specified mirroring strategy can be achieved + """ + if not self.has_mirrors: + return + total_hosts = len(self.target_hosts) + total_primary_segments = sum(len(h.primary_segments) + for h in self.existing_hosts) + if total_hosts < 2: + raise StateValidationError( + """Cannot support target mirroring strategy on given configuration. All + primaries will be at single host.""" + ) + + primaries_per_host = total_primary_segments // total_hosts + if self.mirror_strategy == MirrorStrategy.SPREAD and primaries_per_host >= total_hosts: + raise StateValidationError( + "Cannot support spread mirroring strategy on given configuration. " + "Use cluster utilities like gpresize or gpexpand to get desired cluster configuration" + ) diff --git a/gpMgmt/bin/gprebalance_modules/shrink.py b/gpMgmt/bin/gprebalance_modules/shrink.py new file mode 100644 index 000000000000..48798267ed6c --- /dev/null +++ b/gpMgmt/bin/gprebalance_modules/shrink.py @@ -0,0 +1,756 @@ +#!/usr/bin/env python3 + +from transitions import Machine +from contextlib import closing +from typing import Any + +try: + from gppylib.commands.unix import * + from gppylib.commands.gp import * + from gppylib.gplog import * + from gppylib.db import dbconn + from gppylib import userinput + from gppylib.gparray import GpArray, Segment + from gppylib.fault_injection import * + from gppylib.userinput import * + from gppylib.commands import base + from gppylib.commands.gp import SEGMENT_STOP_TIMEOUT_DEFAULT, SegmentStop + from gppylib.system.environment import * + from gprebalance_modules.planner import * + from gprebalance_modules.rebalance_schema import RebalanceSchema, STATE_NOT_DEFINED +except ImportError as e: + sys.exit('ERROR: Cannot import modules. Please check that you have sourced greenplum_path.sh. Detail: ' + str(e)) + + +DBNAME = 'postgres' + +def print_progress(pool: WorkerPool, interval: int = 10) -> None: + """ + Waits for a WorkerPool to complete, printing a progress percentage marker + once at the beginning of the call, and thereafter at the provided interval + (default ten seconds). A final 100% marker is printed upon completion. + """ + def print_completed_percentage() -> bool: + # pool.completed can change asynchronously; save its value. + completed = pool.completed + + pct = 0 + if pool.assigned: + pct = float(completed) / pool.assigned + + pool.logger.info(f'{pct:.2%} of jobs completed') + return completed >= pool.assigned + + # print_completed_percentage() returns True if we're done. + while not print_completed_percentage(): + if pool.join(interval): + return + +class GGShrink: + timeout = SEGMENT_STOP_TIMEOUT_DEFAULT + stop_mode = 'fast' + + states = [ + 'STATE_START', + 'STATE_CHECK_PREVIOUS_RUN', + 'STATE_END', + 'STATE_ERROR', + 'STATE_END_FROM_ROLLBACK' + ] + + # Note: order of states in the list below is important, + # as we rely on it when recover from an interrupted state. + # These states are separated from 'states' above, because + # these states exist after the shrink schema is created, + # so they are reflected in the status table inside the schema, + # and we can re-enter the interrupted state. + states_main_shrink_flow = [ + 'STATE_BACKUP_CATALOG_AND_UPDATE_TARGET_SEGMENT_COUNT_STARTED', + 'STATE_BACKUP_CATALOG_AND_UPDATE_TARGET_SEGMENT_COUNT_DONE', + 'STATE_PREPARE_SHRINK_SCHEMA_STARTED', + 'STATE_PREPARE_SHRINK_SCHEMA_DONE', + 'STATE_SHRINK_TABLES_STARTED', + 'STATE_SHRINK_TABLES_DONE', + 'STATE_SHRINK_CATALOG_STARTED', + 'STATE_SHRINK_CATALOG_DONE', + 'STATE_SHRINK_SEGMENTS_STOP_STARTED', + 'STATE_SHRINK_SEGMENTS_STOP_DONE', + 'STATE_SHRINK_DONE' + ] + + # Note: order of states in the list below is important, + # as we rely on it when recover from an interrupted state. + # These states are separated from 'states' and 'states_main_shrink_flow' + # above in order to support re-enter the interrupted state of rollback. + states_rollback_flow = [ + 'STATE_SHRINK_ROLLBACK_RESTORE_TARGET_SEGMENT_COUNT_START', + 'STATE_SHRINK_ROLLBACK_RESTORE_TARGET_SEGMENT_COUNT_DONE', + 'STATE_SHRINK_ROLLBACK_PREPARE_SCHEMA_START', + 'STATE_SHRINK_ROLLBACK_PREPARE_SCHEMA_DONE', + 'STATE_SHRINK_ROLLBACK_SHRINKED_TABLES_START', + 'STATE_SHRINK_ROLLBACK_SHRINKED_TABLES_DONE', + 'STATE_SHRINK_ROLLBACK_DROP_SCHEMA_START', + 'STATE_SHRINK_ROLLBACK_DROP_SCHEMA_DONE' + ] + + transitions = [ + { + 'trigger': 'start', + 'source': 'STATE_START', + 'dest': 'STATE_CHECK_PREVIOUS_RUN' + }, + { + 'trigger': 'move_to_STATE_BACKUP_CATALOG_AND_UPDATE_TARGET_SEGMENT_COUNT_STARTED', + 'source': 'STATE_CHECK_PREVIOUS_RUN', + 'dest': 'STATE_BACKUP_CATALOG_AND_UPDATE_TARGET_SEGMENT_COUNT_STARTED' + }, + { + 'trigger': 'move_to_STATE_BACKUP_CATALOG_AND_UPDATE_TARGET_SEGMENT_COUNT_DONE', + 'source': 'STATE_BACKUP_CATALOG_AND_UPDATE_TARGET_SEGMENT_COUNT_STARTED', + 'dest': 'STATE_BACKUP_CATALOG_AND_UPDATE_TARGET_SEGMENT_COUNT_DONE' + }, + { + 'trigger': 'move_to_STATE_PREPARE_SHRINK_SCHEMA_STARTED', + 'source': 'STATE_BACKUP_CATALOG_AND_UPDATE_TARGET_SEGMENT_COUNT_DONE', + 'dest': 'STATE_PREPARE_SHRINK_SCHEMA_STARTED' + }, + { + 'trigger': 'move_to_STATE_PREPARE_SHRINK_SCHEMA_DONE', + 'source': 'STATE_PREPARE_SHRINK_SCHEMA_STARTED', + 'dest': 'STATE_PREPARE_SHRINK_SCHEMA_DONE' + }, + { + 'trigger': 'move_to_STATE_SHRINK_TABLES_STARTED', + 'source': 'STATE_PREPARE_SHRINK_SCHEMA_DONE', + 'dest': 'STATE_SHRINK_TABLES_STARTED' + }, + { + 'trigger': 'move_to_STATE_SHRINK_TABLES_DONE', + 'source': 'STATE_SHRINK_TABLES_STARTED', + 'dest': 'STATE_SHRINK_TABLES_DONE' + }, + { + 'trigger': 'move_to_STATE_SHRINK_CATALOG_STARTED', + 'source': 'STATE_SHRINK_TABLES_DONE', + 'dest': 'STATE_SHRINK_CATALOG_STARTED' + }, + { + 'trigger': 'move_to_STATE_SHRINK_CATALOG_DONE', + 'source': 'STATE_SHRINK_CATALOG_STARTED', + 'dest': 'STATE_SHRINK_CATALOG_DONE' + }, + { + 'trigger': 'move_to_STATE_SHRINK_SEGMENTS_STOP_STARTED', + 'source': 'STATE_SHRINK_CATALOG_DONE', + 'dest': 'STATE_SHRINK_SEGMENTS_STOP_STARTED' + }, + { + 'trigger': 'move_to_STATE_SHRINK_SEGMENTS_STOP_DONE', + 'source': 'STATE_SHRINK_SEGMENTS_STOP_STARTED', + 'dest': 'STATE_SHRINK_SEGMENTS_STOP_DONE' + }, + { + 'trigger': 'move_to_STATE_SHRINK_DONE', + 'source': 'STATE_SHRINK_SEGMENTS_STOP_DONE', + 'dest': 'STATE_SHRINK_DONE' + }, + { + 'trigger': 'move_to_STATE_END', + 'source': ['STATE_SHRINK_DONE', 'STATE_CHECK_PREVIOUS_RUN', 'STATE_END_FROM_ROLLBACK'], + 'dest': 'STATE_END' + }, + { + 'trigger': 'move_to_STATE_SHRINK_ROLLBACK_RESTORE_TARGET_SEGMENT_COUNT_START', + 'source': 'STATE_CHECK_PREVIOUS_RUN', + 'dest': 'STATE_SHRINK_ROLLBACK_RESTORE_TARGET_SEGMENT_COUNT_START' + }, + { + 'trigger': 'move_to_STATE_SHRINK_ROLLBACK_RESTORE_TARGET_SEGMENT_COUNT_DONE', + 'source': 'STATE_SHRINK_ROLLBACK_RESTORE_TARGET_SEGMENT_COUNT_START', + 'dest': 'STATE_SHRINK_ROLLBACK_RESTORE_TARGET_SEGMENT_COUNT_DONE' + }, + { + 'trigger': 'move_to_STATE_SHRINK_ROLLBACK_PREPARE_SCHEMA_START', + 'source': 'STATE_SHRINK_ROLLBACK_RESTORE_TARGET_SEGMENT_COUNT_DONE', + 'dest': 'STATE_SHRINK_ROLLBACK_PREPARE_SCHEMA_START' + }, + { + 'trigger': 'move_to_STATE_SHRINK_ROLLBACK_PREPARE_SCHEMA_DONE', + 'source': 'STATE_SHRINK_ROLLBACK_PREPARE_SCHEMA_START', + 'dest': 'STATE_SHRINK_ROLLBACK_PREPARE_SCHEMA_DONE' + }, + { + 'trigger': 'move_to_STATE_SHRINK_ROLLBACK_SHRINKED_TABLES_START', + 'source': 'STATE_SHRINK_ROLLBACK_PREPARE_SCHEMA_DONE', + 'dest': 'STATE_SHRINK_ROLLBACK_SHRINKED_TABLES_START' + }, + { + 'trigger': 'move_to_STATE_SHRINK_ROLLBACK_SHRINKED_TABLES_DONE', + 'source': 'STATE_SHRINK_ROLLBACK_SHRINKED_TABLES_START', + 'dest': 'STATE_SHRINK_ROLLBACK_SHRINKED_TABLES_DONE' + }, + { + 'trigger': 'move_to_STATE_SHRINK_ROLLBACK_DROP_SCHEMA_START', + 'source': 'STATE_SHRINK_ROLLBACK_SHRINKED_TABLES_DONE', + 'dest': 'STATE_SHRINK_ROLLBACK_DROP_SCHEMA_START' + }, + { + 'trigger': 'move_to_STATE_SHRINK_ROLLBACK_DROP_SCHEMA_DONE', + 'source': 'STATE_SHRINK_ROLLBACK_DROP_SCHEMA_START', + 'dest': 'STATE_SHRINK_ROLLBACK_DROP_SCHEMA_DONE' + }, + { + 'trigger': 'move_to_STATE_END_FROM_ROLLBACK', + 'source': ['STATE_SHRINK_ROLLBACK_DROP_SCHEMA_DONE', 'STATE_CHECK_PREVIOUS_RUN', 'STATE_START'], + 'dest': 'STATE_END_FROM_ROLLBACK' + }, + { + 'trigger': 'move_to_STATE_ERROR', + 'source': '*', + 'dest': 'STATE_ERROR' + } + ] + + def __init__(self, conn: dbconn.Connection, + schema: RebalanceSchema, logger: Any, options: Any, gpEnv: GpCoordinatorEnvironment, gpArray: gparray.GpArray, gpArrayDumpFilename: str) -> None: + self.logger = logger + self.options = options + self.gpEnv = gpEnv + self.conn = conn + self.shutdown_requested = False + self.workers_for_tables_rebalance = None + self.workers_for_segment_stop = None + self.gparray = gpArray + self.gparray_dump_file = gpArrayDumpFilename + self.rebalance_schema = schema + self.shrink_plan = None + self.dumped_gparray = gparray.GpArray.initFromFile(self.gparray_dump_file) if os.path.exists(self.gparray_dump_file) else None + + self.machine = Machine(model = self, + queued=True, + states = self.states + self.states_main_shrink_flow + self.states_rollback_flow, + transitions = self.transitions, + initial = 'STATE_START', + before_state_change = 'on_every_state') + + def run(self, shrinkPlan: ShrinkPlan) -> None: + self.shrink_plan = shrinkPlan + self.trigger('start') + + def rollback(self, shrinkPlan: ShrinkPlan) -> None: + self.shrink_plan = shrinkPlan + if not self.rebalance_schema.schemaExists(): + self.logger.info("Rebalance schema doesn't exist. Can't perform rollback.") + self.trigger('move_to_STATE_END_FROM_ROLLBACK') + return + else: + state_from_prev_run = self.rebalance_schema.getShrinkStateFromPreviousRun() + if state_from_prev_run != STATE_NOT_DEFINED: + # check maybe the state is the final one + if self.state_is_final(state_from_prev_run): + self.logger.info("Previous run was completed successfully. Can't perform rollback.") + self.trigger('move_to_STATE_END_FROM_ROLLBACK') + return + + if not self.state_can_rollback(state_from_prev_run) or self.is_gp_segment_configuration_shrinked(): + self.logger.info("Can't perform rollback as the catalog is already updated") + self.trigger('move_to_STATE_END_FROM_ROLLBACK') + return + + self.trigger('start') + + def get_state_after_interrupt(self, prev_state) -> str: + prev_idx = self.states_main_shrink_flow.index(prev_state) + lower = self.states_main_shrink_flow.index('STATE_BACKUP_CATALOG_AND_UPDATE_TARGET_SEGMENT_COUNT_STARTED') + upper = self.states_main_shrink_flow.index('STATE_SHRINK_TABLES_DONE') + if prev_idx >= lower and prev_idx <= upper: + + #if shrink is interrupted after catalog update and before the state is logged + if prev_state == 'STATE_SHRINK_TABLES_DONE' and \ + self.dumped_gparray is not None \ + and self.gparray.get_segment_count() + self.shrink_plan.target_segment_count == self.dumped_gparray.get_segment_count(): + return 'STATE_SHRINK_CATALOG_DONE' + + row = dbconn.queryRow(self.conn, 'SELECT gp_toolkit.gp_rebalance_numsegments_is_set();') + # means that target rebalance numsegments is reset, and new tables are created at old segment count + if bool(row[0]) is False: + self.logger.info("Cluster restarted after previous run, trying to repopulate the relation queue") + return 'STATE_BACKUP_CATALOG_AND_UPDATE_TARGET_SEGMENT_COUNT_STARTED' + + return self.states_main_shrink_flow[prev_idx + 1] + + def on_every_state(self) -> None: + if self.shutdown_requested: + self.logger.info('Shrink was interrupted') + raise Exception('Shrink was interrupted') + + assert self.state in self.states + self.states_main_shrink_flow + self.states_rollback_flow + + if self.state in self.states_main_shrink_flow + self.states_rollback_flow: + self.rebalance_schema.storeShrinkState(self.state) + + def cleanup(self, prev_run_was_complete: bool) -> None: + if not prev_run_was_complete: + self.logger.warning("ggrebalance hasn't finished shrink process properly. Previous run was interrupted. " + "Some unbalanced tables can still exist.") + + # get default num segments + dbconn.execSQL(self.conn, 'BEGIN') + dbconn.execSQL(self.conn, 'SELECT gp_expand_lock_catalog()') + row = dbconn.queryRow(self.conn, 'SELECT gp_toolkit.gp_rebalance_numsegments_is_set()') + numsegments_is_set = bool(row[0]) + dbconn.execSQL(self.conn, 'END') + + if numsegments_is_set: + self.logger.warning('Current numsegments is not equal to default value.') + self.logger.info('Suggestion: explicitly reset the value before cleanup. Note: cluster restart will implicitly reset the value.') + + if (self.options.interactive and + not userinput.ask_yesno(None, "\nContinue with cleanup?", 'Y')): + self.logger.info('Cleanup was interrupted...') + return + + if (numsegments_is_set and + (not self.options.interactive or userinput.ask_yesno(None, "\nReset numsegments to default?", 'Y'))): + dbconn.execSQL(self.conn, 'BEGIN') + dbconn.execSQL(self.conn, 'SELECT gp_expand_lock_catalog()') + dbconn.execSQL(self.conn, 'SELECT gp_toolkit.gp_reset_rebalance_numsegments()') + dbconn.execSQL(self.conn, 'COMMIT') + self.logger.info('Reset numsegments to default is done.') + + if os.path.exists(self.gparray_dump_file): + os.remove(self.gparray_dump_file) + + def state_is_final(self, state: str) -> bool: + return state == self.states_main_shrink_flow[-1] + + # state callbacks start here + + @wrap_state_func_with_faults + def on_enter_STATE_CHECK_PREVIOUS_RUN(self) -> None: + assert self.rebalance_schema.schemaExists() + # check whether we can get the state where we stopped in previous run + # in order to proceed from the same point + state_from_prev_run = self.rebalance_schema.getShrinkStateFromPreviousRun() + # check maybe the state is the final one + if self.state_is_final(state_from_prev_run): + if self.options.rollback_required: + self.logger.info(f"Previous run was completed successfully. Can't perform rollback.") + self.trigger('move_to_STATE_END_FROM_ROLLBACK') + else: + if state_from_prev_run in self.states_rollback_flow: + self.logger.info('Continue interrupted shrink rollback operation...') + self.logger.info(f"Previous run stopped after state '{state_from_prev_run}', trying to continue from the next state...") + try: + next_state = self.states_rollback_flow[ self.states_rollback_flow.index(state_from_prev_run) + 1 ] + except: + self.logger.error("Can't determine next rollback state") + self.trigger('move_to_STATE_ERROR') + return + else: + if self.options.rollback_required: + self.trigger('move_to_STATE_SHRINK_ROLLBACK_RESTORE_TARGET_SEGMENT_COUNT_START') + return + + # no state so far, so start from the beginning + if state_from_prev_run == STATE_NOT_DEFINED: + self.trigger('move_to_STATE_BACKUP_CATALOG_AND_UPDATE_TARGET_SEGMENT_COUNT_STARTED') + return + + self.logger.info('Continue interrupted shrink operation...') + self.logger.info(f"Previous run stopped after state '{state_from_prev_run}', trying to continue from the next state...") + try: + next_state = self.get_state_after_interrupt(state_from_prev_run) + except: + self.logger.error("Can't determine next state. Try to execute cleanup.") + self.trigger('move_to_STATE_ERROR') + return + + # use auto to_«state» method to recover + self.trigger(f'to_{next_state}') + + @wrap_state_func_with_faults + def on_enter_STATE_BACKUP_CATALOG_AND_UPDATE_TARGET_SEGMENT_COUNT_STARTED(self) -> None: + dbconn.execSQL(self.conn, 'BEGIN') + dbconn.execSQL(self.conn, 'SELECT gp_expand_lock_catalog()') + dbconn.execSQL(self.conn, 'CHECKPOINT') + dbconn.execSQL(self.conn, f'SELECT gp_toolkit.gp_set_rebalance_numsegments({self.shrink_plan.getTargetSegmentCount()})') + + self.gparray.dumpToFile(self.gparray_dump_file) + + # Rebalance the status tables we've created previously right here before we start to rebalance all other tables. + self.rebalance_schema.rebalanceSchema(self.shrink_plan.getTargetSegmentCount()) + + dbconn.execSQL(self.conn, 'COMMIT') + + self.trigger('move_to_STATE_BACKUP_CATALOG_AND_UPDATE_TARGET_SEGMENT_COUNT_DONE') + + @wrap_state_func_with_faults + def on_enter_STATE_BACKUP_CATALOG_AND_UPDATE_TARGET_SEGMENT_COUNT_DONE(self) -> None: + self.logger.info(f'Updated target segment count to {self.shrink_plan.getTargetSegmentCount()}') + self.trigger('move_to_STATE_PREPARE_SHRINK_SCHEMA_STARTED') + + @wrap_state_func_with_faults + def on_enter_STATE_PREPARE_SHRINK_SCHEMA_STARTED(self) -> None: + self.prepare_shrink_schema(False) + self.trigger('move_to_STATE_PREPARE_SHRINK_SCHEMA_DONE') + + @wrap_state_func_with_faults + def on_enter_STATE_PREPARE_SHRINK_SCHEMA_DONE(self) -> None: + self.logger.info(f'Initiated list of tables to rebalance') + self.trigger('move_to_STATE_SHRINK_TABLES_STARTED') + + @wrap_state_func_with_faults + def on_enter_STATE_SHRINK_TABLES_STARTED(self) -> None: + self.logger.info('Start tables rebalance for shrink') + # perform 'ALTER TABLE REBALANCE' for all not yet processed tables + self.rebalance_tables('none', 'done', self.shrink_plan.getTargetSegmentCount()) + self.trigger('move_to_STATE_SHRINK_TABLES_DONE') + + @wrap_state_func_with_faults + def on_enter_STATE_SHRINK_TABLES_DONE(self) -> None: + self.logger.info('Tables rebalance complete') + self.trigger('move_to_STATE_SHRINK_CATALOG_STARTED') + + @wrap_state_func_with_faults + def on_enter_STATE_SHRINK_CATALOG_STARTED(self) -> None: + self.logger.info('Start catalog shrink') + + ## Shrink catalog + dbconn.execSQL(self.conn, 'BEGIN') + dbconn.execSQL(self.conn, 'SELECT gp_expand_lock_catalog()') + dbconn.execSQL(self.conn, f'DELETE FROM gp_segment_configuration WHERE content >= {self.shrink_plan.getTargetSegmentCount()}') + dbconn.execSQL(self.conn, 'CHECKPOINT') + dbconn.execSQL(self.conn, 'SELECT gp_expand_bump_version()') + dbconn.execSQL(self.conn, 'SELECT gp_toolkit.gp_reset_rebalance_numsegments()') + dbconn.execSQL(self.conn, 'COMMIT') + + self.trigger('move_to_STATE_SHRINK_CATALOG_DONE') + + @wrap_state_func_with_faults + def on_enter_STATE_SHRINK_CATALOG_DONE(self) -> None: + self.logger.info('Catalog shrink complete') + self.trigger('move_to_STATE_SHRINK_SEGMENTS_STOP_STARTED') + + @wrap_state_func_with_faults + def on_enter_STATE_SHRINK_SEGMENTS_STOP_STARTED(self) -> None: + self.logger.info('Stopping shrinked segments...') + + gp_array = self.dumped_gparray + + if gp_array is None: + gp_array = self.gparray + + segments_to_stop = gp_array.get_segment_count() - self.shrink_plan.getTargetSegmentCount() + self.workers_for_segment_stop = WorkerPool(numWorkers=min(segments_to_stop, self.options.batch_size)) + + # Stop primaries first, and mirrors after primaries, + # to avoid hanging replication processes + seg_roles = [gparray.ROLE_PRIMARY, gparray.ROLE_MIRROR] + for seg_role in seg_roles: + self.logger.info(f"Prepare to stop segments with role '{seg_role}'") + for seg in gp_array.getSegDbList(): + if (seg.getSegmentContentId() >= self.shrink_plan.getTargetSegmentCount() and + seg.getSegmentRole() == seg_role and seg.isSegmentUp()): + cmd = self.SegmentStopAfterShrink(self, seg) + self.workers_for_segment_stop.addCommand(cmd) + if self.shutdown_requested: + break + print_progress(self.workers_for_segment_stop, interval=1) + + self.workers_for_segment_stop.haltWork() + self.workers_for_segment_stop.joinWorkers() + + for task in self.workers_for_segment_stop.getCompletedItems(): + if not task.was_successful(): + self.logger.warning('Failed to stop segments') + + self.workers_for_segment_stop = None + + self.trigger('move_to_STATE_SHRINK_SEGMENTS_STOP_DONE') + + @wrap_state_func_with_faults + def on_enter_STATE_SHRINK_SEGMENTS_STOP_DONE(self) -> None: + self.logger.info('Shrinked segments were stopped') + self.trigger('move_to_STATE_SHRINK_DONE') + + @wrap_state_func_with_faults + def on_enter_STATE_SHRINK_DONE(self) -> None: + os.remove(self.gparray_dump_file) + self.logger.info('Shrink is complete') + self.trigger('move_to_STATE_END') + + @wrap_state_func_with_faults + def on_enter_STATE_SHRINK_ROLLBACK_RESTORE_TARGET_SEGMENT_COUNT_START(self) -> None: + dbconn.execSQL(self.conn, 'BEGIN') + dbconn.execSQL(self.conn, 'SELECT gp_expand_lock_catalog()') + dbconn.execSQL(self.conn, 'SELECT gp_toolkit.gp_reset_rebalance_numsegments()') + # Store state here in case we fail before we enter 'on_every_state()' + # because after COMMIT we are on a one-way road of rollback. + self.rebalance_schema.storeShrinkState(self.state) + dbconn.execSQL(self.conn, 'COMMIT') + + self.trigger('move_to_STATE_SHRINK_ROLLBACK_RESTORE_TARGET_SEGMENT_COUNT_DONE') + + @wrap_state_func_with_faults + def on_enter_STATE_SHRINK_ROLLBACK_RESTORE_TARGET_SEGMENT_COUNT_DONE(self) -> None: + self.trigger('move_to_STATE_SHRINK_ROLLBACK_PREPARE_SCHEMA_START') + + @wrap_state_func_with_faults + def on_enter_STATE_SHRINK_ROLLBACK_PREPARE_SCHEMA_START(self) -> None: + self.prepare_shrink_schema(True) + self.trigger('move_to_STATE_SHRINK_ROLLBACK_PREPARE_SCHEMA_DONE') + + @wrap_state_func_with_faults + def on_enter_STATE_SHRINK_ROLLBACK_PREPARE_SCHEMA_DONE(self) -> None: + self.trigger('move_to_STATE_SHRINK_ROLLBACK_SHRINKED_TABLES_START') + + @wrap_state_func_with_faults + def on_enter_STATE_SHRINK_ROLLBACK_SHRINKED_TABLES_START(self) -> None: + self.logger.info('Start tables rebalance for rollback') + # perform 'ALTER TABLE REBALANCE' for all not yet processed tables + self.rebalance_tables('done', 'none', self.gparray.get_segment_count()) + self.trigger('move_to_STATE_SHRINK_ROLLBACK_SHRINKED_TABLES_DONE') + + @wrap_state_func_with_faults + def on_enter_STATE_SHRINK_ROLLBACK_SHRINKED_TABLES_DONE(self) -> None: + self.trigger('move_to_STATE_SHRINK_ROLLBACK_DROP_SCHEMA_START') + + @wrap_state_func_with_faults + def on_enter_STATE_SHRINK_ROLLBACK_DROP_SCHEMA_START(self) -> None: + if os.path.exists(self.gparray_dump_file): + os.remove(self.gparray_dump_file) + self.rebalance_schema.dropSchema() + self.trigger('move_to_STATE_SHRINK_ROLLBACK_DROP_SCHEMA_DONE') + + @wrap_state_func_with_faults + def on_enter_STATE_SHRINK_ROLLBACK_DROP_SCHEMA_DONE(self) -> None: + self.logger.info('Rollback is complete.') + self.trigger('move_to_STATE_END_FROM_ROLLBACK') + + @wrap_state_func_with_faults + def on_enter_STATE_END_FROM_ROLLBACK(self) -> None: + self.trigger('move_to_STATE_END') + + @wrap_state_func_with_faults + def on_enter_STATE_END(self) -> None: + pass + + @wrap_state_func_with_faults + def on_enter_STATE_ERROR(self) -> None: + raise Exception('Shrink entered STATE_ERROR') + + # state callbacks end here + + class SegmentStopAfterShrink(SegmentStop): + def __init__(self, shrink: 'GGShrink', segment: Segment) -> None: + self.shrink = shrink + self.segment = segment + if self.segment.isSegmentPrimary(): + name = f'stop primary (content {self.segment.getSegmentContentId()}, dbid {self.segment.getSegmentDbId()})' + else: + name = f'stop mirror (content {self.segment.getSegmentContentId()}, dbid {self.segment.getSegmentDbId()})' + self.checkRunningSegment = SegmentIsShutDown(name, self.segment.getSegmentDataDirectory(), base.REMOTE, self.segment.getSegmentHostName()) + SegmentStop.__init__(self, + name, + self.segment.getSegmentDataDirectory(), + self.shrink.stop_mode, + False, + base.REMOTE, + self.segment.getSegmentHostName(), + self.shrink.timeout) + + # decorator to inject a fault before running SegmentStopAfterShrink for a specific dbid + def wrap_segment_stop_with_faults(fun): + def func_with_faults(self): + try: + inject_fault(f'fault_segment_stop_dbid_{self.segment.getSegmentDbId()}') + except Exception as e: + os.kill(os.getpid(), signal.SIGINT) + return + fun(self) + return func_with_faults + + @wrap_segment_stop_with_faults + def run(self) -> None: + self.shrink.logger.info(f'Stopping shrinked segment {str(self.segment)}') + self.checkRunningSegment.run() + if self.checkRunningSegment.is_shutdown(): + self.shrink.logger.info(f'Segment {str(self.segment)} is already down') + self.set_results(CommandResult(0, b'', b'', True, False)) + else: + try: + SegmentStop.run(self, validateAfter = True) + except ExecutionError: + self.shrink.logger.info(f'Failed to stop shrinked segment {str(self.segment)}') + return + self.shrink.logger.info(f'Stopped shrinked segment {str(self.segment)}') + + class TableRebalanceTask(SQLCommand): + def __init__(self, + shrink: 'GGShrink', + db_name: str, + schema_name: str, + rel_name: str, + target_segment_count: int, + table_status_after_rebalance: str) -> None: + self.shrink = shrink + self.db_name = db_name + self.schema_name = schema_name + self.rel_name = rel_name + self.target_segment_count = target_segment_count + self.table_status_after_rebalance = table_status_after_rebalance + SQLCommand.__init__(self, f'task rebalance for {self.db_name}.{self.schema_name}.{self.rel_name}') + + # decorator to inject a fault before running TableRebalanceTask for a specific {db_name, schema_name, rel_name} + def wrap_table_rebalance_with_faults(fun): + def func_with_faults(self, attempt: int): + inject_fault(f'fault_rebalance_table_{self.db_name}.{self.schema_name}.{self.rel_name}') + fun(self, attempt) + return func_with_faults + + def table_exists(self, conn: dbconn.Connection, schema_name: str, rel_name: str) -> bool: + if dbconn.querySingleton(conn, f""" + SELECT count(1) + FROM pg_class c JOIN pg_namespace n ON c.relnamespace = n.oid + WHERE c.relname = '{rel_name}' AND n.nspname = '{schema_name}' AND c.relnamespace = n.oid + """) == 0: + return False + return True + + def db_exists(self, conn: dbconn.Connection, db_name: str) -> bool: + if dbconn.querySingleton(conn, f"""SELECT count(*) FROM pg_database WHERE datname = '{db_name}'""") == 0: + return False + return True + + @wrap_table_rebalance_with_faults + def process_table(self, attempt: int) -> None: + self.shrink.logger.info(f'Start table rebalance for "{self.db_name}"."{self.schema_name}"."{self.rel_name}" to {self.target_segment_count} segments (attempt {attempt})') + if self.db_exists(self.shrink.rebalance_schema.conn, self.db_name): + dburl = dbconn.DbURL(dbname=self.db_name, port=self.shrink.gpEnv.getCoordinatorPort()) + with closing(dbconn.connect(dburl, encoding='UTF8')) as conn: + dbconn.execSQL(conn, 'BEGIN') + + table_exists = self.table_exists(conn, self.schema_name, self.rel_name) + if table_exists: + dbconn.execSQL(conn, + f'''ALTER TABLE "{self.schema_name}"."{self.rel_name}" + REBALANCE {self.target_segment_count}''') + if self.shrink.options.analyze: + dbconn.execSQL(conn, + f'''ANALYZE "{self.schema_name}"."{self.rel_name}"''') + else: + self.shrink.logger.info(f'''Table "{self.db_name}"."{self.schema_name}"."{self.rel_name}" doesn't exist, skipping actual rebalance''') + + self.shrink.rebalance_schema.setStatusForTableToRebalance(self.db_name, self.schema_name, self.rel_name, self.table_status_after_rebalance) + dbconn.execSQL(conn, 'COMMIT') + else: + self.shrink.logger.info(f'''DB "{self.db_name}" doesn't exist, skipping actual rebalance for "{self.schema_name}"."{self.rel_name}"''') + self.shrink.logger.info(f'Complete table rebalance for "{self.db_name}"."{self.schema_name}"."{self.rel_name}"') + self.set_results(CommandResult(0, b'', b'', True, False)) + + def run(self) -> None: + # Give 2 attempts to process a table. It is needed, when, for example, + # other session opens a transaction after we have created the rebalance table + # list, drops the table before we started to rebalance it, and commits the + # transaction when we've started to rebalance the table. + attempt_max_cnt = 2 + for i in range(attempt_max_cnt): + attempt = i + 1 + try: + self.process_table(attempt) + except Exception as e: + if attempt < attempt_max_cnt: + logger.warning(f"{str(e)}") + else: + logger.error(f"{str(e)}") + raise Exception(f'Failed to process the db object for {attempt_max_cnt} attempts') + continue + break + + def prepare_shrink_schema(self, is_rollback: bool) -> None: + status = 'done' if is_rollback else 'none' + cmp = '<=' if is_rollback else '>' + + dbconn.execSQL(self.conn, 'BEGIN') + + # cleanup list of tables that require rebalance + # for the case we re-enter this state after we were interrupted right after it + self.rebalance_schema.clearTablesToRebalanceWithStatus(status) + + cursor = dbconn.query(self.conn, 'SELECT datname FROM pg_database') + databases_to_process = [] + for record in cursor: + database_name = record[0] + if database_name != 'template0': + databases_to_process.append(database_name) + + for db in databases_to_process: + dburl = dbconn.DbURL(dbname=db, port=self.gpEnv.getCoordinatorPort()) + with closing(dbconn.connect(dburl, encoding='UTF8')) as conn: + cursor = dbconn.query(conn, + f'''SELECT n.nspname, c.relname, c.relkind, pe.writable is not null as external_writable + FROM pg_class c + JOIN pg_namespace n ON c.relnamespace = n.oid + JOIN gp_distribution_policy p ON c.oid = p.localoid + LEFT JOIN pg_exttable pe on (c.oid=pe.reloid and pe.writable) + WHERE c.relkind IN ('r', 'p', 'm', 'f') AND c.relispartition = FALSE AND + c.relpersistence != 't' AND + p.numsegments {cmp} {self.shrink_plan.getTargetSegmentCount()} AND + n.nspname NOT IN ('pg_catalog', 'information_schema', '{self.rebalance_schema.getSchemaName()}')''') + for schema_name, rel_name, rel_kind, external_writable in cursor: + if rel_kind == 'f' and not external_writable: + continue + self.rebalance_schema.addTableToRebalance(db, schema_name, rel_name, status) + + dbconn.execSQL(self.conn, 'COMMIT') + + def rebalance_tables(self, original_status: str, target_status: str, target_segment_count: int) -> None: + cursor = self.rebalance_schema.getTablesToRebalanceWithStatus(original_status) + + self.logger.info(f'Tables to process {cursor.rowcount}') + + if cursor.rowcount > 0: + self.workers_for_tables_rebalance = WorkerPool(numWorkers=min(cursor.rowcount, self.options.parallel)) + + for db_name, schema_name, rel_name in cursor: + task = self.TableRebalanceTask(self, + db_name, + schema_name, + rel_name, + target_segment_count, + target_status) + self.workers_for_tables_rebalance.addCommand(task) + + print_progress(self.workers_for_tables_rebalance, interval=1) + + self.workers_for_tables_rebalance.haltWork() + self.workers_for_tables_rebalance.joinWorkers() + + for task in self.workers_for_tables_rebalance.getCompletedItems(): + if not task.was_successful(): + raise Exception(f'Failed to do ALTER REBALANCE: {task.get_results().stderr}') + + self.workers_for_tables_rebalance = None + + def state_can_rollback(self, state: str) -> bool: + if (state in self.states_main_shrink_flow): + if self.states_main_shrink_flow.index(state) <= self.states_main_shrink_flow.index('STATE_SHRINK_TABLES_DONE'): + return True + return False + + def is_gp_segment_configuration_shrinked(self) -> bool: + if self.dumped_gparray is None: + return False + return self.dumped_gparray.get_segment_count() != self.gparray.get_segment_count() + + def shutdown(self) -> None: + if self.workers_for_tables_rebalance != None: + self.workers_for_tables_rebalance.haltWork() + self.workers_for_tables_rebalance.joinWorkers() + + if self.workers_for_segment_stop != None: + self.workers_for_segment_stop.haltWork() + self.workers_for_segment_stop.joinWorkers() + + self.shutdown_requested = True diff --git a/gpMgmt/bin/gprebalance_modules/solver.py b/gpMgmt/bin/gprebalance_modules/solver.py new file mode 100644 index 000000000000..f74c93c83b9c --- /dev/null +++ b/gpMgmt/bin/gprebalance_modules/solver.py @@ -0,0 +1,1526 @@ +from dataclasses import dataclass +from enum import Enum +import math +import random +from typing import NewType, Optional, Set, Dict, List, Tuple, Union +from collections import defaultdict +import time + +# 0 <= hostid <= n_hosts_initial +HostId = NewType('HostId', int) +# 0 <= contentid <= n_segments +ContentId = NewType('ContentId', int) +# number of moves +Cost = NewType('Cost', int) +# load +Load = NewType('Load', int) +# {'contendid' - > (primary host, mirror host)} +Solution = Dict[ContentId, Tuple[HostId, HostId]] + +@dataclass +class SolverConfig: + """ + Configuration for the balancing problem + Attributes: + n_segments: planned numsegments after shrink/expand + n_hosts_target: target number of hosts + n_hosts_initial: initial number of hosts + initial_primary_mapping: initial_primary_mapping[contentid] = hostid + initial_mirror_mapping: initial_mirror_mapping[contentid] = hostid, 0 <= hostid <= n_hosts_initial + strategy: mirroring strategy + """ + n_segments: int + n_hosts_target: int + n_hosts_initial: int + initial_primary_mapping: List[HostId] + initial_mirror_mapping: List[HostId] + strategy: str + +class GreedySolver: + """ + Greedy construction of a balanced primary/mirror assignment + with an optional ALNS improvement phase. + + Segments: + 0 .. n_segments-1 + + Hosts: + initial hosts: 0 .. n_hosts_initial-1 + target hosts: 0 .. n_hosts_target-1 (subset of initial hosts) + + Constraints: + - Each segment has exactly one primary host and one mirror host. + - Primary host != Mirror host for every segment. + - Each target host has: + target_primary_load primaries + target_primary_load mirrors + => total 2 * target_primary_load segments (primary+mirror) per host. + - `strategy`: + 'grouped': all segments with same primary host share a single mirror host + 'spread' : all segments with same primary host have distinct mirror hosts + """ + def __init__(self, + config: SolverConfig, + run_improve: bool = True, + printing: bool = False, + seed: int = None): + + self.n_segments = config.n_segments + self.n_hosts_target = config.n_hosts_target + self.n_hosts_initial = config.n_hosts_initial + self.initial_primary_mapping = config.initial_primary_mapping + self.initial_mirror_mapping = config.initial_mirror_mapping + self.strategy = config.strategy + + self.run_improve = run_improve + self.printing = printing + self.target_primary_load = self.n_segments // self.n_hosts_target + self.target_load = 2 * self.n_segments // self.n_hosts_target + + if self.n_hosts_target < 2: + raise ValueError("Cannot balance to single host") + + if self.n_segments % self.n_hosts_target != 0: + raise ValueError(f"Cannot evenly distribute {self.n_segments}" + f"segments across {self.n_hosts_target} hosts") + + if self.strategy == 'spread': + if self.target_primary_load > self.n_hosts_target - 1: + raise ValueError("Cannot follow spread mirroring strategy") + + # Best known solution + self.best_primary_mapping: List[HostId] | None = None + self.best_mirror_mapping: List[HostId] | None = None + self.best_cost = None + + # seed to use in ALNS + self.seed = seed + + def solve(self) -> Tuple[Solution, Cost]: + """ + Build an initial greedy solution and optionally improve it with ALNS. + + Returns: + (solution, cost) where solution is a mapping from ContentId to + (primary_host, mirror_host), and cost is number of moves from + the initial placement. + """ + # Phase 1: Balance primaries + primary_mapping = self._balance_primaries() + + # Phase 2: Assign mirrors + mirror_mapping = self._assign_mirror_hosts(primary_mapping) + + # Optional improvement by ALNS. + if self.run_improve: + # at least 10 iterations per segment + config = ALNSConfig(max_iterations=10 * self.n_segments) + alns = ALNS(self, config) + if self.seed: + random.seed(self.seed) + primary_mapping, mirror_mapping = alns.optimize(primary_mapping, mirror_mapping) + + solution: Solution = { + ContentId(i): (HostId(primary_mapping[i]), HostId(mirror_mapping[i])) + for i in range(self.n_segments) + } + + cost = self._calculate_cost(primary_mapping, mirror_mapping) + + assert(self._validate_solution(solution)) + + self.best_primary_mapping = primary_mapping + self.best_mirror_mapping = mirror_mapping + + return solution, cost + + # --------------------------------------------------------------------- # + # Phase 1: Primaries + # --------------------------------------------------------------------- # + def _balance_primaries(self) -> List[HostId]: + """ + Assign primaries to target hosts, keeping: + - all host loads equal to target_primary_load, + - as many primaries as possible on their original host, + - moving out primaries from: + * hosts not in target count, + * overloaded hosts. + """ + primary_mapping = [-1] * self.n_segments + + # Count initial load only on target hosts. + initial_load = [0] * self.n_hosts_initial + for p in self.initial_primary_mapping: + if not self.is_decomissioned(p): + initial_load[p] += 1 + + # Segment processing order: + # must_move first (out-of-target or overloaded), + # then by descending load on their original host (move from heavier first). + segment_order = [] + for i in range(self.n_segments): + orig_host = self.initial_primary_mapping[i] + must_move = ( + self.is_decomissioned(orig_host) + or initial_load[orig_host] > self.target_primary_load + ) + segment_order.append((must_move, initial_load[orig_host], i)) + + segment_order.sort(reverse=True, key=lambda x: (x[0], x[1])) + + # Deficit + # Calculate how many segments each host needs to receive + deficit = [] + for h in range(self.n_hosts_target): + if h < self.n_hosts_initial: + deficit.append(self.target_primary_load - initial_load[h]) + else: + deficit.append(self.target_primary_load) + + # Current primary load for each target host. + current_load = [0] * self.n_hosts_target + + for _, _, seg_id in segment_order: + orig_host = self.initial_primary_mapping[seg_id] + + # Try to keep on original host if possible + if (not self.is_decomissioned(orig_host) and + current_load[orig_host] < self.target_primary_load): + host = orig_host + else: + # Find host with largest remaining deficit + host = max(range(self.n_hosts_target), key=lambda h: deficit[h]) + + primary_mapping[seg_id] = host + current_load[host] += 1 + deficit[host] -= 1 + + return primary_mapping + + def is_decomissioned(self, host: HostId): + return host >= self.n_hosts_target + + # --------------------------------------------------------------------- # + # Phase 2: Mirrors + # --------------------------------------------------------------------- # + def _assign_mirror_hosts(self, primary_mapping: List[HostId]) -> List[HostId]: + """ + Assign mirror hosts for all segments, respecting: + - load balance + - primary host != mirror host + - chosen mirroring strategy (grouped/spread) + - minimizing deviation from initial_mirror_mapping where possible. + """ + mirror_mapping = [-1] * self.n_segments + mirror_load = [0] * self.n_hosts_target + + # Group segments by primary host + groups: Dict[HostId, List[ContentId]] = defaultdict(list) + for seg in range(self.n_segments): + groups[primary_mapping[seg]].append(seg) + + if self.strategy == 'grouped': + self._assign_mirrors_grouped(primary_mapping, mirror_mapping, mirror_load, groups) + elif self.strategy == 'spread': + self._assign_mirrors_spread(primary_mapping, mirror_mapping, mirror_load) + + return mirror_mapping + + def _assign_mirrors_grouped(self, + primary_mapping: List[HostId], + mirror_mapping: List[HostId], + mirror_load: List[Load], + groups: Dict[HostId, List[int]]): + """ + Grouped strategy: + + All segments that share a primary host p_host must share the same + mirror host m_host (per-group mirroring). + """ + # Track mirror choice for each primary host. + phost_to_mhost: Dict[HostId, HostId] = {} + + preferences = self._compute_mirror_preferences(groups) + + sorted_p_hosts = sorted(groups.keys(), + key=lambda p: -max(preferences[p].values(), default=0)) + + for p_host in sorted_p_hosts: + segments = groups[p_host] + best_mirror_host = self._select_group_mirror( + p_host=p_host, + mirror_mapping=mirror_mapping, + mirror_load=mirror_load, + phost_to_mhost=phost_to_mhost, + groups=groups, + preferences=preferences + ) + # Fallback to straightforward assignment (shouldn't happen) + if best_mirror_host is None: + self._fill_naive_grouped(primary_mapping, mirror_mapping) + return + # Assign the group to chosen mirror. + for seg in segments: + mirror_mapping[seg] = best_mirror_host + mirror_load[best_mirror_host] += 1 + + phost_to_mhost[p_host] = best_mirror_host + + def _fill_naive_grouped(self, + primary_mapping: List[HostId], + mirror_mapping: List[HostId]): + """ + GROUPED: Mirrors from same primary host lie at the same host + """ + for content, primary_host in enumerate(primary_mapping): + mirror_mapping[content] = (primary_host + 1) % self.n_hosts_target + + def _fill_naive_spread(self, + primary_mapping: List[HostId], + mirror_mapping: List[HostId]): + """ + SPREAD: Mirrors from same primary host spread across different hosts + """ + segments_per_primary = defaultdict(int) + + for content_id, primary_host in enumerate(primary_mapping): + # Calculate local index within primary host's segments + local_idx = segments_per_primary[primary_host] + segments_per_primary[primary_host] += 1 + + # Formula: mirror = (primary + local_idx + 1) % n_hosts + mirror_host = (primary_host + 1 + local_idx) % self.n_hosts_target + + # Ensure no colocation + if mirror_host == primary_host: + mirror_host = (mirror_host + 1) % self.n_hosts_target + + mirror_mapping[content_id] = mirror_host + + def _compute_mirror_preferences(self, + groups: Dict[HostId, List[ContentId]]) -> Dict[HostId, Dict[HostId, int]]: + """ + Precompute uses (preferences) for each primary host -> possible mirror host. + Returns: {p_host: {m_host: use_count}} + """ + preferences: Dict[HostId, Dict[HostId, int]] = defaultdict(lambda: defaultdict(int)) + for p_host, segments in groups.items(): + for seg in segments: + orig_mirror_host = self.initial_mirror_mapping[seg] + if not self.is_decomissioned(orig_mirror_host) and orig_mirror_host != p_host: + preferences[p_host][orig_mirror_host] += 1 + return preferences + + def _select_group_mirror(self, + p_host: HostId, + mirror_mapping: List[HostId], + mirror_load: List[Load], + phost_to_mhost: Dict[HostId, HostId], + groups: Dict[HostId, List[ContentId]], + preferences: Dict[HostId, Dict[HostId, int]]) -> HostId: + """ + Pick a mirror host for a primary group (grouped strategy), + with the following priority: + 1. Most-used original mirror host. + 2. Least loaded available host with least contention by other primaries. + 3. Swap another already-assigned group with current in + case of conflict when the only mirror host is primary one. + """ + + mirror_uses = preferences[p_host] + segs = groups[p_host] + group_size = len(segs) + assigned_p_hosts = set(phost_to_mhost.keys()) + + best_mirror_host: HostId | None = None + + # Priority 1: Most used original mirror host (if has capacity) + if mirror_uses: + candidates_with_capacity = [] + for h, uses in mirror_uses.items(): + if mirror_load[h] + group_size <= self.target_primary_load: + # Contention: sum of uses from unassigned groups for this host + contention = sum(preferences[other_p].get(h, 0) + for other_p in preferences + if other_p != p_host and other_p not in assigned_p_hosts) + candidates_with_capacity.append((uses, contention, mirror_load[h], h)) + if candidates_with_capacity: + candidates_with_capacity.sort(key=lambda x: (-x[0], x[1], x[2], x[3])) + best_mirror_host = candidates_with_capacity[0][3] + + # Priority 2: Least loaded host + if best_mirror_host is None: + available_hosts = [] + for h in range(self.n_hosts_target): + if h != p_host and mirror_load[h] + group_size <= self.target_primary_load: + contention = sum(preferences[other_p].get(h, 0) + for other_p in preferences + if other_p != p_host and other_p not in assigned_p_hosts) + available_hosts.append((contention, mirror_load[h], h)) + if available_hosts: + best_mirror_host = min(available_hosts)[2] + + # Priority 3: DEADLOCK - Try swapping with already assigned group + if best_mirror_host is None: + best_mirror_host = self._swap_to_resolve_deadlock( + blocked_p_host=p_host, + mirror_load=mirror_load, + phost_to_mhost=phost_to_mhost, + groups=groups, + mirror_mapping=mirror_mapping) + + return best_mirror_host + + def _swap_to_resolve_deadlock(self, + blocked_p_host: HostId, + mirror_load: List[Load], + phost_to_mhost: Dict[HostId, HostId], + groups: Dict[HostId, List[ContentId]], + mirror_mapping: List[HostId]): + """ + Resolve a deadlock in grouped mirroring by moving another + mirror group to a different mirror. + + Idea: + - Look at already assigned primary hosts (other_p_host). + - For each, see if we can move that group to some alternative mirror. + - If that frees enough capacity on its current mirror for blocked_p_host, + we do the move and return the freed mirror as the new mirror for + blocked_p_host. + """ + + for other_p_host, current_mirror_host in phost_to_mhost.items(): + if other_p_host == blocked_p_host: + continue + + other_size = len(groups[other_p_host]) + + # Find alternative mirror where to move other_p_host's mirror group + alternative_mirror_host = next( + (h for h in range(self.n_hosts_target) + if h != other_p_host and # Can't be other's primary + mirror_load[h] + other_size <= self.target_primary_load), + None + ) + + if alternative_mirror_host is None: + continue + + space_after_move = mirror_load[current_mirror_host] - other_size + if space_after_move + len(groups[blocked_p_host]) > self.target_primary_load: + continue + + # Swap: move other_p_host to alternative_mirror + for seg in groups[other_p_host]: + mirror_mapping[seg] = alternative_mirror_host + + mirror_load[current_mirror_host] -= other_size + mirror_load[alternative_mirror_host] += other_size + phost_to_mhost[other_p_host] = alternative_mirror_host + + return current_mirror_host + + return None + + def _assign_mirrors_spread(self, + primary_mapping: List[HostId], + mirror_mapping: List[HostId], + mirror_load: List[Load]): + """ + Spread strategy: + + For each primary host p_host, the mirrors of all segments in that group + must be distinct and not equal to p_host. + """ + + used_in_group: Dict[HostId, Set[HostId]] = defaultdict(set) + + # Phase 1: Try to assign segments to their original mirrors + unassigned: List[ContentId] = [] + + for seg in range(self.n_segments): + p_host = primary_mapping[seg] + orig_mirror_host = self.initial_mirror_mapping[seg] + + # Check if original mirror is valid and available + can_use_original = ( + not self.is_decomissioned(orig_mirror_host) and + orig_mirror_host != p_host and + orig_mirror_host not in used_in_group[p_host] and + mirror_load[orig_mirror_host] < self.target_primary_load + ) + + if can_use_original: + mirror_mapping[seg] = orig_mirror_host + mirror_load[orig_mirror_host] += 1 + used_in_group[p_host].add(orig_mirror_host) + else: + unassigned.append(seg) + + # Phase 2: Assign remaining segments with load balancing + for seg in unassigned: + p_host = primary_mapping[seg] + + available = [ + h for h in range(self.n_hosts_target) + if (h != p_host and + h not in used_in_group[p_host] and + mirror_load[h] < self.target_primary_load) + ] + + if available: + best_host = min(available, key=lambda h: mirror_load[h]) + mirror_mapping[seg] = best_host + mirror_load[best_host] += 1 + used_in_group[p_host].add(best_host) + else: + # Priority 3: DEADLOCK - try swap + best_host = self._resolve_spread_deadlock_for_segment(seg, + primary_mapping, + mirror_mapping, + mirror_load, + used_in_group) + + if best_host is None: + self._fill_naive_spread(primary_mapping, mirror_mapping) + return + + mirror_mapping[seg] = best_host + mirror_load[best_host] += 1 + used_in_group[p_host].add(best_host) + + def _resolve_spread_deadlock_for_segment(self, + seg: ContentId, + primary_mapping: List[HostId], + mirror_mapping: List[HostId], + mirror_load: List[Load], + used_in_group: Dict[HostId, Set[HostId]], + ) -> Optional[HostId]: + """ + Attempt to resolve a deadlock for one segment in the spread strategy + by relocating another segment's mirror to a host with capacity. + """ + p_host = primary_mapping[seg] + + hosts_with_capacity = [ + h for h in range(self.n_hosts_target) + if mirror_load[h] < self.target_primary_load] + + if not hosts_with_capacity: + return None + + # Find hosts we could use (not in our group, at any load level) + candidate_hosts = [ + h for h in range(self.n_hosts_target) + if h != p_host and h not in used_in_group[p_host] + ] + # Try to find a segment using one of these hosts that can move mirror to p_host + for candidate_host in candidate_hosts: + # Find segments currently using candidate_host as mirror + for other_seg in range(self.n_segments): + if mirror_mapping[other_seg] != candidate_host: + continue + + other_p_host = primary_mapping[other_seg] + # Check if other_seg can use p_host as mirror + for dest_host in hosts_with_capacity: + # Check if other_seg can move to dest_host + if dest_host == other_p_host: + continue + if dest_host in used_in_group[other_p_host]: + continue + + # Perform the swap + # Remove other_seg from candidate_host + used_in_group[other_p_host].remove(candidate_host) + mirror_load[candidate_host] -= 1 + + # Move other_seg to dest_host + mirror_mapping[other_seg] = dest_host + used_in_group[other_p_host].add(dest_host) + mirror_load[dest_host] += 1 + + return candidate_host + + return None + + def _calculate_cost(self, primary: List[HostId], mirror: List[HostId]) -> int: + """ + Calculate movement cost + """ + return sum( + (1 if primary[i] != self.initial_primary_mapping[i] else 0) + + (1 if mirror[i] != self.initial_mirror_mapping[i] else 0) + for i in range(self.n_segments) + ) + + def _validate_solution(self, solution: Solution) -> bool: + """ + Validate that solution satisfies all constraints + """ + + # Check 1: All segments assigned + if len(solution) != self.n_segments: + return False + + # Check 2: Primary host != Mirror host + for i, (p, m) in solution.items(): + if p == m: + return False + + # Check 3: Load balance + load = [0] * (self.n_hosts_target) + for i, (p, m) in solution.items(): + load[p] += 1 + load[m] += 1 + + for h in range(self.n_hosts_target): + if load[h] != self.target_primary_load * 2 : + return False + + # Check 4: Strategy constraints + segments_by_host = defaultdict(list) + for i, (p, m) in solution.items(): + segments_by_host[p].append((i, m)) + + for _, segs in segments_by_host.items(): + if len(segs) < 2: + continue + + mirror_hsts = [r for (i, r) in segs] + + if self.strategy == 'grouped': + if len(set(mirror_hsts)) > 1: + return False + elif self.strategy == 'spread': + if len(set(mirror_hsts)) != len(mirror_hsts): + return False + + return True + + +@dataclass +class ALNSConfig: + """ + ALNS algorithm configuration. + """ + max_iterations: int = 1000 + timeout: float = 60.0 + temperature_decay: float = 0.95 + min_temperature: float = 0.01 + local_search_frequency: int = 5 + stagnation_threshold: int = 30 + restart_threshold: int = 20 + + +@dataclass +class SearchState: + """ + Current search state and statistics. + """ + current_primary_mapping: List[HostId] + current_mirror_mapping: List[HostId] + current_cost: Cost + best_primary_mapping: List[HostId] + best_mirror_mapping: List[HostId] + best_cost: Cost + iteration: int = 0 + stagnation_count: int = 0 + last_improvement_iter: int = 0 + + def get_temperature(self, config: ALNSConfig) -> float: + """ + Calculate current temperature. + """ + temp = 1.0 * (config.temperature_decay ** self.iteration) + return max(config.min_temperature, temp) + + def get_destroy_size(self, config: ALNSConfig) -> float: + """ + Calculate adaptive destroy size based on progress. + """ + if self.iteration - self.last_improvement_iter > config.restart_threshold: + return random.uniform(0.20, 0.40) # Stuck: larger neighborhoods + elif self.iteration < config.max_iterations * 0.3: + return random.uniform(0.15, 0.30) # Early phase: explore + else: + return random.uniform(0.10, 0.20) # Late phase: intensify + + def update_best(self, primary_mapping: List[HostId], mirror_mapping: List[HostId], cost: Cost) -> bool: + """ + Update best solution found. + """ + if cost < self.best_cost: + self.best_primary_mapping = primary_mapping[:] + self.best_mirror_mapping = mirror_mapping[:] + self.best_cost = cost + self.last_improvement_iter = self.iteration + return True + return False + + def should_restart(self, config: ALNSConfig) -> bool: + """ + Check if search should restart from best. + """ + return self.stagnation_count > config.stagnation_threshold + + def restart_from_best(self): + """ + Restart search from best solution. + """ + self.current_primary_mapping = self.best_primary_mapping[:] + self.current_mirror_mapping = self.best_mirror_mapping[:] + self.current_cost = self.best_cost + self.stagnation_count = 0 + + +class ALNSDestroyMethod(Enum): + GROUP_DESTROY = 'group_destroy' + BAD_SEGMENTS = 'bad_segments' + SHAW_REMOVAL = 'shaw_removal' + RANDOM_SEGMENTS = 'random_segments' + + +class ALNSDestroyOperators: + """ + Collection of destroy operators for ALNS. + """ + def __init__(self, n_segments: int, strategy: str, + initial_primary: List[HostId], + initial_mirror: List[HostId]): + self.n_segments = n_segments + self.strategy = strategy + self.initial_primary = initial_primary + self.initial_mirror = initial_mirror + + def select_method(self, stagnation: int) -> ALNSDestroyMethod: + """ + Select destroy method based on search state. + """ + if stagnation > 15: + # Stuck: use aggressive methods + return random.choice([ALNSDestroyMethod.GROUP_DESTROY, ALNSDestroyMethod.RANDOM_SEGMENTS]) + + if self.strategy == 'grouped': + # For grouped, favor group-based destroy + return random.choices( + [ALNSDestroyMethod.GROUP_DESTROY, + ALNSDestroyMethod.BAD_SEGMENTS, + ALNSDestroyMethod.SHAW_REMOVAL, + ALNSDestroyMethod.RANDOM_SEGMENTS], + weights=[0.4, 0.3, 0.2, 0.1] + )[0] + else: + # For spread, favor segment-based methods + return random.choices( + [ALNSDestroyMethod.BAD_SEGMENTS, + ALNSDestroyMethod.SHAW_REMOVAL, + ALNSDestroyMethod.RANDOM_SEGMENTS], + weights=[0.4, 0.4, 0.2] + )[0] + + def destroy_random(self, destroy_size: float) -> Set[ContentId]: + """ + Destroy random segments. + """ + n_destroy = max(1, int(self.n_segments * destroy_size)) + return set(random.sample(range(self.n_segments), n_destroy)) + + def destroy_primary_groups(self, primary: List[HostId], mirror: List[HostId], + destroy_size: float) -> Set[ContentId]: + """ + Destroy complete primary groups (for grouped strategy). + """ + groups = defaultdict(list) + for seg in range(self.n_segments): + groups[primary[seg]].append(seg) + + # Score groups by badness + primary_badness = self._calculate_group_badness(groups, primary, mirror) + + # Select groups probabilistically + return self._select_groups_by_badness(groups, primary_badness, destroy_size) + + def destroy_bad_segments(self, primary: List[HostId], mirror: List[HostId], + destroy_size: float) -> Set[ContentId]: + """ + Destroy segments that differ from initial placement. + """ + n_destroy = max(1, int(self.n_segments * destroy_size)) + + # Find segments that moved from original position + bad_segments = [] + for seg in range(self.n_segments): + badness = 0 + if primary[seg] != self.initial_primary[seg]: + badness += 1 + if mirror[seg] != self.initial_mirror[seg]: + badness += 1 + + if badness > 0: + bad_segments.append((badness, seg)) + + if not bad_segments: + return self.destroy_random(destroy_size) + + return self._select_bad_segments_with_relatedness(bad_segments, primary, mirror, n_destroy) + + def shaw_removal(self, primary: List[HostId], mirror: List[HostId], + destroy_size: float) -> Set[ContentId]: + """ + Remove related segments (same primary or mirror host). + """ + n_destroy = max(1, int(self.n_segments * destroy_size)) + + seed_seg = random.randint(0, self.n_segments - 1) + destroyed = {seed_seg} + + # Calculate relatedness scores + relatedness = [] + for seg in range(self.n_segments): + if seg == seed_seg: + continue + + score = 0 + if primary[seg] == primary[seed_seg]: + score += 2 + if mirror[seg] == mirror[seed_seg]: + score += 1 + + relatedness.append((score, seg)) + + # Sort by relatedness and take most related + relatedness.sort(reverse=True) + for _, seg in relatedness[:n_destroy-1]: + destroyed.add(seg) + + return destroyed + + def _calculate_group_badness(self, groups: Dict[HostId, List[ContentId]], + primary: List[HostId], mirror: List[HostId]) -> Dict[HostId, float]: + """ + Calculate badness score for each primary group. + (how many segments deviate from original) + """ + badness = {} + for p_host, segments in groups.items(): + moved_mirrors = sum(1 for seg in segments + if mirror[seg] != self.initial_mirror[seg]) + moved_primaries = sum(1 for seg in segments + if primary[seg] != self.initial_primary[seg]) + + total_moved = moved_mirrors + moved_primaries + badness[p_host] = total_moved / len(segments) if segments else 0 + + return badness + + def _select_groups_by_badness(self, groups: Dict[HostId, List[ContentId]], + badness: Dict[HostId, float], + destroy_size: float) -> Set[ContentId]: + """ + Select primary groups weighted by badness. + """ + n_destroy = max(1, int(self.n_segments * destroy_size)) + primaries = list(groups.keys()) + weights = [badness.get(p, 0.0) + 0.1 for p in primaries] # Add exploration constant 0.1 + + destroyed = set() + available_primaries = primaries[:] + available_weights = weights[:] + + attempts = 0 + while len(destroyed) < n_destroy and attempts < 20 and available_primaries: + p_host = random.choices(available_primaries, weights=available_weights)[0] + destroyed.update(groups[p_host]) + + # Remove from further selection + idx = available_primaries.index(p_host) + available_primaries.pop(idx) + available_weights.pop(idx) + attempts += 1 + + # Trim to size if necessary + if len(destroyed) > n_destroy: + destroyed = set(random.sample(list(destroyed), n_destroy)) + + return destroyed + + def _select_bad_segments_with_relatedness(self, bad_segments: List[Tuple[int, ContentId]], + primary: List[HostId], mirror: List[HostId], + n_destroy: int) -> Set[ContentId]: + """ + Select bad segments and add related ones. + """ + bad_segments.sort(reverse=True) + destroyed = set() + + # Start with worst segments + for _, seg in bad_segments[:min(n_destroy, len(bad_segments))]: + destroyed.add(seg) + + # Add related segments to reach quota + if len(destroyed) < n_destroy: + seed_segments = list(destroyed) + for seed_seg in seed_segments: + if len(destroyed) >= n_destroy: + break + + for seg in range(self.n_segments): + if seg in destroyed or len(destroyed) >= n_destroy: + continue + + if (primary[seg] == primary[seed_seg] or + mirror[seg] == mirror[seed_seg]): + destroyed.add(seg) + + return destroyed + + +class ALNSRepairMethod(Enum): + GREEDY_REPAIR = 'repair_greedy' + CONSTRAINT_REPAIR = 'repair_constrained' + REGRET_REPAIR = "repair_regret" + + +class ALNSRepairOperators: + """Repair operators for ALNS.""" + + def __init__(self, n_segments: int, + n_hosts: int, strategy: str, + initial_primary_mapping: List[HostId], + initial_mirror_mapping: List[HostId], + target_load: Load): + self.n_segments = n_segments + self.n_hosts = n_hosts + self.strategy = strategy + self.initial_primary_mapping = initial_primary_mapping + self.initial_mirror_mapping = initial_mirror_mapping + self.target_load = target_load + self._host_set = set(range(self.n_hosts)) + + def select_method(self, iteration: int, max_iterations: int) -> ALNSRepairMethod: + """ + Select destroy method based on search state. + """ + # Phase 1: Early exploration (first 20% of iterations) + if iteration < max_iterations * 0.2: + return random.choices( + [ALNSRepairMethod.GREEDY_REPAIR, + ALNSRepairMethod.CONSTRAINT_REPAIR], + weights=[0.9, 0.1] + )[0] + + # Phase 2: Quality optimization (middle 60%) + elif iteration < max_iterations * 0.8: + if self.strategy == 'spread': + # Spread is harder - more Most Constrained + return random.choices( + [ALNSRepairMethod.CONSTRAINT_REPAIR, + ALNSRepairMethod.GREEDY_REPAIR, + ALNSRepairMethod.REGRET_REPAIR], + weights=[0.3, 0.5, 0.2] + )[0] + else: + # Grouped is easier - more Greedy/Regret + return random.choices( + [ALNSRepairMethod.GREEDY_REPAIR, + ALNSRepairMethod.REGRET_REPAIR, + ALNSRepairMethod.CONSTRAINT_REPAIR], + weights=[0.4, 0.4, 0.2] + )[0] + + # Phase 3: Final intensification (last 20%) + else: + return random.choices( + [ALNSRepairMethod.REGRET_REPAIR, + ALNSRepairMethod.GREEDY_REPAIR, + ALNSRepairMethod.CONSTRAINT_REPAIR], + weights=[0.5, 0.3, 0.2] + )[0] + + def repair_greedy(self, primary_mapping: List[HostId], mirror_mapping: List[HostId], + destroyed: Set[ContentId]) -> Tuple[List[HostId], List[HostId]]: + """ + Greedy: Assign each segment to cheapest valid option immediately. + O(n * h²) complexity. Fast, reliable. + """ + new_primary_mapping = primary_mapping[:] + new_mirror_mapping = mirror_mapping[:] + + # Clear destroyed segments + for seg in destroyed: + new_primary_mapping[seg] = -1 + new_mirror_mapping[seg] = -1 + + # Initialize load tracker + primary_capacity = [self.target_load] * self.n_hosts + mirror_capacity = [self.target_load] * self.n_hosts + + for seg in range(self.n_segments): + if seg not in destroyed and new_primary_mapping[seg] != -1: + primary_capacity[new_primary_mapping[seg]] -= 1 + mirror_capacity[new_mirror_mapping[seg]] -= 1 + + # Build constraint cache + if self.strategy == 'grouped': + primary_to_mirror = {} + for seg in range(self.n_segments): + if seg not in destroyed and new_primary_mapping[seg] != -1: + primary_to_mirror[new_primary_mapping[seg]] = new_mirror_mapping[seg] + elif self.strategy == 'spread': + primary_to_used_mirrors = defaultdict(set) + for seg in range(self.n_segments): + if seg not in destroyed and new_primary_mapping[seg] != -1: + primary_to_used_mirrors[new_primary_mapping[seg]].add(new_mirror_mapping[seg]) + + # Shuffle to avoid bias + destroyed_list = list(destroyed) + random.shuffle(destroyed_list) + + for seg in destroyed_list: + orig_p = self.initial_primary_mapping[seg] + orig_m = self.initial_mirror_mapping[seg] + best_cost = float('inf') + best_p, best_m = -1, -1 + # Try each primary + for p_host in range(self.n_hosts): + if primary_capacity[p_host] <= 0: + continue + + # Get valid mirrors for this primary + valid_mirrors = self._get_valid_mirrors( + p_host, mirror_capacity, + primary_to_mirror if self.strategy == 'grouped' else primary_to_used_mirrors + ) + if not valid_mirrors: + continue + + # Calculate primary cost + p_cost = 0 if p_host == orig_p else 1 + # Try each valid mirror + for m_host in valid_mirrors: + m_cost = 0 if m_host == orig_m else 1 + total_cost = p_cost + m_cost + if total_cost < best_cost: + best_cost = total_cost + best_p, best_m = p_host, m_host + # Early exit on perfect match + if total_cost == 0: + break + + if best_cost == 0: + break + + # Assign if found + if best_p != -1: + new_primary_mapping[seg] = best_p + new_mirror_mapping[seg] = best_m + primary_capacity[best_p] -= 1 + mirror_capacity[best_m] -= 1 + # Update cache + if self.strategy == 'grouped' and best_p not in primary_to_mirror: + primary_to_mirror[best_p] = best_m + elif self.strategy == 'spread': + primary_to_used_mirrors[best_p].add(best_m) + + return new_primary_mapping, new_mirror_mapping + + def _get_valid_mirrors(self, + primary_host: HostId, + mirror_capacity: List[int], + constraints_cache: Union[Dict[HostId, HostId], Dict[HostId, Set[HostId]]]) -> Set[HostId]: + """ + Fast lookup of valid mirror hosts for a given primary host. + + Args: + primary_host: The primary host we want to assign mirror hostto + mirror_capacity: Remaining capacity on each mirror host + constraints_cache: Either: + - For grouped: Dict[primary -> mirror] mapping + - For spread: Dict[primary -> Set[used_mirrors]] + + Returns: + Set of valid mirror host ids that can be used + """ + + if self.strategy == 'grouped': + # GROUPED STRATEGY: Each primary must use exactly ONE mirror for all segments + + if primary_host in constraints_cache: + # This primary already has an assigned mirror + existing_mirror = constraints_cache[primary_host] + + # Can only use this existing mirror if it has capacity + if mirror_capacity[existing_mirror] > 0: + return {existing_mirror} + else: + return set() # No valid mirrors (existing one is full) + + else: + # New primary: can use any mirror with capacity (except itself) + valid = set() + for m_host in range(self.n_hosts): + if m_host != primary_host and mirror_capacity[m_host] > 0: + valid.add(m_host) + return valid + + elif self.strategy == 'spread': + # SPREAD STRATEGY: Each primary must use DIFFERENT mirrors for each segment + + used_mirrors = constraints_cache.get(primary_host, set()) + + valid = set() + for m_host in range(self.n_hosts): + if (m_host != primary_host and + m_host not in used_mirrors and + mirror_capacity[m_host] > 0): + valid.add(m_host) + + return valid + + def repair_most_constrained(self, primary_mapping: List[HostId], mirror_mapping: List[HostId], + destroyed: Set[ContentId]) -> Tuple[List[HostId], List[HostId]]: + """ + Most Constrained: Assign segments with fewest valid options first. + Useful for sread strategy. + O(n**2 * h**2) complexity. + """ + new_primary_mapping = primary_mapping[:] + new_mirror_mapping = mirror_mapping[:] + + for seg in destroyed: + new_primary_mapping[seg] = -1 + new_mirror_mapping[seg] = -1 + + primary_capacity = [self.target_load] * self.n_hosts + mirror_capacity = [self.target_load] * self.n_hosts + + for seg in range(self.n_segments): + if seg not in destroyed and new_primary_mapping[seg] != -1: + primary_capacity[new_primary_mapping[seg]] -= 1 + mirror_capacity[new_mirror_mapping[seg]] -= 1 + + # Build constraint cache + if self.strategy == 'grouped': + primary_to_mirror = {} + for seg in range(self.n_segments): + if seg not in destroyed and new_primary_mapping[seg] != -1: + primary_to_mirror[new_primary_mapping[seg]] = new_mirror_mapping[seg] + elif self.strategy == 'spread': + primary_to_used_mirrors = defaultdict(set) + for seg in range(self.n_segments): + if seg not in destroyed and new_primary_mapping[seg] != -1: + primary_to_used_mirrors[new_primary_mapping[seg]].add(new_mirror_mapping[seg]) + + unassigned = set(destroyed) + + while unassigned: + # Evaluate ALL unassigned segments + candidates = [] + + for seg in unassigned: + orig_p = self.initial_primary_mapping[seg] + orig_m = self.initial_mirror_mapping[seg] + + # Find ALL valid (primary, mirror) pairs for this segment + valid_options = [] + + for p_host in range(self.n_hosts): + if primary_capacity[p_host] <= 0: + continue + + # Get valid mirrors dynamically + valid_mirrors = self._get_valid_mirrors( + p_host, mirror_capacity, + primary_to_mirror if self.strategy == 'grouped' else primary_to_used_mirrors + ) + + for m_host in valid_mirrors: + # Calculate cost for this option + cost = (0 if p_host == orig_p else 1) + (0 if m_host == orig_m else 1) + valid_options.append((cost, p_host, m_host)) + + if not valid_options: + # No valid options - this segment is problematic + # Add with infinity to handle at end (might fail gracefully) + candidates.append((float('inf'), float('inf'), seg, None)) + else: + # Sort by cost and pick best + valid_options.sort() + best_cost, best_p, best_m = valid_options[0] + option_count = len(valid_options) + + # Store: (option_count, best_cost, segment, best_assignment) + candidates.append((option_count, best_cost, seg, (best_p, best_m))) + + if not candidates: + break + + # Sort: by option_count (ascending), by cost (ascending) + # This ensures we assign the hardest segment first + candidates.sort(key=lambda x: (x[0], x[1])) + + # Process most constrained segment + option_count, cost, seg, assignment = candidates[0] + + if assignment is None: + # No valid assignment possible - skip or handle + unassigned.remove(seg) + continue + + best_p, best_m = assignment + + # Assign + new_primary_mapping[seg] = best_p + new_mirror_mapping[seg] = best_m + primary_capacity[best_p] -= 1 + mirror_capacity[best_m] -= 1 + + # Update cache + if self.strategy == 'grouped' and best_p not in primary_to_mirror: + primary_to_mirror[best_p] = best_m + elif self.strategy == 'spread': + primary_to_used_mirrors[best_p].add(best_m) + + unassigned.remove(seg) + + return new_primary_mapping, new_mirror_mapping + + def repair_regret(self, primary_mapping: List[HostId], mirror_mapping: List[HostId], + destroyed: Set[ContentId]) -> Tuple[List[HostId], List[HostId]]: + """ + Regret: Assign segment with highest regret score (best - 2nd_best) first. + O(n**2 * h**2) complexity. + """ + new_primary_mapping = primary_mapping[:] + new_mirror_mapping = mirror_mapping[:] + + for seg in destroyed: + new_primary_mapping[seg] = -1 + new_mirror_mapping[seg] = -1 + + primary_capacity = [self.target_load] * self.n_hosts + mirror_capacity = [self.target_load] * self.n_hosts + + for seg in range(self.n_segments): + if seg not in destroyed and new_primary_mapping[seg] != -1: + primary_capacity[new_primary_mapping[seg]] -= 1 + mirror_capacity[new_mirror_mapping[seg]] -= 1 + + # Build constraint cache + if self.strategy == 'grouped': + primary_to_mirror = {} + for seg in range(self.n_segments): + if seg not in destroyed and new_primary_mapping[seg] != -1: + primary_to_mirror[new_primary_mapping[seg]] = new_mirror_mapping[seg] + elif self.strategy == 'spread': + primary_to_used_mirrors = defaultdict(set) + for seg in range(self.n_segments): + if seg not in destroyed and new_primary_mapping[seg] != -1: + primary_to_used_mirrors[new_primary_mapping[seg]].add(new_mirror_mapping[seg]) + + unassigned = list(destroyed) + + while unassigned: + best_regret = -1 + best_seg = None + best_assignment = None + + # Calculate regret for each unassigned segment + for seg in unassigned: + orig_p = self.initial_primary_mapping[seg] + orig_m = self.initial_mirror_mapping[seg] + + # Find all valid options + options = [] + + for p_host in range(self.n_hosts): + if primary_capacity[p_host] <= 0: + continue + + valid_mirrors = self._get_valid_mirrors( + p_host, mirror_capacity, + primary_to_mirror if self.strategy == 'grouped' else primary_to_used_mirrors + ) + + for m_host in valid_mirrors: + cost = (0 if p_host == orig_p else 1) + (0 if m_host == orig_m else 1) + options.append((cost, p_host, m_host)) + + if len(options) == 0: + continue # Skip infeasible segments + + # Sort by cost + options.sort() + + # Calculate regret + if len(options) == 1: + # Only one option = infinite regret + regret = float('inf') + else: + # Regret = difference between best and 2nd best + regret = options[1][0] - options[0][0] + + # Track segment with highest regret + if regret > best_regret: + best_regret = regret + best_seg = seg + best_assignment = options[0] # (cost, p, m) + + if best_seg is None: + break + + # Assign the segment with highest regret + _, best_p, best_m = best_assignment + + new_primary_mapping[best_seg] = best_p + new_mirror_mapping[best_seg] = best_m + primary_capacity[best_p] -= 1 + mirror_capacity[best_m] -= 1 + + # Update cache + if self.strategy == 'grouped' and best_p not in primary_to_mirror: + primary_to_mirror[best_p] = best_m + elif self.strategy == 'spread': + primary_to_used_mirrors[best_p].add(best_m) + + unassigned.remove(best_seg) + + return new_primary_mapping, new_mirror_mapping + + +class ALNS: + """ + Adaptive Large Neighborhood Search (ALNS) for refining segment + rebalancing solutions produced by GreedySolver. + + Uses: + - several destroy operators + - several repair operators + - simulated annealing for acceptance + - occasional local search (mirror swaps in grouped) + """ + def __init__(self, + solver: GreedySolver, + config: ALNSConfig = None): + self.solver = solver + self.config = config or ALNSConfig() + + self.solver = solver + self.n_segments = solver.n_segments + self.n_hosts = solver.n_hosts_target + self.strategy = solver.strategy + self.initial_primary_mapping = solver.initial_primary_mapping + self.initial_mirror_mapping = solver.initial_mirror_mapping + self.target_primary_load = solver.target_primary_load + self.printing = solver.printing + + self.start_time = None + + self.destroy_ops = ALNSDestroyOperators(self.n_segments, self.strategy, + self.initial_primary_mapping, self.initial_mirror_mapping) + + self.repair_ops = ALNSRepairOperators( + self.n_segments, self.n_hosts, self.strategy, + self.initial_primary_mapping, self.initial_mirror_mapping, + self.solver.target_primary_load + ) + + def optimize(self, + primary_mapping: List[HostId], + mirror_mapping: List[HostId]) -> Tuple[List[HostId], List[HostId]]: + """ + ALNS. + Uses all destroy/repair strategies with SA acceptance. + """ + self.start_time = time.time() + + initial_cost = self.solver._calculate_cost(primary_mapping, mirror_mapping) + + state = SearchState(current_primary_mapping=primary_mapping[:], + current_mirror_mapping=mirror_mapping[:], + current_cost=initial_cost, + best_primary_mapping=primary_mapping[:], + best_mirror_mapping=mirror_mapping[:], + best_cost=initial_cost) + + if self.solver.printing: + print(f"Initial cost: {state.best_cost}") + + for state.iteration in range(self.config.max_iterations): + if self._timeout_reached(): + if self.printing: + print(f"\n Time limit reached ({self.config.timeout}s) at {state.iteration} iteration") + return state.best_primary_mapping, state.best_mirror_mapping + + new_primary_mapping, new_mirror_mapping, destroy_method, repair_method = self._generate_neighbor(state) + + # Validate + if not self._is_valid(new_primary_mapping, new_mirror_mapping): + state.stagnation_count += 1 + continue + + new_cost = self.solver._calculate_cost(new_primary_mapping, new_mirror_mapping) + + # SA acceptance + if self._accept_move(state.current_cost, new_cost, state.get_temperature(self.config)): + # Accept move + state.current_primary_mapping = new_primary_mapping + state.current_mirror_mapping = new_mirror_mapping + state.current_cost = new_cost + state.stagnation_count = 0 + + if state.update_best(new_primary_mapping, new_mirror_mapping, new_cost): + if self.printing: + print(f"Iter {state.iteration} ({destroy_method.value}) " + f"({repair_method.value}): NEW BEST = {state.best_cost}") + else: + state.stagnation_count += 1 + + # Restart from best if stuck + if state.should_restart(self.config): + if self.printing: + print(f"Iter {state.iteration}: Restarting from best (cost={state.best_cost})") + state.restart_from_best() + + if self.printing: + print(f"Final cost: {state.best_cost}") + + return state.best_primary_mapping, state.best_mirror_mapping + + def _generate_neighbor(self, state: SearchState) -> Tuple[List[HostId], List[HostId], + ALNSDestroyMethod, ALNSRepairMethod]: + """ + Generate a neighbor solution using destroy/repair. + """ + destroy_size = state.get_destroy_size(self.config) + destroy_method = self.destroy_ops.select_method(state.stagnation_count) + + # Apply destroy + destroyed = self._apply_destroy(state.current_primary_mapping, state.current_mirror_mapping, + destroy_method, destroy_size) + + + repair_method = self.repair_ops.select_method(state.iteration, self.config.max_iterations) + + new_primary_mapping, new_mirror_mapping = self._apply_repair(state.current_primary_mapping, + state.current_mirror_mapping, + repair_method, destroyed) + + # Apply local search periodically + if (self.strategy == 'grouped' and + state.stagnation_count < 10 and + state.iteration % self.config.local_search_frequency == 0): + new_primary_mapping, new_mirror_mapping = self._local_mirror_swap(new_primary_mapping, new_mirror_mapping) + + return new_primary_mapping, new_mirror_mapping, destroy_method, repair_method + + def _apply_destroy(self, primary: List[HostId], mirror: List[HostId], + method: ALNSDestroyMethod, destroy_size: float) -> Set[ContentId]: + """ + Apply the selected destroy method. + """ + if method == ALNSDestroyMethod.GROUP_DESTROY and self.strategy == 'grouped': + return self.destroy_ops.destroy_primary_groups(primary, mirror, destroy_size) + elif method == ALNSDestroyMethod.BAD_SEGMENTS: + return self.destroy_ops.destroy_bad_segments(primary, mirror, destroy_size) + elif method == ALNSDestroyMethod.SHAW_REMOVAL: + return self.destroy_ops.shaw_removal(primary, mirror, destroy_size) + else: # RANDOM_SEGMENTS + return self.destroy_ops.destroy_random(destroy_size) + + # REPAIR + def _apply_repair(self, + primary: List[HostId], + mirror: List[HostId], + method: ALNSRepairMethod, + destroyed: Set[ContentId]) -> Tuple[List[HostId], List[HostId]]: + if method == ALNSRepairMethod.GREEDY_REPAIR: + return self.repair_ops.repair_greedy(primary, mirror, destroyed) + elif method == ALNSRepairMethod.CONSTRAINT_REPAIR: + return self.repair_ops.repair_most_constrained(primary, mirror, destroyed) + elif method == ALNSRepairMethod.REGRET_REPAIR: + return self.repair_ops.repair_regret(primary, mirror, destroyed) + + # LOCAL SEARCH + def _local_mirror_swap(self, primary_mapping: List[HostId], + mirror_mapping: List[HostId]) -> Tuple[List[HostId], List[HostId]]: + """ + Local search: swap mirror assignments between two primary groups. + Only for grouped strategy. + """ + if self.strategy != 'grouped': + return primary_mapping, mirror_mapping + + # Build primary -> mirror mapping + groups = defaultdict(list) + ph_to_mh = {} + + for seg in range(self.n_segments): + p = primary_mapping[seg] + m = mirror_mapping[seg] + groups[p].append(seg) + ph_to_mh[p] = m + + current_cost = self.solver._calculate_cost(primary_mapping, mirror_mapping) + best_mirror_mapping = mirror_mapping[:] + best_cost = current_cost + + # Try swapping mirrors between random pairs + primaries = list(groups.keys()) + + if len(primaries) < 2: + return primary_mapping, mirror_mapping + + attempts = min(20, len(primaries) * (len(primaries) - 1) // 2) + + for _ in range(attempts): + p1, p2 = random.sample(primaries, 2) + m1 = ph_to_mh[p1] + m2 = ph_to_mh[p2] + + # Check if swap is valid (doesn't violate primary != mirror) + if m1 == p2 or m2 == p1: + continue + + # Create candidate solution with swapped mirrors + candidate_mirror_mapping = mirror_mapping[:] + for seg in groups[p1]: + candidate_mirror_mapping[seg] = m2 + for seg in groups[p2]: + candidate_mirror_mapping[seg] = m1 + + candidate_cost = self.solver._calculate_cost(primary_mapping, candidate_mirror_mapping) + + if candidate_cost < best_cost: + best_mirror_mapping = candidate_mirror_mapping + best_cost = candidate_cost + + return primary_mapping, best_mirror_mapping + + # UTILITIES + def _accept_move(self, current_cost: Cost, new_cost: Cost, temperature: float) -> bool: + """ + Simulated Annealing acceptance criterion. + """ + if new_cost < current_cost: + return True + elif new_cost == current_cost: + return random.random() < 0.6 + else: + delta = (new_cost - current_cost) / self.n_segments + delta_capped = min(delta, 5.0) + return random.random() < math.exp(-delta_capped / temperature) + + def _is_valid(self, primary_mapping: List[HostId], mirror_mapping: List[HostId]) -> bool: + """ + Validation check. + """ + solution = {i: (primary_mapping[i], mirror_mapping[i]) for i in range(self.n_segments)} + return self.solver._validate_solution(solution) + + def _timeout_reached(self) -> bool: + """Check if time limit exceeded.""" + if self.start_time is None or self.config.timeout is None: + return False + return time.time() - self.start_time > self.config.timeout diff --git a/gpMgmt/bin/gprebalance_modules/test/__init__.py b/gpMgmt/bin/gprebalance_modules/test/__init__.py new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/gpMgmt/bin/gprebalance_modules/test/config.py b/gpMgmt/bin/gprebalance_modules/test/config.py new file mode 100644 index 000000000000..be6b0bac957e --- /dev/null +++ b/gpMgmt/bin/gprebalance_modules/test/config.py @@ -0,0 +1,41 @@ +import os + +from gppylib.gparray import Segment, GpArray +from gprebalance_modules.planner import ConfigurationEncoder, Planner, HostResolver + + +def initGparrayFromFile(basename): + filename = os.path.dirname(__file__) + \ + "/data/" + basename + ".array" + segdbs = [] + with open(filename, 'r') as fp: + for line in fp: + if not line.lstrip().startswith('#'): + segdbs.append(Segment.initFromString(line)) + return GpArray(segdbs, segdbs) + +def getEncoding(file, strat, target_hosts, add_hosts, remove_hosts): + def inner(func): + def wrapper(self, *args, **kwargs): + gparray = initGparrayFromFile(file) + target_hostnames = None + add_hostnames = None + remove_hostnames = None + if target_hosts: + target_hostnames = [h.strip() for h in target_hosts.split(',')] + if add_hosts: + add_hostnames = [h.strip() for h in add_hosts.split(',')] + if remove_hosts: + remove_hostnames = [h.strip() for h in remove_hosts.split(',')] + self.encoding = ConfigurationEncoder.encode_configuration(gparray, + Planner.get_target_hosts( + array=gparray, + target_hostname_list=target_hostnames, + add_hostname_list=add_hostnames, + remove_hostname_list=remove_hostnames)[0], + strat) + res = func(self, *args, **kwargs) + self.encoding = None + return res + return wrapper + return inner diff --git a/gpMgmt/bin/gprebalance_modules/test/data/1000_50_balanced_grouped.array b/gpMgmt/bin/gprebalance_modules/test/data/1000_50_balanced_grouped.array new file mode 100644 index 000000000000..0af454dd30d1 --- /dev/null +++ b/gpMgmt/bin/gprebalance_modules/test/data/1000_50_balanced_grouped.array @@ -0,0 +1,2052 @@ +# Coordinator +1|-1|p|p|s|u|cdw|cdw|5432|/data/coordinator +# SDW1 +2|0|p|p|s|u|sdw1|sdw1|7000|/data/primary0 +3|1|p|p|s|u|sdw1|sdw1|7001|/data/primary1 +4|2|p|p|s|u|sdw1|sdw1|7002|/data/primary2 +5|3|p|p|s|u|sdw1|sdw1|7003|/data/primary3 +6|4|p|p|s|u|sdw1|sdw1|7004|/data/primary4 +7|5|p|p|s|u|sdw1|sdw1|7005|/data/primary5 +8|6|p|p|s|u|sdw1|sdw1|7006|/data/primary6 +9|7|p|p|s|u|sdw1|sdw1|7007|/data/primary7 +10|8|p|p|s|u|sdw1|sdw1|7008|/data/primary8 +11|9|p|p|s|u|sdw1|sdw1|7009|/data/primary9 +12|10|p|p|s|u|sdw1|sdw1|7010|/data/primary10 +13|11|p|p|s|u|sdw1|sdw1|7011|/data/primary11 +14|12|p|p|s|u|sdw1|sdw1|7012|/data/primary12 +15|13|p|p|s|u|sdw1|sdw1|7013|/data/primary13 +16|14|p|p|s|u|sdw1|sdw1|7014|/data/primary14 +17|15|p|p|s|u|sdw1|sdw1|7015|/data/primary15 +18|16|p|p|s|u|sdw1|sdw1|7016|/data/primary16 +19|17|p|p|s|u|sdw1|sdw1|7017|/data/primary17 +20|18|p|p|s|u|sdw1|sdw1|7018|/data/primary18 +21|19|p|p|s|u|sdw1|sdw1|7019|/data/primary19 +1982|980|m|m|s|u|sdw1|sdw1|8980|/data/mirror980 +1983|981|m|m|s|u|sdw1|sdw1|8981|/data/mirror981 +1984|982|m|m|s|u|sdw1|sdw1|8982|/data/mirror982 +1985|983|m|m|s|u|sdw1|sdw1|8983|/data/mirror983 +1986|984|m|m|s|u|sdw1|sdw1|8984|/data/mirror984 +1987|985|m|m|s|u|sdw1|sdw1|8985|/data/mirror985 +1988|986|m|m|s|u|sdw1|sdw1|8986|/data/mirror986 +1989|987|m|m|s|u|sdw1|sdw1|8987|/data/mirror987 +1990|988|m|m|s|u|sdw1|sdw1|8988|/data/mirror988 +1991|989|m|m|s|u|sdw1|sdw1|8989|/data/mirror989 +1992|990|m|m|s|u|sdw1|sdw1|8990|/data/mirror990 +1993|991|m|m|s|u|sdw1|sdw1|8991|/data/mirror991 +1994|992|m|m|s|u|sdw1|sdw1|8992|/data/mirror992 +1995|993|m|m|s|u|sdw1|sdw1|8993|/data/mirror993 +1996|994|m|m|s|u|sdw1|sdw1|8994|/data/mirror994 +1997|995|m|m|s|u|sdw1|sdw1|8995|/data/mirror995 +1998|996|m|m|s|u|sdw1|sdw1|8996|/data/mirror996 +1999|997|m|m|s|u|sdw1|sdw1|8997|/data/mirror997 +2000|998|m|m|s|u|sdw1|sdw1|8998|/data/mirror998 +2001|999|m|m|s|u|sdw1|sdw1|8999|/data/mirror999 +# SDW10 +182|180|p|p|s|u|sdw10|sdw10|7180|/data/primary180 +183|181|p|p|s|u|sdw10|sdw10|7181|/data/primary181 +184|182|p|p|s|u|sdw10|sdw10|7182|/data/primary182 +185|183|p|p|s|u|sdw10|sdw10|7183|/data/primary183 +186|184|p|p|s|u|sdw10|sdw10|7184|/data/primary184 +187|185|p|p|s|u|sdw10|sdw10|7185|/data/primary185 +188|186|p|p|s|u|sdw10|sdw10|7186|/data/primary186 +189|187|p|p|s|u|sdw10|sdw10|7187|/data/primary187 +190|188|p|p|s|u|sdw10|sdw10|7188|/data/primary188 +191|189|p|p|s|u|sdw10|sdw10|7189|/data/primary189 +192|190|p|p|s|u|sdw10|sdw10|7190|/data/primary190 +193|191|p|p|s|u|sdw10|sdw10|7191|/data/primary191 +194|192|p|p|s|u|sdw10|sdw10|7192|/data/primary192 +195|193|p|p|s|u|sdw10|sdw10|7193|/data/primary193 +196|194|p|p|s|u|sdw10|sdw10|7194|/data/primary194 +197|195|p|p|s|u|sdw10|sdw10|7195|/data/primary195 +198|196|p|p|s|u|sdw10|sdw10|7196|/data/primary196 +199|197|p|p|s|u|sdw10|sdw10|7197|/data/primary197 +200|198|p|p|s|u|sdw10|sdw10|7198|/data/primary198 +201|199|p|p|s|u|sdw10|sdw10|7199|/data/primary199 +1162|160|m|m|s|u|sdw10|sdw10|8160|/data/mirror160 +1163|161|m|m|s|u|sdw10|sdw10|8161|/data/mirror161 +1164|162|m|m|s|u|sdw10|sdw10|8162|/data/mirror162 +1165|163|m|m|s|u|sdw10|sdw10|8163|/data/mirror163 +1166|164|m|m|s|u|sdw10|sdw10|8164|/data/mirror164 +1167|165|m|m|s|u|sdw10|sdw10|8165|/data/mirror165 +1168|166|m|m|s|u|sdw10|sdw10|8166|/data/mirror166 +1169|167|m|m|s|u|sdw10|sdw10|8167|/data/mirror167 +1170|168|m|m|s|u|sdw10|sdw10|8168|/data/mirror168 +1171|169|m|m|s|u|sdw10|sdw10|8169|/data/mirror169 +1172|170|m|m|s|u|sdw10|sdw10|8170|/data/mirror170 +1173|171|m|m|s|u|sdw10|sdw10|8171|/data/mirror171 +1174|172|m|m|s|u|sdw10|sdw10|8172|/data/mirror172 +1175|173|m|m|s|u|sdw10|sdw10|8173|/data/mirror173 +1176|174|m|m|s|u|sdw10|sdw10|8174|/data/mirror174 +1177|175|m|m|s|u|sdw10|sdw10|8175|/data/mirror175 +1178|176|m|m|s|u|sdw10|sdw10|8176|/data/mirror176 +1179|177|m|m|s|u|sdw10|sdw10|8177|/data/mirror177 +1180|178|m|m|s|u|sdw10|sdw10|8178|/data/mirror178 +1181|179|m|m|s|u|sdw10|sdw10|8179|/data/mirror179 +# SDW11 +202|200|p|p|s|u|sdw11|sdw11|7200|/data/primary200 +203|201|p|p|s|u|sdw11|sdw11|7201|/data/primary201 +204|202|p|p|s|u|sdw11|sdw11|7202|/data/primary202 +205|203|p|p|s|u|sdw11|sdw11|7203|/data/primary203 +206|204|p|p|s|u|sdw11|sdw11|7204|/data/primary204 +207|205|p|p|s|u|sdw11|sdw11|7205|/data/primary205 +208|206|p|p|s|u|sdw11|sdw11|7206|/data/primary206 +209|207|p|p|s|u|sdw11|sdw11|7207|/data/primary207 +210|208|p|p|s|u|sdw11|sdw11|7208|/data/primary208 +211|209|p|p|s|u|sdw11|sdw11|7209|/data/primary209 +212|210|p|p|s|u|sdw11|sdw11|7210|/data/primary210 +213|211|p|p|s|u|sdw11|sdw11|7211|/data/primary211 +214|212|p|p|s|u|sdw11|sdw11|7212|/data/primary212 +215|213|p|p|s|u|sdw11|sdw11|7213|/data/primary213 +216|214|p|p|s|u|sdw11|sdw11|7214|/data/primary214 +217|215|p|p|s|u|sdw11|sdw11|7215|/data/primary215 +218|216|p|p|s|u|sdw11|sdw11|7216|/data/primary216 +219|217|p|p|s|u|sdw11|sdw11|7217|/data/primary217 +220|218|p|p|s|u|sdw11|sdw11|7218|/data/primary218 +221|219|p|p|s|u|sdw11|sdw11|7219|/data/primary219 +1182|180|m|m|s|u|sdw11|sdw11|8180|/data/mirror180 +1183|181|m|m|s|u|sdw11|sdw11|8181|/data/mirror181 +1184|182|m|m|s|u|sdw11|sdw11|8182|/data/mirror182 +1185|183|m|m|s|u|sdw11|sdw11|8183|/data/mirror183 +1186|184|m|m|s|u|sdw11|sdw11|8184|/data/mirror184 +1187|185|m|m|s|u|sdw11|sdw11|8185|/data/mirror185 +1188|186|m|m|s|u|sdw11|sdw11|8186|/data/mirror186 +1189|187|m|m|s|u|sdw11|sdw11|8187|/data/mirror187 +1190|188|m|m|s|u|sdw11|sdw11|8188|/data/mirror188 +1191|189|m|m|s|u|sdw11|sdw11|8189|/data/mirror189 +1192|190|m|m|s|u|sdw11|sdw11|8190|/data/mirror190 +1193|191|m|m|s|u|sdw11|sdw11|8191|/data/mirror191 +1194|192|m|m|s|u|sdw11|sdw11|8192|/data/mirror192 +1195|193|m|m|s|u|sdw11|sdw11|8193|/data/mirror193 +1196|194|m|m|s|u|sdw11|sdw11|8194|/data/mirror194 +1197|195|m|m|s|u|sdw11|sdw11|8195|/data/mirror195 +1198|196|m|m|s|u|sdw11|sdw11|8196|/data/mirror196 +1199|197|m|m|s|u|sdw11|sdw11|8197|/data/mirror197 +1200|198|m|m|s|u|sdw11|sdw11|8198|/data/mirror198 +1201|199|m|m|s|u|sdw11|sdw11|8199|/data/mirror199 +# SDW12 +222|220|p|p|s|u|sdw12|sdw12|7220|/data/primary220 +223|221|p|p|s|u|sdw12|sdw12|7221|/data/primary221 +224|222|p|p|s|u|sdw12|sdw12|7222|/data/primary222 +225|223|p|p|s|u|sdw12|sdw12|7223|/data/primary223 +226|224|p|p|s|u|sdw12|sdw12|7224|/data/primary224 +227|225|p|p|s|u|sdw12|sdw12|7225|/data/primary225 +228|226|p|p|s|u|sdw12|sdw12|7226|/data/primary226 +229|227|p|p|s|u|sdw12|sdw12|7227|/data/primary227 +230|228|p|p|s|u|sdw12|sdw12|7228|/data/primary228 +231|229|p|p|s|u|sdw12|sdw12|7229|/data/primary229 +232|230|p|p|s|u|sdw12|sdw12|7230|/data/primary230 +233|231|p|p|s|u|sdw12|sdw12|7231|/data/primary231 +234|232|p|p|s|u|sdw12|sdw12|7232|/data/primary232 +235|233|p|p|s|u|sdw12|sdw12|7233|/data/primary233 +236|234|p|p|s|u|sdw12|sdw12|7234|/data/primary234 +237|235|p|p|s|u|sdw12|sdw12|7235|/data/primary235 +238|236|p|p|s|u|sdw12|sdw12|7236|/data/primary236 +239|237|p|p|s|u|sdw12|sdw12|7237|/data/primary237 +240|238|p|p|s|u|sdw12|sdw12|7238|/data/primary238 +241|239|p|p|s|u|sdw12|sdw12|7239|/data/primary239 +1202|200|m|m|s|u|sdw12|sdw12|8200|/data/mirror200 +1203|201|m|m|s|u|sdw12|sdw12|8201|/data/mirror201 +1204|202|m|m|s|u|sdw12|sdw12|8202|/data/mirror202 +1205|203|m|m|s|u|sdw12|sdw12|8203|/data/mirror203 +1206|204|m|m|s|u|sdw12|sdw12|8204|/data/mirror204 +1207|205|m|m|s|u|sdw12|sdw12|8205|/data/mirror205 +1208|206|m|m|s|u|sdw12|sdw12|8206|/data/mirror206 +1209|207|m|m|s|u|sdw12|sdw12|8207|/data/mirror207 +1210|208|m|m|s|u|sdw12|sdw12|8208|/data/mirror208 +1211|209|m|m|s|u|sdw12|sdw12|8209|/data/mirror209 +1212|210|m|m|s|u|sdw12|sdw12|8210|/data/mirror210 +1213|211|m|m|s|u|sdw12|sdw12|8211|/data/mirror211 +1214|212|m|m|s|u|sdw12|sdw12|8212|/data/mirror212 +1215|213|m|m|s|u|sdw12|sdw12|8213|/data/mirror213 +1216|214|m|m|s|u|sdw12|sdw12|8214|/data/mirror214 +1217|215|m|m|s|u|sdw12|sdw12|8215|/data/mirror215 +1218|216|m|m|s|u|sdw12|sdw12|8216|/data/mirror216 +1219|217|m|m|s|u|sdw12|sdw12|8217|/data/mirror217 +1220|218|m|m|s|u|sdw12|sdw12|8218|/data/mirror218 +1221|219|m|m|s|u|sdw12|sdw12|8219|/data/mirror219 +# SDW13 +242|240|p|p|s|u|sdw13|sdw13|7240|/data/primary240 +243|241|p|p|s|u|sdw13|sdw13|7241|/data/primary241 +244|242|p|p|s|u|sdw13|sdw13|7242|/data/primary242 +245|243|p|p|s|u|sdw13|sdw13|7243|/data/primary243 +246|244|p|p|s|u|sdw13|sdw13|7244|/data/primary244 +247|245|p|p|s|u|sdw13|sdw13|7245|/data/primary245 +248|246|p|p|s|u|sdw13|sdw13|7246|/data/primary246 +249|247|p|p|s|u|sdw13|sdw13|7247|/data/primary247 +250|248|p|p|s|u|sdw13|sdw13|7248|/data/primary248 +251|249|p|p|s|u|sdw13|sdw13|7249|/data/primary249 +252|250|p|p|s|u|sdw13|sdw13|7250|/data/primary250 +253|251|p|p|s|u|sdw13|sdw13|7251|/data/primary251 +254|252|p|p|s|u|sdw13|sdw13|7252|/data/primary252 +255|253|p|p|s|u|sdw13|sdw13|7253|/data/primary253 +256|254|p|p|s|u|sdw13|sdw13|7254|/data/primary254 +257|255|p|p|s|u|sdw13|sdw13|7255|/data/primary255 +258|256|p|p|s|u|sdw13|sdw13|7256|/data/primary256 +259|257|p|p|s|u|sdw13|sdw13|7257|/data/primary257 +260|258|p|p|s|u|sdw13|sdw13|7258|/data/primary258 +261|259|p|p|s|u|sdw13|sdw13|7259|/data/primary259 +1222|220|m|m|s|u|sdw13|sdw13|8220|/data/mirror220 +1223|221|m|m|s|u|sdw13|sdw13|8221|/data/mirror221 +1224|222|m|m|s|u|sdw13|sdw13|8222|/data/mirror222 +1225|223|m|m|s|u|sdw13|sdw13|8223|/data/mirror223 +1226|224|m|m|s|u|sdw13|sdw13|8224|/data/mirror224 +1227|225|m|m|s|u|sdw13|sdw13|8225|/data/mirror225 +1228|226|m|m|s|u|sdw13|sdw13|8226|/data/mirror226 +1229|227|m|m|s|u|sdw13|sdw13|8227|/data/mirror227 +1230|228|m|m|s|u|sdw13|sdw13|8228|/data/mirror228 +1231|229|m|m|s|u|sdw13|sdw13|8229|/data/mirror229 +1232|230|m|m|s|u|sdw13|sdw13|8230|/data/mirror230 +1233|231|m|m|s|u|sdw13|sdw13|8231|/data/mirror231 +1234|232|m|m|s|u|sdw13|sdw13|8232|/data/mirror232 +1235|233|m|m|s|u|sdw13|sdw13|8233|/data/mirror233 +1236|234|m|m|s|u|sdw13|sdw13|8234|/data/mirror234 +1237|235|m|m|s|u|sdw13|sdw13|8235|/data/mirror235 +1238|236|m|m|s|u|sdw13|sdw13|8236|/data/mirror236 +1239|237|m|m|s|u|sdw13|sdw13|8237|/data/mirror237 +1240|238|m|m|s|u|sdw13|sdw13|8238|/data/mirror238 +1241|239|m|m|s|u|sdw13|sdw13|8239|/data/mirror239 +# SDW14 +262|260|p|p|s|u|sdw14|sdw14|7260|/data/primary260 +263|261|p|p|s|u|sdw14|sdw14|7261|/data/primary261 +264|262|p|p|s|u|sdw14|sdw14|7262|/data/primary262 +265|263|p|p|s|u|sdw14|sdw14|7263|/data/primary263 +266|264|p|p|s|u|sdw14|sdw14|7264|/data/primary264 +267|265|p|p|s|u|sdw14|sdw14|7265|/data/primary265 +268|266|p|p|s|u|sdw14|sdw14|7266|/data/primary266 +269|267|p|p|s|u|sdw14|sdw14|7267|/data/primary267 +270|268|p|p|s|u|sdw14|sdw14|7268|/data/primary268 +271|269|p|p|s|u|sdw14|sdw14|7269|/data/primary269 +272|270|p|p|s|u|sdw14|sdw14|7270|/data/primary270 +273|271|p|p|s|u|sdw14|sdw14|7271|/data/primary271 +274|272|p|p|s|u|sdw14|sdw14|7272|/data/primary272 +275|273|p|p|s|u|sdw14|sdw14|7273|/data/primary273 +276|274|p|p|s|u|sdw14|sdw14|7274|/data/primary274 +277|275|p|p|s|u|sdw14|sdw14|7275|/data/primary275 +278|276|p|p|s|u|sdw14|sdw14|7276|/data/primary276 +279|277|p|p|s|u|sdw14|sdw14|7277|/data/primary277 +280|278|p|p|s|u|sdw14|sdw14|7278|/data/primary278 +281|279|p|p|s|u|sdw14|sdw14|7279|/data/primary279 +1242|240|m|m|s|u|sdw14|sdw14|8240|/data/mirror240 +1243|241|m|m|s|u|sdw14|sdw14|8241|/data/mirror241 +1244|242|m|m|s|u|sdw14|sdw14|8242|/data/mirror242 +1245|243|m|m|s|u|sdw14|sdw14|8243|/data/mirror243 +1246|244|m|m|s|u|sdw14|sdw14|8244|/data/mirror244 +1247|245|m|m|s|u|sdw14|sdw14|8245|/data/mirror245 +1248|246|m|m|s|u|sdw14|sdw14|8246|/data/mirror246 +1249|247|m|m|s|u|sdw14|sdw14|8247|/data/mirror247 +1250|248|m|m|s|u|sdw14|sdw14|8248|/data/mirror248 +1251|249|m|m|s|u|sdw14|sdw14|8249|/data/mirror249 +1252|250|m|m|s|u|sdw14|sdw14|8250|/data/mirror250 +1253|251|m|m|s|u|sdw14|sdw14|8251|/data/mirror251 +1254|252|m|m|s|u|sdw14|sdw14|8252|/data/mirror252 +1255|253|m|m|s|u|sdw14|sdw14|8253|/data/mirror253 +1256|254|m|m|s|u|sdw14|sdw14|8254|/data/mirror254 +1257|255|m|m|s|u|sdw14|sdw14|8255|/data/mirror255 +1258|256|m|m|s|u|sdw14|sdw14|8256|/data/mirror256 +1259|257|m|m|s|u|sdw14|sdw14|8257|/data/mirror257 +1260|258|m|m|s|u|sdw14|sdw14|8258|/data/mirror258 +1261|259|m|m|s|u|sdw14|sdw14|8259|/data/mirror259 +# SDW15 +282|280|p|p|s|u|sdw15|sdw15|7280|/data/primary280 +283|281|p|p|s|u|sdw15|sdw15|7281|/data/primary281 +284|282|p|p|s|u|sdw15|sdw15|7282|/data/primary282 +285|283|p|p|s|u|sdw15|sdw15|7283|/data/primary283 +286|284|p|p|s|u|sdw15|sdw15|7284|/data/primary284 +287|285|p|p|s|u|sdw15|sdw15|7285|/data/primary285 +288|286|p|p|s|u|sdw15|sdw15|7286|/data/primary286 +289|287|p|p|s|u|sdw15|sdw15|7287|/data/primary287 +290|288|p|p|s|u|sdw15|sdw15|7288|/data/primary288 +291|289|p|p|s|u|sdw15|sdw15|7289|/data/primary289 +292|290|p|p|s|u|sdw15|sdw15|7290|/data/primary290 +293|291|p|p|s|u|sdw15|sdw15|7291|/data/primary291 +294|292|p|p|s|u|sdw15|sdw15|7292|/data/primary292 +295|293|p|p|s|u|sdw15|sdw15|7293|/data/primary293 +296|294|p|p|s|u|sdw15|sdw15|7294|/data/primary294 +297|295|p|p|s|u|sdw15|sdw15|7295|/data/primary295 +298|296|p|p|s|u|sdw15|sdw15|7296|/data/primary296 +299|297|p|p|s|u|sdw15|sdw15|7297|/data/primary297 +300|298|p|p|s|u|sdw15|sdw15|7298|/data/primary298 +301|299|p|p|s|u|sdw15|sdw15|7299|/data/primary299 +1262|260|m|m|s|u|sdw15|sdw15|8260|/data/mirror260 +1263|261|m|m|s|u|sdw15|sdw15|8261|/data/mirror261 +1264|262|m|m|s|u|sdw15|sdw15|8262|/data/mirror262 +1265|263|m|m|s|u|sdw15|sdw15|8263|/data/mirror263 +1266|264|m|m|s|u|sdw15|sdw15|8264|/data/mirror264 +1267|265|m|m|s|u|sdw15|sdw15|8265|/data/mirror265 +1268|266|m|m|s|u|sdw15|sdw15|8266|/data/mirror266 +1269|267|m|m|s|u|sdw15|sdw15|8267|/data/mirror267 +1270|268|m|m|s|u|sdw15|sdw15|8268|/data/mirror268 +1271|269|m|m|s|u|sdw15|sdw15|8269|/data/mirror269 +1272|270|m|m|s|u|sdw15|sdw15|8270|/data/mirror270 +1273|271|m|m|s|u|sdw15|sdw15|8271|/data/mirror271 +1274|272|m|m|s|u|sdw15|sdw15|8272|/data/mirror272 +1275|273|m|m|s|u|sdw15|sdw15|8273|/data/mirror273 +1276|274|m|m|s|u|sdw15|sdw15|8274|/data/mirror274 +1277|275|m|m|s|u|sdw15|sdw15|8275|/data/mirror275 +1278|276|m|m|s|u|sdw15|sdw15|8276|/data/mirror276 +1279|277|m|m|s|u|sdw15|sdw15|8277|/data/mirror277 +1280|278|m|m|s|u|sdw15|sdw15|8278|/data/mirror278 +1281|279|m|m|s|u|sdw15|sdw15|8279|/data/mirror279 +# SDW16 +302|300|p|p|s|u|sdw16|sdw16|7300|/data/primary300 +303|301|p|p|s|u|sdw16|sdw16|7301|/data/primary301 +304|302|p|p|s|u|sdw16|sdw16|7302|/data/primary302 +305|303|p|p|s|u|sdw16|sdw16|7303|/data/primary303 +306|304|p|p|s|u|sdw16|sdw16|7304|/data/primary304 +307|305|p|p|s|u|sdw16|sdw16|7305|/data/primary305 +308|306|p|p|s|u|sdw16|sdw16|7306|/data/primary306 +309|307|p|p|s|u|sdw16|sdw16|7307|/data/primary307 +310|308|p|p|s|u|sdw16|sdw16|7308|/data/primary308 +311|309|p|p|s|u|sdw16|sdw16|7309|/data/primary309 +312|310|p|p|s|u|sdw16|sdw16|7310|/data/primary310 +313|311|p|p|s|u|sdw16|sdw16|7311|/data/primary311 +314|312|p|p|s|u|sdw16|sdw16|7312|/data/primary312 +315|313|p|p|s|u|sdw16|sdw16|7313|/data/primary313 +316|314|p|p|s|u|sdw16|sdw16|7314|/data/primary314 +317|315|p|p|s|u|sdw16|sdw16|7315|/data/primary315 +318|316|p|p|s|u|sdw16|sdw16|7316|/data/primary316 +319|317|p|p|s|u|sdw16|sdw16|7317|/data/primary317 +320|318|p|p|s|u|sdw16|sdw16|7318|/data/primary318 +321|319|p|p|s|u|sdw16|sdw16|7319|/data/primary319 +1282|280|m|m|s|u|sdw16|sdw16|8280|/data/mirror280 +1283|281|m|m|s|u|sdw16|sdw16|8281|/data/mirror281 +1284|282|m|m|s|u|sdw16|sdw16|8282|/data/mirror282 +1285|283|m|m|s|u|sdw16|sdw16|8283|/data/mirror283 +1286|284|m|m|s|u|sdw16|sdw16|8284|/data/mirror284 +1287|285|m|m|s|u|sdw16|sdw16|8285|/data/mirror285 +1288|286|m|m|s|u|sdw16|sdw16|8286|/data/mirror286 +1289|287|m|m|s|u|sdw16|sdw16|8287|/data/mirror287 +1290|288|m|m|s|u|sdw16|sdw16|8288|/data/mirror288 +1291|289|m|m|s|u|sdw16|sdw16|8289|/data/mirror289 +1292|290|m|m|s|u|sdw16|sdw16|8290|/data/mirror290 +1293|291|m|m|s|u|sdw16|sdw16|8291|/data/mirror291 +1294|292|m|m|s|u|sdw16|sdw16|8292|/data/mirror292 +1295|293|m|m|s|u|sdw16|sdw16|8293|/data/mirror293 +1296|294|m|m|s|u|sdw16|sdw16|8294|/data/mirror294 +1297|295|m|m|s|u|sdw16|sdw16|8295|/data/mirror295 +1298|296|m|m|s|u|sdw16|sdw16|8296|/data/mirror296 +1299|297|m|m|s|u|sdw16|sdw16|8297|/data/mirror297 +1300|298|m|m|s|u|sdw16|sdw16|8298|/data/mirror298 +1301|299|m|m|s|u|sdw16|sdw16|8299|/data/mirror299 +# SDW17 +322|320|p|p|s|u|sdw17|sdw17|7320|/data/primary320 +323|321|p|p|s|u|sdw17|sdw17|7321|/data/primary321 +324|322|p|p|s|u|sdw17|sdw17|7322|/data/primary322 +325|323|p|p|s|u|sdw17|sdw17|7323|/data/primary323 +326|324|p|p|s|u|sdw17|sdw17|7324|/data/primary324 +327|325|p|p|s|u|sdw17|sdw17|7325|/data/primary325 +328|326|p|p|s|u|sdw17|sdw17|7326|/data/primary326 +329|327|p|p|s|u|sdw17|sdw17|7327|/data/primary327 +330|328|p|p|s|u|sdw17|sdw17|7328|/data/primary328 +331|329|p|p|s|u|sdw17|sdw17|7329|/data/primary329 +332|330|p|p|s|u|sdw17|sdw17|7330|/data/primary330 +333|331|p|p|s|u|sdw17|sdw17|7331|/data/primary331 +334|332|p|p|s|u|sdw17|sdw17|7332|/data/primary332 +335|333|p|p|s|u|sdw17|sdw17|7333|/data/primary333 +336|334|p|p|s|u|sdw17|sdw17|7334|/data/primary334 +337|335|p|p|s|u|sdw17|sdw17|7335|/data/primary335 +338|336|p|p|s|u|sdw17|sdw17|7336|/data/primary336 +339|337|p|p|s|u|sdw17|sdw17|7337|/data/primary337 +340|338|p|p|s|u|sdw17|sdw17|7338|/data/primary338 +341|339|p|p|s|u|sdw17|sdw17|7339|/data/primary339 +1302|300|m|m|s|u|sdw17|sdw17|8300|/data/mirror300 +1303|301|m|m|s|u|sdw17|sdw17|8301|/data/mirror301 +1304|302|m|m|s|u|sdw17|sdw17|8302|/data/mirror302 +1305|303|m|m|s|u|sdw17|sdw17|8303|/data/mirror303 +1306|304|m|m|s|u|sdw17|sdw17|8304|/data/mirror304 +1307|305|m|m|s|u|sdw17|sdw17|8305|/data/mirror305 +1308|306|m|m|s|u|sdw17|sdw17|8306|/data/mirror306 +1309|307|m|m|s|u|sdw17|sdw17|8307|/data/mirror307 +1310|308|m|m|s|u|sdw17|sdw17|8308|/data/mirror308 +1311|309|m|m|s|u|sdw17|sdw17|8309|/data/mirror309 +1312|310|m|m|s|u|sdw17|sdw17|8310|/data/mirror310 +1313|311|m|m|s|u|sdw17|sdw17|8311|/data/mirror311 +1314|312|m|m|s|u|sdw17|sdw17|8312|/data/mirror312 +1315|313|m|m|s|u|sdw17|sdw17|8313|/data/mirror313 +1316|314|m|m|s|u|sdw17|sdw17|8314|/data/mirror314 +1317|315|m|m|s|u|sdw17|sdw17|8315|/data/mirror315 +1318|316|m|m|s|u|sdw17|sdw17|8316|/data/mirror316 +1319|317|m|m|s|u|sdw17|sdw17|8317|/data/mirror317 +1320|318|m|m|s|u|sdw17|sdw17|8318|/data/mirror318 +1321|319|m|m|s|u|sdw17|sdw17|8319|/data/mirror319 +# SDW18 +342|340|p|p|s|u|sdw18|sdw18|7340|/data/primary340 +343|341|p|p|s|u|sdw18|sdw18|7341|/data/primary341 +344|342|p|p|s|u|sdw18|sdw18|7342|/data/primary342 +345|343|p|p|s|u|sdw18|sdw18|7343|/data/primary343 +346|344|p|p|s|u|sdw18|sdw18|7344|/data/primary344 +347|345|p|p|s|u|sdw18|sdw18|7345|/data/primary345 +348|346|p|p|s|u|sdw18|sdw18|7346|/data/primary346 +349|347|p|p|s|u|sdw18|sdw18|7347|/data/primary347 +350|348|p|p|s|u|sdw18|sdw18|7348|/data/primary348 +351|349|p|p|s|u|sdw18|sdw18|7349|/data/primary349 +352|350|p|p|s|u|sdw18|sdw18|7350|/data/primary350 +353|351|p|p|s|u|sdw18|sdw18|7351|/data/primary351 +354|352|p|p|s|u|sdw18|sdw18|7352|/data/primary352 +355|353|p|p|s|u|sdw18|sdw18|7353|/data/primary353 +356|354|p|p|s|u|sdw18|sdw18|7354|/data/primary354 +357|355|p|p|s|u|sdw18|sdw18|7355|/data/primary355 +358|356|p|p|s|u|sdw18|sdw18|7356|/data/primary356 +359|357|p|p|s|u|sdw18|sdw18|7357|/data/primary357 +360|358|p|p|s|u|sdw18|sdw18|7358|/data/primary358 +361|359|p|p|s|u|sdw18|sdw18|7359|/data/primary359 +1322|320|m|m|s|u|sdw18|sdw18|8320|/data/mirror320 +1323|321|m|m|s|u|sdw18|sdw18|8321|/data/mirror321 +1324|322|m|m|s|u|sdw18|sdw18|8322|/data/mirror322 +1325|323|m|m|s|u|sdw18|sdw18|8323|/data/mirror323 +1326|324|m|m|s|u|sdw18|sdw18|8324|/data/mirror324 +1327|325|m|m|s|u|sdw18|sdw18|8325|/data/mirror325 +1328|326|m|m|s|u|sdw18|sdw18|8326|/data/mirror326 +1329|327|m|m|s|u|sdw18|sdw18|8327|/data/mirror327 +1330|328|m|m|s|u|sdw18|sdw18|8328|/data/mirror328 +1331|329|m|m|s|u|sdw18|sdw18|8329|/data/mirror329 +1332|330|m|m|s|u|sdw18|sdw18|8330|/data/mirror330 +1333|331|m|m|s|u|sdw18|sdw18|8331|/data/mirror331 +1334|332|m|m|s|u|sdw18|sdw18|8332|/data/mirror332 +1335|333|m|m|s|u|sdw18|sdw18|8333|/data/mirror333 +1336|334|m|m|s|u|sdw18|sdw18|8334|/data/mirror334 +1337|335|m|m|s|u|sdw18|sdw18|8335|/data/mirror335 +1338|336|m|m|s|u|sdw18|sdw18|8336|/data/mirror336 +1339|337|m|m|s|u|sdw18|sdw18|8337|/data/mirror337 +1340|338|m|m|s|u|sdw18|sdw18|8338|/data/mirror338 +1341|339|m|m|s|u|sdw18|sdw18|8339|/data/mirror339 +# SDW19 +362|360|p|p|s|u|sdw19|sdw19|7360|/data/primary360 +363|361|p|p|s|u|sdw19|sdw19|7361|/data/primary361 +364|362|p|p|s|u|sdw19|sdw19|7362|/data/primary362 +365|363|p|p|s|u|sdw19|sdw19|7363|/data/primary363 +366|364|p|p|s|u|sdw19|sdw19|7364|/data/primary364 +367|365|p|p|s|u|sdw19|sdw19|7365|/data/primary365 +368|366|p|p|s|u|sdw19|sdw19|7366|/data/primary366 +369|367|p|p|s|u|sdw19|sdw19|7367|/data/primary367 +370|368|p|p|s|u|sdw19|sdw19|7368|/data/primary368 +371|369|p|p|s|u|sdw19|sdw19|7369|/data/primary369 +372|370|p|p|s|u|sdw19|sdw19|7370|/data/primary370 +373|371|p|p|s|u|sdw19|sdw19|7371|/data/primary371 +374|372|p|p|s|u|sdw19|sdw19|7372|/data/primary372 +375|373|p|p|s|u|sdw19|sdw19|7373|/data/primary373 +376|374|p|p|s|u|sdw19|sdw19|7374|/data/primary374 +377|375|p|p|s|u|sdw19|sdw19|7375|/data/primary375 +378|376|p|p|s|u|sdw19|sdw19|7376|/data/primary376 +379|377|p|p|s|u|sdw19|sdw19|7377|/data/primary377 +380|378|p|p|s|u|sdw19|sdw19|7378|/data/primary378 +381|379|p|p|s|u|sdw19|sdw19|7379|/data/primary379 +1342|340|m|m|s|u|sdw19|sdw19|8340|/data/mirror340 +1343|341|m|m|s|u|sdw19|sdw19|8341|/data/mirror341 +1344|342|m|m|s|u|sdw19|sdw19|8342|/data/mirror342 +1345|343|m|m|s|u|sdw19|sdw19|8343|/data/mirror343 +1346|344|m|m|s|u|sdw19|sdw19|8344|/data/mirror344 +1347|345|m|m|s|u|sdw19|sdw19|8345|/data/mirror345 +1348|346|m|m|s|u|sdw19|sdw19|8346|/data/mirror346 +1349|347|m|m|s|u|sdw19|sdw19|8347|/data/mirror347 +1350|348|m|m|s|u|sdw19|sdw19|8348|/data/mirror348 +1351|349|m|m|s|u|sdw19|sdw19|8349|/data/mirror349 +1352|350|m|m|s|u|sdw19|sdw19|8350|/data/mirror350 +1353|351|m|m|s|u|sdw19|sdw19|8351|/data/mirror351 +1354|352|m|m|s|u|sdw19|sdw19|8352|/data/mirror352 +1355|353|m|m|s|u|sdw19|sdw19|8353|/data/mirror353 +1356|354|m|m|s|u|sdw19|sdw19|8354|/data/mirror354 +1357|355|m|m|s|u|sdw19|sdw19|8355|/data/mirror355 +1358|356|m|m|s|u|sdw19|sdw19|8356|/data/mirror356 +1359|357|m|m|s|u|sdw19|sdw19|8357|/data/mirror357 +1360|358|m|m|s|u|sdw19|sdw19|8358|/data/mirror358 +1361|359|m|m|s|u|sdw19|sdw19|8359|/data/mirror359 +# SDW2 +22|20|p|p|s|u|sdw2|sdw2|7020|/data/primary20 +23|21|p|p|s|u|sdw2|sdw2|7021|/data/primary21 +24|22|p|p|s|u|sdw2|sdw2|7022|/data/primary22 +25|23|p|p|s|u|sdw2|sdw2|7023|/data/primary23 +26|24|p|p|s|u|sdw2|sdw2|7024|/data/primary24 +27|25|p|p|s|u|sdw2|sdw2|7025|/data/primary25 +28|26|p|p|s|u|sdw2|sdw2|7026|/data/primary26 +29|27|p|p|s|u|sdw2|sdw2|7027|/data/primary27 +30|28|p|p|s|u|sdw2|sdw2|7028|/data/primary28 +31|29|p|p|s|u|sdw2|sdw2|7029|/data/primary29 +32|30|p|p|s|u|sdw2|sdw2|7030|/data/primary30 +33|31|p|p|s|u|sdw2|sdw2|7031|/data/primary31 +34|32|p|p|s|u|sdw2|sdw2|7032|/data/primary32 +35|33|p|p|s|u|sdw2|sdw2|7033|/data/primary33 +36|34|p|p|s|u|sdw2|sdw2|7034|/data/primary34 +37|35|p|p|s|u|sdw2|sdw2|7035|/data/primary35 +38|36|p|p|s|u|sdw2|sdw2|7036|/data/primary36 +39|37|p|p|s|u|sdw2|sdw2|7037|/data/primary37 +40|38|p|p|s|u|sdw2|sdw2|7038|/data/primary38 +41|39|p|p|s|u|sdw2|sdw2|7039|/data/primary39 +1002|0|m|m|s|u|sdw2|sdw2|8000|/data/mirror0 +1003|1|m|m|s|u|sdw2|sdw2|8001|/data/mirror1 +1004|2|m|m|s|u|sdw2|sdw2|8002|/data/mirror2 +1005|3|m|m|s|u|sdw2|sdw2|8003|/data/mirror3 +1006|4|m|m|s|u|sdw2|sdw2|8004|/data/mirror4 +1007|5|m|m|s|u|sdw2|sdw2|8005|/data/mirror5 +1008|6|m|m|s|u|sdw2|sdw2|8006|/data/mirror6 +1009|7|m|m|s|u|sdw2|sdw2|8007|/data/mirror7 +1010|8|m|m|s|u|sdw2|sdw2|8008|/data/mirror8 +1011|9|m|m|s|u|sdw2|sdw2|8009|/data/mirror9 +1012|10|m|m|s|u|sdw2|sdw2|8010|/data/mirror10 +1013|11|m|m|s|u|sdw2|sdw2|8011|/data/mirror11 +1014|12|m|m|s|u|sdw2|sdw2|8012|/data/mirror12 +1015|13|m|m|s|u|sdw2|sdw2|8013|/data/mirror13 +1016|14|m|m|s|u|sdw2|sdw2|8014|/data/mirror14 +1017|15|m|m|s|u|sdw2|sdw2|8015|/data/mirror15 +1018|16|m|m|s|u|sdw2|sdw2|8016|/data/mirror16 +1019|17|m|m|s|u|sdw2|sdw2|8017|/data/mirror17 +1020|18|m|m|s|u|sdw2|sdw2|8018|/data/mirror18 +1021|19|m|m|s|u|sdw2|sdw2|8019|/data/mirror19 +# SDW20 +382|380|p|p|s|u|sdw20|sdw20|7380|/data/primary380 +383|381|p|p|s|u|sdw20|sdw20|7381|/data/primary381 +384|382|p|p|s|u|sdw20|sdw20|7382|/data/primary382 +385|383|p|p|s|u|sdw20|sdw20|7383|/data/primary383 +386|384|p|p|s|u|sdw20|sdw20|7384|/data/primary384 +387|385|p|p|s|u|sdw20|sdw20|7385|/data/primary385 +388|386|p|p|s|u|sdw20|sdw20|7386|/data/primary386 +389|387|p|p|s|u|sdw20|sdw20|7387|/data/primary387 +390|388|p|p|s|u|sdw20|sdw20|7388|/data/primary388 +391|389|p|p|s|u|sdw20|sdw20|7389|/data/primary389 +392|390|p|p|s|u|sdw20|sdw20|7390|/data/primary390 +393|391|p|p|s|u|sdw20|sdw20|7391|/data/primary391 +394|392|p|p|s|u|sdw20|sdw20|7392|/data/primary392 +395|393|p|p|s|u|sdw20|sdw20|7393|/data/primary393 +396|394|p|p|s|u|sdw20|sdw20|7394|/data/primary394 +397|395|p|p|s|u|sdw20|sdw20|7395|/data/primary395 +398|396|p|p|s|u|sdw20|sdw20|7396|/data/primary396 +399|397|p|p|s|u|sdw20|sdw20|7397|/data/primary397 +400|398|p|p|s|u|sdw20|sdw20|7398|/data/primary398 +401|399|p|p|s|u|sdw20|sdw20|7399|/data/primary399 +1362|360|m|m|s|u|sdw20|sdw20|8360|/data/mirror360 +1363|361|m|m|s|u|sdw20|sdw20|8361|/data/mirror361 +1364|362|m|m|s|u|sdw20|sdw20|8362|/data/mirror362 +1365|363|m|m|s|u|sdw20|sdw20|8363|/data/mirror363 +1366|364|m|m|s|u|sdw20|sdw20|8364|/data/mirror364 +1367|365|m|m|s|u|sdw20|sdw20|8365|/data/mirror365 +1368|366|m|m|s|u|sdw20|sdw20|8366|/data/mirror366 +1369|367|m|m|s|u|sdw20|sdw20|8367|/data/mirror367 +1370|368|m|m|s|u|sdw20|sdw20|8368|/data/mirror368 +1371|369|m|m|s|u|sdw20|sdw20|8369|/data/mirror369 +1372|370|m|m|s|u|sdw20|sdw20|8370|/data/mirror370 +1373|371|m|m|s|u|sdw20|sdw20|8371|/data/mirror371 +1374|372|m|m|s|u|sdw20|sdw20|8372|/data/mirror372 +1375|373|m|m|s|u|sdw20|sdw20|8373|/data/mirror373 +1376|374|m|m|s|u|sdw20|sdw20|8374|/data/mirror374 +1377|375|m|m|s|u|sdw20|sdw20|8375|/data/mirror375 +1378|376|m|m|s|u|sdw20|sdw20|8376|/data/mirror376 +1379|377|m|m|s|u|sdw20|sdw20|8377|/data/mirror377 +1380|378|m|m|s|u|sdw20|sdw20|8378|/data/mirror378 +1381|379|m|m|s|u|sdw20|sdw20|8379|/data/mirror379 +# SDW21 +402|400|p|p|s|u|sdw21|sdw21|7400|/data/primary400 +403|401|p|p|s|u|sdw21|sdw21|7401|/data/primary401 +404|402|p|p|s|u|sdw21|sdw21|7402|/data/primary402 +405|403|p|p|s|u|sdw21|sdw21|7403|/data/primary403 +406|404|p|p|s|u|sdw21|sdw21|7404|/data/primary404 +407|405|p|p|s|u|sdw21|sdw21|7405|/data/primary405 +408|406|p|p|s|u|sdw21|sdw21|7406|/data/primary406 +409|407|p|p|s|u|sdw21|sdw21|7407|/data/primary407 +410|408|p|p|s|u|sdw21|sdw21|7408|/data/primary408 +411|409|p|p|s|u|sdw21|sdw21|7409|/data/primary409 +412|410|p|p|s|u|sdw21|sdw21|7410|/data/primary410 +413|411|p|p|s|u|sdw21|sdw21|7411|/data/primary411 +414|412|p|p|s|u|sdw21|sdw21|7412|/data/primary412 +415|413|p|p|s|u|sdw21|sdw21|7413|/data/primary413 +416|414|p|p|s|u|sdw21|sdw21|7414|/data/primary414 +417|415|p|p|s|u|sdw21|sdw21|7415|/data/primary415 +418|416|p|p|s|u|sdw21|sdw21|7416|/data/primary416 +419|417|p|p|s|u|sdw21|sdw21|7417|/data/primary417 +420|418|p|p|s|u|sdw21|sdw21|7418|/data/primary418 +421|419|p|p|s|u|sdw21|sdw21|7419|/data/primary419 +1382|380|m|m|s|u|sdw21|sdw21|8380|/data/mirror380 +1383|381|m|m|s|u|sdw21|sdw21|8381|/data/mirror381 +1384|382|m|m|s|u|sdw21|sdw21|8382|/data/mirror382 +1385|383|m|m|s|u|sdw21|sdw21|8383|/data/mirror383 +1386|384|m|m|s|u|sdw21|sdw21|8384|/data/mirror384 +1387|385|m|m|s|u|sdw21|sdw21|8385|/data/mirror385 +1388|386|m|m|s|u|sdw21|sdw21|8386|/data/mirror386 +1389|387|m|m|s|u|sdw21|sdw21|8387|/data/mirror387 +1390|388|m|m|s|u|sdw21|sdw21|8388|/data/mirror388 +1391|389|m|m|s|u|sdw21|sdw21|8389|/data/mirror389 +1392|390|m|m|s|u|sdw21|sdw21|8390|/data/mirror390 +1393|391|m|m|s|u|sdw21|sdw21|8391|/data/mirror391 +1394|392|m|m|s|u|sdw21|sdw21|8392|/data/mirror392 +1395|393|m|m|s|u|sdw21|sdw21|8393|/data/mirror393 +1396|394|m|m|s|u|sdw21|sdw21|8394|/data/mirror394 +1397|395|m|m|s|u|sdw21|sdw21|8395|/data/mirror395 +1398|396|m|m|s|u|sdw21|sdw21|8396|/data/mirror396 +1399|397|m|m|s|u|sdw21|sdw21|8397|/data/mirror397 +1400|398|m|m|s|u|sdw21|sdw21|8398|/data/mirror398 +1401|399|m|m|s|u|sdw21|sdw21|8399|/data/mirror399 +# SDW22 +422|420|p|p|s|u|sdw22|sdw22|7420|/data/primary420 +423|421|p|p|s|u|sdw22|sdw22|7421|/data/primary421 +424|422|p|p|s|u|sdw22|sdw22|7422|/data/primary422 +425|423|p|p|s|u|sdw22|sdw22|7423|/data/primary423 +426|424|p|p|s|u|sdw22|sdw22|7424|/data/primary424 +427|425|p|p|s|u|sdw22|sdw22|7425|/data/primary425 +428|426|p|p|s|u|sdw22|sdw22|7426|/data/primary426 +429|427|p|p|s|u|sdw22|sdw22|7427|/data/primary427 +430|428|p|p|s|u|sdw22|sdw22|7428|/data/primary428 +431|429|p|p|s|u|sdw22|sdw22|7429|/data/primary429 +432|430|p|p|s|u|sdw22|sdw22|7430|/data/primary430 +433|431|p|p|s|u|sdw22|sdw22|7431|/data/primary431 +434|432|p|p|s|u|sdw22|sdw22|7432|/data/primary432 +435|433|p|p|s|u|sdw22|sdw22|7433|/data/primary433 +436|434|p|p|s|u|sdw22|sdw22|7434|/data/primary434 +437|435|p|p|s|u|sdw22|sdw22|7435|/data/primary435 +438|436|p|p|s|u|sdw22|sdw22|7436|/data/primary436 +439|437|p|p|s|u|sdw22|sdw22|7437|/data/primary437 +440|438|p|p|s|u|sdw22|sdw22|7438|/data/primary438 +441|439|p|p|s|u|sdw22|sdw22|7439|/data/primary439 +1402|400|m|m|s|u|sdw22|sdw22|8400|/data/mirror400 +1403|401|m|m|s|u|sdw22|sdw22|8401|/data/mirror401 +1404|402|m|m|s|u|sdw22|sdw22|8402|/data/mirror402 +1405|403|m|m|s|u|sdw22|sdw22|8403|/data/mirror403 +1406|404|m|m|s|u|sdw22|sdw22|8404|/data/mirror404 +1407|405|m|m|s|u|sdw22|sdw22|8405|/data/mirror405 +1408|406|m|m|s|u|sdw22|sdw22|8406|/data/mirror406 +1409|407|m|m|s|u|sdw22|sdw22|8407|/data/mirror407 +1410|408|m|m|s|u|sdw22|sdw22|8408|/data/mirror408 +1411|409|m|m|s|u|sdw22|sdw22|8409|/data/mirror409 +1412|410|m|m|s|u|sdw22|sdw22|8410|/data/mirror410 +1413|411|m|m|s|u|sdw22|sdw22|8411|/data/mirror411 +1414|412|m|m|s|u|sdw22|sdw22|8412|/data/mirror412 +1415|413|m|m|s|u|sdw22|sdw22|8413|/data/mirror413 +1416|414|m|m|s|u|sdw22|sdw22|8414|/data/mirror414 +1417|415|m|m|s|u|sdw22|sdw22|8415|/data/mirror415 +1418|416|m|m|s|u|sdw22|sdw22|8416|/data/mirror416 +1419|417|m|m|s|u|sdw22|sdw22|8417|/data/mirror417 +1420|418|m|m|s|u|sdw22|sdw22|8418|/data/mirror418 +1421|419|m|m|s|u|sdw22|sdw22|8419|/data/mirror419 +# SDW23 +442|440|p|p|s|u|sdw23|sdw23|7440|/data/primary440 +443|441|p|p|s|u|sdw23|sdw23|7441|/data/primary441 +444|442|p|p|s|u|sdw23|sdw23|7442|/data/primary442 +445|443|p|p|s|u|sdw23|sdw23|7443|/data/primary443 +446|444|p|p|s|u|sdw23|sdw23|7444|/data/primary444 +447|445|p|p|s|u|sdw23|sdw23|7445|/data/primary445 +448|446|p|p|s|u|sdw23|sdw23|7446|/data/primary446 +449|447|p|p|s|u|sdw23|sdw23|7447|/data/primary447 +450|448|p|p|s|u|sdw23|sdw23|7448|/data/primary448 +451|449|p|p|s|u|sdw23|sdw23|7449|/data/primary449 +452|450|p|p|s|u|sdw23|sdw23|7450|/data/primary450 +453|451|p|p|s|u|sdw23|sdw23|7451|/data/primary451 +454|452|p|p|s|u|sdw23|sdw23|7452|/data/primary452 +455|453|p|p|s|u|sdw23|sdw23|7453|/data/primary453 +456|454|p|p|s|u|sdw23|sdw23|7454|/data/primary454 +457|455|p|p|s|u|sdw23|sdw23|7455|/data/primary455 +458|456|p|p|s|u|sdw23|sdw23|7456|/data/primary456 +459|457|p|p|s|u|sdw23|sdw23|7457|/data/primary457 +460|458|p|p|s|u|sdw23|sdw23|7458|/data/primary458 +461|459|p|p|s|u|sdw23|sdw23|7459|/data/primary459 +1422|420|m|m|s|u|sdw23|sdw23|8420|/data/mirror420 +1423|421|m|m|s|u|sdw23|sdw23|8421|/data/mirror421 +1424|422|m|m|s|u|sdw23|sdw23|8422|/data/mirror422 +1425|423|m|m|s|u|sdw23|sdw23|8423|/data/mirror423 +1426|424|m|m|s|u|sdw23|sdw23|8424|/data/mirror424 +1427|425|m|m|s|u|sdw23|sdw23|8425|/data/mirror425 +1428|426|m|m|s|u|sdw23|sdw23|8426|/data/mirror426 +1429|427|m|m|s|u|sdw23|sdw23|8427|/data/mirror427 +1430|428|m|m|s|u|sdw23|sdw23|8428|/data/mirror428 +1431|429|m|m|s|u|sdw23|sdw23|8429|/data/mirror429 +1432|430|m|m|s|u|sdw23|sdw23|8430|/data/mirror430 +1433|431|m|m|s|u|sdw23|sdw23|8431|/data/mirror431 +1434|432|m|m|s|u|sdw23|sdw23|8432|/data/mirror432 +1435|433|m|m|s|u|sdw23|sdw23|8433|/data/mirror433 +1436|434|m|m|s|u|sdw23|sdw23|8434|/data/mirror434 +1437|435|m|m|s|u|sdw23|sdw23|8435|/data/mirror435 +1438|436|m|m|s|u|sdw23|sdw23|8436|/data/mirror436 +1439|437|m|m|s|u|sdw23|sdw23|8437|/data/mirror437 +1440|438|m|m|s|u|sdw23|sdw23|8438|/data/mirror438 +1441|439|m|m|s|u|sdw23|sdw23|8439|/data/mirror439 +# SDW24 +462|460|p|p|s|u|sdw24|sdw24|7460|/data/primary460 +463|461|p|p|s|u|sdw24|sdw24|7461|/data/primary461 +464|462|p|p|s|u|sdw24|sdw24|7462|/data/primary462 +465|463|p|p|s|u|sdw24|sdw24|7463|/data/primary463 +466|464|p|p|s|u|sdw24|sdw24|7464|/data/primary464 +467|465|p|p|s|u|sdw24|sdw24|7465|/data/primary465 +468|466|p|p|s|u|sdw24|sdw24|7466|/data/primary466 +469|467|p|p|s|u|sdw24|sdw24|7467|/data/primary467 +470|468|p|p|s|u|sdw24|sdw24|7468|/data/primary468 +471|469|p|p|s|u|sdw24|sdw24|7469|/data/primary469 +472|470|p|p|s|u|sdw24|sdw24|7470|/data/primary470 +473|471|p|p|s|u|sdw24|sdw24|7471|/data/primary471 +474|472|p|p|s|u|sdw24|sdw24|7472|/data/primary472 +475|473|p|p|s|u|sdw24|sdw24|7473|/data/primary473 +476|474|p|p|s|u|sdw24|sdw24|7474|/data/primary474 +477|475|p|p|s|u|sdw24|sdw24|7475|/data/primary475 +478|476|p|p|s|u|sdw24|sdw24|7476|/data/primary476 +479|477|p|p|s|u|sdw24|sdw24|7477|/data/primary477 +480|478|p|p|s|u|sdw24|sdw24|7478|/data/primary478 +481|479|p|p|s|u|sdw24|sdw24|7479|/data/primary479 +1442|440|m|m|s|u|sdw24|sdw24|8440|/data/mirror440 +1443|441|m|m|s|u|sdw24|sdw24|8441|/data/mirror441 +1444|442|m|m|s|u|sdw24|sdw24|8442|/data/mirror442 +1445|443|m|m|s|u|sdw24|sdw24|8443|/data/mirror443 +1446|444|m|m|s|u|sdw24|sdw24|8444|/data/mirror444 +1447|445|m|m|s|u|sdw24|sdw24|8445|/data/mirror445 +1448|446|m|m|s|u|sdw24|sdw24|8446|/data/mirror446 +1449|447|m|m|s|u|sdw24|sdw24|8447|/data/mirror447 +1450|448|m|m|s|u|sdw24|sdw24|8448|/data/mirror448 +1451|449|m|m|s|u|sdw24|sdw24|8449|/data/mirror449 +1452|450|m|m|s|u|sdw24|sdw24|8450|/data/mirror450 +1453|451|m|m|s|u|sdw24|sdw24|8451|/data/mirror451 +1454|452|m|m|s|u|sdw24|sdw24|8452|/data/mirror452 +1455|453|m|m|s|u|sdw24|sdw24|8453|/data/mirror453 +1456|454|m|m|s|u|sdw24|sdw24|8454|/data/mirror454 +1457|455|m|m|s|u|sdw24|sdw24|8455|/data/mirror455 +1458|456|m|m|s|u|sdw24|sdw24|8456|/data/mirror456 +1459|457|m|m|s|u|sdw24|sdw24|8457|/data/mirror457 +1460|458|m|m|s|u|sdw24|sdw24|8458|/data/mirror458 +1461|459|m|m|s|u|sdw24|sdw24|8459|/data/mirror459 +# SDW25 +482|480|p|p|s|u|sdw25|sdw25|7480|/data/primary480 +483|481|p|p|s|u|sdw25|sdw25|7481|/data/primary481 +484|482|p|p|s|u|sdw25|sdw25|7482|/data/primary482 +485|483|p|p|s|u|sdw25|sdw25|7483|/data/primary483 +486|484|p|p|s|u|sdw25|sdw25|7484|/data/primary484 +487|485|p|p|s|u|sdw25|sdw25|7485|/data/primary485 +488|486|p|p|s|u|sdw25|sdw25|7486|/data/primary486 +489|487|p|p|s|u|sdw25|sdw25|7487|/data/primary487 +490|488|p|p|s|u|sdw25|sdw25|7488|/data/primary488 +491|489|p|p|s|u|sdw25|sdw25|7489|/data/primary489 +492|490|p|p|s|u|sdw25|sdw25|7490|/data/primary490 +493|491|p|p|s|u|sdw25|sdw25|7491|/data/primary491 +494|492|p|p|s|u|sdw25|sdw25|7492|/data/primary492 +495|493|p|p|s|u|sdw25|sdw25|7493|/data/primary493 +496|494|p|p|s|u|sdw25|sdw25|7494|/data/primary494 +497|495|p|p|s|u|sdw25|sdw25|7495|/data/primary495 +498|496|p|p|s|u|sdw25|sdw25|7496|/data/primary496 +499|497|p|p|s|u|sdw25|sdw25|7497|/data/primary497 +500|498|p|p|s|u|sdw25|sdw25|7498|/data/primary498 +501|499|p|p|s|u|sdw25|sdw25|7499|/data/primary499 +1462|460|m|m|s|u|sdw25|sdw25|8460|/data/mirror460 +1463|461|m|m|s|u|sdw25|sdw25|8461|/data/mirror461 +1464|462|m|m|s|u|sdw25|sdw25|8462|/data/mirror462 +1465|463|m|m|s|u|sdw25|sdw25|8463|/data/mirror463 +1466|464|m|m|s|u|sdw25|sdw25|8464|/data/mirror464 +1467|465|m|m|s|u|sdw25|sdw25|8465|/data/mirror465 +1468|466|m|m|s|u|sdw25|sdw25|8466|/data/mirror466 +1469|467|m|m|s|u|sdw25|sdw25|8467|/data/mirror467 +1470|468|m|m|s|u|sdw25|sdw25|8468|/data/mirror468 +1471|469|m|m|s|u|sdw25|sdw25|8469|/data/mirror469 +1472|470|m|m|s|u|sdw25|sdw25|8470|/data/mirror470 +1473|471|m|m|s|u|sdw25|sdw25|8471|/data/mirror471 +1474|472|m|m|s|u|sdw25|sdw25|8472|/data/mirror472 +1475|473|m|m|s|u|sdw25|sdw25|8473|/data/mirror473 +1476|474|m|m|s|u|sdw25|sdw25|8474|/data/mirror474 +1477|475|m|m|s|u|sdw25|sdw25|8475|/data/mirror475 +1478|476|m|m|s|u|sdw25|sdw25|8476|/data/mirror476 +1479|477|m|m|s|u|sdw25|sdw25|8477|/data/mirror477 +1480|478|m|m|s|u|sdw25|sdw25|8478|/data/mirror478 +1481|479|m|m|s|u|sdw25|sdw25|8479|/data/mirror479 +# SDW26 +502|500|p|p|s|u|sdw26|sdw26|7500|/data/primary500 +503|501|p|p|s|u|sdw26|sdw26|7501|/data/primary501 +504|502|p|p|s|u|sdw26|sdw26|7502|/data/primary502 +505|503|p|p|s|u|sdw26|sdw26|7503|/data/primary503 +506|504|p|p|s|u|sdw26|sdw26|7504|/data/primary504 +507|505|p|p|s|u|sdw26|sdw26|7505|/data/primary505 +508|506|p|p|s|u|sdw26|sdw26|7506|/data/primary506 +509|507|p|p|s|u|sdw26|sdw26|7507|/data/primary507 +510|508|p|p|s|u|sdw26|sdw26|7508|/data/primary508 +511|509|p|p|s|u|sdw26|sdw26|7509|/data/primary509 +512|510|p|p|s|u|sdw26|sdw26|7510|/data/primary510 +513|511|p|p|s|u|sdw26|sdw26|7511|/data/primary511 +514|512|p|p|s|u|sdw26|sdw26|7512|/data/primary512 +515|513|p|p|s|u|sdw26|sdw26|7513|/data/primary513 +516|514|p|p|s|u|sdw26|sdw26|7514|/data/primary514 +517|515|p|p|s|u|sdw26|sdw26|7515|/data/primary515 +518|516|p|p|s|u|sdw26|sdw26|7516|/data/primary516 +519|517|p|p|s|u|sdw26|sdw26|7517|/data/primary517 +520|518|p|p|s|u|sdw26|sdw26|7518|/data/primary518 +521|519|p|p|s|u|sdw26|sdw26|7519|/data/primary519 +1482|480|m|m|s|u|sdw26|sdw26|8480|/data/mirror480 +1483|481|m|m|s|u|sdw26|sdw26|8481|/data/mirror481 +1484|482|m|m|s|u|sdw26|sdw26|8482|/data/mirror482 +1485|483|m|m|s|u|sdw26|sdw26|8483|/data/mirror483 +1486|484|m|m|s|u|sdw26|sdw26|8484|/data/mirror484 +1487|485|m|m|s|u|sdw26|sdw26|8485|/data/mirror485 +1488|486|m|m|s|u|sdw26|sdw26|8486|/data/mirror486 +1489|487|m|m|s|u|sdw26|sdw26|8487|/data/mirror487 +1490|488|m|m|s|u|sdw26|sdw26|8488|/data/mirror488 +1491|489|m|m|s|u|sdw26|sdw26|8489|/data/mirror489 +1492|490|m|m|s|u|sdw26|sdw26|8490|/data/mirror490 +1493|491|m|m|s|u|sdw26|sdw26|8491|/data/mirror491 +1494|492|m|m|s|u|sdw26|sdw26|8492|/data/mirror492 +1495|493|m|m|s|u|sdw26|sdw26|8493|/data/mirror493 +1496|494|m|m|s|u|sdw26|sdw26|8494|/data/mirror494 +1497|495|m|m|s|u|sdw26|sdw26|8495|/data/mirror495 +1498|496|m|m|s|u|sdw26|sdw26|8496|/data/mirror496 +1499|497|m|m|s|u|sdw26|sdw26|8497|/data/mirror497 +1500|498|m|m|s|u|sdw26|sdw26|8498|/data/mirror498 +1501|499|m|m|s|u|sdw26|sdw26|8499|/data/mirror499 +# SDW27 +522|520|p|p|s|u|sdw27|sdw27|7520|/data/primary520 +523|521|p|p|s|u|sdw27|sdw27|7521|/data/primary521 +524|522|p|p|s|u|sdw27|sdw27|7522|/data/primary522 +525|523|p|p|s|u|sdw27|sdw27|7523|/data/primary523 +526|524|p|p|s|u|sdw27|sdw27|7524|/data/primary524 +527|525|p|p|s|u|sdw27|sdw27|7525|/data/primary525 +528|526|p|p|s|u|sdw27|sdw27|7526|/data/primary526 +529|527|p|p|s|u|sdw27|sdw27|7527|/data/primary527 +530|528|p|p|s|u|sdw27|sdw27|7528|/data/primary528 +531|529|p|p|s|u|sdw27|sdw27|7529|/data/primary529 +532|530|p|p|s|u|sdw27|sdw27|7530|/data/primary530 +533|531|p|p|s|u|sdw27|sdw27|7531|/data/primary531 +534|532|p|p|s|u|sdw27|sdw27|7532|/data/primary532 +535|533|p|p|s|u|sdw27|sdw27|7533|/data/primary533 +536|534|p|p|s|u|sdw27|sdw27|7534|/data/primary534 +537|535|p|p|s|u|sdw27|sdw27|7535|/data/primary535 +538|536|p|p|s|u|sdw27|sdw27|7536|/data/primary536 +539|537|p|p|s|u|sdw27|sdw27|7537|/data/primary537 +540|538|p|p|s|u|sdw27|sdw27|7538|/data/primary538 +541|539|p|p|s|u|sdw27|sdw27|7539|/data/primary539 +1502|500|m|m|s|u|sdw27|sdw27|8500|/data/mirror500 +1503|501|m|m|s|u|sdw27|sdw27|8501|/data/mirror501 +1504|502|m|m|s|u|sdw27|sdw27|8502|/data/mirror502 +1505|503|m|m|s|u|sdw27|sdw27|8503|/data/mirror503 +1506|504|m|m|s|u|sdw27|sdw27|8504|/data/mirror504 +1507|505|m|m|s|u|sdw27|sdw27|8505|/data/mirror505 +1508|506|m|m|s|u|sdw27|sdw27|8506|/data/mirror506 +1509|507|m|m|s|u|sdw27|sdw27|8507|/data/mirror507 +1510|508|m|m|s|u|sdw27|sdw27|8508|/data/mirror508 +1511|509|m|m|s|u|sdw27|sdw27|8509|/data/mirror509 +1512|510|m|m|s|u|sdw27|sdw27|8510|/data/mirror510 +1513|511|m|m|s|u|sdw27|sdw27|8511|/data/mirror511 +1514|512|m|m|s|u|sdw27|sdw27|8512|/data/mirror512 +1515|513|m|m|s|u|sdw27|sdw27|8513|/data/mirror513 +1516|514|m|m|s|u|sdw27|sdw27|8514|/data/mirror514 +1517|515|m|m|s|u|sdw27|sdw27|8515|/data/mirror515 +1518|516|m|m|s|u|sdw27|sdw27|8516|/data/mirror516 +1519|517|m|m|s|u|sdw27|sdw27|8517|/data/mirror517 +1520|518|m|m|s|u|sdw27|sdw27|8518|/data/mirror518 +1521|519|m|m|s|u|sdw27|sdw27|8519|/data/mirror519 +# SDW28 +542|540|p|p|s|u|sdw28|sdw28|7540|/data/primary540 +543|541|p|p|s|u|sdw28|sdw28|7541|/data/primary541 +544|542|p|p|s|u|sdw28|sdw28|7542|/data/primary542 +545|543|p|p|s|u|sdw28|sdw28|7543|/data/primary543 +546|544|p|p|s|u|sdw28|sdw28|7544|/data/primary544 +547|545|p|p|s|u|sdw28|sdw28|7545|/data/primary545 +548|546|p|p|s|u|sdw28|sdw28|7546|/data/primary546 +549|547|p|p|s|u|sdw28|sdw28|7547|/data/primary547 +550|548|p|p|s|u|sdw28|sdw28|7548|/data/primary548 +551|549|p|p|s|u|sdw28|sdw28|7549|/data/primary549 +552|550|p|p|s|u|sdw28|sdw28|7550|/data/primary550 +553|551|p|p|s|u|sdw28|sdw28|7551|/data/primary551 +554|552|p|p|s|u|sdw28|sdw28|7552|/data/primary552 +555|553|p|p|s|u|sdw28|sdw28|7553|/data/primary553 +556|554|p|p|s|u|sdw28|sdw28|7554|/data/primary554 +557|555|p|p|s|u|sdw28|sdw28|7555|/data/primary555 +558|556|p|p|s|u|sdw28|sdw28|7556|/data/primary556 +559|557|p|p|s|u|sdw28|sdw28|7557|/data/primary557 +560|558|p|p|s|u|sdw28|sdw28|7558|/data/primary558 +561|559|p|p|s|u|sdw28|sdw28|7559|/data/primary559 +1522|520|m|m|s|u|sdw28|sdw28|8520|/data/mirror520 +1523|521|m|m|s|u|sdw28|sdw28|8521|/data/mirror521 +1524|522|m|m|s|u|sdw28|sdw28|8522|/data/mirror522 +1525|523|m|m|s|u|sdw28|sdw28|8523|/data/mirror523 +1526|524|m|m|s|u|sdw28|sdw28|8524|/data/mirror524 +1527|525|m|m|s|u|sdw28|sdw28|8525|/data/mirror525 +1528|526|m|m|s|u|sdw28|sdw28|8526|/data/mirror526 +1529|527|m|m|s|u|sdw28|sdw28|8527|/data/mirror527 +1530|528|m|m|s|u|sdw28|sdw28|8528|/data/mirror528 +1531|529|m|m|s|u|sdw28|sdw28|8529|/data/mirror529 +1532|530|m|m|s|u|sdw28|sdw28|8530|/data/mirror530 +1533|531|m|m|s|u|sdw28|sdw28|8531|/data/mirror531 +1534|532|m|m|s|u|sdw28|sdw28|8532|/data/mirror532 +1535|533|m|m|s|u|sdw28|sdw28|8533|/data/mirror533 +1536|534|m|m|s|u|sdw28|sdw28|8534|/data/mirror534 +1537|535|m|m|s|u|sdw28|sdw28|8535|/data/mirror535 +1538|536|m|m|s|u|sdw28|sdw28|8536|/data/mirror536 +1539|537|m|m|s|u|sdw28|sdw28|8537|/data/mirror537 +1540|538|m|m|s|u|sdw28|sdw28|8538|/data/mirror538 +1541|539|m|m|s|u|sdw28|sdw28|8539|/data/mirror539 +# SDW29 +562|560|p|p|s|u|sdw29|sdw29|7560|/data/primary560 +563|561|p|p|s|u|sdw29|sdw29|7561|/data/primary561 +564|562|p|p|s|u|sdw29|sdw29|7562|/data/primary562 +565|563|p|p|s|u|sdw29|sdw29|7563|/data/primary563 +566|564|p|p|s|u|sdw29|sdw29|7564|/data/primary564 +567|565|p|p|s|u|sdw29|sdw29|7565|/data/primary565 +568|566|p|p|s|u|sdw29|sdw29|7566|/data/primary566 +569|567|p|p|s|u|sdw29|sdw29|7567|/data/primary567 +570|568|p|p|s|u|sdw29|sdw29|7568|/data/primary568 +571|569|p|p|s|u|sdw29|sdw29|7569|/data/primary569 +572|570|p|p|s|u|sdw29|sdw29|7570|/data/primary570 +573|571|p|p|s|u|sdw29|sdw29|7571|/data/primary571 +574|572|p|p|s|u|sdw29|sdw29|7572|/data/primary572 +575|573|p|p|s|u|sdw29|sdw29|7573|/data/primary573 +576|574|p|p|s|u|sdw29|sdw29|7574|/data/primary574 +577|575|p|p|s|u|sdw29|sdw29|7575|/data/primary575 +578|576|p|p|s|u|sdw29|sdw29|7576|/data/primary576 +579|577|p|p|s|u|sdw29|sdw29|7577|/data/primary577 +580|578|p|p|s|u|sdw29|sdw29|7578|/data/primary578 +581|579|p|p|s|u|sdw29|sdw29|7579|/data/primary579 +1542|540|m|m|s|u|sdw29|sdw29|8540|/data/mirror540 +1543|541|m|m|s|u|sdw29|sdw29|8541|/data/mirror541 +1544|542|m|m|s|u|sdw29|sdw29|8542|/data/mirror542 +1545|543|m|m|s|u|sdw29|sdw29|8543|/data/mirror543 +1546|544|m|m|s|u|sdw29|sdw29|8544|/data/mirror544 +1547|545|m|m|s|u|sdw29|sdw29|8545|/data/mirror545 +1548|546|m|m|s|u|sdw29|sdw29|8546|/data/mirror546 +1549|547|m|m|s|u|sdw29|sdw29|8547|/data/mirror547 +1550|548|m|m|s|u|sdw29|sdw29|8548|/data/mirror548 +1551|549|m|m|s|u|sdw29|sdw29|8549|/data/mirror549 +1552|550|m|m|s|u|sdw29|sdw29|8550|/data/mirror550 +1553|551|m|m|s|u|sdw29|sdw29|8551|/data/mirror551 +1554|552|m|m|s|u|sdw29|sdw29|8552|/data/mirror552 +1555|553|m|m|s|u|sdw29|sdw29|8553|/data/mirror553 +1556|554|m|m|s|u|sdw29|sdw29|8554|/data/mirror554 +1557|555|m|m|s|u|sdw29|sdw29|8555|/data/mirror555 +1558|556|m|m|s|u|sdw29|sdw29|8556|/data/mirror556 +1559|557|m|m|s|u|sdw29|sdw29|8557|/data/mirror557 +1560|558|m|m|s|u|sdw29|sdw29|8558|/data/mirror558 +1561|559|m|m|s|u|sdw29|sdw29|8559|/data/mirror559 +# SDW3 +42|40|p|p|s|u|sdw3|sdw3|7040|/data/primary40 +43|41|p|p|s|u|sdw3|sdw3|7041|/data/primary41 +44|42|p|p|s|u|sdw3|sdw3|7042|/data/primary42 +45|43|p|p|s|u|sdw3|sdw3|7043|/data/primary43 +46|44|p|p|s|u|sdw3|sdw3|7044|/data/primary44 +47|45|p|p|s|u|sdw3|sdw3|7045|/data/primary45 +48|46|p|p|s|u|sdw3|sdw3|7046|/data/primary46 +49|47|p|p|s|u|sdw3|sdw3|7047|/data/primary47 +50|48|p|p|s|u|sdw3|sdw3|7048|/data/primary48 +51|49|p|p|s|u|sdw3|sdw3|7049|/data/primary49 +52|50|p|p|s|u|sdw3|sdw3|7050|/data/primary50 +53|51|p|p|s|u|sdw3|sdw3|7051|/data/primary51 +54|52|p|p|s|u|sdw3|sdw3|7052|/data/primary52 +55|53|p|p|s|u|sdw3|sdw3|7053|/data/primary53 +56|54|p|p|s|u|sdw3|sdw3|7054|/data/primary54 +57|55|p|p|s|u|sdw3|sdw3|7055|/data/primary55 +58|56|p|p|s|u|sdw3|sdw3|7056|/data/primary56 +59|57|p|p|s|u|sdw3|sdw3|7057|/data/primary57 +60|58|p|p|s|u|sdw3|sdw3|7058|/data/primary58 +61|59|p|p|s|u|sdw3|sdw3|7059|/data/primary59 +1022|20|m|m|s|u|sdw3|sdw3|8020|/data/mirror20 +1023|21|m|m|s|u|sdw3|sdw3|8021|/data/mirror21 +1024|22|m|m|s|u|sdw3|sdw3|8022|/data/mirror22 +1025|23|m|m|s|u|sdw3|sdw3|8023|/data/mirror23 +1026|24|m|m|s|u|sdw3|sdw3|8024|/data/mirror24 +1027|25|m|m|s|u|sdw3|sdw3|8025|/data/mirror25 +1028|26|m|m|s|u|sdw3|sdw3|8026|/data/mirror26 +1029|27|m|m|s|u|sdw3|sdw3|8027|/data/mirror27 +1030|28|m|m|s|u|sdw3|sdw3|8028|/data/mirror28 +1031|29|m|m|s|u|sdw3|sdw3|8029|/data/mirror29 +1032|30|m|m|s|u|sdw3|sdw3|8030|/data/mirror30 +1033|31|m|m|s|u|sdw3|sdw3|8031|/data/mirror31 +1034|32|m|m|s|u|sdw3|sdw3|8032|/data/mirror32 +1035|33|m|m|s|u|sdw3|sdw3|8033|/data/mirror33 +1036|34|m|m|s|u|sdw3|sdw3|8034|/data/mirror34 +1037|35|m|m|s|u|sdw3|sdw3|8035|/data/mirror35 +1038|36|m|m|s|u|sdw3|sdw3|8036|/data/mirror36 +1039|37|m|m|s|u|sdw3|sdw3|8037|/data/mirror37 +1040|38|m|m|s|u|sdw3|sdw3|8038|/data/mirror38 +1041|39|m|m|s|u|sdw3|sdw3|8039|/data/mirror39 +# SDW30 +582|580|p|p|s|u|sdw30|sdw30|7580|/data/primary580 +583|581|p|p|s|u|sdw30|sdw30|7581|/data/primary581 +584|582|p|p|s|u|sdw30|sdw30|7582|/data/primary582 +585|583|p|p|s|u|sdw30|sdw30|7583|/data/primary583 +586|584|p|p|s|u|sdw30|sdw30|7584|/data/primary584 +587|585|p|p|s|u|sdw30|sdw30|7585|/data/primary585 +588|586|p|p|s|u|sdw30|sdw30|7586|/data/primary586 +589|587|p|p|s|u|sdw30|sdw30|7587|/data/primary587 +590|588|p|p|s|u|sdw30|sdw30|7588|/data/primary588 +591|589|p|p|s|u|sdw30|sdw30|7589|/data/primary589 +592|590|p|p|s|u|sdw30|sdw30|7590|/data/primary590 +593|591|p|p|s|u|sdw30|sdw30|7591|/data/primary591 +594|592|p|p|s|u|sdw30|sdw30|7592|/data/primary592 +595|593|p|p|s|u|sdw30|sdw30|7593|/data/primary593 +596|594|p|p|s|u|sdw30|sdw30|7594|/data/primary594 +597|595|p|p|s|u|sdw30|sdw30|7595|/data/primary595 +598|596|p|p|s|u|sdw30|sdw30|7596|/data/primary596 +599|597|p|p|s|u|sdw30|sdw30|7597|/data/primary597 +600|598|p|p|s|u|sdw30|sdw30|7598|/data/primary598 +601|599|p|p|s|u|sdw30|sdw30|7599|/data/primary599 +1562|560|m|m|s|u|sdw30|sdw30|8560|/data/mirror560 +1563|561|m|m|s|u|sdw30|sdw30|8561|/data/mirror561 +1564|562|m|m|s|u|sdw30|sdw30|8562|/data/mirror562 +1565|563|m|m|s|u|sdw30|sdw30|8563|/data/mirror563 +1566|564|m|m|s|u|sdw30|sdw30|8564|/data/mirror564 +1567|565|m|m|s|u|sdw30|sdw30|8565|/data/mirror565 +1568|566|m|m|s|u|sdw30|sdw30|8566|/data/mirror566 +1569|567|m|m|s|u|sdw30|sdw30|8567|/data/mirror567 +1570|568|m|m|s|u|sdw30|sdw30|8568|/data/mirror568 +1571|569|m|m|s|u|sdw30|sdw30|8569|/data/mirror569 +1572|570|m|m|s|u|sdw30|sdw30|8570|/data/mirror570 +1573|571|m|m|s|u|sdw30|sdw30|8571|/data/mirror571 +1574|572|m|m|s|u|sdw30|sdw30|8572|/data/mirror572 +1575|573|m|m|s|u|sdw30|sdw30|8573|/data/mirror573 +1576|574|m|m|s|u|sdw30|sdw30|8574|/data/mirror574 +1577|575|m|m|s|u|sdw30|sdw30|8575|/data/mirror575 +1578|576|m|m|s|u|sdw30|sdw30|8576|/data/mirror576 +1579|577|m|m|s|u|sdw30|sdw30|8577|/data/mirror577 +1580|578|m|m|s|u|sdw30|sdw30|8578|/data/mirror578 +1581|579|m|m|s|u|sdw30|sdw30|8579|/data/mirror579 +# SDW31 +602|600|p|p|s|u|sdw31|sdw31|7600|/data/primary600 +603|601|p|p|s|u|sdw31|sdw31|7601|/data/primary601 +604|602|p|p|s|u|sdw31|sdw31|7602|/data/primary602 +605|603|p|p|s|u|sdw31|sdw31|7603|/data/primary603 +606|604|p|p|s|u|sdw31|sdw31|7604|/data/primary604 +607|605|p|p|s|u|sdw31|sdw31|7605|/data/primary605 +608|606|p|p|s|u|sdw31|sdw31|7606|/data/primary606 +609|607|p|p|s|u|sdw31|sdw31|7607|/data/primary607 +610|608|p|p|s|u|sdw31|sdw31|7608|/data/primary608 +611|609|p|p|s|u|sdw31|sdw31|7609|/data/primary609 +612|610|p|p|s|u|sdw31|sdw31|7610|/data/primary610 +613|611|p|p|s|u|sdw31|sdw31|7611|/data/primary611 +614|612|p|p|s|u|sdw31|sdw31|7612|/data/primary612 +615|613|p|p|s|u|sdw31|sdw31|7613|/data/primary613 +616|614|p|p|s|u|sdw31|sdw31|7614|/data/primary614 +617|615|p|p|s|u|sdw31|sdw31|7615|/data/primary615 +618|616|p|p|s|u|sdw31|sdw31|7616|/data/primary616 +619|617|p|p|s|u|sdw31|sdw31|7617|/data/primary617 +620|618|p|p|s|u|sdw31|sdw31|7618|/data/primary618 +621|619|p|p|s|u|sdw31|sdw31|7619|/data/primary619 +1582|580|m|m|s|u|sdw31|sdw31|8580|/data/mirror580 +1583|581|m|m|s|u|sdw31|sdw31|8581|/data/mirror581 +1584|582|m|m|s|u|sdw31|sdw31|8582|/data/mirror582 +1585|583|m|m|s|u|sdw31|sdw31|8583|/data/mirror583 +1586|584|m|m|s|u|sdw31|sdw31|8584|/data/mirror584 +1587|585|m|m|s|u|sdw31|sdw31|8585|/data/mirror585 +1588|586|m|m|s|u|sdw31|sdw31|8586|/data/mirror586 +1589|587|m|m|s|u|sdw31|sdw31|8587|/data/mirror587 +1590|588|m|m|s|u|sdw31|sdw31|8588|/data/mirror588 +1591|589|m|m|s|u|sdw31|sdw31|8589|/data/mirror589 +1592|590|m|m|s|u|sdw31|sdw31|8590|/data/mirror590 +1593|591|m|m|s|u|sdw31|sdw31|8591|/data/mirror591 +1594|592|m|m|s|u|sdw31|sdw31|8592|/data/mirror592 +1595|593|m|m|s|u|sdw31|sdw31|8593|/data/mirror593 +1596|594|m|m|s|u|sdw31|sdw31|8594|/data/mirror594 +1597|595|m|m|s|u|sdw31|sdw31|8595|/data/mirror595 +1598|596|m|m|s|u|sdw31|sdw31|8596|/data/mirror596 +1599|597|m|m|s|u|sdw31|sdw31|8597|/data/mirror597 +1600|598|m|m|s|u|sdw31|sdw31|8598|/data/mirror598 +1601|599|m|m|s|u|sdw31|sdw31|8599|/data/mirror599 +# SDW32 +622|620|p|p|s|u|sdw32|sdw32|7620|/data/primary620 +623|621|p|p|s|u|sdw32|sdw32|7621|/data/primary621 +624|622|p|p|s|u|sdw32|sdw32|7622|/data/primary622 +625|623|p|p|s|u|sdw32|sdw32|7623|/data/primary623 +626|624|p|p|s|u|sdw32|sdw32|7624|/data/primary624 +627|625|p|p|s|u|sdw32|sdw32|7625|/data/primary625 +628|626|p|p|s|u|sdw32|sdw32|7626|/data/primary626 +629|627|p|p|s|u|sdw32|sdw32|7627|/data/primary627 +630|628|p|p|s|u|sdw32|sdw32|7628|/data/primary628 +631|629|p|p|s|u|sdw32|sdw32|7629|/data/primary629 +632|630|p|p|s|u|sdw32|sdw32|7630|/data/primary630 +633|631|p|p|s|u|sdw32|sdw32|7631|/data/primary631 +634|632|p|p|s|u|sdw32|sdw32|7632|/data/primary632 +635|633|p|p|s|u|sdw32|sdw32|7633|/data/primary633 +636|634|p|p|s|u|sdw32|sdw32|7634|/data/primary634 +637|635|p|p|s|u|sdw32|sdw32|7635|/data/primary635 +638|636|p|p|s|u|sdw32|sdw32|7636|/data/primary636 +639|637|p|p|s|u|sdw32|sdw32|7637|/data/primary637 +640|638|p|p|s|u|sdw32|sdw32|7638|/data/primary638 +641|639|p|p|s|u|sdw32|sdw32|7639|/data/primary639 +1602|600|m|m|s|u|sdw32|sdw32|8600|/data/mirror600 +1603|601|m|m|s|u|sdw32|sdw32|8601|/data/mirror601 +1604|602|m|m|s|u|sdw32|sdw32|8602|/data/mirror602 +1605|603|m|m|s|u|sdw32|sdw32|8603|/data/mirror603 +1606|604|m|m|s|u|sdw32|sdw32|8604|/data/mirror604 +1607|605|m|m|s|u|sdw32|sdw32|8605|/data/mirror605 +1608|606|m|m|s|u|sdw32|sdw32|8606|/data/mirror606 +1609|607|m|m|s|u|sdw32|sdw32|8607|/data/mirror607 +1610|608|m|m|s|u|sdw32|sdw32|8608|/data/mirror608 +1611|609|m|m|s|u|sdw32|sdw32|8609|/data/mirror609 +1612|610|m|m|s|u|sdw32|sdw32|8610|/data/mirror610 +1613|611|m|m|s|u|sdw32|sdw32|8611|/data/mirror611 +1614|612|m|m|s|u|sdw32|sdw32|8612|/data/mirror612 +1615|613|m|m|s|u|sdw32|sdw32|8613|/data/mirror613 +1616|614|m|m|s|u|sdw32|sdw32|8614|/data/mirror614 +1617|615|m|m|s|u|sdw32|sdw32|8615|/data/mirror615 +1618|616|m|m|s|u|sdw32|sdw32|8616|/data/mirror616 +1619|617|m|m|s|u|sdw32|sdw32|8617|/data/mirror617 +1620|618|m|m|s|u|sdw32|sdw32|8618|/data/mirror618 +1621|619|m|m|s|u|sdw32|sdw32|8619|/data/mirror619 +# SDW33 +642|640|p|p|s|u|sdw33|sdw33|7640|/data/primary640 +643|641|p|p|s|u|sdw33|sdw33|7641|/data/primary641 +644|642|p|p|s|u|sdw33|sdw33|7642|/data/primary642 +645|643|p|p|s|u|sdw33|sdw33|7643|/data/primary643 +646|644|p|p|s|u|sdw33|sdw33|7644|/data/primary644 +647|645|p|p|s|u|sdw33|sdw33|7645|/data/primary645 +648|646|p|p|s|u|sdw33|sdw33|7646|/data/primary646 +649|647|p|p|s|u|sdw33|sdw33|7647|/data/primary647 +650|648|p|p|s|u|sdw33|sdw33|7648|/data/primary648 +651|649|p|p|s|u|sdw33|sdw33|7649|/data/primary649 +652|650|p|p|s|u|sdw33|sdw33|7650|/data/primary650 +653|651|p|p|s|u|sdw33|sdw33|7651|/data/primary651 +654|652|p|p|s|u|sdw33|sdw33|7652|/data/primary652 +655|653|p|p|s|u|sdw33|sdw33|7653|/data/primary653 +656|654|p|p|s|u|sdw33|sdw33|7654|/data/primary654 +657|655|p|p|s|u|sdw33|sdw33|7655|/data/primary655 +658|656|p|p|s|u|sdw33|sdw33|7656|/data/primary656 +659|657|p|p|s|u|sdw33|sdw33|7657|/data/primary657 +660|658|p|p|s|u|sdw33|sdw33|7658|/data/primary658 +661|659|p|p|s|u|sdw33|sdw33|7659|/data/primary659 +1622|620|m|m|s|u|sdw33|sdw33|8620|/data/mirror620 +1623|621|m|m|s|u|sdw33|sdw33|8621|/data/mirror621 +1624|622|m|m|s|u|sdw33|sdw33|8622|/data/mirror622 +1625|623|m|m|s|u|sdw33|sdw33|8623|/data/mirror623 +1626|624|m|m|s|u|sdw33|sdw33|8624|/data/mirror624 +1627|625|m|m|s|u|sdw33|sdw33|8625|/data/mirror625 +1628|626|m|m|s|u|sdw33|sdw33|8626|/data/mirror626 +1629|627|m|m|s|u|sdw33|sdw33|8627|/data/mirror627 +1630|628|m|m|s|u|sdw33|sdw33|8628|/data/mirror628 +1631|629|m|m|s|u|sdw33|sdw33|8629|/data/mirror629 +1632|630|m|m|s|u|sdw33|sdw33|8630|/data/mirror630 +1633|631|m|m|s|u|sdw33|sdw33|8631|/data/mirror631 +1634|632|m|m|s|u|sdw33|sdw33|8632|/data/mirror632 +1635|633|m|m|s|u|sdw33|sdw33|8633|/data/mirror633 +1636|634|m|m|s|u|sdw33|sdw33|8634|/data/mirror634 +1637|635|m|m|s|u|sdw33|sdw33|8635|/data/mirror635 +1638|636|m|m|s|u|sdw33|sdw33|8636|/data/mirror636 +1639|637|m|m|s|u|sdw33|sdw33|8637|/data/mirror637 +1640|638|m|m|s|u|sdw33|sdw33|8638|/data/mirror638 +1641|639|m|m|s|u|sdw33|sdw33|8639|/data/mirror639 +# SDW34 +662|660|p|p|s|u|sdw34|sdw34|7660|/data/primary660 +663|661|p|p|s|u|sdw34|sdw34|7661|/data/primary661 +664|662|p|p|s|u|sdw34|sdw34|7662|/data/primary662 +665|663|p|p|s|u|sdw34|sdw34|7663|/data/primary663 +666|664|p|p|s|u|sdw34|sdw34|7664|/data/primary664 +667|665|p|p|s|u|sdw34|sdw34|7665|/data/primary665 +668|666|p|p|s|u|sdw34|sdw34|7666|/data/primary666 +669|667|p|p|s|u|sdw34|sdw34|7667|/data/primary667 +670|668|p|p|s|u|sdw34|sdw34|7668|/data/primary668 +671|669|p|p|s|u|sdw34|sdw34|7669|/data/primary669 +672|670|p|p|s|u|sdw34|sdw34|7670|/data/primary670 +673|671|p|p|s|u|sdw34|sdw34|7671|/data/primary671 +674|672|p|p|s|u|sdw34|sdw34|7672|/data/primary672 +675|673|p|p|s|u|sdw34|sdw34|7673|/data/primary673 +676|674|p|p|s|u|sdw34|sdw34|7674|/data/primary674 +677|675|p|p|s|u|sdw34|sdw34|7675|/data/primary675 +678|676|p|p|s|u|sdw34|sdw34|7676|/data/primary676 +679|677|p|p|s|u|sdw34|sdw34|7677|/data/primary677 +680|678|p|p|s|u|sdw34|sdw34|7678|/data/primary678 +681|679|p|p|s|u|sdw34|sdw34|7679|/data/primary679 +1642|640|m|m|s|u|sdw34|sdw34|8640|/data/mirror640 +1643|641|m|m|s|u|sdw34|sdw34|8641|/data/mirror641 +1644|642|m|m|s|u|sdw34|sdw34|8642|/data/mirror642 +1645|643|m|m|s|u|sdw34|sdw34|8643|/data/mirror643 +1646|644|m|m|s|u|sdw34|sdw34|8644|/data/mirror644 +1647|645|m|m|s|u|sdw34|sdw34|8645|/data/mirror645 +1648|646|m|m|s|u|sdw34|sdw34|8646|/data/mirror646 +1649|647|m|m|s|u|sdw34|sdw34|8647|/data/mirror647 +1650|648|m|m|s|u|sdw34|sdw34|8648|/data/mirror648 +1651|649|m|m|s|u|sdw34|sdw34|8649|/data/mirror649 +1652|650|m|m|s|u|sdw34|sdw34|8650|/data/mirror650 +1653|651|m|m|s|u|sdw34|sdw34|8651|/data/mirror651 +1654|652|m|m|s|u|sdw34|sdw34|8652|/data/mirror652 +1655|653|m|m|s|u|sdw34|sdw34|8653|/data/mirror653 +1656|654|m|m|s|u|sdw34|sdw34|8654|/data/mirror654 +1657|655|m|m|s|u|sdw34|sdw34|8655|/data/mirror655 +1658|656|m|m|s|u|sdw34|sdw34|8656|/data/mirror656 +1659|657|m|m|s|u|sdw34|sdw34|8657|/data/mirror657 +1660|658|m|m|s|u|sdw34|sdw34|8658|/data/mirror658 +1661|659|m|m|s|u|sdw34|sdw34|8659|/data/mirror659 +# SDW35 +682|680|p|p|s|u|sdw35|sdw35|7680|/data/primary680 +683|681|p|p|s|u|sdw35|sdw35|7681|/data/primary681 +684|682|p|p|s|u|sdw35|sdw35|7682|/data/primary682 +685|683|p|p|s|u|sdw35|sdw35|7683|/data/primary683 +686|684|p|p|s|u|sdw35|sdw35|7684|/data/primary684 +687|685|p|p|s|u|sdw35|sdw35|7685|/data/primary685 +688|686|p|p|s|u|sdw35|sdw35|7686|/data/primary686 +689|687|p|p|s|u|sdw35|sdw35|7687|/data/primary687 +690|688|p|p|s|u|sdw35|sdw35|7688|/data/primary688 +691|689|p|p|s|u|sdw35|sdw35|7689|/data/primary689 +692|690|p|p|s|u|sdw35|sdw35|7690|/data/primary690 +693|691|p|p|s|u|sdw35|sdw35|7691|/data/primary691 +694|692|p|p|s|u|sdw35|sdw35|7692|/data/primary692 +695|693|p|p|s|u|sdw35|sdw35|7693|/data/primary693 +696|694|p|p|s|u|sdw35|sdw35|7694|/data/primary694 +697|695|p|p|s|u|sdw35|sdw35|7695|/data/primary695 +698|696|p|p|s|u|sdw35|sdw35|7696|/data/primary696 +699|697|p|p|s|u|sdw35|sdw35|7697|/data/primary697 +700|698|p|p|s|u|sdw35|sdw35|7698|/data/primary698 +701|699|p|p|s|u|sdw35|sdw35|7699|/data/primary699 +1662|660|m|m|s|u|sdw35|sdw35|8660|/data/mirror660 +1663|661|m|m|s|u|sdw35|sdw35|8661|/data/mirror661 +1664|662|m|m|s|u|sdw35|sdw35|8662|/data/mirror662 +1665|663|m|m|s|u|sdw35|sdw35|8663|/data/mirror663 +1666|664|m|m|s|u|sdw35|sdw35|8664|/data/mirror664 +1667|665|m|m|s|u|sdw35|sdw35|8665|/data/mirror665 +1668|666|m|m|s|u|sdw35|sdw35|8666|/data/mirror666 +1669|667|m|m|s|u|sdw35|sdw35|8667|/data/mirror667 +1670|668|m|m|s|u|sdw35|sdw35|8668|/data/mirror668 +1671|669|m|m|s|u|sdw35|sdw35|8669|/data/mirror669 +1672|670|m|m|s|u|sdw35|sdw35|8670|/data/mirror670 +1673|671|m|m|s|u|sdw35|sdw35|8671|/data/mirror671 +1674|672|m|m|s|u|sdw35|sdw35|8672|/data/mirror672 +1675|673|m|m|s|u|sdw35|sdw35|8673|/data/mirror673 +1676|674|m|m|s|u|sdw35|sdw35|8674|/data/mirror674 +1677|675|m|m|s|u|sdw35|sdw35|8675|/data/mirror675 +1678|676|m|m|s|u|sdw35|sdw35|8676|/data/mirror676 +1679|677|m|m|s|u|sdw35|sdw35|8677|/data/mirror677 +1680|678|m|m|s|u|sdw35|sdw35|8678|/data/mirror678 +1681|679|m|m|s|u|sdw35|sdw35|8679|/data/mirror679 +# SDW36 +702|700|p|p|s|u|sdw36|sdw36|7700|/data/primary700 +703|701|p|p|s|u|sdw36|sdw36|7701|/data/primary701 +704|702|p|p|s|u|sdw36|sdw36|7702|/data/primary702 +705|703|p|p|s|u|sdw36|sdw36|7703|/data/primary703 +706|704|p|p|s|u|sdw36|sdw36|7704|/data/primary704 +707|705|p|p|s|u|sdw36|sdw36|7705|/data/primary705 +708|706|p|p|s|u|sdw36|sdw36|7706|/data/primary706 +709|707|p|p|s|u|sdw36|sdw36|7707|/data/primary707 +710|708|p|p|s|u|sdw36|sdw36|7708|/data/primary708 +711|709|p|p|s|u|sdw36|sdw36|7709|/data/primary709 +712|710|p|p|s|u|sdw36|sdw36|7710|/data/primary710 +713|711|p|p|s|u|sdw36|sdw36|7711|/data/primary711 +714|712|p|p|s|u|sdw36|sdw36|7712|/data/primary712 +715|713|p|p|s|u|sdw36|sdw36|7713|/data/primary713 +716|714|p|p|s|u|sdw36|sdw36|7714|/data/primary714 +717|715|p|p|s|u|sdw36|sdw36|7715|/data/primary715 +718|716|p|p|s|u|sdw36|sdw36|7716|/data/primary716 +719|717|p|p|s|u|sdw36|sdw36|7717|/data/primary717 +720|718|p|p|s|u|sdw36|sdw36|7718|/data/primary718 +721|719|p|p|s|u|sdw36|sdw36|7719|/data/primary719 +1682|680|m|m|s|u|sdw36|sdw36|8680|/data/mirror680 +1683|681|m|m|s|u|sdw36|sdw36|8681|/data/mirror681 +1684|682|m|m|s|u|sdw36|sdw36|8682|/data/mirror682 +1685|683|m|m|s|u|sdw36|sdw36|8683|/data/mirror683 +1686|684|m|m|s|u|sdw36|sdw36|8684|/data/mirror684 +1687|685|m|m|s|u|sdw36|sdw36|8685|/data/mirror685 +1688|686|m|m|s|u|sdw36|sdw36|8686|/data/mirror686 +1689|687|m|m|s|u|sdw36|sdw36|8687|/data/mirror687 +1690|688|m|m|s|u|sdw36|sdw36|8688|/data/mirror688 +1691|689|m|m|s|u|sdw36|sdw36|8689|/data/mirror689 +1692|690|m|m|s|u|sdw36|sdw36|8690|/data/mirror690 +1693|691|m|m|s|u|sdw36|sdw36|8691|/data/mirror691 +1694|692|m|m|s|u|sdw36|sdw36|8692|/data/mirror692 +1695|693|m|m|s|u|sdw36|sdw36|8693|/data/mirror693 +1696|694|m|m|s|u|sdw36|sdw36|8694|/data/mirror694 +1697|695|m|m|s|u|sdw36|sdw36|8695|/data/mirror695 +1698|696|m|m|s|u|sdw36|sdw36|8696|/data/mirror696 +1699|697|m|m|s|u|sdw36|sdw36|8697|/data/mirror697 +1700|698|m|m|s|u|sdw36|sdw36|8698|/data/mirror698 +1701|699|m|m|s|u|sdw36|sdw36|8699|/data/mirror699 +# SDW37 +722|720|p|p|s|u|sdw37|sdw37|7720|/data/primary720 +723|721|p|p|s|u|sdw37|sdw37|7721|/data/primary721 +724|722|p|p|s|u|sdw37|sdw37|7722|/data/primary722 +725|723|p|p|s|u|sdw37|sdw37|7723|/data/primary723 +726|724|p|p|s|u|sdw37|sdw37|7724|/data/primary724 +727|725|p|p|s|u|sdw37|sdw37|7725|/data/primary725 +728|726|p|p|s|u|sdw37|sdw37|7726|/data/primary726 +729|727|p|p|s|u|sdw37|sdw37|7727|/data/primary727 +730|728|p|p|s|u|sdw37|sdw37|7728|/data/primary728 +731|729|p|p|s|u|sdw37|sdw37|7729|/data/primary729 +732|730|p|p|s|u|sdw37|sdw37|7730|/data/primary730 +733|731|p|p|s|u|sdw37|sdw37|7731|/data/primary731 +734|732|p|p|s|u|sdw37|sdw37|7732|/data/primary732 +735|733|p|p|s|u|sdw37|sdw37|7733|/data/primary733 +736|734|p|p|s|u|sdw37|sdw37|7734|/data/primary734 +737|735|p|p|s|u|sdw37|sdw37|7735|/data/primary735 +738|736|p|p|s|u|sdw37|sdw37|7736|/data/primary736 +739|737|p|p|s|u|sdw37|sdw37|7737|/data/primary737 +740|738|p|p|s|u|sdw37|sdw37|7738|/data/primary738 +741|739|p|p|s|u|sdw37|sdw37|7739|/data/primary739 +1702|700|m|m|s|u|sdw37|sdw37|8700|/data/mirror700 +1703|701|m|m|s|u|sdw37|sdw37|8701|/data/mirror701 +1704|702|m|m|s|u|sdw37|sdw37|8702|/data/mirror702 +1705|703|m|m|s|u|sdw37|sdw37|8703|/data/mirror703 +1706|704|m|m|s|u|sdw37|sdw37|8704|/data/mirror704 +1707|705|m|m|s|u|sdw37|sdw37|8705|/data/mirror705 +1708|706|m|m|s|u|sdw37|sdw37|8706|/data/mirror706 +1709|707|m|m|s|u|sdw37|sdw37|8707|/data/mirror707 +1710|708|m|m|s|u|sdw37|sdw37|8708|/data/mirror708 +1711|709|m|m|s|u|sdw37|sdw37|8709|/data/mirror709 +1712|710|m|m|s|u|sdw37|sdw37|8710|/data/mirror710 +1713|711|m|m|s|u|sdw37|sdw37|8711|/data/mirror711 +1714|712|m|m|s|u|sdw37|sdw37|8712|/data/mirror712 +1715|713|m|m|s|u|sdw37|sdw37|8713|/data/mirror713 +1716|714|m|m|s|u|sdw37|sdw37|8714|/data/mirror714 +1717|715|m|m|s|u|sdw37|sdw37|8715|/data/mirror715 +1718|716|m|m|s|u|sdw37|sdw37|8716|/data/mirror716 +1719|717|m|m|s|u|sdw37|sdw37|8717|/data/mirror717 +1720|718|m|m|s|u|sdw37|sdw37|8718|/data/mirror718 +1721|719|m|m|s|u|sdw37|sdw37|8719|/data/mirror719 +# SDW38 +742|740|p|p|s|u|sdw38|sdw38|7740|/data/primary740 +743|741|p|p|s|u|sdw38|sdw38|7741|/data/primary741 +744|742|p|p|s|u|sdw38|sdw38|7742|/data/primary742 +745|743|p|p|s|u|sdw38|sdw38|7743|/data/primary743 +746|744|p|p|s|u|sdw38|sdw38|7744|/data/primary744 +747|745|p|p|s|u|sdw38|sdw38|7745|/data/primary745 +748|746|p|p|s|u|sdw38|sdw38|7746|/data/primary746 +749|747|p|p|s|u|sdw38|sdw38|7747|/data/primary747 +750|748|p|p|s|u|sdw38|sdw38|7748|/data/primary748 +751|749|p|p|s|u|sdw38|sdw38|7749|/data/primary749 +752|750|p|p|s|u|sdw38|sdw38|7750|/data/primary750 +753|751|p|p|s|u|sdw38|sdw38|7751|/data/primary751 +754|752|p|p|s|u|sdw38|sdw38|7752|/data/primary752 +755|753|p|p|s|u|sdw38|sdw38|7753|/data/primary753 +756|754|p|p|s|u|sdw38|sdw38|7754|/data/primary754 +757|755|p|p|s|u|sdw38|sdw38|7755|/data/primary755 +758|756|p|p|s|u|sdw38|sdw38|7756|/data/primary756 +759|757|p|p|s|u|sdw38|sdw38|7757|/data/primary757 +760|758|p|p|s|u|sdw38|sdw38|7758|/data/primary758 +761|759|p|p|s|u|sdw38|sdw38|7759|/data/primary759 +1722|720|m|m|s|u|sdw38|sdw38|8720|/data/mirror720 +1723|721|m|m|s|u|sdw38|sdw38|8721|/data/mirror721 +1724|722|m|m|s|u|sdw38|sdw38|8722|/data/mirror722 +1725|723|m|m|s|u|sdw38|sdw38|8723|/data/mirror723 +1726|724|m|m|s|u|sdw38|sdw38|8724|/data/mirror724 +1727|725|m|m|s|u|sdw38|sdw38|8725|/data/mirror725 +1728|726|m|m|s|u|sdw38|sdw38|8726|/data/mirror726 +1729|727|m|m|s|u|sdw38|sdw38|8727|/data/mirror727 +1730|728|m|m|s|u|sdw38|sdw38|8728|/data/mirror728 +1731|729|m|m|s|u|sdw38|sdw38|8729|/data/mirror729 +1732|730|m|m|s|u|sdw38|sdw38|8730|/data/mirror730 +1733|731|m|m|s|u|sdw38|sdw38|8731|/data/mirror731 +1734|732|m|m|s|u|sdw38|sdw38|8732|/data/mirror732 +1735|733|m|m|s|u|sdw38|sdw38|8733|/data/mirror733 +1736|734|m|m|s|u|sdw38|sdw38|8734|/data/mirror734 +1737|735|m|m|s|u|sdw38|sdw38|8735|/data/mirror735 +1738|736|m|m|s|u|sdw38|sdw38|8736|/data/mirror736 +1739|737|m|m|s|u|sdw38|sdw38|8737|/data/mirror737 +1740|738|m|m|s|u|sdw38|sdw38|8738|/data/mirror738 +1741|739|m|m|s|u|sdw38|sdw38|8739|/data/mirror739 +# SDW39 +762|760|p|p|s|u|sdw39|sdw39|7760|/data/primary760 +763|761|p|p|s|u|sdw39|sdw39|7761|/data/primary761 +764|762|p|p|s|u|sdw39|sdw39|7762|/data/primary762 +765|763|p|p|s|u|sdw39|sdw39|7763|/data/primary763 +766|764|p|p|s|u|sdw39|sdw39|7764|/data/primary764 +767|765|p|p|s|u|sdw39|sdw39|7765|/data/primary765 +768|766|p|p|s|u|sdw39|sdw39|7766|/data/primary766 +769|767|p|p|s|u|sdw39|sdw39|7767|/data/primary767 +770|768|p|p|s|u|sdw39|sdw39|7768|/data/primary768 +771|769|p|p|s|u|sdw39|sdw39|7769|/data/primary769 +772|770|p|p|s|u|sdw39|sdw39|7770|/data/primary770 +773|771|p|p|s|u|sdw39|sdw39|7771|/data/primary771 +774|772|p|p|s|u|sdw39|sdw39|7772|/data/primary772 +775|773|p|p|s|u|sdw39|sdw39|7773|/data/primary773 +776|774|p|p|s|u|sdw39|sdw39|7774|/data/primary774 +777|775|p|p|s|u|sdw39|sdw39|7775|/data/primary775 +778|776|p|p|s|u|sdw39|sdw39|7776|/data/primary776 +779|777|p|p|s|u|sdw39|sdw39|7777|/data/primary777 +780|778|p|p|s|u|sdw39|sdw39|7778|/data/primary778 +781|779|p|p|s|u|sdw39|sdw39|7779|/data/primary779 +1742|740|m|m|s|u|sdw39|sdw39|8740|/data/mirror740 +1743|741|m|m|s|u|sdw39|sdw39|8741|/data/mirror741 +1744|742|m|m|s|u|sdw39|sdw39|8742|/data/mirror742 +1745|743|m|m|s|u|sdw39|sdw39|8743|/data/mirror743 +1746|744|m|m|s|u|sdw39|sdw39|8744|/data/mirror744 +1747|745|m|m|s|u|sdw39|sdw39|8745|/data/mirror745 +1748|746|m|m|s|u|sdw39|sdw39|8746|/data/mirror746 +1749|747|m|m|s|u|sdw39|sdw39|8747|/data/mirror747 +1750|748|m|m|s|u|sdw39|sdw39|8748|/data/mirror748 +1751|749|m|m|s|u|sdw39|sdw39|8749|/data/mirror749 +1752|750|m|m|s|u|sdw39|sdw39|8750|/data/mirror750 +1753|751|m|m|s|u|sdw39|sdw39|8751|/data/mirror751 +1754|752|m|m|s|u|sdw39|sdw39|8752|/data/mirror752 +1755|753|m|m|s|u|sdw39|sdw39|8753|/data/mirror753 +1756|754|m|m|s|u|sdw39|sdw39|8754|/data/mirror754 +1757|755|m|m|s|u|sdw39|sdw39|8755|/data/mirror755 +1758|756|m|m|s|u|sdw39|sdw39|8756|/data/mirror756 +1759|757|m|m|s|u|sdw39|sdw39|8757|/data/mirror757 +1760|758|m|m|s|u|sdw39|sdw39|8758|/data/mirror758 +1761|759|m|m|s|u|sdw39|sdw39|8759|/data/mirror759 +# SDW4 +62|60|p|p|s|u|sdw4|sdw4|7060|/data/primary60 +63|61|p|p|s|u|sdw4|sdw4|7061|/data/primary61 +64|62|p|p|s|u|sdw4|sdw4|7062|/data/primary62 +65|63|p|p|s|u|sdw4|sdw4|7063|/data/primary63 +66|64|p|p|s|u|sdw4|sdw4|7064|/data/primary64 +67|65|p|p|s|u|sdw4|sdw4|7065|/data/primary65 +68|66|p|p|s|u|sdw4|sdw4|7066|/data/primary66 +69|67|p|p|s|u|sdw4|sdw4|7067|/data/primary67 +70|68|p|p|s|u|sdw4|sdw4|7068|/data/primary68 +71|69|p|p|s|u|sdw4|sdw4|7069|/data/primary69 +72|70|p|p|s|u|sdw4|sdw4|7070|/data/primary70 +73|71|p|p|s|u|sdw4|sdw4|7071|/data/primary71 +74|72|p|p|s|u|sdw4|sdw4|7072|/data/primary72 +75|73|p|p|s|u|sdw4|sdw4|7073|/data/primary73 +76|74|p|p|s|u|sdw4|sdw4|7074|/data/primary74 +77|75|p|p|s|u|sdw4|sdw4|7075|/data/primary75 +78|76|p|p|s|u|sdw4|sdw4|7076|/data/primary76 +79|77|p|p|s|u|sdw4|sdw4|7077|/data/primary77 +80|78|p|p|s|u|sdw4|sdw4|7078|/data/primary78 +81|79|p|p|s|u|sdw4|sdw4|7079|/data/primary79 +1042|40|m|m|s|u|sdw4|sdw4|8040|/data/mirror40 +1043|41|m|m|s|u|sdw4|sdw4|8041|/data/mirror41 +1044|42|m|m|s|u|sdw4|sdw4|8042|/data/mirror42 +1045|43|m|m|s|u|sdw4|sdw4|8043|/data/mirror43 +1046|44|m|m|s|u|sdw4|sdw4|8044|/data/mirror44 +1047|45|m|m|s|u|sdw4|sdw4|8045|/data/mirror45 +1048|46|m|m|s|u|sdw4|sdw4|8046|/data/mirror46 +1049|47|m|m|s|u|sdw4|sdw4|8047|/data/mirror47 +1050|48|m|m|s|u|sdw4|sdw4|8048|/data/mirror48 +1051|49|m|m|s|u|sdw4|sdw4|8049|/data/mirror49 +1052|50|m|m|s|u|sdw4|sdw4|8050|/data/mirror50 +1053|51|m|m|s|u|sdw4|sdw4|8051|/data/mirror51 +1054|52|m|m|s|u|sdw4|sdw4|8052|/data/mirror52 +1055|53|m|m|s|u|sdw4|sdw4|8053|/data/mirror53 +1056|54|m|m|s|u|sdw4|sdw4|8054|/data/mirror54 +1057|55|m|m|s|u|sdw4|sdw4|8055|/data/mirror55 +1058|56|m|m|s|u|sdw4|sdw4|8056|/data/mirror56 +1059|57|m|m|s|u|sdw4|sdw4|8057|/data/mirror57 +1060|58|m|m|s|u|sdw4|sdw4|8058|/data/mirror58 +1061|59|m|m|s|u|sdw4|sdw4|8059|/data/mirror59 +# SDW40 +782|780|p|p|s|u|sdw40|sdw40|7780|/data/primary780 +783|781|p|p|s|u|sdw40|sdw40|7781|/data/primary781 +784|782|p|p|s|u|sdw40|sdw40|7782|/data/primary782 +785|783|p|p|s|u|sdw40|sdw40|7783|/data/primary783 +786|784|p|p|s|u|sdw40|sdw40|7784|/data/primary784 +787|785|p|p|s|u|sdw40|sdw40|7785|/data/primary785 +788|786|p|p|s|u|sdw40|sdw40|7786|/data/primary786 +789|787|p|p|s|u|sdw40|sdw40|7787|/data/primary787 +790|788|p|p|s|u|sdw40|sdw40|7788|/data/primary788 +791|789|p|p|s|u|sdw40|sdw40|7789|/data/primary789 +792|790|p|p|s|u|sdw40|sdw40|7790|/data/primary790 +793|791|p|p|s|u|sdw40|sdw40|7791|/data/primary791 +794|792|p|p|s|u|sdw40|sdw40|7792|/data/primary792 +795|793|p|p|s|u|sdw40|sdw40|7793|/data/primary793 +796|794|p|p|s|u|sdw40|sdw40|7794|/data/primary794 +797|795|p|p|s|u|sdw40|sdw40|7795|/data/primary795 +798|796|p|p|s|u|sdw40|sdw40|7796|/data/primary796 +799|797|p|p|s|u|sdw40|sdw40|7797|/data/primary797 +800|798|p|p|s|u|sdw40|sdw40|7798|/data/primary798 +801|799|p|p|s|u|sdw40|sdw40|7799|/data/primary799 +1762|760|m|m|s|u|sdw40|sdw40|8760|/data/mirror760 +1763|761|m|m|s|u|sdw40|sdw40|8761|/data/mirror761 +1764|762|m|m|s|u|sdw40|sdw40|8762|/data/mirror762 +1765|763|m|m|s|u|sdw40|sdw40|8763|/data/mirror763 +1766|764|m|m|s|u|sdw40|sdw40|8764|/data/mirror764 +1767|765|m|m|s|u|sdw40|sdw40|8765|/data/mirror765 +1768|766|m|m|s|u|sdw40|sdw40|8766|/data/mirror766 +1769|767|m|m|s|u|sdw40|sdw40|8767|/data/mirror767 +1770|768|m|m|s|u|sdw40|sdw40|8768|/data/mirror768 +1771|769|m|m|s|u|sdw40|sdw40|8769|/data/mirror769 +1772|770|m|m|s|u|sdw40|sdw40|8770|/data/mirror770 +1773|771|m|m|s|u|sdw40|sdw40|8771|/data/mirror771 +1774|772|m|m|s|u|sdw40|sdw40|8772|/data/mirror772 +1775|773|m|m|s|u|sdw40|sdw40|8773|/data/mirror773 +1776|774|m|m|s|u|sdw40|sdw40|8774|/data/mirror774 +1777|775|m|m|s|u|sdw40|sdw40|8775|/data/mirror775 +1778|776|m|m|s|u|sdw40|sdw40|8776|/data/mirror776 +1779|777|m|m|s|u|sdw40|sdw40|8777|/data/mirror777 +1780|778|m|m|s|u|sdw40|sdw40|8778|/data/mirror778 +1781|779|m|m|s|u|sdw40|sdw40|8779|/data/mirror779 +# SDW41 +802|800|p|p|s|u|sdw41|sdw41|7800|/data/primary800 +803|801|p|p|s|u|sdw41|sdw41|7801|/data/primary801 +804|802|p|p|s|u|sdw41|sdw41|7802|/data/primary802 +805|803|p|p|s|u|sdw41|sdw41|7803|/data/primary803 +806|804|p|p|s|u|sdw41|sdw41|7804|/data/primary804 +807|805|p|p|s|u|sdw41|sdw41|7805|/data/primary805 +808|806|p|p|s|u|sdw41|sdw41|7806|/data/primary806 +809|807|p|p|s|u|sdw41|sdw41|7807|/data/primary807 +810|808|p|p|s|u|sdw41|sdw41|7808|/data/primary808 +811|809|p|p|s|u|sdw41|sdw41|7809|/data/primary809 +812|810|p|p|s|u|sdw41|sdw41|7810|/data/primary810 +813|811|p|p|s|u|sdw41|sdw41|7811|/data/primary811 +814|812|p|p|s|u|sdw41|sdw41|7812|/data/primary812 +815|813|p|p|s|u|sdw41|sdw41|7813|/data/primary813 +816|814|p|p|s|u|sdw41|sdw41|7814|/data/primary814 +817|815|p|p|s|u|sdw41|sdw41|7815|/data/primary815 +818|816|p|p|s|u|sdw41|sdw41|7816|/data/primary816 +819|817|p|p|s|u|sdw41|sdw41|7817|/data/primary817 +820|818|p|p|s|u|sdw41|sdw41|7818|/data/primary818 +821|819|p|p|s|u|sdw41|sdw41|7819|/data/primary819 +1782|780|m|m|s|u|sdw41|sdw41|8780|/data/mirror780 +1783|781|m|m|s|u|sdw41|sdw41|8781|/data/mirror781 +1784|782|m|m|s|u|sdw41|sdw41|8782|/data/mirror782 +1785|783|m|m|s|u|sdw41|sdw41|8783|/data/mirror783 +1786|784|m|m|s|u|sdw41|sdw41|8784|/data/mirror784 +1787|785|m|m|s|u|sdw41|sdw41|8785|/data/mirror785 +1788|786|m|m|s|u|sdw41|sdw41|8786|/data/mirror786 +1789|787|m|m|s|u|sdw41|sdw41|8787|/data/mirror787 +1790|788|m|m|s|u|sdw41|sdw41|8788|/data/mirror788 +1791|789|m|m|s|u|sdw41|sdw41|8789|/data/mirror789 +1792|790|m|m|s|u|sdw41|sdw41|8790|/data/mirror790 +1793|791|m|m|s|u|sdw41|sdw41|8791|/data/mirror791 +1794|792|m|m|s|u|sdw41|sdw41|8792|/data/mirror792 +1795|793|m|m|s|u|sdw41|sdw41|8793|/data/mirror793 +1796|794|m|m|s|u|sdw41|sdw41|8794|/data/mirror794 +1797|795|m|m|s|u|sdw41|sdw41|8795|/data/mirror795 +1798|796|m|m|s|u|sdw41|sdw41|8796|/data/mirror796 +1799|797|m|m|s|u|sdw41|sdw41|8797|/data/mirror797 +1800|798|m|m|s|u|sdw41|sdw41|8798|/data/mirror798 +1801|799|m|m|s|u|sdw41|sdw41|8799|/data/mirror799 +# SDW42 +822|820|p|p|s|u|sdw42|sdw42|7820|/data/primary820 +823|821|p|p|s|u|sdw42|sdw42|7821|/data/primary821 +824|822|p|p|s|u|sdw42|sdw42|7822|/data/primary822 +825|823|p|p|s|u|sdw42|sdw42|7823|/data/primary823 +826|824|p|p|s|u|sdw42|sdw42|7824|/data/primary824 +827|825|p|p|s|u|sdw42|sdw42|7825|/data/primary825 +828|826|p|p|s|u|sdw42|sdw42|7826|/data/primary826 +829|827|p|p|s|u|sdw42|sdw42|7827|/data/primary827 +830|828|p|p|s|u|sdw42|sdw42|7828|/data/primary828 +831|829|p|p|s|u|sdw42|sdw42|7829|/data/primary829 +832|830|p|p|s|u|sdw42|sdw42|7830|/data/primary830 +833|831|p|p|s|u|sdw42|sdw42|7831|/data/primary831 +834|832|p|p|s|u|sdw42|sdw42|7832|/data/primary832 +835|833|p|p|s|u|sdw42|sdw42|7833|/data/primary833 +836|834|p|p|s|u|sdw42|sdw42|7834|/data/primary834 +837|835|p|p|s|u|sdw42|sdw42|7835|/data/primary835 +838|836|p|p|s|u|sdw42|sdw42|7836|/data/primary836 +839|837|p|p|s|u|sdw42|sdw42|7837|/data/primary837 +840|838|p|p|s|u|sdw42|sdw42|7838|/data/primary838 +841|839|p|p|s|u|sdw42|sdw42|7839|/data/primary839 +1802|800|m|m|s|u|sdw42|sdw42|8800|/data/mirror800 +1803|801|m|m|s|u|sdw42|sdw42|8801|/data/mirror801 +1804|802|m|m|s|u|sdw42|sdw42|8802|/data/mirror802 +1805|803|m|m|s|u|sdw42|sdw42|8803|/data/mirror803 +1806|804|m|m|s|u|sdw42|sdw42|8804|/data/mirror804 +1807|805|m|m|s|u|sdw42|sdw42|8805|/data/mirror805 +1808|806|m|m|s|u|sdw42|sdw42|8806|/data/mirror806 +1809|807|m|m|s|u|sdw42|sdw42|8807|/data/mirror807 +1810|808|m|m|s|u|sdw42|sdw42|8808|/data/mirror808 +1811|809|m|m|s|u|sdw42|sdw42|8809|/data/mirror809 +1812|810|m|m|s|u|sdw42|sdw42|8810|/data/mirror810 +1813|811|m|m|s|u|sdw42|sdw42|8811|/data/mirror811 +1814|812|m|m|s|u|sdw42|sdw42|8812|/data/mirror812 +1815|813|m|m|s|u|sdw42|sdw42|8813|/data/mirror813 +1816|814|m|m|s|u|sdw42|sdw42|8814|/data/mirror814 +1817|815|m|m|s|u|sdw42|sdw42|8815|/data/mirror815 +1818|816|m|m|s|u|sdw42|sdw42|8816|/data/mirror816 +1819|817|m|m|s|u|sdw42|sdw42|8817|/data/mirror817 +1820|818|m|m|s|u|sdw42|sdw42|8818|/data/mirror818 +1821|819|m|m|s|u|sdw42|sdw42|8819|/data/mirror819 +# SDW43 +842|840|p|p|s|u|sdw43|sdw43|7840|/data/primary840 +843|841|p|p|s|u|sdw43|sdw43|7841|/data/primary841 +844|842|p|p|s|u|sdw43|sdw43|7842|/data/primary842 +845|843|p|p|s|u|sdw43|sdw43|7843|/data/primary843 +846|844|p|p|s|u|sdw43|sdw43|7844|/data/primary844 +847|845|p|p|s|u|sdw43|sdw43|7845|/data/primary845 +848|846|p|p|s|u|sdw43|sdw43|7846|/data/primary846 +849|847|p|p|s|u|sdw43|sdw43|7847|/data/primary847 +850|848|p|p|s|u|sdw43|sdw43|7848|/data/primary848 +851|849|p|p|s|u|sdw43|sdw43|7849|/data/primary849 +852|850|p|p|s|u|sdw43|sdw43|7850|/data/primary850 +853|851|p|p|s|u|sdw43|sdw43|7851|/data/primary851 +854|852|p|p|s|u|sdw43|sdw43|7852|/data/primary852 +855|853|p|p|s|u|sdw43|sdw43|7853|/data/primary853 +856|854|p|p|s|u|sdw43|sdw43|7854|/data/primary854 +857|855|p|p|s|u|sdw43|sdw43|7855|/data/primary855 +858|856|p|p|s|u|sdw43|sdw43|7856|/data/primary856 +859|857|p|p|s|u|sdw43|sdw43|7857|/data/primary857 +860|858|p|p|s|u|sdw43|sdw43|7858|/data/primary858 +861|859|p|p|s|u|sdw43|sdw43|7859|/data/primary859 +1822|820|m|m|s|u|sdw43|sdw43|8820|/data/mirror820 +1823|821|m|m|s|u|sdw43|sdw43|8821|/data/mirror821 +1824|822|m|m|s|u|sdw43|sdw43|8822|/data/mirror822 +1825|823|m|m|s|u|sdw43|sdw43|8823|/data/mirror823 +1826|824|m|m|s|u|sdw43|sdw43|8824|/data/mirror824 +1827|825|m|m|s|u|sdw43|sdw43|8825|/data/mirror825 +1828|826|m|m|s|u|sdw43|sdw43|8826|/data/mirror826 +1829|827|m|m|s|u|sdw43|sdw43|8827|/data/mirror827 +1830|828|m|m|s|u|sdw43|sdw43|8828|/data/mirror828 +1831|829|m|m|s|u|sdw43|sdw43|8829|/data/mirror829 +1832|830|m|m|s|u|sdw43|sdw43|8830|/data/mirror830 +1833|831|m|m|s|u|sdw43|sdw43|8831|/data/mirror831 +1834|832|m|m|s|u|sdw43|sdw43|8832|/data/mirror832 +1835|833|m|m|s|u|sdw43|sdw43|8833|/data/mirror833 +1836|834|m|m|s|u|sdw43|sdw43|8834|/data/mirror834 +1837|835|m|m|s|u|sdw43|sdw43|8835|/data/mirror835 +1838|836|m|m|s|u|sdw43|sdw43|8836|/data/mirror836 +1839|837|m|m|s|u|sdw43|sdw43|8837|/data/mirror837 +1840|838|m|m|s|u|sdw43|sdw43|8838|/data/mirror838 +1841|839|m|m|s|u|sdw43|sdw43|8839|/data/mirror839 +# SDW44 +862|860|p|p|s|u|sdw44|sdw44|7860|/data/primary860 +863|861|p|p|s|u|sdw44|sdw44|7861|/data/primary861 +864|862|p|p|s|u|sdw44|sdw44|7862|/data/primary862 +865|863|p|p|s|u|sdw44|sdw44|7863|/data/primary863 +866|864|p|p|s|u|sdw44|sdw44|7864|/data/primary864 +867|865|p|p|s|u|sdw44|sdw44|7865|/data/primary865 +868|866|p|p|s|u|sdw44|sdw44|7866|/data/primary866 +869|867|p|p|s|u|sdw44|sdw44|7867|/data/primary867 +870|868|p|p|s|u|sdw44|sdw44|7868|/data/primary868 +871|869|p|p|s|u|sdw44|sdw44|7869|/data/primary869 +872|870|p|p|s|u|sdw44|sdw44|7870|/data/primary870 +873|871|p|p|s|u|sdw44|sdw44|7871|/data/primary871 +874|872|p|p|s|u|sdw44|sdw44|7872|/data/primary872 +875|873|p|p|s|u|sdw44|sdw44|7873|/data/primary873 +876|874|p|p|s|u|sdw44|sdw44|7874|/data/primary874 +877|875|p|p|s|u|sdw44|sdw44|7875|/data/primary875 +878|876|p|p|s|u|sdw44|sdw44|7876|/data/primary876 +879|877|p|p|s|u|sdw44|sdw44|7877|/data/primary877 +880|878|p|p|s|u|sdw44|sdw44|7878|/data/primary878 +881|879|p|p|s|u|sdw44|sdw44|7879|/data/primary879 +1842|840|m|m|s|u|sdw44|sdw44|8840|/data/mirror840 +1843|841|m|m|s|u|sdw44|sdw44|8841|/data/mirror841 +1844|842|m|m|s|u|sdw44|sdw44|8842|/data/mirror842 +1845|843|m|m|s|u|sdw44|sdw44|8843|/data/mirror843 +1846|844|m|m|s|u|sdw44|sdw44|8844|/data/mirror844 +1847|845|m|m|s|u|sdw44|sdw44|8845|/data/mirror845 +1848|846|m|m|s|u|sdw44|sdw44|8846|/data/mirror846 +1849|847|m|m|s|u|sdw44|sdw44|8847|/data/mirror847 +1850|848|m|m|s|u|sdw44|sdw44|8848|/data/mirror848 +1851|849|m|m|s|u|sdw44|sdw44|8849|/data/mirror849 +1852|850|m|m|s|u|sdw44|sdw44|8850|/data/mirror850 +1853|851|m|m|s|u|sdw44|sdw44|8851|/data/mirror851 +1854|852|m|m|s|u|sdw44|sdw44|8852|/data/mirror852 +1855|853|m|m|s|u|sdw44|sdw44|8853|/data/mirror853 +1856|854|m|m|s|u|sdw44|sdw44|8854|/data/mirror854 +1857|855|m|m|s|u|sdw44|sdw44|8855|/data/mirror855 +1858|856|m|m|s|u|sdw44|sdw44|8856|/data/mirror856 +1859|857|m|m|s|u|sdw44|sdw44|8857|/data/mirror857 +1860|858|m|m|s|u|sdw44|sdw44|8858|/data/mirror858 +1861|859|m|m|s|u|sdw44|sdw44|8859|/data/mirror859 +# SDW45 +882|880|p|p|s|u|sdw45|sdw45|7880|/data/primary880 +883|881|p|p|s|u|sdw45|sdw45|7881|/data/primary881 +884|882|p|p|s|u|sdw45|sdw45|7882|/data/primary882 +885|883|p|p|s|u|sdw45|sdw45|7883|/data/primary883 +886|884|p|p|s|u|sdw45|sdw45|7884|/data/primary884 +887|885|p|p|s|u|sdw45|sdw45|7885|/data/primary885 +888|886|p|p|s|u|sdw45|sdw45|7886|/data/primary886 +889|887|p|p|s|u|sdw45|sdw45|7887|/data/primary887 +890|888|p|p|s|u|sdw45|sdw45|7888|/data/primary888 +891|889|p|p|s|u|sdw45|sdw45|7889|/data/primary889 +892|890|p|p|s|u|sdw45|sdw45|7890|/data/primary890 +893|891|p|p|s|u|sdw45|sdw45|7891|/data/primary891 +894|892|p|p|s|u|sdw45|sdw45|7892|/data/primary892 +895|893|p|p|s|u|sdw45|sdw45|7893|/data/primary893 +896|894|p|p|s|u|sdw45|sdw45|7894|/data/primary894 +897|895|p|p|s|u|sdw45|sdw45|7895|/data/primary895 +898|896|p|p|s|u|sdw45|sdw45|7896|/data/primary896 +899|897|p|p|s|u|sdw45|sdw45|7897|/data/primary897 +900|898|p|p|s|u|sdw45|sdw45|7898|/data/primary898 +901|899|p|p|s|u|sdw45|sdw45|7899|/data/primary899 +1862|860|m|m|s|u|sdw45|sdw45|8860|/data/mirror860 +1863|861|m|m|s|u|sdw45|sdw45|8861|/data/mirror861 +1864|862|m|m|s|u|sdw45|sdw45|8862|/data/mirror862 +1865|863|m|m|s|u|sdw45|sdw45|8863|/data/mirror863 +1866|864|m|m|s|u|sdw45|sdw45|8864|/data/mirror864 +1867|865|m|m|s|u|sdw45|sdw45|8865|/data/mirror865 +1868|866|m|m|s|u|sdw45|sdw45|8866|/data/mirror866 +1869|867|m|m|s|u|sdw45|sdw45|8867|/data/mirror867 +1870|868|m|m|s|u|sdw45|sdw45|8868|/data/mirror868 +1871|869|m|m|s|u|sdw45|sdw45|8869|/data/mirror869 +1872|870|m|m|s|u|sdw45|sdw45|8870|/data/mirror870 +1873|871|m|m|s|u|sdw45|sdw45|8871|/data/mirror871 +1874|872|m|m|s|u|sdw45|sdw45|8872|/data/mirror872 +1875|873|m|m|s|u|sdw45|sdw45|8873|/data/mirror873 +1876|874|m|m|s|u|sdw45|sdw45|8874|/data/mirror874 +1877|875|m|m|s|u|sdw45|sdw45|8875|/data/mirror875 +1878|876|m|m|s|u|sdw45|sdw45|8876|/data/mirror876 +1879|877|m|m|s|u|sdw45|sdw45|8877|/data/mirror877 +1880|878|m|m|s|u|sdw45|sdw45|8878|/data/mirror878 +1881|879|m|m|s|u|sdw45|sdw45|8879|/data/mirror879 +# SDW46 +902|900|p|p|s|u|sdw46|sdw46|7900|/data/primary900 +903|901|p|p|s|u|sdw46|sdw46|7901|/data/primary901 +904|902|p|p|s|u|sdw46|sdw46|7902|/data/primary902 +905|903|p|p|s|u|sdw46|sdw46|7903|/data/primary903 +906|904|p|p|s|u|sdw46|sdw46|7904|/data/primary904 +907|905|p|p|s|u|sdw46|sdw46|7905|/data/primary905 +908|906|p|p|s|u|sdw46|sdw46|7906|/data/primary906 +909|907|p|p|s|u|sdw46|sdw46|7907|/data/primary907 +910|908|p|p|s|u|sdw46|sdw46|7908|/data/primary908 +911|909|p|p|s|u|sdw46|sdw46|7909|/data/primary909 +912|910|p|p|s|u|sdw46|sdw46|7910|/data/primary910 +913|911|p|p|s|u|sdw46|sdw46|7911|/data/primary911 +914|912|p|p|s|u|sdw46|sdw46|7912|/data/primary912 +915|913|p|p|s|u|sdw46|sdw46|7913|/data/primary913 +916|914|p|p|s|u|sdw46|sdw46|7914|/data/primary914 +917|915|p|p|s|u|sdw46|sdw46|7915|/data/primary915 +918|916|p|p|s|u|sdw46|sdw46|7916|/data/primary916 +919|917|p|p|s|u|sdw46|sdw46|7917|/data/primary917 +920|918|p|p|s|u|sdw46|sdw46|7918|/data/primary918 +921|919|p|p|s|u|sdw46|sdw46|7919|/data/primary919 +1882|880|m|m|s|u|sdw46|sdw46|8880|/data/mirror880 +1883|881|m|m|s|u|sdw46|sdw46|8881|/data/mirror881 +1884|882|m|m|s|u|sdw46|sdw46|8882|/data/mirror882 +1885|883|m|m|s|u|sdw46|sdw46|8883|/data/mirror883 +1886|884|m|m|s|u|sdw46|sdw46|8884|/data/mirror884 +1887|885|m|m|s|u|sdw46|sdw46|8885|/data/mirror885 +1888|886|m|m|s|u|sdw46|sdw46|8886|/data/mirror886 +1889|887|m|m|s|u|sdw46|sdw46|8887|/data/mirror887 +1890|888|m|m|s|u|sdw46|sdw46|8888|/data/mirror888 +1891|889|m|m|s|u|sdw46|sdw46|8889|/data/mirror889 +1892|890|m|m|s|u|sdw46|sdw46|8890|/data/mirror890 +1893|891|m|m|s|u|sdw46|sdw46|8891|/data/mirror891 +1894|892|m|m|s|u|sdw46|sdw46|8892|/data/mirror892 +1895|893|m|m|s|u|sdw46|sdw46|8893|/data/mirror893 +1896|894|m|m|s|u|sdw46|sdw46|8894|/data/mirror894 +1897|895|m|m|s|u|sdw46|sdw46|8895|/data/mirror895 +1898|896|m|m|s|u|sdw46|sdw46|8896|/data/mirror896 +1899|897|m|m|s|u|sdw46|sdw46|8897|/data/mirror897 +1900|898|m|m|s|u|sdw46|sdw46|8898|/data/mirror898 +1901|899|m|m|s|u|sdw46|sdw46|8899|/data/mirror899 +# SDW47 +922|920|p|p|s|u|sdw47|sdw47|7920|/data/primary920 +923|921|p|p|s|u|sdw47|sdw47|7921|/data/primary921 +924|922|p|p|s|u|sdw47|sdw47|7922|/data/primary922 +925|923|p|p|s|u|sdw47|sdw47|7923|/data/primary923 +926|924|p|p|s|u|sdw47|sdw47|7924|/data/primary924 +927|925|p|p|s|u|sdw47|sdw47|7925|/data/primary925 +928|926|p|p|s|u|sdw47|sdw47|7926|/data/primary926 +929|927|p|p|s|u|sdw47|sdw47|7927|/data/primary927 +930|928|p|p|s|u|sdw47|sdw47|7928|/data/primary928 +931|929|p|p|s|u|sdw47|sdw47|7929|/data/primary929 +932|930|p|p|s|u|sdw47|sdw47|7930|/data/primary930 +933|931|p|p|s|u|sdw47|sdw47|7931|/data/primary931 +934|932|p|p|s|u|sdw47|sdw47|7932|/data/primary932 +935|933|p|p|s|u|sdw47|sdw47|7933|/data/primary933 +936|934|p|p|s|u|sdw47|sdw47|7934|/data/primary934 +937|935|p|p|s|u|sdw47|sdw47|7935|/data/primary935 +938|936|p|p|s|u|sdw47|sdw47|7936|/data/primary936 +939|937|p|p|s|u|sdw47|sdw47|7937|/data/primary937 +940|938|p|p|s|u|sdw47|sdw47|7938|/data/primary938 +941|939|p|p|s|u|sdw47|sdw47|7939|/data/primary939 +1902|900|m|m|s|u|sdw47|sdw47|8900|/data/mirror900 +1903|901|m|m|s|u|sdw47|sdw47|8901|/data/mirror901 +1904|902|m|m|s|u|sdw47|sdw47|8902|/data/mirror902 +1905|903|m|m|s|u|sdw47|sdw47|8903|/data/mirror903 +1906|904|m|m|s|u|sdw47|sdw47|8904|/data/mirror904 +1907|905|m|m|s|u|sdw47|sdw47|8905|/data/mirror905 +1908|906|m|m|s|u|sdw47|sdw47|8906|/data/mirror906 +1909|907|m|m|s|u|sdw47|sdw47|8907|/data/mirror907 +1910|908|m|m|s|u|sdw47|sdw47|8908|/data/mirror908 +1911|909|m|m|s|u|sdw47|sdw47|8909|/data/mirror909 +1912|910|m|m|s|u|sdw47|sdw47|8910|/data/mirror910 +1913|911|m|m|s|u|sdw47|sdw47|8911|/data/mirror911 +1914|912|m|m|s|u|sdw47|sdw47|8912|/data/mirror912 +1915|913|m|m|s|u|sdw47|sdw47|8913|/data/mirror913 +1916|914|m|m|s|u|sdw47|sdw47|8914|/data/mirror914 +1917|915|m|m|s|u|sdw47|sdw47|8915|/data/mirror915 +1918|916|m|m|s|u|sdw47|sdw47|8916|/data/mirror916 +1919|917|m|m|s|u|sdw47|sdw47|8917|/data/mirror917 +1920|918|m|m|s|u|sdw47|sdw47|8918|/data/mirror918 +1921|919|m|m|s|u|sdw47|sdw47|8919|/data/mirror919 +# SDW48 +942|940|p|p|s|u|sdw48|sdw48|7940|/data/primary940 +943|941|p|p|s|u|sdw48|sdw48|7941|/data/primary941 +944|942|p|p|s|u|sdw48|sdw48|7942|/data/primary942 +945|943|p|p|s|u|sdw48|sdw48|7943|/data/primary943 +946|944|p|p|s|u|sdw48|sdw48|7944|/data/primary944 +947|945|p|p|s|u|sdw48|sdw48|7945|/data/primary945 +948|946|p|p|s|u|sdw48|sdw48|7946|/data/primary946 +949|947|p|p|s|u|sdw48|sdw48|7947|/data/primary947 +950|948|p|p|s|u|sdw48|sdw48|7948|/data/primary948 +951|949|p|p|s|u|sdw48|sdw48|7949|/data/primary949 +952|950|p|p|s|u|sdw48|sdw48|7950|/data/primary950 +953|951|p|p|s|u|sdw48|sdw48|7951|/data/primary951 +954|952|p|p|s|u|sdw48|sdw48|7952|/data/primary952 +955|953|p|p|s|u|sdw48|sdw48|7953|/data/primary953 +956|954|p|p|s|u|sdw48|sdw48|7954|/data/primary954 +957|955|p|p|s|u|sdw48|sdw48|7955|/data/primary955 +958|956|p|p|s|u|sdw48|sdw48|7956|/data/primary956 +959|957|p|p|s|u|sdw48|sdw48|7957|/data/primary957 +960|958|p|p|s|u|sdw48|sdw48|7958|/data/primary958 +961|959|p|p|s|u|sdw48|sdw48|7959|/data/primary959 +1922|920|m|m|s|u|sdw48|sdw48|8920|/data/mirror920 +1923|921|m|m|s|u|sdw48|sdw48|8921|/data/mirror921 +1924|922|m|m|s|u|sdw48|sdw48|8922|/data/mirror922 +1925|923|m|m|s|u|sdw48|sdw48|8923|/data/mirror923 +1926|924|m|m|s|u|sdw48|sdw48|8924|/data/mirror924 +1927|925|m|m|s|u|sdw48|sdw48|8925|/data/mirror925 +1928|926|m|m|s|u|sdw48|sdw48|8926|/data/mirror926 +1929|927|m|m|s|u|sdw48|sdw48|8927|/data/mirror927 +1930|928|m|m|s|u|sdw48|sdw48|8928|/data/mirror928 +1931|929|m|m|s|u|sdw48|sdw48|8929|/data/mirror929 +1932|930|m|m|s|u|sdw48|sdw48|8930|/data/mirror930 +1933|931|m|m|s|u|sdw48|sdw48|8931|/data/mirror931 +1934|932|m|m|s|u|sdw48|sdw48|8932|/data/mirror932 +1935|933|m|m|s|u|sdw48|sdw48|8933|/data/mirror933 +1936|934|m|m|s|u|sdw48|sdw48|8934|/data/mirror934 +1937|935|m|m|s|u|sdw48|sdw48|8935|/data/mirror935 +1938|936|m|m|s|u|sdw48|sdw48|8936|/data/mirror936 +1939|937|m|m|s|u|sdw48|sdw48|8937|/data/mirror937 +1940|938|m|m|s|u|sdw48|sdw48|8938|/data/mirror938 +1941|939|m|m|s|u|sdw48|sdw48|8939|/data/mirror939 +# SDW49 +962|960|p|p|s|u|sdw49|sdw49|7960|/data/primary960 +963|961|p|p|s|u|sdw49|sdw49|7961|/data/primary961 +964|962|p|p|s|u|sdw49|sdw49|7962|/data/primary962 +965|963|p|p|s|u|sdw49|sdw49|7963|/data/primary963 +966|964|p|p|s|u|sdw49|sdw49|7964|/data/primary964 +967|965|p|p|s|u|sdw49|sdw49|7965|/data/primary965 +968|966|p|p|s|u|sdw49|sdw49|7966|/data/primary966 +969|967|p|p|s|u|sdw49|sdw49|7967|/data/primary967 +970|968|p|p|s|u|sdw49|sdw49|7968|/data/primary968 +971|969|p|p|s|u|sdw49|sdw49|7969|/data/primary969 +972|970|p|p|s|u|sdw49|sdw49|7970|/data/primary970 +973|971|p|p|s|u|sdw49|sdw49|7971|/data/primary971 +974|972|p|p|s|u|sdw49|sdw49|7972|/data/primary972 +975|973|p|p|s|u|sdw49|sdw49|7973|/data/primary973 +976|974|p|p|s|u|sdw49|sdw49|7974|/data/primary974 +977|975|p|p|s|u|sdw49|sdw49|7975|/data/primary975 +978|976|p|p|s|u|sdw49|sdw49|7976|/data/primary976 +979|977|p|p|s|u|sdw49|sdw49|7977|/data/primary977 +980|978|p|p|s|u|sdw49|sdw49|7978|/data/primary978 +981|979|p|p|s|u|sdw49|sdw49|7979|/data/primary979 +1942|940|m|m|s|u|sdw49|sdw49|8940|/data/mirror940 +1943|941|m|m|s|u|sdw49|sdw49|8941|/data/mirror941 +1944|942|m|m|s|u|sdw49|sdw49|8942|/data/mirror942 +1945|943|m|m|s|u|sdw49|sdw49|8943|/data/mirror943 +1946|944|m|m|s|u|sdw49|sdw49|8944|/data/mirror944 +1947|945|m|m|s|u|sdw49|sdw49|8945|/data/mirror945 +1948|946|m|m|s|u|sdw49|sdw49|8946|/data/mirror946 +1949|947|m|m|s|u|sdw49|sdw49|8947|/data/mirror947 +1950|948|m|m|s|u|sdw49|sdw49|8948|/data/mirror948 +1951|949|m|m|s|u|sdw49|sdw49|8949|/data/mirror949 +1952|950|m|m|s|u|sdw49|sdw49|8950|/data/mirror950 +1953|951|m|m|s|u|sdw49|sdw49|8951|/data/mirror951 +1954|952|m|m|s|u|sdw49|sdw49|8952|/data/mirror952 +1955|953|m|m|s|u|sdw49|sdw49|8953|/data/mirror953 +1956|954|m|m|s|u|sdw49|sdw49|8954|/data/mirror954 +1957|955|m|m|s|u|sdw49|sdw49|8955|/data/mirror955 +1958|956|m|m|s|u|sdw49|sdw49|8956|/data/mirror956 +1959|957|m|m|s|u|sdw49|sdw49|8957|/data/mirror957 +1960|958|m|m|s|u|sdw49|sdw49|8958|/data/mirror958 +1961|959|m|m|s|u|sdw49|sdw49|8959|/data/mirror959 +# SDW5 +82|80|p|p|s|u|sdw5|sdw5|7080|/data/primary80 +83|81|p|p|s|u|sdw5|sdw5|7081|/data/primary81 +84|82|p|p|s|u|sdw5|sdw5|7082|/data/primary82 +85|83|p|p|s|u|sdw5|sdw5|7083|/data/primary83 +86|84|p|p|s|u|sdw5|sdw5|7084|/data/primary84 +87|85|p|p|s|u|sdw5|sdw5|7085|/data/primary85 +88|86|p|p|s|u|sdw5|sdw5|7086|/data/primary86 +89|87|p|p|s|u|sdw5|sdw5|7087|/data/primary87 +90|88|p|p|s|u|sdw5|sdw5|7088|/data/primary88 +91|89|p|p|s|u|sdw5|sdw5|7089|/data/primary89 +92|90|p|p|s|u|sdw5|sdw5|7090|/data/primary90 +93|91|p|p|s|u|sdw5|sdw5|7091|/data/primary91 +94|92|p|p|s|u|sdw5|sdw5|7092|/data/primary92 +95|93|p|p|s|u|sdw5|sdw5|7093|/data/primary93 +96|94|p|p|s|u|sdw5|sdw5|7094|/data/primary94 +97|95|p|p|s|u|sdw5|sdw5|7095|/data/primary95 +98|96|p|p|s|u|sdw5|sdw5|7096|/data/primary96 +99|97|p|p|s|u|sdw5|sdw5|7097|/data/primary97 +100|98|p|p|s|u|sdw5|sdw5|7098|/data/primary98 +101|99|p|p|s|u|sdw5|sdw5|7099|/data/primary99 +1062|60|m|m|s|u|sdw5|sdw5|8060|/data/mirror60 +1063|61|m|m|s|u|sdw5|sdw5|8061|/data/mirror61 +1064|62|m|m|s|u|sdw5|sdw5|8062|/data/mirror62 +1065|63|m|m|s|u|sdw5|sdw5|8063|/data/mirror63 +1066|64|m|m|s|u|sdw5|sdw5|8064|/data/mirror64 +1067|65|m|m|s|u|sdw5|sdw5|8065|/data/mirror65 +1068|66|m|m|s|u|sdw5|sdw5|8066|/data/mirror66 +1069|67|m|m|s|u|sdw5|sdw5|8067|/data/mirror67 +1070|68|m|m|s|u|sdw5|sdw5|8068|/data/mirror68 +1071|69|m|m|s|u|sdw5|sdw5|8069|/data/mirror69 +1072|70|m|m|s|u|sdw5|sdw5|8070|/data/mirror70 +1073|71|m|m|s|u|sdw5|sdw5|8071|/data/mirror71 +1074|72|m|m|s|u|sdw5|sdw5|8072|/data/mirror72 +1075|73|m|m|s|u|sdw5|sdw5|8073|/data/mirror73 +1076|74|m|m|s|u|sdw5|sdw5|8074|/data/mirror74 +1077|75|m|m|s|u|sdw5|sdw5|8075|/data/mirror75 +1078|76|m|m|s|u|sdw5|sdw5|8076|/data/mirror76 +1079|77|m|m|s|u|sdw5|sdw5|8077|/data/mirror77 +1080|78|m|m|s|u|sdw5|sdw5|8078|/data/mirror78 +1081|79|m|m|s|u|sdw5|sdw5|8079|/data/mirror79 +# SDW50 +982|980|p|p|s|u|sdw50|sdw50|7980|/data/primary980 +983|981|p|p|s|u|sdw50|sdw50|7981|/data/primary981 +984|982|p|p|s|u|sdw50|sdw50|7982|/data/primary982 +985|983|p|p|s|u|sdw50|sdw50|7983|/data/primary983 +986|984|p|p|s|u|sdw50|sdw50|7984|/data/primary984 +987|985|p|p|s|u|sdw50|sdw50|7985|/data/primary985 +988|986|p|p|s|u|sdw50|sdw50|7986|/data/primary986 +989|987|p|p|s|u|sdw50|sdw50|7987|/data/primary987 +990|988|p|p|s|u|sdw50|sdw50|7988|/data/primary988 +991|989|p|p|s|u|sdw50|sdw50|7989|/data/primary989 +992|990|p|p|s|u|sdw50|sdw50|7990|/data/primary990 +993|991|p|p|s|u|sdw50|sdw50|7991|/data/primary991 +994|992|p|p|s|u|sdw50|sdw50|7992|/data/primary992 +995|993|p|p|s|u|sdw50|sdw50|7993|/data/primary993 +996|994|p|p|s|u|sdw50|sdw50|7994|/data/primary994 +997|995|p|p|s|u|sdw50|sdw50|7995|/data/primary995 +998|996|p|p|s|u|sdw50|sdw50|7996|/data/primary996 +999|997|p|p|s|u|sdw50|sdw50|7997|/data/primary997 +1000|998|p|p|s|u|sdw50|sdw50|7998|/data/primary998 +1001|999|p|p|s|u|sdw50|sdw50|7999|/data/primary999 +1962|960|m|m|s|u|sdw50|sdw50|8960|/data/mirror960 +1963|961|m|m|s|u|sdw50|sdw50|8961|/data/mirror961 +1964|962|m|m|s|u|sdw50|sdw50|8962|/data/mirror962 +1965|963|m|m|s|u|sdw50|sdw50|8963|/data/mirror963 +1966|964|m|m|s|u|sdw50|sdw50|8964|/data/mirror964 +1967|965|m|m|s|u|sdw50|sdw50|8965|/data/mirror965 +1968|966|m|m|s|u|sdw50|sdw50|8966|/data/mirror966 +1969|967|m|m|s|u|sdw50|sdw50|8967|/data/mirror967 +1970|968|m|m|s|u|sdw50|sdw50|8968|/data/mirror968 +1971|969|m|m|s|u|sdw50|sdw50|8969|/data/mirror969 +1972|970|m|m|s|u|sdw50|sdw50|8970|/data/mirror970 +1973|971|m|m|s|u|sdw50|sdw50|8971|/data/mirror971 +1974|972|m|m|s|u|sdw50|sdw50|8972|/data/mirror972 +1975|973|m|m|s|u|sdw50|sdw50|8973|/data/mirror973 +1976|974|m|m|s|u|sdw50|sdw50|8974|/data/mirror974 +1977|975|m|m|s|u|sdw50|sdw50|8975|/data/mirror975 +1978|976|m|m|s|u|sdw50|sdw50|8976|/data/mirror976 +1979|977|m|m|s|u|sdw50|sdw50|8977|/data/mirror977 +1980|978|m|m|s|u|sdw50|sdw50|8978|/data/mirror978 +1981|979|m|m|s|u|sdw50|sdw50|8979|/data/mirror979 +# SDW6 +102|100|p|p|s|u|sdw6|sdw6|7100|/data/primary100 +103|101|p|p|s|u|sdw6|sdw6|7101|/data/primary101 +104|102|p|p|s|u|sdw6|sdw6|7102|/data/primary102 +105|103|p|p|s|u|sdw6|sdw6|7103|/data/primary103 +106|104|p|p|s|u|sdw6|sdw6|7104|/data/primary104 +107|105|p|p|s|u|sdw6|sdw6|7105|/data/primary105 +108|106|p|p|s|u|sdw6|sdw6|7106|/data/primary106 +109|107|p|p|s|u|sdw6|sdw6|7107|/data/primary107 +110|108|p|p|s|u|sdw6|sdw6|7108|/data/primary108 +111|109|p|p|s|u|sdw6|sdw6|7109|/data/primary109 +112|110|p|p|s|u|sdw6|sdw6|7110|/data/primary110 +113|111|p|p|s|u|sdw6|sdw6|7111|/data/primary111 +114|112|p|p|s|u|sdw6|sdw6|7112|/data/primary112 +115|113|p|p|s|u|sdw6|sdw6|7113|/data/primary113 +116|114|p|p|s|u|sdw6|sdw6|7114|/data/primary114 +117|115|p|p|s|u|sdw6|sdw6|7115|/data/primary115 +118|116|p|p|s|u|sdw6|sdw6|7116|/data/primary116 +119|117|p|p|s|u|sdw6|sdw6|7117|/data/primary117 +120|118|p|p|s|u|sdw6|sdw6|7118|/data/primary118 +121|119|p|p|s|u|sdw6|sdw6|7119|/data/primary119 +1082|80|m|m|s|u|sdw6|sdw6|8080|/data/mirror80 +1083|81|m|m|s|u|sdw6|sdw6|8081|/data/mirror81 +1084|82|m|m|s|u|sdw6|sdw6|8082|/data/mirror82 +1085|83|m|m|s|u|sdw6|sdw6|8083|/data/mirror83 +1086|84|m|m|s|u|sdw6|sdw6|8084|/data/mirror84 +1087|85|m|m|s|u|sdw6|sdw6|8085|/data/mirror85 +1088|86|m|m|s|u|sdw6|sdw6|8086|/data/mirror86 +1089|87|m|m|s|u|sdw6|sdw6|8087|/data/mirror87 +1090|88|m|m|s|u|sdw6|sdw6|8088|/data/mirror88 +1091|89|m|m|s|u|sdw6|sdw6|8089|/data/mirror89 +1092|90|m|m|s|u|sdw6|sdw6|8090|/data/mirror90 +1093|91|m|m|s|u|sdw6|sdw6|8091|/data/mirror91 +1094|92|m|m|s|u|sdw6|sdw6|8092|/data/mirror92 +1095|93|m|m|s|u|sdw6|sdw6|8093|/data/mirror93 +1096|94|m|m|s|u|sdw6|sdw6|8094|/data/mirror94 +1097|95|m|m|s|u|sdw6|sdw6|8095|/data/mirror95 +1098|96|m|m|s|u|sdw6|sdw6|8096|/data/mirror96 +1099|97|m|m|s|u|sdw6|sdw6|8097|/data/mirror97 +1100|98|m|m|s|u|sdw6|sdw6|8098|/data/mirror98 +1101|99|m|m|s|u|sdw6|sdw6|8099|/data/mirror99 +# SDW7 +122|120|p|p|s|u|sdw7|sdw7|7120|/data/primary120 +123|121|p|p|s|u|sdw7|sdw7|7121|/data/primary121 +124|122|p|p|s|u|sdw7|sdw7|7122|/data/primary122 +125|123|p|p|s|u|sdw7|sdw7|7123|/data/primary123 +126|124|p|p|s|u|sdw7|sdw7|7124|/data/primary124 +127|125|p|p|s|u|sdw7|sdw7|7125|/data/primary125 +128|126|p|p|s|u|sdw7|sdw7|7126|/data/primary126 +129|127|p|p|s|u|sdw7|sdw7|7127|/data/primary127 +130|128|p|p|s|u|sdw7|sdw7|7128|/data/primary128 +131|129|p|p|s|u|sdw7|sdw7|7129|/data/primary129 +132|130|p|p|s|u|sdw7|sdw7|7130|/data/primary130 +133|131|p|p|s|u|sdw7|sdw7|7131|/data/primary131 +134|132|p|p|s|u|sdw7|sdw7|7132|/data/primary132 +135|133|p|p|s|u|sdw7|sdw7|7133|/data/primary133 +136|134|p|p|s|u|sdw7|sdw7|7134|/data/primary134 +137|135|p|p|s|u|sdw7|sdw7|7135|/data/primary135 +138|136|p|p|s|u|sdw7|sdw7|7136|/data/primary136 +139|137|p|p|s|u|sdw7|sdw7|7137|/data/primary137 +140|138|p|p|s|u|sdw7|sdw7|7138|/data/primary138 +141|139|p|p|s|u|sdw7|sdw7|7139|/data/primary139 +1102|100|m|m|s|u|sdw7|sdw7|8100|/data/mirror100 +1103|101|m|m|s|u|sdw7|sdw7|8101|/data/mirror101 +1104|102|m|m|s|u|sdw7|sdw7|8102|/data/mirror102 +1105|103|m|m|s|u|sdw7|sdw7|8103|/data/mirror103 +1106|104|m|m|s|u|sdw7|sdw7|8104|/data/mirror104 +1107|105|m|m|s|u|sdw7|sdw7|8105|/data/mirror105 +1108|106|m|m|s|u|sdw7|sdw7|8106|/data/mirror106 +1109|107|m|m|s|u|sdw7|sdw7|8107|/data/mirror107 +1110|108|m|m|s|u|sdw7|sdw7|8108|/data/mirror108 +1111|109|m|m|s|u|sdw7|sdw7|8109|/data/mirror109 +1112|110|m|m|s|u|sdw7|sdw7|8110|/data/mirror110 +1113|111|m|m|s|u|sdw7|sdw7|8111|/data/mirror111 +1114|112|m|m|s|u|sdw7|sdw7|8112|/data/mirror112 +1115|113|m|m|s|u|sdw7|sdw7|8113|/data/mirror113 +1116|114|m|m|s|u|sdw7|sdw7|8114|/data/mirror114 +1117|115|m|m|s|u|sdw7|sdw7|8115|/data/mirror115 +1118|116|m|m|s|u|sdw7|sdw7|8116|/data/mirror116 +1119|117|m|m|s|u|sdw7|sdw7|8117|/data/mirror117 +1120|118|m|m|s|u|sdw7|sdw7|8118|/data/mirror118 +1121|119|m|m|s|u|sdw7|sdw7|8119|/data/mirror119 +# SDW8 +142|140|p|p|s|u|sdw8|sdw8|7140|/data/primary140 +143|141|p|p|s|u|sdw8|sdw8|7141|/data/primary141 +144|142|p|p|s|u|sdw8|sdw8|7142|/data/primary142 +145|143|p|p|s|u|sdw8|sdw8|7143|/data/primary143 +146|144|p|p|s|u|sdw8|sdw8|7144|/data/primary144 +147|145|p|p|s|u|sdw8|sdw8|7145|/data/primary145 +148|146|p|p|s|u|sdw8|sdw8|7146|/data/primary146 +149|147|p|p|s|u|sdw8|sdw8|7147|/data/primary147 +150|148|p|p|s|u|sdw8|sdw8|7148|/data/primary148 +151|149|p|p|s|u|sdw8|sdw8|7149|/data/primary149 +152|150|p|p|s|u|sdw8|sdw8|7150|/data/primary150 +153|151|p|p|s|u|sdw8|sdw8|7151|/data/primary151 +154|152|p|p|s|u|sdw8|sdw8|7152|/data/primary152 +155|153|p|p|s|u|sdw8|sdw8|7153|/data/primary153 +156|154|p|p|s|u|sdw8|sdw8|7154|/data/primary154 +157|155|p|p|s|u|sdw8|sdw8|7155|/data/primary155 +158|156|p|p|s|u|sdw8|sdw8|7156|/data/primary156 +159|157|p|p|s|u|sdw8|sdw8|7157|/data/primary157 +160|158|p|p|s|u|sdw8|sdw8|7158|/data/primary158 +161|159|p|p|s|u|sdw8|sdw8|7159|/data/primary159 +1122|120|m|m|s|u|sdw8|sdw8|8120|/data/mirror120 +1123|121|m|m|s|u|sdw8|sdw8|8121|/data/mirror121 +1124|122|m|m|s|u|sdw8|sdw8|8122|/data/mirror122 +1125|123|m|m|s|u|sdw8|sdw8|8123|/data/mirror123 +1126|124|m|m|s|u|sdw8|sdw8|8124|/data/mirror124 +1127|125|m|m|s|u|sdw8|sdw8|8125|/data/mirror125 +1128|126|m|m|s|u|sdw8|sdw8|8126|/data/mirror126 +1129|127|m|m|s|u|sdw8|sdw8|8127|/data/mirror127 +1130|128|m|m|s|u|sdw8|sdw8|8128|/data/mirror128 +1131|129|m|m|s|u|sdw8|sdw8|8129|/data/mirror129 +1132|130|m|m|s|u|sdw8|sdw8|8130|/data/mirror130 +1133|131|m|m|s|u|sdw8|sdw8|8131|/data/mirror131 +1134|132|m|m|s|u|sdw8|sdw8|8132|/data/mirror132 +1135|133|m|m|s|u|sdw8|sdw8|8133|/data/mirror133 +1136|134|m|m|s|u|sdw8|sdw8|8134|/data/mirror134 +1137|135|m|m|s|u|sdw8|sdw8|8135|/data/mirror135 +1138|136|m|m|s|u|sdw8|sdw8|8136|/data/mirror136 +1139|137|m|m|s|u|sdw8|sdw8|8137|/data/mirror137 +1140|138|m|m|s|u|sdw8|sdw8|8138|/data/mirror138 +1141|139|m|m|s|u|sdw8|sdw8|8139|/data/mirror139 +# SDW9 +162|160|p|p|s|u|sdw9|sdw9|7160|/data/primary160 +163|161|p|p|s|u|sdw9|sdw9|7161|/data/primary161 +164|162|p|p|s|u|sdw9|sdw9|7162|/data/primary162 +165|163|p|p|s|u|sdw9|sdw9|7163|/data/primary163 +166|164|p|p|s|u|sdw9|sdw9|7164|/data/primary164 +167|165|p|p|s|u|sdw9|sdw9|7165|/data/primary165 +168|166|p|p|s|u|sdw9|sdw9|7166|/data/primary166 +169|167|p|p|s|u|sdw9|sdw9|7167|/data/primary167 +170|168|p|p|s|u|sdw9|sdw9|7168|/data/primary168 +171|169|p|p|s|u|sdw9|sdw9|7169|/data/primary169 +172|170|p|p|s|u|sdw9|sdw9|7170|/data/primary170 +173|171|p|p|s|u|sdw9|sdw9|7171|/data/primary171 +174|172|p|p|s|u|sdw9|sdw9|7172|/data/primary172 +175|173|p|p|s|u|sdw9|sdw9|7173|/data/primary173 +176|174|p|p|s|u|sdw9|sdw9|7174|/data/primary174 +177|175|p|p|s|u|sdw9|sdw9|7175|/data/primary175 +178|176|p|p|s|u|sdw9|sdw9|7176|/data/primary176 +179|177|p|p|s|u|sdw9|sdw9|7177|/data/primary177 +180|178|p|p|s|u|sdw9|sdw9|7178|/data/primary178 +181|179|p|p|s|u|sdw9|sdw9|7179|/data/primary179 +1142|140|m|m|s|u|sdw9|sdw9|8140|/data/mirror140 +1143|141|m|m|s|u|sdw9|sdw9|8141|/data/mirror141 +1144|142|m|m|s|u|sdw9|sdw9|8142|/data/mirror142 +1145|143|m|m|s|u|sdw9|sdw9|8143|/data/mirror143 +1146|144|m|m|s|u|sdw9|sdw9|8144|/data/mirror144 +1147|145|m|m|s|u|sdw9|sdw9|8145|/data/mirror145 +1148|146|m|m|s|u|sdw9|sdw9|8146|/data/mirror146 +1149|147|m|m|s|u|sdw9|sdw9|8147|/data/mirror147 +1150|148|m|m|s|u|sdw9|sdw9|8148|/data/mirror148 +1151|149|m|m|s|u|sdw9|sdw9|8149|/data/mirror149 +1152|150|m|m|s|u|sdw9|sdw9|8150|/data/mirror150 +1153|151|m|m|s|u|sdw9|sdw9|8151|/data/mirror151 +1154|152|m|m|s|u|sdw9|sdw9|8152|/data/mirror152 +1155|153|m|m|s|u|sdw9|sdw9|8153|/data/mirror153 +1156|154|m|m|s|u|sdw9|sdw9|8154|/data/mirror154 +1157|155|m|m|s|u|sdw9|sdw9|8155|/data/mirror155 +1158|156|m|m|s|u|sdw9|sdw9|8156|/data/mirror156 +1159|157|m|m|s|u|sdw9|sdw9|8157|/data/mirror157 +1160|158|m|m|s|u|sdw9|sdw9|8158|/data/mirror158 +1161|159|m|m|s|u|sdw9|sdw9|8159|/data/mirror159 diff --git a/gpMgmt/bin/gprebalance_modules/test/data/1000_50_balanced_spread.array b/gpMgmt/bin/gprebalance_modules/test/data/1000_50_balanced_spread.array new file mode 100644 index 000000000000..4d62750e35f5 --- /dev/null +++ b/gpMgmt/bin/gprebalance_modules/test/data/1000_50_balanced_spread.array @@ -0,0 +1,2052 @@ +# Coordinator +1|-1|p|p|s|u|cdw|cdw|5432|/data/coordinator +# SDW1 +2|0|p|p|s|u|sdw1|sdw1|7000|/data/primary0 +3|1|p|p|s|u|sdw1|sdw1|7001|/data/primary1 +4|2|p|p|s|u|sdw1|sdw1|7002|/data/primary2 +5|3|p|p|s|u|sdw1|sdw1|7003|/data/primary3 +6|4|p|p|s|u|sdw1|sdw1|7004|/data/primary4 +7|5|p|p|s|u|sdw1|sdw1|7005|/data/primary5 +8|6|p|p|s|u|sdw1|sdw1|7006|/data/primary6 +9|7|p|p|s|u|sdw1|sdw1|7007|/data/primary7 +10|8|p|p|s|u|sdw1|sdw1|7008|/data/primary8 +11|9|p|p|s|u|sdw1|sdw1|7009|/data/primary9 +12|10|p|p|s|u|sdw1|sdw1|7010|/data/primary10 +13|11|p|p|s|u|sdw1|sdw1|7011|/data/primary11 +14|12|p|p|s|u|sdw1|sdw1|7012|/data/primary12 +15|13|p|p|s|u|sdw1|sdw1|7013|/data/primary13 +16|14|p|p|s|u|sdw1|sdw1|7014|/data/primary14 +17|15|p|p|s|u|sdw1|sdw1|7015|/data/primary15 +18|16|p|p|s|u|sdw1|sdw1|7016|/data/primary16 +19|17|p|p|s|u|sdw1|sdw1|7017|/data/primary17 +20|18|p|p|s|u|sdw1|sdw1|7018|/data/primary18 +21|19|p|p|s|u|sdw1|sdw1|7019|/data/primary19 +1621|619|m|m|s|u|sdw1|sdw1|8619|/data/mirror619 +1640|638|m|m|s|u|sdw1|sdw1|8638|/data/mirror638 +1659|657|m|m|s|u|sdw1|sdw1|8657|/data/mirror657 +1678|676|m|m|s|u|sdw1|sdw1|8676|/data/mirror676 +1697|695|m|m|s|u|sdw1|sdw1|8695|/data/mirror695 +1716|714|m|m|s|u|sdw1|sdw1|8714|/data/mirror714 +1735|733|m|m|s|u|sdw1|sdw1|8733|/data/mirror733 +1754|752|m|m|s|u|sdw1|sdw1|8752|/data/mirror752 +1773|771|m|m|s|u|sdw1|sdw1|8771|/data/mirror771 +1792|790|m|m|s|u|sdw1|sdw1|8790|/data/mirror790 +1811|809|m|m|s|u|sdw1|sdw1|8809|/data/mirror809 +1830|828|m|m|s|u|sdw1|sdw1|8828|/data/mirror828 +1849|847|m|m|s|u|sdw1|sdw1|8847|/data/mirror847 +1868|866|m|m|s|u|sdw1|sdw1|8866|/data/mirror866 +1887|885|m|m|s|u|sdw1|sdw1|8885|/data/mirror885 +1906|904|m|m|s|u|sdw1|sdw1|8904|/data/mirror904 +1925|923|m|m|s|u|sdw1|sdw1|8923|/data/mirror923 +1944|942|m|m|s|u|sdw1|sdw1|8942|/data/mirror942 +1963|961|m|m|s|u|sdw1|sdw1|8961|/data/mirror961 +1982|980|m|m|s|u|sdw1|sdw1|8980|/data/mirror980 +# SDW10 +182|180|p|p|s|u|sdw10|sdw10|7180|/data/primary180 +183|181|p|p|s|u|sdw10|sdw10|7181|/data/primary181 +184|182|p|p|s|u|sdw10|sdw10|7182|/data/primary182 +185|183|p|p|s|u|sdw10|sdw10|7183|/data/primary183 +186|184|p|p|s|u|sdw10|sdw10|7184|/data/primary184 +187|185|p|p|s|u|sdw10|sdw10|7185|/data/primary185 +188|186|p|p|s|u|sdw10|sdw10|7186|/data/primary186 +189|187|p|p|s|u|sdw10|sdw10|7187|/data/primary187 +190|188|p|p|s|u|sdw10|sdw10|7188|/data/primary188 +191|189|p|p|s|u|sdw10|sdw10|7189|/data/primary189 +192|190|p|p|s|u|sdw10|sdw10|7190|/data/primary190 +193|191|p|p|s|u|sdw10|sdw10|7191|/data/primary191 +194|192|p|p|s|u|sdw10|sdw10|7192|/data/primary192 +195|193|p|p|s|u|sdw10|sdw10|7193|/data/primary193 +196|194|p|p|s|u|sdw10|sdw10|7194|/data/primary194 +197|195|p|p|s|u|sdw10|sdw10|7195|/data/primary195 +198|196|p|p|s|u|sdw10|sdw10|7196|/data/primary196 +199|197|p|p|s|u|sdw10|sdw10|7197|/data/primary197 +200|198|p|p|s|u|sdw10|sdw10|7198|/data/primary198 +201|199|p|p|s|u|sdw10|sdw10|7199|/data/primary199 +1010|8|m|m|s|u|sdw10|sdw10|8008|/data/mirror8 +1029|27|m|m|s|u|sdw10|sdw10|8027|/data/mirror27 +1048|46|m|m|s|u|sdw10|sdw10|8046|/data/mirror46 +1067|65|m|m|s|u|sdw10|sdw10|8065|/data/mirror65 +1086|84|m|m|s|u|sdw10|sdw10|8084|/data/mirror84 +1105|103|m|m|s|u|sdw10|sdw10|8103|/data/mirror103 +1124|122|m|m|s|u|sdw10|sdw10|8122|/data/mirror122 +1143|141|m|m|s|u|sdw10|sdw10|8141|/data/mirror141 +1162|160|m|m|s|u|sdw10|sdw10|8160|/data/mirror160 +1801|799|m|m|s|u|sdw10|sdw10|8799|/data/mirror799 +1820|818|m|m|s|u|sdw10|sdw10|8818|/data/mirror818 +1839|837|m|m|s|u|sdw10|sdw10|8837|/data/mirror837 +1858|856|m|m|s|u|sdw10|sdw10|8856|/data/mirror856 +1877|875|m|m|s|u|sdw10|sdw10|8875|/data/mirror875 +1896|894|m|m|s|u|sdw10|sdw10|8894|/data/mirror894 +1915|913|m|m|s|u|sdw10|sdw10|8913|/data/mirror913 +1934|932|m|m|s|u|sdw10|sdw10|8932|/data/mirror932 +1953|951|m|m|s|u|sdw10|sdw10|8951|/data/mirror951 +1972|970|m|m|s|u|sdw10|sdw10|8970|/data/mirror970 +1991|989|m|m|s|u|sdw10|sdw10|8989|/data/mirror989 +# SDW11 +202|200|p|p|s|u|sdw11|sdw11|7200|/data/primary200 +203|201|p|p|s|u|sdw11|sdw11|7201|/data/primary201 +204|202|p|p|s|u|sdw11|sdw11|7202|/data/primary202 +205|203|p|p|s|u|sdw11|sdw11|7203|/data/primary203 +206|204|p|p|s|u|sdw11|sdw11|7204|/data/primary204 +207|205|p|p|s|u|sdw11|sdw11|7205|/data/primary205 +208|206|p|p|s|u|sdw11|sdw11|7206|/data/primary206 +209|207|p|p|s|u|sdw11|sdw11|7207|/data/primary207 +210|208|p|p|s|u|sdw11|sdw11|7208|/data/primary208 +211|209|p|p|s|u|sdw11|sdw11|7209|/data/primary209 +212|210|p|p|s|u|sdw11|sdw11|7210|/data/primary210 +213|211|p|p|s|u|sdw11|sdw11|7211|/data/primary211 +214|212|p|p|s|u|sdw11|sdw11|7212|/data/primary212 +215|213|p|p|s|u|sdw11|sdw11|7213|/data/primary213 +216|214|p|p|s|u|sdw11|sdw11|7214|/data/primary214 +217|215|p|p|s|u|sdw11|sdw11|7215|/data/primary215 +218|216|p|p|s|u|sdw11|sdw11|7216|/data/primary216 +219|217|p|p|s|u|sdw11|sdw11|7217|/data/primary217 +220|218|p|p|s|u|sdw11|sdw11|7218|/data/primary218 +221|219|p|p|s|u|sdw11|sdw11|7219|/data/primary219 +1011|9|m|m|s|u|sdw11|sdw11|8009|/data/mirror9 +1030|28|m|m|s|u|sdw11|sdw11|8028|/data/mirror28 +1049|47|m|m|s|u|sdw11|sdw11|8047|/data/mirror47 +1068|66|m|m|s|u|sdw11|sdw11|8066|/data/mirror66 +1087|85|m|m|s|u|sdw11|sdw11|8085|/data/mirror85 +1106|104|m|m|s|u|sdw11|sdw11|8104|/data/mirror104 +1125|123|m|m|s|u|sdw11|sdw11|8123|/data/mirror123 +1144|142|m|m|s|u|sdw11|sdw11|8142|/data/mirror142 +1163|161|m|m|s|u|sdw11|sdw11|8161|/data/mirror161 +1182|180|m|m|s|u|sdw11|sdw11|8180|/data/mirror180 +1821|819|m|m|s|u|sdw11|sdw11|8819|/data/mirror819 +1840|838|m|m|s|u|sdw11|sdw11|8838|/data/mirror838 +1859|857|m|m|s|u|sdw11|sdw11|8857|/data/mirror857 +1878|876|m|m|s|u|sdw11|sdw11|8876|/data/mirror876 +1897|895|m|m|s|u|sdw11|sdw11|8895|/data/mirror895 +1916|914|m|m|s|u|sdw11|sdw11|8914|/data/mirror914 +1935|933|m|m|s|u|sdw11|sdw11|8933|/data/mirror933 +1954|952|m|m|s|u|sdw11|sdw11|8952|/data/mirror952 +1973|971|m|m|s|u|sdw11|sdw11|8971|/data/mirror971 +1992|990|m|m|s|u|sdw11|sdw11|8990|/data/mirror990 +# SDW12 +222|220|p|p|s|u|sdw12|sdw12|7220|/data/primary220 +223|221|p|p|s|u|sdw12|sdw12|7221|/data/primary221 +224|222|p|p|s|u|sdw12|sdw12|7222|/data/primary222 +225|223|p|p|s|u|sdw12|sdw12|7223|/data/primary223 +226|224|p|p|s|u|sdw12|sdw12|7224|/data/primary224 +227|225|p|p|s|u|sdw12|sdw12|7225|/data/primary225 +228|226|p|p|s|u|sdw12|sdw12|7226|/data/primary226 +229|227|p|p|s|u|sdw12|sdw12|7227|/data/primary227 +230|228|p|p|s|u|sdw12|sdw12|7228|/data/primary228 +231|229|p|p|s|u|sdw12|sdw12|7229|/data/primary229 +232|230|p|p|s|u|sdw12|sdw12|7230|/data/primary230 +233|231|p|p|s|u|sdw12|sdw12|7231|/data/primary231 +234|232|p|p|s|u|sdw12|sdw12|7232|/data/primary232 +235|233|p|p|s|u|sdw12|sdw12|7233|/data/primary233 +236|234|p|p|s|u|sdw12|sdw12|7234|/data/primary234 +237|235|p|p|s|u|sdw12|sdw12|7235|/data/primary235 +238|236|p|p|s|u|sdw12|sdw12|7236|/data/primary236 +239|237|p|p|s|u|sdw12|sdw12|7237|/data/primary237 +240|238|p|p|s|u|sdw12|sdw12|7238|/data/primary238 +241|239|p|p|s|u|sdw12|sdw12|7239|/data/primary239 +1012|10|m|m|s|u|sdw12|sdw12|8010|/data/mirror10 +1031|29|m|m|s|u|sdw12|sdw12|8029|/data/mirror29 +1050|48|m|m|s|u|sdw12|sdw12|8048|/data/mirror48 +1069|67|m|m|s|u|sdw12|sdw12|8067|/data/mirror67 +1088|86|m|m|s|u|sdw12|sdw12|8086|/data/mirror86 +1107|105|m|m|s|u|sdw12|sdw12|8105|/data/mirror105 +1126|124|m|m|s|u|sdw12|sdw12|8124|/data/mirror124 +1145|143|m|m|s|u|sdw12|sdw12|8143|/data/mirror143 +1164|162|m|m|s|u|sdw12|sdw12|8162|/data/mirror162 +1183|181|m|m|s|u|sdw12|sdw12|8181|/data/mirror181 +1202|200|m|m|s|u|sdw12|sdw12|8200|/data/mirror200 +1841|839|m|m|s|u|sdw12|sdw12|8839|/data/mirror839 +1860|858|m|m|s|u|sdw12|sdw12|8858|/data/mirror858 +1879|877|m|m|s|u|sdw12|sdw12|8877|/data/mirror877 +1898|896|m|m|s|u|sdw12|sdw12|8896|/data/mirror896 +1917|915|m|m|s|u|sdw12|sdw12|8915|/data/mirror915 +1936|934|m|m|s|u|sdw12|sdw12|8934|/data/mirror934 +1955|953|m|m|s|u|sdw12|sdw12|8953|/data/mirror953 +1974|972|m|m|s|u|sdw12|sdw12|8972|/data/mirror972 +1993|991|m|m|s|u|sdw12|sdw12|8991|/data/mirror991 +# SDW13 +242|240|p|p|s|u|sdw13|sdw13|7240|/data/primary240 +243|241|p|p|s|u|sdw13|sdw13|7241|/data/primary241 +244|242|p|p|s|u|sdw13|sdw13|7242|/data/primary242 +245|243|p|p|s|u|sdw13|sdw13|7243|/data/primary243 +246|244|p|p|s|u|sdw13|sdw13|7244|/data/primary244 +247|245|p|p|s|u|sdw13|sdw13|7245|/data/primary245 +248|246|p|p|s|u|sdw13|sdw13|7246|/data/primary246 +249|247|p|p|s|u|sdw13|sdw13|7247|/data/primary247 +250|248|p|p|s|u|sdw13|sdw13|7248|/data/primary248 +251|249|p|p|s|u|sdw13|sdw13|7249|/data/primary249 +252|250|p|p|s|u|sdw13|sdw13|7250|/data/primary250 +253|251|p|p|s|u|sdw13|sdw13|7251|/data/primary251 +254|252|p|p|s|u|sdw13|sdw13|7252|/data/primary252 +255|253|p|p|s|u|sdw13|sdw13|7253|/data/primary253 +256|254|p|p|s|u|sdw13|sdw13|7254|/data/primary254 +257|255|p|p|s|u|sdw13|sdw13|7255|/data/primary255 +258|256|p|p|s|u|sdw13|sdw13|7256|/data/primary256 +259|257|p|p|s|u|sdw13|sdw13|7257|/data/primary257 +260|258|p|p|s|u|sdw13|sdw13|7258|/data/primary258 +261|259|p|p|s|u|sdw13|sdw13|7259|/data/primary259 +1013|11|m|m|s|u|sdw13|sdw13|8011|/data/mirror11 +1032|30|m|m|s|u|sdw13|sdw13|8030|/data/mirror30 +1051|49|m|m|s|u|sdw13|sdw13|8049|/data/mirror49 +1070|68|m|m|s|u|sdw13|sdw13|8068|/data/mirror68 +1089|87|m|m|s|u|sdw13|sdw13|8087|/data/mirror87 +1108|106|m|m|s|u|sdw13|sdw13|8106|/data/mirror106 +1127|125|m|m|s|u|sdw13|sdw13|8125|/data/mirror125 +1146|144|m|m|s|u|sdw13|sdw13|8144|/data/mirror144 +1165|163|m|m|s|u|sdw13|sdw13|8163|/data/mirror163 +1184|182|m|m|s|u|sdw13|sdw13|8182|/data/mirror182 +1203|201|m|m|s|u|sdw13|sdw13|8201|/data/mirror201 +1222|220|m|m|s|u|sdw13|sdw13|8220|/data/mirror220 +1861|859|m|m|s|u|sdw13|sdw13|8859|/data/mirror859 +1880|878|m|m|s|u|sdw13|sdw13|8878|/data/mirror878 +1899|897|m|m|s|u|sdw13|sdw13|8897|/data/mirror897 +1918|916|m|m|s|u|sdw13|sdw13|8916|/data/mirror916 +1937|935|m|m|s|u|sdw13|sdw13|8935|/data/mirror935 +1956|954|m|m|s|u|sdw13|sdw13|8954|/data/mirror954 +1975|973|m|m|s|u|sdw13|sdw13|8973|/data/mirror973 +1994|992|m|m|s|u|sdw13|sdw13|8992|/data/mirror992 +# SDW14 +262|260|p|p|s|u|sdw14|sdw14|7260|/data/primary260 +263|261|p|p|s|u|sdw14|sdw14|7261|/data/primary261 +264|262|p|p|s|u|sdw14|sdw14|7262|/data/primary262 +265|263|p|p|s|u|sdw14|sdw14|7263|/data/primary263 +266|264|p|p|s|u|sdw14|sdw14|7264|/data/primary264 +267|265|p|p|s|u|sdw14|sdw14|7265|/data/primary265 +268|266|p|p|s|u|sdw14|sdw14|7266|/data/primary266 +269|267|p|p|s|u|sdw14|sdw14|7267|/data/primary267 +270|268|p|p|s|u|sdw14|sdw14|7268|/data/primary268 +271|269|p|p|s|u|sdw14|sdw14|7269|/data/primary269 +272|270|p|p|s|u|sdw14|sdw14|7270|/data/primary270 +273|271|p|p|s|u|sdw14|sdw14|7271|/data/primary271 +274|272|p|p|s|u|sdw14|sdw14|7272|/data/primary272 +275|273|p|p|s|u|sdw14|sdw14|7273|/data/primary273 +276|274|p|p|s|u|sdw14|sdw14|7274|/data/primary274 +277|275|p|p|s|u|sdw14|sdw14|7275|/data/primary275 +278|276|p|p|s|u|sdw14|sdw14|7276|/data/primary276 +279|277|p|p|s|u|sdw14|sdw14|7277|/data/primary277 +280|278|p|p|s|u|sdw14|sdw14|7278|/data/primary278 +281|279|p|p|s|u|sdw14|sdw14|7279|/data/primary279 +1014|12|m|m|s|u|sdw14|sdw14|8012|/data/mirror12 +1033|31|m|m|s|u|sdw14|sdw14|8031|/data/mirror31 +1052|50|m|m|s|u|sdw14|sdw14|8050|/data/mirror50 +1071|69|m|m|s|u|sdw14|sdw14|8069|/data/mirror69 +1090|88|m|m|s|u|sdw14|sdw14|8088|/data/mirror88 +1109|107|m|m|s|u|sdw14|sdw14|8107|/data/mirror107 +1128|126|m|m|s|u|sdw14|sdw14|8126|/data/mirror126 +1147|145|m|m|s|u|sdw14|sdw14|8145|/data/mirror145 +1166|164|m|m|s|u|sdw14|sdw14|8164|/data/mirror164 +1185|183|m|m|s|u|sdw14|sdw14|8183|/data/mirror183 +1204|202|m|m|s|u|sdw14|sdw14|8202|/data/mirror202 +1223|221|m|m|s|u|sdw14|sdw14|8221|/data/mirror221 +1242|240|m|m|s|u|sdw14|sdw14|8240|/data/mirror240 +1881|879|m|m|s|u|sdw14|sdw14|8879|/data/mirror879 +1900|898|m|m|s|u|sdw14|sdw14|8898|/data/mirror898 +1919|917|m|m|s|u|sdw14|sdw14|8917|/data/mirror917 +1938|936|m|m|s|u|sdw14|sdw14|8936|/data/mirror936 +1957|955|m|m|s|u|sdw14|sdw14|8955|/data/mirror955 +1976|974|m|m|s|u|sdw14|sdw14|8974|/data/mirror974 +1995|993|m|m|s|u|sdw14|sdw14|8993|/data/mirror993 +# SDW15 +282|280|p|p|s|u|sdw15|sdw15|7280|/data/primary280 +283|281|p|p|s|u|sdw15|sdw15|7281|/data/primary281 +284|282|p|p|s|u|sdw15|sdw15|7282|/data/primary282 +285|283|p|p|s|u|sdw15|sdw15|7283|/data/primary283 +286|284|p|p|s|u|sdw15|sdw15|7284|/data/primary284 +287|285|p|p|s|u|sdw15|sdw15|7285|/data/primary285 +288|286|p|p|s|u|sdw15|sdw15|7286|/data/primary286 +289|287|p|p|s|u|sdw15|sdw15|7287|/data/primary287 +290|288|p|p|s|u|sdw15|sdw15|7288|/data/primary288 +291|289|p|p|s|u|sdw15|sdw15|7289|/data/primary289 +292|290|p|p|s|u|sdw15|sdw15|7290|/data/primary290 +293|291|p|p|s|u|sdw15|sdw15|7291|/data/primary291 +294|292|p|p|s|u|sdw15|sdw15|7292|/data/primary292 +295|293|p|p|s|u|sdw15|sdw15|7293|/data/primary293 +296|294|p|p|s|u|sdw15|sdw15|7294|/data/primary294 +297|295|p|p|s|u|sdw15|sdw15|7295|/data/primary295 +298|296|p|p|s|u|sdw15|sdw15|7296|/data/primary296 +299|297|p|p|s|u|sdw15|sdw15|7297|/data/primary297 +300|298|p|p|s|u|sdw15|sdw15|7298|/data/primary298 +301|299|p|p|s|u|sdw15|sdw15|7299|/data/primary299 +1015|13|m|m|s|u|sdw15|sdw15|8013|/data/mirror13 +1034|32|m|m|s|u|sdw15|sdw15|8032|/data/mirror32 +1053|51|m|m|s|u|sdw15|sdw15|8051|/data/mirror51 +1072|70|m|m|s|u|sdw15|sdw15|8070|/data/mirror70 +1091|89|m|m|s|u|sdw15|sdw15|8089|/data/mirror89 +1110|108|m|m|s|u|sdw15|sdw15|8108|/data/mirror108 +1129|127|m|m|s|u|sdw15|sdw15|8127|/data/mirror127 +1148|146|m|m|s|u|sdw15|sdw15|8146|/data/mirror146 +1167|165|m|m|s|u|sdw15|sdw15|8165|/data/mirror165 +1186|184|m|m|s|u|sdw15|sdw15|8184|/data/mirror184 +1205|203|m|m|s|u|sdw15|sdw15|8203|/data/mirror203 +1224|222|m|m|s|u|sdw15|sdw15|8222|/data/mirror222 +1243|241|m|m|s|u|sdw15|sdw15|8241|/data/mirror241 +1262|260|m|m|s|u|sdw15|sdw15|8260|/data/mirror260 +1901|899|m|m|s|u|sdw15|sdw15|8899|/data/mirror899 +1920|918|m|m|s|u|sdw15|sdw15|8918|/data/mirror918 +1939|937|m|m|s|u|sdw15|sdw15|8937|/data/mirror937 +1958|956|m|m|s|u|sdw15|sdw15|8956|/data/mirror956 +1977|975|m|m|s|u|sdw15|sdw15|8975|/data/mirror975 +1996|994|m|m|s|u|sdw15|sdw15|8994|/data/mirror994 +# SDW16 +302|300|p|p|s|u|sdw16|sdw16|7300|/data/primary300 +303|301|p|p|s|u|sdw16|sdw16|7301|/data/primary301 +304|302|p|p|s|u|sdw16|sdw16|7302|/data/primary302 +305|303|p|p|s|u|sdw16|sdw16|7303|/data/primary303 +306|304|p|p|s|u|sdw16|sdw16|7304|/data/primary304 +307|305|p|p|s|u|sdw16|sdw16|7305|/data/primary305 +308|306|p|p|s|u|sdw16|sdw16|7306|/data/primary306 +309|307|p|p|s|u|sdw16|sdw16|7307|/data/primary307 +310|308|p|p|s|u|sdw16|sdw16|7308|/data/primary308 +311|309|p|p|s|u|sdw16|sdw16|7309|/data/primary309 +312|310|p|p|s|u|sdw16|sdw16|7310|/data/primary310 +313|311|p|p|s|u|sdw16|sdw16|7311|/data/primary311 +314|312|p|p|s|u|sdw16|sdw16|7312|/data/primary312 +315|313|p|p|s|u|sdw16|sdw16|7313|/data/primary313 +316|314|p|p|s|u|sdw16|sdw16|7314|/data/primary314 +317|315|p|p|s|u|sdw16|sdw16|7315|/data/primary315 +318|316|p|p|s|u|sdw16|sdw16|7316|/data/primary316 +319|317|p|p|s|u|sdw16|sdw16|7317|/data/primary317 +320|318|p|p|s|u|sdw16|sdw16|7318|/data/primary318 +321|319|p|p|s|u|sdw16|sdw16|7319|/data/primary319 +1016|14|m|m|s|u|sdw16|sdw16|8014|/data/mirror14 +1035|33|m|m|s|u|sdw16|sdw16|8033|/data/mirror33 +1054|52|m|m|s|u|sdw16|sdw16|8052|/data/mirror52 +1073|71|m|m|s|u|sdw16|sdw16|8071|/data/mirror71 +1092|90|m|m|s|u|sdw16|sdw16|8090|/data/mirror90 +1111|109|m|m|s|u|sdw16|sdw16|8109|/data/mirror109 +1130|128|m|m|s|u|sdw16|sdw16|8128|/data/mirror128 +1149|147|m|m|s|u|sdw16|sdw16|8147|/data/mirror147 +1168|166|m|m|s|u|sdw16|sdw16|8166|/data/mirror166 +1187|185|m|m|s|u|sdw16|sdw16|8185|/data/mirror185 +1206|204|m|m|s|u|sdw16|sdw16|8204|/data/mirror204 +1225|223|m|m|s|u|sdw16|sdw16|8223|/data/mirror223 +1244|242|m|m|s|u|sdw16|sdw16|8242|/data/mirror242 +1263|261|m|m|s|u|sdw16|sdw16|8261|/data/mirror261 +1282|280|m|m|s|u|sdw16|sdw16|8280|/data/mirror280 +1921|919|m|m|s|u|sdw16|sdw16|8919|/data/mirror919 +1940|938|m|m|s|u|sdw16|sdw16|8938|/data/mirror938 +1959|957|m|m|s|u|sdw16|sdw16|8957|/data/mirror957 +1978|976|m|m|s|u|sdw16|sdw16|8976|/data/mirror976 +1997|995|m|m|s|u|sdw16|sdw16|8995|/data/mirror995 +# SDW17 +322|320|p|p|s|u|sdw17|sdw17|7320|/data/primary320 +323|321|p|p|s|u|sdw17|sdw17|7321|/data/primary321 +324|322|p|p|s|u|sdw17|sdw17|7322|/data/primary322 +325|323|p|p|s|u|sdw17|sdw17|7323|/data/primary323 +326|324|p|p|s|u|sdw17|sdw17|7324|/data/primary324 +327|325|p|p|s|u|sdw17|sdw17|7325|/data/primary325 +328|326|p|p|s|u|sdw17|sdw17|7326|/data/primary326 +329|327|p|p|s|u|sdw17|sdw17|7327|/data/primary327 +330|328|p|p|s|u|sdw17|sdw17|7328|/data/primary328 +331|329|p|p|s|u|sdw17|sdw17|7329|/data/primary329 +332|330|p|p|s|u|sdw17|sdw17|7330|/data/primary330 +333|331|p|p|s|u|sdw17|sdw17|7331|/data/primary331 +334|332|p|p|s|u|sdw17|sdw17|7332|/data/primary332 +335|333|p|p|s|u|sdw17|sdw17|7333|/data/primary333 +336|334|p|p|s|u|sdw17|sdw17|7334|/data/primary334 +337|335|p|p|s|u|sdw17|sdw17|7335|/data/primary335 +338|336|p|p|s|u|sdw17|sdw17|7336|/data/primary336 +339|337|p|p|s|u|sdw17|sdw17|7337|/data/primary337 +340|338|p|p|s|u|sdw17|sdw17|7338|/data/primary338 +341|339|p|p|s|u|sdw17|sdw17|7339|/data/primary339 +1017|15|m|m|s|u|sdw17|sdw17|8015|/data/mirror15 +1036|34|m|m|s|u|sdw17|sdw17|8034|/data/mirror34 +1055|53|m|m|s|u|sdw17|sdw17|8053|/data/mirror53 +1074|72|m|m|s|u|sdw17|sdw17|8072|/data/mirror72 +1093|91|m|m|s|u|sdw17|sdw17|8091|/data/mirror91 +1112|110|m|m|s|u|sdw17|sdw17|8110|/data/mirror110 +1131|129|m|m|s|u|sdw17|sdw17|8129|/data/mirror129 +1150|148|m|m|s|u|sdw17|sdw17|8148|/data/mirror148 +1169|167|m|m|s|u|sdw17|sdw17|8167|/data/mirror167 +1188|186|m|m|s|u|sdw17|sdw17|8186|/data/mirror186 +1207|205|m|m|s|u|sdw17|sdw17|8205|/data/mirror205 +1226|224|m|m|s|u|sdw17|sdw17|8224|/data/mirror224 +1245|243|m|m|s|u|sdw17|sdw17|8243|/data/mirror243 +1264|262|m|m|s|u|sdw17|sdw17|8262|/data/mirror262 +1283|281|m|m|s|u|sdw17|sdw17|8281|/data/mirror281 +1302|300|m|m|s|u|sdw17|sdw17|8300|/data/mirror300 +1941|939|m|m|s|u|sdw17|sdw17|8939|/data/mirror939 +1960|958|m|m|s|u|sdw17|sdw17|8958|/data/mirror958 +1979|977|m|m|s|u|sdw17|sdw17|8977|/data/mirror977 +1998|996|m|m|s|u|sdw17|sdw17|8996|/data/mirror996 +# SDW18 +342|340|p|p|s|u|sdw18|sdw18|7340|/data/primary340 +343|341|p|p|s|u|sdw18|sdw18|7341|/data/primary341 +344|342|p|p|s|u|sdw18|sdw18|7342|/data/primary342 +345|343|p|p|s|u|sdw18|sdw18|7343|/data/primary343 +346|344|p|p|s|u|sdw18|sdw18|7344|/data/primary344 +347|345|p|p|s|u|sdw18|sdw18|7345|/data/primary345 +348|346|p|p|s|u|sdw18|sdw18|7346|/data/primary346 +349|347|p|p|s|u|sdw18|sdw18|7347|/data/primary347 +350|348|p|p|s|u|sdw18|sdw18|7348|/data/primary348 +351|349|p|p|s|u|sdw18|sdw18|7349|/data/primary349 +352|350|p|p|s|u|sdw18|sdw18|7350|/data/primary350 +353|351|p|p|s|u|sdw18|sdw18|7351|/data/primary351 +354|352|p|p|s|u|sdw18|sdw18|7352|/data/primary352 +355|353|p|p|s|u|sdw18|sdw18|7353|/data/primary353 +356|354|p|p|s|u|sdw18|sdw18|7354|/data/primary354 +357|355|p|p|s|u|sdw18|sdw18|7355|/data/primary355 +358|356|p|p|s|u|sdw18|sdw18|7356|/data/primary356 +359|357|p|p|s|u|sdw18|sdw18|7357|/data/primary357 +360|358|p|p|s|u|sdw18|sdw18|7358|/data/primary358 +361|359|p|p|s|u|sdw18|sdw18|7359|/data/primary359 +1018|16|m|m|s|u|sdw18|sdw18|8016|/data/mirror16 +1037|35|m|m|s|u|sdw18|sdw18|8035|/data/mirror35 +1056|54|m|m|s|u|sdw18|sdw18|8054|/data/mirror54 +1075|73|m|m|s|u|sdw18|sdw18|8073|/data/mirror73 +1094|92|m|m|s|u|sdw18|sdw18|8092|/data/mirror92 +1113|111|m|m|s|u|sdw18|sdw18|8111|/data/mirror111 +1132|130|m|m|s|u|sdw18|sdw18|8130|/data/mirror130 +1151|149|m|m|s|u|sdw18|sdw18|8149|/data/mirror149 +1170|168|m|m|s|u|sdw18|sdw18|8168|/data/mirror168 +1189|187|m|m|s|u|sdw18|sdw18|8187|/data/mirror187 +1208|206|m|m|s|u|sdw18|sdw18|8206|/data/mirror206 +1227|225|m|m|s|u|sdw18|sdw18|8225|/data/mirror225 +1246|244|m|m|s|u|sdw18|sdw18|8244|/data/mirror244 +1265|263|m|m|s|u|sdw18|sdw18|8263|/data/mirror263 +1284|282|m|m|s|u|sdw18|sdw18|8282|/data/mirror282 +1303|301|m|m|s|u|sdw18|sdw18|8301|/data/mirror301 +1322|320|m|m|s|u|sdw18|sdw18|8320|/data/mirror320 +1961|959|m|m|s|u|sdw18|sdw18|8959|/data/mirror959 +1980|978|m|m|s|u|sdw18|sdw18|8978|/data/mirror978 +1999|997|m|m|s|u|sdw18|sdw18|8997|/data/mirror997 +# SDW19 +362|360|p|p|s|u|sdw19|sdw19|7360|/data/primary360 +363|361|p|p|s|u|sdw19|sdw19|7361|/data/primary361 +364|362|p|p|s|u|sdw19|sdw19|7362|/data/primary362 +365|363|p|p|s|u|sdw19|sdw19|7363|/data/primary363 +366|364|p|p|s|u|sdw19|sdw19|7364|/data/primary364 +367|365|p|p|s|u|sdw19|sdw19|7365|/data/primary365 +368|366|p|p|s|u|sdw19|sdw19|7366|/data/primary366 +369|367|p|p|s|u|sdw19|sdw19|7367|/data/primary367 +370|368|p|p|s|u|sdw19|sdw19|7368|/data/primary368 +371|369|p|p|s|u|sdw19|sdw19|7369|/data/primary369 +372|370|p|p|s|u|sdw19|sdw19|7370|/data/primary370 +373|371|p|p|s|u|sdw19|sdw19|7371|/data/primary371 +374|372|p|p|s|u|sdw19|sdw19|7372|/data/primary372 +375|373|p|p|s|u|sdw19|sdw19|7373|/data/primary373 +376|374|p|p|s|u|sdw19|sdw19|7374|/data/primary374 +377|375|p|p|s|u|sdw19|sdw19|7375|/data/primary375 +378|376|p|p|s|u|sdw19|sdw19|7376|/data/primary376 +379|377|p|p|s|u|sdw19|sdw19|7377|/data/primary377 +380|378|p|p|s|u|sdw19|sdw19|7378|/data/primary378 +381|379|p|p|s|u|sdw19|sdw19|7379|/data/primary379 +1019|17|m|m|s|u|sdw19|sdw19|8017|/data/mirror17 +1038|36|m|m|s|u|sdw19|sdw19|8036|/data/mirror36 +1057|55|m|m|s|u|sdw19|sdw19|8055|/data/mirror55 +1076|74|m|m|s|u|sdw19|sdw19|8074|/data/mirror74 +1095|93|m|m|s|u|sdw19|sdw19|8093|/data/mirror93 +1114|112|m|m|s|u|sdw19|sdw19|8112|/data/mirror112 +1133|131|m|m|s|u|sdw19|sdw19|8131|/data/mirror131 +1152|150|m|m|s|u|sdw19|sdw19|8150|/data/mirror150 +1171|169|m|m|s|u|sdw19|sdw19|8169|/data/mirror169 +1190|188|m|m|s|u|sdw19|sdw19|8188|/data/mirror188 +1209|207|m|m|s|u|sdw19|sdw19|8207|/data/mirror207 +1228|226|m|m|s|u|sdw19|sdw19|8226|/data/mirror226 +1247|245|m|m|s|u|sdw19|sdw19|8245|/data/mirror245 +1266|264|m|m|s|u|sdw19|sdw19|8264|/data/mirror264 +1285|283|m|m|s|u|sdw19|sdw19|8283|/data/mirror283 +1304|302|m|m|s|u|sdw19|sdw19|8302|/data/mirror302 +1323|321|m|m|s|u|sdw19|sdw19|8321|/data/mirror321 +1342|340|m|m|s|u|sdw19|sdw19|8340|/data/mirror340 +1981|979|m|m|s|u|sdw19|sdw19|8979|/data/mirror979 +2000|998|m|m|s|u|sdw19|sdw19|8998|/data/mirror998 +# SDW2 +22|20|p|p|s|u|sdw2|sdw2|7020|/data/primary20 +23|21|p|p|s|u|sdw2|sdw2|7021|/data/primary21 +24|22|p|p|s|u|sdw2|sdw2|7022|/data/primary22 +25|23|p|p|s|u|sdw2|sdw2|7023|/data/primary23 +26|24|p|p|s|u|sdw2|sdw2|7024|/data/primary24 +27|25|p|p|s|u|sdw2|sdw2|7025|/data/primary25 +28|26|p|p|s|u|sdw2|sdw2|7026|/data/primary26 +29|27|p|p|s|u|sdw2|sdw2|7027|/data/primary27 +30|28|p|p|s|u|sdw2|sdw2|7028|/data/primary28 +31|29|p|p|s|u|sdw2|sdw2|7029|/data/primary29 +32|30|p|p|s|u|sdw2|sdw2|7030|/data/primary30 +33|31|p|p|s|u|sdw2|sdw2|7031|/data/primary31 +34|32|p|p|s|u|sdw2|sdw2|7032|/data/primary32 +35|33|p|p|s|u|sdw2|sdw2|7033|/data/primary33 +36|34|p|p|s|u|sdw2|sdw2|7034|/data/primary34 +37|35|p|p|s|u|sdw2|sdw2|7035|/data/primary35 +38|36|p|p|s|u|sdw2|sdw2|7036|/data/primary36 +39|37|p|p|s|u|sdw2|sdw2|7037|/data/primary37 +40|38|p|p|s|u|sdw2|sdw2|7038|/data/primary38 +41|39|p|p|s|u|sdw2|sdw2|7039|/data/primary39 +1002|0|m|m|s|u|sdw2|sdw2|8000|/data/mirror0 +1641|639|m|m|s|u|sdw2|sdw2|8639|/data/mirror639 +1660|658|m|m|s|u|sdw2|sdw2|8658|/data/mirror658 +1679|677|m|m|s|u|sdw2|sdw2|8677|/data/mirror677 +1698|696|m|m|s|u|sdw2|sdw2|8696|/data/mirror696 +1717|715|m|m|s|u|sdw2|sdw2|8715|/data/mirror715 +1736|734|m|m|s|u|sdw2|sdw2|8734|/data/mirror734 +1755|753|m|m|s|u|sdw2|sdw2|8753|/data/mirror753 +1774|772|m|m|s|u|sdw2|sdw2|8772|/data/mirror772 +1793|791|m|m|s|u|sdw2|sdw2|8791|/data/mirror791 +1812|810|m|m|s|u|sdw2|sdw2|8810|/data/mirror810 +1831|829|m|m|s|u|sdw2|sdw2|8829|/data/mirror829 +1850|848|m|m|s|u|sdw2|sdw2|8848|/data/mirror848 +1869|867|m|m|s|u|sdw2|sdw2|8867|/data/mirror867 +1888|886|m|m|s|u|sdw2|sdw2|8886|/data/mirror886 +1907|905|m|m|s|u|sdw2|sdw2|8905|/data/mirror905 +1926|924|m|m|s|u|sdw2|sdw2|8924|/data/mirror924 +1945|943|m|m|s|u|sdw2|sdw2|8943|/data/mirror943 +1964|962|m|m|s|u|sdw2|sdw2|8962|/data/mirror962 +1983|981|m|m|s|u|sdw2|sdw2|8981|/data/mirror981 +# SDW20 +382|380|p|p|s|u|sdw20|sdw20|7380|/data/primary380 +383|381|p|p|s|u|sdw20|sdw20|7381|/data/primary381 +384|382|p|p|s|u|sdw20|sdw20|7382|/data/primary382 +385|383|p|p|s|u|sdw20|sdw20|7383|/data/primary383 +386|384|p|p|s|u|sdw20|sdw20|7384|/data/primary384 +387|385|p|p|s|u|sdw20|sdw20|7385|/data/primary385 +388|386|p|p|s|u|sdw20|sdw20|7386|/data/primary386 +389|387|p|p|s|u|sdw20|sdw20|7387|/data/primary387 +390|388|p|p|s|u|sdw20|sdw20|7388|/data/primary388 +391|389|p|p|s|u|sdw20|sdw20|7389|/data/primary389 +392|390|p|p|s|u|sdw20|sdw20|7390|/data/primary390 +393|391|p|p|s|u|sdw20|sdw20|7391|/data/primary391 +394|392|p|p|s|u|sdw20|sdw20|7392|/data/primary392 +395|393|p|p|s|u|sdw20|sdw20|7393|/data/primary393 +396|394|p|p|s|u|sdw20|sdw20|7394|/data/primary394 +397|395|p|p|s|u|sdw20|sdw20|7395|/data/primary395 +398|396|p|p|s|u|sdw20|sdw20|7396|/data/primary396 +399|397|p|p|s|u|sdw20|sdw20|7397|/data/primary397 +400|398|p|p|s|u|sdw20|sdw20|7398|/data/primary398 +401|399|p|p|s|u|sdw20|sdw20|7399|/data/primary399 +1020|18|m|m|s|u|sdw20|sdw20|8018|/data/mirror18 +1039|37|m|m|s|u|sdw20|sdw20|8037|/data/mirror37 +1058|56|m|m|s|u|sdw20|sdw20|8056|/data/mirror56 +1077|75|m|m|s|u|sdw20|sdw20|8075|/data/mirror75 +1096|94|m|m|s|u|sdw20|sdw20|8094|/data/mirror94 +1115|113|m|m|s|u|sdw20|sdw20|8113|/data/mirror113 +1134|132|m|m|s|u|sdw20|sdw20|8132|/data/mirror132 +1153|151|m|m|s|u|sdw20|sdw20|8151|/data/mirror151 +1172|170|m|m|s|u|sdw20|sdw20|8170|/data/mirror170 +1191|189|m|m|s|u|sdw20|sdw20|8189|/data/mirror189 +1210|208|m|m|s|u|sdw20|sdw20|8208|/data/mirror208 +1229|227|m|m|s|u|sdw20|sdw20|8227|/data/mirror227 +1248|246|m|m|s|u|sdw20|sdw20|8246|/data/mirror246 +1267|265|m|m|s|u|sdw20|sdw20|8265|/data/mirror265 +1286|284|m|m|s|u|sdw20|sdw20|8284|/data/mirror284 +1305|303|m|m|s|u|sdw20|sdw20|8303|/data/mirror303 +1324|322|m|m|s|u|sdw20|sdw20|8322|/data/mirror322 +1343|341|m|m|s|u|sdw20|sdw20|8341|/data/mirror341 +1362|360|m|m|s|u|sdw20|sdw20|8360|/data/mirror360 +2001|999|m|m|s|u|sdw20|sdw20|8999|/data/mirror999 +# SDW21 +402|400|p|p|s|u|sdw21|sdw21|7400|/data/primary400 +403|401|p|p|s|u|sdw21|sdw21|7401|/data/primary401 +404|402|p|p|s|u|sdw21|sdw21|7402|/data/primary402 +405|403|p|p|s|u|sdw21|sdw21|7403|/data/primary403 +406|404|p|p|s|u|sdw21|sdw21|7404|/data/primary404 +407|405|p|p|s|u|sdw21|sdw21|7405|/data/primary405 +408|406|p|p|s|u|sdw21|sdw21|7406|/data/primary406 +409|407|p|p|s|u|sdw21|sdw21|7407|/data/primary407 +410|408|p|p|s|u|sdw21|sdw21|7408|/data/primary408 +411|409|p|p|s|u|sdw21|sdw21|7409|/data/primary409 +412|410|p|p|s|u|sdw21|sdw21|7410|/data/primary410 +413|411|p|p|s|u|sdw21|sdw21|7411|/data/primary411 +414|412|p|p|s|u|sdw21|sdw21|7412|/data/primary412 +415|413|p|p|s|u|sdw21|sdw21|7413|/data/primary413 +416|414|p|p|s|u|sdw21|sdw21|7414|/data/primary414 +417|415|p|p|s|u|sdw21|sdw21|7415|/data/primary415 +418|416|p|p|s|u|sdw21|sdw21|7416|/data/primary416 +419|417|p|p|s|u|sdw21|sdw21|7417|/data/primary417 +420|418|p|p|s|u|sdw21|sdw21|7418|/data/primary418 +421|419|p|p|s|u|sdw21|sdw21|7419|/data/primary419 +1021|19|m|m|s|u|sdw21|sdw21|8019|/data/mirror19 +1040|38|m|m|s|u|sdw21|sdw21|8038|/data/mirror38 +1059|57|m|m|s|u|sdw21|sdw21|8057|/data/mirror57 +1078|76|m|m|s|u|sdw21|sdw21|8076|/data/mirror76 +1097|95|m|m|s|u|sdw21|sdw21|8095|/data/mirror95 +1116|114|m|m|s|u|sdw21|sdw21|8114|/data/mirror114 +1135|133|m|m|s|u|sdw21|sdw21|8133|/data/mirror133 +1154|152|m|m|s|u|sdw21|sdw21|8152|/data/mirror152 +1173|171|m|m|s|u|sdw21|sdw21|8171|/data/mirror171 +1192|190|m|m|s|u|sdw21|sdw21|8190|/data/mirror190 +1211|209|m|m|s|u|sdw21|sdw21|8209|/data/mirror209 +1230|228|m|m|s|u|sdw21|sdw21|8228|/data/mirror228 +1249|247|m|m|s|u|sdw21|sdw21|8247|/data/mirror247 +1268|266|m|m|s|u|sdw21|sdw21|8266|/data/mirror266 +1287|285|m|m|s|u|sdw21|sdw21|8285|/data/mirror285 +1306|304|m|m|s|u|sdw21|sdw21|8304|/data/mirror304 +1325|323|m|m|s|u|sdw21|sdw21|8323|/data/mirror323 +1344|342|m|m|s|u|sdw21|sdw21|8342|/data/mirror342 +1363|361|m|m|s|u|sdw21|sdw21|8361|/data/mirror361 +1382|380|m|m|s|u|sdw21|sdw21|8380|/data/mirror380 +# SDW22 +422|420|p|p|s|u|sdw22|sdw22|7420|/data/primary420 +423|421|p|p|s|u|sdw22|sdw22|7421|/data/primary421 +424|422|p|p|s|u|sdw22|sdw22|7422|/data/primary422 +425|423|p|p|s|u|sdw22|sdw22|7423|/data/primary423 +426|424|p|p|s|u|sdw22|sdw22|7424|/data/primary424 +427|425|p|p|s|u|sdw22|sdw22|7425|/data/primary425 +428|426|p|p|s|u|sdw22|sdw22|7426|/data/primary426 +429|427|p|p|s|u|sdw22|sdw22|7427|/data/primary427 +430|428|p|p|s|u|sdw22|sdw22|7428|/data/primary428 +431|429|p|p|s|u|sdw22|sdw22|7429|/data/primary429 +432|430|p|p|s|u|sdw22|sdw22|7430|/data/primary430 +433|431|p|p|s|u|sdw22|sdw22|7431|/data/primary431 +434|432|p|p|s|u|sdw22|sdw22|7432|/data/primary432 +435|433|p|p|s|u|sdw22|sdw22|7433|/data/primary433 +436|434|p|p|s|u|sdw22|sdw22|7434|/data/primary434 +437|435|p|p|s|u|sdw22|sdw22|7435|/data/primary435 +438|436|p|p|s|u|sdw22|sdw22|7436|/data/primary436 +439|437|p|p|s|u|sdw22|sdw22|7437|/data/primary437 +440|438|p|p|s|u|sdw22|sdw22|7438|/data/primary438 +441|439|p|p|s|u|sdw22|sdw22|7439|/data/primary439 +1041|39|m|m|s|u|sdw22|sdw22|8039|/data/mirror39 +1060|58|m|m|s|u|sdw22|sdw22|8058|/data/mirror58 +1079|77|m|m|s|u|sdw22|sdw22|8077|/data/mirror77 +1098|96|m|m|s|u|sdw22|sdw22|8096|/data/mirror96 +1117|115|m|m|s|u|sdw22|sdw22|8115|/data/mirror115 +1136|134|m|m|s|u|sdw22|sdw22|8134|/data/mirror134 +1155|153|m|m|s|u|sdw22|sdw22|8153|/data/mirror153 +1174|172|m|m|s|u|sdw22|sdw22|8172|/data/mirror172 +1193|191|m|m|s|u|sdw22|sdw22|8191|/data/mirror191 +1212|210|m|m|s|u|sdw22|sdw22|8210|/data/mirror210 +1231|229|m|m|s|u|sdw22|sdw22|8229|/data/mirror229 +1250|248|m|m|s|u|sdw22|sdw22|8248|/data/mirror248 +1269|267|m|m|s|u|sdw22|sdw22|8267|/data/mirror267 +1288|286|m|m|s|u|sdw22|sdw22|8286|/data/mirror286 +1307|305|m|m|s|u|sdw22|sdw22|8305|/data/mirror305 +1326|324|m|m|s|u|sdw22|sdw22|8324|/data/mirror324 +1345|343|m|m|s|u|sdw22|sdw22|8343|/data/mirror343 +1364|362|m|m|s|u|sdw22|sdw22|8362|/data/mirror362 +1383|381|m|m|s|u|sdw22|sdw22|8381|/data/mirror381 +1402|400|m|m|s|u|sdw22|sdw22|8400|/data/mirror400 +# SDW23 +442|440|p|p|s|u|sdw23|sdw23|7440|/data/primary440 +443|441|p|p|s|u|sdw23|sdw23|7441|/data/primary441 +444|442|p|p|s|u|sdw23|sdw23|7442|/data/primary442 +445|443|p|p|s|u|sdw23|sdw23|7443|/data/primary443 +446|444|p|p|s|u|sdw23|sdw23|7444|/data/primary444 +447|445|p|p|s|u|sdw23|sdw23|7445|/data/primary445 +448|446|p|p|s|u|sdw23|sdw23|7446|/data/primary446 +449|447|p|p|s|u|sdw23|sdw23|7447|/data/primary447 +450|448|p|p|s|u|sdw23|sdw23|7448|/data/primary448 +451|449|p|p|s|u|sdw23|sdw23|7449|/data/primary449 +452|450|p|p|s|u|sdw23|sdw23|7450|/data/primary450 +453|451|p|p|s|u|sdw23|sdw23|7451|/data/primary451 +454|452|p|p|s|u|sdw23|sdw23|7452|/data/primary452 +455|453|p|p|s|u|sdw23|sdw23|7453|/data/primary453 +456|454|p|p|s|u|sdw23|sdw23|7454|/data/primary454 +457|455|p|p|s|u|sdw23|sdw23|7455|/data/primary455 +458|456|p|p|s|u|sdw23|sdw23|7456|/data/primary456 +459|457|p|p|s|u|sdw23|sdw23|7457|/data/primary457 +460|458|p|p|s|u|sdw23|sdw23|7458|/data/primary458 +461|459|p|p|s|u|sdw23|sdw23|7459|/data/primary459 +1061|59|m|m|s|u|sdw23|sdw23|8059|/data/mirror59 +1080|78|m|m|s|u|sdw23|sdw23|8078|/data/mirror78 +1099|97|m|m|s|u|sdw23|sdw23|8097|/data/mirror97 +1118|116|m|m|s|u|sdw23|sdw23|8116|/data/mirror116 +1137|135|m|m|s|u|sdw23|sdw23|8135|/data/mirror135 +1156|154|m|m|s|u|sdw23|sdw23|8154|/data/mirror154 +1175|173|m|m|s|u|sdw23|sdw23|8173|/data/mirror173 +1194|192|m|m|s|u|sdw23|sdw23|8192|/data/mirror192 +1213|211|m|m|s|u|sdw23|sdw23|8211|/data/mirror211 +1232|230|m|m|s|u|sdw23|sdw23|8230|/data/mirror230 +1251|249|m|m|s|u|sdw23|sdw23|8249|/data/mirror249 +1270|268|m|m|s|u|sdw23|sdw23|8268|/data/mirror268 +1289|287|m|m|s|u|sdw23|sdw23|8287|/data/mirror287 +1308|306|m|m|s|u|sdw23|sdw23|8306|/data/mirror306 +1327|325|m|m|s|u|sdw23|sdw23|8325|/data/mirror325 +1346|344|m|m|s|u|sdw23|sdw23|8344|/data/mirror344 +1365|363|m|m|s|u|sdw23|sdw23|8363|/data/mirror363 +1384|382|m|m|s|u|sdw23|sdw23|8382|/data/mirror382 +1403|401|m|m|s|u|sdw23|sdw23|8401|/data/mirror401 +1422|420|m|m|s|u|sdw23|sdw23|8420|/data/mirror420 +# SDW24 +462|460|p|p|s|u|sdw24|sdw24|7460|/data/primary460 +463|461|p|p|s|u|sdw24|sdw24|7461|/data/primary461 +464|462|p|p|s|u|sdw24|sdw24|7462|/data/primary462 +465|463|p|p|s|u|sdw24|sdw24|7463|/data/primary463 +466|464|p|p|s|u|sdw24|sdw24|7464|/data/primary464 +467|465|p|p|s|u|sdw24|sdw24|7465|/data/primary465 +468|466|p|p|s|u|sdw24|sdw24|7466|/data/primary466 +469|467|p|p|s|u|sdw24|sdw24|7467|/data/primary467 +470|468|p|p|s|u|sdw24|sdw24|7468|/data/primary468 +471|469|p|p|s|u|sdw24|sdw24|7469|/data/primary469 +472|470|p|p|s|u|sdw24|sdw24|7470|/data/primary470 +473|471|p|p|s|u|sdw24|sdw24|7471|/data/primary471 +474|472|p|p|s|u|sdw24|sdw24|7472|/data/primary472 +475|473|p|p|s|u|sdw24|sdw24|7473|/data/primary473 +476|474|p|p|s|u|sdw24|sdw24|7474|/data/primary474 +477|475|p|p|s|u|sdw24|sdw24|7475|/data/primary475 +478|476|p|p|s|u|sdw24|sdw24|7476|/data/primary476 +479|477|p|p|s|u|sdw24|sdw24|7477|/data/primary477 +480|478|p|p|s|u|sdw24|sdw24|7478|/data/primary478 +481|479|p|p|s|u|sdw24|sdw24|7479|/data/primary479 +1081|79|m|m|s|u|sdw24|sdw24|8079|/data/mirror79 +1100|98|m|m|s|u|sdw24|sdw24|8098|/data/mirror98 +1119|117|m|m|s|u|sdw24|sdw24|8117|/data/mirror117 +1138|136|m|m|s|u|sdw24|sdw24|8136|/data/mirror136 +1157|155|m|m|s|u|sdw24|sdw24|8155|/data/mirror155 +1176|174|m|m|s|u|sdw24|sdw24|8174|/data/mirror174 +1195|193|m|m|s|u|sdw24|sdw24|8193|/data/mirror193 +1214|212|m|m|s|u|sdw24|sdw24|8212|/data/mirror212 +1233|231|m|m|s|u|sdw24|sdw24|8231|/data/mirror231 +1252|250|m|m|s|u|sdw24|sdw24|8250|/data/mirror250 +1271|269|m|m|s|u|sdw24|sdw24|8269|/data/mirror269 +1290|288|m|m|s|u|sdw24|sdw24|8288|/data/mirror288 +1309|307|m|m|s|u|sdw24|sdw24|8307|/data/mirror307 +1328|326|m|m|s|u|sdw24|sdw24|8326|/data/mirror326 +1347|345|m|m|s|u|sdw24|sdw24|8345|/data/mirror345 +1366|364|m|m|s|u|sdw24|sdw24|8364|/data/mirror364 +1385|383|m|m|s|u|sdw24|sdw24|8383|/data/mirror383 +1404|402|m|m|s|u|sdw24|sdw24|8402|/data/mirror402 +1423|421|m|m|s|u|sdw24|sdw24|8421|/data/mirror421 +1442|440|m|m|s|u|sdw24|sdw24|8440|/data/mirror440 +# SDW25 +482|480|p|p|s|u|sdw25|sdw25|7480|/data/primary480 +483|481|p|p|s|u|sdw25|sdw25|7481|/data/primary481 +484|482|p|p|s|u|sdw25|sdw25|7482|/data/primary482 +485|483|p|p|s|u|sdw25|sdw25|7483|/data/primary483 +486|484|p|p|s|u|sdw25|sdw25|7484|/data/primary484 +487|485|p|p|s|u|sdw25|sdw25|7485|/data/primary485 +488|486|p|p|s|u|sdw25|sdw25|7486|/data/primary486 +489|487|p|p|s|u|sdw25|sdw25|7487|/data/primary487 +490|488|p|p|s|u|sdw25|sdw25|7488|/data/primary488 +491|489|p|p|s|u|sdw25|sdw25|7489|/data/primary489 +492|490|p|p|s|u|sdw25|sdw25|7490|/data/primary490 +493|491|p|p|s|u|sdw25|sdw25|7491|/data/primary491 +494|492|p|p|s|u|sdw25|sdw25|7492|/data/primary492 +495|493|p|p|s|u|sdw25|sdw25|7493|/data/primary493 +496|494|p|p|s|u|sdw25|sdw25|7494|/data/primary494 +497|495|p|p|s|u|sdw25|sdw25|7495|/data/primary495 +498|496|p|p|s|u|sdw25|sdw25|7496|/data/primary496 +499|497|p|p|s|u|sdw25|sdw25|7497|/data/primary497 +500|498|p|p|s|u|sdw25|sdw25|7498|/data/primary498 +501|499|p|p|s|u|sdw25|sdw25|7499|/data/primary499 +1101|99|m|m|s|u|sdw25|sdw25|8099|/data/mirror99 +1120|118|m|m|s|u|sdw25|sdw25|8118|/data/mirror118 +1139|137|m|m|s|u|sdw25|sdw25|8137|/data/mirror137 +1158|156|m|m|s|u|sdw25|sdw25|8156|/data/mirror156 +1177|175|m|m|s|u|sdw25|sdw25|8175|/data/mirror175 +1196|194|m|m|s|u|sdw25|sdw25|8194|/data/mirror194 +1215|213|m|m|s|u|sdw25|sdw25|8213|/data/mirror213 +1234|232|m|m|s|u|sdw25|sdw25|8232|/data/mirror232 +1253|251|m|m|s|u|sdw25|sdw25|8251|/data/mirror251 +1272|270|m|m|s|u|sdw25|sdw25|8270|/data/mirror270 +1291|289|m|m|s|u|sdw25|sdw25|8289|/data/mirror289 +1310|308|m|m|s|u|sdw25|sdw25|8308|/data/mirror308 +1329|327|m|m|s|u|sdw25|sdw25|8327|/data/mirror327 +1348|346|m|m|s|u|sdw25|sdw25|8346|/data/mirror346 +1367|365|m|m|s|u|sdw25|sdw25|8365|/data/mirror365 +1386|384|m|m|s|u|sdw25|sdw25|8384|/data/mirror384 +1405|403|m|m|s|u|sdw25|sdw25|8403|/data/mirror403 +1424|422|m|m|s|u|sdw25|sdw25|8422|/data/mirror422 +1443|441|m|m|s|u|sdw25|sdw25|8441|/data/mirror441 +1462|460|m|m|s|u|sdw25|sdw25|8460|/data/mirror460 +# SDW26 +502|500|p|p|s|u|sdw26|sdw26|7500|/data/primary500 +503|501|p|p|s|u|sdw26|sdw26|7501|/data/primary501 +504|502|p|p|s|u|sdw26|sdw26|7502|/data/primary502 +505|503|p|p|s|u|sdw26|sdw26|7503|/data/primary503 +506|504|p|p|s|u|sdw26|sdw26|7504|/data/primary504 +507|505|p|p|s|u|sdw26|sdw26|7505|/data/primary505 +508|506|p|p|s|u|sdw26|sdw26|7506|/data/primary506 +509|507|p|p|s|u|sdw26|sdw26|7507|/data/primary507 +510|508|p|p|s|u|sdw26|sdw26|7508|/data/primary508 +511|509|p|p|s|u|sdw26|sdw26|7509|/data/primary509 +512|510|p|p|s|u|sdw26|sdw26|7510|/data/primary510 +513|511|p|p|s|u|sdw26|sdw26|7511|/data/primary511 +514|512|p|p|s|u|sdw26|sdw26|7512|/data/primary512 +515|513|p|p|s|u|sdw26|sdw26|7513|/data/primary513 +516|514|p|p|s|u|sdw26|sdw26|7514|/data/primary514 +517|515|p|p|s|u|sdw26|sdw26|7515|/data/primary515 +518|516|p|p|s|u|sdw26|sdw26|7516|/data/primary516 +519|517|p|p|s|u|sdw26|sdw26|7517|/data/primary517 +520|518|p|p|s|u|sdw26|sdw26|7518|/data/primary518 +521|519|p|p|s|u|sdw26|sdw26|7519|/data/primary519 +1121|119|m|m|s|u|sdw26|sdw26|8119|/data/mirror119 +1140|138|m|m|s|u|sdw26|sdw26|8138|/data/mirror138 +1159|157|m|m|s|u|sdw26|sdw26|8157|/data/mirror157 +1178|176|m|m|s|u|sdw26|sdw26|8176|/data/mirror176 +1197|195|m|m|s|u|sdw26|sdw26|8195|/data/mirror195 +1216|214|m|m|s|u|sdw26|sdw26|8214|/data/mirror214 +1235|233|m|m|s|u|sdw26|sdw26|8233|/data/mirror233 +1254|252|m|m|s|u|sdw26|sdw26|8252|/data/mirror252 +1273|271|m|m|s|u|sdw26|sdw26|8271|/data/mirror271 +1292|290|m|m|s|u|sdw26|sdw26|8290|/data/mirror290 +1311|309|m|m|s|u|sdw26|sdw26|8309|/data/mirror309 +1330|328|m|m|s|u|sdw26|sdw26|8328|/data/mirror328 +1349|347|m|m|s|u|sdw26|sdw26|8347|/data/mirror347 +1368|366|m|m|s|u|sdw26|sdw26|8366|/data/mirror366 +1387|385|m|m|s|u|sdw26|sdw26|8385|/data/mirror385 +1406|404|m|m|s|u|sdw26|sdw26|8404|/data/mirror404 +1425|423|m|m|s|u|sdw26|sdw26|8423|/data/mirror423 +1444|442|m|m|s|u|sdw26|sdw26|8442|/data/mirror442 +1463|461|m|m|s|u|sdw26|sdw26|8461|/data/mirror461 +1482|480|m|m|s|u|sdw26|sdw26|8480|/data/mirror480 +# SDW27 +522|520|p|p|s|u|sdw27|sdw27|7520|/data/primary520 +523|521|p|p|s|u|sdw27|sdw27|7521|/data/primary521 +524|522|p|p|s|u|sdw27|sdw27|7522|/data/primary522 +525|523|p|p|s|u|sdw27|sdw27|7523|/data/primary523 +526|524|p|p|s|u|sdw27|sdw27|7524|/data/primary524 +527|525|p|p|s|u|sdw27|sdw27|7525|/data/primary525 +528|526|p|p|s|u|sdw27|sdw27|7526|/data/primary526 +529|527|p|p|s|u|sdw27|sdw27|7527|/data/primary527 +530|528|p|p|s|u|sdw27|sdw27|7528|/data/primary528 +531|529|p|p|s|u|sdw27|sdw27|7529|/data/primary529 +532|530|p|p|s|u|sdw27|sdw27|7530|/data/primary530 +533|531|p|p|s|u|sdw27|sdw27|7531|/data/primary531 +534|532|p|p|s|u|sdw27|sdw27|7532|/data/primary532 +535|533|p|p|s|u|sdw27|sdw27|7533|/data/primary533 +536|534|p|p|s|u|sdw27|sdw27|7534|/data/primary534 +537|535|p|p|s|u|sdw27|sdw27|7535|/data/primary535 +538|536|p|p|s|u|sdw27|sdw27|7536|/data/primary536 +539|537|p|p|s|u|sdw27|sdw27|7537|/data/primary537 +540|538|p|p|s|u|sdw27|sdw27|7538|/data/primary538 +541|539|p|p|s|u|sdw27|sdw27|7539|/data/primary539 +1141|139|m|m|s|u|sdw27|sdw27|8139|/data/mirror139 +1160|158|m|m|s|u|sdw27|sdw27|8158|/data/mirror158 +1179|177|m|m|s|u|sdw27|sdw27|8177|/data/mirror177 +1198|196|m|m|s|u|sdw27|sdw27|8196|/data/mirror196 +1217|215|m|m|s|u|sdw27|sdw27|8215|/data/mirror215 +1236|234|m|m|s|u|sdw27|sdw27|8234|/data/mirror234 +1255|253|m|m|s|u|sdw27|sdw27|8253|/data/mirror253 +1274|272|m|m|s|u|sdw27|sdw27|8272|/data/mirror272 +1293|291|m|m|s|u|sdw27|sdw27|8291|/data/mirror291 +1312|310|m|m|s|u|sdw27|sdw27|8310|/data/mirror310 +1331|329|m|m|s|u|sdw27|sdw27|8329|/data/mirror329 +1350|348|m|m|s|u|sdw27|sdw27|8348|/data/mirror348 +1369|367|m|m|s|u|sdw27|sdw27|8367|/data/mirror367 +1388|386|m|m|s|u|sdw27|sdw27|8386|/data/mirror386 +1407|405|m|m|s|u|sdw27|sdw27|8405|/data/mirror405 +1426|424|m|m|s|u|sdw27|sdw27|8424|/data/mirror424 +1445|443|m|m|s|u|sdw27|sdw27|8443|/data/mirror443 +1464|462|m|m|s|u|sdw27|sdw27|8462|/data/mirror462 +1483|481|m|m|s|u|sdw27|sdw27|8481|/data/mirror481 +1502|500|m|m|s|u|sdw27|sdw27|8500|/data/mirror500 +# SDW28 +542|540|p|p|s|u|sdw28|sdw28|7540|/data/primary540 +543|541|p|p|s|u|sdw28|sdw28|7541|/data/primary541 +544|542|p|p|s|u|sdw28|sdw28|7542|/data/primary542 +545|543|p|p|s|u|sdw28|sdw28|7543|/data/primary543 +546|544|p|p|s|u|sdw28|sdw28|7544|/data/primary544 +547|545|p|p|s|u|sdw28|sdw28|7545|/data/primary545 +548|546|p|p|s|u|sdw28|sdw28|7546|/data/primary546 +549|547|p|p|s|u|sdw28|sdw28|7547|/data/primary547 +550|548|p|p|s|u|sdw28|sdw28|7548|/data/primary548 +551|549|p|p|s|u|sdw28|sdw28|7549|/data/primary549 +552|550|p|p|s|u|sdw28|sdw28|7550|/data/primary550 +553|551|p|p|s|u|sdw28|sdw28|7551|/data/primary551 +554|552|p|p|s|u|sdw28|sdw28|7552|/data/primary552 +555|553|p|p|s|u|sdw28|sdw28|7553|/data/primary553 +556|554|p|p|s|u|sdw28|sdw28|7554|/data/primary554 +557|555|p|p|s|u|sdw28|sdw28|7555|/data/primary555 +558|556|p|p|s|u|sdw28|sdw28|7556|/data/primary556 +559|557|p|p|s|u|sdw28|sdw28|7557|/data/primary557 +560|558|p|p|s|u|sdw28|sdw28|7558|/data/primary558 +561|559|p|p|s|u|sdw28|sdw28|7559|/data/primary559 +1161|159|m|m|s|u|sdw28|sdw28|8159|/data/mirror159 +1180|178|m|m|s|u|sdw28|sdw28|8178|/data/mirror178 +1199|197|m|m|s|u|sdw28|sdw28|8197|/data/mirror197 +1218|216|m|m|s|u|sdw28|sdw28|8216|/data/mirror216 +1237|235|m|m|s|u|sdw28|sdw28|8235|/data/mirror235 +1256|254|m|m|s|u|sdw28|sdw28|8254|/data/mirror254 +1275|273|m|m|s|u|sdw28|sdw28|8273|/data/mirror273 +1294|292|m|m|s|u|sdw28|sdw28|8292|/data/mirror292 +1313|311|m|m|s|u|sdw28|sdw28|8311|/data/mirror311 +1332|330|m|m|s|u|sdw28|sdw28|8330|/data/mirror330 +1351|349|m|m|s|u|sdw28|sdw28|8349|/data/mirror349 +1370|368|m|m|s|u|sdw28|sdw28|8368|/data/mirror368 +1389|387|m|m|s|u|sdw28|sdw28|8387|/data/mirror387 +1408|406|m|m|s|u|sdw28|sdw28|8406|/data/mirror406 +1427|425|m|m|s|u|sdw28|sdw28|8425|/data/mirror425 +1446|444|m|m|s|u|sdw28|sdw28|8444|/data/mirror444 +1465|463|m|m|s|u|sdw28|sdw28|8463|/data/mirror463 +1484|482|m|m|s|u|sdw28|sdw28|8482|/data/mirror482 +1503|501|m|m|s|u|sdw28|sdw28|8501|/data/mirror501 +1522|520|m|m|s|u|sdw28|sdw28|8520|/data/mirror520 +# SDW29 +562|560|p|p|s|u|sdw29|sdw29|7560|/data/primary560 +563|561|p|p|s|u|sdw29|sdw29|7561|/data/primary561 +564|562|p|p|s|u|sdw29|sdw29|7562|/data/primary562 +565|563|p|p|s|u|sdw29|sdw29|7563|/data/primary563 +566|564|p|p|s|u|sdw29|sdw29|7564|/data/primary564 +567|565|p|p|s|u|sdw29|sdw29|7565|/data/primary565 +568|566|p|p|s|u|sdw29|sdw29|7566|/data/primary566 +569|567|p|p|s|u|sdw29|sdw29|7567|/data/primary567 +570|568|p|p|s|u|sdw29|sdw29|7568|/data/primary568 +571|569|p|p|s|u|sdw29|sdw29|7569|/data/primary569 +572|570|p|p|s|u|sdw29|sdw29|7570|/data/primary570 +573|571|p|p|s|u|sdw29|sdw29|7571|/data/primary571 +574|572|p|p|s|u|sdw29|sdw29|7572|/data/primary572 +575|573|p|p|s|u|sdw29|sdw29|7573|/data/primary573 +576|574|p|p|s|u|sdw29|sdw29|7574|/data/primary574 +577|575|p|p|s|u|sdw29|sdw29|7575|/data/primary575 +578|576|p|p|s|u|sdw29|sdw29|7576|/data/primary576 +579|577|p|p|s|u|sdw29|sdw29|7577|/data/primary577 +580|578|p|p|s|u|sdw29|sdw29|7578|/data/primary578 +581|579|p|p|s|u|sdw29|sdw29|7579|/data/primary579 +1181|179|m|m|s|u|sdw29|sdw29|8179|/data/mirror179 +1200|198|m|m|s|u|sdw29|sdw29|8198|/data/mirror198 +1219|217|m|m|s|u|sdw29|sdw29|8217|/data/mirror217 +1238|236|m|m|s|u|sdw29|sdw29|8236|/data/mirror236 +1257|255|m|m|s|u|sdw29|sdw29|8255|/data/mirror255 +1276|274|m|m|s|u|sdw29|sdw29|8274|/data/mirror274 +1295|293|m|m|s|u|sdw29|sdw29|8293|/data/mirror293 +1314|312|m|m|s|u|sdw29|sdw29|8312|/data/mirror312 +1333|331|m|m|s|u|sdw29|sdw29|8331|/data/mirror331 +1352|350|m|m|s|u|sdw29|sdw29|8350|/data/mirror350 +1371|369|m|m|s|u|sdw29|sdw29|8369|/data/mirror369 +1390|388|m|m|s|u|sdw29|sdw29|8388|/data/mirror388 +1409|407|m|m|s|u|sdw29|sdw29|8407|/data/mirror407 +1428|426|m|m|s|u|sdw29|sdw29|8426|/data/mirror426 +1447|445|m|m|s|u|sdw29|sdw29|8445|/data/mirror445 +1466|464|m|m|s|u|sdw29|sdw29|8464|/data/mirror464 +1485|483|m|m|s|u|sdw29|sdw29|8483|/data/mirror483 +1504|502|m|m|s|u|sdw29|sdw29|8502|/data/mirror502 +1523|521|m|m|s|u|sdw29|sdw29|8521|/data/mirror521 +1542|540|m|m|s|u|sdw29|sdw29|8540|/data/mirror540 +# SDW3 +42|40|p|p|s|u|sdw3|sdw3|7040|/data/primary40 +43|41|p|p|s|u|sdw3|sdw3|7041|/data/primary41 +44|42|p|p|s|u|sdw3|sdw3|7042|/data/primary42 +45|43|p|p|s|u|sdw3|sdw3|7043|/data/primary43 +46|44|p|p|s|u|sdw3|sdw3|7044|/data/primary44 +47|45|p|p|s|u|sdw3|sdw3|7045|/data/primary45 +48|46|p|p|s|u|sdw3|sdw3|7046|/data/primary46 +49|47|p|p|s|u|sdw3|sdw3|7047|/data/primary47 +50|48|p|p|s|u|sdw3|sdw3|7048|/data/primary48 +51|49|p|p|s|u|sdw3|sdw3|7049|/data/primary49 +52|50|p|p|s|u|sdw3|sdw3|7050|/data/primary50 +53|51|p|p|s|u|sdw3|sdw3|7051|/data/primary51 +54|52|p|p|s|u|sdw3|sdw3|7052|/data/primary52 +55|53|p|p|s|u|sdw3|sdw3|7053|/data/primary53 +56|54|p|p|s|u|sdw3|sdw3|7054|/data/primary54 +57|55|p|p|s|u|sdw3|sdw3|7055|/data/primary55 +58|56|p|p|s|u|sdw3|sdw3|7056|/data/primary56 +59|57|p|p|s|u|sdw3|sdw3|7057|/data/primary57 +60|58|p|p|s|u|sdw3|sdw3|7058|/data/primary58 +61|59|p|p|s|u|sdw3|sdw3|7059|/data/primary59 +1003|1|m|m|s|u|sdw3|sdw3|8001|/data/mirror1 +1022|20|m|m|s|u|sdw3|sdw3|8020|/data/mirror20 +1661|659|m|m|s|u|sdw3|sdw3|8659|/data/mirror659 +1680|678|m|m|s|u|sdw3|sdw3|8678|/data/mirror678 +1699|697|m|m|s|u|sdw3|sdw3|8697|/data/mirror697 +1718|716|m|m|s|u|sdw3|sdw3|8716|/data/mirror716 +1737|735|m|m|s|u|sdw3|sdw3|8735|/data/mirror735 +1756|754|m|m|s|u|sdw3|sdw3|8754|/data/mirror754 +1775|773|m|m|s|u|sdw3|sdw3|8773|/data/mirror773 +1794|792|m|m|s|u|sdw3|sdw3|8792|/data/mirror792 +1813|811|m|m|s|u|sdw3|sdw3|8811|/data/mirror811 +1832|830|m|m|s|u|sdw3|sdw3|8830|/data/mirror830 +1851|849|m|m|s|u|sdw3|sdw3|8849|/data/mirror849 +1870|868|m|m|s|u|sdw3|sdw3|8868|/data/mirror868 +1889|887|m|m|s|u|sdw3|sdw3|8887|/data/mirror887 +1908|906|m|m|s|u|sdw3|sdw3|8906|/data/mirror906 +1927|925|m|m|s|u|sdw3|sdw3|8925|/data/mirror925 +1946|944|m|m|s|u|sdw3|sdw3|8944|/data/mirror944 +1965|963|m|m|s|u|sdw3|sdw3|8963|/data/mirror963 +1984|982|m|m|s|u|sdw3|sdw3|8982|/data/mirror982 +# SDW30 +582|580|p|p|s|u|sdw30|sdw30|7580|/data/primary580 +583|581|p|p|s|u|sdw30|sdw30|7581|/data/primary581 +584|582|p|p|s|u|sdw30|sdw30|7582|/data/primary582 +585|583|p|p|s|u|sdw30|sdw30|7583|/data/primary583 +586|584|p|p|s|u|sdw30|sdw30|7584|/data/primary584 +587|585|p|p|s|u|sdw30|sdw30|7585|/data/primary585 +588|586|p|p|s|u|sdw30|sdw30|7586|/data/primary586 +589|587|p|p|s|u|sdw30|sdw30|7587|/data/primary587 +590|588|p|p|s|u|sdw30|sdw30|7588|/data/primary588 +591|589|p|p|s|u|sdw30|sdw30|7589|/data/primary589 +592|590|p|p|s|u|sdw30|sdw30|7590|/data/primary590 +593|591|p|p|s|u|sdw30|sdw30|7591|/data/primary591 +594|592|p|p|s|u|sdw30|sdw30|7592|/data/primary592 +595|593|p|p|s|u|sdw30|sdw30|7593|/data/primary593 +596|594|p|p|s|u|sdw30|sdw30|7594|/data/primary594 +597|595|p|p|s|u|sdw30|sdw30|7595|/data/primary595 +598|596|p|p|s|u|sdw30|sdw30|7596|/data/primary596 +599|597|p|p|s|u|sdw30|sdw30|7597|/data/primary597 +600|598|p|p|s|u|sdw30|sdw30|7598|/data/primary598 +601|599|p|p|s|u|sdw30|sdw30|7599|/data/primary599 +1201|199|m|m|s|u|sdw30|sdw30|8199|/data/mirror199 +1220|218|m|m|s|u|sdw30|sdw30|8218|/data/mirror218 +1239|237|m|m|s|u|sdw30|sdw30|8237|/data/mirror237 +1258|256|m|m|s|u|sdw30|sdw30|8256|/data/mirror256 +1277|275|m|m|s|u|sdw30|sdw30|8275|/data/mirror275 +1296|294|m|m|s|u|sdw30|sdw30|8294|/data/mirror294 +1315|313|m|m|s|u|sdw30|sdw30|8313|/data/mirror313 +1334|332|m|m|s|u|sdw30|sdw30|8332|/data/mirror332 +1353|351|m|m|s|u|sdw30|sdw30|8351|/data/mirror351 +1372|370|m|m|s|u|sdw30|sdw30|8370|/data/mirror370 +1391|389|m|m|s|u|sdw30|sdw30|8389|/data/mirror389 +1410|408|m|m|s|u|sdw30|sdw30|8408|/data/mirror408 +1429|427|m|m|s|u|sdw30|sdw30|8427|/data/mirror427 +1448|446|m|m|s|u|sdw30|sdw30|8446|/data/mirror446 +1467|465|m|m|s|u|sdw30|sdw30|8465|/data/mirror465 +1486|484|m|m|s|u|sdw30|sdw30|8484|/data/mirror484 +1505|503|m|m|s|u|sdw30|sdw30|8503|/data/mirror503 +1524|522|m|m|s|u|sdw30|sdw30|8522|/data/mirror522 +1543|541|m|m|s|u|sdw30|sdw30|8541|/data/mirror541 +1562|560|m|m|s|u|sdw30|sdw30|8560|/data/mirror560 +# SDW31 +602|600|p|p|s|u|sdw31|sdw31|7600|/data/primary600 +603|601|p|p|s|u|sdw31|sdw31|7601|/data/primary601 +604|602|p|p|s|u|sdw31|sdw31|7602|/data/primary602 +605|603|p|p|s|u|sdw31|sdw31|7603|/data/primary603 +606|604|p|p|s|u|sdw31|sdw31|7604|/data/primary604 +607|605|p|p|s|u|sdw31|sdw31|7605|/data/primary605 +608|606|p|p|s|u|sdw31|sdw31|7606|/data/primary606 +609|607|p|p|s|u|sdw31|sdw31|7607|/data/primary607 +610|608|p|p|s|u|sdw31|sdw31|7608|/data/primary608 +611|609|p|p|s|u|sdw31|sdw31|7609|/data/primary609 +612|610|p|p|s|u|sdw31|sdw31|7610|/data/primary610 +613|611|p|p|s|u|sdw31|sdw31|7611|/data/primary611 +614|612|p|p|s|u|sdw31|sdw31|7612|/data/primary612 +615|613|p|p|s|u|sdw31|sdw31|7613|/data/primary613 +616|614|p|p|s|u|sdw31|sdw31|7614|/data/primary614 +617|615|p|p|s|u|sdw31|sdw31|7615|/data/primary615 +618|616|p|p|s|u|sdw31|sdw31|7616|/data/primary616 +619|617|p|p|s|u|sdw31|sdw31|7617|/data/primary617 +620|618|p|p|s|u|sdw31|sdw31|7618|/data/primary618 +621|619|p|p|s|u|sdw31|sdw31|7619|/data/primary619 +1221|219|m|m|s|u|sdw31|sdw31|8219|/data/mirror219 +1240|238|m|m|s|u|sdw31|sdw31|8238|/data/mirror238 +1259|257|m|m|s|u|sdw31|sdw31|8257|/data/mirror257 +1278|276|m|m|s|u|sdw31|sdw31|8276|/data/mirror276 +1297|295|m|m|s|u|sdw31|sdw31|8295|/data/mirror295 +1316|314|m|m|s|u|sdw31|sdw31|8314|/data/mirror314 +1335|333|m|m|s|u|sdw31|sdw31|8333|/data/mirror333 +1354|352|m|m|s|u|sdw31|sdw31|8352|/data/mirror352 +1373|371|m|m|s|u|sdw31|sdw31|8371|/data/mirror371 +1392|390|m|m|s|u|sdw31|sdw31|8390|/data/mirror390 +1411|409|m|m|s|u|sdw31|sdw31|8409|/data/mirror409 +1430|428|m|m|s|u|sdw31|sdw31|8428|/data/mirror428 +1449|447|m|m|s|u|sdw31|sdw31|8447|/data/mirror447 +1468|466|m|m|s|u|sdw31|sdw31|8466|/data/mirror466 +1487|485|m|m|s|u|sdw31|sdw31|8485|/data/mirror485 +1506|504|m|m|s|u|sdw31|sdw31|8504|/data/mirror504 +1525|523|m|m|s|u|sdw31|sdw31|8523|/data/mirror523 +1544|542|m|m|s|u|sdw31|sdw31|8542|/data/mirror542 +1563|561|m|m|s|u|sdw31|sdw31|8561|/data/mirror561 +1582|580|m|m|s|u|sdw31|sdw31|8580|/data/mirror580 +# SDW32 +622|620|p|p|s|u|sdw32|sdw32|7620|/data/primary620 +623|621|p|p|s|u|sdw32|sdw32|7621|/data/primary621 +624|622|p|p|s|u|sdw32|sdw32|7622|/data/primary622 +625|623|p|p|s|u|sdw32|sdw32|7623|/data/primary623 +626|624|p|p|s|u|sdw32|sdw32|7624|/data/primary624 +627|625|p|p|s|u|sdw32|sdw32|7625|/data/primary625 +628|626|p|p|s|u|sdw32|sdw32|7626|/data/primary626 +629|627|p|p|s|u|sdw32|sdw32|7627|/data/primary627 +630|628|p|p|s|u|sdw32|sdw32|7628|/data/primary628 +631|629|p|p|s|u|sdw32|sdw32|7629|/data/primary629 +632|630|p|p|s|u|sdw32|sdw32|7630|/data/primary630 +633|631|p|p|s|u|sdw32|sdw32|7631|/data/primary631 +634|632|p|p|s|u|sdw32|sdw32|7632|/data/primary632 +635|633|p|p|s|u|sdw32|sdw32|7633|/data/primary633 +636|634|p|p|s|u|sdw32|sdw32|7634|/data/primary634 +637|635|p|p|s|u|sdw32|sdw32|7635|/data/primary635 +638|636|p|p|s|u|sdw32|sdw32|7636|/data/primary636 +639|637|p|p|s|u|sdw32|sdw32|7637|/data/primary637 +640|638|p|p|s|u|sdw32|sdw32|7638|/data/primary638 +641|639|p|p|s|u|sdw32|sdw32|7639|/data/primary639 +1241|239|m|m|s|u|sdw32|sdw32|8239|/data/mirror239 +1260|258|m|m|s|u|sdw32|sdw32|8258|/data/mirror258 +1279|277|m|m|s|u|sdw32|sdw32|8277|/data/mirror277 +1298|296|m|m|s|u|sdw32|sdw32|8296|/data/mirror296 +1317|315|m|m|s|u|sdw32|sdw32|8315|/data/mirror315 +1336|334|m|m|s|u|sdw32|sdw32|8334|/data/mirror334 +1355|353|m|m|s|u|sdw32|sdw32|8353|/data/mirror353 +1374|372|m|m|s|u|sdw32|sdw32|8372|/data/mirror372 +1393|391|m|m|s|u|sdw32|sdw32|8391|/data/mirror391 +1412|410|m|m|s|u|sdw32|sdw32|8410|/data/mirror410 +1431|429|m|m|s|u|sdw32|sdw32|8429|/data/mirror429 +1450|448|m|m|s|u|sdw32|sdw32|8448|/data/mirror448 +1469|467|m|m|s|u|sdw32|sdw32|8467|/data/mirror467 +1488|486|m|m|s|u|sdw32|sdw32|8486|/data/mirror486 +1507|505|m|m|s|u|sdw32|sdw32|8505|/data/mirror505 +1526|524|m|m|s|u|sdw32|sdw32|8524|/data/mirror524 +1545|543|m|m|s|u|sdw32|sdw32|8543|/data/mirror543 +1564|562|m|m|s|u|sdw32|sdw32|8562|/data/mirror562 +1583|581|m|m|s|u|sdw32|sdw32|8581|/data/mirror581 +1602|600|m|m|s|u|sdw32|sdw32|8600|/data/mirror600 +# SDW33 +642|640|p|p|s|u|sdw33|sdw33|7640|/data/primary640 +643|641|p|p|s|u|sdw33|sdw33|7641|/data/primary641 +644|642|p|p|s|u|sdw33|sdw33|7642|/data/primary642 +645|643|p|p|s|u|sdw33|sdw33|7643|/data/primary643 +646|644|p|p|s|u|sdw33|sdw33|7644|/data/primary644 +647|645|p|p|s|u|sdw33|sdw33|7645|/data/primary645 +648|646|p|p|s|u|sdw33|sdw33|7646|/data/primary646 +649|647|p|p|s|u|sdw33|sdw33|7647|/data/primary647 +650|648|p|p|s|u|sdw33|sdw33|7648|/data/primary648 +651|649|p|p|s|u|sdw33|sdw33|7649|/data/primary649 +652|650|p|p|s|u|sdw33|sdw33|7650|/data/primary650 +653|651|p|p|s|u|sdw33|sdw33|7651|/data/primary651 +654|652|p|p|s|u|sdw33|sdw33|7652|/data/primary652 +655|653|p|p|s|u|sdw33|sdw33|7653|/data/primary653 +656|654|p|p|s|u|sdw33|sdw33|7654|/data/primary654 +657|655|p|p|s|u|sdw33|sdw33|7655|/data/primary655 +658|656|p|p|s|u|sdw33|sdw33|7656|/data/primary656 +659|657|p|p|s|u|sdw33|sdw33|7657|/data/primary657 +660|658|p|p|s|u|sdw33|sdw33|7658|/data/primary658 +661|659|p|p|s|u|sdw33|sdw33|7659|/data/primary659 +1261|259|m|m|s|u|sdw33|sdw33|8259|/data/mirror259 +1280|278|m|m|s|u|sdw33|sdw33|8278|/data/mirror278 +1299|297|m|m|s|u|sdw33|sdw33|8297|/data/mirror297 +1318|316|m|m|s|u|sdw33|sdw33|8316|/data/mirror316 +1337|335|m|m|s|u|sdw33|sdw33|8335|/data/mirror335 +1356|354|m|m|s|u|sdw33|sdw33|8354|/data/mirror354 +1375|373|m|m|s|u|sdw33|sdw33|8373|/data/mirror373 +1394|392|m|m|s|u|sdw33|sdw33|8392|/data/mirror392 +1413|411|m|m|s|u|sdw33|sdw33|8411|/data/mirror411 +1432|430|m|m|s|u|sdw33|sdw33|8430|/data/mirror430 +1451|449|m|m|s|u|sdw33|sdw33|8449|/data/mirror449 +1470|468|m|m|s|u|sdw33|sdw33|8468|/data/mirror468 +1489|487|m|m|s|u|sdw33|sdw33|8487|/data/mirror487 +1508|506|m|m|s|u|sdw33|sdw33|8506|/data/mirror506 +1527|525|m|m|s|u|sdw33|sdw33|8525|/data/mirror525 +1546|544|m|m|s|u|sdw33|sdw33|8544|/data/mirror544 +1565|563|m|m|s|u|sdw33|sdw33|8563|/data/mirror563 +1584|582|m|m|s|u|sdw33|sdw33|8582|/data/mirror582 +1603|601|m|m|s|u|sdw33|sdw33|8601|/data/mirror601 +1622|620|m|m|s|u|sdw33|sdw33|8620|/data/mirror620 +# SDW34 +662|660|p|p|s|u|sdw34|sdw34|7660|/data/primary660 +663|661|p|p|s|u|sdw34|sdw34|7661|/data/primary661 +664|662|p|p|s|u|sdw34|sdw34|7662|/data/primary662 +665|663|p|p|s|u|sdw34|sdw34|7663|/data/primary663 +666|664|p|p|s|u|sdw34|sdw34|7664|/data/primary664 +667|665|p|p|s|u|sdw34|sdw34|7665|/data/primary665 +668|666|p|p|s|u|sdw34|sdw34|7666|/data/primary666 +669|667|p|p|s|u|sdw34|sdw34|7667|/data/primary667 +670|668|p|p|s|u|sdw34|sdw34|7668|/data/primary668 +671|669|p|p|s|u|sdw34|sdw34|7669|/data/primary669 +672|670|p|p|s|u|sdw34|sdw34|7670|/data/primary670 +673|671|p|p|s|u|sdw34|sdw34|7671|/data/primary671 +674|672|p|p|s|u|sdw34|sdw34|7672|/data/primary672 +675|673|p|p|s|u|sdw34|sdw34|7673|/data/primary673 +676|674|p|p|s|u|sdw34|sdw34|7674|/data/primary674 +677|675|p|p|s|u|sdw34|sdw34|7675|/data/primary675 +678|676|p|p|s|u|sdw34|sdw34|7676|/data/primary676 +679|677|p|p|s|u|sdw34|sdw34|7677|/data/primary677 +680|678|p|p|s|u|sdw34|sdw34|7678|/data/primary678 +681|679|p|p|s|u|sdw34|sdw34|7679|/data/primary679 +1281|279|m|m|s|u|sdw34|sdw34|8279|/data/mirror279 +1300|298|m|m|s|u|sdw34|sdw34|8298|/data/mirror298 +1319|317|m|m|s|u|sdw34|sdw34|8317|/data/mirror317 +1338|336|m|m|s|u|sdw34|sdw34|8336|/data/mirror336 +1357|355|m|m|s|u|sdw34|sdw34|8355|/data/mirror355 +1376|374|m|m|s|u|sdw34|sdw34|8374|/data/mirror374 +1395|393|m|m|s|u|sdw34|sdw34|8393|/data/mirror393 +1414|412|m|m|s|u|sdw34|sdw34|8412|/data/mirror412 +1433|431|m|m|s|u|sdw34|sdw34|8431|/data/mirror431 +1452|450|m|m|s|u|sdw34|sdw34|8450|/data/mirror450 +1471|469|m|m|s|u|sdw34|sdw34|8469|/data/mirror469 +1490|488|m|m|s|u|sdw34|sdw34|8488|/data/mirror488 +1509|507|m|m|s|u|sdw34|sdw34|8507|/data/mirror507 +1528|526|m|m|s|u|sdw34|sdw34|8526|/data/mirror526 +1547|545|m|m|s|u|sdw34|sdw34|8545|/data/mirror545 +1566|564|m|m|s|u|sdw34|sdw34|8564|/data/mirror564 +1585|583|m|m|s|u|sdw34|sdw34|8583|/data/mirror583 +1604|602|m|m|s|u|sdw34|sdw34|8602|/data/mirror602 +1623|621|m|m|s|u|sdw34|sdw34|8621|/data/mirror621 +1642|640|m|m|s|u|sdw34|sdw34|8640|/data/mirror640 +# SDW35 +682|680|p|p|s|u|sdw35|sdw35|7680|/data/primary680 +683|681|p|p|s|u|sdw35|sdw35|7681|/data/primary681 +684|682|p|p|s|u|sdw35|sdw35|7682|/data/primary682 +685|683|p|p|s|u|sdw35|sdw35|7683|/data/primary683 +686|684|p|p|s|u|sdw35|sdw35|7684|/data/primary684 +687|685|p|p|s|u|sdw35|sdw35|7685|/data/primary685 +688|686|p|p|s|u|sdw35|sdw35|7686|/data/primary686 +689|687|p|p|s|u|sdw35|sdw35|7687|/data/primary687 +690|688|p|p|s|u|sdw35|sdw35|7688|/data/primary688 +691|689|p|p|s|u|sdw35|sdw35|7689|/data/primary689 +692|690|p|p|s|u|sdw35|sdw35|7690|/data/primary690 +693|691|p|p|s|u|sdw35|sdw35|7691|/data/primary691 +694|692|p|p|s|u|sdw35|sdw35|7692|/data/primary692 +695|693|p|p|s|u|sdw35|sdw35|7693|/data/primary693 +696|694|p|p|s|u|sdw35|sdw35|7694|/data/primary694 +697|695|p|p|s|u|sdw35|sdw35|7695|/data/primary695 +698|696|p|p|s|u|sdw35|sdw35|7696|/data/primary696 +699|697|p|p|s|u|sdw35|sdw35|7697|/data/primary697 +700|698|p|p|s|u|sdw35|sdw35|7698|/data/primary698 +701|699|p|p|s|u|sdw35|sdw35|7699|/data/primary699 +1301|299|m|m|s|u|sdw35|sdw35|8299|/data/mirror299 +1320|318|m|m|s|u|sdw35|sdw35|8318|/data/mirror318 +1339|337|m|m|s|u|sdw35|sdw35|8337|/data/mirror337 +1358|356|m|m|s|u|sdw35|sdw35|8356|/data/mirror356 +1377|375|m|m|s|u|sdw35|sdw35|8375|/data/mirror375 +1396|394|m|m|s|u|sdw35|sdw35|8394|/data/mirror394 +1415|413|m|m|s|u|sdw35|sdw35|8413|/data/mirror413 +1434|432|m|m|s|u|sdw35|sdw35|8432|/data/mirror432 +1453|451|m|m|s|u|sdw35|sdw35|8451|/data/mirror451 +1472|470|m|m|s|u|sdw35|sdw35|8470|/data/mirror470 +1491|489|m|m|s|u|sdw35|sdw35|8489|/data/mirror489 +1510|508|m|m|s|u|sdw35|sdw35|8508|/data/mirror508 +1529|527|m|m|s|u|sdw35|sdw35|8527|/data/mirror527 +1548|546|m|m|s|u|sdw35|sdw35|8546|/data/mirror546 +1567|565|m|m|s|u|sdw35|sdw35|8565|/data/mirror565 +1586|584|m|m|s|u|sdw35|sdw35|8584|/data/mirror584 +1605|603|m|m|s|u|sdw35|sdw35|8603|/data/mirror603 +1624|622|m|m|s|u|sdw35|sdw35|8622|/data/mirror622 +1643|641|m|m|s|u|sdw35|sdw35|8641|/data/mirror641 +1662|660|m|m|s|u|sdw35|sdw35|8660|/data/mirror660 +# SDW36 +702|700|p|p|s|u|sdw36|sdw36|7700|/data/primary700 +703|701|p|p|s|u|sdw36|sdw36|7701|/data/primary701 +704|702|p|p|s|u|sdw36|sdw36|7702|/data/primary702 +705|703|p|p|s|u|sdw36|sdw36|7703|/data/primary703 +706|704|p|p|s|u|sdw36|sdw36|7704|/data/primary704 +707|705|p|p|s|u|sdw36|sdw36|7705|/data/primary705 +708|706|p|p|s|u|sdw36|sdw36|7706|/data/primary706 +709|707|p|p|s|u|sdw36|sdw36|7707|/data/primary707 +710|708|p|p|s|u|sdw36|sdw36|7708|/data/primary708 +711|709|p|p|s|u|sdw36|sdw36|7709|/data/primary709 +712|710|p|p|s|u|sdw36|sdw36|7710|/data/primary710 +713|711|p|p|s|u|sdw36|sdw36|7711|/data/primary711 +714|712|p|p|s|u|sdw36|sdw36|7712|/data/primary712 +715|713|p|p|s|u|sdw36|sdw36|7713|/data/primary713 +716|714|p|p|s|u|sdw36|sdw36|7714|/data/primary714 +717|715|p|p|s|u|sdw36|sdw36|7715|/data/primary715 +718|716|p|p|s|u|sdw36|sdw36|7716|/data/primary716 +719|717|p|p|s|u|sdw36|sdw36|7717|/data/primary717 +720|718|p|p|s|u|sdw36|sdw36|7718|/data/primary718 +721|719|p|p|s|u|sdw36|sdw36|7719|/data/primary719 +1321|319|m|m|s|u|sdw36|sdw36|8319|/data/mirror319 +1340|338|m|m|s|u|sdw36|sdw36|8338|/data/mirror338 +1359|357|m|m|s|u|sdw36|sdw36|8357|/data/mirror357 +1378|376|m|m|s|u|sdw36|sdw36|8376|/data/mirror376 +1397|395|m|m|s|u|sdw36|sdw36|8395|/data/mirror395 +1416|414|m|m|s|u|sdw36|sdw36|8414|/data/mirror414 +1435|433|m|m|s|u|sdw36|sdw36|8433|/data/mirror433 +1454|452|m|m|s|u|sdw36|sdw36|8452|/data/mirror452 +1473|471|m|m|s|u|sdw36|sdw36|8471|/data/mirror471 +1492|490|m|m|s|u|sdw36|sdw36|8490|/data/mirror490 +1511|509|m|m|s|u|sdw36|sdw36|8509|/data/mirror509 +1530|528|m|m|s|u|sdw36|sdw36|8528|/data/mirror528 +1549|547|m|m|s|u|sdw36|sdw36|8547|/data/mirror547 +1568|566|m|m|s|u|sdw36|sdw36|8566|/data/mirror566 +1587|585|m|m|s|u|sdw36|sdw36|8585|/data/mirror585 +1606|604|m|m|s|u|sdw36|sdw36|8604|/data/mirror604 +1625|623|m|m|s|u|sdw36|sdw36|8623|/data/mirror623 +1644|642|m|m|s|u|sdw36|sdw36|8642|/data/mirror642 +1663|661|m|m|s|u|sdw36|sdw36|8661|/data/mirror661 +1682|680|m|m|s|u|sdw36|sdw36|8680|/data/mirror680 +# SDW37 +722|720|p|p|s|u|sdw37|sdw37|7720|/data/primary720 +723|721|p|p|s|u|sdw37|sdw37|7721|/data/primary721 +724|722|p|p|s|u|sdw37|sdw37|7722|/data/primary722 +725|723|p|p|s|u|sdw37|sdw37|7723|/data/primary723 +726|724|p|p|s|u|sdw37|sdw37|7724|/data/primary724 +727|725|p|p|s|u|sdw37|sdw37|7725|/data/primary725 +728|726|p|p|s|u|sdw37|sdw37|7726|/data/primary726 +729|727|p|p|s|u|sdw37|sdw37|7727|/data/primary727 +730|728|p|p|s|u|sdw37|sdw37|7728|/data/primary728 +731|729|p|p|s|u|sdw37|sdw37|7729|/data/primary729 +732|730|p|p|s|u|sdw37|sdw37|7730|/data/primary730 +733|731|p|p|s|u|sdw37|sdw37|7731|/data/primary731 +734|732|p|p|s|u|sdw37|sdw37|7732|/data/primary732 +735|733|p|p|s|u|sdw37|sdw37|7733|/data/primary733 +736|734|p|p|s|u|sdw37|sdw37|7734|/data/primary734 +737|735|p|p|s|u|sdw37|sdw37|7735|/data/primary735 +738|736|p|p|s|u|sdw37|sdw37|7736|/data/primary736 +739|737|p|p|s|u|sdw37|sdw37|7737|/data/primary737 +740|738|p|p|s|u|sdw37|sdw37|7738|/data/primary738 +741|739|p|p|s|u|sdw37|sdw37|7739|/data/primary739 +1341|339|m|m|s|u|sdw37|sdw37|8339|/data/mirror339 +1360|358|m|m|s|u|sdw37|sdw37|8358|/data/mirror358 +1379|377|m|m|s|u|sdw37|sdw37|8377|/data/mirror377 +1398|396|m|m|s|u|sdw37|sdw37|8396|/data/mirror396 +1417|415|m|m|s|u|sdw37|sdw37|8415|/data/mirror415 +1436|434|m|m|s|u|sdw37|sdw37|8434|/data/mirror434 +1455|453|m|m|s|u|sdw37|sdw37|8453|/data/mirror453 +1474|472|m|m|s|u|sdw37|sdw37|8472|/data/mirror472 +1493|491|m|m|s|u|sdw37|sdw37|8491|/data/mirror491 +1512|510|m|m|s|u|sdw37|sdw37|8510|/data/mirror510 +1531|529|m|m|s|u|sdw37|sdw37|8529|/data/mirror529 +1550|548|m|m|s|u|sdw37|sdw37|8548|/data/mirror548 +1569|567|m|m|s|u|sdw37|sdw37|8567|/data/mirror567 +1588|586|m|m|s|u|sdw37|sdw37|8586|/data/mirror586 +1607|605|m|m|s|u|sdw37|sdw37|8605|/data/mirror605 +1626|624|m|m|s|u|sdw37|sdw37|8624|/data/mirror624 +1645|643|m|m|s|u|sdw37|sdw37|8643|/data/mirror643 +1664|662|m|m|s|u|sdw37|sdw37|8662|/data/mirror662 +1683|681|m|m|s|u|sdw37|sdw37|8681|/data/mirror681 +1702|700|m|m|s|u|sdw37|sdw37|8700|/data/mirror700 +# SDW38 +742|740|p|p|s|u|sdw38|sdw38|7740|/data/primary740 +743|741|p|p|s|u|sdw38|sdw38|7741|/data/primary741 +744|742|p|p|s|u|sdw38|sdw38|7742|/data/primary742 +745|743|p|p|s|u|sdw38|sdw38|7743|/data/primary743 +746|744|p|p|s|u|sdw38|sdw38|7744|/data/primary744 +747|745|p|p|s|u|sdw38|sdw38|7745|/data/primary745 +748|746|p|p|s|u|sdw38|sdw38|7746|/data/primary746 +749|747|p|p|s|u|sdw38|sdw38|7747|/data/primary747 +750|748|p|p|s|u|sdw38|sdw38|7748|/data/primary748 +751|749|p|p|s|u|sdw38|sdw38|7749|/data/primary749 +752|750|p|p|s|u|sdw38|sdw38|7750|/data/primary750 +753|751|p|p|s|u|sdw38|sdw38|7751|/data/primary751 +754|752|p|p|s|u|sdw38|sdw38|7752|/data/primary752 +755|753|p|p|s|u|sdw38|sdw38|7753|/data/primary753 +756|754|p|p|s|u|sdw38|sdw38|7754|/data/primary754 +757|755|p|p|s|u|sdw38|sdw38|7755|/data/primary755 +758|756|p|p|s|u|sdw38|sdw38|7756|/data/primary756 +759|757|p|p|s|u|sdw38|sdw38|7757|/data/primary757 +760|758|p|p|s|u|sdw38|sdw38|7758|/data/primary758 +761|759|p|p|s|u|sdw38|sdw38|7759|/data/primary759 +1361|359|m|m|s|u|sdw38|sdw38|8359|/data/mirror359 +1380|378|m|m|s|u|sdw38|sdw38|8378|/data/mirror378 +1399|397|m|m|s|u|sdw38|sdw38|8397|/data/mirror397 +1418|416|m|m|s|u|sdw38|sdw38|8416|/data/mirror416 +1437|435|m|m|s|u|sdw38|sdw38|8435|/data/mirror435 +1456|454|m|m|s|u|sdw38|sdw38|8454|/data/mirror454 +1475|473|m|m|s|u|sdw38|sdw38|8473|/data/mirror473 +1494|492|m|m|s|u|sdw38|sdw38|8492|/data/mirror492 +1513|511|m|m|s|u|sdw38|sdw38|8511|/data/mirror511 +1532|530|m|m|s|u|sdw38|sdw38|8530|/data/mirror530 +1551|549|m|m|s|u|sdw38|sdw38|8549|/data/mirror549 +1570|568|m|m|s|u|sdw38|sdw38|8568|/data/mirror568 +1589|587|m|m|s|u|sdw38|sdw38|8587|/data/mirror587 +1608|606|m|m|s|u|sdw38|sdw38|8606|/data/mirror606 +1627|625|m|m|s|u|sdw38|sdw38|8625|/data/mirror625 +1646|644|m|m|s|u|sdw38|sdw38|8644|/data/mirror644 +1665|663|m|m|s|u|sdw38|sdw38|8663|/data/mirror663 +1684|682|m|m|s|u|sdw38|sdw38|8682|/data/mirror682 +1703|701|m|m|s|u|sdw38|sdw38|8701|/data/mirror701 +1722|720|m|m|s|u|sdw38|sdw38|8720|/data/mirror720 +# SDW39 +762|760|p|p|s|u|sdw39|sdw39|7760|/data/primary760 +763|761|p|p|s|u|sdw39|sdw39|7761|/data/primary761 +764|762|p|p|s|u|sdw39|sdw39|7762|/data/primary762 +765|763|p|p|s|u|sdw39|sdw39|7763|/data/primary763 +766|764|p|p|s|u|sdw39|sdw39|7764|/data/primary764 +767|765|p|p|s|u|sdw39|sdw39|7765|/data/primary765 +768|766|p|p|s|u|sdw39|sdw39|7766|/data/primary766 +769|767|p|p|s|u|sdw39|sdw39|7767|/data/primary767 +770|768|p|p|s|u|sdw39|sdw39|7768|/data/primary768 +771|769|p|p|s|u|sdw39|sdw39|7769|/data/primary769 +772|770|p|p|s|u|sdw39|sdw39|7770|/data/primary770 +773|771|p|p|s|u|sdw39|sdw39|7771|/data/primary771 +774|772|p|p|s|u|sdw39|sdw39|7772|/data/primary772 +775|773|p|p|s|u|sdw39|sdw39|7773|/data/primary773 +776|774|p|p|s|u|sdw39|sdw39|7774|/data/primary774 +777|775|p|p|s|u|sdw39|sdw39|7775|/data/primary775 +778|776|p|p|s|u|sdw39|sdw39|7776|/data/primary776 +779|777|p|p|s|u|sdw39|sdw39|7777|/data/primary777 +780|778|p|p|s|u|sdw39|sdw39|7778|/data/primary778 +781|779|p|p|s|u|sdw39|sdw39|7779|/data/primary779 +1381|379|m|m|s|u|sdw39|sdw39|8379|/data/mirror379 +1400|398|m|m|s|u|sdw39|sdw39|8398|/data/mirror398 +1419|417|m|m|s|u|sdw39|sdw39|8417|/data/mirror417 +1438|436|m|m|s|u|sdw39|sdw39|8436|/data/mirror436 +1457|455|m|m|s|u|sdw39|sdw39|8455|/data/mirror455 +1476|474|m|m|s|u|sdw39|sdw39|8474|/data/mirror474 +1495|493|m|m|s|u|sdw39|sdw39|8493|/data/mirror493 +1514|512|m|m|s|u|sdw39|sdw39|8512|/data/mirror512 +1533|531|m|m|s|u|sdw39|sdw39|8531|/data/mirror531 +1552|550|m|m|s|u|sdw39|sdw39|8550|/data/mirror550 +1571|569|m|m|s|u|sdw39|sdw39|8569|/data/mirror569 +1590|588|m|m|s|u|sdw39|sdw39|8588|/data/mirror588 +1609|607|m|m|s|u|sdw39|sdw39|8607|/data/mirror607 +1628|626|m|m|s|u|sdw39|sdw39|8626|/data/mirror626 +1647|645|m|m|s|u|sdw39|sdw39|8645|/data/mirror645 +1666|664|m|m|s|u|sdw39|sdw39|8664|/data/mirror664 +1685|683|m|m|s|u|sdw39|sdw39|8683|/data/mirror683 +1704|702|m|m|s|u|sdw39|sdw39|8702|/data/mirror702 +1723|721|m|m|s|u|sdw39|sdw39|8721|/data/mirror721 +1742|740|m|m|s|u|sdw39|sdw39|8740|/data/mirror740 +# SDW4 +62|60|p|p|s|u|sdw4|sdw4|7060|/data/primary60 +63|61|p|p|s|u|sdw4|sdw4|7061|/data/primary61 +64|62|p|p|s|u|sdw4|sdw4|7062|/data/primary62 +65|63|p|p|s|u|sdw4|sdw4|7063|/data/primary63 +66|64|p|p|s|u|sdw4|sdw4|7064|/data/primary64 +67|65|p|p|s|u|sdw4|sdw4|7065|/data/primary65 +68|66|p|p|s|u|sdw4|sdw4|7066|/data/primary66 +69|67|p|p|s|u|sdw4|sdw4|7067|/data/primary67 +70|68|p|p|s|u|sdw4|sdw4|7068|/data/primary68 +71|69|p|p|s|u|sdw4|sdw4|7069|/data/primary69 +72|70|p|p|s|u|sdw4|sdw4|7070|/data/primary70 +73|71|p|p|s|u|sdw4|sdw4|7071|/data/primary71 +74|72|p|p|s|u|sdw4|sdw4|7072|/data/primary72 +75|73|p|p|s|u|sdw4|sdw4|7073|/data/primary73 +76|74|p|p|s|u|sdw4|sdw4|7074|/data/primary74 +77|75|p|p|s|u|sdw4|sdw4|7075|/data/primary75 +78|76|p|p|s|u|sdw4|sdw4|7076|/data/primary76 +79|77|p|p|s|u|sdw4|sdw4|7077|/data/primary77 +80|78|p|p|s|u|sdw4|sdw4|7078|/data/primary78 +81|79|p|p|s|u|sdw4|sdw4|7079|/data/primary79 +1004|2|m|m|s|u|sdw4|sdw4|8002|/data/mirror2 +1023|21|m|m|s|u|sdw4|sdw4|8021|/data/mirror21 +1042|40|m|m|s|u|sdw4|sdw4|8040|/data/mirror40 +1681|679|m|m|s|u|sdw4|sdw4|8679|/data/mirror679 +1700|698|m|m|s|u|sdw4|sdw4|8698|/data/mirror698 +1719|717|m|m|s|u|sdw4|sdw4|8717|/data/mirror717 +1738|736|m|m|s|u|sdw4|sdw4|8736|/data/mirror736 +1757|755|m|m|s|u|sdw4|sdw4|8755|/data/mirror755 +1776|774|m|m|s|u|sdw4|sdw4|8774|/data/mirror774 +1795|793|m|m|s|u|sdw4|sdw4|8793|/data/mirror793 +1814|812|m|m|s|u|sdw4|sdw4|8812|/data/mirror812 +1833|831|m|m|s|u|sdw4|sdw4|8831|/data/mirror831 +1852|850|m|m|s|u|sdw4|sdw4|8850|/data/mirror850 +1871|869|m|m|s|u|sdw4|sdw4|8869|/data/mirror869 +1890|888|m|m|s|u|sdw4|sdw4|8888|/data/mirror888 +1909|907|m|m|s|u|sdw4|sdw4|8907|/data/mirror907 +1928|926|m|m|s|u|sdw4|sdw4|8926|/data/mirror926 +1947|945|m|m|s|u|sdw4|sdw4|8945|/data/mirror945 +1966|964|m|m|s|u|sdw4|sdw4|8964|/data/mirror964 +1985|983|m|m|s|u|sdw4|sdw4|8983|/data/mirror983 +# SDW40 +782|780|p|p|s|u|sdw40|sdw40|7780|/data/primary780 +783|781|p|p|s|u|sdw40|sdw40|7781|/data/primary781 +784|782|p|p|s|u|sdw40|sdw40|7782|/data/primary782 +785|783|p|p|s|u|sdw40|sdw40|7783|/data/primary783 +786|784|p|p|s|u|sdw40|sdw40|7784|/data/primary784 +787|785|p|p|s|u|sdw40|sdw40|7785|/data/primary785 +788|786|p|p|s|u|sdw40|sdw40|7786|/data/primary786 +789|787|p|p|s|u|sdw40|sdw40|7787|/data/primary787 +790|788|p|p|s|u|sdw40|sdw40|7788|/data/primary788 +791|789|p|p|s|u|sdw40|sdw40|7789|/data/primary789 +792|790|p|p|s|u|sdw40|sdw40|7790|/data/primary790 +793|791|p|p|s|u|sdw40|sdw40|7791|/data/primary791 +794|792|p|p|s|u|sdw40|sdw40|7792|/data/primary792 +795|793|p|p|s|u|sdw40|sdw40|7793|/data/primary793 +796|794|p|p|s|u|sdw40|sdw40|7794|/data/primary794 +797|795|p|p|s|u|sdw40|sdw40|7795|/data/primary795 +798|796|p|p|s|u|sdw40|sdw40|7796|/data/primary796 +799|797|p|p|s|u|sdw40|sdw40|7797|/data/primary797 +800|798|p|p|s|u|sdw40|sdw40|7798|/data/primary798 +801|799|p|p|s|u|sdw40|sdw40|7799|/data/primary799 +1401|399|m|m|s|u|sdw40|sdw40|8399|/data/mirror399 +1420|418|m|m|s|u|sdw40|sdw40|8418|/data/mirror418 +1439|437|m|m|s|u|sdw40|sdw40|8437|/data/mirror437 +1458|456|m|m|s|u|sdw40|sdw40|8456|/data/mirror456 +1477|475|m|m|s|u|sdw40|sdw40|8475|/data/mirror475 +1496|494|m|m|s|u|sdw40|sdw40|8494|/data/mirror494 +1515|513|m|m|s|u|sdw40|sdw40|8513|/data/mirror513 +1534|532|m|m|s|u|sdw40|sdw40|8532|/data/mirror532 +1553|551|m|m|s|u|sdw40|sdw40|8551|/data/mirror551 +1572|570|m|m|s|u|sdw40|sdw40|8570|/data/mirror570 +1591|589|m|m|s|u|sdw40|sdw40|8589|/data/mirror589 +1610|608|m|m|s|u|sdw40|sdw40|8608|/data/mirror608 +1629|627|m|m|s|u|sdw40|sdw40|8627|/data/mirror627 +1648|646|m|m|s|u|sdw40|sdw40|8646|/data/mirror646 +1667|665|m|m|s|u|sdw40|sdw40|8665|/data/mirror665 +1686|684|m|m|s|u|sdw40|sdw40|8684|/data/mirror684 +1705|703|m|m|s|u|sdw40|sdw40|8703|/data/mirror703 +1724|722|m|m|s|u|sdw40|sdw40|8722|/data/mirror722 +1743|741|m|m|s|u|sdw40|sdw40|8741|/data/mirror741 +1762|760|m|m|s|u|sdw40|sdw40|8760|/data/mirror760 +# SDW41 +802|800|p|p|s|u|sdw41|sdw41|7800|/data/primary800 +803|801|p|p|s|u|sdw41|sdw41|7801|/data/primary801 +804|802|p|p|s|u|sdw41|sdw41|7802|/data/primary802 +805|803|p|p|s|u|sdw41|sdw41|7803|/data/primary803 +806|804|p|p|s|u|sdw41|sdw41|7804|/data/primary804 +807|805|p|p|s|u|sdw41|sdw41|7805|/data/primary805 +808|806|p|p|s|u|sdw41|sdw41|7806|/data/primary806 +809|807|p|p|s|u|sdw41|sdw41|7807|/data/primary807 +810|808|p|p|s|u|sdw41|sdw41|7808|/data/primary808 +811|809|p|p|s|u|sdw41|sdw41|7809|/data/primary809 +812|810|p|p|s|u|sdw41|sdw41|7810|/data/primary810 +813|811|p|p|s|u|sdw41|sdw41|7811|/data/primary811 +814|812|p|p|s|u|sdw41|sdw41|7812|/data/primary812 +815|813|p|p|s|u|sdw41|sdw41|7813|/data/primary813 +816|814|p|p|s|u|sdw41|sdw41|7814|/data/primary814 +817|815|p|p|s|u|sdw41|sdw41|7815|/data/primary815 +818|816|p|p|s|u|sdw41|sdw41|7816|/data/primary816 +819|817|p|p|s|u|sdw41|sdw41|7817|/data/primary817 +820|818|p|p|s|u|sdw41|sdw41|7818|/data/primary818 +821|819|p|p|s|u|sdw41|sdw41|7819|/data/primary819 +1421|419|m|m|s|u|sdw41|sdw41|8419|/data/mirror419 +1440|438|m|m|s|u|sdw41|sdw41|8438|/data/mirror438 +1459|457|m|m|s|u|sdw41|sdw41|8457|/data/mirror457 +1478|476|m|m|s|u|sdw41|sdw41|8476|/data/mirror476 +1497|495|m|m|s|u|sdw41|sdw41|8495|/data/mirror495 +1516|514|m|m|s|u|sdw41|sdw41|8514|/data/mirror514 +1535|533|m|m|s|u|sdw41|sdw41|8533|/data/mirror533 +1554|552|m|m|s|u|sdw41|sdw41|8552|/data/mirror552 +1573|571|m|m|s|u|sdw41|sdw41|8571|/data/mirror571 +1592|590|m|m|s|u|sdw41|sdw41|8590|/data/mirror590 +1611|609|m|m|s|u|sdw41|sdw41|8609|/data/mirror609 +1630|628|m|m|s|u|sdw41|sdw41|8628|/data/mirror628 +1649|647|m|m|s|u|sdw41|sdw41|8647|/data/mirror647 +1668|666|m|m|s|u|sdw41|sdw41|8666|/data/mirror666 +1687|685|m|m|s|u|sdw41|sdw41|8685|/data/mirror685 +1706|704|m|m|s|u|sdw41|sdw41|8704|/data/mirror704 +1725|723|m|m|s|u|sdw41|sdw41|8723|/data/mirror723 +1744|742|m|m|s|u|sdw41|sdw41|8742|/data/mirror742 +1763|761|m|m|s|u|sdw41|sdw41|8761|/data/mirror761 +1782|780|m|m|s|u|sdw41|sdw41|8780|/data/mirror780 +# SDW42 +822|820|p|p|s|u|sdw42|sdw42|7820|/data/primary820 +823|821|p|p|s|u|sdw42|sdw42|7821|/data/primary821 +824|822|p|p|s|u|sdw42|sdw42|7822|/data/primary822 +825|823|p|p|s|u|sdw42|sdw42|7823|/data/primary823 +826|824|p|p|s|u|sdw42|sdw42|7824|/data/primary824 +827|825|p|p|s|u|sdw42|sdw42|7825|/data/primary825 +828|826|p|p|s|u|sdw42|sdw42|7826|/data/primary826 +829|827|p|p|s|u|sdw42|sdw42|7827|/data/primary827 +830|828|p|p|s|u|sdw42|sdw42|7828|/data/primary828 +831|829|p|p|s|u|sdw42|sdw42|7829|/data/primary829 +832|830|p|p|s|u|sdw42|sdw42|7830|/data/primary830 +833|831|p|p|s|u|sdw42|sdw42|7831|/data/primary831 +834|832|p|p|s|u|sdw42|sdw42|7832|/data/primary832 +835|833|p|p|s|u|sdw42|sdw42|7833|/data/primary833 +836|834|p|p|s|u|sdw42|sdw42|7834|/data/primary834 +837|835|p|p|s|u|sdw42|sdw42|7835|/data/primary835 +838|836|p|p|s|u|sdw42|sdw42|7836|/data/primary836 +839|837|p|p|s|u|sdw42|sdw42|7837|/data/primary837 +840|838|p|p|s|u|sdw42|sdw42|7838|/data/primary838 +841|839|p|p|s|u|sdw42|sdw42|7839|/data/primary839 +1441|439|m|m|s|u|sdw42|sdw42|8439|/data/mirror439 +1460|458|m|m|s|u|sdw42|sdw42|8458|/data/mirror458 +1479|477|m|m|s|u|sdw42|sdw42|8477|/data/mirror477 +1498|496|m|m|s|u|sdw42|sdw42|8496|/data/mirror496 +1517|515|m|m|s|u|sdw42|sdw42|8515|/data/mirror515 +1536|534|m|m|s|u|sdw42|sdw42|8534|/data/mirror534 +1555|553|m|m|s|u|sdw42|sdw42|8553|/data/mirror553 +1574|572|m|m|s|u|sdw42|sdw42|8572|/data/mirror572 +1593|591|m|m|s|u|sdw42|sdw42|8591|/data/mirror591 +1612|610|m|m|s|u|sdw42|sdw42|8610|/data/mirror610 +1631|629|m|m|s|u|sdw42|sdw42|8629|/data/mirror629 +1650|648|m|m|s|u|sdw42|sdw42|8648|/data/mirror648 +1669|667|m|m|s|u|sdw42|sdw42|8667|/data/mirror667 +1688|686|m|m|s|u|sdw42|sdw42|8686|/data/mirror686 +1707|705|m|m|s|u|sdw42|sdw42|8705|/data/mirror705 +1726|724|m|m|s|u|sdw42|sdw42|8724|/data/mirror724 +1745|743|m|m|s|u|sdw42|sdw42|8743|/data/mirror743 +1764|762|m|m|s|u|sdw42|sdw42|8762|/data/mirror762 +1783|781|m|m|s|u|sdw42|sdw42|8781|/data/mirror781 +1802|800|m|m|s|u|sdw42|sdw42|8800|/data/mirror800 +# SDW43 +842|840|p|p|s|u|sdw43|sdw43|7840|/data/primary840 +843|841|p|p|s|u|sdw43|sdw43|7841|/data/primary841 +844|842|p|p|s|u|sdw43|sdw43|7842|/data/primary842 +845|843|p|p|s|u|sdw43|sdw43|7843|/data/primary843 +846|844|p|p|s|u|sdw43|sdw43|7844|/data/primary844 +847|845|p|p|s|u|sdw43|sdw43|7845|/data/primary845 +848|846|p|p|s|u|sdw43|sdw43|7846|/data/primary846 +849|847|p|p|s|u|sdw43|sdw43|7847|/data/primary847 +850|848|p|p|s|u|sdw43|sdw43|7848|/data/primary848 +851|849|p|p|s|u|sdw43|sdw43|7849|/data/primary849 +852|850|p|p|s|u|sdw43|sdw43|7850|/data/primary850 +853|851|p|p|s|u|sdw43|sdw43|7851|/data/primary851 +854|852|p|p|s|u|sdw43|sdw43|7852|/data/primary852 +855|853|p|p|s|u|sdw43|sdw43|7853|/data/primary853 +856|854|p|p|s|u|sdw43|sdw43|7854|/data/primary854 +857|855|p|p|s|u|sdw43|sdw43|7855|/data/primary855 +858|856|p|p|s|u|sdw43|sdw43|7856|/data/primary856 +859|857|p|p|s|u|sdw43|sdw43|7857|/data/primary857 +860|858|p|p|s|u|sdw43|sdw43|7858|/data/primary858 +861|859|p|p|s|u|sdw43|sdw43|7859|/data/primary859 +1461|459|m|m|s|u|sdw43|sdw43|8459|/data/mirror459 +1480|478|m|m|s|u|sdw43|sdw43|8478|/data/mirror478 +1499|497|m|m|s|u|sdw43|sdw43|8497|/data/mirror497 +1518|516|m|m|s|u|sdw43|sdw43|8516|/data/mirror516 +1537|535|m|m|s|u|sdw43|sdw43|8535|/data/mirror535 +1556|554|m|m|s|u|sdw43|sdw43|8554|/data/mirror554 +1575|573|m|m|s|u|sdw43|sdw43|8573|/data/mirror573 +1594|592|m|m|s|u|sdw43|sdw43|8592|/data/mirror592 +1613|611|m|m|s|u|sdw43|sdw43|8611|/data/mirror611 +1632|630|m|m|s|u|sdw43|sdw43|8630|/data/mirror630 +1651|649|m|m|s|u|sdw43|sdw43|8649|/data/mirror649 +1670|668|m|m|s|u|sdw43|sdw43|8668|/data/mirror668 +1689|687|m|m|s|u|sdw43|sdw43|8687|/data/mirror687 +1708|706|m|m|s|u|sdw43|sdw43|8706|/data/mirror706 +1727|725|m|m|s|u|sdw43|sdw43|8725|/data/mirror725 +1746|744|m|m|s|u|sdw43|sdw43|8744|/data/mirror744 +1765|763|m|m|s|u|sdw43|sdw43|8763|/data/mirror763 +1784|782|m|m|s|u|sdw43|sdw43|8782|/data/mirror782 +1803|801|m|m|s|u|sdw43|sdw43|8801|/data/mirror801 +1822|820|m|m|s|u|sdw43|sdw43|8820|/data/mirror820 +# SDW44 +862|860|p|p|s|u|sdw44|sdw44|7860|/data/primary860 +863|861|p|p|s|u|sdw44|sdw44|7861|/data/primary861 +864|862|p|p|s|u|sdw44|sdw44|7862|/data/primary862 +865|863|p|p|s|u|sdw44|sdw44|7863|/data/primary863 +866|864|p|p|s|u|sdw44|sdw44|7864|/data/primary864 +867|865|p|p|s|u|sdw44|sdw44|7865|/data/primary865 +868|866|p|p|s|u|sdw44|sdw44|7866|/data/primary866 +869|867|p|p|s|u|sdw44|sdw44|7867|/data/primary867 +870|868|p|p|s|u|sdw44|sdw44|7868|/data/primary868 +871|869|p|p|s|u|sdw44|sdw44|7869|/data/primary869 +872|870|p|p|s|u|sdw44|sdw44|7870|/data/primary870 +873|871|p|p|s|u|sdw44|sdw44|7871|/data/primary871 +874|872|p|p|s|u|sdw44|sdw44|7872|/data/primary872 +875|873|p|p|s|u|sdw44|sdw44|7873|/data/primary873 +876|874|p|p|s|u|sdw44|sdw44|7874|/data/primary874 +877|875|p|p|s|u|sdw44|sdw44|7875|/data/primary875 +878|876|p|p|s|u|sdw44|sdw44|7876|/data/primary876 +879|877|p|p|s|u|sdw44|sdw44|7877|/data/primary877 +880|878|p|p|s|u|sdw44|sdw44|7878|/data/primary878 +881|879|p|p|s|u|sdw44|sdw44|7879|/data/primary879 +1481|479|m|m|s|u|sdw44|sdw44|8479|/data/mirror479 +1500|498|m|m|s|u|sdw44|sdw44|8498|/data/mirror498 +1519|517|m|m|s|u|sdw44|sdw44|8517|/data/mirror517 +1538|536|m|m|s|u|sdw44|sdw44|8536|/data/mirror536 +1557|555|m|m|s|u|sdw44|sdw44|8555|/data/mirror555 +1576|574|m|m|s|u|sdw44|sdw44|8574|/data/mirror574 +1595|593|m|m|s|u|sdw44|sdw44|8593|/data/mirror593 +1614|612|m|m|s|u|sdw44|sdw44|8612|/data/mirror612 +1633|631|m|m|s|u|sdw44|sdw44|8631|/data/mirror631 +1652|650|m|m|s|u|sdw44|sdw44|8650|/data/mirror650 +1671|669|m|m|s|u|sdw44|sdw44|8669|/data/mirror669 +1690|688|m|m|s|u|sdw44|sdw44|8688|/data/mirror688 +1709|707|m|m|s|u|sdw44|sdw44|8707|/data/mirror707 +1728|726|m|m|s|u|sdw44|sdw44|8726|/data/mirror726 +1747|745|m|m|s|u|sdw44|sdw44|8745|/data/mirror745 +1766|764|m|m|s|u|sdw44|sdw44|8764|/data/mirror764 +1785|783|m|m|s|u|sdw44|sdw44|8783|/data/mirror783 +1804|802|m|m|s|u|sdw44|sdw44|8802|/data/mirror802 +1823|821|m|m|s|u|sdw44|sdw44|8821|/data/mirror821 +1842|840|m|m|s|u|sdw44|sdw44|8840|/data/mirror840 +# SDW45 +882|880|p|p|s|u|sdw45|sdw45|7880|/data/primary880 +883|881|p|p|s|u|sdw45|sdw45|7881|/data/primary881 +884|882|p|p|s|u|sdw45|sdw45|7882|/data/primary882 +885|883|p|p|s|u|sdw45|sdw45|7883|/data/primary883 +886|884|p|p|s|u|sdw45|sdw45|7884|/data/primary884 +887|885|p|p|s|u|sdw45|sdw45|7885|/data/primary885 +888|886|p|p|s|u|sdw45|sdw45|7886|/data/primary886 +889|887|p|p|s|u|sdw45|sdw45|7887|/data/primary887 +890|888|p|p|s|u|sdw45|sdw45|7888|/data/primary888 +891|889|p|p|s|u|sdw45|sdw45|7889|/data/primary889 +892|890|p|p|s|u|sdw45|sdw45|7890|/data/primary890 +893|891|p|p|s|u|sdw45|sdw45|7891|/data/primary891 +894|892|p|p|s|u|sdw45|sdw45|7892|/data/primary892 +895|893|p|p|s|u|sdw45|sdw45|7893|/data/primary893 +896|894|p|p|s|u|sdw45|sdw45|7894|/data/primary894 +897|895|p|p|s|u|sdw45|sdw45|7895|/data/primary895 +898|896|p|p|s|u|sdw45|sdw45|7896|/data/primary896 +899|897|p|p|s|u|sdw45|sdw45|7897|/data/primary897 +900|898|p|p|s|u|sdw45|sdw45|7898|/data/primary898 +901|899|p|p|s|u|sdw45|sdw45|7899|/data/primary899 +1501|499|m|m|s|u|sdw45|sdw45|8499|/data/mirror499 +1520|518|m|m|s|u|sdw45|sdw45|8518|/data/mirror518 +1539|537|m|m|s|u|sdw45|sdw45|8537|/data/mirror537 +1558|556|m|m|s|u|sdw45|sdw45|8556|/data/mirror556 +1577|575|m|m|s|u|sdw45|sdw45|8575|/data/mirror575 +1596|594|m|m|s|u|sdw45|sdw45|8594|/data/mirror594 +1615|613|m|m|s|u|sdw45|sdw45|8613|/data/mirror613 +1634|632|m|m|s|u|sdw45|sdw45|8632|/data/mirror632 +1653|651|m|m|s|u|sdw45|sdw45|8651|/data/mirror651 +1672|670|m|m|s|u|sdw45|sdw45|8670|/data/mirror670 +1691|689|m|m|s|u|sdw45|sdw45|8689|/data/mirror689 +1710|708|m|m|s|u|sdw45|sdw45|8708|/data/mirror708 +1729|727|m|m|s|u|sdw45|sdw45|8727|/data/mirror727 +1748|746|m|m|s|u|sdw45|sdw45|8746|/data/mirror746 +1767|765|m|m|s|u|sdw45|sdw45|8765|/data/mirror765 +1786|784|m|m|s|u|sdw45|sdw45|8784|/data/mirror784 +1805|803|m|m|s|u|sdw45|sdw45|8803|/data/mirror803 +1824|822|m|m|s|u|sdw45|sdw45|8822|/data/mirror822 +1843|841|m|m|s|u|sdw45|sdw45|8841|/data/mirror841 +1862|860|m|m|s|u|sdw45|sdw45|8860|/data/mirror860 +# SDW46 +902|900|p|p|s|u|sdw46|sdw46|7900|/data/primary900 +903|901|p|p|s|u|sdw46|sdw46|7901|/data/primary901 +904|902|p|p|s|u|sdw46|sdw46|7902|/data/primary902 +905|903|p|p|s|u|sdw46|sdw46|7903|/data/primary903 +906|904|p|p|s|u|sdw46|sdw46|7904|/data/primary904 +907|905|p|p|s|u|sdw46|sdw46|7905|/data/primary905 +908|906|p|p|s|u|sdw46|sdw46|7906|/data/primary906 +909|907|p|p|s|u|sdw46|sdw46|7907|/data/primary907 +910|908|p|p|s|u|sdw46|sdw46|7908|/data/primary908 +911|909|p|p|s|u|sdw46|sdw46|7909|/data/primary909 +912|910|p|p|s|u|sdw46|sdw46|7910|/data/primary910 +913|911|p|p|s|u|sdw46|sdw46|7911|/data/primary911 +914|912|p|p|s|u|sdw46|sdw46|7912|/data/primary912 +915|913|p|p|s|u|sdw46|sdw46|7913|/data/primary913 +916|914|p|p|s|u|sdw46|sdw46|7914|/data/primary914 +917|915|p|p|s|u|sdw46|sdw46|7915|/data/primary915 +918|916|p|p|s|u|sdw46|sdw46|7916|/data/primary916 +919|917|p|p|s|u|sdw46|sdw46|7917|/data/primary917 +920|918|p|p|s|u|sdw46|sdw46|7918|/data/primary918 +921|919|p|p|s|u|sdw46|sdw46|7919|/data/primary919 +1521|519|m|m|s|u|sdw46|sdw46|8519|/data/mirror519 +1540|538|m|m|s|u|sdw46|sdw46|8538|/data/mirror538 +1559|557|m|m|s|u|sdw46|sdw46|8557|/data/mirror557 +1578|576|m|m|s|u|sdw46|sdw46|8576|/data/mirror576 +1597|595|m|m|s|u|sdw46|sdw46|8595|/data/mirror595 +1616|614|m|m|s|u|sdw46|sdw46|8614|/data/mirror614 +1635|633|m|m|s|u|sdw46|sdw46|8633|/data/mirror633 +1654|652|m|m|s|u|sdw46|sdw46|8652|/data/mirror652 +1673|671|m|m|s|u|sdw46|sdw46|8671|/data/mirror671 +1692|690|m|m|s|u|sdw46|sdw46|8690|/data/mirror690 +1711|709|m|m|s|u|sdw46|sdw46|8709|/data/mirror709 +1730|728|m|m|s|u|sdw46|sdw46|8728|/data/mirror728 +1749|747|m|m|s|u|sdw46|sdw46|8747|/data/mirror747 +1768|766|m|m|s|u|sdw46|sdw46|8766|/data/mirror766 +1787|785|m|m|s|u|sdw46|sdw46|8785|/data/mirror785 +1806|804|m|m|s|u|sdw46|sdw46|8804|/data/mirror804 +1825|823|m|m|s|u|sdw46|sdw46|8823|/data/mirror823 +1844|842|m|m|s|u|sdw46|sdw46|8842|/data/mirror842 +1863|861|m|m|s|u|sdw46|sdw46|8861|/data/mirror861 +1882|880|m|m|s|u|sdw46|sdw46|8880|/data/mirror880 +# SDW47 +922|920|p|p|s|u|sdw47|sdw47|7920|/data/primary920 +923|921|p|p|s|u|sdw47|sdw47|7921|/data/primary921 +924|922|p|p|s|u|sdw47|sdw47|7922|/data/primary922 +925|923|p|p|s|u|sdw47|sdw47|7923|/data/primary923 +926|924|p|p|s|u|sdw47|sdw47|7924|/data/primary924 +927|925|p|p|s|u|sdw47|sdw47|7925|/data/primary925 +928|926|p|p|s|u|sdw47|sdw47|7926|/data/primary926 +929|927|p|p|s|u|sdw47|sdw47|7927|/data/primary927 +930|928|p|p|s|u|sdw47|sdw47|7928|/data/primary928 +931|929|p|p|s|u|sdw47|sdw47|7929|/data/primary929 +932|930|p|p|s|u|sdw47|sdw47|7930|/data/primary930 +933|931|p|p|s|u|sdw47|sdw47|7931|/data/primary931 +934|932|p|p|s|u|sdw47|sdw47|7932|/data/primary932 +935|933|p|p|s|u|sdw47|sdw47|7933|/data/primary933 +936|934|p|p|s|u|sdw47|sdw47|7934|/data/primary934 +937|935|p|p|s|u|sdw47|sdw47|7935|/data/primary935 +938|936|p|p|s|u|sdw47|sdw47|7936|/data/primary936 +939|937|p|p|s|u|sdw47|sdw47|7937|/data/primary937 +940|938|p|p|s|u|sdw47|sdw47|7938|/data/primary938 +941|939|p|p|s|u|sdw47|sdw47|7939|/data/primary939 +1541|539|m|m|s|u|sdw47|sdw47|8539|/data/mirror539 +1560|558|m|m|s|u|sdw47|sdw47|8558|/data/mirror558 +1579|577|m|m|s|u|sdw47|sdw47|8577|/data/mirror577 +1598|596|m|m|s|u|sdw47|sdw47|8596|/data/mirror596 +1617|615|m|m|s|u|sdw47|sdw47|8615|/data/mirror615 +1636|634|m|m|s|u|sdw47|sdw47|8634|/data/mirror634 +1655|653|m|m|s|u|sdw47|sdw47|8653|/data/mirror653 +1674|672|m|m|s|u|sdw47|sdw47|8672|/data/mirror672 +1693|691|m|m|s|u|sdw47|sdw47|8691|/data/mirror691 +1712|710|m|m|s|u|sdw47|sdw47|8710|/data/mirror710 +1731|729|m|m|s|u|sdw47|sdw47|8729|/data/mirror729 +1750|748|m|m|s|u|sdw47|sdw47|8748|/data/mirror748 +1769|767|m|m|s|u|sdw47|sdw47|8767|/data/mirror767 +1788|786|m|m|s|u|sdw47|sdw47|8786|/data/mirror786 +1807|805|m|m|s|u|sdw47|sdw47|8805|/data/mirror805 +1826|824|m|m|s|u|sdw47|sdw47|8824|/data/mirror824 +1845|843|m|m|s|u|sdw47|sdw47|8843|/data/mirror843 +1864|862|m|m|s|u|sdw47|sdw47|8862|/data/mirror862 +1883|881|m|m|s|u|sdw47|sdw47|8881|/data/mirror881 +1902|900|m|m|s|u|sdw47|sdw47|8900|/data/mirror900 +# SDW48 +942|940|p|p|s|u|sdw48|sdw48|7940|/data/primary940 +943|941|p|p|s|u|sdw48|sdw48|7941|/data/primary941 +944|942|p|p|s|u|sdw48|sdw48|7942|/data/primary942 +945|943|p|p|s|u|sdw48|sdw48|7943|/data/primary943 +946|944|p|p|s|u|sdw48|sdw48|7944|/data/primary944 +947|945|p|p|s|u|sdw48|sdw48|7945|/data/primary945 +948|946|p|p|s|u|sdw48|sdw48|7946|/data/primary946 +949|947|p|p|s|u|sdw48|sdw48|7947|/data/primary947 +950|948|p|p|s|u|sdw48|sdw48|7948|/data/primary948 +951|949|p|p|s|u|sdw48|sdw48|7949|/data/primary949 +952|950|p|p|s|u|sdw48|sdw48|7950|/data/primary950 +953|951|p|p|s|u|sdw48|sdw48|7951|/data/primary951 +954|952|p|p|s|u|sdw48|sdw48|7952|/data/primary952 +955|953|p|p|s|u|sdw48|sdw48|7953|/data/primary953 +956|954|p|p|s|u|sdw48|sdw48|7954|/data/primary954 +957|955|p|p|s|u|sdw48|sdw48|7955|/data/primary955 +958|956|p|p|s|u|sdw48|sdw48|7956|/data/primary956 +959|957|p|p|s|u|sdw48|sdw48|7957|/data/primary957 +960|958|p|p|s|u|sdw48|sdw48|7958|/data/primary958 +961|959|p|p|s|u|sdw48|sdw48|7959|/data/primary959 +1561|559|m|m|s|u|sdw48|sdw48|8559|/data/mirror559 +1580|578|m|m|s|u|sdw48|sdw48|8578|/data/mirror578 +1599|597|m|m|s|u|sdw48|sdw48|8597|/data/mirror597 +1618|616|m|m|s|u|sdw48|sdw48|8616|/data/mirror616 +1637|635|m|m|s|u|sdw48|sdw48|8635|/data/mirror635 +1656|654|m|m|s|u|sdw48|sdw48|8654|/data/mirror654 +1675|673|m|m|s|u|sdw48|sdw48|8673|/data/mirror673 +1694|692|m|m|s|u|sdw48|sdw48|8692|/data/mirror692 +1713|711|m|m|s|u|sdw48|sdw48|8711|/data/mirror711 +1732|730|m|m|s|u|sdw48|sdw48|8730|/data/mirror730 +1751|749|m|m|s|u|sdw48|sdw48|8749|/data/mirror749 +1770|768|m|m|s|u|sdw48|sdw48|8768|/data/mirror768 +1789|787|m|m|s|u|sdw48|sdw48|8787|/data/mirror787 +1808|806|m|m|s|u|sdw48|sdw48|8806|/data/mirror806 +1827|825|m|m|s|u|sdw48|sdw48|8825|/data/mirror825 +1846|844|m|m|s|u|sdw48|sdw48|8844|/data/mirror844 +1865|863|m|m|s|u|sdw48|sdw48|8863|/data/mirror863 +1884|882|m|m|s|u|sdw48|sdw48|8882|/data/mirror882 +1903|901|m|m|s|u|sdw48|sdw48|8901|/data/mirror901 +1922|920|m|m|s|u|sdw48|sdw48|8920|/data/mirror920 +# SDW49 +962|960|p|p|s|u|sdw49|sdw49|7960|/data/primary960 +963|961|p|p|s|u|sdw49|sdw49|7961|/data/primary961 +964|962|p|p|s|u|sdw49|sdw49|7962|/data/primary962 +965|963|p|p|s|u|sdw49|sdw49|7963|/data/primary963 +966|964|p|p|s|u|sdw49|sdw49|7964|/data/primary964 +967|965|p|p|s|u|sdw49|sdw49|7965|/data/primary965 +968|966|p|p|s|u|sdw49|sdw49|7966|/data/primary966 +969|967|p|p|s|u|sdw49|sdw49|7967|/data/primary967 +970|968|p|p|s|u|sdw49|sdw49|7968|/data/primary968 +971|969|p|p|s|u|sdw49|sdw49|7969|/data/primary969 +972|970|p|p|s|u|sdw49|sdw49|7970|/data/primary970 +973|971|p|p|s|u|sdw49|sdw49|7971|/data/primary971 +974|972|p|p|s|u|sdw49|sdw49|7972|/data/primary972 +975|973|p|p|s|u|sdw49|sdw49|7973|/data/primary973 +976|974|p|p|s|u|sdw49|sdw49|7974|/data/primary974 +977|975|p|p|s|u|sdw49|sdw49|7975|/data/primary975 +978|976|p|p|s|u|sdw49|sdw49|7976|/data/primary976 +979|977|p|p|s|u|sdw49|sdw49|7977|/data/primary977 +980|978|p|p|s|u|sdw49|sdw49|7978|/data/primary978 +981|979|p|p|s|u|sdw49|sdw49|7979|/data/primary979 +1581|579|m|m|s|u|sdw49|sdw49|8579|/data/mirror579 +1600|598|m|m|s|u|sdw49|sdw49|8598|/data/mirror598 +1619|617|m|m|s|u|sdw49|sdw49|8617|/data/mirror617 +1638|636|m|m|s|u|sdw49|sdw49|8636|/data/mirror636 +1657|655|m|m|s|u|sdw49|sdw49|8655|/data/mirror655 +1676|674|m|m|s|u|sdw49|sdw49|8674|/data/mirror674 +1695|693|m|m|s|u|sdw49|sdw49|8693|/data/mirror693 +1714|712|m|m|s|u|sdw49|sdw49|8712|/data/mirror712 +1733|731|m|m|s|u|sdw49|sdw49|8731|/data/mirror731 +1752|750|m|m|s|u|sdw49|sdw49|8750|/data/mirror750 +1771|769|m|m|s|u|sdw49|sdw49|8769|/data/mirror769 +1790|788|m|m|s|u|sdw49|sdw49|8788|/data/mirror788 +1809|807|m|m|s|u|sdw49|sdw49|8807|/data/mirror807 +1828|826|m|m|s|u|sdw49|sdw49|8826|/data/mirror826 +1847|845|m|m|s|u|sdw49|sdw49|8845|/data/mirror845 +1866|864|m|m|s|u|sdw49|sdw49|8864|/data/mirror864 +1885|883|m|m|s|u|sdw49|sdw49|8883|/data/mirror883 +1904|902|m|m|s|u|sdw49|sdw49|8902|/data/mirror902 +1923|921|m|m|s|u|sdw49|sdw49|8921|/data/mirror921 +1942|940|m|m|s|u|sdw49|sdw49|8940|/data/mirror940 +# SDW5 +82|80|p|p|s|u|sdw5|sdw5|7080|/data/primary80 +83|81|p|p|s|u|sdw5|sdw5|7081|/data/primary81 +84|82|p|p|s|u|sdw5|sdw5|7082|/data/primary82 +85|83|p|p|s|u|sdw5|sdw5|7083|/data/primary83 +86|84|p|p|s|u|sdw5|sdw5|7084|/data/primary84 +87|85|p|p|s|u|sdw5|sdw5|7085|/data/primary85 +88|86|p|p|s|u|sdw5|sdw5|7086|/data/primary86 +89|87|p|p|s|u|sdw5|sdw5|7087|/data/primary87 +90|88|p|p|s|u|sdw5|sdw5|7088|/data/primary88 +91|89|p|p|s|u|sdw5|sdw5|7089|/data/primary89 +92|90|p|p|s|u|sdw5|sdw5|7090|/data/primary90 +93|91|p|p|s|u|sdw5|sdw5|7091|/data/primary91 +94|92|p|p|s|u|sdw5|sdw5|7092|/data/primary92 +95|93|p|p|s|u|sdw5|sdw5|7093|/data/primary93 +96|94|p|p|s|u|sdw5|sdw5|7094|/data/primary94 +97|95|p|p|s|u|sdw5|sdw5|7095|/data/primary95 +98|96|p|p|s|u|sdw5|sdw5|7096|/data/primary96 +99|97|p|p|s|u|sdw5|sdw5|7097|/data/primary97 +100|98|p|p|s|u|sdw5|sdw5|7098|/data/primary98 +101|99|p|p|s|u|sdw5|sdw5|7099|/data/primary99 +1005|3|m|m|s|u|sdw5|sdw5|8003|/data/mirror3 +1024|22|m|m|s|u|sdw5|sdw5|8022|/data/mirror22 +1043|41|m|m|s|u|sdw5|sdw5|8041|/data/mirror41 +1062|60|m|m|s|u|sdw5|sdw5|8060|/data/mirror60 +1701|699|m|m|s|u|sdw5|sdw5|8699|/data/mirror699 +1720|718|m|m|s|u|sdw5|sdw5|8718|/data/mirror718 +1739|737|m|m|s|u|sdw5|sdw5|8737|/data/mirror737 +1758|756|m|m|s|u|sdw5|sdw5|8756|/data/mirror756 +1777|775|m|m|s|u|sdw5|sdw5|8775|/data/mirror775 +1796|794|m|m|s|u|sdw5|sdw5|8794|/data/mirror794 +1815|813|m|m|s|u|sdw5|sdw5|8813|/data/mirror813 +1834|832|m|m|s|u|sdw5|sdw5|8832|/data/mirror832 +1853|851|m|m|s|u|sdw5|sdw5|8851|/data/mirror851 +1872|870|m|m|s|u|sdw5|sdw5|8870|/data/mirror870 +1891|889|m|m|s|u|sdw5|sdw5|8889|/data/mirror889 +1910|908|m|m|s|u|sdw5|sdw5|8908|/data/mirror908 +1929|927|m|m|s|u|sdw5|sdw5|8927|/data/mirror927 +1948|946|m|m|s|u|sdw5|sdw5|8946|/data/mirror946 +1967|965|m|m|s|u|sdw5|sdw5|8965|/data/mirror965 +1986|984|m|m|s|u|sdw5|sdw5|8984|/data/mirror984 +# SDW50 +982|980|p|p|s|u|sdw50|sdw50|7980|/data/primary980 +983|981|p|p|s|u|sdw50|sdw50|7981|/data/primary981 +984|982|p|p|s|u|sdw50|sdw50|7982|/data/primary982 +985|983|p|p|s|u|sdw50|sdw50|7983|/data/primary983 +986|984|p|p|s|u|sdw50|sdw50|7984|/data/primary984 +987|985|p|p|s|u|sdw50|sdw50|7985|/data/primary985 +988|986|p|p|s|u|sdw50|sdw50|7986|/data/primary986 +989|987|p|p|s|u|sdw50|sdw50|7987|/data/primary987 +990|988|p|p|s|u|sdw50|sdw50|7988|/data/primary988 +991|989|p|p|s|u|sdw50|sdw50|7989|/data/primary989 +992|990|p|p|s|u|sdw50|sdw50|7990|/data/primary990 +993|991|p|p|s|u|sdw50|sdw50|7991|/data/primary991 +994|992|p|p|s|u|sdw50|sdw50|7992|/data/primary992 +995|993|p|p|s|u|sdw50|sdw50|7993|/data/primary993 +996|994|p|p|s|u|sdw50|sdw50|7994|/data/primary994 +997|995|p|p|s|u|sdw50|sdw50|7995|/data/primary995 +998|996|p|p|s|u|sdw50|sdw50|7996|/data/primary996 +999|997|p|p|s|u|sdw50|sdw50|7997|/data/primary997 +1000|998|p|p|s|u|sdw50|sdw50|7998|/data/primary998 +1001|999|p|p|s|u|sdw50|sdw50|7999|/data/primary999 +1601|599|m|m|s|u|sdw50|sdw50|8599|/data/mirror599 +1620|618|m|m|s|u|sdw50|sdw50|8618|/data/mirror618 +1639|637|m|m|s|u|sdw50|sdw50|8637|/data/mirror637 +1658|656|m|m|s|u|sdw50|sdw50|8656|/data/mirror656 +1677|675|m|m|s|u|sdw50|sdw50|8675|/data/mirror675 +1696|694|m|m|s|u|sdw50|sdw50|8694|/data/mirror694 +1715|713|m|m|s|u|sdw50|sdw50|8713|/data/mirror713 +1734|732|m|m|s|u|sdw50|sdw50|8732|/data/mirror732 +1753|751|m|m|s|u|sdw50|sdw50|8751|/data/mirror751 +1772|770|m|m|s|u|sdw50|sdw50|8770|/data/mirror770 +1791|789|m|m|s|u|sdw50|sdw50|8789|/data/mirror789 +1810|808|m|m|s|u|sdw50|sdw50|8808|/data/mirror808 +1829|827|m|m|s|u|sdw50|sdw50|8827|/data/mirror827 +1848|846|m|m|s|u|sdw50|sdw50|8846|/data/mirror846 +1867|865|m|m|s|u|sdw50|sdw50|8865|/data/mirror865 +1886|884|m|m|s|u|sdw50|sdw50|8884|/data/mirror884 +1905|903|m|m|s|u|sdw50|sdw50|8903|/data/mirror903 +1924|922|m|m|s|u|sdw50|sdw50|8922|/data/mirror922 +1943|941|m|m|s|u|sdw50|sdw50|8941|/data/mirror941 +1962|960|m|m|s|u|sdw50|sdw50|8960|/data/mirror960 +# SDW6 +102|100|p|p|s|u|sdw6|sdw6|7100|/data/primary100 +103|101|p|p|s|u|sdw6|sdw6|7101|/data/primary101 +104|102|p|p|s|u|sdw6|sdw6|7102|/data/primary102 +105|103|p|p|s|u|sdw6|sdw6|7103|/data/primary103 +106|104|p|p|s|u|sdw6|sdw6|7104|/data/primary104 +107|105|p|p|s|u|sdw6|sdw6|7105|/data/primary105 +108|106|p|p|s|u|sdw6|sdw6|7106|/data/primary106 +109|107|p|p|s|u|sdw6|sdw6|7107|/data/primary107 +110|108|p|p|s|u|sdw6|sdw6|7108|/data/primary108 +111|109|p|p|s|u|sdw6|sdw6|7109|/data/primary109 +112|110|p|p|s|u|sdw6|sdw6|7110|/data/primary110 +113|111|p|p|s|u|sdw6|sdw6|7111|/data/primary111 +114|112|p|p|s|u|sdw6|sdw6|7112|/data/primary112 +115|113|p|p|s|u|sdw6|sdw6|7113|/data/primary113 +116|114|p|p|s|u|sdw6|sdw6|7114|/data/primary114 +117|115|p|p|s|u|sdw6|sdw6|7115|/data/primary115 +118|116|p|p|s|u|sdw6|sdw6|7116|/data/primary116 +119|117|p|p|s|u|sdw6|sdw6|7117|/data/primary117 +120|118|p|p|s|u|sdw6|sdw6|7118|/data/primary118 +121|119|p|p|s|u|sdw6|sdw6|7119|/data/primary119 +1006|4|m|m|s|u|sdw6|sdw6|8004|/data/mirror4 +1025|23|m|m|s|u|sdw6|sdw6|8023|/data/mirror23 +1044|42|m|m|s|u|sdw6|sdw6|8042|/data/mirror42 +1063|61|m|m|s|u|sdw6|sdw6|8061|/data/mirror61 +1082|80|m|m|s|u|sdw6|sdw6|8080|/data/mirror80 +1721|719|m|m|s|u|sdw6|sdw6|8719|/data/mirror719 +1740|738|m|m|s|u|sdw6|sdw6|8738|/data/mirror738 +1759|757|m|m|s|u|sdw6|sdw6|8757|/data/mirror757 +1778|776|m|m|s|u|sdw6|sdw6|8776|/data/mirror776 +1797|795|m|m|s|u|sdw6|sdw6|8795|/data/mirror795 +1816|814|m|m|s|u|sdw6|sdw6|8814|/data/mirror814 +1835|833|m|m|s|u|sdw6|sdw6|8833|/data/mirror833 +1854|852|m|m|s|u|sdw6|sdw6|8852|/data/mirror852 +1873|871|m|m|s|u|sdw6|sdw6|8871|/data/mirror871 +1892|890|m|m|s|u|sdw6|sdw6|8890|/data/mirror890 +1911|909|m|m|s|u|sdw6|sdw6|8909|/data/mirror909 +1930|928|m|m|s|u|sdw6|sdw6|8928|/data/mirror928 +1949|947|m|m|s|u|sdw6|sdw6|8947|/data/mirror947 +1968|966|m|m|s|u|sdw6|sdw6|8966|/data/mirror966 +1987|985|m|m|s|u|sdw6|sdw6|8985|/data/mirror985 +# SDW7 +122|120|p|p|s|u|sdw7|sdw7|7120|/data/primary120 +123|121|p|p|s|u|sdw7|sdw7|7121|/data/primary121 +124|122|p|p|s|u|sdw7|sdw7|7122|/data/primary122 +125|123|p|p|s|u|sdw7|sdw7|7123|/data/primary123 +126|124|p|p|s|u|sdw7|sdw7|7124|/data/primary124 +127|125|p|p|s|u|sdw7|sdw7|7125|/data/primary125 +128|126|p|p|s|u|sdw7|sdw7|7126|/data/primary126 +129|127|p|p|s|u|sdw7|sdw7|7127|/data/primary127 +130|128|p|p|s|u|sdw7|sdw7|7128|/data/primary128 +131|129|p|p|s|u|sdw7|sdw7|7129|/data/primary129 +132|130|p|p|s|u|sdw7|sdw7|7130|/data/primary130 +133|131|p|p|s|u|sdw7|sdw7|7131|/data/primary131 +134|132|p|p|s|u|sdw7|sdw7|7132|/data/primary132 +135|133|p|p|s|u|sdw7|sdw7|7133|/data/primary133 +136|134|p|p|s|u|sdw7|sdw7|7134|/data/primary134 +137|135|p|p|s|u|sdw7|sdw7|7135|/data/primary135 +138|136|p|p|s|u|sdw7|sdw7|7136|/data/primary136 +139|137|p|p|s|u|sdw7|sdw7|7137|/data/primary137 +140|138|p|p|s|u|sdw7|sdw7|7138|/data/primary138 +141|139|p|p|s|u|sdw7|sdw7|7139|/data/primary139 +1007|5|m|m|s|u|sdw7|sdw7|8005|/data/mirror5 +1026|24|m|m|s|u|sdw7|sdw7|8024|/data/mirror24 +1045|43|m|m|s|u|sdw7|sdw7|8043|/data/mirror43 +1064|62|m|m|s|u|sdw7|sdw7|8062|/data/mirror62 +1083|81|m|m|s|u|sdw7|sdw7|8081|/data/mirror81 +1102|100|m|m|s|u|sdw7|sdw7|8100|/data/mirror100 +1741|739|m|m|s|u|sdw7|sdw7|8739|/data/mirror739 +1760|758|m|m|s|u|sdw7|sdw7|8758|/data/mirror758 +1779|777|m|m|s|u|sdw7|sdw7|8777|/data/mirror777 +1798|796|m|m|s|u|sdw7|sdw7|8796|/data/mirror796 +1817|815|m|m|s|u|sdw7|sdw7|8815|/data/mirror815 +1836|834|m|m|s|u|sdw7|sdw7|8834|/data/mirror834 +1855|853|m|m|s|u|sdw7|sdw7|8853|/data/mirror853 +1874|872|m|m|s|u|sdw7|sdw7|8872|/data/mirror872 +1893|891|m|m|s|u|sdw7|sdw7|8891|/data/mirror891 +1912|910|m|m|s|u|sdw7|sdw7|8910|/data/mirror910 +1931|929|m|m|s|u|sdw7|sdw7|8929|/data/mirror929 +1950|948|m|m|s|u|sdw7|sdw7|8948|/data/mirror948 +1969|967|m|m|s|u|sdw7|sdw7|8967|/data/mirror967 +1988|986|m|m|s|u|sdw7|sdw7|8986|/data/mirror986 +# SDW8 +142|140|p|p|s|u|sdw8|sdw8|7140|/data/primary140 +143|141|p|p|s|u|sdw8|sdw8|7141|/data/primary141 +144|142|p|p|s|u|sdw8|sdw8|7142|/data/primary142 +145|143|p|p|s|u|sdw8|sdw8|7143|/data/primary143 +146|144|p|p|s|u|sdw8|sdw8|7144|/data/primary144 +147|145|p|p|s|u|sdw8|sdw8|7145|/data/primary145 +148|146|p|p|s|u|sdw8|sdw8|7146|/data/primary146 +149|147|p|p|s|u|sdw8|sdw8|7147|/data/primary147 +150|148|p|p|s|u|sdw8|sdw8|7148|/data/primary148 +151|149|p|p|s|u|sdw8|sdw8|7149|/data/primary149 +152|150|p|p|s|u|sdw8|sdw8|7150|/data/primary150 +153|151|p|p|s|u|sdw8|sdw8|7151|/data/primary151 +154|152|p|p|s|u|sdw8|sdw8|7152|/data/primary152 +155|153|p|p|s|u|sdw8|sdw8|7153|/data/primary153 +156|154|p|p|s|u|sdw8|sdw8|7154|/data/primary154 +157|155|p|p|s|u|sdw8|sdw8|7155|/data/primary155 +158|156|p|p|s|u|sdw8|sdw8|7156|/data/primary156 +159|157|p|p|s|u|sdw8|sdw8|7157|/data/primary157 +160|158|p|p|s|u|sdw8|sdw8|7158|/data/primary158 +161|159|p|p|s|u|sdw8|sdw8|7159|/data/primary159 +1008|6|m|m|s|u|sdw8|sdw8|8006|/data/mirror6 +1027|25|m|m|s|u|sdw8|sdw8|8025|/data/mirror25 +1046|44|m|m|s|u|sdw8|sdw8|8044|/data/mirror44 +1065|63|m|m|s|u|sdw8|sdw8|8063|/data/mirror63 +1084|82|m|m|s|u|sdw8|sdw8|8082|/data/mirror82 +1103|101|m|m|s|u|sdw8|sdw8|8101|/data/mirror101 +1122|120|m|m|s|u|sdw8|sdw8|8120|/data/mirror120 +1761|759|m|m|s|u|sdw8|sdw8|8759|/data/mirror759 +1780|778|m|m|s|u|sdw8|sdw8|8778|/data/mirror778 +1799|797|m|m|s|u|sdw8|sdw8|8797|/data/mirror797 +1818|816|m|m|s|u|sdw8|sdw8|8816|/data/mirror816 +1837|835|m|m|s|u|sdw8|sdw8|8835|/data/mirror835 +1856|854|m|m|s|u|sdw8|sdw8|8854|/data/mirror854 +1875|873|m|m|s|u|sdw8|sdw8|8873|/data/mirror873 +1894|892|m|m|s|u|sdw8|sdw8|8892|/data/mirror892 +1913|911|m|m|s|u|sdw8|sdw8|8911|/data/mirror911 +1932|930|m|m|s|u|sdw8|sdw8|8930|/data/mirror930 +1951|949|m|m|s|u|sdw8|sdw8|8949|/data/mirror949 +1970|968|m|m|s|u|sdw8|sdw8|8968|/data/mirror968 +1989|987|m|m|s|u|sdw8|sdw8|8987|/data/mirror987 +# SDW9 +162|160|p|p|s|u|sdw9|sdw9|7160|/data/primary160 +163|161|p|p|s|u|sdw9|sdw9|7161|/data/primary161 +164|162|p|p|s|u|sdw9|sdw9|7162|/data/primary162 +165|163|p|p|s|u|sdw9|sdw9|7163|/data/primary163 +166|164|p|p|s|u|sdw9|sdw9|7164|/data/primary164 +167|165|p|p|s|u|sdw9|sdw9|7165|/data/primary165 +168|166|p|p|s|u|sdw9|sdw9|7166|/data/primary166 +169|167|p|p|s|u|sdw9|sdw9|7167|/data/primary167 +170|168|p|p|s|u|sdw9|sdw9|7168|/data/primary168 +171|169|p|p|s|u|sdw9|sdw9|7169|/data/primary169 +172|170|p|p|s|u|sdw9|sdw9|7170|/data/primary170 +173|171|p|p|s|u|sdw9|sdw9|7171|/data/primary171 +174|172|p|p|s|u|sdw9|sdw9|7172|/data/primary172 +175|173|p|p|s|u|sdw9|sdw9|7173|/data/primary173 +176|174|p|p|s|u|sdw9|sdw9|7174|/data/primary174 +177|175|p|p|s|u|sdw9|sdw9|7175|/data/primary175 +178|176|p|p|s|u|sdw9|sdw9|7176|/data/primary176 +179|177|p|p|s|u|sdw9|sdw9|7177|/data/primary177 +180|178|p|p|s|u|sdw9|sdw9|7178|/data/primary178 +181|179|p|p|s|u|sdw9|sdw9|7179|/data/primary179 +1009|7|m|m|s|u|sdw9|sdw9|8007|/data/mirror7 +1028|26|m|m|s|u|sdw9|sdw9|8026|/data/mirror26 +1047|45|m|m|s|u|sdw9|sdw9|8045|/data/mirror45 +1066|64|m|m|s|u|sdw9|sdw9|8064|/data/mirror64 +1085|83|m|m|s|u|sdw9|sdw9|8083|/data/mirror83 +1104|102|m|m|s|u|sdw9|sdw9|8102|/data/mirror102 +1123|121|m|m|s|u|sdw9|sdw9|8121|/data/mirror121 +1142|140|m|m|s|u|sdw9|sdw9|8140|/data/mirror140 +1781|779|m|m|s|u|sdw9|sdw9|8779|/data/mirror779 +1800|798|m|m|s|u|sdw9|sdw9|8798|/data/mirror798 +1819|817|m|m|s|u|sdw9|sdw9|8817|/data/mirror817 +1838|836|m|m|s|u|sdw9|sdw9|8836|/data/mirror836 +1857|855|m|m|s|u|sdw9|sdw9|8855|/data/mirror855 +1876|874|m|m|s|u|sdw9|sdw9|8874|/data/mirror874 +1895|893|m|m|s|u|sdw9|sdw9|8893|/data/mirror893 +1914|912|m|m|s|u|sdw9|sdw9|8912|/data/mirror912 +1933|931|m|m|s|u|sdw9|sdw9|8931|/data/mirror931 +1952|950|m|m|s|u|sdw9|sdw9|8950|/data/mirror950 +1971|969|m|m|s|u|sdw9|sdw9|8969|/data/mirror969 +1990|988|m|m|s|u|sdw9|sdw9|8988|/data/mirror988 diff --git a/gpMgmt/bin/gprebalance_modules/test/data/1000_50_unbalanced_grouped.array b/gpMgmt/bin/gprebalance_modules/test/data/1000_50_unbalanced_grouped.array new file mode 100644 index 000000000000..e986a3f9df95 --- /dev/null +++ b/gpMgmt/bin/gprebalance_modules/test/data/1000_50_unbalanced_grouped.array @@ -0,0 +1,2052 @@ +# Coordinator +1|-1|p|p|s|u|cdw|cdw|5432|/data/coordinator +# SDW1 +2|0|p|p|s|u|sdw1|sdw1|7000|/data/primary0 +3|1|p|p|s|u|sdw1|sdw1|7001|/data/primary1 +4|2|p|p|s|u|sdw1|sdw1|7002|/data/primary2 +5|3|p|p|s|u|sdw1|sdw1|7003|/data/primary3 +6|4|p|p|s|u|sdw1|sdw1|7004|/data/primary4 +7|5|p|p|s|u|sdw1|sdw1|7005|/data/primary5 +8|6|p|p|s|u|sdw1|sdw1|7006|/data/primary6 +9|7|p|p|s|u|sdw1|sdw1|7007|/data/primary7 +10|8|p|p|s|u|sdw1|sdw1|7008|/data/primary8 +11|9|p|p|s|u|sdw1|sdw1|7009|/data/primary9 +12|10|p|p|s|u|sdw1|sdw1|7010|/data/primary10 +13|11|p|p|s|u|sdw1|sdw1|7011|/data/primary11 +14|12|p|p|s|u|sdw1|sdw1|7012|/data/primary12 +15|13|p|p|s|u|sdw1|sdw1|7013|/data/primary13 +16|14|p|p|s|u|sdw1|sdw1|7014|/data/primary14 +17|15|p|p|s|u|sdw1|sdw1|7015|/data/primary15 +18|16|p|p|s|u|sdw1|sdw1|7016|/data/primary16 +19|17|p|p|s|u|sdw1|sdw1|7017|/data/primary17 +20|18|p|p|s|u|sdw1|sdw1|7018|/data/primary18 +1097|95|m|m|s|u|sdw1|sdw1|8095|/data/mirror95 +1098|96|m|m|s|u|sdw1|sdw1|8096|/data/mirror96 +1099|97|m|m|s|u|sdw1|sdw1|8097|/data/mirror97 +1100|98|m|m|s|u|sdw1|sdw1|8098|/data/mirror98 +1101|99|m|m|s|u|sdw1|sdw1|8099|/data/mirror99 +1102|100|m|m|s|u|sdw1|sdw1|8100|/data/mirror100 +1103|101|m|m|s|u|sdw1|sdw1|8101|/data/mirror101 +1104|102|m|m|s|u|sdw1|sdw1|8102|/data/mirror102 +1105|103|m|m|s|u|sdw1|sdw1|8103|/data/mirror103 +1106|104|m|m|s|u|sdw1|sdw1|8104|/data/mirror104 +1107|105|m|m|s|u|sdw1|sdw1|8105|/data/mirror105 +1108|106|m|m|s|u|sdw1|sdw1|8106|/data/mirror106 +1109|107|m|m|s|u|sdw1|sdw1|8107|/data/mirror107 +1110|108|m|m|s|u|sdw1|sdw1|8108|/data/mirror108 +1111|109|m|m|s|u|sdw1|sdw1|8109|/data/mirror109 +1112|110|m|m|s|u|sdw1|sdw1|8110|/data/mirror110 +1113|111|m|m|s|u|sdw1|sdw1|8111|/data/mirror111 +# SDW10 +175|173|p|p|s|u|sdw10|sdw10|7173|/data/primary173 +176|174|p|p|s|u|sdw10|sdw10|7174|/data/primary174 +177|175|p|p|s|u|sdw10|sdw10|7175|/data/primary175 +178|176|p|p|s|u|sdw10|sdw10|7176|/data/primary176 +179|177|p|p|s|u|sdw10|sdw10|7177|/data/primary177 +180|178|p|p|s|u|sdw10|sdw10|7178|/data/primary178 +181|179|p|p|s|u|sdw10|sdw10|7179|/data/primary179 +182|180|p|p|s|u|sdw10|sdw10|7180|/data/primary180 +183|181|p|p|s|u|sdw10|sdw10|7181|/data/primary181 +184|182|p|p|s|u|sdw10|sdw10|7182|/data/primary182 +185|183|p|p|s|u|sdw10|sdw10|7183|/data/primary183 +186|184|p|p|s|u|sdw10|sdw10|7184|/data/primary184 +187|185|p|p|s|u|sdw10|sdw10|7185|/data/primary185 +188|186|p|p|s|u|sdw10|sdw10|7186|/data/primary186 +189|187|p|p|s|u|sdw10|sdw10|7187|/data/primary187 +190|188|p|p|s|u|sdw10|sdw10|7188|/data/primary188 +191|189|p|p|s|u|sdw10|sdw10|7189|/data/primary189 +192|190|p|p|s|u|sdw10|sdw10|7190|/data/primary190 +193|191|p|p|s|u|sdw10|sdw10|7191|/data/primary191 +194|192|p|p|s|u|sdw10|sdw10|7192|/data/primary192 +195|193|p|p|s|u|sdw10|sdw10|7193|/data/primary193 +196|194|p|p|s|u|sdw10|sdw10|7194|/data/primary194 +197|195|p|p|s|u|sdw10|sdw10|7195|/data/primary195 +198|196|p|p|s|u|sdw10|sdw10|7196|/data/primary196 +1132|130|m|m|s|u|sdw10|sdw10|8130|/data/mirror130 +1133|131|m|m|s|u|sdw10|sdw10|8131|/data/mirror131 +1134|132|m|m|s|u|sdw10|sdw10|8132|/data/mirror132 +1135|133|m|m|s|u|sdw10|sdw10|8133|/data/mirror133 +1136|134|m|m|s|u|sdw10|sdw10|8134|/data/mirror134 +1137|135|m|m|s|u|sdw10|sdw10|8135|/data/mirror135 +1138|136|m|m|s|u|sdw10|sdw10|8136|/data/mirror136 +1139|137|m|m|s|u|sdw10|sdw10|8137|/data/mirror137 +1140|138|m|m|s|u|sdw10|sdw10|8138|/data/mirror138 +1141|139|m|m|s|u|sdw10|sdw10|8139|/data/mirror139 +1142|140|m|m|s|u|sdw10|sdw10|8140|/data/mirror140 +1143|141|m|m|s|u|sdw10|sdw10|8141|/data/mirror141 +1144|142|m|m|s|u|sdw10|sdw10|8142|/data/mirror142 +1145|143|m|m|s|u|sdw10|sdw10|8143|/data/mirror143 +1146|144|m|m|s|u|sdw10|sdw10|8144|/data/mirror144 +1147|145|m|m|s|u|sdw10|sdw10|8145|/data/mirror145 +1148|146|m|m|s|u|sdw10|sdw10|8146|/data/mirror146 +1149|147|m|m|s|u|sdw10|sdw10|8147|/data/mirror147 +1150|148|m|m|s|u|sdw10|sdw10|8148|/data/mirror148 +1151|149|m|m|s|u|sdw10|sdw10|8149|/data/mirror149 +# SDW11 +199|197|p|p|s|u|sdw11|sdw11|7197|/data/primary197 +200|198|p|p|s|u|sdw11|sdw11|7198|/data/primary198 +201|199|p|p|s|u|sdw11|sdw11|7199|/data/primary199 +202|200|p|p|s|u|sdw11|sdw11|7200|/data/primary200 +203|201|p|p|s|u|sdw11|sdw11|7201|/data/primary201 +204|202|p|p|s|u|sdw11|sdw11|7202|/data/primary202 +205|203|p|p|s|u|sdw11|sdw11|7203|/data/primary203 +206|204|p|p|s|u|sdw11|sdw11|7204|/data/primary204 +207|205|p|p|s|u|sdw11|sdw11|7205|/data/primary205 +208|206|p|p|s|u|sdw11|sdw11|7206|/data/primary206 +209|207|p|p|s|u|sdw11|sdw11|7207|/data/primary207 +210|208|p|p|s|u|sdw11|sdw11|7208|/data/primary208 +211|209|p|p|s|u|sdw11|sdw11|7209|/data/primary209 +212|210|p|p|s|u|sdw11|sdw11|7210|/data/primary210 +213|211|p|p|s|u|sdw11|sdw11|7211|/data/primary211 +214|212|p|p|s|u|sdw11|sdw11|7212|/data/primary212 +215|213|p|p|s|u|sdw11|sdw11|7213|/data/primary213 +216|214|p|p|s|u|sdw11|sdw11|7214|/data/primary214 +217|215|p|p|s|u|sdw11|sdw11|7215|/data/primary215 +218|216|p|p|s|u|sdw11|sdw11|7216|/data/primary216 +1152|150|m|m|s|u|sdw11|sdw11|8150|/data/mirror150 +1153|151|m|m|s|u|sdw11|sdw11|8151|/data/mirror151 +1154|152|m|m|s|u|sdw11|sdw11|8152|/data/mirror152 +1155|153|m|m|s|u|sdw11|sdw11|8153|/data/mirror153 +1156|154|m|m|s|u|sdw11|sdw11|8154|/data/mirror154 +1157|155|m|m|s|u|sdw11|sdw11|8155|/data/mirror155 +1158|156|m|m|s|u|sdw11|sdw11|8156|/data/mirror156 +1159|157|m|m|s|u|sdw11|sdw11|8157|/data/mirror157 +1160|158|m|m|s|u|sdw11|sdw11|8158|/data/mirror158 +1161|159|m|m|s|u|sdw11|sdw11|8159|/data/mirror159 +1162|160|m|m|s|u|sdw11|sdw11|8160|/data/mirror160 +1163|161|m|m|s|u|sdw11|sdw11|8161|/data/mirror161 +1164|162|m|m|s|u|sdw11|sdw11|8162|/data/mirror162 +1165|163|m|m|s|u|sdw11|sdw11|8163|/data/mirror163 +1166|164|m|m|s|u|sdw11|sdw11|8164|/data/mirror164 +1167|165|m|m|s|u|sdw11|sdw11|8165|/data/mirror165 +1168|166|m|m|s|u|sdw11|sdw11|8166|/data/mirror166 +1169|167|m|m|s|u|sdw11|sdw11|8167|/data/mirror167 +1170|168|m|m|s|u|sdw11|sdw11|8168|/data/mirror168 +1171|169|m|m|s|u|sdw11|sdw11|8169|/data/mirror169 +1172|170|m|m|s|u|sdw11|sdw11|8170|/data/mirror170 +1173|171|m|m|s|u|sdw11|sdw11|8171|/data/mirror171 +1174|172|m|m|s|u|sdw11|sdw11|8172|/data/mirror172 +# SDW12 +219|217|p|p|s|u|sdw12|sdw12|7217|/data/primary217 +220|218|p|p|s|u|sdw12|sdw12|7218|/data/primary218 +221|219|p|p|s|u|sdw12|sdw12|7219|/data/primary219 +222|220|p|p|s|u|sdw12|sdw12|7220|/data/primary220 +223|221|p|p|s|u|sdw12|sdw12|7221|/data/primary221 +224|222|p|p|s|u|sdw12|sdw12|7222|/data/primary222 +225|223|p|p|s|u|sdw12|sdw12|7223|/data/primary223 +226|224|p|p|s|u|sdw12|sdw12|7224|/data/primary224 +227|225|p|p|s|u|sdw12|sdw12|7225|/data/primary225 +228|226|p|p|s|u|sdw12|sdw12|7226|/data/primary226 +229|227|p|p|s|u|sdw12|sdw12|7227|/data/primary227 +230|228|p|p|s|u|sdw12|sdw12|7228|/data/primary228 +231|229|p|p|s|u|sdw12|sdw12|7229|/data/primary229 +232|230|p|p|s|u|sdw12|sdw12|7230|/data/primary230 +233|231|p|p|s|u|sdw12|sdw12|7231|/data/primary231 +234|232|p|p|s|u|sdw12|sdw12|7232|/data/primary232 +235|233|p|p|s|u|sdw12|sdw12|7233|/data/primary233 +236|234|p|p|s|u|sdw12|sdw12|7234|/data/primary234 +237|235|p|p|s|u|sdw12|sdw12|7235|/data/primary235 +238|236|p|p|s|u|sdw12|sdw12|7236|/data/primary236 +239|237|p|p|s|u|sdw12|sdw12|7237|/data/primary237 +240|238|p|p|s|u|sdw12|sdw12|7238|/data/primary238 +1199|197|m|m|s|u|sdw12|sdw12|8197|/data/mirror197 +1200|198|m|m|s|u|sdw12|sdw12|8198|/data/mirror198 +1201|199|m|m|s|u|sdw12|sdw12|8199|/data/mirror199 +1202|200|m|m|s|u|sdw12|sdw12|8200|/data/mirror200 +1203|201|m|m|s|u|sdw12|sdw12|8201|/data/mirror201 +1204|202|m|m|s|u|sdw12|sdw12|8202|/data/mirror202 +1205|203|m|m|s|u|sdw12|sdw12|8203|/data/mirror203 +1206|204|m|m|s|u|sdw12|sdw12|8204|/data/mirror204 +1207|205|m|m|s|u|sdw12|sdw12|8205|/data/mirror205 +1208|206|m|m|s|u|sdw12|sdw12|8206|/data/mirror206 +1209|207|m|m|s|u|sdw12|sdw12|8207|/data/mirror207 +1210|208|m|m|s|u|sdw12|sdw12|8208|/data/mirror208 +1211|209|m|m|s|u|sdw12|sdw12|8209|/data/mirror209 +1212|210|m|m|s|u|sdw12|sdw12|8210|/data/mirror210 +1213|211|m|m|s|u|sdw12|sdw12|8211|/data/mirror211 +1214|212|m|m|s|u|sdw12|sdw12|8212|/data/mirror212 +1215|213|m|m|s|u|sdw12|sdw12|8213|/data/mirror213 +1216|214|m|m|s|u|sdw12|sdw12|8214|/data/mirror214 +1217|215|m|m|s|u|sdw12|sdw12|8215|/data/mirror215 +1218|216|m|m|s|u|sdw12|sdw12|8216|/data/mirror216 +# SDW13 +241|239|p|p|s|u|sdw13|sdw13|7239|/data/primary239 +242|240|p|p|s|u|sdw13|sdw13|7240|/data/primary240 +243|241|p|p|s|u|sdw13|sdw13|7241|/data/primary241 +244|242|p|p|s|u|sdw13|sdw13|7242|/data/primary242 +245|243|p|p|s|u|sdw13|sdw13|7243|/data/primary243 +246|244|p|p|s|u|sdw13|sdw13|7244|/data/primary244 +247|245|p|p|s|u|sdw13|sdw13|7245|/data/primary245 +248|246|p|p|s|u|sdw13|sdw13|7246|/data/primary246 +249|247|p|p|s|u|sdw13|sdw13|7247|/data/primary247 +250|248|p|p|s|u|sdw13|sdw13|7248|/data/primary248 +251|249|p|p|s|u|sdw13|sdw13|7249|/data/primary249 +252|250|p|p|s|u|sdw13|sdw13|7250|/data/primary250 +253|251|p|p|s|u|sdw13|sdw13|7251|/data/primary251 +254|252|p|p|s|u|sdw13|sdw13|7252|/data/primary252 +255|253|p|p|s|u|sdw13|sdw13|7253|/data/primary253 +256|254|p|p|s|u|sdw13|sdw13|7254|/data/primary254 +257|255|p|p|s|u|sdw13|sdw13|7255|/data/primary255 +258|256|p|p|s|u|sdw13|sdw13|7256|/data/primary256 +259|257|p|p|s|u|sdw13|sdw13|7257|/data/primary257 +260|258|p|p|s|u|sdw13|sdw13|7258|/data/primary258 +261|259|p|p|s|u|sdw13|sdw13|7259|/data/primary259 +262|260|p|p|s|u|sdw13|sdw13|7260|/data/primary260 +263|261|p|p|s|u|sdw13|sdw13|7261|/data/primary261 +264|262|p|p|s|u|sdw13|sdw13|7262|/data/primary262 +1283|281|m|m|s|u|sdw13|sdw13|8281|/data/mirror281 +1284|282|m|m|s|u|sdw13|sdw13|8282|/data/mirror282 +1285|283|m|m|s|u|sdw13|sdw13|8283|/data/mirror283 +1286|284|m|m|s|u|sdw13|sdw13|8284|/data/mirror284 +1287|285|m|m|s|u|sdw13|sdw13|8285|/data/mirror285 +1288|286|m|m|s|u|sdw13|sdw13|8286|/data/mirror286 +1289|287|m|m|s|u|sdw13|sdw13|8287|/data/mirror287 +1290|288|m|m|s|u|sdw13|sdw13|8288|/data/mirror288 +1291|289|m|m|s|u|sdw13|sdw13|8289|/data/mirror289 +1292|290|m|m|s|u|sdw13|sdw13|8290|/data/mirror290 +1293|291|m|m|s|u|sdw13|sdw13|8291|/data/mirror291 +1294|292|m|m|s|u|sdw13|sdw13|8292|/data/mirror292 +1295|293|m|m|s|u|sdw13|sdw13|8293|/data/mirror293 +1296|294|m|m|s|u|sdw13|sdw13|8294|/data/mirror294 +1297|295|m|m|s|u|sdw13|sdw13|8295|/data/mirror295 +1298|296|m|m|s|u|sdw13|sdw13|8296|/data/mirror296 +1299|297|m|m|s|u|sdw13|sdw13|8297|/data/mirror297 +1300|298|m|m|s|u|sdw13|sdw13|8298|/data/mirror298 +1301|299|m|m|s|u|sdw13|sdw13|8299|/data/mirror299 +1302|300|m|m|s|u|sdw13|sdw13|8300|/data/mirror300 +1303|301|m|m|s|u|sdw13|sdw13|8301|/data/mirror301 +# SDW14 +265|263|p|p|s|u|sdw14|sdw14|7263|/data/primary263 +266|264|p|p|s|u|sdw14|sdw14|7264|/data/primary264 +267|265|p|p|s|u|sdw14|sdw14|7265|/data/primary265 +268|266|p|p|s|u|sdw14|sdw14|7266|/data/primary266 +269|267|p|p|s|u|sdw14|sdw14|7267|/data/primary267 +270|268|p|p|s|u|sdw14|sdw14|7268|/data/primary268 +271|269|p|p|s|u|sdw14|sdw14|7269|/data/primary269 +272|270|p|p|s|u|sdw14|sdw14|7270|/data/primary270 +273|271|p|p|s|u|sdw14|sdw14|7271|/data/primary271 +274|272|p|p|s|u|sdw14|sdw14|7272|/data/primary272 +275|273|p|p|s|u|sdw14|sdw14|7273|/data/primary273 +276|274|p|p|s|u|sdw14|sdw14|7274|/data/primary274 +277|275|p|p|s|u|sdw14|sdw14|7275|/data/primary275 +278|276|p|p|s|u|sdw14|sdw14|7276|/data/primary276 +279|277|p|p|s|u|sdw14|sdw14|7277|/data/primary277 +280|278|p|p|s|u|sdw14|sdw14|7278|/data/primary278 +281|279|p|p|s|u|sdw14|sdw14|7279|/data/primary279 +282|280|p|p|s|u|sdw14|sdw14|7280|/data/primary280 +1241|239|m|m|s|u|sdw14|sdw14|8239|/data/mirror239 +1242|240|m|m|s|u|sdw14|sdw14|8240|/data/mirror240 +1243|241|m|m|s|u|sdw14|sdw14|8241|/data/mirror241 +1244|242|m|m|s|u|sdw14|sdw14|8242|/data/mirror242 +1245|243|m|m|s|u|sdw14|sdw14|8243|/data/mirror243 +1246|244|m|m|s|u|sdw14|sdw14|8244|/data/mirror244 +1247|245|m|m|s|u|sdw14|sdw14|8245|/data/mirror245 +1248|246|m|m|s|u|sdw14|sdw14|8246|/data/mirror246 +1249|247|m|m|s|u|sdw14|sdw14|8247|/data/mirror247 +1250|248|m|m|s|u|sdw14|sdw14|8248|/data/mirror248 +1251|249|m|m|s|u|sdw14|sdw14|8249|/data/mirror249 +1252|250|m|m|s|u|sdw14|sdw14|8250|/data/mirror250 +1253|251|m|m|s|u|sdw14|sdw14|8251|/data/mirror251 +1254|252|m|m|s|u|sdw14|sdw14|8252|/data/mirror252 +1255|253|m|m|s|u|sdw14|sdw14|8253|/data/mirror253 +1256|254|m|m|s|u|sdw14|sdw14|8254|/data/mirror254 +1257|255|m|m|s|u|sdw14|sdw14|8255|/data/mirror255 +1258|256|m|m|s|u|sdw14|sdw14|8256|/data/mirror256 +1259|257|m|m|s|u|sdw14|sdw14|8257|/data/mirror257 +1260|258|m|m|s|u|sdw14|sdw14|8258|/data/mirror258 +1261|259|m|m|s|u|sdw14|sdw14|8259|/data/mirror259 +1262|260|m|m|s|u|sdw14|sdw14|8260|/data/mirror260 +1263|261|m|m|s|u|sdw14|sdw14|8261|/data/mirror261 +1264|262|m|m|s|u|sdw14|sdw14|8262|/data/mirror262 +# SDW15 +283|281|p|p|s|u|sdw15|sdw15|7281|/data/primary281 +284|282|p|p|s|u|sdw15|sdw15|7282|/data/primary282 +285|283|p|p|s|u|sdw15|sdw15|7283|/data/primary283 +286|284|p|p|s|u|sdw15|sdw15|7284|/data/primary284 +287|285|p|p|s|u|sdw15|sdw15|7285|/data/primary285 +288|286|p|p|s|u|sdw15|sdw15|7286|/data/primary286 +289|287|p|p|s|u|sdw15|sdw15|7287|/data/primary287 +290|288|p|p|s|u|sdw15|sdw15|7288|/data/primary288 +291|289|p|p|s|u|sdw15|sdw15|7289|/data/primary289 +292|290|p|p|s|u|sdw15|sdw15|7290|/data/primary290 +293|291|p|p|s|u|sdw15|sdw15|7291|/data/primary291 +294|292|p|p|s|u|sdw15|sdw15|7292|/data/primary292 +295|293|p|p|s|u|sdw15|sdw15|7293|/data/primary293 +296|294|p|p|s|u|sdw15|sdw15|7294|/data/primary294 +297|295|p|p|s|u|sdw15|sdw15|7295|/data/primary295 +298|296|p|p|s|u|sdw15|sdw15|7296|/data/primary296 +299|297|p|p|s|u|sdw15|sdw15|7297|/data/primary297 +300|298|p|p|s|u|sdw15|sdw15|7298|/data/primary298 +301|299|p|p|s|u|sdw15|sdw15|7299|/data/primary299 +302|300|p|p|s|u|sdw15|sdw15|7300|/data/primary300 +303|301|p|p|s|u|sdw15|sdw15|7301|/data/primary301 +1265|263|m|m|s|u|sdw15|sdw15|8263|/data/mirror263 +1266|264|m|m|s|u|sdw15|sdw15|8264|/data/mirror264 +1267|265|m|m|s|u|sdw15|sdw15|8265|/data/mirror265 +1268|266|m|m|s|u|sdw15|sdw15|8266|/data/mirror266 +1269|267|m|m|s|u|sdw15|sdw15|8267|/data/mirror267 +1270|268|m|m|s|u|sdw15|sdw15|8268|/data/mirror268 +1271|269|m|m|s|u|sdw15|sdw15|8269|/data/mirror269 +1272|270|m|m|s|u|sdw15|sdw15|8270|/data/mirror270 +1273|271|m|m|s|u|sdw15|sdw15|8271|/data/mirror271 +1274|272|m|m|s|u|sdw15|sdw15|8272|/data/mirror272 +1275|273|m|m|s|u|sdw15|sdw15|8273|/data/mirror273 +1276|274|m|m|s|u|sdw15|sdw15|8274|/data/mirror274 +1277|275|m|m|s|u|sdw15|sdw15|8275|/data/mirror275 +1278|276|m|m|s|u|sdw15|sdw15|8276|/data/mirror276 +1279|277|m|m|s|u|sdw15|sdw15|8277|/data/mirror277 +1280|278|m|m|s|u|sdw15|sdw15|8278|/data/mirror278 +1281|279|m|m|s|u|sdw15|sdw15|8279|/data/mirror279 +1282|280|m|m|s|u|sdw15|sdw15|8280|/data/mirror280 +# SDW16 +304|302|p|p|s|u|sdw16|sdw16|7302|/data/primary302 +305|303|p|p|s|u|sdw16|sdw16|7303|/data/primary303 +306|304|p|p|s|u|sdw16|sdw16|7304|/data/primary304 +307|305|p|p|s|u|sdw16|sdw16|7305|/data/primary305 +308|306|p|p|s|u|sdw16|sdw16|7306|/data/primary306 +309|307|p|p|s|u|sdw16|sdw16|7307|/data/primary307 +310|308|p|p|s|u|sdw16|sdw16|7308|/data/primary308 +311|309|p|p|s|u|sdw16|sdw16|7309|/data/primary309 +312|310|p|p|s|u|sdw16|sdw16|7310|/data/primary310 +313|311|p|p|s|u|sdw16|sdw16|7311|/data/primary311 +314|312|p|p|s|u|sdw16|sdw16|7312|/data/primary312 +315|313|p|p|s|u|sdw16|sdw16|7313|/data/primary313 +316|314|p|p|s|u|sdw16|sdw16|7314|/data/primary314 +317|315|p|p|s|u|sdw16|sdw16|7315|/data/primary315 +318|316|p|p|s|u|sdw16|sdw16|7316|/data/primary316 +319|317|p|p|s|u|sdw16|sdw16|7317|/data/primary317 +320|318|p|p|s|u|sdw16|sdw16|7318|/data/primary318 +321|319|p|p|s|u|sdw16|sdw16|7319|/data/primary319 +322|320|p|p|s|u|sdw16|sdw16|7320|/data/primary320 +323|321|p|p|s|u|sdw16|sdw16|7321|/data/primary321 +324|322|p|p|s|u|sdw16|sdw16|7322|/data/primary322 +325|323|p|p|s|u|sdw16|sdw16|7323|/data/primary323 +326|324|p|p|s|u|sdw16|sdw16|7324|/data/primary324 +327|325|p|p|s|u|sdw16|sdw16|7325|/data/primary325 +1343|341|m|m|s|u|sdw16|sdw16|8341|/data/mirror341 +1344|342|m|m|s|u|sdw16|sdw16|8342|/data/mirror342 +1345|343|m|m|s|u|sdw16|sdw16|8343|/data/mirror343 +1346|344|m|m|s|u|sdw16|sdw16|8344|/data/mirror344 +1347|345|m|m|s|u|sdw16|sdw16|8345|/data/mirror345 +1348|346|m|m|s|u|sdw16|sdw16|8346|/data/mirror346 +1349|347|m|m|s|u|sdw16|sdw16|8347|/data/mirror347 +1350|348|m|m|s|u|sdw16|sdw16|8348|/data/mirror348 +1351|349|m|m|s|u|sdw16|sdw16|8349|/data/mirror349 +1352|350|m|m|s|u|sdw16|sdw16|8350|/data/mirror350 +1353|351|m|m|s|u|sdw16|sdw16|8351|/data/mirror351 +1354|352|m|m|s|u|sdw16|sdw16|8352|/data/mirror352 +1355|353|m|m|s|u|sdw16|sdw16|8353|/data/mirror353 +1356|354|m|m|s|u|sdw16|sdw16|8354|/data/mirror354 +1357|355|m|m|s|u|sdw16|sdw16|8355|/data/mirror355 +1358|356|m|m|s|u|sdw16|sdw16|8356|/data/mirror356 +1359|357|m|m|s|u|sdw16|sdw16|8357|/data/mirror357 +1360|358|m|m|s|u|sdw16|sdw16|8358|/data/mirror358 +# SDW17 +328|326|p|p|s|u|sdw17|sdw17|7326|/data/primary326 +329|327|p|p|s|u|sdw17|sdw17|7327|/data/primary327 +330|328|p|p|s|u|sdw17|sdw17|7328|/data/primary328 +331|329|p|p|s|u|sdw17|sdw17|7329|/data/primary329 +332|330|p|p|s|u|sdw17|sdw17|7330|/data/primary330 +333|331|p|p|s|u|sdw17|sdw17|7331|/data/primary331 +334|332|p|p|s|u|sdw17|sdw17|7332|/data/primary332 +335|333|p|p|s|u|sdw17|sdw17|7333|/data/primary333 +336|334|p|p|s|u|sdw17|sdw17|7334|/data/primary334 +337|335|p|p|s|u|sdw17|sdw17|7335|/data/primary335 +338|336|p|p|s|u|sdw17|sdw17|7336|/data/primary336 +339|337|p|p|s|u|sdw17|sdw17|7337|/data/primary337 +340|338|p|p|s|u|sdw17|sdw17|7338|/data/primary338 +341|339|p|p|s|u|sdw17|sdw17|7339|/data/primary339 +342|340|p|p|s|u|sdw17|sdw17|7340|/data/primary340 +1361|359|m|m|s|u|sdw17|sdw17|8359|/data/mirror359 +1362|360|m|m|s|u|sdw17|sdw17|8360|/data/mirror360 +1363|361|m|m|s|u|sdw17|sdw17|8361|/data/mirror361 +1364|362|m|m|s|u|sdw17|sdw17|8362|/data/mirror362 +1365|363|m|m|s|u|sdw17|sdw17|8363|/data/mirror363 +1366|364|m|m|s|u|sdw17|sdw17|8364|/data/mirror364 +1367|365|m|m|s|u|sdw17|sdw17|8365|/data/mirror365 +1368|366|m|m|s|u|sdw17|sdw17|8366|/data/mirror366 +1369|367|m|m|s|u|sdw17|sdw17|8367|/data/mirror367 +1370|368|m|m|s|u|sdw17|sdw17|8368|/data/mirror368 +1371|369|m|m|s|u|sdw17|sdw17|8369|/data/mirror369 +1372|370|m|m|s|u|sdw17|sdw17|8370|/data/mirror370 +1373|371|m|m|s|u|sdw17|sdw17|8371|/data/mirror371 +1374|372|m|m|s|u|sdw17|sdw17|8372|/data/mirror372 +1375|373|m|m|s|u|sdw17|sdw17|8373|/data/mirror373 +1376|374|m|m|s|u|sdw17|sdw17|8374|/data/mirror374 +1377|375|m|m|s|u|sdw17|sdw17|8375|/data/mirror375 +1378|376|m|m|s|u|sdw17|sdw17|8376|/data/mirror376 +1379|377|m|m|s|u|sdw17|sdw17|8377|/data/mirror377 +# SDW18 +343|341|p|p|s|u|sdw18|sdw18|7341|/data/primary341 +344|342|p|p|s|u|sdw18|sdw18|7342|/data/primary342 +345|343|p|p|s|u|sdw18|sdw18|7343|/data/primary343 +346|344|p|p|s|u|sdw18|sdw18|7344|/data/primary344 +347|345|p|p|s|u|sdw18|sdw18|7345|/data/primary345 +348|346|p|p|s|u|sdw18|sdw18|7346|/data/primary346 +349|347|p|p|s|u|sdw18|sdw18|7347|/data/primary347 +350|348|p|p|s|u|sdw18|sdw18|7348|/data/primary348 +351|349|p|p|s|u|sdw18|sdw18|7349|/data/primary349 +352|350|p|p|s|u|sdw18|sdw18|7350|/data/primary350 +353|351|p|p|s|u|sdw18|sdw18|7351|/data/primary351 +354|352|p|p|s|u|sdw18|sdw18|7352|/data/primary352 +355|353|p|p|s|u|sdw18|sdw18|7353|/data/primary353 +356|354|p|p|s|u|sdw18|sdw18|7354|/data/primary354 +357|355|p|p|s|u|sdw18|sdw18|7355|/data/primary355 +358|356|p|p|s|u|sdw18|sdw18|7356|/data/primary356 +359|357|p|p|s|u|sdw18|sdw18|7357|/data/primary357 +360|358|p|p|s|u|sdw18|sdw18|7358|/data/primary358 +1328|326|m|m|s|u|sdw18|sdw18|8326|/data/mirror326 +1329|327|m|m|s|u|sdw18|sdw18|8327|/data/mirror327 +1330|328|m|m|s|u|sdw18|sdw18|8328|/data/mirror328 +1331|329|m|m|s|u|sdw18|sdw18|8329|/data/mirror329 +1332|330|m|m|s|u|sdw18|sdw18|8330|/data/mirror330 +1333|331|m|m|s|u|sdw18|sdw18|8331|/data/mirror331 +1334|332|m|m|s|u|sdw18|sdw18|8332|/data/mirror332 +1335|333|m|m|s|u|sdw18|sdw18|8333|/data/mirror333 +1336|334|m|m|s|u|sdw18|sdw18|8334|/data/mirror334 +1337|335|m|m|s|u|sdw18|sdw18|8335|/data/mirror335 +1338|336|m|m|s|u|sdw18|sdw18|8336|/data/mirror336 +1339|337|m|m|s|u|sdw18|sdw18|8337|/data/mirror337 +1340|338|m|m|s|u|sdw18|sdw18|8338|/data/mirror338 +1341|339|m|m|s|u|sdw18|sdw18|8339|/data/mirror339 +1342|340|m|m|s|u|sdw18|sdw18|8340|/data/mirror340 +# SDW19 +361|359|p|p|s|u|sdw19|sdw19|7359|/data/primary359 +362|360|p|p|s|u|sdw19|sdw19|7360|/data/primary360 +363|361|p|p|s|u|sdw19|sdw19|7361|/data/primary361 +364|362|p|p|s|u|sdw19|sdw19|7362|/data/primary362 +365|363|p|p|s|u|sdw19|sdw19|7363|/data/primary363 +366|364|p|p|s|u|sdw19|sdw19|7364|/data/primary364 +367|365|p|p|s|u|sdw19|sdw19|7365|/data/primary365 +368|366|p|p|s|u|sdw19|sdw19|7366|/data/primary366 +369|367|p|p|s|u|sdw19|sdw19|7367|/data/primary367 +370|368|p|p|s|u|sdw19|sdw19|7368|/data/primary368 +371|369|p|p|s|u|sdw19|sdw19|7369|/data/primary369 +372|370|p|p|s|u|sdw19|sdw19|7370|/data/primary370 +373|371|p|p|s|u|sdw19|sdw19|7371|/data/primary371 +374|372|p|p|s|u|sdw19|sdw19|7372|/data/primary372 +375|373|p|p|s|u|sdw19|sdw19|7373|/data/primary373 +376|374|p|p|s|u|sdw19|sdw19|7374|/data/primary374 +377|375|p|p|s|u|sdw19|sdw19|7375|/data/primary375 +378|376|p|p|s|u|sdw19|sdw19|7376|/data/primary376 +379|377|p|p|s|u|sdw19|sdw19|7377|/data/primary377 +1304|302|m|m|s|u|sdw19|sdw19|8302|/data/mirror302 +1305|303|m|m|s|u|sdw19|sdw19|8303|/data/mirror303 +1306|304|m|m|s|u|sdw19|sdw19|8304|/data/mirror304 +1307|305|m|m|s|u|sdw19|sdw19|8305|/data/mirror305 +1308|306|m|m|s|u|sdw19|sdw19|8306|/data/mirror306 +1309|307|m|m|s|u|sdw19|sdw19|8307|/data/mirror307 +1310|308|m|m|s|u|sdw19|sdw19|8308|/data/mirror308 +1311|309|m|m|s|u|sdw19|sdw19|8309|/data/mirror309 +1312|310|m|m|s|u|sdw19|sdw19|8310|/data/mirror310 +1313|311|m|m|s|u|sdw19|sdw19|8311|/data/mirror311 +1314|312|m|m|s|u|sdw19|sdw19|8312|/data/mirror312 +1315|313|m|m|s|u|sdw19|sdw19|8313|/data/mirror313 +1316|314|m|m|s|u|sdw19|sdw19|8314|/data/mirror314 +1317|315|m|m|s|u|sdw19|sdw19|8315|/data/mirror315 +1318|316|m|m|s|u|sdw19|sdw19|8316|/data/mirror316 +1319|317|m|m|s|u|sdw19|sdw19|8317|/data/mirror317 +1320|318|m|m|s|u|sdw19|sdw19|8318|/data/mirror318 +1321|319|m|m|s|u|sdw19|sdw19|8319|/data/mirror319 +1322|320|m|m|s|u|sdw19|sdw19|8320|/data/mirror320 +1323|321|m|m|s|u|sdw19|sdw19|8321|/data/mirror321 +1324|322|m|m|s|u|sdw19|sdw19|8322|/data/mirror322 +1325|323|m|m|s|u|sdw19|sdw19|8323|/data/mirror323 +1326|324|m|m|s|u|sdw19|sdw19|8324|/data/mirror324 +1327|325|m|m|s|u|sdw19|sdw19|8325|/data/mirror325 +# SDW2 +21|19|p|p|s|u|sdw2|sdw2|7019|/data/primary19 +22|20|p|p|s|u|sdw2|sdw2|7020|/data/primary20 +23|21|p|p|s|u|sdw2|sdw2|7021|/data/primary21 +24|22|p|p|s|u|sdw2|sdw2|7022|/data/primary22 +25|23|p|p|s|u|sdw2|sdw2|7023|/data/primary23 +26|24|p|p|s|u|sdw2|sdw2|7024|/data/primary24 +27|25|p|p|s|u|sdw2|sdw2|7025|/data/primary25 +28|26|p|p|s|u|sdw2|sdw2|7026|/data/primary26 +29|27|p|p|s|u|sdw2|sdw2|7027|/data/primary27 +30|28|p|p|s|u|sdw2|sdw2|7028|/data/primary28 +31|29|p|p|s|u|sdw2|sdw2|7029|/data/primary29 +32|30|p|p|s|u|sdw2|sdw2|7030|/data/primary30 +33|31|p|p|s|u|sdw2|sdw2|7031|/data/primary31 +34|32|p|p|s|u|sdw2|sdw2|7032|/data/primary32 +35|33|p|p|s|u|sdw2|sdw2|7033|/data/primary33 +36|34|p|p|s|u|sdw2|sdw2|7034|/data/primary34 +37|35|p|p|s|u|sdw2|sdw2|7035|/data/primary35 +38|36|p|p|s|u|sdw2|sdw2|7036|/data/primary36 +39|37|p|p|s|u|sdw2|sdw2|7037|/data/primary37 +40|38|p|p|s|u|sdw2|sdw2|7038|/data/primary38 +41|39|p|p|s|u|sdw2|sdw2|7039|/data/primary39 +1042|40|m|m|s|u|sdw2|sdw2|8040|/data/mirror40 +1043|41|m|m|s|u|sdw2|sdw2|8041|/data/mirror41 +1044|42|m|m|s|u|sdw2|sdw2|8042|/data/mirror42 +1045|43|m|m|s|u|sdw2|sdw2|8043|/data/mirror43 +1046|44|m|m|s|u|sdw2|sdw2|8044|/data/mirror44 +1047|45|m|m|s|u|sdw2|sdw2|8045|/data/mirror45 +1048|46|m|m|s|u|sdw2|sdw2|8046|/data/mirror46 +1049|47|m|m|s|u|sdw2|sdw2|8047|/data/mirror47 +1050|48|m|m|s|u|sdw2|sdw2|8048|/data/mirror48 +1051|49|m|m|s|u|sdw2|sdw2|8049|/data/mirror49 +1052|50|m|m|s|u|sdw2|sdw2|8050|/data/mirror50 +1053|51|m|m|s|u|sdw2|sdw2|8051|/data/mirror51 +1054|52|m|m|s|u|sdw2|sdw2|8052|/data/mirror52 +1055|53|m|m|s|u|sdw2|sdw2|8053|/data/mirror53 +1965|963|m|m|s|u|sdw2|sdw2|8963|/data/mirror963 +1966|964|m|m|s|u|sdw2|sdw2|8964|/data/mirror964 +1967|965|m|m|s|u|sdw2|sdw2|8965|/data/mirror965 +1968|966|m|m|s|u|sdw2|sdw2|8966|/data/mirror966 +1969|967|m|m|s|u|sdw2|sdw2|8967|/data/mirror967 +1970|968|m|m|s|u|sdw2|sdw2|8968|/data/mirror968 +1971|969|m|m|s|u|sdw2|sdw2|8969|/data/mirror969 +1972|970|m|m|s|u|sdw2|sdw2|8970|/data/mirror970 +1973|971|m|m|s|u|sdw2|sdw2|8971|/data/mirror971 +1974|972|m|m|s|u|sdw2|sdw2|8972|/data/mirror972 +1975|973|m|m|s|u|sdw2|sdw2|8973|/data/mirror973 +1976|974|m|m|s|u|sdw2|sdw2|8974|/data/mirror974 +1977|975|m|m|s|u|sdw2|sdw2|8975|/data/mirror975 +1978|976|m|m|s|u|sdw2|sdw2|8976|/data/mirror976 +1979|977|m|m|s|u|sdw2|sdw2|8977|/data/mirror977 +1980|978|m|m|s|u|sdw2|sdw2|8978|/data/mirror978 +1981|979|m|m|s|u|sdw2|sdw2|8979|/data/mirror979 +1982|980|m|m|s|u|sdw2|sdw2|8980|/data/mirror980 +1983|981|m|m|s|u|sdw2|sdw2|8981|/data/mirror981 +1984|982|m|m|s|u|sdw2|sdw2|8982|/data/mirror982 +1985|983|m|m|s|u|sdw2|sdw2|8983|/data/mirror983 +# SDW20 +380|378|p|p|s|u|sdw20|sdw20|7378|/data/primary378 +381|379|p|p|s|u|sdw20|sdw20|7379|/data/primary379 +382|380|p|p|s|u|sdw20|sdw20|7380|/data/primary380 +383|381|p|p|s|u|sdw20|sdw20|7381|/data/primary381 +384|382|p|p|s|u|sdw20|sdw20|7382|/data/primary382 +385|383|p|p|s|u|sdw20|sdw20|7383|/data/primary383 +386|384|p|p|s|u|sdw20|sdw20|7384|/data/primary384 +387|385|p|p|s|u|sdw20|sdw20|7385|/data/primary385 +388|386|p|p|s|u|sdw20|sdw20|7386|/data/primary386 +389|387|p|p|s|u|sdw20|sdw20|7387|/data/primary387 +390|388|p|p|s|u|sdw20|sdw20|7388|/data/primary388 +391|389|p|p|s|u|sdw20|sdw20|7389|/data/primary389 +392|390|p|p|s|u|sdw20|sdw20|7390|/data/primary390 +393|391|p|p|s|u|sdw20|sdw20|7391|/data/primary391 +394|392|p|p|s|u|sdw20|sdw20|7392|/data/primary392 +395|393|p|p|s|u|sdw20|sdw20|7393|/data/primary393 +396|394|p|p|s|u|sdw20|sdw20|7394|/data/primary394 +397|395|p|p|s|u|sdw20|sdw20|7395|/data/primary395 +398|396|p|p|s|u|sdw20|sdw20|7396|/data/primary396 +399|397|p|p|s|u|sdw20|sdw20|7397|/data/primary397 +400|398|p|p|s|u|sdw20|sdw20|7398|/data/primary398 +401|399|p|p|s|u|sdw20|sdw20|7399|/data/primary399 +402|400|p|p|s|u|sdw20|sdw20|7400|/data/primary400 +1403|401|m|m|s|u|sdw20|sdw20|8401|/data/mirror401 +1404|402|m|m|s|u|sdw20|sdw20|8402|/data/mirror402 +1405|403|m|m|s|u|sdw20|sdw20|8403|/data/mirror403 +1406|404|m|m|s|u|sdw20|sdw20|8404|/data/mirror404 +1407|405|m|m|s|u|sdw20|sdw20|8405|/data/mirror405 +1408|406|m|m|s|u|sdw20|sdw20|8406|/data/mirror406 +1409|407|m|m|s|u|sdw20|sdw20|8407|/data/mirror407 +1410|408|m|m|s|u|sdw20|sdw20|8408|/data/mirror408 +1411|409|m|m|s|u|sdw20|sdw20|8409|/data/mirror409 +1412|410|m|m|s|u|sdw20|sdw20|8410|/data/mirror410 +1413|411|m|m|s|u|sdw20|sdw20|8411|/data/mirror411 +1414|412|m|m|s|u|sdw20|sdw20|8412|/data/mirror412 +1415|413|m|m|s|u|sdw20|sdw20|8413|/data/mirror413 +1416|414|m|m|s|u|sdw20|sdw20|8414|/data/mirror414 +1417|415|m|m|s|u|sdw20|sdw20|8415|/data/mirror415 +1418|416|m|m|s|u|sdw20|sdw20|8416|/data/mirror416 +1419|417|m|m|s|u|sdw20|sdw20|8417|/data/mirror417 +1420|418|m|m|s|u|sdw20|sdw20|8418|/data/mirror418 +1421|419|m|m|s|u|sdw20|sdw20|8419|/data/mirror419 +1422|420|m|m|s|u|sdw20|sdw20|8420|/data/mirror420 +1423|421|m|m|s|u|sdw20|sdw20|8421|/data/mirror421 +# SDW21 +403|401|p|p|s|u|sdw21|sdw21|7401|/data/primary401 +404|402|p|p|s|u|sdw21|sdw21|7402|/data/primary402 +405|403|p|p|s|u|sdw21|sdw21|7403|/data/primary403 +406|404|p|p|s|u|sdw21|sdw21|7404|/data/primary404 +407|405|p|p|s|u|sdw21|sdw21|7405|/data/primary405 +408|406|p|p|s|u|sdw21|sdw21|7406|/data/primary406 +409|407|p|p|s|u|sdw21|sdw21|7407|/data/primary407 +410|408|p|p|s|u|sdw21|sdw21|7408|/data/primary408 +411|409|p|p|s|u|sdw21|sdw21|7409|/data/primary409 +412|410|p|p|s|u|sdw21|sdw21|7410|/data/primary410 +413|411|p|p|s|u|sdw21|sdw21|7411|/data/primary411 +414|412|p|p|s|u|sdw21|sdw21|7412|/data/primary412 +415|413|p|p|s|u|sdw21|sdw21|7413|/data/primary413 +416|414|p|p|s|u|sdw21|sdw21|7414|/data/primary414 +417|415|p|p|s|u|sdw21|sdw21|7415|/data/primary415 +418|416|p|p|s|u|sdw21|sdw21|7416|/data/primary416 +419|417|p|p|s|u|sdw21|sdw21|7417|/data/primary417 +420|418|p|p|s|u|sdw21|sdw21|7418|/data/primary418 +421|419|p|p|s|u|sdw21|sdw21|7419|/data/primary419 +422|420|p|p|s|u|sdw21|sdw21|7420|/data/primary420 +423|421|p|p|s|u|sdw21|sdw21|7421|/data/primary421 +1444|442|m|m|s|u|sdw21|sdw21|8442|/data/mirror442 +1445|443|m|m|s|u|sdw21|sdw21|8443|/data/mirror443 +1446|444|m|m|s|u|sdw21|sdw21|8444|/data/mirror444 +1447|445|m|m|s|u|sdw21|sdw21|8445|/data/mirror445 +1448|446|m|m|s|u|sdw21|sdw21|8446|/data/mirror446 +1449|447|m|m|s|u|sdw21|sdw21|8447|/data/mirror447 +1450|448|m|m|s|u|sdw21|sdw21|8448|/data/mirror448 +1451|449|m|m|s|u|sdw21|sdw21|8449|/data/mirror449 +1452|450|m|m|s|u|sdw21|sdw21|8450|/data/mirror450 +1453|451|m|m|s|u|sdw21|sdw21|8451|/data/mirror451 +1454|452|m|m|s|u|sdw21|sdw21|8452|/data/mirror452 +1455|453|m|m|s|u|sdw21|sdw21|8453|/data/mirror453 +1456|454|m|m|s|u|sdw21|sdw21|8454|/data/mirror454 +1457|455|m|m|s|u|sdw21|sdw21|8455|/data/mirror455 +1458|456|m|m|s|u|sdw21|sdw21|8456|/data/mirror456 +1459|457|m|m|s|u|sdw21|sdw21|8457|/data/mirror457 +1460|458|m|m|s|u|sdw21|sdw21|8458|/data/mirror458 +1461|459|m|m|s|u|sdw21|sdw21|8459|/data/mirror459 +1462|460|m|m|s|u|sdw21|sdw21|8460|/data/mirror460 +# SDW22 +424|422|p|p|s|u|sdw22|sdw22|7422|/data/primary422 +425|423|p|p|s|u|sdw22|sdw22|7423|/data/primary423 +426|424|p|p|s|u|sdw22|sdw22|7424|/data/primary424 +427|425|p|p|s|u|sdw22|sdw22|7425|/data/primary425 +428|426|p|p|s|u|sdw22|sdw22|7426|/data/primary426 +429|427|p|p|s|u|sdw22|sdw22|7427|/data/primary427 +430|428|p|p|s|u|sdw22|sdw22|7428|/data/primary428 +431|429|p|p|s|u|sdw22|sdw22|7429|/data/primary429 +432|430|p|p|s|u|sdw22|sdw22|7430|/data/primary430 +433|431|p|p|s|u|sdw22|sdw22|7431|/data/primary431 +434|432|p|p|s|u|sdw22|sdw22|7432|/data/primary432 +435|433|p|p|s|u|sdw22|sdw22|7433|/data/primary433 +436|434|p|p|s|u|sdw22|sdw22|7434|/data/primary434 +437|435|p|p|s|u|sdw22|sdw22|7435|/data/primary435 +438|436|p|p|s|u|sdw22|sdw22|7436|/data/primary436 +439|437|p|p|s|u|sdw22|sdw22|7437|/data/primary437 +440|438|p|p|s|u|sdw22|sdw22|7438|/data/primary438 +441|439|p|p|s|u|sdw22|sdw22|7439|/data/primary439 +442|440|p|p|s|u|sdw22|sdw22|7440|/data/primary440 +443|441|p|p|s|u|sdw22|sdw22|7441|/data/primary441 +1380|378|m|m|s|u|sdw22|sdw22|8378|/data/mirror378 +1381|379|m|m|s|u|sdw22|sdw22|8379|/data/mirror379 +1382|380|m|m|s|u|sdw22|sdw22|8380|/data/mirror380 +1383|381|m|m|s|u|sdw22|sdw22|8381|/data/mirror381 +1384|382|m|m|s|u|sdw22|sdw22|8382|/data/mirror382 +1385|383|m|m|s|u|sdw22|sdw22|8383|/data/mirror383 +1386|384|m|m|s|u|sdw22|sdw22|8384|/data/mirror384 +1387|385|m|m|s|u|sdw22|sdw22|8385|/data/mirror385 +1388|386|m|m|s|u|sdw22|sdw22|8386|/data/mirror386 +1389|387|m|m|s|u|sdw22|sdw22|8387|/data/mirror387 +1390|388|m|m|s|u|sdw22|sdw22|8388|/data/mirror388 +1391|389|m|m|s|u|sdw22|sdw22|8389|/data/mirror389 +1392|390|m|m|s|u|sdw22|sdw22|8390|/data/mirror390 +1393|391|m|m|s|u|sdw22|sdw22|8391|/data/mirror391 +1394|392|m|m|s|u|sdw22|sdw22|8392|/data/mirror392 +1395|393|m|m|s|u|sdw22|sdw22|8393|/data/mirror393 +1396|394|m|m|s|u|sdw22|sdw22|8394|/data/mirror394 +1397|395|m|m|s|u|sdw22|sdw22|8395|/data/mirror395 +1398|396|m|m|s|u|sdw22|sdw22|8396|/data/mirror396 +1399|397|m|m|s|u|sdw22|sdw22|8397|/data/mirror397 +1400|398|m|m|s|u|sdw22|sdw22|8398|/data/mirror398 +1401|399|m|m|s|u|sdw22|sdw22|8399|/data/mirror399 +1402|400|m|m|s|u|sdw22|sdw22|8400|/data/mirror400 +# SDW23 +444|442|p|p|s|u|sdw23|sdw23|7442|/data/primary442 +445|443|p|p|s|u|sdw23|sdw23|7443|/data/primary443 +446|444|p|p|s|u|sdw23|sdw23|7444|/data/primary444 +447|445|p|p|s|u|sdw23|sdw23|7445|/data/primary445 +448|446|p|p|s|u|sdw23|sdw23|7446|/data/primary446 +449|447|p|p|s|u|sdw23|sdw23|7447|/data/primary447 +450|448|p|p|s|u|sdw23|sdw23|7448|/data/primary448 +451|449|p|p|s|u|sdw23|sdw23|7449|/data/primary449 +452|450|p|p|s|u|sdw23|sdw23|7450|/data/primary450 +453|451|p|p|s|u|sdw23|sdw23|7451|/data/primary451 +454|452|p|p|s|u|sdw23|sdw23|7452|/data/primary452 +455|453|p|p|s|u|sdw23|sdw23|7453|/data/primary453 +456|454|p|p|s|u|sdw23|sdw23|7454|/data/primary454 +457|455|p|p|s|u|sdw23|sdw23|7455|/data/primary455 +458|456|p|p|s|u|sdw23|sdw23|7456|/data/primary456 +459|457|p|p|s|u|sdw23|sdw23|7457|/data/primary457 +460|458|p|p|s|u|sdw23|sdw23|7458|/data/primary458 +461|459|p|p|s|u|sdw23|sdw23|7459|/data/primary459 +462|460|p|p|s|u|sdw23|sdw23|7460|/data/primary460 +1424|422|m|m|s|u|sdw23|sdw23|8422|/data/mirror422 +1425|423|m|m|s|u|sdw23|sdw23|8423|/data/mirror423 +1426|424|m|m|s|u|sdw23|sdw23|8424|/data/mirror424 +1427|425|m|m|s|u|sdw23|sdw23|8425|/data/mirror425 +1428|426|m|m|s|u|sdw23|sdw23|8426|/data/mirror426 +1429|427|m|m|s|u|sdw23|sdw23|8427|/data/mirror427 +1430|428|m|m|s|u|sdw23|sdw23|8428|/data/mirror428 +1431|429|m|m|s|u|sdw23|sdw23|8429|/data/mirror429 +1432|430|m|m|s|u|sdw23|sdw23|8430|/data/mirror430 +1433|431|m|m|s|u|sdw23|sdw23|8431|/data/mirror431 +1434|432|m|m|s|u|sdw23|sdw23|8432|/data/mirror432 +1435|433|m|m|s|u|sdw23|sdw23|8433|/data/mirror433 +1436|434|m|m|s|u|sdw23|sdw23|8434|/data/mirror434 +1437|435|m|m|s|u|sdw23|sdw23|8435|/data/mirror435 +1438|436|m|m|s|u|sdw23|sdw23|8436|/data/mirror436 +1439|437|m|m|s|u|sdw23|sdw23|8437|/data/mirror437 +1440|438|m|m|s|u|sdw23|sdw23|8438|/data/mirror438 +1441|439|m|m|s|u|sdw23|sdw23|8439|/data/mirror439 +1442|440|m|m|s|u|sdw23|sdw23|8440|/data/mirror440 +1443|441|m|m|s|u|sdw23|sdw23|8441|/data/mirror441 +# SDW24 +463|461|p|p|s|u|sdw24|sdw24|7461|/data/primary461 +464|462|p|p|s|u|sdw24|sdw24|7462|/data/primary462 +465|463|p|p|s|u|sdw24|sdw24|7463|/data/primary463 +466|464|p|p|s|u|sdw24|sdw24|7464|/data/primary464 +467|465|p|p|s|u|sdw24|sdw24|7465|/data/primary465 +468|466|p|p|s|u|sdw24|sdw24|7466|/data/primary466 +469|467|p|p|s|u|sdw24|sdw24|7467|/data/primary467 +470|468|p|p|s|u|sdw24|sdw24|7468|/data/primary468 +471|469|p|p|s|u|sdw24|sdw24|7469|/data/primary469 +472|470|p|p|s|u|sdw24|sdw24|7470|/data/primary470 +473|471|p|p|s|u|sdw24|sdw24|7471|/data/primary471 +474|472|p|p|s|u|sdw24|sdw24|7472|/data/primary472 +475|473|p|p|s|u|sdw24|sdw24|7473|/data/primary473 +476|474|p|p|s|u|sdw24|sdw24|7474|/data/primary474 +477|475|p|p|s|u|sdw24|sdw24|7475|/data/primary475 +478|476|p|p|s|u|sdw24|sdw24|7476|/data/primary476 +479|477|p|p|s|u|sdw24|sdw24|7477|/data/primary477 +480|478|p|p|s|u|sdw24|sdw24|7478|/data/primary478 +481|479|p|p|s|u|sdw24|sdw24|7479|/data/primary479 +482|480|p|p|s|u|sdw24|sdw24|7480|/data/primary480 +483|481|p|p|s|u|sdw24|sdw24|7481|/data/primary481 +484|482|p|p|s|u|sdw24|sdw24|7482|/data/primary482 +485|483|p|p|s|u|sdw24|sdw24|7483|/data/primary483 +1486|484|m|m|s|u|sdw24|sdw24|8484|/data/mirror484 +1487|485|m|m|s|u|sdw24|sdw24|8485|/data/mirror485 +1488|486|m|m|s|u|sdw24|sdw24|8486|/data/mirror486 +1489|487|m|m|s|u|sdw24|sdw24|8487|/data/mirror487 +1490|488|m|m|s|u|sdw24|sdw24|8488|/data/mirror488 +1491|489|m|m|s|u|sdw24|sdw24|8489|/data/mirror489 +1492|490|m|m|s|u|sdw24|sdw24|8490|/data/mirror490 +1493|491|m|m|s|u|sdw24|sdw24|8491|/data/mirror491 +1494|492|m|m|s|u|sdw24|sdw24|8492|/data/mirror492 +1495|493|m|m|s|u|sdw24|sdw24|8493|/data/mirror493 +1496|494|m|m|s|u|sdw24|sdw24|8494|/data/mirror494 +1497|495|m|m|s|u|sdw24|sdw24|8495|/data/mirror495 +1498|496|m|m|s|u|sdw24|sdw24|8496|/data/mirror496 +1499|497|m|m|s|u|sdw24|sdw24|8497|/data/mirror497 +1500|498|m|m|s|u|sdw24|sdw24|8498|/data/mirror498 +1501|499|m|m|s|u|sdw24|sdw24|8499|/data/mirror499 +1502|500|m|m|s|u|sdw24|sdw24|8500|/data/mirror500 +1503|501|m|m|s|u|sdw24|sdw24|8501|/data/mirror501 +1504|502|m|m|s|u|sdw24|sdw24|8502|/data/mirror502 +1505|503|m|m|s|u|sdw24|sdw24|8503|/data/mirror503 +1506|504|m|m|s|u|sdw24|sdw24|8504|/data/mirror504 +1507|505|m|m|s|u|sdw24|sdw24|8505|/data/mirror505 +1508|506|m|m|s|u|sdw24|sdw24|8506|/data/mirror506 +1509|507|m|m|s|u|sdw24|sdw24|8507|/data/mirror507 +1510|508|m|m|s|u|sdw24|sdw24|8508|/data/mirror508 +# SDW25 +486|484|p|p|s|u|sdw25|sdw25|7484|/data/primary484 +487|485|p|p|s|u|sdw25|sdw25|7485|/data/primary485 +488|486|p|p|s|u|sdw25|sdw25|7486|/data/primary486 +489|487|p|p|s|u|sdw25|sdw25|7487|/data/primary487 +490|488|p|p|s|u|sdw25|sdw25|7488|/data/primary488 +491|489|p|p|s|u|sdw25|sdw25|7489|/data/primary489 +492|490|p|p|s|u|sdw25|sdw25|7490|/data/primary490 +493|491|p|p|s|u|sdw25|sdw25|7491|/data/primary491 +494|492|p|p|s|u|sdw25|sdw25|7492|/data/primary492 +495|493|p|p|s|u|sdw25|sdw25|7493|/data/primary493 +496|494|p|p|s|u|sdw25|sdw25|7494|/data/primary494 +497|495|p|p|s|u|sdw25|sdw25|7495|/data/primary495 +498|496|p|p|s|u|sdw25|sdw25|7496|/data/primary496 +499|497|p|p|s|u|sdw25|sdw25|7497|/data/primary497 +500|498|p|p|s|u|sdw25|sdw25|7498|/data/primary498 +501|499|p|p|s|u|sdw25|sdw25|7499|/data/primary499 +502|500|p|p|s|u|sdw25|sdw25|7500|/data/primary500 +503|501|p|p|s|u|sdw25|sdw25|7501|/data/primary501 +504|502|p|p|s|u|sdw25|sdw25|7502|/data/primary502 +505|503|p|p|s|u|sdw25|sdw25|7503|/data/primary503 +506|504|p|p|s|u|sdw25|sdw25|7504|/data/primary504 +507|505|p|p|s|u|sdw25|sdw25|7505|/data/primary505 +508|506|p|p|s|u|sdw25|sdw25|7506|/data/primary506 +509|507|p|p|s|u|sdw25|sdw25|7507|/data/primary507 +510|508|p|p|s|u|sdw25|sdw25|7508|/data/primary508 +1573|571|m|m|s|u|sdw25|sdw25|8571|/data/mirror571 +1574|572|m|m|s|u|sdw25|sdw25|8572|/data/mirror572 +1575|573|m|m|s|u|sdw25|sdw25|8573|/data/mirror573 +1576|574|m|m|s|u|sdw25|sdw25|8574|/data/mirror574 +1577|575|m|m|s|u|sdw25|sdw25|8575|/data/mirror575 +1578|576|m|m|s|u|sdw25|sdw25|8576|/data/mirror576 +1579|577|m|m|s|u|sdw25|sdw25|8577|/data/mirror577 +1580|578|m|m|s|u|sdw25|sdw25|8578|/data/mirror578 +1581|579|m|m|s|u|sdw25|sdw25|8579|/data/mirror579 +1582|580|m|m|s|u|sdw25|sdw25|8580|/data/mirror580 +1583|581|m|m|s|u|sdw25|sdw25|8581|/data/mirror581 +1584|582|m|m|s|u|sdw25|sdw25|8582|/data/mirror582 +1585|583|m|m|s|u|sdw25|sdw25|8583|/data/mirror583 +1586|584|m|m|s|u|sdw25|sdw25|8584|/data/mirror584 +1587|585|m|m|s|u|sdw25|sdw25|8585|/data/mirror585 +1588|586|m|m|s|u|sdw25|sdw25|8586|/data/mirror586 +1589|587|m|m|s|u|sdw25|sdw25|8587|/data/mirror587 +1590|588|m|m|s|u|sdw25|sdw25|8588|/data/mirror588 +1591|589|m|m|s|u|sdw25|sdw25|8589|/data/mirror589 +1592|590|m|m|s|u|sdw25|sdw25|8590|/data/mirror590 +1593|591|m|m|s|u|sdw25|sdw25|8591|/data/mirror591 +# SDW26 +511|509|p|p|s|u|sdw26|sdw26|7509|/data/primary509 +512|510|p|p|s|u|sdw26|sdw26|7510|/data/primary510 +513|511|p|p|s|u|sdw26|sdw26|7511|/data/primary511 +514|512|p|p|s|u|sdw26|sdw26|7512|/data/primary512 +515|513|p|p|s|u|sdw26|sdw26|7513|/data/primary513 +516|514|p|p|s|u|sdw26|sdw26|7514|/data/primary514 +517|515|p|p|s|u|sdw26|sdw26|7515|/data/primary515 +518|516|p|p|s|u|sdw26|sdw26|7516|/data/primary516 +519|517|p|p|s|u|sdw26|sdw26|7517|/data/primary517 +520|518|p|p|s|u|sdw26|sdw26|7518|/data/primary518 +521|519|p|p|s|u|sdw26|sdw26|7519|/data/primary519 +522|520|p|p|s|u|sdw26|sdw26|7520|/data/primary520 +523|521|p|p|s|u|sdw26|sdw26|7521|/data/primary521 +524|522|p|p|s|u|sdw26|sdw26|7522|/data/primary522 +525|523|p|p|s|u|sdw26|sdw26|7523|/data/primary523 +526|524|p|p|s|u|sdw26|sdw26|7524|/data/primary524 +527|525|p|p|s|u|sdw26|sdw26|7525|/data/primary525 +528|526|p|p|s|u|sdw26|sdw26|7526|/data/primary526 +529|527|p|p|s|u|sdw26|sdw26|7527|/data/primary527 +530|528|p|p|s|u|sdw26|sdw26|7528|/data/primary528 +1531|529|m|m|s|u|sdw26|sdw26|8529|/data/mirror529 +1532|530|m|m|s|u|sdw26|sdw26|8530|/data/mirror530 +1533|531|m|m|s|u|sdw26|sdw26|8531|/data/mirror531 +1534|532|m|m|s|u|sdw26|sdw26|8532|/data/mirror532 +1535|533|m|m|s|u|sdw26|sdw26|8533|/data/mirror533 +1536|534|m|m|s|u|sdw26|sdw26|8534|/data/mirror534 +1537|535|m|m|s|u|sdw26|sdw26|8535|/data/mirror535 +1538|536|m|m|s|u|sdw26|sdw26|8536|/data/mirror536 +1539|537|m|m|s|u|sdw26|sdw26|8537|/data/mirror537 +1540|538|m|m|s|u|sdw26|sdw26|8538|/data/mirror538 +1541|539|m|m|s|u|sdw26|sdw26|8539|/data/mirror539 +1542|540|m|m|s|u|sdw26|sdw26|8540|/data/mirror540 +1543|541|m|m|s|u|sdw26|sdw26|8541|/data/mirror541 +1544|542|m|m|s|u|sdw26|sdw26|8542|/data/mirror542 +1545|543|m|m|s|u|sdw26|sdw26|8543|/data/mirror543 +1546|544|m|m|s|u|sdw26|sdw26|8544|/data/mirror544 +1547|545|m|m|s|u|sdw26|sdw26|8545|/data/mirror545 +1548|546|m|m|s|u|sdw26|sdw26|8546|/data/mirror546 +# SDW27 +531|529|p|p|s|u|sdw27|sdw27|7529|/data/primary529 +532|530|p|p|s|u|sdw27|sdw27|7530|/data/primary530 +533|531|p|p|s|u|sdw27|sdw27|7531|/data/primary531 +534|532|p|p|s|u|sdw27|sdw27|7532|/data/primary532 +535|533|p|p|s|u|sdw27|sdw27|7533|/data/primary533 +536|534|p|p|s|u|sdw27|sdw27|7534|/data/primary534 +537|535|p|p|s|u|sdw27|sdw27|7535|/data/primary535 +538|536|p|p|s|u|sdw27|sdw27|7536|/data/primary536 +539|537|p|p|s|u|sdw27|sdw27|7537|/data/primary537 +540|538|p|p|s|u|sdw27|sdw27|7538|/data/primary538 +541|539|p|p|s|u|sdw27|sdw27|7539|/data/primary539 +542|540|p|p|s|u|sdw27|sdw27|7540|/data/primary540 +543|541|p|p|s|u|sdw27|sdw27|7541|/data/primary541 +544|542|p|p|s|u|sdw27|sdw27|7542|/data/primary542 +545|543|p|p|s|u|sdw27|sdw27|7543|/data/primary543 +546|544|p|p|s|u|sdw27|sdw27|7544|/data/primary544 +547|545|p|p|s|u|sdw27|sdw27|7545|/data/primary545 +548|546|p|p|s|u|sdw27|sdw27|7546|/data/primary546 +1463|461|m|m|s|u|sdw27|sdw27|8461|/data/mirror461 +1464|462|m|m|s|u|sdw27|sdw27|8462|/data/mirror462 +1465|463|m|m|s|u|sdw27|sdw27|8463|/data/mirror463 +1466|464|m|m|s|u|sdw27|sdw27|8464|/data/mirror464 +1467|465|m|m|s|u|sdw27|sdw27|8465|/data/mirror465 +1468|466|m|m|s|u|sdw27|sdw27|8466|/data/mirror466 +1469|467|m|m|s|u|sdw27|sdw27|8467|/data/mirror467 +1470|468|m|m|s|u|sdw27|sdw27|8468|/data/mirror468 +1471|469|m|m|s|u|sdw27|sdw27|8469|/data/mirror469 +1472|470|m|m|s|u|sdw27|sdw27|8470|/data/mirror470 +1473|471|m|m|s|u|sdw27|sdw27|8471|/data/mirror471 +1474|472|m|m|s|u|sdw27|sdw27|8472|/data/mirror472 +1475|473|m|m|s|u|sdw27|sdw27|8473|/data/mirror473 +1476|474|m|m|s|u|sdw27|sdw27|8474|/data/mirror474 +1477|475|m|m|s|u|sdw27|sdw27|8475|/data/mirror475 +1478|476|m|m|s|u|sdw27|sdw27|8476|/data/mirror476 +1479|477|m|m|s|u|sdw27|sdw27|8477|/data/mirror477 +1480|478|m|m|s|u|sdw27|sdw27|8478|/data/mirror478 +1481|479|m|m|s|u|sdw27|sdw27|8479|/data/mirror479 +1482|480|m|m|s|u|sdw27|sdw27|8480|/data/mirror480 +1483|481|m|m|s|u|sdw27|sdw27|8481|/data/mirror481 +1484|482|m|m|s|u|sdw27|sdw27|8482|/data/mirror482 +1485|483|m|m|s|u|sdw27|sdw27|8483|/data/mirror483 +# SDW28 +549|547|p|p|s|u|sdw28|sdw28|7547|/data/primary547 +550|548|p|p|s|u|sdw28|sdw28|7548|/data/primary548 +551|549|p|p|s|u|sdw28|sdw28|7549|/data/primary549 +552|550|p|p|s|u|sdw28|sdw28|7550|/data/primary550 +553|551|p|p|s|u|sdw28|sdw28|7551|/data/primary551 +554|552|p|p|s|u|sdw28|sdw28|7552|/data/primary552 +555|553|p|p|s|u|sdw28|sdw28|7553|/data/primary553 +556|554|p|p|s|u|sdw28|sdw28|7554|/data/primary554 +557|555|p|p|s|u|sdw28|sdw28|7555|/data/primary555 +558|556|p|p|s|u|sdw28|sdw28|7556|/data/primary556 +559|557|p|p|s|u|sdw28|sdw28|7557|/data/primary557 +560|558|p|p|s|u|sdw28|sdw28|7558|/data/primary558 +561|559|p|p|s|u|sdw28|sdw28|7559|/data/primary559 +562|560|p|p|s|u|sdw28|sdw28|7560|/data/primary560 +563|561|p|p|s|u|sdw28|sdw28|7561|/data/primary561 +564|562|p|p|s|u|sdw28|sdw28|7562|/data/primary562 +565|563|p|p|s|u|sdw28|sdw28|7563|/data/primary563 +566|564|p|p|s|u|sdw28|sdw28|7564|/data/primary564 +567|565|p|p|s|u|sdw28|sdw28|7565|/data/primary565 +568|566|p|p|s|u|sdw28|sdw28|7566|/data/primary566 +569|567|p|p|s|u|sdw28|sdw28|7567|/data/primary567 +570|568|p|p|s|u|sdw28|sdw28|7568|/data/primary568 +571|569|p|p|s|u|sdw28|sdw28|7569|/data/primary569 +572|570|p|p|s|u|sdw28|sdw28|7570|/data/primary570 +1511|509|m|m|s|u|sdw28|sdw28|8509|/data/mirror509 +1512|510|m|m|s|u|sdw28|sdw28|8510|/data/mirror510 +1513|511|m|m|s|u|sdw28|sdw28|8511|/data/mirror511 +1514|512|m|m|s|u|sdw28|sdw28|8512|/data/mirror512 +1515|513|m|m|s|u|sdw28|sdw28|8513|/data/mirror513 +1516|514|m|m|s|u|sdw28|sdw28|8514|/data/mirror514 +1517|515|m|m|s|u|sdw28|sdw28|8515|/data/mirror515 +1518|516|m|m|s|u|sdw28|sdw28|8516|/data/mirror516 +1519|517|m|m|s|u|sdw28|sdw28|8517|/data/mirror517 +1520|518|m|m|s|u|sdw28|sdw28|8518|/data/mirror518 +1521|519|m|m|s|u|sdw28|sdw28|8519|/data/mirror519 +1522|520|m|m|s|u|sdw28|sdw28|8520|/data/mirror520 +1523|521|m|m|s|u|sdw28|sdw28|8521|/data/mirror521 +1524|522|m|m|s|u|sdw28|sdw28|8522|/data/mirror522 +1525|523|m|m|s|u|sdw28|sdw28|8523|/data/mirror523 +1526|524|m|m|s|u|sdw28|sdw28|8524|/data/mirror524 +1527|525|m|m|s|u|sdw28|sdw28|8525|/data/mirror525 +1528|526|m|m|s|u|sdw28|sdw28|8526|/data/mirror526 +1529|527|m|m|s|u|sdw28|sdw28|8527|/data/mirror527 +1530|528|m|m|s|u|sdw28|sdw28|8528|/data/mirror528 +# SDW29 +573|571|p|p|s|u|sdw29|sdw29|7571|/data/primary571 +574|572|p|p|s|u|sdw29|sdw29|7572|/data/primary572 +575|573|p|p|s|u|sdw29|sdw29|7573|/data/primary573 +576|574|p|p|s|u|sdw29|sdw29|7574|/data/primary574 +577|575|p|p|s|u|sdw29|sdw29|7575|/data/primary575 +578|576|p|p|s|u|sdw29|sdw29|7576|/data/primary576 +579|577|p|p|s|u|sdw29|sdw29|7577|/data/primary577 +580|578|p|p|s|u|sdw29|sdw29|7578|/data/primary578 +581|579|p|p|s|u|sdw29|sdw29|7579|/data/primary579 +582|580|p|p|s|u|sdw29|sdw29|7580|/data/primary580 +583|581|p|p|s|u|sdw29|sdw29|7581|/data/primary581 +584|582|p|p|s|u|sdw29|sdw29|7582|/data/primary582 +585|583|p|p|s|u|sdw29|sdw29|7583|/data/primary583 +586|584|p|p|s|u|sdw29|sdw29|7584|/data/primary584 +587|585|p|p|s|u|sdw29|sdw29|7585|/data/primary585 +588|586|p|p|s|u|sdw29|sdw29|7586|/data/primary586 +589|587|p|p|s|u|sdw29|sdw29|7587|/data/primary587 +590|588|p|p|s|u|sdw29|sdw29|7588|/data/primary588 +591|589|p|p|s|u|sdw29|sdw29|7589|/data/primary589 +592|590|p|p|s|u|sdw29|sdw29|7590|/data/primary590 +593|591|p|p|s|u|sdw29|sdw29|7591|/data/primary591 +1677|675|m|m|s|u|sdw29|sdw29|8675|/data/mirror675 +1678|676|m|m|s|u|sdw29|sdw29|8676|/data/mirror676 +1679|677|m|m|s|u|sdw29|sdw29|8677|/data/mirror677 +1680|678|m|m|s|u|sdw29|sdw29|8678|/data/mirror678 +1681|679|m|m|s|u|sdw29|sdw29|8679|/data/mirror679 +1682|680|m|m|s|u|sdw29|sdw29|8680|/data/mirror680 +1683|681|m|m|s|u|sdw29|sdw29|8681|/data/mirror681 +1684|682|m|m|s|u|sdw29|sdw29|8682|/data/mirror682 +1685|683|m|m|s|u|sdw29|sdw29|8683|/data/mirror683 +1686|684|m|m|s|u|sdw29|sdw29|8684|/data/mirror684 +1687|685|m|m|s|u|sdw29|sdw29|8685|/data/mirror685 +1688|686|m|m|s|u|sdw29|sdw29|8686|/data/mirror686 +1689|687|m|m|s|u|sdw29|sdw29|8687|/data/mirror687 +1690|688|m|m|s|u|sdw29|sdw29|8688|/data/mirror688 +1691|689|m|m|s|u|sdw29|sdw29|8689|/data/mirror689 +1692|690|m|m|s|u|sdw29|sdw29|8690|/data/mirror690 +1693|691|m|m|s|u|sdw29|sdw29|8691|/data/mirror691 +1694|692|m|m|s|u|sdw29|sdw29|8692|/data/mirror692 +1695|693|m|m|s|u|sdw29|sdw29|8693|/data/mirror693 +1696|694|m|m|s|u|sdw29|sdw29|8694|/data/mirror694 +1697|695|m|m|s|u|sdw29|sdw29|8695|/data/mirror695 +1698|696|m|m|s|u|sdw29|sdw29|8696|/data/mirror696 +1699|697|m|m|s|u|sdw29|sdw29|8697|/data/mirror697 +1700|698|m|m|s|u|sdw29|sdw29|8698|/data/mirror698 +1701|699|m|m|s|u|sdw29|sdw29|8699|/data/mirror699 +1702|700|m|m|s|u|sdw29|sdw29|8700|/data/mirror700 +# SDW3 +42|40|p|p|s|u|sdw3|sdw3|7040|/data/primary40 +43|41|p|p|s|u|sdw3|sdw3|7041|/data/primary41 +44|42|p|p|s|u|sdw3|sdw3|7042|/data/primary42 +45|43|p|p|s|u|sdw3|sdw3|7043|/data/primary43 +46|44|p|p|s|u|sdw3|sdw3|7044|/data/primary44 +47|45|p|p|s|u|sdw3|sdw3|7045|/data/primary45 +48|46|p|p|s|u|sdw3|sdw3|7046|/data/primary46 +49|47|p|p|s|u|sdw3|sdw3|7047|/data/primary47 +50|48|p|p|s|u|sdw3|sdw3|7048|/data/primary48 +51|49|p|p|s|u|sdw3|sdw3|7049|/data/primary49 +52|50|p|p|s|u|sdw3|sdw3|7050|/data/primary50 +53|51|p|p|s|u|sdw3|sdw3|7051|/data/primary51 +54|52|p|p|s|u|sdw3|sdw3|7052|/data/primary52 +55|53|p|p|s|u|sdw3|sdw3|7053|/data/primary53 +1002|0|m|m|s|u|sdw3|sdw3|8000|/data/mirror0 +1003|1|m|m|s|u|sdw3|sdw3|8001|/data/mirror1 +1004|2|m|m|s|u|sdw3|sdw3|8002|/data/mirror2 +1005|3|m|m|s|u|sdw3|sdw3|8003|/data/mirror3 +1006|4|m|m|s|u|sdw3|sdw3|8004|/data/mirror4 +1007|5|m|m|s|u|sdw3|sdw3|8005|/data/mirror5 +1008|6|m|m|s|u|sdw3|sdw3|8006|/data/mirror6 +1009|7|m|m|s|u|sdw3|sdw3|8007|/data/mirror7 +1010|8|m|m|s|u|sdw3|sdw3|8008|/data/mirror8 +1011|9|m|m|s|u|sdw3|sdw3|8009|/data/mirror9 +1012|10|m|m|s|u|sdw3|sdw3|8010|/data/mirror10 +1013|11|m|m|s|u|sdw3|sdw3|8011|/data/mirror11 +1014|12|m|m|s|u|sdw3|sdw3|8012|/data/mirror12 +1015|13|m|m|s|u|sdw3|sdw3|8013|/data/mirror13 +1016|14|m|m|s|u|sdw3|sdw3|8014|/data/mirror14 +1017|15|m|m|s|u|sdw3|sdw3|8015|/data/mirror15 +1018|16|m|m|s|u|sdw3|sdw3|8016|/data/mirror16 +1019|17|m|m|s|u|sdw3|sdw3|8017|/data/mirror17 +1020|18|m|m|s|u|sdw3|sdw3|8018|/data/mirror18 +# SDW30 +594|592|p|p|s|u|sdw30|sdw30|7592|/data/primary592 +595|593|p|p|s|u|sdw30|sdw30|7593|/data/primary593 +596|594|p|p|s|u|sdw30|sdw30|7594|/data/primary594 +597|595|p|p|s|u|sdw30|sdw30|7595|/data/primary595 +598|596|p|p|s|u|sdw30|sdw30|7596|/data/primary596 +599|597|p|p|s|u|sdw30|sdw30|7597|/data/primary597 +600|598|p|p|s|u|sdw30|sdw30|7598|/data/primary598 +601|599|p|p|s|u|sdw30|sdw30|7599|/data/primary599 +602|600|p|p|s|u|sdw30|sdw30|7600|/data/primary600 +603|601|p|p|s|u|sdw30|sdw30|7601|/data/primary601 +604|602|p|p|s|u|sdw30|sdw30|7602|/data/primary602 +605|603|p|p|s|u|sdw30|sdw30|7603|/data/primary603 +606|604|p|p|s|u|sdw30|sdw30|7604|/data/primary604 +607|605|p|p|s|u|sdw30|sdw30|7605|/data/primary605 +608|606|p|p|s|u|sdw30|sdw30|7606|/data/primary606 +609|607|p|p|s|u|sdw30|sdw30|7607|/data/primary607 +610|608|p|p|s|u|sdw30|sdw30|7608|/data/primary608 +611|609|p|p|s|u|sdw30|sdw30|7609|/data/primary609 +612|610|p|p|s|u|sdw30|sdw30|7610|/data/primary610 +1549|547|m|m|s|u|sdw30|sdw30|8547|/data/mirror547 +1550|548|m|m|s|u|sdw30|sdw30|8548|/data/mirror548 +1551|549|m|m|s|u|sdw30|sdw30|8549|/data/mirror549 +1552|550|m|m|s|u|sdw30|sdw30|8550|/data/mirror550 +1553|551|m|m|s|u|sdw30|sdw30|8551|/data/mirror551 +1554|552|m|m|s|u|sdw30|sdw30|8552|/data/mirror552 +1555|553|m|m|s|u|sdw30|sdw30|8553|/data/mirror553 +1556|554|m|m|s|u|sdw30|sdw30|8554|/data/mirror554 +1557|555|m|m|s|u|sdw30|sdw30|8555|/data/mirror555 +1558|556|m|m|s|u|sdw30|sdw30|8556|/data/mirror556 +1559|557|m|m|s|u|sdw30|sdw30|8557|/data/mirror557 +1560|558|m|m|s|u|sdw30|sdw30|8558|/data/mirror558 +1561|559|m|m|s|u|sdw30|sdw30|8559|/data/mirror559 +1562|560|m|m|s|u|sdw30|sdw30|8560|/data/mirror560 +1563|561|m|m|s|u|sdw30|sdw30|8561|/data/mirror561 +1564|562|m|m|s|u|sdw30|sdw30|8562|/data/mirror562 +1565|563|m|m|s|u|sdw30|sdw30|8563|/data/mirror563 +1566|564|m|m|s|u|sdw30|sdw30|8564|/data/mirror564 +1567|565|m|m|s|u|sdw30|sdw30|8565|/data/mirror565 +1568|566|m|m|s|u|sdw30|sdw30|8566|/data/mirror566 +1569|567|m|m|s|u|sdw30|sdw30|8567|/data/mirror567 +1570|568|m|m|s|u|sdw30|sdw30|8568|/data/mirror568 +1571|569|m|m|s|u|sdw30|sdw30|8569|/data/mirror569 +1572|570|m|m|s|u|sdw30|sdw30|8570|/data/mirror570 +# SDW31 +613|611|p|p|s|u|sdw31|sdw31|7611|/data/primary611 +614|612|p|p|s|u|sdw31|sdw31|7612|/data/primary612 +615|613|p|p|s|u|sdw31|sdw31|7613|/data/primary613 +616|614|p|p|s|u|sdw31|sdw31|7614|/data/primary614 +617|615|p|p|s|u|sdw31|sdw31|7615|/data/primary615 +618|616|p|p|s|u|sdw31|sdw31|7616|/data/primary616 +619|617|p|p|s|u|sdw31|sdw31|7617|/data/primary617 +620|618|p|p|s|u|sdw31|sdw31|7618|/data/primary618 +621|619|p|p|s|u|sdw31|sdw31|7619|/data/primary619 +622|620|p|p|s|u|sdw31|sdw31|7620|/data/primary620 +623|621|p|p|s|u|sdw31|sdw31|7621|/data/primary621 +624|622|p|p|s|u|sdw31|sdw31|7622|/data/primary622 +625|623|p|p|s|u|sdw31|sdw31|7623|/data/primary623 +626|624|p|p|s|u|sdw31|sdw31|7624|/data/primary624 +627|625|p|p|s|u|sdw31|sdw31|7625|/data/primary625 +1628|626|m|m|s|u|sdw31|sdw31|8626|/data/mirror626 +1629|627|m|m|s|u|sdw31|sdw31|8627|/data/mirror627 +1630|628|m|m|s|u|sdw31|sdw31|8628|/data/mirror628 +1631|629|m|m|s|u|sdw31|sdw31|8629|/data/mirror629 +1632|630|m|m|s|u|sdw31|sdw31|8630|/data/mirror630 +1633|631|m|m|s|u|sdw31|sdw31|8631|/data/mirror631 +1634|632|m|m|s|u|sdw31|sdw31|8632|/data/mirror632 +1635|633|m|m|s|u|sdw31|sdw31|8633|/data/mirror633 +1636|634|m|m|s|u|sdw31|sdw31|8634|/data/mirror634 +1637|635|m|m|s|u|sdw31|sdw31|8635|/data/mirror635 +1638|636|m|m|s|u|sdw31|sdw31|8636|/data/mirror636 +1639|637|m|m|s|u|sdw31|sdw31|8637|/data/mirror637 +1640|638|m|m|s|u|sdw31|sdw31|8638|/data/mirror638 +1641|639|m|m|s|u|sdw31|sdw31|8639|/data/mirror639 +1642|640|m|m|s|u|sdw31|sdw31|8640|/data/mirror640 +1643|641|m|m|s|u|sdw31|sdw31|8641|/data/mirror641 +1644|642|m|m|s|u|sdw31|sdw31|8642|/data/mirror642 +1645|643|m|m|s|u|sdw31|sdw31|8643|/data/mirror643 +# SDW32 +628|626|p|p|s|u|sdw32|sdw32|7626|/data/primary626 +629|627|p|p|s|u|sdw32|sdw32|7627|/data/primary627 +630|628|p|p|s|u|sdw32|sdw32|7628|/data/primary628 +631|629|p|p|s|u|sdw32|sdw32|7629|/data/primary629 +632|630|p|p|s|u|sdw32|sdw32|7630|/data/primary630 +633|631|p|p|s|u|sdw32|sdw32|7631|/data/primary631 +634|632|p|p|s|u|sdw32|sdw32|7632|/data/primary632 +635|633|p|p|s|u|sdw32|sdw32|7633|/data/primary633 +636|634|p|p|s|u|sdw32|sdw32|7634|/data/primary634 +637|635|p|p|s|u|sdw32|sdw32|7635|/data/primary635 +638|636|p|p|s|u|sdw32|sdw32|7636|/data/primary636 +639|637|p|p|s|u|sdw32|sdw32|7637|/data/primary637 +640|638|p|p|s|u|sdw32|sdw32|7638|/data/primary638 +641|639|p|p|s|u|sdw32|sdw32|7639|/data/primary639 +642|640|p|p|s|u|sdw32|sdw32|7640|/data/primary640 +643|641|p|p|s|u|sdw32|sdw32|7641|/data/primary641 +644|642|p|p|s|u|sdw32|sdw32|7642|/data/primary642 +645|643|p|p|s|u|sdw32|sdw32|7643|/data/primary643 +1594|592|m|m|s|u|sdw32|sdw32|8592|/data/mirror592 +1595|593|m|m|s|u|sdw32|sdw32|8593|/data/mirror593 +1596|594|m|m|s|u|sdw32|sdw32|8594|/data/mirror594 +1597|595|m|m|s|u|sdw32|sdw32|8595|/data/mirror595 +1598|596|m|m|s|u|sdw32|sdw32|8596|/data/mirror596 +1599|597|m|m|s|u|sdw32|sdw32|8597|/data/mirror597 +1600|598|m|m|s|u|sdw32|sdw32|8598|/data/mirror598 +1601|599|m|m|s|u|sdw32|sdw32|8599|/data/mirror599 +1602|600|m|m|s|u|sdw32|sdw32|8600|/data/mirror600 +1603|601|m|m|s|u|sdw32|sdw32|8601|/data/mirror601 +1604|602|m|m|s|u|sdw32|sdw32|8602|/data/mirror602 +1605|603|m|m|s|u|sdw32|sdw32|8603|/data/mirror603 +1606|604|m|m|s|u|sdw32|sdw32|8604|/data/mirror604 +1607|605|m|m|s|u|sdw32|sdw32|8605|/data/mirror605 +1608|606|m|m|s|u|sdw32|sdw32|8606|/data/mirror606 +1609|607|m|m|s|u|sdw32|sdw32|8607|/data/mirror607 +1610|608|m|m|s|u|sdw32|sdw32|8608|/data/mirror608 +1611|609|m|m|s|u|sdw32|sdw32|8609|/data/mirror609 +1612|610|m|m|s|u|sdw32|sdw32|8610|/data/mirror610 +# SDW33 +646|644|p|p|s|u|sdw33|sdw33|7644|/data/primary644 +647|645|p|p|s|u|sdw33|sdw33|7645|/data/primary645 +648|646|p|p|s|u|sdw33|sdw33|7646|/data/primary646 +649|647|p|p|s|u|sdw33|sdw33|7647|/data/primary647 +650|648|p|p|s|u|sdw33|sdw33|7648|/data/primary648 +651|649|p|p|s|u|sdw33|sdw33|7649|/data/primary649 +652|650|p|p|s|u|sdw33|sdw33|7650|/data/primary650 +653|651|p|p|s|u|sdw33|sdw33|7651|/data/primary651 +654|652|p|p|s|u|sdw33|sdw33|7652|/data/primary652 +655|653|p|p|s|u|sdw33|sdw33|7653|/data/primary653 +656|654|p|p|s|u|sdw33|sdw33|7654|/data/primary654 +657|655|p|p|s|u|sdw33|sdw33|7655|/data/primary655 +658|656|p|p|s|u|sdw33|sdw33|7656|/data/primary656 +659|657|p|p|s|u|sdw33|sdw33|7657|/data/primary657 +660|658|p|p|s|u|sdw33|sdw33|7658|/data/primary658 +1613|611|m|m|s|u|sdw33|sdw33|8611|/data/mirror611 +1614|612|m|m|s|u|sdw33|sdw33|8612|/data/mirror612 +1615|613|m|m|s|u|sdw33|sdw33|8613|/data/mirror613 +1616|614|m|m|s|u|sdw33|sdw33|8614|/data/mirror614 +1617|615|m|m|s|u|sdw33|sdw33|8615|/data/mirror615 +1618|616|m|m|s|u|sdw33|sdw33|8616|/data/mirror616 +1619|617|m|m|s|u|sdw33|sdw33|8617|/data/mirror617 +1620|618|m|m|s|u|sdw33|sdw33|8618|/data/mirror618 +1621|619|m|m|s|u|sdw33|sdw33|8619|/data/mirror619 +1622|620|m|m|s|u|sdw33|sdw33|8620|/data/mirror620 +1623|621|m|m|s|u|sdw33|sdw33|8621|/data/mirror621 +1624|622|m|m|s|u|sdw33|sdw33|8622|/data/mirror622 +1625|623|m|m|s|u|sdw33|sdw33|8623|/data/mirror623 +1626|624|m|m|s|u|sdw33|sdw33|8624|/data/mirror624 +1627|625|m|m|s|u|sdw33|sdw33|8625|/data/mirror625 +# SDW34 +661|659|p|p|s|u|sdw34|sdw34|7659|/data/primary659 +662|660|p|p|s|u|sdw34|sdw34|7660|/data/primary660 +663|661|p|p|s|u|sdw34|sdw34|7661|/data/primary661 +664|662|p|p|s|u|sdw34|sdw34|7662|/data/primary662 +665|663|p|p|s|u|sdw34|sdw34|7663|/data/primary663 +666|664|p|p|s|u|sdw34|sdw34|7664|/data/primary664 +667|665|p|p|s|u|sdw34|sdw34|7665|/data/primary665 +668|666|p|p|s|u|sdw34|sdw34|7666|/data/primary666 +669|667|p|p|s|u|sdw34|sdw34|7667|/data/primary667 +670|668|p|p|s|u|sdw34|sdw34|7668|/data/primary668 +671|669|p|p|s|u|sdw34|sdw34|7669|/data/primary669 +672|670|p|p|s|u|sdw34|sdw34|7670|/data/primary670 +673|671|p|p|s|u|sdw34|sdw34|7671|/data/primary671 +674|672|p|p|s|u|sdw34|sdw34|7672|/data/primary672 +675|673|p|p|s|u|sdw34|sdw34|7673|/data/primary673 +676|674|p|p|s|u|sdw34|sdw34|7674|/data/primary674 +1646|644|m|m|s|u|sdw34|sdw34|8644|/data/mirror644 +1647|645|m|m|s|u|sdw34|sdw34|8645|/data/mirror645 +1648|646|m|m|s|u|sdw34|sdw34|8646|/data/mirror646 +1649|647|m|m|s|u|sdw34|sdw34|8647|/data/mirror647 +1650|648|m|m|s|u|sdw34|sdw34|8648|/data/mirror648 +1651|649|m|m|s|u|sdw34|sdw34|8649|/data/mirror649 +1652|650|m|m|s|u|sdw34|sdw34|8650|/data/mirror650 +1653|651|m|m|s|u|sdw34|sdw34|8651|/data/mirror651 +1654|652|m|m|s|u|sdw34|sdw34|8652|/data/mirror652 +1655|653|m|m|s|u|sdw34|sdw34|8653|/data/mirror653 +1656|654|m|m|s|u|sdw34|sdw34|8654|/data/mirror654 +1657|655|m|m|s|u|sdw34|sdw34|8655|/data/mirror655 +1658|656|m|m|s|u|sdw34|sdw34|8656|/data/mirror656 +1659|657|m|m|s|u|sdw34|sdw34|8657|/data/mirror657 +1660|658|m|m|s|u|sdw34|sdw34|8658|/data/mirror658 +# SDW35 +677|675|p|p|s|u|sdw35|sdw35|7675|/data/primary675 +678|676|p|p|s|u|sdw35|sdw35|7676|/data/primary676 +679|677|p|p|s|u|sdw35|sdw35|7677|/data/primary677 +680|678|p|p|s|u|sdw35|sdw35|7678|/data/primary678 +681|679|p|p|s|u|sdw35|sdw35|7679|/data/primary679 +682|680|p|p|s|u|sdw35|sdw35|7680|/data/primary680 +683|681|p|p|s|u|sdw35|sdw35|7681|/data/primary681 +684|682|p|p|s|u|sdw35|sdw35|7682|/data/primary682 +685|683|p|p|s|u|sdw35|sdw35|7683|/data/primary683 +686|684|p|p|s|u|sdw35|sdw35|7684|/data/primary684 +687|685|p|p|s|u|sdw35|sdw35|7685|/data/primary685 +688|686|p|p|s|u|sdw35|sdw35|7686|/data/primary686 +689|687|p|p|s|u|sdw35|sdw35|7687|/data/primary687 +690|688|p|p|s|u|sdw35|sdw35|7688|/data/primary688 +691|689|p|p|s|u|sdw35|sdw35|7689|/data/primary689 +692|690|p|p|s|u|sdw35|sdw35|7690|/data/primary690 +693|691|p|p|s|u|sdw35|sdw35|7691|/data/primary691 +694|692|p|p|s|u|sdw35|sdw35|7692|/data/primary692 +695|693|p|p|s|u|sdw35|sdw35|7693|/data/primary693 +696|694|p|p|s|u|sdw35|sdw35|7694|/data/primary694 +697|695|p|p|s|u|sdw35|sdw35|7695|/data/primary695 +698|696|p|p|s|u|sdw35|sdw35|7696|/data/primary696 +699|697|p|p|s|u|sdw35|sdw35|7697|/data/primary697 +700|698|p|p|s|u|sdw35|sdw35|7698|/data/primary698 +701|699|p|p|s|u|sdw35|sdw35|7699|/data/primary699 +702|700|p|p|s|u|sdw35|sdw35|7700|/data/primary700 +1661|659|m|m|s|u|sdw35|sdw35|8659|/data/mirror659 +1662|660|m|m|s|u|sdw35|sdw35|8660|/data/mirror660 +1663|661|m|m|s|u|sdw35|sdw35|8661|/data/mirror661 +1664|662|m|m|s|u|sdw35|sdw35|8662|/data/mirror662 +1665|663|m|m|s|u|sdw35|sdw35|8663|/data/mirror663 +1666|664|m|m|s|u|sdw35|sdw35|8664|/data/mirror664 +1667|665|m|m|s|u|sdw35|sdw35|8665|/data/mirror665 +1668|666|m|m|s|u|sdw35|sdw35|8666|/data/mirror666 +1669|667|m|m|s|u|sdw35|sdw35|8667|/data/mirror667 +1670|668|m|m|s|u|sdw35|sdw35|8668|/data/mirror668 +1671|669|m|m|s|u|sdw35|sdw35|8669|/data/mirror669 +1672|670|m|m|s|u|sdw35|sdw35|8670|/data/mirror670 +1673|671|m|m|s|u|sdw35|sdw35|8671|/data/mirror671 +1674|672|m|m|s|u|sdw35|sdw35|8672|/data/mirror672 +1675|673|m|m|s|u|sdw35|sdw35|8673|/data/mirror673 +1676|674|m|m|s|u|sdw35|sdw35|8674|/data/mirror674 +# SDW36 +703|701|p|p|s|u|sdw36|sdw36|7701|/data/primary701 +704|702|p|p|s|u|sdw36|sdw36|7702|/data/primary702 +705|703|p|p|s|u|sdw36|sdw36|7703|/data/primary703 +706|704|p|p|s|u|sdw36|sdw36|7704|/data/primary704 +707|705|p|p|s|u|sdw36|sdw36|7705|/data/primary705 +708|706|p|p|s|u|sdw36|sdw36|7706|/data/primary706 +709|707|p|p|s|u|sdw36|sdw36|7707|/data/primary707 +710|708|p|p|s|u|sdw36|sdw36|7708|/data/primary708 +711|709|p|p|s|u|sdw36|sdw36|7709|/data/primary709 +712|710|p|p|s|u|sdw36|sdw36|7710|/data/primary710 +713|711|p|p|s|u|sdw36|sdw36|7711|/data/primary711 +714|712|p|p|s|u|sdw36|sdw36|7712|/data/primary712 +715|713|p|p|s|u|sdw36|sdw36|7713|/data/primary713 +716|714|p|p|s|u|sdw36|sdw36|7714|/data/primary714 +717|715|p|p|s|u|sdw36|sdw36|7715|/data/primary715 +718|716|p|p|s|u|sdw36|sdw36|7716|/data/primary716 +719|717|p|p|s|u|sdw36|sdw36|7717|/data/primary717 +1746|744|m|m|s|u|sdw36|sdw36|8744|/data/mirror744 +1747|745|m|m|s|u|sdw36|sdw36|8745|/data/mirror745 +1748|746|m|m|s|u|sdw36|sdw36|8746|/data/mirror746 +1749|747|m|m|s|u|sdw36|sdw36|8747|/data/mirror747 +1750|748|m|m|s|u|sdw36|sdw36|8748|/data/mirror748 +1751|749|m|m|s|u|sdw36|sdw36|8749|/data/mirror749 +1752|750|m|m|s|u|sdw36|sdw36|8750|/data/mirror750 +1753|751|m|m|s|u|sdw36|sdw36|8751|/data/mirror751 +1754|752|m|m|s|u|sdw36|sdw36|8752|/data/mirror752 +1755|753|m|m|s|u|sdw36|sdw36|8753|/data/mirror753 +1756|754|m|m|s|u|sdw36|sdw36|8754|/data/mirror754 +1757|755|m|m|s|u|sdw36|sdw36|8755|/data/mirror755 +1758|756|m|m|s|u|sdw36|sdw36|8756|/data/mirror756 +1759|757|m|m|s|u|sdw36|sdw36|8757|/data/mirror757 +1760|758|m|m|s|u|sdw36|sdw36|8758|/data/mirror758 +1761|759|m|m|s|u|sdw36|sdw36|8759|/data/mirror759 +1762|760|m|m|s|u|sdw36|sdw36|8760|/data/mirror760 +1763|761|m|m|s|u|sdw36|sdw36|8761|/data/mirror761 +1764|762|m|m|s|u|sdw36|sdw36|8762|/data/mirror762 +1765|763|m|m|s|u|sdw36|sdw36|8763|/data/mirror763 +1766|764|m|m|s|u|sdw36|sdw36|8764|/data/mirror764 +1767|765|m|m|s|u|sdw36|sdw36|8765|/data/mirror765 +1768|766|m|m|s|u|sdw36|sdw36|8766|/data/mirror766 +# SDW37 +720|718|p|p|s|u|sdw37|sdw37|7718|/data/primary718 +721|719|p|p|s|u|sdw37|sdw37|7719|/data/primary719 +722|720|p|p|s|u|sdw37|sdw37|7720|/data/primary720 +723|721|p|p|s|u|sdw37|sdw37|7721|/data/primary721 +724|722|p|p|s|u|sdw37|sdw37|7722|/data/primary722 +725|723|p|p|s|u|sdw37|sdw37|7723|/data/primary723 +726|724|p|p|s|u|sdw37|sdw37|7724|/data/primary724 +727|725|p|p|s|u|sdw37|sdw37|7725|/data/primary725 +728|726|p|p|s|u|sdw37|sdw37|7726|/data/primary726 +729|727|p|p|s|u|sdw37|sdw37|7727|/data/primary727 +730|728|p|p|s|u|sdw37|sdw37|7728|/data/primary728 +731|729|p|p|s|u|sdw37|sdw37|7729|/data/primary729 +732|730|p|p|s|u|sdw37|sdw37|7730|/data/primary730 +733|731|p|p|s|u|sdw37|sdw37|7731|/data/primary731 +734|732|p|p|s|u|sdw37|sdw37|7732|/data/primary732 +735|733|p|p|s|u|sdw37|sdw37|7733|/data/primary733 +736|734|p|p|s|u|sdw37|sdw37|7734|/data/primary734 +737|735|p|p|s|u|sdw37|sdw37|7735|/data/primary735 +738|736|p|p|s|u|sdw37|sdw37|7736|/data/primary736 +739|737|p|p|s|u|sdw37|sdw37|7737|/data/primary737 +740|738|p|p|s|u|sdw37|sdw37|7738|/data/primary738 +741|739|p|p|s|u|sdw37|sdw37|7739|/data/primary739 +742|740|p|p|s|u|sdw37|sdw37|7740|/data/primary740 +743|741|p|p|s|u|sdw37|sdw37|7741|/data/primary741 +744|742|p|p|s|u|sdw37|sdw37|7742|/data/primary742 +745|743|p|p|s|u|sdw37|sdw37|7743|/data/primary743 +1814|812|m|m|s|u|sdw37|sdw37|8812|/data/mirror812 +1815|813|m|m|s|u|sdw37|sdw37|8813|/data/mirror813 +1816|814|m|m|s|u|sdw37|sdw37|8814|/data/mirror814 +1817|815|m|m|s|u|sdw37|sdw37|8815|/data/mirror815 +1818|816|m|m|s|u|sdw37|sdw37|8816|/data/mirror816 +1819|817|m|m|s|u|sdw37|sdw37|8817|/data/mirror817 +1820|818|m|m|s|u|sdw37|sdw37|8818|/data/mirror818 +1821|819|m|m|s|u|sdw37|sdw37|8819|/data/mirror819 +1822|820|m|m|s|u|sdw37|sdw37|8820|/data/mirror820 +1823|821|m|m|s|u|sdw37|sdw37|8821|/data/mirror821 +1824|822|m|m|s|u|sdw37|sdw37|8822|/data/mirror822 +1825|823|m|m|s|u|sdw37|sdw37|8823|/data/mirror823 +1826|824|m|m|s|u|sdw37|sdw37|8824|/data/mirror824 +1827|825|m|m|s|u|sdw37|sdw37|8825|/data/mirror825 +1828|826|m|m|s|u|sdw37|sdw37|8826|/data/mirror826 +1829|827|m|m|s|u|sdw37|sdw37|8827|/data/mirror827 +1830|828|m|m|s|u|sdw37|sdw37|8828|/data/mirror828 +1831|829|m|m|s|u|sdw37|sdw37|8829|/data/mirror829 +1832|830|m|m|s|u|sdw37|sdw37|8830|/data/mirror830 +1833|831|m|m|s|u|sdw37|sdw37|8831|/data/mirror831 +1834|832|m|m|s|u|sdw37|sdw37|8832|/data/mirror832 +# SDW38 +746|744|p|p|s|u|sdw38|sdw38|7744|/data/primary744 +747|745|p|p|s|u|sdw38|sdw38|7745|/data/primary745 +748|746|p|p|s|u|sdw38|sdw38|7746|/data/primary746 +749|747|p|p|s|u|sdw38|sdw38|7747|/data/primary747 +750|748|p|p|s|u|sdw38|sdw38|7748|/data/primary748 +751|749|p|p|s|u|sdw38|sdw38|7749|/data/primary749 +752|750|p|p|s|u|sdw38|sdw38|7750|/data/primary750 +753|751|p|p|s|u|sdw38|sdw38|7751|/data/primary751 +754|752|p|p|s|u|sdw38|sdw38|7752|/data/primary752 +755|753|p|p|s|u|sdw38|sdw38|7753|/data/primary753 +756|754|p|p|s|u|sdw38|sdw38|7754|/data/primary754 +757|755|p|p|s|u|sdw38|sdw38|7755|/data/primary755 +758|756|p|p|s|u|sdw38|sdw38|7756|/data/primary756 +759|757|p|p|s|u|sdw38|sdw38|7757|/data/primary757 +760|758|p|p|s|u|sdw38|sdw38|7758|/data/primary758 +761|759|p|p|s|u|sdw38|sdw38|7759|/data/primary759 +762|760|p|p|s|u|sdw38|sdw38|7760|/data/primary760 +763|761|p|p|s|u|sdw38|sdw38|7761|/data/primary761 +764|762|p|p|s|u|sdw38|sdw38|7762|/data/primary762 +765|763|p|p|s|u|sdw38|sdw38|7763|/data/primary763 +766|764|p|p|s|u|sdw38|sdw38|7764|/data/primary764 +767|765|p|p|s|u|sdw38|sdw38|7765|/data/primary765 +768|766|p|p|s|u|sdw38|sdw38|7766|/data/primary766 +1703|701|m|m|s|u|sdw38|sdw38|8701|/data/mirror701 +1704|702|m|m|s|u|sdw38|sdw38|8702|/data/mirror702 +1705|703|m|m|s|u|sdw38|sdw38|8703|/data/mirror703 +1706|704|m|m|s|u|sdw38|sdw38|8704|/data/mirror704 +1707|705|m|m|s|u|sdw38|sdw38|8705|/data/mirror705 +1708|706|m|m|s|u|sdw38|sdw38|8706|/data/mirror706 +1709|707|m|m|s|u|sdw38|sdw38|8707|/data/mirror707 +1710|708|m|m|s|u|sdw38|sdw38|8708|/data/mirror708 +1711|709|m|m|s|u|sdw38|sdw38|8709|/data/mirror709 +1712|710|m|m|s|u|sdw38|sdw38|8710|/data/mirror710 +1713|711|m|m|s|u|sdw38|sdw38|8711|/data/mirror711 +1714|712|m|m|s|u|sdw38|sdw38|8712|/data/mirror712 +1715|713|m|m|s|u|sdw38|sdw38|8713|/data/mirror713 +1716|714|m|m|s|u|sdw38|sdw38|8714|/data/mirror714 +1717|715|m|m|s|u|sdw38|sdw38|8715|/data/mirror715 +1718|716|m|m|s|u|sdw38|sdw38|8716|/data/mirror716 +1719|717|m|m|s|u|sdw38|sdw38|8717|/data/mirror717 +# SDW39 +769|767|p|p|s|u|sdw39|sdw39|7767|/data/primary767 +770|768|p|p|s|u|sdw39|sdw39|7768|/data/primary768 +771|769|p|p|s|u|sdw39|sdw39|7769|/data/primary769 +772|770|p|p|s|u|sdw39|sdw39|7770|/data/primary770 +773|771|p|p|s|u|sdw39|sdw39|7771|/data/primary771 +774|772|p|p|s|u|sdw39|sdw39|7772|/data/primary772 +775|773|p|p|s|u|sdw39|sdw39|7773|/data/primary773 +776|774|p|p|s|u|sdw39|sdw39|7774|/data/primary774 +777|775|p|p|s|u|sdw39|sdw39|7775|/data/primary775 +778|776|p|p|s|u|sdw39|sdw39|7776|/data/primary776 +779|777|p|p|s|u|sdw39|sdw39|7777|/data/primary777 +780|778|p|p|s|u|sdw39|sdw39|7778|/data/primary778 +781|779|p|p|s|u|sdw39|sdw39|7779|/data/primary779 +782|780|p|p|s|u|sdw39|sdw39|7780|/data/primary780 +783|781|p|p|s|u|sdw39|sdw39|7781|/data/primary781 +784|782|p|p|s|u|sdw39|sdw39|7782|/data/primary782 +785|783|p|p|s|u|sdw39|sdw39|7783|/data/primary783 +786|784|p|p|s|u|sdw39|sdw39|7784|/data/primary784 +787|785|p|p|s|u|sdw39|sdw39|7785|/data/primary785 +788|786|p|p|s|u|sdw39|sdw39|7786|/data/primary786 +789|787|p|p|s|u|sdw39|sdw39|7787|/data/primary787 +1856|854|m|m|s|u|sdw39|sdw39|8854|/data/mirror854 +1857|855|m|m|s|u|sdw39|sdw39|8855|/data/mirror855 +1858|856|m|m|s|u|sdw39|sdw39|8856|/data/mirror856 +1859|857|m|m|s|u|sdw39|sdw39|8857|/data/mirror857 +1860|858|m|m|s|u|sdw39|sdw39|8858|/data/mirror858 +1861|859|m|m|s|u|sdw39|sdw39|8859|/data/mirror859 +1862|860|m|m|s|u|sdw39|sdw39|8860|/data/mirror860 +1863|861|m|m|s|u|sdw39|sdw39|8861|/data/mirror861 +1864|862|m|m|s|u|sdw39|sdw39|8862|/data/mirror862 +1865|863|m|m|s|u|sdw39|sdw39|8863|/data/mirror863 +1866|864|m|m|s|u|sdw39|sdw39|8864|/data/mirror864 +1867|865|m|m|s|u|sdw39|sdw39|8865|/data/mirror865 +1868|866|m|m|s|u|sdw39|sdw39|8866|/data/mirror866 +1869|867|m|m|s|u|sdw39|sdw39|8867|/data/mirror867 +1870|868|m|m|s|u|sdw39|sdw39|8868|/data/mirror868 +1871|869|m|m|s|u|sdw39|sdw39|8869|/data/mirror869 +1872|870|m|m|s|u|sdw39|sdw39|8870|/data/mirror870 +1873|871|m|m|s|u|sdw39|sdw39|8871|/data/mirror871 +# SDW4 +56|54|p|p|s|u|sdw4|sdw4|7054|/data/primary54 +57|55|p|p|s|u|sdw4|sdw4|7055|/data/primary55 +58|56|p|p|s|u|sdw4|sdw4|7056|/data/primary56 +59|57|p|p|s|u|sdw4|sdw4|7057|/data/primary57 +60|58|p|p|s|u|sdw4|sdw4|7058|/data/primary58 +61|59|p|p|s|u|sdw4|sdw4|7059|/data/primary59 +62|60|p|p|s|u|sdw4|sdw4|7060|/data/primary60 +63|61|p|p|s|u|sdw4|sdw4|7061|/data/primary61 +64|62|p|p|s|u|sdw4|sdw4|7062|/data/primary62 +65|63|p|p|s|u|sdw4|sdw4|7063|/data/primary63 +66|64|p|p|s|u|sdw4|sdw4|7064|/data/primary64 +67|65|p|p|s|u|sdw4|sdw4|7065|/data/primary65 +68|66|p|p|s|u|sdw4|sdw4|7066|/data/primary66 +69|67|p|p|s|u|sdw4|sdw4|7067|/data/primary67 +70|68|p|p|s|u|sdw4|sdw4|7068|/data/primary68 +71|69|p|p|s|u|sdw4|sdw4|7069|/data/primary69 +72|70|p|p|s|u|sdw4|sdw4|7070|/data/primary70 +73|71|p|p|s|u|sdw4|sdw4|7071|/data/primary71 +1021|19|m|m|s|u|sdw4|sdw4|8019|/data/mirror19 +1022|20|m|m|s|u|sdw4|sdw4|8020|/data/mirror20 +1023|21|m|m|s|u|sdw4|sdw4|8021|/data/mirror21 +1024|22|m|m|s|u|sdw4|sdw4|8022|/data/mirror22 +1025|23|m|m|s|u|sdw4|sdw4|8023|/data/mirror23 +1026|24|m|m|s|u|sdw4|sdw4|8024|/data/mirror24 +1027|25|m|m|s|u|sdw4|sdw4|8025|/data/mirror25 +1028|26|m|m|s|u|sdw4|sdw4|8026|/data/mirror26 +1029|27|m|m|s|u|sdw4|sdw4|8027|/data/mirror27 +1030|28|m|m|s|u|sdw4|sdw4|8028|/data/mirror28 +1031|29|m|m|s|u|sdw4|sdw4|8029|/data/mirror29 +1032|30|m|m|s|u|sdw4|sdw4|8030|/data/mirror30 +1033|31|m|m|s|u|sdw4|sdw4|8031|/data/mirror31 +1034|32|m|m|s|u|sdw4|sdw4|8032|/data/mirror32 +1035|33|m|m|s|u|sdw4|sdw4|8033|/data/mirror33 +1036|34|m|m|s|u|sdw4|sdw4|8034|/data/mirror34 +1037|35|m|m|s|u|sdw4|sdw4|8035|/data/mirror35 +1038|36|m|m|s|u|sdw4|sdw4|8036|/data/mirror36 +1039|37|m|m|s|u|sdw4|sdw4|8037|/data/mirror37 +1040|38|m|m|s|u|sdw4|sdw4|8038|/data/mirror38 +1041|39|m|m|s|u|sdw4|sdw4|8039|/data/mirror39 +# SDW40 +790|788|p|p|s|u|sdw40|sdw40|7788|/data/primary788 +791|789|p|p|s|u|sdw40|sdw40|7789|/data/primary789 +792|790|p|p|s|u|sdw40|sdw40|7790|/data/primary790 +793|791|p|p|s|u|sdw40|sdw40|7791|/data/primary791 +794|792|p|p|s|u|sdw40|sdw40|7792|/data/primary792 +795|793|p|p|s|u|sdw40|sdw40|7793|/data/primary793 +796|794|p|p|s|u|sdw40|sdw40|7794|/data/primary794 +797|795|p|p|s|u|sdw40|sdw40|7795|/data/primary795 +798|796|p|p|s|u|sdw40|sdw40|7796|/data/primary796 +799|797|p|p|s|u|sdw40|sdw40|7797|/data/primary797 +800|798|p|p|s|u|sdw40|sdw40|7798|/data/primary798 +801|799|p|p|s|u|sdw40|sdw40|7799|/data/primary799 +802|800|p|p|s|u|sdw40|sdw40|7800|/data/primary800 +803|801|p|p|s|u|sdw40|sdw40|7801|/data/primary801 +804|802|p|p|s|u|sdw40|sdw40|7802|/data/primary802 +805|803|p|p|s|u|sdw40|sdw40|7803|/data/primary803 +806|804|p|p|s|u|sdw40|sdw40|7804|/data/primary804 +807|805|p|p|s|u|sdw40|sdw40|7805|/data/primary805 +808|806|p|p|s|u|sdw40|sdw40|7806|/data/primary806 +809|807|p|p|s|u|sdw40|sdw40|7807|/data/primary807 +810|808|p|p|s|u|sdw40|sdw40|7808|/data/primary808 +811|809|p|p|s|u|sdw40|sdw40|7809|/data/primary809 +812|810|p|p|s|u|sdw40|sdw40|7810|/data/primary810 +813|811|p|p|s|u|sdw40|sdw40|7811|/data/primary811 +1720|718|m|m|s|u|sdw40|sdw40|8718|/data/mirror718 +1721|719|m|m|s|u|sdw40|sdw40|8719|/data/mirror719 +1722|720|m|m|s|u|sdw40|sdw40|8720|/data/mirror720 +1723|721|m|m|s|u|sdw40|sdw40|8721|/data/mirror721 +1724|722|m|m|s|u|sdw40|sdw40|8722|/data/mirror722 +1725|723|m|m|s|u|sdw40|sdw40|8723|/data/mirror723 +1726|724|m|m|s|u|sdw40|sdw40|8724|/data/mirror724 +1727|725|m|m|s|u|sdw40|sdw40|8725|/data/mirror725 +1728|726|m|m|s|u|sdw40|sdw40|8726|/data/mirror726 +1729|727|m|m|s|u|sdw40|sdw40|8727|/data/mirror727 +1730|728|m|m|s|u|sdw40|sdw40|8728|/data/mirror728 +1731|729|m|m|s|u|sdw40|sdw40|8729|/data/mirror729 +1732|730|m|m|s|u|sdw40|sdw40|8730|/data/mirror730 +1733|731|m|m|s|u|sdw40|sdw40|8731|/data/mirror731 +1734|732|m|m|s|u|sdw40|sdw40|8732|/data/mirror732 +1735|733|m|m|s|u|sdw40|sdw40|8733|/data/mirror733 +1736|734|m|m|s|u|sdw40|sdw40|8734|/data/mirror734 +1737|735|m|m|s|u|sdw40|sdw40|8735|/data/mirror735 +1738|736|m|m|s|u|sdw40|sdw40|8736|/data/mirror736 +1739|737|m|m|s|u|sdw40|sdw40|8737|/data/mirror737 +1740|738|m|m|s|u|sdw40|sdw40|8738|/data/mirror738 +1741|739|m|m|s|u|sdw40|sdw40|8739|/data/mirror739 +1742|740|m|m|s|u|sdw40|sdw40|8740|/data/mirror740 +1743|741|m|m|s|u|sdw40|sdw40|8741|/data/mirror741 +1744|742|m|m|s|u|sdw40|sdw40|8742|/data/mirror742 +1745|743|m|m|s|u|sdw40|sdw40|8743|/data/mirror743 +# SDW41 +814|812|p|p|s|u|sdw41|sdw41|7812|/data/primary812 +815|813|p|p|s|u|sdw41|sdw41|7813|/data/primary813 +816|814|p|p|s|u|sdw41|sdw41|7814|/data/primary814 +817|815|p|p|s|u|sdw41|sdw41|7815|/data/primary815 +818|816|p|p|s|u|sdw41|sdw41|7816|/data/primary816 +819|817|p|p|s|u|sdw41|sdw41|7817|/data/primary817 +820|818|p|p|s|u|sdw41|sdw41|7818|/data/primary818 +821|819|p|p|s|u|sdw41|sdw41|7819|/data/primary819 +822|820|p|p|s|u|sdw41|sdw41|7820|/data/primary820 +823|821|p|p|s|u|sdw41|sdw41|7821|/data/primary821 +824|822|p|p|s|u|sdw41|sdw41|7822|/data/primary822 +825|823|p|p|s|u|sdw41|sdw41|7823|/data/primary823 +826|824|p|p|s|u|sdw41|sdw41|7824|/data/primary824 +827|825|p|p|s|u|sdw41|sdw41|7825|/data/primary825 +828|826|p|p|s|u|sdw41|sdw41|7826|/data/primary826 +829|827|p|p|s|u|sdw41|sdw41|7827|/data/primary827 +830|828|p|p|s|u|sdw41|sdw41|7828|/data/primary828 +831|829|p|p|s|u|sdw41|sdw41|7829|/data/primary829 +832|830|p|p|s|u|sdw41|sdw41|7830|/data/primary830 +833|831|p|p|s|u|sdw41|sdw41|7831|/data/primary831 +834|832|p|p|s|u|sdw41|sdw41|7832|/data/primary832 +1769|767|m|m|s|u|sdw41|sdw41|8767|/data/mirror767 +1770|768|m|m|s|u|sdw41|sdw41|8768|/data/mirror768 +1771|769|m|m|s|u|sdw41|sdw41|8769|/data/mirror769 +1772|770|m|m|s|u|sdw41|sdw41|8770|/data/mirror770 +1773|771|m|m|s|u|sdw41|sdw41|8771|/data/mirror771 +1774|772|m|m|s|u|sdw41|sdw41|8772|/data/mirror772 +1775|773|m|m|s|u|sdw41|sdw41|8773|/data/mirror773 +1776|774|m|m|s|u|sdw41|sdw41|8774|/data/mirror774 +1777|775|m|m|s|u|sdw41|sdw41|8775|/data/mirror775 +1778|776|m|m|s|u|sdw41|sdw41|8776|/data/mirror776 +1779|777|m|m|s|u|sdw41|sdw41|8777|/data/mirror777 +1780|778|m|m|s|u|sdw41|sdw41|8778|/data/mirror778 +1781|779|m|m|s|u|sdw41|sdw41|8779|/data/mirror779 +1782|780|m|m|s|u|sdw41|sdw41|8780|/data/mirror780 +1783|781|m|m|s|u|sdw41|sdw41|8781|/data/mirror781 +1784|782|m|m|s|u|sdw41|sdw41|8782|/data/mirror782 +1785|783|m|m|s|u|sdw41|sdw41|8783|/data/mirror783 +1786|784|m|m|s|u|sdw41|sdw41|8784|/data/mirror784 +1787|785|m|m|s|u|sdw41|sdw41|8785|/data/mirror785 +1788|786|m|m|s|u|sdw41|sdw41|8786|/data/mirror786 +1789|787|m|m|s|u|sdw41|sdw41|8787|/data/mirror787 +# SDW42 +835|833|p|p|s|u|sdw42|sdw42|7833|/data/primary833 +836|834|p|p|s|u|sdw42|sdw42|7834|/data/primary834 +837|835|p|p|s|u|sdw42|sdw42|7835|/data/primary835 +838|836|p|p|s|u|sdw42|sdw42|7836|/data/primary836 +839|837|p|p|s|u|sdw42|sdw42|7837|/data/primary837 +840|838|p|p|s|u|sdw42|sdw42|7838|/data/primary838 +841|839|p|p|s|u|sdw42|sdw42|7839|/data/primary839 +842|840|p|p|s|u|sdw42|sdw42|7840|/data/primary840 +843|841|p|p|s|u|sdw42|sdw42|7841|/data/primary841 +844|842|p|p|s|u|sdw42|sdw42|7842|/data/primary842 +845|843|p|p|s|u|sdw42|sdw42|7843|/data/primary843 +846|844|p|p|s|u|sdw42|sdw42|7844|/data/primary844 +847|845|p|p|s|u|sdw42|sdw42|7845|/data/primary845 +848|846|p|p|s|u|sdw42|sdw42|7846|/data/primary846 +849|847|p|p|s|u|sdw42|sdw42|7847|/data/primary847 +850|848|p|p|s|u|sdw42|sdw42|7848|/data/primary848 +851|849|p|p|s|u|sdw42|sdw42|7849|/data/primary849 +852|850|p|p|s|u|sdw42|sdw42|7850|/data/primary850 +853|851|p|p|s|u|sdw42|sdw42|7851|/data/primary851 +854|852|p|p|s|u|sdw42|sdw42|7852|/data/primary852 +855|853|p|p|s|u|sdw42|sdw42|7853|/data/primary853 +1790|788|m|m|s|u|sdw42|sdw42|8788|/data/mirror788 +1791|789|m|m|s|u|sdw42|sdw42|8789|/data/mirror789 +1792|790|m|m|s|u|sdw42|sdw42|8790|/data/mirror790 +1793|791|m|m|s|u|sdw42|sdw42|8791|/data/mirror791 +1794|792|m|m|s|u|sdw42|sdw42|8792|/data/mirror792 +1795|793|m|m|s|u|sdw42|sdw42|8793|/data/mirror793 +1796|794|m|m|s|u|sdw42|sdw42|8794|/data/mirror794 +1797|795|m|m|s|u|sdw42|sdw42|8795|/data/mirror795 +1798|796|m|m|s|u|sdw42|sdw42|8796|/data/mirror796 +1799|797|m|m|s|u|sdw42|sdw42|8797|/data/mirror797 +1800|798|m|m|s|u|sdw42|sdw42|8798|/data/mirror798 +1801|799|m|m|s|u|sdw42|sdw42|8799|/data/mirror799 +1802|800|m|m|s|u|sdw42|sdw42|8800|/data/mirror800 +1803|801|m|m|s|u|sdw42|sdw42|8801|/data/mirror801 +1804|802|m|m|s|u|sdw42|sdw42|8802|/data/mirror802 +1805|803|m|m|s|u|sdw42|sdw42|8803|/data/mirror803 +1806|804|m|m|s|u|sdw42|sdw42|8804|/data/mirror804 +1807|805|m|m|s|u|sdw42|sdw42|8805|/data/mirror805 +1808|806|m|m|s|u|sdw42|sdw42|8806|/data/mirror806 +1809|807|m|m|s|u|sdw42|sdw42|8807|/data/mirror807 +1810|808|m|m|s|u|sdw42|sdw42|8808|/data/mirror808 +1811|809|m|m|s|u|sdw42|sdw42|8809|/data/mirror809 +1812|810|m|m|s|u|sdw42|sdw42|8810|/data/mirror810 +1813|811|m|m|s|u|sdw42|sdw42|8811|/data/mirror811 +# SDW43 +856|854|p|p|s|u|sdw43|sdw43|7854|/data/primary854 +857|855|p|p|s|u|sdw43|sdw43|7855|/data/primary855 +858|856|p|p|s|u|sdw43|sdw43|7856|/data/primary856 +859|857|p|p|s|u|sdw43|sdw43|7857|/data/primary857 +860|858|p|p|s|u|sdw43|sdw43|7858|/data/primary858 +861|859|p|p|s|u|sdw43|sdw43|7859|/data/primary859 +862|860|p|p|s|u|sdw43|sdw43|7860|/data/primary860 +863|861|p|p|s|u|sdw43|sdw43|7861|/data/primary861 +864|862|p|p|s|u|sdw43|sdw43|7862|/data/primary862 +865|863|p|p|s|u|sdw43|sdw43|7863|/data/primary863 +866|864|p|p|s|u|sdw43|sdw43|7864|/data/primary864 +867|865|p|p|s|u|sdw43|sdw43|7865|/data/primary865 +868|866|p|p|s|u|sdw43|sdw43|7866|/data/primary866 +869|867|p|p|s|u|sdw43|sdw43|7867|/data/primary867 +870|868|p|p|s|u|sdw43|sdw43|7868|/data/primary868 +871|869|p|p|s|u|sdw43|sdw43|7869|/data/primary869 +872|870|p|p|s|u|sdw43|sdw43|7870|/data/primary870 +873|871|p|p|s|u|sdw43|sdw43|7871|/data/primary871 +1835|833|m|m|s|u|sdw43|sdw43|8833|/data/mirror833 +1836|834|m|m|s|u|sdw43|sdw43|8834|/data/mirror834 +1837|835|m|m|s|u|sdw43|sdw43|8835|/data/mirror835 +1838|836|m|m|s|u|sdw43|sdw43|8836|/data/mirror836 +1839|837|m|m|s|u|sdw43|sdw43|8837|/data/mirror837 +1840|838|m|m|s|u|sdw43|sdw43|8838|/data/mirror838 +1841|839|m|m|s|u|sdw43|sdw43|8839|/data/mirror839 +1842|840|m|m|s|u|sdw43|sdw43|8840|/data/mirror840 +1843|841|m|m|s|u|sdw43|sdw43|8841|/data/mirror841 +1844|842|m|m|s|u|sdw43|sdw43|8842|/data/mirror842 +1845|843|m|m|s|u|sdw43|sdw43|8843|/data/mirror843 +1846|844|m|m|s|u|sdw43|sdw43|8844|/data/mirror844 +1847|845|m|m|s|u|sdw43|sdw43|8845|/data/mirror845 +1848|846|m|m|s|u|sdw43|sdw43|8846|/data/mirror846 +1849|847|m|m|s|u|sdw43|sdw43|8847|/data/mirror847 +1850|848|m|m|s|u|sdw43|sdw43|8848|/data/mirror848 +1851|849|m|m|s|u|sdw43|sdw43|8849|/data/mirror849 +1852|850|m|m|s|u|sdw43|sdw43|8850|/data/mirror850 +1853|851|m|m|s|u|sdw43|sdw43|8851|/data/mirror851 +1854|852|m|m|s|u|sdw43|sdw43|8852|/data/mirror852 +1855|853|m|m|s|u|sdw43|sdw43|8853|/data/mirror853 +# SDW44 +874|872|p|p|s|u|sdw44|sdw44|7872|/data/primary872 +875|873|p|p|s|u|sdw44|sdw44|7873|/data/primary873 +876|874|p|p|s|u|sdw44|sdw44|7874|/data/primary874 +877|875|p|p|s|u|sdw44|sdw44|7875|/data/primary875 +878|876|p|p|s|u|sdw44|sdw44|7876|/data/primary876 +879|877|p|p|s|u|sdw44|sdw44|7877|/data/primary877 +880|878|p|p|s|u|sdw44|sdw44|7878|/data/primary878 +881|879|p|p|s|u|sdw44|sdw44|7879|/data/primary879 +882|880|p|p|s|u|sdw44|sdw44|7880|/data/primary880 +883|881|p|p|s|u|sdw44|sdw44|7881|/data/primary881 +884|882|p|p|s|u|sdw44|sdw44|7882|/data/primary882 +885|883|p|p|s|u|sdw44|sdw44|7883|/data/primary883 +886|884|p|p|s|u|sdw44|sdw44|7884|/data/primary884 +887|885|p|p|s|u|sdw44|sdw44|7885|/data/primary885 +888|886|p|p|s|u|sdw44|sdw44|7886|/data/primary886 +889|887|p|p|s|u|sdw44|sdw44|7887|/data/primary887 +890|888|p|p|s|u|sdw44|sdw44|7888|/data/primary888 +891|889|p|p|s|u|sdw44|sdw44|7889|/data/primary889 +892|890|p|p|s|u|sdw44|sdw44|7890|/data/primary890 +893|891|p|p|s|u|sdw44|sdw44|7891|/data/primary891 +894|892|p|p|s|u|sdw44|sdw44|7892|/data/primary892 +1986|984|m|m|s|u|sdw44|sdw44|8984|/data/mirror984 +1987|985|m|m|s|u|sdw44|sdw44|8985|/data/mirror985 +1988|986|m|m|s|u|sdw44|sdw44|8986|/data/mirror986 +1989|987|m|m|s|u|sdw44|sdw44|8987|/data/mirror987 +1990|988|m|m|s|u|sdw44|sdw44|8988|/data/mirror988 +1991|989|m|m|s|u|sdw44|sdw44|8989|/data/mirror989 +1992|990|m|m|s|u|sdw44|sdw44|8990|/data/mirror990 +1993|991|m|m|s|u|sdw44|sdw44|8991|/data/mirror991 +1994|992|m|m|s|u|sdw44|sdw44|8992|/data/mirror992 +1995|993|m|m|s|u|sdw44|sdw44|8993|/data/mirror993 +1996|994|m|m|s|u|sdw44|sdw44|8994|/data/mirror994 +1997|995|m|m|s|u|sdw44|sdw44|8995|/data/mirror995 +1998|996|m|m|s|u|sdw44|sdw44|8996|/data/mirror996 +1999|997|m|m|s|u|sdw44|sdw44|8997|/data/mirror997 +2000|998|m|m|s|u|sdw44|sdw44|8998|/data/mirror998 +2001|999|m|m|s|u|sdw44|sdw44|8999|/data/mirror999 +# SDW45 +895|893|p|p|s|u|sdw45|sdw45|7893|/data/primary893 +896|894|p|p|s|u|sdw45|sdw45|7894|/data/primary894 +897|895|p|p|s|u|sdw45|sdw45|7895|/data/primary895 +898|896|p|p|s|u|sdw45|sdw45|7896|/data/primary896 +899|897|p|p|s|u|sdw45|sdw45|7897|/data/primary897 +900|898|p|p|s|u|sdw45|sdw45|7898|/data/primary898 +901|899|p|p|s|u|sdw45|sdw45|7899|/data/primary899 +902|900|p|p|s|u|sdw45|sdw45|7900|/data/primary900 +903|901|p|p|s|u|sdw45|sdw45|7901|/data/primary901 +904|902|p|p|s|u|sdw45|sdw45|7902|/data/primary902 +905|903|p|p|s|u|sdw45|sdw45|7903|/data/primary903 +906|904|p|p|s|u|sdw45|sdw45|7904|/data/primary904 +907|905|p|p|s|u|sdw45|sdw45|7905|/data/primary905 +908|906|p|p|s|u|sdw45|sdw45|7906|/data/primary906 +909|907|p|p|s|u|sdw45|sdw45|7907|/data/primary907 +910|908|p|p|s|u|sdw45|sdw45|7908|/data/primary908 +911|909|p|p|s|u|sdw45|sdw45|7909|/data/primary909 +912|910|p|p|s|u|sdw45|sdw45|7910|/data/primary910 +913|911|p|p|s|u|sdw45|sdw45|7911|/data/primary911 +1931|929|m|m|s|u|sdw45|sdw45|8929|/data/mirror929 +1932|930|m|m|s|u|sdw45|sdw45|8930|/data/mirror930 +1933|931|m|m|s|u|sdw45|sdw45|8931|/data/mirror931 +1934|932|m|m|s|u|sdw45|sdw45|8932|/data/mirror932 +1935|933|m|m|s|u|sdw45|sdw45|8933|/data/mirror933 +1936|934|m|m|s|u|sdw45|sdw45|8934|/data/mirror934 +1937|935|m|m|s|u|sdw45|sdw45|8935|/data/mirror935 +1938|936|m|m|s|u|sdw45|sdw45|8936|/data/mirror936 +1939|937|m|m|s|u|sdw45|sdw45|8937|/data/mirror937 +1940|938|m|m|s|u|sdw45|sdw45|8938|/data/mirror938 +1941|939|m|m|s|u|sdw45|sdw45|8939|/data/mirror939 +1942|940|m|m|s|u|sdw45|sdw45|8940|/data/mirror940 +1943|941|m|m|s|u|sdw45|sdw45|8941|/data/mirror941 +1944|942|m|m|s|u|sdw45|sdw45|8942|/data/mirror942 +1945|943|m|m|s|u|sdw45|sdw45|8943|/data/mirror943 +1946|944|m|m|s|u|sdw45|sdw45|8944|/data/mirror944 +1947|945|m|m|s|u|sdw45|sdw45|8945|/data/mirror945 +# SDW46 +914|912|p|p|s|u|sdw46|sdw46|7912|/data/primary912 +915|913|p|p|s|u|sdw46|sdw46|7913|/data/primary913 +916|914|p|p|s|u|sdw46|sdw46|7914|/data/primary914 +917|915|p|p|s|u|sdw46|sdw46|7915|/data/primary915 +918|916|p|p|s|u|sdw46|sdw46|7916|/data/primary916 +919|917|p|p|s|u|sdw46|sdw46|7917|/data/primary917 +920|918|p|p|s|u|sdw46|sdw46|7918|/data/primary918 +921|919|p|p|s|u|sdw46|sdw46|7919|/data/primary919 +922|920|p|p|s|u|sdw46|sdw46|7920|/data/primary920 +923|921|p|p|s|u|sdw46|sdw46|7921|/data/primary921 +924|922|p|p|s|u|sdw46|sdw46|7922|/data/primary922 +925|923|p|p|s|u|sdw46|sdw46|7923|/data/primary923 +926|924|p|p|s|u|sdw46|sdw46|7924|/data/primary924 +927|925|p|p|s|u|sdw46|sdw46|7925|/data/primary925 +928|926|p|p|s|u|sdw46|sdw46|7926|/data/primary926 +929|927|p|p|s|u|sdw46|sdw46|7927|/data/primary927 +930|928|p|p|s|u|sdw46|sdw46|7928|/data/primary928 +1874|872|m|m|s|u|sdw46|sdw46|8872|/data/mirror872 +1875|873|m|m|s|u|sdw46|sdw46|8873|/data/mirror873 +1876|874|m|m|s|u|sdw46|sdw46|8874|/data/mirror874 +1877|875|m|m|s|u|sdw46|sdw46|8875|/data/mirror875 +1878|876|m|m|s|u|sdw46|sdw46|8876|/data/mirror876 +1879|877|m|m|s|u|sdw46|sdw46|8877|/data/mirror877 +1880|878|m|m|s|u|sdw46|sdw46|8878|/data/mirror878 +1881|879|m|m|s|u|sdw46|sdw46|8879|/data/mirror879 +1882|880|m|m|s|u|sdw46|sdw46|8880|/data/mirror880 +1883|881|m|m|s|u|sdw46|sdw46|8881|/data/mirror881 +1884|882|m|m|s|u|sdw46|sdw46|8882|/data/mirror882 +1885|883|m|m|s|u|sdw46|sdw46|8883|/data/mirror883 +1886|884|m|m|s|u|sdw46|sdw46|8884|/data/mirror884 +1887|885|m|m|s|u|sdw46|sdw46|8885|/data/mirror885 +1888|886|m|m|s|u|sdw46|sdw46|8886|/data/mirror886 +1889|887|m|m|s|u|sdw46|sdw46|8887|/data/mirror887 +1890|888|m|m|s|u|sdw46|sdw46|8888|/data/mirror888 +1891|889|m|m|s|u|sdw46|sdw46|8889|/data/mirror889 +1892|890|m|m|s|u|sdw46|sdw46|8890|/data/mirror890 +1893|891|m|m|s|u|sdw46|sdw46|8891|/data/mirror891 +1894|892|m|m|s|u|sdw46|sdw46|8892|/data/mirror892 +# SDW47 +931|929|p|p|s|u|sdw47|sdw47|7929|/data/primary929 +932|930|p|p|s|u|sdw47|sdw47|7930|/data/primary930 +933|931|p|p|s|u|sdw47|sdw47|7931|/data/primary931 +934|932|p|p|s|u|sdw47|sdw47|7932|/data/primary932 +935|933|p|p|s|u|sdw47|sdw47|7933|/data/primary933 +936|934|p|p|s|u|sdw47|sdw47|7934|/data/primary934 +937|935|p|p|s|u|sdw47|sdw47|7935|/data/primary935 +938|936|p|p|s|u|sdw47|sdw47|7936|/data/primary936 +939|937|p|p|s|u|sdw47|sdw47|7937|/data/primary937 +940|938|p|p|s|u|sdw47|sdw47|7938|/data/primary938 +941|939|p|p|s|u|sdw47|sdw47|7939|/data/primary939 +942|940|p|p|s|u|sdw47|sdw47|7940|/data/primary940 +943|941|p|p|s|u|sdw47|sdw47|7941|/data/primary941 +944|942|p|p|s|u|sdw47|sdw47|7942|/data/primary942 +945|943|p|p|s|u|sdw47|sdw47|7943|/data/primary943 +946|944|p|p|s|u|sdw47|sdw47|7944|/data/primary944 +947|945|p|p|s|u|sdw47|sdw47|7945|/data/primary945 +1895|893|m|m|s|u|sdw47|sdw47|8893|/data/mirror893 +1896|894|m|m|s|u|sdw47|sdw47|8894|/data/mirror894 +1897|895|m|m|s|u|sdw47|sdw47|8895|/data/mirror895 +1898|896|m|m|s|u|sdw47|sdw47|8896|/data/mirror896 +1899|897|m|m|s|u|sdw47|sdw47|8897|/data/mirror897 +1900|898|m|m|s|u|sdw47|sdw47|8898|/data/mirror898 +1901|899|m|m|s|u|sdw47|sdw47|8899|/data/mirror899 +1902|900|m|m|s|u|sdw47|sdw47|8900|/data/mirror900 +1903|901|m|m|s|u|sdw47|sdw47|8901|/data/mirror901 +1904|902|m|m|s|u|sdw47|sdw47|8902|/data/mirror902 +1905|903|m|m|s|u|sdw47|sdw47|8903|/data/mirror903 +1906|904|m|m|s|u|sdw47|sdw47|8904|/data/mirror904 +1907|905|m|m|s|u|sdw47|sdw47|8905|/data/mirror905 +1908|906|m|m|s|u|sdw47|sdw47|8906|/data/mirror906 +1909|907|m|m|s|u|sdw47|sdw47|8907|/data/mirror907 +1910|908|m|m|s|u|sdw47|sdw47|8908|/data/mirror908 +1911|909|m|m|s|u|sdw47|sdw47|8909|/data/mirror909 +1912|910|m|m|s|u|sdw47|sdw47|8910|/data/mirror910 +1913|911|m|m|s|u|sdw47|sdw47|8911|/data/mirror911 +# SDW48 +948|946|p|p|s|u|sdw48|sdw48|7946|/data/primary946 +949|947|p|p|s|u|sdw48|sdw48|7947|/data/primary947 +950|948|p|p|s|u|sdw48|sdw48|7948|/data/primary948 +951|949|p|p|s|u|sdw48|sdw48|7949|/data/primary949 +952|950|p|p|s|u|sdw48|sdw48|7950|/data/primary950 +953|951|p|p|s|u|sdw48|sdw48|7951|/data/primary951 +954|952|p|p|s|u|sdw48|sdw48|7952|/data/primary952 +955|953|p|p|s|u|sdw48|sdw48|7953|/data/primary953 +956|954|p|p|s|u|sdw48|sdw48|7954|/data/primary954 +957|955|p|p|s|u|sdw48|sdw48|7955|/data/primary955 +958|956|p|p|s|u|sdw48|sdw48|7956|/data/primary956 +959|957|p|p|s|u|sdw48|sdw48|7957|/data/primary957 +960|958|p|p|s|u|sdw48|sdw48|7958|/data/primary958 +961|959|p|p|s|u|sdw48|sdw48|7959|/data/primary959 +962|960|p|p|s|u|sdw48|sdw48|7960|/data/primary960 +963|961|p|p|s|u|sdw48|sdw48|7961|/data/primary961 +964|962|p|p|s|u|sdw48|sdw48|7962|/data/primary962 +1914|912|m|m|s|u|sdw48|sdw48|8912|/data/mirror912 +1915|913|m|m|s|u|sdw48|sdw48|8913|/data/mirror913 +1916|914|m|m|s|u|sdw48|sdw48|8914|/data/mirror914 +1917|915|m|m|s|u|sdw48|sdw48|8915|/data/mirror915 +1918|916|m|m|s|u|sdw48|sdw48|8916|/data/mirror916 +1919|917|m|m|s|u|sdw48|sdw48|8917|/data/mirror917 +1920|918|m|m|s|u|sdw48|sdw48|8918|/data/mirror918 +1921|919|m|m|s|u|sdw48|sdw48|8919|/data/mirror919 +1922|920|m|m|s|u|sdw48|sdw48|8920|/data/mirror920 +1923|921|m|m|s|u|sdw48|sdw48|8921|/data/mirror921 +1924|922|m|m|s|u|sdw48|sdw48|8922|/data/mirror922 +1925|923|m|m|s|u|sdw48|sdw48|8923|/data/mirror923 +1926|924|m|m|s|u|sdw48|sdw48|8924|/data/mirror924 +1927|925|m|m|s|u|sdw48|sdw48|8925|/data/mirror925 +1928|926|m|m|s|u|sdw48|sdw48|8926|/data/mirror926 +1929|927|m|m|s|u|sdw48|sdw48|8927|/data/mirror927 +1930|928|m|m|s|u|sdw48|sdw48|8928|/data/mirror928 +# SDW49 +965|963|p|p|s|u|sdw49|sdw49|7963|/data/primary963 +966|964|p|p|s|u|sdw49|sdw49|7964|/data/primary964 +967|965|p|p|s|u|sdw49|sdw49|7965|/data/primary965 +968|966|p|p|s|u|sdw49|sdw49|7966|/data/primary966 +969|967|p|p|s|u|sdw49|sdw49|7967|/data/primary967 +970|968|p|p|s|u|sdw49|sdw49|7968|/data/primary968 +971|969|p|p|s|u|sdw49|sdw49|7969|/data/primary969 +972|970|p|p|s|u|sdw49|sdw49|7970|/data/primary970 +973|971|p|p|s|u|sdw49|sdw49|7971|/data/primary971 +974|972|p|p|s|u|sdw49|sdw49|7972|/data/primary972 +975|973|p|p|s|u|sdw49|sdw49|7973|/data/primary973 +976|974|p|p|s|u|sdw49|sdw49|7974|/data/primary974 +977|975|p|p|s|u|sdw49|sdw49|7975|/data/primary975 +978|976|p|p|s|u|sdw49|sdw49|7976|/data/primary976 +979|977|p|p|s|u|sdw49|sdw49|7977|/data/primary977 +980|978|p|p|s|u|sdw49|sdw49|7978|/data/primary978 +981|979|p|p|s|u|sdw49|sdw49|7979|/data/primary979 +982|980|p|p|s|u|sdw49|sdw49|7980|/data/primary980 +983|981|p|p|s|u|sdw49|sdw49|7981|/data/primary981 +984|982|p|p|s|u|sdw49|sdw49|7982|/data/primary982 +985|983|p|p|s|u|sdw49|sdw49|7983|/data/primary983 +1948|946|m|m|s|u|sdw49|sdw49|8946|/data/mirror946 +1949|947|m|m|s|u|sdw49|sdw49|8947|/data/mirror947 +1950|948|m|m|s|u|sdw49|sdw49|8948|/data/mirror948 +1951|949|m|m|s|u|sdw49|sdw49|8949|/data/mirror949 +1952|950|m|m|s|u|sdw49|sdw49|8950|/data/mirror950 +1953|951|m|m|s|u|sdw49|sdw49|8951|/data/mirror951 +1954|952|m|m|s|u|sdw49|sdw49|8952|/data/mirror952 +1955|953|m|m|s|u|sdw49|sdw49|8953|/data/mirror953 +1956|954|m|m|s|u|sdw49|sdw49|8954|/data/mirror954 +1957|955|m|m|s|u|sdw49|sdw49|8955|/data/mirror955 +1958|956|m|m|s|u|sdw49|sdw49|8956|/data/mirror956 +1959|957|m|m|s|u|sdw49|sdw49|8957|/data/mirror957 +1960|958|m|m|s|u|sdw49|sdw49|8958|/data/mirror958 +1961|959|m|m|s|u|sdw49|sdw49|8959|/data/mirror959 +1962|960|m|m|s|u|sdw49|sdw49|8960|/data/mirror960 +1963|961|m|m|s|u|sdw49|sdw49|8961|/data/mirror961 +1964|962|m|m|s|u|sdw49|sdw49|8962|/data/mirror962 +# SDW5 +74|72|p|p|s|u|sdw5|sdw5|7072|/data/primary72 +75|73|p|p|s|u|sdw5|sdw5|7073|/data/primary73 +76|74|p|p|s|u|sdw5|sdw5|7074|/data/primary74 +77|75|p|p|s|u|sdw5|sdw5|7075|/data/primary75 +78|76|p|p|s|u|sdw5|sdw5|7076|/data/primary76 +79|77|p|p|s|u|sdw5|sdw5|7077|/data/primary77 +80|78|p|p|s|u|sdw5|sdw5|7078|/data/primary78 +81|79|p|p|s|u|sdw5|sdw5|7079|/data/primary79 +82|80|p|p|s|u|sdw5|sdw5|7080|/data/primary80 +83|81|p|p|s|u|sdw5|sdw5|7081|/data/primary81 +84|82|p|p|s|u|sdw5|sdw5|7082|/data/primary82 +85|83|p|p|s|u|sdw5|sdw5|7083|/data/primary83 +86|84|p|p|s|u|sdw5|sdw5|7084|/data/primary84 +87|85|p|p|s|u|sdw5|sdw5|7085|/data/primary85 +88|86|p|p|s|u|sdw5|sdw5|7086|/data/primary86 +89|87|p|p|s|u|sdw5|sdw5|7087|/data/primary87 +90|88|p|p|s|u|sdw5|sdw5|7088|/data/primary88 +91|89|p|p|s|u|sdw5|sdw5|7089|/data/primary89 +92|90|p|p|s|u|sdw5|sdw5|7090|/data/primary90 +93|91|p|p|s|u|sdw5|sdw5|7091|/data/primary91 +94|92|p|p|s|u|sdw5|sdw5|7092|/data/primary92 +95|93|p|p|s|u|sdw5|sdw5|7093|/data/primary93 +96|94|p|p|s|u|sdw5|sdw5|7094|/data/primary94 +1056|54|m|m|s|u|sdw5|sdw5|8054|/data/mirror54 +1057|55|m|m|s|u|sdw5|sdw5|8055|/data/mirror55 +1058|56|m|m|s|u|sdw5|sdw5|8056|/data/mirror56 +1059|57|m|m|s|u|sdw5|sdw5|8057|/data/mirror57 +1060|58|m|m|s|u|sdw5|sdw5|8058|/data/mirror58 +1061|59|m|m|s|u|sdw5|sdw5|8059|/data/mirror59 +1062|60|m|m|s|u|sdw5|sdw5|8060|/data/mirror60 +1063|61|m|m|s|u|sdw5|sdw5|8061|/data/mirror61 +1064|62|m|m|s|u|sdw5|sdw5|8062|/data/mirror62 +1065|63|m|m|s|u|sdw5|sdw5|8063|/data/mirror63 +1066|64|m|m|s|u|sdw5|sdw5|8064|/data/mirror64 +1067|65|m|m|s|u|sdw5|sdw5|8065|/data/mirror65 +1068|66|m|m|s|u|sdw5|sdw5|8066|/data/mirror66 +1069|67|m|m|s|u|sdw5|sdw5|8067|/data/mirror67 +1070|68|m|m|s|u|sdw5|sdw5|8068|/data/mirror68 +1071|69|m|m|s|u|sdw5|sdw5|8069|/data/mirror69 +1072|70|m|m|s|u|sdw5|sdw5|8070|/data/mirror70 +1073|71|m|m|s|u|sdw5|sdw5|8071|/data/mirror71 +# SDW50 +986|984|p|p|s|u|sdw50|sdw50|7984|/data/primary984 +987|985|p|p|s|u|sdw50|sdw50|7985|/data/primary985 +988|986|p|p|s|u|sdw50|sdw50|7986|/data/primary986 +989|987|p|p|s|u|sdw50|sdw50|7987|/data/primary987 +990|988|p|p|s|u|sdw50|sdw50|7988|/data/primary988 +991|989|p|p|s|u|sdw50|sdw50|7989|/data/primary989 +992|990|p|p|s|u|sdw50|sdw50|7990|/data/primary990 +993|991|p|p|s|u|sdw50|sdw50|7991|/data/primary991 +994|992|p|p|s|u|sdw50|sdw50|7992|/data/primary992 +995|993|p|p|s|u|sdw50|sdw50|7993|/data/primary993 +996|994|p|p|s|u|sdw50|sdw50|7994|/data/primary994 +997|995|p|p|s|u|sdw50|sdw50|7995|/data/primary995 +998|996|p|p|s|u|sdw50|sdw50|7996|/data/primary996 +999|997|p|p|s|u|sdw50|sdw50|7997|/data/primary997 +1000|998|p|p|s|u|sdw50|sdw50|7998|/data/primary998 +1001|999|p|p|s|u|sdw50|sdw50|7999|/data/primary999 +# SDW6 +97|95|p|p|s|u|sdw6|sdw6|7095|/data/primary95 +98|96|p|p|s|u|sdw6|sdw6|7096|/data/primary96 +99|97|p|p|s|u|sdw6|sdw6|7097|/data/primary97 +100|98|p|p|s|u|sdw6|sdw6|7098|/data/primary98 +101|99|p|p|s|u|sdw6|sdw6|7099|/data/primary99 +102|100|p|p|s|u|sdw6|sdw6|7100|/data/primary100 +103|101|p|p|s|u|sdw6|sdw6|7101|/data/primary101 +104|102|p|p|s|u|sdw6|sdw6|7102|/data/primary102 +105|103|p|p|s|u|sdw6|sdw6|7103|/data/primary103 +106|104|p|p|s|u|sdw6|sdw6|7104|/data/primary104 +107|105|p|p|s|u|sdw6|sdw6|7105|/data/primary105 +108|106|p|p|s|u|sdw6|sdw6|7106|/data/primary106 +109|107|p|p|s|u|sdw6|sdw6|7107|/data/primary107 +110|108|p|p|s|u|sdw6|sdw6|7108|/data/primary108 +111|109|p|p|s|u|sdw6|sdw6|7109|/data/primary109 +112|110|p|p|s|u|sdw6|sdw6|7110|/data/primary110 +113|111|p|p|s|u|sdw6|sdw6|7111|/data/primary111 +1074|72|m|m|s|u|sdw6|sdw6|8072|/data/mirror72 +1075|73|m|m|s|u|sdw6|sdw6|8073|/data/mirror73 +1076|74|m|m|s|u|sdw6|sdw6|8074|/data/mirror74 +1077|75|m|m|s|u|sdw6|sdw6|8075|/data/mirror75 +1078|76|m|m|s|u|sdw6|sdw6|8076|/data/mirror76 +1079|77|m|m|s|u|sdw6|sdw6|8077|/data/mirror77 +1080|78|m|m|s|u|sdw6|sdw6|8078|/data/mirror78 +1081|79|m|m|s|u|sdw6|sdw6|8079|/data/mirror79 +1082|80|m|m|s|u|sdw6|sdw6|8080|/data/mirror80 +1083|81|m|m|s|u|sdw6|sdw6|8081|/data/mirror81 +1084|82|m|m|s|u|sdw6|sdw6|8082|/data/mirror82 +1085|83|m|m|s|u|sdw6|sdw6|8083|/data/mirror83 +1086|84|m|m|s|u|sdw6|sdw6|8084|/data/mirror84 +1087|85|m|m|s|u|sdw6|sdw6|8085|/data/mirror85 +1088|86|m|m|s|u|sdw6|sdw6|8086|/data/mirror86 +1089|87|m|m|s|u|sdw6|sdw6|8087|/data/mirror87 +1090|88|m|m|s|u|sdw6|sdw6|8088|/data/mirror88 +1091|89|m|m|s|u|sdw6|sdw6|8089|/data/mirror89 +1092|90|m|m|s|u|sdw6|sdw6|8090|/data/mirror90 +1093|91|m|m|s|u|sdw6|sdw6|8091|/data/mirror91 +1094|92|m|m|s|u|sdw6|sdw6|8092|/data/mirror92 +1095|93|m|m|s|u|sdw6|sdw6|8093|/data/mirror93 +1096|94|m|m|s|u|sdw6|sdw6|8094|/data/mirror94 +# SDW7 +114|112|p|p|s|u|sdw7|sdw7|7112|/data/primary112 +115|113|p|p|s|u|sdw7|sdw7|7113|/data/primary113 +116|114|p|p|s|u|sdw7|sdw7|7114|/data/primary114 +117|115|p|p|s|u|sdw7|sdw7|7115|/data/primary115 +118|116|p|p|s|u|sdw7|sdw7|7116|/data/primary116 +119|117|p|p|s|u|sdw7|sdw7|7117|/data/primary117 +120|118|p|p|s|u|sdw7|sdw7|7118|/data/primary118 +121|119|p|p|s|u|sdw7|sdw7|7119|/data/primary119 +122|120|p|p|s|u|sdw7|sdw7|7120|/data/primary120 +123|121|p|p|s|u|sdw7|sdw7|7121|/data/primary121 +124|122|p|p|s|u|sdw7|sdw7|7122|/data/primary122 +125|123|p|p|s|u|sdw7|sdw7|7123|/data/primary123 +126|124|p|p|s|u|sdw7|sdw7|7124|/data/primary124 +127|125|p|p|s|u|sdw7|sdw7|7125|/data/primary125 +128|126|p|p|s|u|sdw7|sdw7|7126|/data/primary126 +129|127|p|p|s|u|sdw7|sdw7|7127|/data/primary127 +130|128|p|p|s|u|sdw7|sdw7|7128|/data/primary128 +131|129|p|p|s|u|sdw7|sdw7|7129|/data/primary129 +1175|173|m|m|s|u|sdw7|sdw7|8173|/data/mirror173 +1176|174|m|m|s|u|sdw7|sdw7|8174|/data/mirror174 +1177|175|m|m|s|u|sdw7|sdw7|8175|/data/mirror175 +1178|176|m|m|s|u|sdw7|sdw7|8176|/data/mirror176 +1179|177|m|m|s|u|sdw7|sdw7|8177|/data/mirror177 +1180|178|m|m|s|u|sdw7|sdw7|8178|/data/mirror178 +1181|179|m|m|s|u|sdw7|sdw7|8179|/data/mirror179 +1182|180|m|m|s|u|sdw7|sdw7|8180|/data/mirror180 +1183|181|m|m|s|u|sdw7|sdw7|8181|/data/mirror181 +1184|182|m|m|s|u|sdw7|sdw7|8182|/data/mirror182 +1185|183|m|m|s|u|sdw7|sdw7|8183|/data/mirror183 +1186|184|m|m|s|u|sdw7|sdw7|8184|/data/mirror184 +1187|185|m|m|s|u|sdw7|sdw7|8185|/data/mirror185 +1188|186|m|m|s|u|sdw7|sdw7|8186|/data/mirror186 +1189|187|m|m|s|u|sdw7|sdw7|8187|/data/mirror187 +1190|188|m|m|s|u|sdw7|sdw7|8188|/data/mirror188 +1191|189|m|m|s|u|sdw7|sdw7|8189|/data/mirror189 +1192|190|m|m|s|u|sdw7|sdw7|8190|/data/mirror190 +1193|191|m|m|s|u|sdw7|sdw7|8191|/data/mirror191 +1194|192|m|m|s|u|sdw7|sdw7|8192|/data/mirror192 +1195|193|m|m|s|u|sdw7|sdw7|8193|/data/mirror193 +1196|194|m|m|s|u|sdw7|sdw7|8194|/data/mirror194 +1197|195|m|m|s|u|sdw7|sdw7|8195|/data/mirror195 +1198|196|m|m|s|u|sdw7|sdw7|8196|/data/mirror196 +# SDW8 +132|130|p|p|s|u|sdw8|sdw8|7130|/data/primary130 +133|131|p|p|s|u|sdw8|sdw8|7131|/data/primary131 +134|132|p|p|s|u|sdw8|sdw8|7132|/data/primary132 +135|133|p|p|s|u|sdw8|sdw8|7133|/data/primary133 +136|134|p|p|s|u|sdw8|sdw8|7134|/data/primary134 +137|135|p|p|s|u|sdw8|sdw8|7135|/data/primary135 +138|136|p|p|s|u|sdw8|sdw8|7136|/data/primary136 +139|137|p|p|s|u|sdw8|sdw8|7137|/data/primary137 +140|138|p|p|s|u|sdw8|sdw8|7138|/data/primary138 +141|139|p|p|s|u|sdw8|sdw8|7139|/data/primary139 +142|140|p|p|s|u|sdw8|sdw8|7140|/data/primary140 +143|141|p|p|s|u|sdw8|sdw8|7141|/data/primary141 +144|142|p|p|s|u|sdw8|sdw8|7142|/data/primary142 +145|143|p|p|s|u|sdw8|sdw8|7143|/data/primary143 +146|144|p|p|s|u|sdw8|sdw8|7144|/data/primary144 +147|145|p|p|s|u|sdw8|sdw8|7145|/data/primary145 +148|146|p|p|s|u|sdw8|sdw8|7146|/data/primary146 +149|147|p|p|s|u|sdw8|sdw8|7147|/data/primary147 +150|148|p|p|s|u|sdw8|sdw8|7148|/data/primary148 +151|149|p|p|s|u|sdw8|sdw8|7149|/data/primary149 +1219|217|m|m|s|u|sdw8|sdw8|8217|/data/mirror217 +1220|218|m|m|s|u|sdw8|sdw8|8218|/data/mirror218 +1221|219|m|m|s|u|sdw8|sdw8|8219|/data/mirror219 +1222|220|m|m|s|u|sdw8|sdw8|8220|/data/mirror220 +1223|221|m|m|s|u|sdw8|sdw8|8221|/data/mirror221 +1224|222|m|m|s|u|sdw8|sdw8|8222|/data/mirror222 +1225|223|m|m|s|u|sdw8|sdw8|8223|/data/mirror223 +1226|224|m|m|s|u|sdw8|sdw8|8224|/data/mirror224 +1227|225|m|m|s|u|sdw8|sdw8|8225|/data/mirror225 +1228|226|m|m|s|u|sdw8|sdw8|8226|/data/mirror226 +1229|227|m|m|s|u|sdw8|sdw8|8227|/data/mirror227 +1230|228|m|m|s|u|sdw8|sdw8|8228|/data/mirror228 +1231|229|m|m|s|u|sdw8|sdw8|8229|/data/mirror229 +1232|230|m|m|s|u|sdw8|sdw8|8230|/data/mirror230 +1233|231|m|m|s|u|sdw8|sdw8|8231|/data/mirror231 +1234|232|m|m|s|u|sdw8|sdw8|8232|/data/mirror232 +1235|233|m|m|s|u|sdw8|sdw8|8233|/data/mirror233 +1236|234|m|m|s|u|sdw8|sdw8|8234|/data/mirror234 +1237|235|m|m|s|u|sdw8|sdw8|8235|/data/mirror235 +1238|236|m|m|s|u|sdw8|sdw8|8236|/data/mirror236 +1239|237|m|m|s|u|sdw8|sdw8|8237|/data/mirror237 +1240|238|m|m|s|u|sdw8|sdw8|8238|/data/mirror238 +# SDW9 +152|150|p|p|s|u|sdw9|sdw9|7150|/data/primary150 +153|151|p|p|s|u|sdw9|sdw9|7151|/data/primary151 +154|152|p|p|s|u|sdw9|sdw9|7152|/data/primary152 +155|153|p|p|s|u|sdw9|sdw9|7153|/data/primary153 +156|154|p|p|s|u|sdw9|sdw9|7154|/data/primary154 +157|155|p|p|s|u|sdw9|sdw9|7155|/data/primary155 +158|156|p|p|s|u|sdw9|sdw9|7156|/data/primary156 +159|157|p|p|s|u|sdw9|sdw9|7157|/data/primary157 +160|158|p|p|s|u|sdw9|sdw9|7158|/data/primary158 +161|159|p|p|s|u|sdw9|sdw9|7159|/data/primary159 +162|160|p|p|s|u|sdw9|sdw9|7160|/data/primary160 +163|161|p|p|s|u|sdw9|sdw9|7161|/data/primary161 +164|162|p|p|s|u|sdw9|sdw9|7162|/data/primary162 +165|163|p|p|s|u|sdw9|sdw9|7163|/data/primary163 +166|164|p|p|s|u|sdw9|sdw9|7164|/data/primary164 +167|165|p|p|s|u|sdw9|sdw9|7165|/data/primary165 +168|166|p|p|s|u|sdw9|sdw9|7166|/data/primary166 +169|167|p|p|s|u|sdw9|sdw9|7167|/data/primary167 +170|168|p|p|s|u|sdw9|sdw9|7168|/data/primary168 +171|169|p|p|s|u|sdw9|sdw9|7169|/data/primary169 +172|170|p|p|s|u|sdw9|sdw9|7170|/data/primary170 +173|171|p|p|s|u|sdw9|sdw9|7171|/data/primary171 +174|172|p|p|s|u|sdw9|sdw9|7172|/data/primary172 +1114|112|m|m|s|u|sdw9|sdw9|8112|/data/mirror112 +1115|113|m|m|s|u|sdw9|sdw9|8113|/data/mirror113 +1116|114|m|m|s|u|sdw9|sdw9|8114|/data/mirror114 +1117|115|m|m|s|u|sdw9|sdw9|8115|/data/mirror115 +1118|116|m|m|s|u|sdw9|sdw9|8116|/data/mirror116 +1119|117|m|m|s|u|sdw9|sdw9|8117|/data/mirror117 +1120|118|m|m|s|u|sdw9|sdw9|8118|/data/mirror118 +1121|119|m|m|s|u|sdw9|sdw9|8119|/data/mirror119 +1122|120|m|m|s|u|sdw9|sdw9|8120|/data/mirror120 +1123|121|m|m|s|u|sdw9|sdw9|8121|/data/mirror121 +1124|122|m|m|s|u|sdw9|sdw9|8122|/data/mirror122 +1125|123|m|m|s|u|sdw9|sdw9|8123|/data/mirror123 +1126|124|m|m|s|u|sdw9|sdw9|8124|/data/mirror124 +1127|125|m|m|s|u|sdw9|sdw9|8125|/data/mirror125 +1128|126|m|m|s|u|sdw9|sdw9|8126|/data/mirror126 +1129|127|m|m|s|u|sdw9|sdw9|8127|/data/mirror127 +1130|128|m|m|s|u|sdw9|sdw9|8128|/data/mirror128 +1131|129|m|m|s|u|sdw9|sdw9|8129|/data/mirror129 diff --git a/gpMgmt/bin/gprebalance_modules/test/data/1000_50_unbalanced_spread.array b/gpMgmt/bin/gprebalance_modules/test/data/1000_50_unbalanced_spread.array new file mode 100644 index 000000000000..38e9b4295181 --- /dev/null +++ b/gpMgmt/bin/gprebalance_modules/test/data/1000_50_unbalanced_spread.array @@ -0,0 +1,2052 @@ +# Coordinator +1|-1|p|p|s|u|cdw|cdw|5432|/data/coordinator +# SDW1 +2|0|p|p|s|u|sdw1|sdw1|7000|/data/primary0 +3|1|p|p|s|u|sdw1|sdw1|7001|/data/primary1 +4|2|p|p|s|u|sdw1|sdw1|7002|/data/primary2 +5|3|p|p|s|u|sdw1|sdw1|7003|/data/primary3 +6|4|p|p|s|u|sdw1|sdw1|7004|/data/primary4 +7|5|p|p|s|u|sdw1|sdw1|7005|/data/primary5 +8|6|p|p|s|u|sdw1|sdw1|7006|/data/primary6 +9|7|p|p|s|u|sdw1|sdw1|7007|/data/primary7 +10|8|p|p|s|u|sdw1|sdw1|7008|/data/primary8 +11|9|p|p|s|u|sdw1|sdw1|7009|/data/primary9 +12|10|p|p|s|u|sdw1|sdw1|7010|/data/primary10 +13|11|p|p|s|u|sdw1|sdw1|7011|/data/primary11 +14|12|p|p|s|u|sdw1|sdw1|7012|/data/primary12 +15|13|p|p|s|u|sdw1|sdw1|7013|/data/primary13 +16|14|p|p|s|u|sdw1|sdw1|7014|/data/primary14 +17|15|p|p|s|u|sdw1|sdw1|7015|/data/primary15 +18|16|p|p|s|u|sdw1|sdw1|7016|/data/primary16 +19|17|p|p|s|u|sdw1|sdw1|7017|/data/primary17 +20|18|p|p|s|u|sdw1|sdw1|7018|/data/primary18 +21|19|p|p|s|u|sdw1|sdw1|7019|/data/primary19 +22|20|p|p|s|u|sdw1|sdw1|7020|/data/primary20 +23|21|p|p|s|u|sdw1|sdw1|7021|/data/primary21 +24|22|p|p|s|u|sdw1|sdw1|7022|/data/primary22 +25|23|p|p|s|u|sdw1|sdw1|7023|/data/primary23 +1026|24|m|m|s|u|sdw1|sdw1|8024|/data/mirror24 +1057|55|m|m|s|u|sdw1|sdw1|8055|/data/mirror55 +1099|97|m|m|s|u|sdw1|sdw1|8097|/data/mirror97 +1134|132|m|m|s|u|sdw1|sdw1|8132|/data/mirror132 +1208|206|m|m|s|u|sdw1|sdw1|8206|/data/mirror206 +1253|251|m|m|s|u|sdw1|sdw1|8251|/data/mirror251 +1288|286|m|m|s|u|sdw1|sdw1|8286|/data/mirror286 +1334|332|m|m|s|u|sdw1|sdw1|8332|/data/mirror332 +1403|401|m|m|s|u|sdw1|sdw1|8401|/data/mirror401 +1453|451|m|m|s|u|sdw1|sdw1|8451|/data/mirror451 +1508|506|m|m|s|u|sdw1|sdw1|8506|/data/mirror506 +1557|555|m|m|s|u|sdw1|sdw1|8555|/data/mirror555 +1591|589|m|m|s|u|sdw1|sdw1|8589|/data/mirror589 +1657|655|m|m|s|u|sdw1|sdw1|8655|/data/mirror655 +1710|708|m|m|s|u|sdw1|sdw1|8708|/data/mirror708 +1755|753|m|m|s|u|sdw1|sdw1|8753|/data/mirror753 +1805|803|m|m|s|u|sdw1|sdw1|8803|/data/mirror803 +1848|846|m|m|s|u|sdw1|sdw1|8846|/data/mirror846 +1906|904|m|m|s|u|sdw1|sdw1|8904|/data/mirror904 +1939|937|m|m|s|u|sdw1|sdw1|8937|/data/mirror937 +# SDW10 +193|191|p|p|s|u|sdw10|sdw10|7191|/data/primary191 +194|192|p|p|s|u|sdw10|sdw10|7192|/data/primary192 +195|193|p|p|s|u|sdw10|sdw10|7193|/data/primary193 +196|194|p|p|s|u|sdw10|sdw10|7194|/data/primary194 +197|195|p|p|s|u|sdw10|sdw10|7195|/data/primary195 +198|196|p|p|s|u|sdw10|sdw10|7196|/data/primary196 +199|197|p|p|s|u|sdw10|sdw10|7197|/data/primary197 +200|198|p|p|s|u|sdw10|sdw10|7198|/data/primary198 +201|199|p|p|s|u|sdw10|sdw10|7199|/data/primary199 +202|200|p|p|s|u|sdw10|sdw10|7200|/data/primary200 +203|201|p|p|s|u|sdw10|sdw10|7201|/data/primary201 +204|202|p|p|s|u|sdw10|sdw10|7202|/data/primary202 +205|203|p|p|s|u|sdw10|sdw10|7203|/data/primary203 +206|204|p|p|s|u|sdw10|sdw10|7204|/data/primary204 +207|205|p|p|s|u|sdw10|sdw10|7205|/data/primary205 +208|206|p|p|s|u|sdw10|sdw10|7206|/data/primary206 +209|207|p|p|s|u|sdw10|sdw10|7207|/data/primary207 +210|208|p|p|s|u|sdw10|sdw10|7208|/data/primary208 +211|209|p|p|s|u|sdw10|sdw10|7209|/data/primary209 +212|210|p|p|s|u|sdw10|sdw10|7210|/data/primary210 +213|211|p|p|s|u|sdw10|sdw10|7211|/data/primary211 +214|212|p|p|s|u|sdw10|sdw10|7212|/data/primary212 +1006|4|m|m|s|u|sdw10|sdw10|8004|/data/mirror4 +1043|41|m|m|s|u|sdw10|sdw10|8041|/data/mirror41 +1103|101|m|m|s|u|sdw10|sdw10|8101|/data/mirror101 +1155|153|m|m|s|u|sdw10|sdw10|8153|/data/mirror153 +1218|216|m|m|s|u|sdw10|sdw10|8216|/data/mirror216 +1259|257|m|m|s|u|sdw10|sdw10|8257|/data/mirror257 +1312|310|m|m|s|u|sdw10|sdw10|8310|/data/mirror310 +1365|363|m|m|s|u|sdw10|sdw10|8363|/data/mirror363 +1413|411|m|m|s|u|sdw10|sdw10|8411|/data/mirror411 +1454|452|m|m|s|u|sdw10|sdw10|8452|/data/mirror452 +1521|519|m|m|s|u|sdw10|sdw10|8519|/data/mirror519 +1564|562|m|m|s|u|sdw10|sdw10|8562|/data/mirror562 +1613|611|m|m|s|u|sdw10|sdw10|8611|/data/mirror611 +1668|666|m|m|s|u|sdw10|sdw10|8666|/data/mirror666 +1717|715|m|m|s|u|sdw10|sdw10|8715|/data/mirror715 +1757|755|m|m|s|u|sdw10|sdw10|8755|/data/mirror755 +1814|812|m|m|s|u|sdw10|sdw10|8812|/data/mirror812 +1865|863|m|m|s|u|sdw10|sdw10|8863|/data/mirror863 +1888|886|m|m|s|u|sdw10|sdw10|8886|/data/mirror886 +1966|964|m|m|s|u|sdw10|sdw10|8964|/data/mirror964 +# SDW11 +215|213|p|p|s|u|sdw11|sdw11|7213|/data/primary213 +216|214|p|p|s|u|sdw11|sdw11|7214|/data/primary214 +217|215|p|p|s|u|sdw11|sdw11|7215|/data/primary215 +218|216|p|p|s|u|sdw11|sdw11|7216|/data/primary216 +219|217|p|p|s|u|sdw11|sdw11|7217|/data/primary217 +220|218|p|p|s|u|sdw11|sdw11|7218|/data/primary218 +221|219|p|p|s|u|sdw11|sdw11|7219|/data/primary219 +222|220|p|p|s|u|sdw11|sdw11|7220|/data/primary220 +223|221|p|p|s|u|sdw11|sdw11|7221|/data/primary221 +224|222|p|p|s|u|sdw11|sdw11|7222|/data/primary222 +225|223|p|p|s|u|sdw11|sdw11|7223|/data/primary223 +226|224|p|p|s|u|sdw11|sdw11|7224|/data/primary224 +227|225|p|p|s|u|sdw11|sdw11|7225|/data/primary225 +228|226|p|p|s|u|sdw11|sdw11|7226|/data/primary226 +229|227|p|p|s|u|sdw11|sdw11|7227|/data/primary227 +230|228|p|p|s|u|sdw11|sdw11|7228|/data/primary228 +231|229|p|p|s|u|sdw11|sdw11|7229|/data/primary229 +232|230|p|p|s|u|sdw11|sdw11|7230|/data/primary230 +233|231|p|p|s|u|sdw11|sdw11|7231|/data/primary231 +1013|11|m|m|s|u|sdw11|sdw11|8011|/data/mirror11 +1065|63|m|m|s|u|sdw11|sdw11|8063|/data/mirror63 +1118|116|m|m|s|u|sdw11|sdw11|8116|/data/mirror116 +1163|161|m|m|s|u|sdw11|sdw11|8161|/data/mirror161 +1202|200|m|m|s|u|sdw11|sdw11|8200|/data/mirror200 +1263|261|m|m|s|u|sdw11|sdw11|8261|/data/mirror261 +1313|311|m|m|s|u|sdw11|sdw11|8311|/data/mirror311 +1339|337|m|m|s|u|sdw11|sdw11|8337|/data/mirror337 +1414|412|m|m|s|u|sdw11|sdw11|8412|/data/mirror412 +1470|468|m|m|s|u|sdw11|sdw11|8468|/data/mirror468 +1499|497|m|m|s|u|sdw11|sdw11|8497|/data/mirror497 +1565|563|m|m|s|u|sdw11|sdw11|8563|/data/mirror563 +1614|612|m|m|s|u|sdw11|sdw11|8612|/data/mirror612 +1654|652|m|m|s|u|sdw11|sdw11|8652|/data/mirror652 +1699|697|m|m|s|u|sdw11|sdw11|8697|/data/mirror697 +1765|763|m|m|s|u|sdw11|sdw11|8763|/data/mirror763 +1789|787|m|m|s|u|sdw11|sdw11|8787|/data/mirror787 +1867|865|m|m|s|u|sdw11|sdw11|8865|/data/mirror865 +1900|898|m|m|s|u|sdw11|sdw11|8898|/data/mirror898 +1967|965|m|m|s|u|sdw11|sdw11|8965|/data/mirror965 +# SDW12 +234|232|p|p|s|u|sdw12|sdw12|7232|/data/primary232 +235|233|p|p|s|u|sdw12|sdw12|7233|/data/primary233 +236|234|p|p|s|u|sdw12|sdw12|7234|/data/primary234 +237|235|p|p|s|u|sdw12|sdw12|7235|/data/primary235 +238|236|p|p|s|u|sdw12|sdw12|7236|/data/primary236 +239|237|p|p|s|u|sdw12|sdw12|7237|/data/primary237 +240|238|p|p|s|u|sdw12|sdw12|7238|/data/primary238 +241|239|p|p|s|u|sdw12|sdw12|7239|/data/primary239 +242|240|p|p|s|u|sdw12|sdw12|7240|/data/primary240 +243|241|p|p|s|u|sdw12|sdw12|7241|/data/primary241 +244|242|p|p|s|u|sdw12|sdw12|7242|/data/primary242 +245|243|p|p|s|u|sdw12|sdw12|7243|/data/primary243 +246|244|p|p|s|u|sdw12|sdw12|7244|/data/primary244 +247|245|p|p|s|u|sdw12|sdw12|7245|/data/primary245 +248|246|p|p|s|u|sdw12|sdw12|7246|/data/primary246 +249|247|p|p|s|u|sdw12|sdw12|7247|/data/primary247 +250|248|p|p|s|u|sdw12|sdw12|7248|/data/primary248 +251|249|p|p|s|u|sdw12|sdw12|7249|/data/primary249 +252|250|p|p|s|u|sdw12|sdw12|7250|/data/primary250 +253|251|p|p|s|u|sdw12|sdw12|7251|/data/primary251 +254|252|p|p|s|u|sdw12|sdw12|7252|/data/primary252 +255|253|p|p|s|u|sdw12|sdw12|7253|/data/primary253 +1007|5|m|m|s|u|sdw12|sdw12|8005|/data/mirror5 +1044|42|m|m|s|u|sdw12|sdw12|8042|/data/mirror42 +1119|117|m|m|s|u|sdw12|sdw12|8117|/data/mirror117 +1156|154|m|m|s|u|sdw12|sdw12|8154|/data/mirror154 +1220|218|m|m|s|u|sdw12|sdw12|8218|/data/mirror218 +1264|262|m|m|s|u|sdw12|sdw12|8262|/data/mirror262 +1315|313|m|m|s|u|sdw12|sdw12|8313|/data/mirror313 +1367|365|m|m|s|u|sdw12|sdw12|8365|/data/mirror365 +1415|413|m|m|s|u|sdw12|sdw12|8413|/data/mirror413 +1455|453|m|m|s|u|sdw12|sdw12|8453|/data/mirror453 +1522|520|m|m|s|u|sdw12|sdw12|8520|/data/mirror520 +1566|564|m|m|s|u|sdw12|sdw12|8564|/data/mirror564 +1615|613|m|m|s|u|sdw12|sdw12|8613|/data/mirror613 +1641|639|m|m|s|u|sdw12|sdw12|8639|/data/mirror639 +1718|716|m|m|s|u|sdw12|sdw12|8716|/data/mirror716 +1758|756|m|m|s|u|sdw12|sdw12|8756|/data/mirror756 +1815|813|m|m|s|u|sdw12|sdw12|8813|/data/mirror813 +1868|866|m|m|s|u|sdw12|sdw12|8866|/data/mirror866 +1916|914|m|m|s|u|sdw12|sdw12|8914|/data/mirror914 +1968|966|m|m|s|u|sdw12|sdw12|8966|/data/mirror966 +# SDW13 +256|254|p|p|s|u|sdw13|sdw13|7254|/data/primary254 +257|255|p|p|s|u|sdw13|sdw13|7255|/data/primary255 +258|256|p|p|s|u|sdw13|sdw13|7256|/data/primary256 +259|257|p|p|s|u|sdw13|sdw13|7257|/data/primary257 +260|258|p|p|s|u|sdw13|sdw13|7258|/data/primary258 +261|259|p|p|s|u|sdw13|sdw13|7259|/data/primary259 +262|260|p|p|s|u|sdw13|sdw13|7260|/data/primary260 +263|261|p|p|s|u|sdw13|sdw13|7261|/data/primary261 +264|262|p|p|s|u|sdw13|sdw13|7262|/data/primary262 +265|263|p|p|s|u|sdw13|sdw13|7263|/data/primary263 +266|264|p|p|s|u|sdw13|sdw13|7264|/data/primary264 +267|265|p|p|s|u|sdw13|sdw13|7265|/data/primary265 +268|266|p|p|s|u|sdw13|sdw13|7266|/data/primary266 +269|267|p|p|s|u|sdw13|sdw13|7267|/data/primary267 +270|268|p|p|s|u|sdw13|sdw13|7268|/data/primary268 +271|269|p|p|s|u|sdw13|sdw13|7269|/data/primary269 +272|270|p|p|s|u|sdw13|sdw13|7270|/data/primary270 +273|271|p|p|s|u|sdw13|sdw13|7271|/data/primary271 +1014|12|m|m|s|u|sdw13|sdw13|8012|/data/mirror12 +1066|64|m|m|s|u|sdw13|sdw13|8064|/data/mirror64 +1120|118|m|m|s|u|sdw13|sdw13|8118|/data/mirror118 +1165|163|m|m|s|u|sdw13|sdw13|8163|/data/mirror163 +1203|201|m|m|s|u|sdw13|sdw13|8201|/data/mirror201 +1274|272|m|m|s|u|sdw13|sdw13|8272|/data/mirror272 +1316|314|m|m|s|u|sdw13|sdw13|8314|/data/mirror314 +1340|338|m|m|s|u|sdw13|sdw13|8338|/data/mirror338 +1405|403|m|m|s|u|sdw13|sdw13|8403|/data/mirror403 +1471|469|m|m|s|u|sdw13|sdw13|8469|/data/mirror469 +1500|498|m|m|s|u|sdw13|sdw13|8498|/data/mirror498 +1567|565|m|m|s|u|sdw13|sdw13|8565|/data/mirror565 +1616|614|m|m|s|u|sdw13|sdw13|8614|/data/mirror614 +1671|669|m|m|s|u|sdw13|sdw13|8669|/data/mirror669 +1719|717|m|m|s|u|sdw13|sdw13|8717|/data/mirror717 +1749|747|m|m|s|u|sdw13|sdw13|8747|/data/mirror747 +1816|814|m|m|s|u|sdw13|sdw13|8814|/data/mirror814 +1842|840|m|m|s|u|sdw13|sdw13|8840|/data/mirror840 +1920|918|m|m|s|u|sdw13|sdw13|8918|/data/mirror918 +1969|967|m|m|s|u|sdw13|sdw13|8967|/data/mirror967 +# SDW14 +274|272|p|p|s|u|sdw14|sdw14|7272|/data/primary272 +275|273|p|p|s|u|sdw14|sdw14|7273|/data/primary273 +276|274|p|p|s|u|sdw14|sdw14|7274|/data/primary274 +277|275|p|p|s|u|sdw14|sdw14|7275|/data/primary275 +278|276|p|p|s|u|sdw14|sdw14|7276|/data/primary276 +279|277|p|p|s|u|sdw14|sdw14|7277|/data/primary277 +280|278|p|p|s|u|sdw14|sdw14|7278|/data/primary278 +281|279|p|p|s|u|sdw14|sdw14|7279|/data/primary279 +282|280|p|p|s|u|sdw14|sdw14|7280|/data/primary280 +283|281|p|p|s|u|sdw14|sdw14|7281|/data/primary281 +284|282|p|p|s|u|sdw14|sdw14|7282|/data/primary282 +285|283|p|p|s|u|sdw14|sdw14|7283|/data/primary283 +286|284|p|p|s|u|sdw14|sdw14|7284|/data/primary284 +287|285|p|p|s|u|sdw14|sdw14|7285|/data/primary285 +288|286|p|p|s|u|sdw14|sdw14|7286|/data/primary286 +289|287|p|p|s|u|sdw14|sdw14|7287|/data/primary287 +290|288|p|p|s|u|sdw14|sdw14|7288|/data/primary288 +291|289|p|p|s|u|sdw14|sdw14|7289|/data/primary289 +292|290|p|p|s|u|sdw14|sdw14|7290|/data/primary290 +293|291|p|p|s|u|sdw14|sdw14|7291|/data/primary291 +294|292|p|p|s|u|sdw14|sdw14|7292|/data/primary292 +295|293|p|p|s|u|sdw14|sdw14|7293|/data/primary293 +296|294|p|p|s|u|sdw14|sdw14|7294|/data/primary294 +297|295|p|p|s|u|sdw14|sdw14|7295|/data/primary295 +298|296|p|p|s|u|sdw14|sdw14|7296|/data/primary296 +299|297|p|p|s|u|sdw14|sdw14|7297|/data/primary297 +300|298|p|p|s|u|sdw14|sdw14|7298|/data/primary298 +1015|13|m|m|s|u|sdw14|sdw14|8013|/data/mirror13 +1045|43|m|m|s|u|sdw14|sdw14|8043|/data/mirror43 +1121|119|m|m|s|u|sdw14|sdw14|8119|/data/mirror119 +1166|164|m|m|s|u|sdw14|sdw14|8164|/data/mirror164 +1222|220|m|m|s|u|sdw14|sdw14|8220|/data/mirror220 +1265|263|m|m|s|u|sdw14|sdw14|8263|/data/mirror263 +1317|315|m|m|s|u|sdw14|sdw14|8315|/data/mirror315 +1366|364|m|m|s|u|sdw14|sdw14|8364|/data/mirror364 +1416|414|m|m|s|u|sdw14|sdw14|8414|/data/mirror414 +1456|454|m|m|s|u|sdw14|sdw14|8454|/data/mirror454 +1513|511|m|m|s|u|sdw14|sdw14|8511|/data/mirror511 +1571|569|m|m|s|u|sdw14|sdw14|8569|/data/mirror569 +1606|604|m|m|s|u|sdw14|sdw14|8604|/data/mirror604 +1642|640|m|m|s|u|sdw14|sdw14|8640|/data/mirror640 +1720|718|m|m|s|u|sdw14|sdw14|8718|/data/mirror718 +1767|765|m|m|s|u|sdw14|sdw14|8765|/data/mirror765 +1817|815|m|m|s|u|sdw14|sdw14|8815|/data/mirror815 +1870|868|m|m|s|u|sdw14|sdw14|8868|/data/mirror868 +1921|919|m|m|s|u|sdw14|sdw14|8919|/data/mirror919 +1970|968|m|m|s|u|sdw14|sdw14|8968|/data/mirror968 +# SDW15 +301|299|p|p|s|u|sdw15|sdw15|7299|/data/primary299 +302|300|p|p|s|u|sdw15|sdw15|7300|/data/primary300 +303|301|p|p|s|u|sdw15|sdw15|7301|/data/primary301 +304|302|p|p|s|u|sdw15|sdw15|7302|/data/primary302 +305|303|p|p|s|u|sdw15|sdw15|7303|/data/primary303 +306|304|p|p|s|u|sdw15|sdw15|7304|/data/primary304 +307|305|p|p|s|u|sdw15|sdw15|7305|/data/primary305 +308|306|p|p|s|u|sdw15|sdw15|7306|/data/primary306 +309|307|p|p|s|u|sdw15|sdw15|7307|/data/primary307 +310|308|p|p|s|u|sdw15|sdw15|7308|/data/primary308 +311|309|p|p|s|u|sdw15|sdw15|7309|/data/primary309 +312|310|p|p|s|u|sdw15|sdw15|7310|/data/primary310 +313|311|p|p|s|u|sdw15|sdw15|7311|/data/primary311 +314|312|p|p|s|u|sdw15|sdw15|7312|/data/primary312 +315|313|p|p|s|u|sdw15|sdw15|7313|/data/primary313 +316|314|p|p|s|u|sdw15|sdw15|7314|/data/primary314 +1016|14|m|m|s|u|sdw15|sdw15|8014|/data/mirror14 +1067|65|m|m|s|u|sdw15|sdw15|8065|/data/mirror65 +1122|120|m|m|s|u|sdw15|sdw15|8120|/data/mirror120 +1169|167|m|m|s|u|sdw15|sdw15|8167|/data/mirror167 +1223|221|m|m|s|u|sdw15|sdw15|8221|/data/mirror221 +1266|264|m|m|s|u|sdw15|sdw15|8264|/data/mirror264 +1319|317|m|m|s|u|sdw15|sdw15|8317|/data/mirror317 +1368|366|m|m|s|u|sdw15|sdw15|8366|/data/mirror366 +1418|416|m|m|s|u|sdw15|sdw15|8416|/data/mirror416 +1468|466|m|m|s|u|sdw15|sdw15|8466|/data/mirror466 +1524|522|m|m|s|u|sdw15|sdw15|8522|/data/mirror522 +1573|571|m|m|s|u|sdw15|sdw15|8571|/data/mirror571 +1617|615|m|m|s|u|sdw15|sdw15|8615|/data/mirror615 +1655|653|m|m|s|u|sdw15|sdw15|8653|/data/mirror653 +1701|699|m|m|s|u|sdw15|sdw15|8699|/data/mirror699 +1768|766|m|m|s|u|sdw15|sdw15|8766|/data/mirror766 +1804|802|m|m|s|u|sdw15|sdw15|8802|/data/mirror802 +1872|870|m|m|s|u|sdw15|sdw15|8870|/data/mirror870 +1912|910|m|m|s|u|sdw15|sdw15|8910|/data/mirror910 +1973|971|m|m|s|u|sdw15|sdw15|8971|/data/mirror971 +# SDW16 +317|315|p|p|s|u|sdw16|sdw16|7315|/data/primary315 +318|316|p|p|s|u|sdw16|sdw16|7316|/data/primary316 +319|317|p|p|s|u|sdw16|sdw16|7317|/data/primary317 +320|318|p|p|s|u|sdw16|sdw16|7318|/data/primary318 +321|319|p|p|s|u|sdw16|sdw16|7319|/data/primary319 +322|320|p|p|s|u|sdw16|sdw16|7320|/data/primary320 +323|321|p|p|s|u|sdw16|sdw16|7321|/data/primary321 +324|322|p|p|s|u|sdw16|sdw16|7322|/data/primary322 +325|323|p|p|s|u|sdw16|sdw16|7323|/data/primary323 +326|324|p|p|s|u|sdw16|sdw16|7324|/data/primary324 +327|325|p|p|s|u|sdw16|sdw16|7325|/data/primary325 +328|326|p|p|s|u|sdw16|sdw16|7326|/data/primary326 +329|327|p|p|s|u|sdw16|sdw16|7327|/data/primary327 +330|328|p|p|s|u|sdw16|sdw16|7328|/data/primary328 +331|329|p|p|s|u|sdw16|sdw16|7329|/data/primary329 +332|330|p|p|s|u|sdw16|sdw16|7330|/data/primary330 +333|331|p|p|s|u|sdw16|sdw16|7331|/data/primary331 +334|332|p|p|s|u|sdw16|sdw16|7332|/data/primary332 +335|333|p|p|s|u|sdw16|sdw16|7333|/data/primary333 +336|334|p|p|s|u|sdw16|sdw16|7334|/data/primary334 +337|335|p|p|s|u|sdw16|sdw16|7335|/data/primary335 +338|336|p|p|s|u|sdw16|sdw16|7336|/data/primary336 +339|337|p|p|s|u|sdw16|sdw16|7337|/data/primary337 +340|338|p|p|s|u|sdw16|sdw16|7338|/data/primary338 +1018|16|m|m|s|u|sdw16|sdw16|8016|/data/mirror16 +1069|67|m|m|s|u|sdw16|sdw16|8067|/data/mirror67 +1123|121|m|m|s|u|sdw16|sdw16|8121|/data/mirror121 +1170|168|m|m|s|u|sdw16|sdw16|8168|/data/mirror168 +1224|222|m|m|s|u|sdw16|sdw16|8222|/data/mirror222 +1267|265|m|m|s|u|sdw16|sdw16|8265|/data/mirror265 +1341|339|m|m|s|u|sdw16|sdw16|8339|/data/mirror339 +1369|367|m|m|s|u|sdw16|sdw16|8367|/data/mirror367 +1419|417|m|m|s|u|sdw16|sdw16|8417|/data/mirror417 +1472|470|m|m|s|u|sdw16|sdw16|8470|/data/mirror470 +1514|512|m|m|s|u|sdw16|sdw16|8512|/data/mirror512 +1553|551|m|m|s|u|sdw16|sdw16|8551|/data/mirror551 +1619|617|m|m|s|u|sdw16|sdw16|8617|/data/mirror617 +1673|671|m|m|s|u|sdw16|sdw16|8671|/data/mirror671 +1721|719|m|m|s|u|sdw16|sdw16|8719|/data/mirror719 +1769|767|m|m|s|u|sdw16|sdw16|8767|/data/mirror767 +1818|816|m|m|s|u|sdw16|sdw16|8816|/data/mirror816 +1873|871|m|m|s|u|sdw16|sdw16|8871|/data/mirror871 +1922|920|m|m|s|u|sdw16|sdw16|8920|/data/mirror920 +1974|972|m|m|s|u|sdw16|sdw16|8972|/data/mirror972 +# SDW17 +341|339|p|p|s|u|sdw17|sdw17|7339|/data/primary339 +342|340|p|p|s|u|sdw17|sdw17|7340|/data/primary340 +343|341|p|p|s|u|sdw17|sdw17|7341|/data/primary341 +344|342|p|p|s|u|sdw17|sdw17|7342|/data/primary342 +345|343|p|p|s|u|sdw17|sdw17|7343|/data/primary343 +346|344|p|p|s|u|sdw17|sdw17|7344|/data/primary344 +347|345|p|p|s|u|sdw17|sdw17|7345|/data/primary345 +348|346|p|p|s|u|sdw17|sdw17|7346|/data/primary346 +349|347|p|p|s|u|sdw17|sdw17|7347|/data/primary347 +350|348|p|p|s|u|sdw17|sdw17|7348|/data/primary348 +351|349|p|p|s|u|sdw17|sdw17|7349|/data/primary349 +352|350|p|p|s|u|sdw17|sdw17|7350|/data/primary350 +353|351|p|p|s|u|sdw17|sdw17|7351|/data/primary351 +354|352|p|p|s|u|sdw17|sdw17|7352|/data/primary352 +355|353|p|p|s|u|sdw17|sdw17|7353|/data/primary353 +356|354|p|p|s|u|sdw17|sdw17|7354|/data/primary354 +357|355|p|p|s|u|sdw17|sdw17|7355|/data/primary355 +358|356|p|p|s|u|sdw17|sdw17|7356|/data/primary356 +359|357|p|p|s|u|sdw17|sdw17|7357|/data/primary357 +360|358|p|p|s|u|sdw17|sdw17|7358|/data/primary358 +361|359|p|p|s|u|sdw17|sdw17|7359|/data/primary359 +362|360|p|p|s|u|sdw17|sdw17|7360|/data/primary360 +363|361|p|p|s|u|sdw17|sdw17|7361|/data/primary361 +364|362|p|p|s|u|sdw17|sdw17|7362|/data/primary362 +1019|17|m|m|s|u|sdw17|sdw17|8017|/data/mirror17 +1070|68|m|m|s|u|sdw17|sdw17|8068|/data/mirror68 +1125|123|m|m|s|u|sdw17|sdw17|8123|/data/mirror123 +1173|171|m|m|s|u|sdw17|sdw17|8171|/data/mirror171 +1205|203|m|m|s|u|sdw17|sdw17|8203|/data/mirror203 +1268|266|m|m|s|u|sdw17|sdw17|8266|/data/mirror266 +1318|316|m|m|s|u|sdw17|sdw17|8316|/data/mirror316 +1370|368|m|m|s|u|sdw17|sdw17|8368|/data/mirror368 +1420|418|m|m|s|u|sdw17|sdw17|8418|/data/mirror418 +1473|471|m|m|s|u|sdw17|sdw17|8471|/data/mirror471 +1502|500|m|m|s|u|sdw17|sdw17|8500|/data/mirror500 +1574|572|m|m|s|u|sdw17|sdw17|8572|/data/mirror572 +1620|618|m|m|s|u|sdw17|sdw17|8618|/data/mirror618 +1667|665|m|m|s|u|sdw17|sdw17|8665|/data/mirror665 +1702|700|m|m|s|u|sdw17|sdw17|8700|/data/mirror700 +1770|768|m|m|s|u|sdw17|sdw17|8768|/data/mirror768 +1819|817|m|m|s|u|sdw17|sdw17|8817|/data/mirror817 +1869|867|m|m|s|u|sdw17|sdw17|8867|/data/mirror867 +1903|901|m|m|s|u|sdw17|sdw17|8901|/data/mirror901 +1955|953|m|m|s|u|sdw17|sdw17|8953|/data/mirror953 +# SDW18 +365|363|p|p|s|u|sdw18|sdw18|7363|/data/primary363 +366|364|p|p|s|u|sdw18|sdw18|7364|/data/primary364 +367|365|p|p|s|u|sdw18|sdw18|7365|/data/primary365 +368|366|p|p|s|u|sdw18|sdw18|7366|/data/primary366 +369|367|p|p|s|u|sdw18|sdw18|7367|/data/primary367 +370|368|p|p|s|u|sdw18|sdw18|7368|/data/primary368 +371|369|p|p|s|u|sdw18|sdw18|7369|/data/primary369 +372|370|p|p|s|u|sdw18|sdw18|7370|/data/primary370 +373|371|p|p|s|u|sdw18|sdw18|7371|/data/primary371 +374|372|p|p|s|u|sdw18|sdw18|7372|/data/primary372 +375|373|p|p|s|u|sdw18|sdw18|7373|/data/primary373 +376|374|p|p|s|u|sdw18|sdw18|7374|/data/primary374 +377|375|p|p|s|u|sdw18|sdw18|7375|/data/primary375 +378|376|p|p|s|u|sdw18|sdw18|7376|/data/primary376 +379|377|p|p|s|u|sdw18|sdw18|7377|/data/primary377 +380|378|p|p|s|u|sdw18|sdw18|7378|/data/primary378 +381|379|p|p|s|u|sdw18|sdw18|7379|/data/primary379 +382|380|p|p|s|u|sdw18|sdw18|7380|/data/primary380 +1010|8|m|m|s|u|sdw18|sdw18|8008|/data/mirror8 +1073|71|m|m|s|u|sdw18|sdw18|8071|/data/mirror71 +1107|105|m|m|s|u|sdw18|sdw18|8105|/data/mirror105 +1172|170|m|m|s|u|sdw18|sdw18|8170|/data/mirror170 +1219|217|m|m|s|u|sdw18|sdw18|8217|/data/mirror217 +1271|269|m|m|s|u|sdw18|sdw18|8269|/data/mirror269 +1321|319|m|m|s|u|sdw18|sdw18|8319|/data/mirror319 +1383|381|m|m|s|u|sdw18|sdw18|8381|/data/mirror381 +1421|419|m|m|s|u|sdw18|sdw18|8419|/data/mirror419 +1475|473|m|m|s|u|sdw18|sdw18|8473|/data/mirror473 +1525|523|m|m|s|u|sdw18|sdw18|8523|/data/mirror523 +1554|552|m|m|s|u|sdw18|sdw18|8552|/data/mirror552 +1621|619|m|m|s|u|sdw18|sdw18|8619|/data/mirror619 +1674|672|m|m|s|u|sdw18|sdw18|8672|/data/mirror672 +1712|710|m|m|s|u|sdw18|sdw18|8710|/data/mirror710 +1771|769|m|m|s|u|sdw18|sdw18|8769|/data/mirror769 +1820|818|m|m|s|u|sdw18|sdw18|8818|/data/mirror818 +1876|874|m|m|s|u|sdw18|sdw18|8874|/data/mirror874 +1923|921|m|m|s|u|sdw18|sdw18|8921|/data/mirror921 +1975|973|m|m|s|u|sdw18|sdw18|8973|/data/mirror973 +# SDW19 +383|381|p|p|s|u|sdw19|sdw19|7381|/data/primary381 +384|382|p|p|s|u|sdw19|sdw19|7382|/data/primary382 +385|383|p|p|s|u|sdw19|sdw19|7383|/data/primary383 +386|384|p|p|s|u|sdw19|sdw19|7384|/data/primary384 +387|385|p|p|s|u|sdw19|sdw19|7385|/data/primary385 +388|386|p|p|s|u|sdw19|sdw19|7386|/data/primary386 +389|387|p|p|s|u|sdw19|sdw19|7387|/data/primary387 +390|388|p|p|s|u|sdw19|sdw19|7388|/data/primary388 +391|389|p|p|s|u|sdw19|sdw19|7389|/data/primary389 +392|390|p|p|s|u|sdw19|sdw19|7390|/data/primary390 +393|391|p|p|s|u|sdw19|sdw19|7391|/data/primary391 +394|392|p|p|s|u|sdw19|sdw19|7392|/data/primary392 +395|393|p|p|s|u|sdw19|sdw19|7393|/data/primary393 +1020|18|m|m|s|u|sdw19|sdw19|8018|/data/mirror18 +1077|75|m|m|s|u|sdw19|sdw19|8075|/data/mirror75 +1126|124|m|m|s|u|sdw19|sdw19|8124|/data/mirror124 +1174|172|m|m|s|u|sdw19|sdw19|8172|/data/mirror172 +1226|224|m|m|s|u|sdw19|sdw19|8224|/data/mirror224 +1272|270|m|m|s|u|sdw19|sdw19|8270|/data/mirror270 +1322|320|m|m|s|u|sdw19|sdw19|8320|/data/mirror320 +1371|369|m|m|s|u|sdw19|sdw19|8369|/data/mirror369 +1422|420|m|m|s|u|sdw19|sdw19|8420|/data/mirror420 +1445|443|m|m|s|u|sdw19|sdw19|8443|/data/mirror443 +1503|501|m|m|s|u|sdw19|sdw19|8501|/data/mirror501 +1575|573|m|m|s|u|sdw19|sdw19|8573|/data/mirror573 +1622|620|m|m|s|u|sdw19|sdw19|8620|/data/mirror620 +1675|673|m|m|s|u|sdw19|sdw19|8673|/data/mirror673 +1703|701|m|m|s|u|sdw19|sdw19|8701|/data/mirror701 +1761|759|m|m|s|u|sdw19|sdw19|8759|/data/mirror759 +1821|819|m|m|s|u|sdw19|sdw19|8819|/data/mirror819 +1877|875|m|m|s|u|sdw19|sdw19|8875|/data/mirror875 +1924|922|m|m|s|u|sdw19|sdw19|8922|/data/mirror922 +1976|974|m|m|s|u|sdw19|sdw19|8974|/data/mirror974 +# SDW2 +26|24|p|p|s|u|sdw2|sdw2|7024|/data/primary24 +27|25|p|p|s|u|sdw2|sdw2|7025|/data/primary25 +28|26|p|p|s|u|sdw2|sdw2|7026|/data/primary26 +29|27|p|p|s|u|sdw2|sdw2|7027|/data/primary27 +30|28|p|p|s|u|sdw2|sdw2|7028|/data/primary28 +31|29|p|p|s|u|sdw2|sdw2|7029|/data/primary29 +32|30|p|p|s|u|sdw2|sdw2|7030|/data/primary30 +33|31|p|p|s|u|sdw2|sdw2|7031|/data/primary31 +34|32|p|p|s|u|sdw2|sdw2|7032|/data/primary32 +35|33|p|p|s|u|sdw2|sdw2|7033|/data/primary33 +36|34|p|p|s|u|sdw2|sdw2|7034|/data/primary34 +37|35|p|p|s|u|sdw2|sdw2|7035|/data/primary35 +38|36|p|p|s|u|sdw2|sdw2|7036|/data/primary36 +39|37|p|p|s|u|sdw2|sdw2|7037|/data/primary37 +40|38|p|p|s|u|sdw2|sdw2|7038|/data/primary38 +41|39|p|p|s|u|sdw2|sdw2|7039|/data/primary39 +42|40|p|p|s|u|sdw2|sdw2|7040|/data/primary40 +43|41|p|p|s|u|sdw2|sdw2|7041|/data/primary41 +44|42|p|p|s|u|sdw2|sdw2|7042|/data/primary42 +45|43|p|p|s|u|sdw2|sdw2|7043|/data/primary43 +46|44|p|p|s|u|sdw2|sdw2|7044|/data/primary44 +47|45|p|p|s|u|sdw2|sdw2|7045|/data/primary45 +48|46|p|p|s|u|sdw2|sdw2|7046|/data/primary46 +49|47|p|p|s|u|sdw2|sdw2|7047|/data/primary47 +1002|0|m|m|s|u|sdw2|sdw2|8000|/data/mirror0 +1058|56|m|m|s|u|sdw2|sdw2|8056|/data/mirror56 +1104|102|m|m|s|u|sdw2|sdw2|8102|/data/mirror102 +1157|155|m|m|s|u|sdw2|sdw2|8155|/data/mirror155 +1187|185|m|m|s|u|sdw2|sdw2|8185|/data/mirror185 +1254|252|m|m|s|u|sdw2|sdw2|8252|/data/mirror252 +1303|301|m|m|s|u|sdw2|sdw2|8301|/data/mirror301 +1356|354|m|m|s|u|sdw2|sdw2|8354|/data/mirror354 +1404|402|m|m|s|u|sdw2|sdw2|8402|/data/mirror402 +1457|455|m|m|s|u|sdw2|sdw2|8455|/data/mirror455 +1510|508|m|m|s|u|sdw2|sdw2|8508|/data/mirror508 +1558|556|m|m|s|u|sdw2|sdw2|8556|/data/mirror556 +1604|602|m|m|s|u|sdw2|sdw2|8602|/data/mirror602 +1658|656|m|m|s|u|sdw2|sdw2|8656|/data/mirror656 +1711|709|m|m|s|u|sdw2|sdw2|8709|/data/mirror709 +1756|754|m|m|s|u|sdw2|sdw2|8754|/data/mirror754 +1806|804|m|m|s|u|sdw2|sdw2|8804|/data/mirror804 +1854|852|m|m|s|u|sdw2|sdw2|8852|/data/mirror852 +1909|907|m|m|s|u|sdw2|sdw2|8907|/data/mirror907 +1956|954|m|m|s|u|sdw2|sdw2|8954|/data/mirror954 +# SDW20 +396|394|p|p|s|u|sdw20|sdw20|7394|/data/primary394 +397|395|p|p|s|u|sdw20|sdw20|7395|/data/primary395 +398|396|p|p|s|u|sdw20|sdw20|7396|/data/primary396 +399|397|p|p|s|u|sdw20|sdw20|7397|/data/primary397 +400|398|p|p|s|u|sdw20|sdw20|7398|/data/primary398 +401|399|p|p|s|u|sdw20|sdw20|7399|/data/primary399 +402|400|p|p|s|u|sdw20|sdw20|7400|/data/primary400 +403|401|p|p|s|u|sdw20|sdw20|7401|/data/primary401 +404|402|p|p|s|u|sdw20|sdw20|7402|/data/primary402 +405|403|p|p|s|u|sdw20|sdw20|7403|/data/primary403 +406|404|p|p|s|u|sdw20|sdw20|7404|/data/primary404 +407|405|p|p|s|u|sdw20|sdw20|7405|/data/primary405 +408|406|p|p|s|u|sdw20|sdw20|7406|/data/primary406 +409|407|p|p|s|u|sdw20|sdw20|7407|/data/primary407 +410|408|p|p|s|u|sdw20|sdw20|7408|/data/primary408 +411|409|p|p|s|u|sdw20|sdw20|7409|/data/primary409 +412|410|p|p|s|u|sdw20|sdw20|7410|/data/primary410 +413|411|p|p|s|u|sdw20|sdw20|7411|/data/primary411 +414|412|p|p|s|u|sdw20|sdw20|7412|/data/primary412 +415|413|p|p|s|u|sdw20|sdw20|7413|/data/primary413 +416|414|p|p|s|u|sdw20|sdw20|7414|/data/primary414 +417|415|p|p|s|u|sdw20|sdw20|7415|/data/primary415 +418|416|p|p|s|u|sdw20|sdw20|7416|/data/primary416 +419|417|p|p|s|u|sdw20|sdw20|7417|/data/primary417 +420|418|p|p|s|u|sdw20|sdw20|7418|/data/primary418 +1021|19|m|m|s|u|sdw20|sdw20|8019|/data/mirror19 +1080|78|m|m|s|u|sdw20|sdw20|8078|/data/mirror78 +1108|106|m|m|s|u|sdw20|sdw20|8106|/data/mirror106 +1160|158|m|m|s|u|sdw20|sdw20|8158|/data/mirror158 +1227|225|m|m|s|u|sdw20|sdw20|8225|/data/mirror225 +1273|271|m|m|s|u|sdw20|sdw20|8271|/data/mirror271 +1323|321|m|m|s|u|sdw20|sdw20|8321|/data/mirror321 +1372|370|m|m|s|u|sdw20|sdw20|8370|/data/mirror370 +1423|421|m|m|s|u|sdw20|sdw20|8421|/data/mirror421 +1476|474|m|m|s|u|sdw20|sdw20|8474|/data/mirror474 +1526|524|m|m|s|u|sdw20|sdw20|8524|/data/mirror524 +1555|553|m|m|s|u|sdw20|sdw20|8553|/data/mirror553 +1623|621|m|m|s|u|sdw20|sdw20|8621|/data/mirror621 +1676|674|m|m|s|u|sdw20|sdw20|8674|/data/mirror674 +1724|722|m|m|s|u|sdw20|sdw20|8722|/data/mirror722 +1772|770|m|m|s|u|sdw20|sdw20|8770|/data/mirror770 +1822|820|m|m|s|u|sdw20|sdw20|8820|/data/mirror820 +1857|855|m|m|s|u|sdw20|sdw20|8855|/data/mirror855 +1914|912|m|m|s|u|sdw20|sdw20|8912|/data/mirror912 +1977|975|m|m|s|u|sdw20|sdw20|8975|/data/mirror975 +# SDW21 +421|419|p|p|s|u|sdw21|sdw21|7419|/data/primary419 +422|420|p|p|s|u|sdw21|sdw21|7420|/data/primary420 +423|421|p|p|s|u|sdw21|sdw21|7421|/data/primary421 +424|422|p|p|s|u|sdw21|sdw21|7422|/data/primary422 +425|423|p|p|s|u|sdw21|sdw21|7423|/data/primary423 +426|424|p|p|s|u|sdw21|sdw21|7424|/data/primary424 +427|425|p|p|s|u|sdw21|sdw21|7425|/data/primary425 +428|426|p|p|s|u|sdw21|sdw21|7426|/data/primary426 +429|427|p|p|s|u|sdw21|sdw21|7427|/data/primary427 +430|428|p|p|s|u|sdw21|sdw21|7428|/data/primary428 +431|429|p|p|s|u|sdw21|sdw21|7429|/data/primary429 +432|430|p|p|s|u|sdw21|sdw21|7430|/data/primary430 +433|431|p|p|s|u|sdw21|sdw21|7431|/data/primary431 +434|432|p|p|s|u|sdw21|sdw21|7432|/data/primary432 +435|433|p|p|s|u|sdw21|sdw21|7433|/data/primary433 +436|434|p|p|s|u|sdw21|sdw21|7434|/data/primary434 +437|435|p|p|s|u|sdw21|sdw21|7435|/data/primary435 +438|436|p|p|s|u|sdw21|sdw21|7436|/data/primary436 +439|437|p|p|s|u|sdw21|sdw21|7437|/data/primary437 +440|438|p|p|s|u|sdw21|sdw21|7438|/data/primary438 +441|439|p|p|s|u|sdw21|sdw21|7439|/data/primary439 +442|440|p|p|s|u|sdw21|sdw21|7440|/data/primary440 +443|441|p|p|s|u|sdw21|sdw21|7441|/data/primary441 +444|442|p|p|s|u|sdw21|sdw21|7442|/data/primary442 +445|443|p|p|s|u|sdw21|sdw21|7443|/data/primary443 +1022|20|m|m|s|u|sdw21|sdw21|8020|/data/mirror20 +1074|72|m|m|s|u|sdw21|sdw21|8072|/data/mirror72 +1127|125|m|m|s|u|sdw21|sdw21|8125|/data/mirror125 +1175|173|m|m|s|u|sdw21|sdw21|8173|/data/mirror173 +1207|205|m|m|s|u|sdw21|sdw21|8205|/data/mirror205 +1275|273|m|m|s|u|sdw21|sdw21|8273|/data/mirror273 +1311|309|m|m|s|u|sdw21|sdw21|8309|/data/mirror309 +1373|371|m|m|s|u|sdw21|sdw21|8371|/data/mirror371 +1446|444|m|m|s|u|sdw21|sdw21|8444|/data/mirror444 +1478|476|m|m|s|u|sdw21|sdw21|8476|/data/mirror476 +1527|525|m|m|s|u|sdw21|sdw21|8525|/data/mirror525 +1576|574|m|m|s|u|sdw21|sdw21|8574|/data/mirror574 +1624|622|m|m|s|u|sdw21|sdw21|8622|/data/mirror622 +1669|667|m|m|s|u|sdw21|sdw21|8667|/data/mirror667 +1725|723|m|m|s|u|sdw21|sdw21|8723|/data/mirror723 +1773|771|m|m|s|u|sdw21|sdw21|8771|/data/mirror771 +1807|805|m|m|s|u|sdw21|sdw21|8805|/data/mirror805 +1878|876|m|m|s|u|sdw21|sdw21|8876|/data/mirror876 +1925|923|m|m|s|u|sdw21|sdw21|8923|/data/mirror923 +1957|955|m|m|s|u|sdw21|sdw21|8955|/data/mirror955 +# SDW22 +446|444|p|p|s|u|sdw22|sdw22|7444|/data/primary444 +447|445|p|p|s|u|sdw22|sdw22|7445|/data/primary445 +448|446|p|p|s|u|sdw22|sdw22|7446|/data/primary446 +449|447|p|p|s|u|sdw22|sdw22|7447|/data/primary447 +450|448|p|p|s|u|sdw22|sdw22|7448|/data/primary448 +451|449|p|p|s|u|sdw22|sdw22|7449|/data/primary449 +452|450|p|p|s|u|sdw22|sdw22|7450|/data/primary450 +453|451|p|p|s|u|sdw22|sdw22|7451|/data/primary451 +454|452|p|p|s|u|sdw22|sdw22|7452|/data/primary452 +455|453|p|p|s|u|sdw22|sdw22|7453|/data/primary453 +456|454|p|p|s|u|sdw22|sdw22|7454|/data/primary454 +457|455|p|p|s|u|sdw22|sdw22|7455|/data/primary455 +458|456|p|p|s|u|sdw22|sdw22|7456|/data/primary456 +459|457|p|p|s|u|sdw22|sdw22|7457|/data/primary457 +460|458|p|p|s|u|sdw22|sdw22|7458|/data/primary458 +461|459|p|p|s|u|sdw22|sdw22|7459|/data/primary459 +462|460|p|p|s|u|sdw22|sdw22|7460|/data/primary460 +463|461|p|p|s|u|sdw22|sdw22|7461|/data/primary461 +464|462|p|p|s|u|sdw22|sdw22|7462|/data/primary462 +1024|22|m|m|s|u|sdw22|sdw22|8022|/data/mirror22 +1061|59|m|m|s|u|sdw22|sdw22|8059|/data/mirror59 +1129|127|m|m|s|u|sdw22|sdw22|8127|/data/mirror127 +1176|174|m|m|s|u|sdw22|sdw22|8174|/data/mirror174 +1229|227|m|m|s|u|sdw22|sdw22|8227|/data/mirror227 +1276|274|m|m|s|u|sdw22|sdw22|8274|/data/mirror274 +1320|318|m|m|s|u|sdw22|sdw22|8318|/data/mirror318 +1374|372|m|m|s|u|sdw22|sdw22|8372|/data/mirror372 +1409|407|m|m|s|u|sdw22|sdw22|8407|/data/mirror407 +1480|478|m|m|s|u|sdw22|sdw22|8478|/data/mirror478 +1516|514|m|m|s|u|sdw22|sdw22|8514|/data/mirror514 +1577|575|m|m|s|u|sdw22|sdw22|8575|/data/mirror575 +1625|623|m|m|s|u|sdw22|sdw22|8623|/data/mirror623 +1677|675|m|m|s|u|sdw22|sdw22|8675|/data/mirror675 +1722|720|m|m|s|u|sdw22|sdw22|8720|/data/mirror720 +1774|772|m|m|s|u|sdw22|sdw22|8772|/data/mirror772 +1823|821|m|m|s|u|sdw22|sdw22|8821|/data/mirror821 +1871|869|m|m|s|u|sdw22|sdw22|8869|/data/mirror869 +1926|924|m|m|s|u|sdw22|sdw22|8924|/data/mirror924 +1979|977|m|m|s|u|sdw22|sdw22|8977|/data/mirror977 +# SDW23 +465|463|p|p|s|u|sdw23|sdw23|7463|/data/primary463 +466|464|p|p|s|u|sdw23|sdw23|7464|/data/primary464 +467|465|p|p|s|u|sdw23|sdw23|7465|/data/primary465 +468|466|p|p|s|u|sdw23|sdw23|7466|/data/primary466 +469|467|p|p|s|u|sdw23|sdw23|7467|/data/primary467 +470|468|p|p|s|u|sdw23|sdw23|7468|/data/primary468 +471|469|p|p|s|u|sdw23|sdw23|7469|/data/primary469 +472|470|p|p|s|u|sdw23|sdw23|7470|/data/primary470 +473|471|p|p|s|u|sdw23|sdw23|7471|/data/primary471 +474|472|p|p|s|u|sdw23|sdw23|7472|/data/primary472 +475|473|p|p|s|u|sdw23|sdw23|7473|/data/primary473 +476|474|p|p|s|u|sdw23|sdw23|7474|/data/primary474 +477|475|p|p|s|u|sdw23|sdw23|7475|/data/primary475 +478|476|p|p|s|u|sdw23|sdw23|7476|/data/primary476 +479|477|p|p|s|u|sdw23|sdw23|7477|/data/primary477 +480|478|p|p|s|u|sdw23|sdw23|7478|/data/primary478 +481|479|p|p|s|u|sdw23|sdw23|7479|/data/primary479 +482|480|p|p|s|u|sdw23|sdw23|7480|/data/primary480 +483|481|p|p|s|u|sdw23|sdw23|7481|/data/primary481 +484|482|p|p|s|u|sdw23|sdw23|7482|/data/primary482 +485|483|p|p|s|u|sdw23|sdw23|7483|/data/primary483 +1025|23|m|m|s|u|sdw23|sdw23|8023|/data/mirror23 +1081|79|m|m|s|u|sdw23|sdw23|8079|/data/mirror79 +1132|130|m|m|s|u|sdw23|sdw23|8130|/data/mirror130 +1179|177|m|m|s|u|sdw23|sdw23|8177|/data/mirror177 +1221|219|m|m|s|u|sdw23|sdw23|8219|/data/mirror219 +1278|276|m|m|s|u|sdw23|sdw23|8276|/data/mirror276 +1324|322|m|m|s|u|sdw23|sdw23|8322|/data/mirror322 +1375|373|m|m|s|u|sdw23|sdw23|8373|/data/mirror373 +1424|422|m|m|s|u|sdw23|sdw23|8422|/data/mirror422 +1486|484|m|m|s|u|sdw23|sdw23|8484|/data/mirror484 +1505|503|m|m|s|u|sdw23|sdw23|8503|/data/mirror503 +1579|577|m|m|s|u|sdw23|sdw23|8577|/data/mirror577 +1626|624|m|m|s|u|sdw23|sdw23|8624|/data/mirror624 +1670|668|m|m|s|u|sdw23|sdw23|8668|/data/mirror668 +1714|712|m|m|s|u|sdw23|sdw23|8712|/data/mirror712 +1775|773|m|m|s|u|sdw23|sdw23|8773|/data/mirror773 +1824|822|m|m|s|u|sdw23|sdw23|8822|/data/mirror822 +1879|877|m|m|s|u|sdw23|sdw23|8877|/data/mirror877 +1927|925|m|m|s|u|sdw23|sdw23|8925|/data/mirror925 +1958|956|m|m|s|u|sdw23|sdw23|8956|/data/mirror956 +# SDW24 +486|484|p|p|s|u|sdw24|sdw24|7484|/data/primary484 +487|485|p|p|s|u|sdw24|sdw24|7485|/data/primary485 +488|486|p|p|s|u|sdw24|sdw24|7486|/data/primary486 +489|487|p|p|s|u|sdw24|sdw24|7487|/data/primary487 +490|488|p|p|s|u|sdw24|sdw24|7488|/data/primary488 +491|489|p|p|s|u|sdw24|sdw24|7489|/data/primary489 +492|490|p|p|s|u|sdw24|sdw24|7490|/data/primary490 +493|491|p|p|s|u|sdw24|sdw24|7491|/data/primary491 +494|492|p|p|s|u|sdw24|sdw24|7492|/data/primary492 +495|493|p|p|s|u|sdw24|sdw24|7493|/data/primary493 +496|494|p|p|s|u|sdw24|sdw24|7494|/data/primary494 +497|495|p|p|s|u|sdw24|sdw24|7495|/data/primary495 +498|496|p|p|s|u|sdw24|sdw24|7496|/data/primary496 +499|497|p|p|s|u|sdw24|sdw24|7497|/data/primary497 +500|498|p|p|s|u|sdw24|sdw24|7498|/data/primary498 +501|499|p|p|s|u|sdw24|sdw24|7499|/data/primary499 +502|500|p|p|s|u|sdw24|sdw24|7500|/data/primary500 +503|501|p|p|s|u|sdw24|sdw24|7501|/data/primary501 +504|502|p|p|s|u|sdw24|sdw24|7502|/data/primary502 +505|503|p|p|s|u|sdw24|sdw24|7503|/data/primary503 +506|504|p|p|s|u|sdw24|sdw24|7504|/data/primary504 +1027|25|m|m|s|u|sdw24|sdw24|8025|/data/mirror25 +1062|60|m|m|s|u|sdw24|sdw24|8060|/data/mirror60 +1110|108|m|m|s|u|sdw24|sdw24|8108|/data/mirror108 +1181|179|m|m|s|u|sdw24|sdw24|8179|/data/mirror179 +1231|229|m|m|s|u|sdw24|sdw24|8229|/data/mirror229 +1280|278|m|m|s|u|sdw24|sdw24|8278|/data/mirror278 +1325|323|m|m|s|u|sdw24|sdw24|8323|/data/mirror323 +1376|374|m|m|s|u|sdw24|sdw24|8374|/data/mirror374 +1425|423|m|m|s|u|sdw24|sdw24|8423|/data/mirror423 +1481|479|m|m|s|u|sdw24|sdw24|8479|/data/mirror479 +1528|526|m|m|s|u|sdw24|sdw24|8526|/data/mirror526 +1580|578|m|m|s|u|sdw24|sdw24|8578|/data/mirror578 +1611|609|m|m|s|u|sdw24|sdw24|8609|/data/mirror609 +1678|676|m|m|s|u|sdw24|sdw24|8676|/data/mirror676 +1726|724|m|m|s|u|sdw24|sdw24|8724|/data/mirror724 +1776|774|m|m|s|u|sdw24|sdw24|8774|/data/mirror774 +1825|823|m|m|s|u|sdw24|sdw24|8823|/data/mirror823 +1883|881|m|m|s|u|sdw24|sdw24|8881|/data/mirror881 +1929|927|m|m|s|u|sdw24|sdw24|8927|/data/mirror927 +1980|978|m|m|s|u|sdw24|sdw24|8978|/data/mirror978 +# SDW25 +507|505|p|p|s|u|sdw25|sdw25|7505|/data/primary505 +508|506|p|p|s|u|sdw25|sdw25|7506|/data/primary506 +509|507|p|p|s|u|sdw25|sdw25|7507|/data/primary507 +510|508|p|p|s|u|sdw25|sdw25|7508|/data/primary508 +511|509|p|p|s|u|sdw25|sdw25|7509|/data/primary509 +512|510|p|p|s|u|sdw25|sdw25|7510|/data/primary510 +513|511|p|p|s|u|sdw25|sdw25|7511|/data/primary511 +514|512|p|p|s|u|sdw25|sdw25|7512|/data/primary512 +515|513|p|p|s|u|sdw25|sdw25|7513|/data/primary513 +516|514|p|p|s|u|sdw25|sdw25|7514|/data/primary514 +517|515|p|p|s|u|sdw25|sdw25|7515|/data/primary515 +518|516|p|p|s|u|sdw25|sdw25|7516|/data/primary516 +519|517|p|p|s|u|sdw25|sdw25|7517|/data/primary517 +520|518|p|p|s|u|sdw25|sdw25|7518|/data/primary518 +521|519|p|p|s|u|sdw25|sdw25|7519|/data/primary519 +522|520|p|p|s|u|sdw25|sdw25|7520|/data/primary520 +523|521|p|p|s|u|sdw25|sdw25|7521|/data/primary521 +1028|26|m|m|s|u|sdw25|sdw25|8026|/data/mirror26 +1075|73|m|m|s|u|sdw25|sdw25|8073|/data/mirror73 +1133|131|m|m|s|u|sdw25|sdw25|8131|/data/mirror131 +1184|182|m|m|s|u|sdw25|sdw25|8182|/data/mirror182 +1209|207|m|m|s|u|sdw25|sdw25|8207|/data/mirror207 +1281|279|m|m|s|u|sdw25|sdw25|8279|/data/mirror279 +1327|325|m|m|s|u|sdw25|sdw25|8325|/data/mirror325 +1358|356|m|m|s|u|sdw25|sdw25|8356|/data/mirror356 +1426|424|m|m|s|u|sdw25|sdw25|8424|/data/mirror424 +1482|480|m|m|s|u|sdw25|sdw25|8480|/data/mirror480 +1529|527|m|m|s|u|sdw25|sdw25|8527|/data/mirror527 +1581|579|m|m|s|u|sdw25|sdw25|8579|/data/mirror579 +1628|626|m|m|s|u|sdw25|sdw25|8626|/data/mirror626 +1679|677|m|m|s|u|sdw25|sdw25|8677|/data/mirror677 +1723|721|m|m|s|u|sdw25|sdw25|8721|/data/mirror721 +1777|775|m|m|s|u|sdw25|sdw25|8775|/data/mirror775 +1826|824|m|m|s|u|sdw25|sdw25|8824|/data/mirror824 +1884|882|m|m|s|u|sdw25|sdw25|8882|/data/mirror882 +1928|926|m|m|s|u|sdw25|sdw25|8926|/data/mirror926 +1981|979|m|m|s|u|sdw25|sdw25|8979|/data/mirror979 +# SDW26 +524|522|p|p|s|u|sdw26|sdw26|7522|/data/primary522 +525|523|p|p|s|u|sdw26|sdw26|7523|/data/primary523 +526|524|p|p|s|u|sdw26|sdw26|7524|/data/primary524 +527|525|p|p|s|u|sdw26|sdw26|7525|/data/primary525 +528|526|p|p|s|u|sdw26|sdw26|7526|/data/primary526 +529|527|p|p|s|u|sdw26|sdw26|7527|/data/primary527 +530|528|p|p|s|u|sdw26|sdw26|7528|/data/primary528 +531|529|p|p|s|u|sdw26|sdw26|7529|/data/primary529 +532|530|p|p|s|u|sdw26|sdw26|7530|/data/primary530 +533|531|p|p|s|u|sdw26|sdw26|7531|/data/primary531 +534|532|p|p|s|u|sdw26|sdw26|7532|/data/primary532 +535|533|p|p|s|u|sdw26|sdw26|7533|/data/primary533 +536|534|p|p|s|u|sdw26|sdw26|7534|/data/primary534 +537|535|p|p|s|u|sdw26|sdw26|7535|/data/primary535 +538|536|p|p|s|u|sdw26|sdw26|7536|/data/primary536 +1029|27|m|m|s|u|sdw26|sdw26|8027|/data/mirror27 +1082|80|m|m|s|u|sdw26|sdw26|8080|/data/mirror80 +1111|109|m|m|s|u|sdw26|sdw26|8109|/data/mirror109 +1185|183|m|m|s|u|sdw26|sdw26|8183|/data/mirror183 +1233|231|m|m|s|u|sdw26|sdw26|8231|/data/mirror231 +1255|253|m|m|s|u|sdw26|sdw26|8253|/data/mirror253 +1328|326|m|m|s|u|sdw26|sdw26|8326|/data/mirror326 +1377|375|m|m|s|u|sdw26|sdw26|8375|/data/mirror375 +1428|426|m|m|s|u|sdw26|sdw26|8426|/data/mirror426 +1461|459|m|m|s|u|sdw26|sdw26|8459|/data/mirror459 +1506|504|m|m|s|u|sdw26|sdw26|8504|/data/mirror504 +1582|580|m|m|s|u|sdw26|sdw26|8580|/data/mirror580 +1629|627|m|m|s|u|sdw26|sdw26|8627|/data/mirror627 +1680|678|m|m|s|u|sdw26|sdw26|8678|/data/mirror678 +1727|725|m|m|s|u|sdw26|sdw26|8725|/data/mirror725 +1778|776|m|m|s|u|sdw26|sdw26|8776|/data/mirror776 +1827|825|m|m|s|u|sdw26|sdw26|8825|/data/mirror825 +1885|883|m|m|s|u|sdw26|sdw26|8883|/data/mirror883 +1917|915|m|m|s|u|sdw26|sdw26|8915|/data/mirror915 +1983|981|m|m|s|u|sdw26|sdw26|8981|/data/mirror981 +# SDW27 +539|537|p|p|s|u|sdw27|sdw27|7537|/data/primary537 +540|538|p|p|s|u|sdw27|sdw27|7538|/data/primary538 +541|539|p|p|s|u|sdw27|sdw27|7539|/data/primary539 +542|540|p|p|s|u|sdw27|sdw27|7540|/data/primary540 +543|541|p|p|s|u|sdw27|sdw27|7541|/data/primary541 +544|542|p|p|s|u|sdw27|sdw27|7542|/data/primary542 +545|543|p|p|s|u|sdw27|sdw27|7543|/data/primary543 +546|544|p|p|s|u|sdw27|sdw27|7544|/data/primary544 +547|545|p|p|s|u|sdw27|sdw27|7545|/data/primary545 +548|546|p|p|s|u|sdw27|sdw27|7546|/data/primary546 +549|547|p|p|s|u|sdw27|sdw27|7547|/data/primary547 +550|548|p|p|s|u|sdw27|sdw27|7548|/data/primary548 +551|549|p|p|s|u|sdw27|sdw27|7549|/data/primary549 +552|550|p|p|s|u|sdw27|sdw27|7550|/data/primary550 +553|551|p|p|s|u|sdw27|sdw27|7551|/data/primary551 +554|552|p|p|s|u|sdw27|sdw27|7552|/data/primary552 +555|553|p|p|s|u|sdw27|sdw27|7553|/data/primary553 +556|554|p|p|s|u|sdw27|sdw27|7554|/data/primary554 +557|555|p|p|s|u|sdw27|sdw27|7555|/data/primary555 +1030|28|m|m|s|u|sdw27|sdw27|8028|/data/mirror28 +1076|74|m|m|s|u|sdw27|sdw27|8074|/data/mirror74 +1136|134|m|m|s|u|sdw27|sdw27|8134|/data/mirror134 +1186|184|m|m|s|u|sdw27|sdw27|8184|/data/mirror184 +1234|232|m|m|s|u|sdw27|sdw27|8232|/data/mirror232 +1277|275|m|m|s|u|sdw27|sdw27|8275|/data/mirror275 +1314|312|m|m|s|u|sdw27|sdw27|8312|/data/mirror312 +1359|357|m|m|s|u|sdw27|sdw27|8357|/data/mirror357 +1429|427|m|m|s|u|sdw27|sdw27|8427|/data/mirror427 +1483|481|m|m|s|u|sdw27|sdw27|8481|/data/mirror481 +1530|528|m|m|s|u|sdw27|sdw27|8528|/data/mirror528 +1568|566|m|m|s|u|sdw27|sdw27|8566|/data/mirror566 +1630|628|m|m|s|u|sdw27|sdw27|8628|/data/mirror628 +1661|659|m|m|s|u|sdw27|sdw27|8659|/data/mirror659 +1728|726|m|m|s|u|sdw27|sdw27|8726|/data/mirror726 +1780|778|m|m|s|u|sdw27|sdw27|8778|/data/mirror778 +1828|826|m|m|s|u|sdw27|sdw27|8826|/data/mirror826 +1886|884|m|m|s|u|sdw27|sdw27|8884|/data/mirror884 +1930|928|m|m|s|u|sdw27|sdw27|8928|/data/mirror928 +1960|958|m|m|s|u|sdw27|sdw27|8958|/data/mirror958 +# SDW28 +558|556|p|p|s|u|sdw28|sdw28|7556|/data/primary556 +559|557|p|p|s|u|sdw28|sdw28|7557|/data/primary557 +560|558|p|p|s|u|sdw28|sdw28|7558|/data/primary558 +561|559|p|p|s|u|sdw28|sdw28|7559|/data/primary559 +562|560|p|p|s|u|sdw28|sdw28|7560|/data/primary560 +563|561|p|p|s|u|sdw28|sdw28|7561|/data/primary561 +564|562|p|p|s|u|sdw28|sdw28|7562|/data/primary562 +565|563|p|p|s|u|sdw28|sdw28|7563|/data/primary563 +566|564|p|p|s|u|sdw28|sdw28|7564|/data/primary564 +567|565|p|p|s|u|sdw28|sdw28|7565|/data/primary565 +568|566|p|p|s|u|sdw28|sdw28|7566|/data/primary566 +569|567|p|p|s|u|sdw28|sdw28|7567|/data/primary567 +570|568|p|p|s|u|sdw28|sdw28|7568|/data/primary568 +571|569|p|p|s|u|sdw28|sdw28|7569|/data/primary569 +572|570|p|p|s|u|sdw28|sdw28|7570|/data/primary570 +573|571|p|p|s|u|sdw28|sdw28|7571|/data/primary571 +574|572|p|p|s|u|sdw28|sdw28|7572|/data/primary572 +575|573|p|p|s|u|sdw28|sdw28|7573|/data/primary573 +576|574|p|p|s|u|sdw28|sdw28|7574|/data/primary574 +577|575|p|p|s|u|sdw28|sdw28|7575|/data/primary575 +578|576|p|p|s|u|sdw28|sdw28|7576|/data/primary576 +579|577|p|p|s|u|sdw28|sdw28|7577|/data/primary577 +580|578|p|p|s|u|sdw28|sdw28|7578|/data/primary578 +1034|32|m|m|s|u|sdw28|sdw28|8032|/data/mirror32 +1083|81|m|m|s|u|sdw28|sdw28|8081|/data/mirror81 +1112|110|m|m|s|u|sdw28|sdw28|8110|/data/mirror110 +1164|162|m|m|s|u|sdw28|sdw28|8162|/data/mirror162 +1236|234|m|m|s|u|sdw28|sdw28|8234|/data/mirror234 +1282|280|m|m|s|u|sdw28|sdw28|8280|/data/mirror280 +1329|327|m|m|s|u|sdw28|sdw28|8327|/data/mirror327 +1378|376|m|m|s|u|sdw28|sdw28|8376|/data/mirror376 +1430|428|m|m|s|u|sdw28|sdw28|8428|/data/mirror428 +1462|460|m|m|s|u|sdw28|sdw28|8460|/data/mirror460 +1532|530|m|m|s|u|sdw28|sdw28|8530|/data/mirror530 +1583|581|m|m|s|u|sdw28|sdw28|8581|/data/mirror581 +1632|630|m|m|s|u|sdw28|sdw28|8630|/data/mirror630 +1672|670|m|m|s|u|sdw28|sdw28|8670|/data/mirror670 +1729|727|m|m|s|u|sdw28|sdw28|8727|/data/mirror727 +1782|780|m|m|s|u|sdw28|sdw28|8780|/data/mirror780 +1830|828|m|m|s|u|sdw28|sdw28|8828|/data/mirror828 +1874|872|m|m|s|u|sdw28|sdw28|8872|/data/mirror872 +1918|916|m|m|s|u|sdw28|sdw28|8916|/data/mirror916 +1984|982|m|m|s|u|sdw28|sdw28|8982|/data/mirror982 +# SDW29 +581|579|p|p|s|u|sdw29|sdw29|7579|/data/primary579 +582|580|p|p|s|u|sdw29|sdw29|7580|/data/primary580 +583|581|p|p|s|u|sdw29|sdw29|7581|/data/primary581 +584|582|p|p|s|u|sdw29|sdw29|7582|/data/primary582 +585|583|p|p|s|u|sdw29|sdw29|7583|/data/primary583 +586|584|p|p|s|u|sdw29|sdw29|7584|/data/primary584 +587|585|p|p|s|u|sdw29|sdw29|7585|/data/primary585 +588|586|p|p|s|u|sdw29|sdw29|7586|/data/primary586 +589|587|p|p|s|u|sdw29|sdw29|7587|/data/primary587 +590|588|p|p|s|u|sdw29|sdw29|7588|/data/primary588 +591|589|p|p|s|u|sdw29|sdw29|7589|/data/primary589 +592|590|p|p|s|u|sdw29|sdw29|7590|/data/primary590 +593|591|p|p|s|u|sdw29|sdw29|7591|/data/primary591 +594|592|p|p|s|u|sdw29|sdw29|7592|/data/primary592 +595|593|p|p|s|u|sdw29|sdw29|7593|/data/primary593 +596|594|p|p|s|u|sdw29|sdw29|7594|/data/primary594 +1036|34|m|m|s|u|sdw29|sdw29|8034|/data/mirror34 +1084|82|m|m|s|u|sdw29|sdw29|8082|/data/mirror82 +1137|135|m|m|s|u|sdw29|sdw29|8135|/data/mirror135 +1188|186|m|m|s|u|sdw29|sdw29|8186|/data/mirror186 +1235|233|m|m|s|u|sdw29|sdw29|8233|/data/mirror233 +1283|281|m|m|s|u|sdw29|sdw29|8281|/data/mirror281 +1330|328|m|m|s|u|sdw29|sdw29|8328|/data/mirror328 +1379|377|m|m|s|u|sdw29|sdw29|8377|/data/mirror377 +1431|429|m|m|s|u|sdw29|sdw29|8429|/data/mirror429 +1485|483|m|m|s|u|sdw29|sdw29|8483|/data/mirror483 +1533|531|m|m|s|u|sdw29|sdw29|8531|/data/mirror531 +1597|595|m|m|s|u|sdw29|sdw29|8595|/data/mirror595 +1633|631|m|m|s|u|sdw29|sdw29|8631|/data/mirror631 +1683|681|m|m|s|u|sdw29|sdw29|8681|/data/mirror681 +1731|729|m|m|s|u|sdw29|sdw29|8729|/data/mirror729 +1766|764|m|m|s|u|sdw29|sdw29|8764|/data/mirror764 +1831|829|m|m|s|u|sdw29|sdw29|8829|/data/mirror829 +1889|887|m|m|s|u|sdw29|sdw29|8887|/data/mirror887 +1931|929|m|m|s|u|sdw29|sdw29|8929|/data/mirror929 +1961|959|m|m|s|u|sdw29|sdw29|8959|/data/mirror959 +# SDW3 +50|48|p|p|s|u|sdw3|sdw3|7048|/data/primary48 +51|49|p|p|s|u|sdw3|sdw3|7049|/data/primary49 +52|50|p|p|s|u|sdw3|sdw3|7050|/data/primary50 +53|51|p|p|s|u|sdw3|sdw3|7051|/data/primary51 +54|52|p|p|s|u|sdw3|sdw3|7052|/data/primary52 +55|53|p|p|s|u|sdw3|sdw3|7053|/data/primary53 +56|54|p|p|s|u|sdw3|sdw3|7054|/data/primary54 +57|55|p|p|s|u|sdw3|sdw3|7055|/data/primary55 +58|56|p|p|s|u|sdw3|sdw3|7056|/data/primary56 +59|57|p|p|s|u|sdw3|sdw3|7057|/data/primary57 +60|58|p|p|s|u|sdw3|sdw3|7058|/data/primary58 +61|59|p|p|s|u|sdw3|sdw3|7059|/data/primary59 +62|60|p|p|s|u|sdw3|sdw3|7060|/data/primary60 +63|61|p|p|s|u|sdw3|sdw3|7061|/data/primary61 +64|62|p|p|s|u|sdw3|sdw3|7062|/data/primary62 +65|63|p|p|s|u|sdw3|sdw3|7063|/data/primary63 +66|64|p|p|s|u|sdw3|sdw3|7064|/data/primary64 +67|65|p|p|s|u|sdw3|sdw3|7065|/data/primary65 +68|66|p|p|s|u|sdw3|sdw3|7066|/data/primary66 +69|67|p|p|s|u|sdw3|sdw3|7067|/data/primary67 +70|68|p|p|s|u|sdw3|sdw3|7068|/data/primary68 +71|69|p|p|s|u|sdw3|sdw3|7069|/data/primary69 +1003|1|m|m|s|u|sdw3|sdw3|8001|/data/mirror1 +1072|70|m|m|s|u|sdw3|sdw3|8070|/data/mirror70 +1105|103|m|m|s|u|sdw3|sdw3|8103|/data/mirror103 +1135|133|m|m|s|u|sdw3|sdw3|8133|/data/mirror133 +1210|208|m|m|s|u|sdw3|sdw3|8208|/data/mirror208 +1256|254|m|m|s|u|sdw3|sdw3|8254|/data/mirror254 +1304|302|m|m|s|u|sdw3|sdw3|8302|/data/mirror302 +1357|355|m|m|s|u|sdw3|sdw3|8355|/data/mirror355 +1406|404|m|m|s|u|sdw3|sdw3|8404|/data/mirror404 +1458|456|m|m|s|u|sdw3|sdw3|8456|/data/mirror456 +1484|482|m|m|s|u|sdw3|sdw3|8482|/data/mirror482 +1559|557|m|m|s|u|sdw3|sdw3|8557|/data/mirror557 +1605|603|m|m|s|u|sdw3|sdw3|8603|/data/mirror603 +1650|648|m|m|s|u|sdw3|sdw3|8648|/data/mirror648 +1713|711|m|m|s|u|sdw3|sdw3|8711|/data/mirror711 +1759|757|m|m|s|u|sdw3|sdw3|8757|/data/mirror757 +1810|808|m|m|s|u|sdw3|sdw3|8808|/data/mirror808 +1855|853|m|m|s|u|sdw3|sdw3|8853|/data/mirror853 +1908|906|m|m|s|u|sdw3|sdw3|8906|/data/mirror906 +1940|938|m|m|s|u|sdw3|sdw3|8938|/data/mirror938 +1998|996|m|m|s|u|sdw3|sdw3|8996|/data/mirror996 +# SDW30 +597|595|p|p|s|u|sdw30|sdw30|7595|/data/primary595 +598|596|p|p|s|u|sdw30|sdw30|7596|/data/primary596 +599|597|p|p|s|u|sdw30|sdw30|7597|/data/primary597 +600|598|p|p|s|u|sdw30|sdw30|7598|/data/primary598 +601|599|p|p|s|u|sdw30|sdw30|7599|/data/primary599 +602|600|p|p|s|u|sdw30|sdw30|7600|/data/primary600 +603|601|p|p|s|u|sdw30|sdw30|7601|/data/primary601 +604|602|p|p|s|u|sdw30|sdw30|7602|/data/primary602 +605|603|p|p|s|u|sdw30|sdw30|7603|/data/primary603 +606|604|p|p|s|u|sdw30|sdw30|7604|/data/primary604 +607|605|p|p|s|u|sdw30|sdw30|7605|/data/primary605 +608|606|p|p|s|u|sdw30|sdw30|7606|/data/primary606 +609|607|p|p|s|u|sdw30|sdw30|7607|/data/primary607 +610|608|p|p|s|u|sdw30|sdw30|7608|/data/primary608 +611|609|p|p|s|u|sdw30|sdw30|7609|/data/primary609 +612|610|p|p|s|u|sdw30|sdw30|7610|/data/primary610 +613|611|p|p|s|u|sdw30|sdw30|7611|/data/primary611 +614|612|p|p|s|u|sdw30|sdw30|7612|/data/primary612 +615|613|p|p|s|u|sdw30|sdw30|7613|/data/primary613 +616|614|p|p|s|u|sdw30|sdw30|7614|/data/primary614 +617|615|p|p|s|u|sdw30|sdw30|7615|/data/primary615 +618|616|p|p|s|u|sdw30|sdw30|7616|/data/primary616 +619|617|p|p|s|u|sdw30|sdw30|7617|/data/primary617 +1037|35|m|m|s|u|sdw30|sdw30|8035|/data/mirror35 +1085|83|m|m|s|u|sdw30|sdw30|8083|/data/mirror83 +1113|111|m|m|s|u|sdw30|sdw30|8111|/data/mirror111 +1177|175|m|m|s|u|sdw30|sdw30|8175|/data/mirror175 +1237|235|m|m|s|u|sdw30|sdw30|8235|/data/mirror235 +1286|284|m|m|s|u|sdw30|sdw30|8284|/data/mirror284 +1332|330|m|m|s|u|sdw30|sdw30|8330|/data/mirror330 +1380|378|m|m|s|u|sdw30|sdw30|8378|/data/mirror378 +1432|430|m|m|s|u|sdw30|sdw30|8430|/data/mirror430 +1463|461|m|m|s|u|sdw30|sdw30|8461|/data/mirror461 +1534|532|m|m|s|u|sdw30|sdw30|8532|/data/mirror532 +1569|567|m|m|s|u|sdw30|sdw30|8567|/data/mirror567 +1635|633|m|m|s|u|sdw30|sdw30|8633|/data/mirror633 +1684|682|m|m|s|u|sdw30|sdw30|8682|/data/mirror682 +1732|730|m|m|s|u|sdw30|sdw30|8730|/data/mirror730 +1783|781|m|m|s|u|sdw30|sdw30|8781|/data/mirror781 +1832|830|m|m|s|u|sdw30|sdw30|8830|/data/mirror830 +1862|860|m|m|s|u|sdw30|sdw30|8860|/data/mirror860 +1919|917|m|m|s|u|sdw30|sdw30|8917|/data/mirror917 +1985|983|m|m|s|u|sdw30|sdw30|8983|/data/mirror983 +# SDW31 +620|618|p|p|s|u|sdw31|sdw31|7618|/data/primary618 +621|619|p|p|s|u|sdw31|sdw31|7619|/data/primary619 +622|620|p|p|s|u|sdw31|sdw31|7620|/data/primary620 +623|621|p|p|s|u|sdw31|sdw31|7621|/data/primary621 +624|622|p|p|s|u|sdw31|sdw31|7622|/data/primary622 +625|623|p|p|s|u|sdw31|sdw31|7623|/data/primary623 +626|624|p|p|s|u|sdw31|sdw31|7624|/data/primary624 +627|625|p|p|s|u|sdw31|sdw31|7625|/data/primary625 +628|626|p|p|s|u|sdw31|sdw31|7626|/data/primary626 +629|627|p|p|s|u|sdw31|sdw31|7627|/data/primary627 +630|628|p|p|s|u|sdw31|sdw31|7628|/data/primary628 +631|629|p|p|s|u|sdw31|sdw31|7629|/data/primary629 +632|630|p|p|s|u|sdw31|sdw31|7630|/data/primary630 +633|631|p|p|s|u|sdw31|sdw31|7631|/data/primary631 +634|632|p|p|s|u|sdw31|sdw31|7632|/data/primary632 +635|633|p|p|s|u|sdw31|sdw31|7633|/data/primary633 +636|634|p|p|s|u|sdw31|sdw31|7634|/data/primary634 +637|635|p|p|s|u|sdw31|sdw31|7635|/data/primary635 +638|636|p|p|s|u|sdw31|sdw31|7636|/data/primary636 +639|637|p|p|s|u|sdw31|sdw31|7637|/data/primary637 +640|638|p|p|s|u|sdw31|sdw31|7638|/data/primary638 +641|639|p|p|s|u|sdw31|sdw31|7639|/data/primary639 +642|640|p|p|s|u|sdw31|sdw31|7640|/data/primary640 +643|641|p|p|s|u|sdw31|sdw31|7641|/data/primary641 +1038|36|m|m|s|u|sdw31|sdw31|8036|/data/mirror36 +1078|76|m|m|s|u|sdw31|sdw31|8076|/data/mirror76 +1124|122|m|m|s|u|sdw31|sdw31|8122|/data/mirror122 +1190|188|m|m|s|u|sdw31|sdw31|8188|/data/mirror188 +1238|236|m|m|s|u|sdw31|sdw31|8236|/data/mirror236 +1279|277|m|m|s|u|sdw31|sdw31|8277|/data/mirror277 +1333|331|m|m|s|u|sdw31|sdw31|8331|/data/mirror331 +1381|379|m|m|s|u|sdw31|sdw31|8379|/data/mirror379 +1433|431|m|m|s|u|sdw31|sdw31|8431|/data/mirror431 +1489|487|m|m|s|u|sdw31|sdw31|8487|/data/mirror487 +1536|534|m|m|s|u|sdw31|sdw31|8534|/data/mirror534 +1584|582|m|m|s|u|sdw31|sdw31|8582|/data/mirror582 +1644|642|m|m|s|u|sdw31|sdw31|8642|/data/mirror642 +1663|661|m|m|s|u|sdw31|sdw31|8661|/data/mirror661 +1733|731|m|m|s|u|sdw31|sdw31|8731|/data/mirror731 +1784|782|m|m|s|u|sdw31|sdw31|8782|/data/mirror782 +1835|833|m|m|s|u|sdw31|sdw31|8833|/data/mirror833 +1875|873|m|m|s|u|sdw31|sdw31|8873|/data/mirror873 +1933|931|m|m|s|u|sdw31|sdw31|8931|/data/mirror931 +1978|976|m|m|s|u|sdw31|sdw31|8976|/data/mirror976 +# SDW32 +644|642|p|p|s|u|sdw32|sdw32|7642|/data/primary642 +645|643|p|p|s|u|sdw32|sdw32|7643|/data/primary643 +646|644|p|p|s|u|sdw32|sdw32|7644|/data/primary644 +647|645|p|p|s|u|sdw32|sdw32|7645|/data/primary645 +648|646|p|p|s|u|sdw32|sdw32|7646|/data/primary646 +649|647|p|p|s|u|sdw32|sdw32|7647|/data/primary647 +650|648|p|p|s|u|sdw32|sdw32|7648|/data/primary648 +651|649|p|p|s|u|sdw32|sdw32|7649|/data/primary649 +652|650|p|p|s|u|sdw32|sdw32|7650|/data/primary650 +653|651|p|p|s|u|sdw32|sdw32|7651|/data/primary651 +654|652|p|p|s|u|sdw32|sdw32|7652|/data/primary652 +655|653|p|p|s|u|sdw32|sdw32|7653|/data/primary653 +656|654|p|p|s|u|sdw32|sdw32|7654|/data/primary654 +657|655|p|p|s|u|sdw32|sdw32|7655|/data/primary655 +658|656|p|p|s|u|sdw32|sdw32|7656|/data/primary656 +659|657|p|p|s|u|sdw32|sdw32|7657|/data/primary657 +660|658|p|p|s|u|sdw32|sdw32|7658|/data/primary658 +661|659|p|p|s|u|sdw32|sdw32|7659|/data/primary659 +662|660|p|p|s|u|sdw32|sdw32|7660|/data/primary660 +663|661|p|p|s|u|sdw32|sdw32|7661|/data/primary661 +1017|15|m|m|s|u|sdw32|sdw32|8015|/data/mirror15 +1086|84|m|m|s|u|sdw32|sdw32|8084|/data/mirror84 +1138|136|m|m|s|u|sdw32|sdw32|8136|/data/mirror136 +1178|176|m|m|s|u|sdw32|sdw32|8176|/data/mirror176 +1225|223|m|m|s|u|sdw32|sdw32|8223|/data/mirror223 +1269|267|m|m|s|u|sdw32|sdw32|8267|/data/mirror267 +1335|333|m|m|s|u|sdw32|sdw32|8333|/data/mirror333 +1382|380|m|m|s|u|sdw32|sdw32|8380|/data/mirror380 +1427|425|m|m|s|u|sdw32|sdw32|8425|/data/mirror425 +1464|462|m|m|s|u|sdw32|sdw32|8462|/data/mirror462 +1537|535|m|m|s|u|sdw32|sdw32|8535|/data/mirror535 +1570|568|m|m|s|u|sdw32|sdw32|8568|/data/mirror568 +1636|634|m|m|s|u|sdw32|sdw32|8634|/data/mirror634 +1685|683|m|m|s|u|sdw32|sdw32|8683|/data/mirror683 +1735|733|m|m|s|u|sdw32|sdw32|8733|/data/mirror733 +1785|783|m|m|s|u|sdw32|sdw32|8783|/data/mirror783 +1836|834|m|m|s|u|sdw32|sdw32|8834|/data/mirror834 +1863|861|m|m|s|u|sdw32|sdw32|8861|/data/mirror861 +1934|932|m|m|s|u|sdw32|sdw32|8932|/data/mirror932 +1986|984|m|m|s|u|sdw32|sdw32|8984|/data/mirror984 +# SDW33 +664|662|p|p|s|u|sdw33|sdw33|7662|/data/primary662 +665|663|p|p|s|u|sdw33|sdw33|7663|/data/primary663 +666|664|p|p|s|u|sdw33|sdw33|7664|/data/primary664 +667|665|p|p|s|u|sdw33|sdw33|7665|/data/primary665 +668|666|p|p|s|u|sdw33|sdw33|7666|/data/primary666 +669|667|p|p|s|u|sdw33|sdw33|7667|/data/primary667 +670|668|p|p|s|u|sdw33|sdw33|7668|/data/primary668 +671|669|p|p|s|u|sdw33|sdw33|7669|/data/primary669 +672|670|p|p|s|u|sdw33|sdw33|7670|/data/primary670 +673|671|p|p|s|u|sdw33|sdw33|7671|/data/primary671 +674|672|p|p|s|u|sdw33|sdw33|7672|/data/primary672 +675|673|p|p|s|u|sdw33|sdw33|7673|/data/primary673 +676|674|p|p|s|u|sdw33|sdw33|7674|/data/primary674 +677|675|p|p|s|u|sdw33|sdw33|7675|/data/primary675 +678|676|p|p|s|u|sdw33|sdw33|7676|/data/primary676 +679|677|p|p|s|u|sdw33|sdw33|7677|/data/primary677 +680|678|p|p|s|u|sdw33|sdw33|7678|/data/primary678 +681|679|p|p|s|u|sdw33|sdw33|7679|/data/primary679 +682|680|p|p|s|u|sdw33|sdw33|7680|/data/primary680 +683|681|p|p|s|u|sdw33|sdw33|7681|/data/primary681 +684|682|p|p|s|u|sdw33|sdw33|7682|/data/primary682 +685|683|p|p|s|u|sdw33|sdw33|7683|/data/primary683 +686|684|p|p|s|u|sdw33|sdw33|7684|/data/primary684 +1039|37|m|m|s|u|sdw33|sdw33|8037|/data/mirror37 +1079|77|m|m|s|u|sdw33|sdw33|8077|/data/mirror77 +1140|138|m|m|s|u|sdw33|sdw33|8138|/data/mirror138 +1191|189|m|m|s|u|sdw33|sdw33|8189|/data/mirror189 +1213|211|m|m|s|u|sdw33|sdw33|8211|/data/mirror211 +1287|285|m|m|s|u|sdw33|sdw33|8285|/data/mirror285 +1336|334|m|m|s|u|sdw33|sdw33|8334|/data/mirror334 +1385|383|m|m|s|u|sdw33|sdw33|8383|/data/mirror383 +1435|433|m|m|s|u|sdw33|sdw33|8433|/data/mirror433 +1474|472|m|m|s|u|sdw33|sdw33|8472|/data/mirror472 +1520|518|m|m|s|u|sdw33|sdw33|8518|/data/mirror518 +1585|583|m|m|s|u|sdw33|sdw33|8583|/data/mirror583 +1627|625|m|m|s|u|sdw33|sdw33|8625|/data/mirror625 +1687|685|m|m|s|u|sdw33|sdw33|8685|/data/mirror685 +1736|734|m|m|s|u|sdw33|sdw33|8734|/data/mirror734 +1786|784|m|m|s|u|sdw33|sdw33|8784|/data/mirror784 +1837|835|m|m|s|u|sdw33|sdw33|8835|/data/mirror835 +1890|888|m|m|s|u|sdw33|sdw33|8888|/data/mirror888 +1941|939|m|m|s|u|sdw33|sdw33|8939|/data/mirror939 +1987|985|m|m|s|u|sdw33|sdw33|8985|/data/mirror985 +# SDW34 +687|685|p|p|s|u|sdw34|sdw34|7685|/data/primary685 +688|686|p|p|s|u|sdw34|sdw34|7686|/data/primary686 +689|687|p|p|s|u|sdw34|sdw34|7687|/data/primary687 +690|688|p|p|s|u|sdw34|sdw34|7688|/data/primary688 +691|689|p|p|s|u|sdw34|sdw34|7689|/data/primary689 +692|690|p|p|s|u|sdw34|sdw34|7690|/data/primary690 +693|691|p|p|s|u|sdw34|sdw34|7691|/data/primary691 +694|692|p|p|s|u|sdw34|sdw34|7692|/data/primary692 +695|693|p|p|s|u|sdw34|sdw34|7693|/data/primary693 +696|694|p|p|s|u|sdw34|sdw34|7694|/data/primary694 +697|695|p|p|s|u|sdw34|sdw34|7695|/data/primary695 +698|696|p|p|s|u|sdw34|sdw34|7696|/data/primary696 +699|697|p|p|s|u|sdw34|sdw34|7697|/data/primary697 +700|698|p|p|s|u|sdw34|sdw34|7698|/data/primary698 +701|699|p|p|s|u|sdw34|sdw34|7699|/data/primary699 +702|700|p|p|s|u|sdw34|sdw34|7700|/data/primary700 +703|701|p|p|s|u|sdw34|sdw34|7701|/data/primary701 +1031|29|m|m|s|u|sdw34|sdw34|8029|/data/mirror29 +1087|85|m|m|s|u|sdw34|sdw34|8085|/data/mirror85 +1139|137|m|m|s|u|sdw34|sdw34|8137|/data/mirror137 +1167|165|m|m|s|u|sdw34|sdw34|8165|/data/mirror165 +1240|238|m|m|s|u|sdw34|sdw34|8238|/data/mirror238 +1270|268|m|m|s|u|sdw34|sdw34|8268|/data/mirror268 +1338|336|m|m|s|u|sdw34|sdw34|8336|/data/mirror336 +1384|382|m|m|s|u|sdw34|sdw34|8382|/data/mirror382 +1436|434|m|m|s|u|sdw34|sdw34|8434|/data/mirror434 +1487|485|m|m|s|u|sdw34|sdw34|8485|/data/mirror485 +1538|536|m|m|s|u|sdw34|sdw34|8536|/data/mirror536 +1587|585|m|m|s|u|sdw34|sdw34|8585|/data/mirror585 +1637|635|m|m|s|u|sdw34|sdw34|8635|/data/mirror635 +1704|702|m|m|s|u|sdw34|sdw34|8702|/data/mirror702 +1737|735|m|m|s|u|sdw34|sdw34|8735|/data/mirror735 +1787|785|m|m|s|u|sdw34|sdw34|8785|/data/mirror785 +1838|836|m|m|s|u|sdw34|sdw34|8836|/data/mirror836 +1864|862|m|m|s|u|sdw34|sdw34|8862|/data/mirror862 +1942|940|m|m|s|u|sdw34|sdw34|8940|/data/mirror940 +1988|986|m|m|s|u|sdw34|sdw34|8986|/data/mirror986 +# SDW35 +704|702|p|p|s|u|sdw35|sdw35|7702|/data/primary702 +705|703|p|p|s|u|sdw35|sdw35|7703|/data/primary703 +706|704|p|p|s|u|sdw35|sdw35|7704|/data/primary704 +707|705|p|p|s|u|sdw35|sdw35|7705|/data/primary705 +708|706|p|p|s|u|sdw35|sdw35|7706|/data/primary706 +709|707|p|p|s|u|sdw35|sdw35|7707|/data/primary707 +710|708|p|p|s|u|sdw35|sdw35|7708|/data/primary708 +711|709|p|p|s|u|sdw35|sdw35|7709|/data/primary709 +712|710|p|p|s|u|sdw35|sdw35|7710|/data/primary710 +713|711|p|p|s|u|sdw35|sdw35|7711|/data/primary711 +714|712|p|p|s|u|sdw35|sdw35|7712|/data/primary712 +715|713|p|p|s|u|sdw35|sdw35|7713|/data/primary713 +716|714|p|p|s|u|sdw35|sdw35|7714|/data/primary714 +717|715|p|p|s|u|sdw35|sdw35|7715|/data/primary715 +718|716|p|p|s|u|sdw35|sdw35|7716|/data/primary716 +719|717|p|p|s|u|sdw35|sdw35|7717|/data/primary717 +1040|38|m|m|s|u|sdw35|sdw35|8038|/data/mirror38 +1088|86|m|m|s|u|sdw35|sdw35|8086|/data/mirror86 +1142|140|m|m|s|u|sdw35|sdw35|8140|/data/mirror140 +1192|190|m|m|s|u|sdw35|sdw35|8190|/data/mirror190 +1214|212|m|m|s|u|sdw35|sdw35|8212|/data/mirror212 +1289|287|m|m|s|u|sdw35|sdw35|8287|/data/mirror287 +1326|324|m|m|s|u|sdw35|sdw35|8324|/data/mirror324 +1386|384|m|m|s|u|sdw35|sdw35|8384|/data/mirror384 +1437|435|m|m|s|u|sdw35|sdw35|8435|/data/mirror435 +1490|488|m|m|s|u|sdw35|sdw35|8488|/data/mirror488 +1539|537|m|m|s|u|sdw35|sdw35|8537|/data/mirror537 +1588|586|m|m|s|u|sdw35|sdw35|8586|/data/mirror586 +1638|636|m|m|s|u|sdw35|sdw35|8636|/data/mirror636 +1690|688|m|m|s|u|sdw35|sdw35|8688|/data/mirror688 +1738|736|m|m|s|u|sdw35|sdw35|8736|/data/mirror736 +1790|788|m|m|s|u|sdw35|sdw35|8788|/data/mirror788 +1839|837|m|m|s|u|sdw35|sdw35|8837|/data/mirror837 +1891|889|m|m|s|u|sdw35|sdw35|8889|/data/mirror889 +1943|941|m|m|s|u|sdw35|sdw35|8941|/data/mirror941 +1990|988|m|m|s|u|sdw35|sdw35|8988|/data/mirror988 +# SDW36 +720|718|p|p|s|u|sdw36|sdw36|7718|/data/primary718 +721|719|p|p|s|u|sdw36|sdw36|7719|/data/primary719 +722|720|p|p|s|u|sdw36|sdw36|7720|/data/primary720 +723|721|p|p|s|u|sdw36|sdw36|7721|/data/primary721 +724|722|p|p|s|u|sdw36|sdw36|7722|/data/primary722 +725|723|p|p|s|u|sdw36|sdw36|7723|/data/primary723 +726|724|p|p|s|u|sdw36|sdw36|7724|/data/primary724 +727|725|p|p|s|u|sdw36|sdw36|7725|/data/primary725 +728|726|p|p|s|u|sdw36|sdw36|7726|/data/primary726 +729|727|p|p|s|u|sdw36|sdw36|7727|/data/primary727 +730|728|p|p|s|u|sdw36|sdw36|7728|/data/primary728 +731|729|p|p|s|u|sdw36|sdw36|7729|/data/primary729 +732|730|p|p|s|u|sdw36|sdw36|7730|/data/primary730 +733|731|p|p|s|u|sdw36|sdw36|7731|/data/primary731 +734|732|p|p|s|u|sdw36|sdw36|7732|/data/primary732 +1032|30|m|m|s|u|sdw36|sdw36|8030|/data/mirror30 +1068|66|m|m|s|u|sdw36|sdw36|8066|/data/mirror66 +1143|141|m|m|s|u|sdw36|sdw36|8141|/data/mirror141 +1168|166|m|m|s|u|sdw36|sdw36|8166|/data/mirror166 +1242|240|m|m|s|u|sdw36|sdw36|8240|/data/mirror240 +1290|288|m|m|s|u|sdw36|sdw36|8288|/data/mirror288 +1342|340|m|m|s|u|sdw36|sdw36|8340|/data/mirror340 +1387|385|m|m|s|u|sdw36|sdw36|8385|/data/mirror385 +1438|436|m|m|s|u|sdw36|sdw36|8436|/data/mirror436 +1488|486|m|m|s|u|sdw36|sdw36|8486|/data/mirror486 +1531|529|m|m|s|u|sdw36|sdw36|8529|/data/mirror529 +1572|570|m|m|s|u|sdw36|sdw36|8570|/data/mirror570 +1639|637|m|m|s|u|sdw36|sdw36|8637|/data/mirror637 +1688|686|m|m|s|u|sdw36|sdw36|8686|/data/mirror686 +1742|740|m|m|s|u|sdw36|sdw36|8740|/data/mirror740 +1791|789|m|m|s|u|sdw36|sdw36|8789|/data/mirror789 +1829|827|m|m|s|u|sdw36|sdw36|8827|/data/mirror827 +1892|890|m|m|s|u|sdw36|sdw36|8890|/data/mirror890 +1932|930|m|m|s|u|sdw36|sdw36|8930|/data/mirror930 +1991|989|m|m|s|u|sdw36|sdw36|8989|/data/mirror989 +# SDW37 +735|733|p|p|s|u|sdw37|sdw37|7733|/data/primary733 +736|734|p|p|s|u|sdw37|sdw37|7734|/data/primary734 +737|735|p|p|s|u|sdw37|sdw37|7735|/data/primary735 +738|736|p|p|s|u|sdw37|sdw37|7736|/data/primary736 +739|737|p|p|s|u|sdw37|sdw37|7737|/data/primary737 +740|738|p|p|s|u|sdw37|sdw37|7738|/data/primary738 +741|739|p|p|s|u|sdw37|sdw37|7739|/data/primary739 +742|740|p|p|s|u|sdw37|sdw37|7740|/data/primary740 +743|741|p|p|s|u|sdw37|sdw37|7741|/data/primary741 +744|742|p|p|s|u|sdw37|sdw37|7742|/data/primary742 +745|743|p|p|s|u|sdw37|sdw37|7743|/data/primary743 +746|744|p|p|s|u|sdw37|sdw37|7744|/data/primary744 +747|745|p|p|s|u|sdw37|sdw37|7745|/data/primary745 +748|746|p|p|s|u|sdw37|sdw37|7746|/data/primary746 +749|747|p|p|s|u|sdw37|sdw37|7747|/data/primary747 +750|748|p|p|s|u|sdw37|sdw37|7748|/data/primary748 +751|749|p|p|s|u|sdw37|sdw37|7749|/data/primary749 +1042|40|m|m|s|u|sdw37|sdw37|8040|/data/mirror40 +1089|87|m|m|s|u|sdw37|sdw37|8087|/data/mirror87 +1144|142|m|m|s|u|sdw37|sdw37|8142|/data/mirror142 +1193|191|m|m|s|u|sdw37|sdw37|8191|/data/mirror191 +1243|241|m|m|s|u|sdw37|sdw37|8241|/data/mirror241 +1291|289|m|m|s|u|sdw37|sdw37|8289|/data/mirror289 +1343|341|m|m|s|u|sdw37|sdw37|8341|/data/mirror341 +1389|387|m|m|s|u|sdw37|sdw37|8387|/data/mirror387 +1439|437|m|m|s|u|sdw37|sdw37|8437|/data/mirror437 +1493|491|m|m|s|u|sdw37|sdw37|8491|/data/mirror491 +1540|538|m|m|s|u|sdw37|sdw37|8538|/data/mirror538 +1589|587|m|m|s|u|sdw37|sdw37|8587|/data/mirror587 +1640|638|m|m|s|u|sdw37|sdw37|8638|/data/mirror638 +1693|691|m|m|s|u|sdw37|sdw37|8691|/data/mirror691 +1752|750|m|m|s|u|sdw37|sdw37|8750|/data/mirror750 +1792|790|m|m|s|u|sdw37|sdw37|8790|/data/mirror790 +1840|838|m|m|s|u|sdw37|sdw37|8838|/data/mirror838 +1894|892|m|m|s|u|sdw37|sdw37|8892|/data/mirror892 +1944|942|m|m|s|u|sdw37|sdw37|8942|/data/mirror942 +1965|963|m|m|s|u|sdw37|sdw37|8963|/data/mirror963 +# SDW38 +752|750|p|p|s|u|sdw38|sdw38|7750|/data/primary750 +753|751|p|p|s|u|sdw38|sdw38|7751|/data/primary751 +754|752|p|p|s|u|sdw38|sdw38|7752|/data/primary752 +755|753|p|p|s|u|sdw38|sdw38|7753|/data/primary753 +756|754|p|p|s|u|sdw38|sdw38|7754|/data/primary754 +757|755|p|p|s|u|sdw38|sdw38|7755|/data/primary755 +758|756|p|p|s|u|sdw38|sdw38|7756|/data/primary756 +759|757|p|p|s|u|sdw38|sdw38|7757|/data/primary757 +760|758|p|p|s|u|sdw38|sdw38|7758|/data/primary758 +761|759|p|p|s|u|sdw38|sdw38|7759|/data/primary759 +762|760|p|p|s|u|sdw38|sdw38|7760|/data/primary760 +763|761|p|p|s|u|sdw38|sdw38|7761|/data/primary761 +764|762|p|p|s|u|sdw38|sdw38|7762|/data/primary762 +765|763|p|p|s|u|sdw38|sdw38|7763|/data/primary763 +766|764|p|p|s|u|sdw38|sdw38|7764|/data/primary764 +1033|31|m|m|s|u|sdw38|sdw38|8031|/data/mirror31 +1090|88|m|m|s|u|sdw38|sdw38|8088|/data/mirror88 +1141|139|m|m|s|u|sdw38|sdw38|8139|/data/mirror139 +1180|178|m|m|s|u|sdw38|sdw38|8178|/data/mirror178 +1244|242|m|m|s|u|sdw38|sdw38|8242|/data/mirror242 +1292|290|m|m|s|u|sdw38|sdw38|8290|/data/mirror290 +1344|342|m|m|s|u|sdw38|sdw38|8342|/data/mirror342 +1390|388|m|m|s|u|sdw38|sdw38|8388|/data/mirror388 +1417|415|m|m|s|u|sdw38|sdw38|8415|/data/mirror415 +1495|493|m|m|s|u|sdw38|sdw38|8493|/data/mirror493 +1541|539|m|m|s|u|sdw38|sdw38|8539|/data/mirror539 +1590|588|m|m|s|u|sdw38|sdw38|8588|/data/mirror588 +1643|641|m|m|s|u|sdw38|sdw38|8641|/data/mirror641 +1689|687|m|m|s|u|sdw38|sdw38|8687|/data/mirror687 +1743|741|m|m|s|u|sdw38|sdw38|8741|/data/mirror741 +1793|791|m|m|s|u|sdw38|sdw38|8791|/data/mirror791 +1841|839|m|m|s|u|sdw38|sdw38|8839|/data/mirror839 +1895|893|m|m|s|u|sdw38|sdw38|8893|/data/mirror893 +1945|943|m|m|s|u|sdw38|sdw38|8943|/data/mirror943 +1992|990|m|m|s|u|sdw38|sdw38|8990|/data/mirror990 +# SDW39 +767|765|p|p|s|u|sdw39|sdw39|7765|/data/primary765 +768|766|p|p|s|u|sdw39|sdw39|7766|/data/primary766 +769|767|p|p|s|u|sdw39|sdw39|7767|/data/primary767 +770|768|p|p|s|u|sdw39|sdw39|7768|/data/primary768 +771|769|p|p|s|u|sdw39|sdw39|7769|/data/primary769 +772|770|p|p|s|u|sdw39|sdw39|7770|/data/primary770 +773|771|p|p|s|u|sdw39|sdw39|7771|/data/primary771 +774|772|p|p|s|u|sdw39|sdw39|7772|/data/primary772 +775|773|p|p|s|u|sdw39|sdw39|7773|/data/primary773 +776|774|p|p|s|u|sdw39|sdw39|7774|/data/primary774 +777|775|p|p|s|u|sdw39|sdw39|7775|/data/primary775 +778|776|p|p|s|u|sdw39|sdw39|7776|/data/primary776 +779|777|p|p|s|u|sdw39|sdw39|7777|/data/primary777 +780|778|p|p|s|u|sdw39|sdw39|7778|/data/primary778 +781|779|p|p|s|u|sdw39|sdw39|7779|/data/primary779 +782|780|p|p|s|u|sdw39|sdw39|7780|/data/primary780 +783|781|p|p|s|u|sdw39|sdw39|7781|/data/primary781 +784|782|p|p|s|u|sdw39|sdw39|7782|/data/primary782 +785|783|p|p|s|u|sdw39|sdw39|7783|/data/primary783 +786|784|p|p|s|u|sdw39|sdw39|7784|/data/primary784 +787|785|p|p|s|u|sdw39|sdw39|7785|/data/primary785 +788|786|p|p|s|u|sdw39|sdw39|7786|/data/primary786 +789|787|p|p|s|u|sdw39|sdw39|7787|/data/primary787 +790|788|p|p|s|u|sdw39|sdw39|7788|/data/primary788 +791|789|p|p|s|u|sdw39|sdw39|7789|/data/primary789 +792|790|p|p|s|u|sdw39|sdw39|7790|/data/primary790 +1046|44|m|m|s|u|sdw39|sdw39|8044|/data/mirror44 +1091|89|m|m|s|u|sdw39|sdw39|8089|/data/mirror89 +1128|126|m|m|s|u|sdw39|sdw39|8126|/data/mirror126 +1194|192|m|m|s|u|sdw39|sdw39|8192|/data/mirror192 +1245|243|m|m|s|u|sdw39|sdw39|8243|/data/mirror243 +1293|291|m|m|s|u|sdw39|sdw39|8291|/data/mirror291 +1345|343|m|m|s|u|sdw39|sdw39|8343|/data/mirror343 +1391|389|m|m|s|u|sdw39|sdw39|8389|/data/mirror389 +1440|438|m|m|s|u|sdw39|sdw39|8438|/data/mirror438 +1477|475|m|m|s|u|sdw39|sdw39|8475|/data/mirror475 +1523|521|m|m|s|u|sdw39|sdw39|8521|/data/mirror521 +1592|590|m|m|s|u|sdw39|sdw39|8590|/data/mirror590 +1618|616|m|m|s|u|sdw39|sdw39|8616|/data/mirror616 +1695|693|m|m|s|u|sdw39|sdw39|8693|/data/mirror693 +1744|742|m|m|s|u|sdw39|sdw39|8742|/data/mirror742 +1794|792|m|m|s|u|sdw39|sdw39|8792|/data/mirror792 +1843|841|m|m|s|u|sdw39|sdw39|8841|/data/mirror841 +1896|894|m|m|s|u|sdw39|sdw39|8894|/data/mirror894 +1946|944|m|m|s|u|sdw39|sdw39|8944|/data/mirror944 +1993|991|m|m|s|u|sdw39|sdw39|8991|/data/mirror991 +# SDW4 +72|70|p|p|s|u|sdw4|sdw4|7070|/data/primary70 +73|71|p|p|s|u|sdw4|sdw4|7071|/data/primary71 +74|72|p|p|s|u|sdw4|sdw4|7072|/data/primary72 +75|73|p|p|s|u|sdw4|sdw4|7073|/data/primary73 +76|74|p|p|s|u|sdw4|sdw4|7074|/data/primary74 +77|75|p|p|s|u|sdw4|sdw4|7075|/data/primary75 +78|76|p|p|s|u|sdw4|sdw4|7076|/data/primary76 +79|77|p|p|s|u|sdw4|sdw4|7077|/data/primary77 +80|78|p|p|s|u|sdw4|sdw4|7078|/data/primary78 +81|79|p|p|s|u|sdw4|sdw4|7079|/data/primary79 +82|80|p|p|s|u|sdw4|sdw4|7080|/data/primary80 +83|81|p|p|s|u|sdw4|sdw4|7081|/data/primary81 +84|82|p|p|s|u|sdw4|sdw4|7082|/data/primary82 +85|83|p|p|s|u|sdw4|sdw4|7083|/data/primary83 +86|84|p|p|s|u|sdw4|sdw4|7084|/data/primary84 +87|85|p|p|s|u|sdw4|sdw4|7085|/data/primary85 +88|86|p|p|s|u|sdw4|sdw4|7086|/data/primary86 +89|87|p|p|s|u|sdw4|sdw4|7087|/data/primary87 +90|88|p|p|s|u|sdw4|sdw4|7088|/data/primary88 +91|89|p|p|s|u|sdw4|sdw4|7089|/data/primary89 +92|90|p|p|s|u|sdw4|sdw4|7090|/data/primary90 +93|91|p|p|s|u|sdw4|sdw4|7091|/data/primary91 +94|92|p|p|s|u|sdw4|sdw4|7092|/data/primary92 +95|93|p|p|s|u|sdw4|sdw4|7093|/data/primary93 +1004|2|m|m|s|u|sdw4|sdw4|8002|/data/mirror2 +1059|57|m|m|s|u|sdw4|sdw4|8057|/data/mirror57 +1106|104|m|m|s|u|sdw4|sdw4|8104|/data/mirror104 +1158|156|m|m|s|u|sdw4|sdw4|8156|/data/mirror156 +1211|209|m|m|s|u|sdw4|sdw4|8209|/data/mirror209 +1257|255|m|m|s|u|sdw4|sdw4|8255|/data/mirror255 +1306|304|m|m|s|u|sdw4|sdw4|8304|/data/mirror304 +1360|358|m|m|s|u|sdw4|sdw4|8358|/data/mirror358 +1407|405|m|m|s|u|sdw4|sdw4|8405|/data/mirror405 +1459|457|m|m|s|u|sdw4|sdw4|8457|/data/mirror457 +1511|509|m|m|s|u|sdw4|sdw4|8509|/data/mirror509 +1547|545|m|m|s|u|sdw4|sdw4|8545|/data/mirror545 +1607|605|m|m|s|u|sdw4|sdw4|8605|/data/mirror605 +1659|657|m|m|s|u|sdw4|sdw4|8657|/data/mirror657 +1696|694|m|m|s|u|sdw4|sdw4|8694|/data/mirror694 +1760|758|m|m|s|u|sdw4|sdw4|8758|/data/mirror758 +1809|807|m|m|s|u|sdw4|sdw4|8807|/data/mirror807 +1856|854|m|m|s|u|sdw4|sdw4|8854|/data/mirror854 +1910|908|m|m|s|u|sdw4|sdw4|8908|/data/mirror908 +1959|957|m|m|s|u|sdw4|sdw4|8957|/data/mirror957 +# SDW40 +793|791|p|p|s|u|sdw40|sdw40|7791|/data/primary791 +794|792|p|p|s|u|sdw40|sdw40|7792|/data/primary792 +795|793|p|p|s|u|sdw40|sdw40|7793|/data/primary793 +796|794|p|p|s|u|sdw40|sdw40|7794|/data/primary794 +797|795|p|p|s|u|sdw40|sdw40|7795|/data/primary795 +798|796|p|p|s|u|sdw40|sdw40|7796|/data/primary796 +799|797|p|p|s|u|sdw40|sdw40|7797|/data/primary797 +800|798|p|p|s|u|sdw40|sdw40|7798|/data/primary798 +801|799|p|p|s|u|sdw40|sdw40|7799|/data/primary799 +802|800|p|p|s|u|sdw40|sdw40|7800|/data/primary800 +803|801|p|p|s|u|sdw40|sdw40|7801|/data/primary801 +804|802|p|p|s|u|sdw40|sdw40|7802|/data/primary802 +805|803|p|p|s|u|sdw40|sdw40|7803|/data/primary803 +806|804|p|p|s|u|sdw40|sdw40|7804|/data/primary804 +807|805|p|p|s|u|sdw40|sdw40|7805|/data/primary805 +1047|45|m|m|s|u|sdw40|sdw40|8045|/data/mirror45 +1092|90|m|m|s|u|sdw40|sdw40|8090|/data/mirror90 +1145|143|m|m|s|u|sdw40|sdw40|8143|/data/mirror143 +1195|193|m|m|s|u|sdw40|sdw40|8193|/data/mirror193 +1228|226|m|m|s|u|sdw40|sdw40|8226|/data/mirror226 +1294|292|m|m|s|u|sdw40|sdw40|8292|/data/mirror292 +1346|344|m|m|s|u|sdw40|sdw40|8344|/data/mirror344 +1392|390|m|m|s|u|sdw40|sdw40|8390|/data/mirror390 +1441|439|m|m|s|u|sdw40|sdw40|8439|/data/mirror439 +1496|494|m|m|s|u|sdw40|sdw40|8494|/data/mirror494 +1543|541|m|m|s|u|sdw40|sdw40|8541|/data/mirror541 +1586|584|m|m|s|u|sdw40|sdw40|8584|/data/mirror584 +1645|643|m|m|s|u|sdw40|sdw40|8643|/data/mirror643 +1697|695|m|m|s|u|sdw40|sdw40|8695|/data/mirror695 +1730|728|m|m|s|u|sdw40|sdw40|8728|/data/mirror728 +1808|806|m|m|s|u|sdw40|sdw40|8806|/data/mirror806 +1844|842|m|m|s|u|sdw40|sdw40|8842|/data/mirror842 +1897|895|m|m|s|u|sdw40|sdw40|8895|/data/mirror895 +1948|946|m|m|s|u|sdw40|sdw40|8946|/data/mirror946 +1982|980|m|m|s|u|sdw40|sdw40|8980|/data/mirror980 +# SDW41 +808|806|p|p|s|u|sdw41|sdw41|7806|/data/primary806 +809|807|p|p|s|u|sdw41|sdw41|7807|/data/primary807 +810|808|p|p|s|u|sdw41|sdw41|7808|/data/primary808 +811|809|p|p|s|u|sdw41|sdw41|7809|/data/primary809 +812|810|p|p|s|u|sdw41|sdw41|7810|/data/primary810 +813|811|p|p|s|u|sdw41|sdw41|7811|/data/primary811 +814|812|p|p|s|u|sdw41|sdw41|7812|/data/primary812 +815|813|p|p|s|u|sdw41|sdw41|7813|/data/primary813 +816|814|p|p|s|u|sdw41|sdw41|7814|/data/primary814 +817|815|p|p|s|u|sdw41|sdw41|7815|/data/primary815 +818|816|p|p|s|u|sdw41|sdw41|7816|/data/primary816 +819|817|p|p|s|u|sdw41|sdw41|7817|/data/primary817 +820|818|p|p|s|u|sdw41|sdw41|7818|/data/primary818 +1048|46|m|m|s|u|sdw41|sdw41|8046|/data/mirror46 +1093|91|m|m|s|u|sdw41|sdw41|8091|/data/mirror91 +1146|144|m|m|s|u|sdw41|sdw41|8144|/data/mirror144 +1196|194|m|m|s|u|sdw41|sdw41|8194|/data/mirror194 +1239|237|m|m|s|u|sdw41|sdw41|8237|/data/mirror237 +1295|293|m|m|s|u|sdw41|sdw41|8293|/data/mirror293 +1348|346|m|m|s|u|sdw41|sdw41|8346|/data/mirror346 +1393|391|m|m|s|u|sdw41|sdw41|8391|/data/mirror391 +1442|440|m|m|s|u|sdw41|sdw41|8440|/data/mirror440 +1497|495|m|m|s|u|sdw41|sdw41|8495|/data/mirror495 +1544|542|m|m|s|u|sdw41|sdw41|8542|/data/mirror542 +1593|591|m|m|s|u|sdw41|sdw41|8591|/data/mirror591 +1647|645|m|m|s|u|sdw41|sdw41|8645|/data/mirror645 +1698|696|m|m|s|u|sdw41|sdw41|8696|/data/mirror696 +1747|745|m|m|s|u|sdw41|sdw41|8745|/data/mirror745 +1779|777|m|m|s|u|sdw41|sdw41|8777|/data/mirror777 +1845|843|m|m|s|u|sdw41|sdw41|8843|/data/mirror843 +1893|891|m|m|s|u|sdw41|sdw41|8891|/data/mirror891 +1950|948|m|m|s|u|sdw41|sdw41|8948|/data/mirror948 +1994|992|m|m|s|u|sdw41|sdw41|8992|/data/mirror992 +# SDW42 +821|819|p|p|s|u|sdw42|sdw42|7819|/data/primary819 +822|820|p|p|s|u|sdw42|sdw42|7820|/data/primary820 +823|821|p|p|s|u|sdw42|sdw42|7821|/data/primary821 +824|822|p|p|s|u|sdw42|sdw42|7822|/data/primary822 +825|823|p|p|s|u|sdw42|sdw42|7823|/data/primary823 +826|824|p|p|s|u|sdw42|sdw42|7824|/data/primary824 +827|825|p|p|s|u|sdw42|sdw42|7825|/data/primary825 +828|826|p|p|s|u|sdw42|sdw42|7826|/data/primary826 +829|827|p|p|s|u|sdw42|sdw42|7827|/data/primary827 +830|828|p|p|s|u|sdw42|sdw42|7828|/data/primary828 +831|829|p|p|s|u|sdw42|sdw42|7829|/data/primary829 +832|830|p|p|s|u|sdw42|sdw42|7830|/data/primary830 +833|831|p|p|s|u|sdw42|sdw42|7831|/data/primary831 +834|832|p|p|s|u|sdw42|sdw42|7832|/data/primary832 +835|833|p|p|s|u|sdw42|sdw42|7833|/data/primary833 +836|834|p|p|s|u|sdw42|sdw42|7834|/data/primary834 +837|835|p|p|s|u|sdw42|sdw42|7835|/data/primary835 +838|836|p|p|s|u|sdw42|sdw42|7836|/data/primary836 +839|837|p|p|s|u|sdw42|sdw42|7837|/data/primary837 +840|838|p|p|s|u|sdw42|sdw42|7838|/data/primary838 +841|839|p|p|s|u|sdw42|sdw42|7839|/data/primary839 +842|840|p|p|s|u|sdw42|sdw42|7840|/data/primary840 +843|841|p|p|s|u|sdw42|sdw42|7841|/data/primary841 +1035|33|m|m|s|u|sdw42|sdw42|8033|/data/mirror33 +1071|69|m|m|s|u|sdw42|sdw42|8069|/data/mirror69 +1147|145|m|m|s|u|sdw42|sdw42|8145|/data/mirror145 +1182|180|m|m|s|u|sdw42|sdw42|8180|/data/mirror180 +1247|245|m|m|s|u|sdw42|sdw42|8245|/data/mirror245 +1296|294|m|m|s|u|sdw42|sdw42|8294|/data/mirror294 +1349|347|m|m|s|u|sdw42|sdw42|8347|/data/mirror347 +1388|386|m|m|s|u|sdw42|sdw42|8386|/data/mirror386 +1443|441|m|m|s|u|sdw42|sdw42|8441|/data/mirror441 +1498|496|m|m|s|u|sdw42|sdw42|8496|/data/mirror496 +1545|543|m|m|s|u|sdw42|sdw42|8543|/data/mirror543 +1594|592|m|m|s|u|sdw42|sdw42|8592|/data/mirror592 +1631|629|m|m|s|u|sdw42|sdw42|8629|/data/mirror629 +1691|689|m|m|s|u|sdw42|sdw42|8689|/data/mirror689 +1739|737|m|m|s|u|sdw42|sdw42|8737|/data/mirror737 +1795|793|m|m|s|u|sdw42|sdw42|8793|/data/mirror793 +1846|844|m|m|s|u|sdw42|sdw42|8844|/data/mirror844 +1898|896|m|m|s|u|sdw42|sdw42|8896|/data/mirror896 +1935|933|m|m|s|u|sdw42|sdw42|8933|/data/mirror933 +1996|994|m|m|s|u|sdw42|sdw42|8994|/data/mirror994 +# SDW43 +844|842|p|p|s|u|sdw43|sdw43|7842|/data/primary842 +845|843|p|p|s|u|sdw43|sdw43|7843|/data/primary843 +846|844|p|p|s|u|sdw43|sdw43|7844|/data/primary844 +847|845|p|p|s|u|sdw43|sdw43|7845|/data/primary845 +848|846|p|p|s|u|sdw43|sdw43|7846|/data/primary846 +849|847|p|p|s|u|sdw43|sdw43|7847|/data/primary847 +850|848|p|p|s|u|sdw43|sdw43|7848|/data/primary848 +851|849|p|p|s|u|sdw43|sdw43|7849|/data/primary849 +852|850|p|p|s|u|sdw43|sdw43|7850|/data/primary850 +853|851|p|p|s|u|sdw43|sdw43|7851|/data/primary851 +854|852|p|p|s|u|sdw43|sdw43|7852|/data/primary852 +855|853|p|p|s|u|sdw43|sdw43|7853|/data/primary853 +856|854|p|p|s|u|sdw43|sdw43|7854|/data/primary854 +857|855|p|p|s|u|sdw43|sdw43|7855|/data/primary855 +858|856|p|p|s|u|sdw43|sdw43|7856|/data/primary856 +859|857|p|p|s|u|sdw43|sdw43|7857|/data/primary857 +860|858|p|p|s|u|sdw43|sdw43|7858|/data/primary858 +861|859|p|p|s|u|sdw43|sdw43|7859|/data/primary859 +862|860|p|p|s|u|sdw43|sdw43|7860|/data/primary860 +863|861|p|p|s|u|sdw43|sdw43|7861|/data/primary861 +864|862|p|p|s|u|sdw43|sdw43|7862|/data/primary862 +865|863|p|p|s|u|sdw43|sdw43|7863|/data/primary863 +1049|47|m|m|s|u|sdw43|sdw43|8047|/data/mirror47 +1094|92|m|m|s|u|sdw43|sdw43|8092|/data/mirror92 +1130|128|m|m|s|u|sdw43|sdw43|8128|/data/mirror128 +1197|195|m|m|s|u|sdw43|sdw43|8195|/data/mirror195 +1248|246|m|m|s|u|sdw43|sdw43|8246|/data/mirror246 +1284|282|m|m|s|u|sdw43|sdw43|8282|/data/mirror282 +1350|348|m|m|s|u|sdw43|sdw43|8348|/data/mirror348 +1394|392|m|m|s|u|sdw43|sdw43|8392|/data/mirror392 +1444|442|m|m|s|u|sdw43|sdw43|8442|/data/mirror442 +1479|477|m|m|s|u|sdw43|sdw43|8477|/data/mirror477 +1542|540|m|m|s|u|sdw43|sdw43|8540|/data/mirror540 +1596|594|m|m|s|u|sdw43|sdw43|8594|/data/mirror594 +1648|646|m|m|s|u|sdw43|sdw43|8646|/data/mirror646 +1700|698|m|m|s|u|sdw43|sdw43|8698|/data/mirror698 +1748|746|m|m|s|u|sdw43|sdw43|8746|/data/mirror746 +1796|794|m|m|s|u|sdw43|sdw43|8794|/data/mirror794 +1866|864|m|m|s|u|sdw43|sdw43|8864|/data/mirror864 +1880|878|m|m|s|u|sdw43|sdw43|8878|/data/mirror878 +1952|950|m|m|s|u|sdw43|sdw43|8950|/data/mirror950 +1997|995|m|m|s|u|sdw43|sdw43|8995|/data/mirror995 +# SDW44 +866|864|p|p|s|u|sdw44|sdw44|7864|/data/primary864 +867|865|p|p|s|u|sdw44|sdw44|7865|/data/primary865 +868|866|p|p|s|u|sdw44|sdw44|7866|/data/primary866 +869|867|p|p|s|u|sdw44|sdw44|7867|/data/primary867 +870|868|p|p|s|u|sdw44|sdw44|7868|/data/primary868 +871|869|p|p|s|u|sdw44|sdw44|7869|/data/primary869 +872|870|p|p|s|u|sdw44|sdw44|7870|/data/primary870 +873|871|p|p|s|u|sdw44|sdw44|7871|/data/primary871 +874|872|p|p|s|u|sdw44|sdw44|7872|/data/primary872 +875|873|p|p|s|u|sdw44|sdw44|7873|/data/primary873 +876|874|p|p|s|u|sdw44|sdw44|7874|/data/primary874 +877|875|p|p|s|u|sdw44|sdw44|7875|/data/primary875 +878|876|p|p|s|u|sdw44|sdw44|7876|/data/primary876 +879|877|p|p|s|u|sdw44|sdw44|7877|/data/primary877 +880|878|p|p|s|u|sdw44|sdw44|7878|/data/primary878 +881|879|p|p|s|u|sdw44|sdw44|7879|/data/primary879 +882|880|p|p|s|u|sdw44|sdw44|7880|/data/primary880 +883|881|p|p|s|u|sdw44|sdw44|7881|/data/primary881 +884|882|p|p|s|u|sdw44|sdw44|7882|/data/primary882 +885|883|p|p|s|u|sdw44|sdw44|7883|/data/primary883 +886|884|p|p|s|u|sdw44|sdw44|7884|/data/primary884 +887|885|p|p|s|u|sdw44|sdw44|7885|/data/primary885 +888|886|p|p|s|u|sdw44|sdw44|7886|/data/primary886 +1023|21|m|m|s|u|sdw44|sdw44|8021|/data/mirror21 +1095|93|m|m|s|u|sdw44|sdw44|8093|/data/mirror93 +1148|146|m|m|s|u|sdw44|sdw44|8146|/data/mirror146 +1183|181|m|m|s|u|sdw44|sdw44|8181|/data/mirror181 +1230|228|m|m|s|u|sdw44|sdw44|8228|/data/mirror228 +1297|295|m|m|s|u|sdw44|sdw44|8295|/data/mirror295 +1351|349|m|m|s|u|sdw44|sdw44|8349|/data/mirror349 +1396|394|m|m|s|u|sdw44|sdw44|8394|/data/mirror394 +1447|445|m|m|s|u|sdw44|sdw44|8445|/data/mirror445 +1491|489|m|m|s|u|sdw44|sdw44|8489|/data/mirror489 +1546|544|m|m|s|u|sdw44|sdw44|8544|/data/mirror544 +1599|597|m|m|s|u|sdw44|sdw44|8597|/data/mirror597 +1646|644|m|m|s|u|sdw44|sdw44|8644|/data/mirror644 +1692|690|m|m|s|u|sdw44|sdw44|8690|/data/mirror690 +1740|738|m|m|s|u|sdw44|sdw44|8738|/data/mirror738 +1797|795|m|m|s|u|sdw44|sdw44|8795|/data/mirror795 +1847|845|m|m|s|u|sdw44|sdw44|8845|/data/mirror845 +1901|899|m|m|s|u|sdw44|sdw44|8899|/data/mirror899 +1936|934|m|m|s|u|sdw44|sdw44|8934|/data/mirror934 +1999|997|m|m|s|u|sdw44|sdw44|8997|/data/mirror997 +# SDW45 +889|887|p|p|s|u|sdw45|sdw45|7887|/data/primary887 +890|888|p|p|s|u|sdw45|sdw45|7888|/data/primary888 +891|889|p|p|s|u|sdw45|sdw45|7889|/data/primary889 +892|890|p|p|s|u|sdw45|sdw45|7890|/data/primary890 +893|891|p|p|s|u|sdw45|sdw45|7891|/data/primary891 +894|892|p|p|s|u|sdw45|sdw45|7892|/data/primary892 +895|893|p|p|s|u|sdw45|sdw45|7893|/data/primary893 +896|894|p|p|s|u|sdw45|sdw45|7894|/data/primary894 +897|895|p|p|s|u|sdw45|sdw45|7895|/data/primary895 +898|896|p|p|s|u|sdw45|sdw45|7896|/data/primary896 +899|897|p|p|s|u|sdw45|sdw45|7897|/data/primary897 +900|898|p|p|s|u|sdw45|sdw45|7898|/data/primary898 +901|899|p|p|s|u|sdw45|sdw45|7899|/data/primary899 +902|900|p|p|s|u|sdw45|sdw45|7900|/data/primary900 +903|901|p|p|s|u|sdw45|sdw45|7901|/data/primary901 +904|902|p|p|s|u|sdw45|sdw45|7902|/data/primary902 +905|903|p|p|s|u|sdw45|sdw45|7903|/data/primary903 +906|904|p|p|s|u|sdw45|sdw45|7904|/data/primary904 +1050|48|m|m|s|u|sdw45|sdw45|8048|/data/mirror48 +1096|94|m|m|s|u|sdw45|sdw45|8094|/data/mirror94 +1131|129|m|m|s|u|sdw45|sdw45|8129|/data/mirror129 +1198|196|m|m|s|u|sdw45|sdw45|8196|/data/mirror196 +1249|247|m|m|s|u|sdw45|sdw45|8247|/data/mirror247 +1285|283|m|m|s|u|sdw45|sdw45|8283|/data/mirror283 +1331|329|m|m|s|u|sdw45|sdw45|8329|/data/mirror329 +1397|395|m|m|s|u|sdw45|sdw45|8395|/data/mirror395 +1448|446|m|m|s|u|sdw45|sdw45|8446|/data/mirror446 +1501|499|m|m|s|u|sdw45|sdw45|8499|/data/mirror499 +1535|533|m|m|s|u|sdw45|sdw45|8533|/data/mirror533 +1598|596|m|m|s|u|sdw45|sdw45|8596|/data/mirror596 +1649|647|m|m|s|u|sdw45|sdw45|8647|/data/mirror647 +1705|703|m|m|s|u|sdw45|sdw45|8703|/data/mirror703 +1750|748|m|m|s|u|sdw45|sdw45|8748|/data/mirror748 +1781|779|m|m|s|u|sdw45|sdw45|8779|/data/mirror779 +1833|831|m|m|s|u|sdw45|sdw45|8831|/data/mirror831 +1907|905|m|m|s|u|sdw45|sdw45|8905|/data/mirror905 +1953|951|m|m|s|u|sdw45|sdw45|8951|/data/mirror951 +2001|999|m|m|s|u|sdw45|sdw45|8999|/data/mirror999 +# SDW46 +907|905|p|p|s|u|sdw46|sdw46|7905|/data/primary905 +908|906|p|p|s|u|sdw46|sdw46|7906|/data/primary906 +909|907|p|p|s|u|sdw46|sdw46|7907|/data/primary907 +910|908|p|p|s|u|sdw46|sdw46|7908|/data/primary908 +911|909|p|p|s|u|sdw46|sdw46|7909|/data/primary909 +912|910|p|p|s|u|sdw46|sdw46|7910|/data/primary910 +913|911|p|p|s|u|sdw46|sdw46|7911|/data/primary911 +914|912|p|p|s|u|sdw46|sdw46|7912|/data/primary912 +915|913|p|p|s|u|sdw46|sdw46|7913|/data/primary913 +916|914|p|p|s|u|sdw46|sdw46|7914|/data/primary914 +917|915|p|p|s|u|sdw46|sdw46|7915|/data/primary915 +918|916|p|p|s|u|sdw46|sdw46|7916|/data/primary916 +919|917|p|p|s|u|sdw46|sdw46|7917|/data/primary917 +920|918|p|p|s|u|sdw46|sdw46|7918|/data/primary918 +921|919|p|p|s|u|sdw46|sdw46|7919|/data/primary919 +922|920|p|p|s|u|sdw46|sdw46|7920|/data/primary920 +923|921|p|p|s|u|sdw46|sdw46|7921|/data/primary921 +924|922|p|p|s|u|sdw46|sdw46|7922|/data/primary922 +925|923|p|p|s|u|sdw46|sdw46|7923|/data/primary923 +926|924|p|p|s|u|sdw46|sdw46|7924|/data/primary924 +1052|50|m|m|s|u|sdw46|sdw46|8050|/data/mirror50 +1097|95|m|m|s|u|sdw46|sdw46|8095|/data/mirror95 +1149|147|m|m|s|u|sdw46|sdw46|8147|/data/mirror147 +1199|197|m|m|s|u|sdw46|sdw46|8197|/data/mirror197 +1241|239|m|m|s|u|sdw46|sdw46|8239|/data/mirror239 +1298|296|m|m|s|u|sdw46|sdw46|8296|/data/mirror296 +1352|350|m|m|s|u|sdw46|sdw46|8350|/data/mirror350 +1398|396|m|m|s|u|sdw46|sdw46|8396|/data/mirror396 +1449|447|m|m|s|u|sdw46|sdw46|8447|/data/mirror447 +1492|490|m|m|s|u|sdw46|sdw46|8490|/data/mirror490 +1548|546|m|m|s|u|sdw46|sdw46|8546|/data/mirror546 +1600|598|m|m|s|u|sdw46|sdw46|8598|/data/mirror598 +1651|649|m|m|s|u|sdw46|sdw46|8649|/data/mirror649 +1706|704|m|m|s|u|sdw46|sdw46|8704|/data/mirror704 +1741|739|m|m|s|u|sdw46|sdw46|8739|/data/mirror739 +1798|796|m|m|s|u|sdw46|sdw46|8796|/data/mirror796 +1849|847|m|m|s|u|sdw46|sdw46|8847|/data/mirror847 +1881|879|m|m|s|u|sdw46|sdw46|8879|/data/mirror879 +1937|935|m|m|s|u|sdw46|sdw46|8935|/data/mirror935 +1995|993|m|m|s|u|sdw46|sdw46|8993|/data/mirror993 +# SDW47 +927|925|p|p|s|u|sdw47|sdw47|7925|/data/primary925 +928|926|p|p|s|u|sdw47|sdw47|7926|/data/primary926 +929|927|p|p|s|u|sdw47|sdw47|7927|/data/primary927 +930|928|p|p|s|u|sdw47|sdw47|7928|/data/primary928 +931|929|p|p|s|u|sdw47|sdw47|7929|/data/primary929 +932|930|p|p|s|u|sdw47|sdw47|7930|/data/primary930 +933|931|p|p|s|u|sdw47|sdw47|7931|/data/primary931 +934|932|p|p|s|u|sdw47|sdw47|7932|/data/primary932 +935|933|p|p|s|u|sdw47|sdw47|7933|/data/primary933 +936|934|p|p|s|u|sdw47|sdw47|7934|/data/primary934 +937|935|p|p|s|u|sdw47|sdw47|7935|/data/primary935 +938|936|p|p|s|u|sdw47|sdw47|7936|/data/primary936 +939|937|p|p|s|u|sdw47|sdw47|7937|/data/primary937 +940|938|p|p|s|u|sdw47|sdw47|7938|/data/primary938 +941|939|p|p|s|u|sdw47|sdw47|7939|/data/primary939 +942|940|p|p|s|u|sdw47|sdw47|7940|/data/primary940 +1051|49|m|m|s|u|sdw47|sdw47|8049|/data/mirror49 +1098|96|m|m|s|u|sdw47|sdw47|8096|/data/mirror96 +1150|148|m|m|s|u|sdw47|sdw47|8148|/data/mirror148 +1200|198|m|m|s|u|sdw47|sdw47|8198|/data/mirror198 +1250|248|m|m|s|u|sdw47|sdw47|8248|/data/mirror248 +1299|297|m|m|s|u|sdw47|sdw47|8297|/data/mirror297 +1353|351|m|m|s|u|sdw47|sdw47|8351|/data/mirror351 +1399|397|m|m|s|u|sdw47|sdw47|8397|/data/mirror397 +1434|432|m|m|s|u|sdw47|sdw47|8432|/data/mirror432 +1504|502|m|m|s|u|sdw47|sdw47|8502|/data/mirror502 +1550|548|m|m|s|u|sdw47|sdw47|8548|/data/mirror548 +1601|599|m|m|s|u|sdw47|sdw47|8599|/data/mirror599 +1652|650|m|m|s|u|sdw47|sdw47|8650|/data/mirror650 +1707|705|m|m|s|u|sdw47|sdw47|8705|/data/mirror705 +1751|749|m|m|s|u|sdw47|sdw47|8749|/data/mirror749 +1799|797|m|m|s|u|sdw47|sdw47|8797|/data/mirror797 +1834|832|m|m|s|u|sdw47|sdw47|8832|/data/mirror832 +1902|900|m|m|s|u|sdw47|sdw47|8900|/data/mirror900 +1947|945|m|m|s|u|sdw47|sdw47|8945|/data/mirror945 +# SDW48 +943|941|p|p|s|u|sdw48|sdw48|7941|/data/primary941 +944|942|p|p|s|u|sdw48|sdw48|7942|/data/primary942 +945|943|p|p|s|u|sdw48|sdw48|7943|/data/primary943 +946|944|p|p|s|u|sdw48|sdw48|7944|/data/primary944 +947|945|p|p|s|u|sdw48|sdw48|7945|/data/primary945 +948|946|p|p|s|u|sdw48|sdw48|7946|/data/primary946 +949|947|p|p|s|u|sdw48|sdw48|7947|/data/primary947 +950|948|p|p|s|u|sdw48|sdw48|7948|/data/primary948 +951|949|p|p|s|u|sdw48|sdw48|7949|/data/primary949 +952|950|p|p|s|u|sdw48|sdw48|7950|/data/primary950 +953|951|p|p|s|u|sdw48|sdw48|7951|/data/primary951 +954|952|p|p|s|u|sdw48|sdw48|7952|/data/primary952 +955|953|p|p|s|u|sdw48|sdw48|7953|/data/primary953 +956|954|p|p|s|u|sdw48|sdw48|7954|/data/primary954 +957|955|p|p|s|u|sdw48|sdw48|7955|/data/primary955 +958|956|p|p|s|u|sdw48|sdw48|7956|/data/primary956 +959|957|p|p|s|u|sdw48|sdw48|7957|/data/primary957 +960|958|p|p|s|u|sdw48|sdw48|7958|/data/primary958 +961|959|p|p|s|u|sdw48|sdw48|7959|/data/primary959 +962|960|p|p|s|u|sdw48|sdw48|7960|/data/primary960 +963|961|p|p|s|u|sdw48|sdw48|7961|/data/primary961 +964|962|p|p|s|u|sdw48|sdw48|7962|/data/primary962 +965|963|p|p|s|u|sdw48|sdw48|7963|/data/primary963 +966|964|p|p|s|u|sdw48|sdw48|7964|/data/primary964 +967|965|p|p|s|u|sdw48|sdw48|7965|/data/primary965 +968|966|p|p|s|u|sdw48|sdw48|7966|/data/primary966 +969|967|p|p|s|u|sdw48|sdw48|7967|/data/primary967 +970|968|p|p|s|u|sdw48|sdw48|7968|/data/primary968 +971|969|p|p|s|u|sdw48|sdw48|7969|/data/primary969 +1053|51|m|m|s|u|sdw48|sdw48|8051|/data/mirror51 +1100|98|m|m|s|u|sdw48|sdw48|8098|/data/mirror98 +1151|149|m|m|s|u|sdw48|sdw48|8149|/data/mirror149 +1201|199|m|m|s|u|sdw48|sdw48|8199|/data/mirror199 +1232|230|m|m|s|u|sdw48|sdw48|8230|/data/mirror230 +1300|298|m|m|s|u|sdw48|sdw48|8298|/data/mirror298 +1347|345|m|m|s|u|sdw48|sdw48|8345|/data/mirror345 +1400|398|m|m|s|u|sdw48|sdw48|8398|/data/mirror398 +1450|448|m|m|s|u|sdw48|sdw48|8448|/data/mirror448 +1507|505|m|m|s|u|sdw48|sdw48|8505|/data/mirror505 +1551|549|m|m|s|u|sdw48|sdw48|8549|/data/mirror549 +1578|576|m|m|s|u|sdw48|sdw48|8576|/data/mirror576 +1634|632|m|m|s|u|sdw48|sdw48|8632|/data/mirror632 +1681|679|m|m|s|u|sdw48|sdw48|8679|/data/mirror679 +1734|732|m|m|s|u|sdw48|sdw48|8732|/data/mirror732 +1800|798|m|m|s|u|sdw48|sdw48|8798|/data/mirror798 +1851|849|m|m|s|u|sdw48|sdw48|8849|/data/mirror849 +1882|880|m|m|s|u|sdw48|sdw48|8880|/data/mirror880 +1972|970|m|m|s|u|sdw48|sdw48|8970|/data/mirror970 +# SDW49 +972|970|p|p|s|u|sdw49|sdw49|7970|/data/primary970 +973|971|p|p|s|u|sdw49|sdw49|7971|/data/primary971 +974|972|p|p|s|u|sdw49|sdw49|7972|/data/primary972 +975|973|p|p|s|u|sdw49|sdw49|7973|/data/primary973 +976|974|p|p|s|u|sdw49|sdw49|7974|/data/primary974 +977|975|p|p|s|u|sdw49|sdw49|7975|/data/primary975 +978|976|p|p|s|u|sdw49|sdw49|7976|/data/primary976 +979|977|p|p|s|u|sdw49|sdw49|7977|/data/primary977 +980|978|p|p|s|u|sdw49|sdw49|7978|/data/primary978 +981|979|p|p|s|u|sdw49|sdw49|7979|/data/primary979 +982|980|p|p|s|u|sdw49|sdw49|7980|/data/primary980 +983|981|p|p|s|u|sdw49|sdw49|7981|/data/primary981 +984|982|p|p|s|u|sdw49|sdw49|7982|/data/primary982 +985|983|p|p|s|u|sdw49|sdw49|7983|/data/primary983 +986|984|p|p|s|u|sdw49|sdw49|7984|/data/primary984 +987|985|p|p|s|u|sdw49|sdw49|7985|/data/primary985 +988|986|p|p|s|u|sdw49|sdw49|7986|/data/primary986 +989|987|p|p|s|u|sdw49|sdw49|7987|/data/primary987 +990|988|p|p|s|u|sdw49|sdw49|7988|/data/primary988 +1054|52|m|m|s|u|sdw49|sdw49|8052|/data/mirror52 +1101|99|m|m|s|u|sdw49|sdw49|8099|/data/mirror99 +1152|150|m|m|s|u|sdw49|sdw49|8150|/data/mirror150 +1204|202|m|m|s|u|sdw49|sdw49|8202|/data/mirror202 +1251|249|m|m|s|u|sdw49|sdw49|8249|/data/mirror249 +1301|299|m|m|s|u|sdw49|sdw49|8299|/data/mirror299 +1354|352|m|m|s|u|sdw49|sdw49|8352|/data/mirror352 +1401|399|m|m|s|u|sdw49|sdw49|8399|/data/mirror399 +1451|449|m|m|s|u|sdw49|sdw49|8449|/data/mirror449 +1509|507|m|m|s|u|sdw49|sdw49|8507|/data/mirror507 +1552|550|m|m|s|u|sdw49|sdw49|8550|/data/mirror550 +1602|600|m|m|s|u|sdw49|sdw49|8600|/data/mirror600 +1653|651|m|m|s|u|sdw49|sdw49|8651|/data/mirror651 +1694|692|m|m|s|u|sdw49|sdw49|8692|/data/mirror692 +1753|751|m|m|s|u|sdw49|sdw49|8751|/data/mirror751 +1802|800|m|m|s|u|sdw49|sdw49|8800|/data/mirror800 +1852|850|m|m|s|u|sdw49|sdw49|8850|/data/mirror850 +1904|902|m|m|s|u|sdw49|sdw49|8902|/data/mirror902 +1938|936|m|m|s|u|sdw49|sdw49|8936|/data/mirror936 +# SDW5 +96|94|p|p|s|u|sdw5|sdw5|7094|/data/primary94 +97|95|p|p|s|u|sdw5|sdw5|7095|/data/primary95 +98|96|p|p|s|u|sdw5|sdw5|7096|/data/primary96 +99|97|p|p|s|u|sdw5|sdw5|7097|/data/primary97 +100|98|p|p|s|u|sdw5|sdw5|7098|/data/primary98 +101|99|p|p|s|u|sdw5|sdw5|7099|/data/primary99 +102|100|p|p|s|u|sdw5|sdw5|7100|/data/primary100 +103|101|p|p|s|u|sdw5|sdw5|7101|/data/primary101 +104|102|p|p|s|u|sdw5|sdw5|7102|/data/primary102 +105|103|p|p|s|u|sdw5|sdw5|7103|/data/primary103 +106|104|p|p|s|u|sdw5|sdw5|7104|/data/primary104 +107|105|p|p|s|u|sdw5|sdw5|7105|/data/primary105 +108|106|p|p|s|u|sdw5|sdw5|7106|/data/primary106 +109|107|p|p|s|u|sdw5|sdw5|7107|/data/primary107 +110|108|p|p|s|u|sdw5|sdw5|7108|/data/primary108 +111|109|p|p|s|u|sdw5|sdw5|7109|/data/primary109 +112|110|p|p|s|u|sdw5|sdw5|7110|/data/primary110 +113|111|p|p|s|u|sdw5|sdw5|7111|/data/primary111 +114|112|p|p|s|u|sdw5|sdw5|7112|/data/primary112 +115|113|p|p|s|u|sdw5|sdw5|7113|/data/primary113 +1005|3|m|m|s|u|sdw5|sdw5|8003|/data/mirror3 +1060|58|m|m|s|u|sdw5|sdw5|8058|/data/mirror58 +1116|114|m|m|s|u|sdw5|sdw5|8114|/data/mirror114 +1153|151|m|m|s|u|sdw5|sdw5|8151|/data/mirror151 +1212|210|m|m|s|u|sdw5|sdw5|8210|/data/mirror210 +1260|258|m|m|s|u|sdw5|sdw5|8258|/data/mirror258 +1307|305|m|m|s|u|sdw5|sdw5|8305|/data/mirror305 +1361|359|m|m|s|u|sdw5|sdw5|8359|/data/mirror359 +1408|406|m|m|s|u|sdw5|sdw5|8406|/data/mirror406 +1460|458|m|m|s|u|sdw5|sdw5|8458|/data/mirror458 +1512|510|m|m|s|u|sdw5|sdw5|8510|/data/mirror510 +1560|558|m|m|s|u|sdw5|sdw5|8558|/data/mirror558 +1608|606|m|m|s|u|sdw5|sdw5|8606|/data/mirror606 +1660|658|m|m|s|u|sdw5|sdw5|8658|/data/mirror658 +1715|713|m|m|s|u|sdw5|sdw5|8713|/data/mirror713 +1745|743|m|m|s|u|sdw5|sdw5|8743|/data/mirror743 +1811|809|m|m|s|u|sdw5|sdw5|8809|/data/mirror809 +1850|848|m|m|s|u|sdw5|sdw5|8848|/data/mirror848 +1911|909|m|m|s|u|sdw5|sdw5|8909|/data/mirror909 +1949|947|m|m|s|u|sdw5|sdw5|8947|/data/mirror947 +# SDW50 +991|989|p|p|s|u|sdw50|sdw50|7989|/data/primary989 +992|990|p|p|s|u|sdw50|sdw50|7990|/data/primary990 +993|991|p|p|s|u|sdw50|sdw50|7991|/data/primary991 +994|992|p|p|s|u|sdw50|sdw50|7992|/data/primary992 +995|993|p|p|s|u|sdw50|sdw50|7993|/data/primary993 +996|994|p|p|s|u|sdw50|sdw50|7994|/data/primary994 +997|995|p|p|s|u|sdw50|sdw50|7995|/data/primary995 +998|996|p|p|s|u|sdw50|sdw50|7996|/data/primary996 +999|997|p|p|s|u|sdw50|sdw50|7997|/data/primary997 +1000|998|p|p|s|u|sdw50|sdw50|7998|/data/primary998 +1001|999|p|p|s|u|sdw50|sdw50|7999|/data/primary999 +1055|53|m|m|s|u|sdw50|sdw50|8053|/data/mirror53 +1102|100|m|m|s|u|sdw50|sdw50|8100|/data/mirror100 +1154|152|m|m|s|u|sdw50|sdw50|8152|/data/mirror152 +1206|204|m|m|s|u|sdw50|sdw50|8204|/data/mirror204 +1252|250|m|m|s|u|sdw50|sdw50|8250|/data/mirror250 +1302|300|m|m|s|u|sdw50|sdw50|8300|/data/mirror300 +1355|353|m|m|s|u|sdw50|sdw50|8353|/data/mirror353 +1402|400|m|m|s|u|sdw50|sdw50|8400|/data/mirror400 +1452|450|m|m|s|u|sdw50|sdw50|8450|/data/mirror450 +1494|492|m|m|s|u|sdw50|sdw50|8492|/data/mirror492 +1556|554|m|m|s|u|sdw50|sdw50|8554|/data/mirror554 +1603|601|m|m|s|u|sdw50|sdw50|8601|/data/mirror601 +1656|654|m|m|s|u|sdw50|sdw50|8654|/data/mirror654 +1682|680|m|m|s|u|sdw50|sdw50|8680|/data/mirror680 +1754|752|m|m|s|u|sdw50|sdw50|8752|/data/mirror752 +1803|801|m|m|s|u|sdw50|sdw50|8801|/data/mirror801 +1853|851|m|m|s|u|sdw50|sdw50|8851|/data/mirror851 +1905|903|m|m|s|u|sdw50|sdw50|8903|/data/mirror903 +1954|952|m|m|s|u|sdw50|sdw50|8952|/data/mirror952 +1971|969|m|m|s|u|sdw50|sdw50|8969|/data/mirror969 +# SDW6 +116|114|p|p|s|u|sdw6|sdw6|7114|/data/primary114 +117|115|p|p|s|u|sdw6|sdw6|7115|/data/primary115 +118|116|p|p|s|u|sdw6|sdw6|7116|/data/primary116 +119|117|p|p|s|u|sdw6|sdw6|7117|/data/primary117 +120|118|p|p|s|u|sdw6|sdw6|7118|/data/primary118 +121|119|p|p|s|u|sdw6|sdw6|7119|/data/primary119 +122|120|p|p|s|u|sdw6|sdw6|7120|/data/primary120 +123|121|p|p|s|u|sdw6|sdw6|7121|/data/primary121 +124|122|p|p|s|u|sdw6|sdw6|7122|/data/primary122 +125|123|p|p|s|u|sdw6|sdw6|7123|/data/primary123 +126|124|p|p|s|u|sdw6|sdw6|7124|/data/primary124 +127|125|p|p|s|u|sdw6|sdw6|7125|/data/primary125 +128|126|p|p|s|u|sdw6|sdw6|7126|/data/primary126 +129|127|p|p|s|u|sdw6|sdw6|7127|/data/primary127 +130|128|p|p|s|u|sdw6|sdw6|7128|/data/primary128 +131|129|p|p|s|u|sdw6|sdw6|7129|/data/primary129 +132|130|p|p|s|u|sdw6|sdw6|7130|/data/primary130 +133|131|p|p|s|u|sdw6|sdw6|7131|/data/primary131 +134|132|p|p|s|u|sdw6|sdw6|7132|/data/primary132 +135|133|p|p|s|u|sdw6|sdw6|7133|/data/primary133 +136|134|p|p|s|u|sdw6|sdw6|7134|/data/primary134 +137|135|p|p|s|u|sdw6|sdw6|7135|/data/primary135 +1008|6|m|m|s|u|sdw6|sdw6|8006|/data/mirror6 +1041|39|m|m|s|u|sdw6|sdw6|8039|/data/mirror39 +1109|107|m|m|s|u|sdw6|sdw6|8107|/data/mirror107 +1159|157|m|m|s|u|sdw6|sdw6|8157|/data/mirror157 +1189|187|m|m|s|u|sdw6|sdw6|8187|/data/mirror187 +1261|259|m|m|s|u|sdw6|sdw6|8259|/data/mirror259 +1308|306|m|m|s|u|sdw6|sdw6|8306|/data/mirror306 +1362|360|m|m|s|u|sdw6|sdw6|8360|/data/mirror360 +1395|393|m|m|s|u|sdw6|sdw6|8393|/data/mirror393 +1465|463|m|m|s|u|sdw6|sdw6|8463|/data/mirror463 +1515|513|m|m|s|u|sdw6|sdw6|8513|/data/mirror513 +1561|559|m|m|s|u|sdw6|sdw6|8559|/data/mirror559 +1609|607|m|m|s|u|sdw6|sdw6|8607|/data/mirror607 +1662|660|m|m|s|u|sdw6|sdw6|8660|/data/mirror660 +1708|706|m|m|s|u|sdw6|sdw6|8706|/data/mirror706 +1762|760|m|m|s|u|sdw6|sdw6|8760|/data/mirror760 +1812|810|m|m|s|u|sdw6|sdw6|8810|/data/mirror810 +1858|856|m|m|s|u|sdw6|sdw6|8856|/data/mirror856 +1913|911|m|m|s|u|sdw6|sdw6|8911|/data/mirror911 +1962|960|m|m|s|u|sdw6|sdw6|8960|/data/mirror960 +1989|987|m|m|s|u|sdw6|sdw6|8987|/data/mirror987 +# SDW7 +138|136|p|p|s|u|sdw7|sdw7|7136|/data/primary136 +139|137|p|p|s|u|sdw7|sdw7|7137|/data/primary137 +140|138|p|p|s|u|sdw7|sdw7|7138|/data/primary138 +141|139|p|p|s|u|sdw7|sdw7|7139|/data/primary139 +142|140|p|p|s|u|sdw7|sdw7|7140|/data/primary140 +143|141|p|p|s|u|sdw7|sdw7|7141|/data/primary141 +144|142|p|p|s|u|sdw7|sdw7|7142|/data/primary142 +145|143|p|p|s|u|sdw7|sdw7|7143|/data/primary143 +146|144|p|p|s|u|sdw7|sdw7|7144|/data/primary144 +147|145|p|p|s|u|sdw7|sdw7|7145|/data/primary145 +148|146|p|p|s|u|sdw7|sdw7|7146|/data/primary146 +149|147|p|p|s|u|sdw7|sdw7|7147|/data/primary147 +1009|7|m|m|s|u|sdw7|sdw7|8007|/data/mirror7 +1063|61|m|m|s|u|sdw7|sdw7|8061|/data/mirror61 +1114|112|m|m|s|u|sdw7|sdw7|8112|/data/mirror112 +1161|159|m|m|s|u|sdw7|sdw7|8159|/data/mirror159 +1215|213|m|m|s|u|sdw7|sdw7|8213|/data/mirror213 +1246|244|m|m|s|u|sdw7|sdw7|8244|/data/mirror244 +1309|307|m|m|s|u|sdw7|sdw7|8307|/data/mirror307 +1337|335|m|m|s|u|sdw7|sdw7|8335|/data/mirror335 +1410|408|m|m|s|u|sdw7|sdw7|8408|/data/mirror408 +1466|464|m|m|s|u|sdw7|sdw7|8464|/data/mirror464 +1517|515|m|m|s|u|sdw7|sdw7|8515|/data/mirror515 +1562|560|m|m|s|u|sdw7|sdw7|8560|/data/mirror560 +1610|608|m|m|s|u|sdw7|sdw7|8608|/data/mirror608 +1664|662|m|m|s|u|sdw7|sdw7|8662|/data/mirror662 +1716|714|m|m|s|u|sdw7|sdw7|8714|/data/mirror714 +1746|744|m|m|s|u|sdw7|sdw7|8744|/data/mirror744 +1801|799|m|m|s|u|sdw7|sdw7|8799|/data/mirror799 +1859|857|m|m|s|u|sdw7|sdw7|8857|/data/mirror857 +1899|897|m|m|s|u|sdw7|sdw7|8897|/data/mirror897 +1963|961|m|m|s|u|sdw7|sdw7|8961|/data/mirror961 +# SDW8 +150|148|p|p|s|u|sdw8|sdw8|7148|/data/primary148 +151|149|p|p|s|u|sdw8|sdw8|7149|/data/primary149 +152|150|p|p|s|u|sdw8|sdw8|7150|/data/primary150 +153|151|p|p|s|u|sdw8|sdw8|7151|/data/primary151 +154|152|p|p|s|u|sdw8|sdw8|7152|/data/primary152 +155|153|p|p|s|u|sdw8|sdw8|7153|/data/primary153 +156|154|p|p|s|u|sdw8|sdw8|7154|/data/primary154 +157|155|p|p|s|u|sdw8|sdw8|7155|/data/primary155 +158|156|p|p|s|u|sdw8|sdw8|7156|/data/primary156 +159|157|p|p|s|u|sdw8|sdw8|7157|/data/primary157 +160|158|p|p|s|u|sdw8|sdw8|7158|/data/primary158 +161|159|p|p|s|u|sdw8|sdw8|7159|/data/primary159 +162|160|p|p|s|u|sdw8|sdw8|7160|/data/primary160 +163|161|p|p|s|u|sdw8|sdw8|7161|/data/primary161 +164|162|p|p|s|u|sdw8|sdw8|7162|/data/primary162 +165|163|p|p|s|u|sdw8|sdw8|7163|/data/primary163 +166|164|p|p|s|u|sdw8|sdw8|7164|/data/primary164 +167|165|p|p|s|u|sdw8|sdw8|7165|/data/primary165 +168|166|p|p|s|u|sdw8|sdw8|7166|/data/primary166 +169|167|p|p|s|u|sdw8|sdw8|7167|/data/primary167 +170|168|p|p|s|u|sdw8|sdw8|7168|/data/primary168 +1011|9|m|m|s|u|sdw8|sdw8|8009|/data/mirror9 +1064|62|m|m|s|u|sdw8|sdw8|8062|/data/mirror62 +1115|113|m|m|s|u|sdw8|sdw8|8113|/data/mirror113 +1171|169|m|m|s|u|sdw8|sdw8|8169|/data/mirror169 +1216|214|m|m|s|u|sdw8|sdw8|8214|/data/mirror214 +1258|256|m|m|s|u|sdw8|sdw8|8256|/data/mirror256 +1305|303|m|m|s|u|sdw8|sdw8|8303|/data/mirror303 +1363|361|m|m|s|u|sdw8|sdw8|8361|/data/mirror361 +1411|409|m|m|s|u|sdw8|sdw8|8409|/data/mirror409 +1467|465|m|m|s|u|sdw8|sdw8|8465|/data/mirror465 +1518|516|m|m|s|u|sdw8|sdw8|8516|/data/mirror516 +1549|547|m|m|s|u|sdw8|sdw8|8547|/data/mirror547 +1612|610|m|m|s|u|sdw8|sdw8|8610|/data/mirror610 +1665|663|m|m|s|u|sdw8|sdw8|8663|/data/mirror663 +1686|684|m|m|s|u|sdw8|sdw8|8684|/data/mirror684 +1763|761|m|m|s|u|sdw8|sdw8|8761|/data/mirror761 +1813|811|m|m|s|u|sdw8|sdw8|8811|/data/mirror811 +1860|858|m|m|s|u|sdw8|sdw8|8858|/data/mirror858 +1887|885|m|m|s|u|sdw8|sdw8|8885|/data/mirror885 +1964|962|m|m|s|u|sdw8|sdw8|8962|/data/mirror962 +2000|998|m|m|s|u|sdw8|sdw8|8998|/data/mirror998 +# SDW9 +171|169|p|p|s|u|sdw9|sdw9|7169|/data/primary169 +172|170|p|p|s|u|sdw9|sdw9|7170|/data/primary170 +173|171|p|p|s|u|sdw9|sdw9|7171|/data/primary171 +174|172|p|p|s|u|sdw9|sdw9|7172|/data/primary172 +175|173|p|p|s|u|sdw9|sdw9|7173|/data/primary173 +176|174|p|p|s|u|sdw9|sdw9|7174|/data/primary174 +177|175|p|p|s|u|sdw9|sdw9|7175|/data/primary175 +178|176|p|p|s|u|sdw9|sdw9|7176|/data/primary176 +179|177|p|p|s|u|sdw9|sdw9|7177|/data/primary177 +180|178|p|p|s|u|sdw9|sdw9|7178|/data/primary178 +181|179|p|p|s|u|sdw9|sdw9|7179|/data/primary179 +182|180|p|p|s|u|sdw9|sdw9|7180|/data/primary180 +183|181|p|p|s|u|sdw9|sdw9|7181|/data/primary181 +184|182|p|p|s|u|sdw9|sdw9|7182|/data/primary182 +185|183|p|p|s|u|sdw9|sdw9|7183|/data/primary183 +186|184|p|p|s|u|sdw9|sdw9|7184|/data/primary184 +187|185|p|p|s|u|sdw9|sdw9|7185|/data/primary185 +188|186|p|p|s|u|sdw9|sdw9|7186|/data/primary186 +189|187|p|p|s|u|sdw9|sdw9|7187|/data/primary187 +190|188|p|p|s|u|sdw9|sdw9|7188|/data/primary188 +191|189|p|p|s|u|sdw9|sdw9|7189|/data/primary189 +192|190|p|p|s|u|sdw9|sdw9|7190|/data/primary190 +1012|10|m|m|s|u|sdw9|sdw9|8010|/data/mirror10 +1056|54|m|m|s|u|sdw9|sdw9|8054|/data/mirror54 +1117|115|m|m|s|u|sdw9|sdw9|8115|/data/mirror115 +1162|160|m|m|s|u|sdw9|sdw9|8160|/data/mirror160 +1217|215|m|m|s|u|sdw9|sdw9|8215|/data/mirror215 +1262|260|m|m|s|u|sdw9|sdw9|8260|/data/mirror260 +1310|308|m|m|s|u|sdw9|sdw9|8308|/data/mirror308 +1364|362|m|m|s|u|sdw9|sdw9|8362|/data/mirror362 +1412|410|m|m|s|u|sdw9|sdw9|8410|/data/mirror410 +1469|467|m|m|s|u|sdw9|sdw9|8467|/data/mirror467 +1519|517|m|m|s|u|sdw9|sdw9|8517|/data/mirror517 +1563|561|m|m|s|u|sdw9|sdw9|8561|/data/mirror561 +1595|593|m|m|s|u|sdw9|sdw9|8593|/data/mirror593 +1666|664|m|m|s|u|sdw9|sdw9|8664|/data/mirror664 +1709|707|m|m|s|u|sdw9|sdw9|8707|/data/mirror707 +1764|762|m|m|s|u|sdw9|sdw9|8762|/data/mirror762 +1788|786|m|m|s|u|sdw9|sdw9|8786|/data/mirror786 +1861|859|m|m|s|u|sdw9|sdw9|8859|/data/mirror859 +1915|913|m|m|s|u|sdw9|sdw9|8913|/data/mirror913 +1951|949|m|m|s|u|sdw9|sdw9|8949|/data/mirror949 diff --git a/gpMgmt/bin/gprebalance_modules/test/data/120_20_balanced_grouped.array b/gpMgmt/bin/gprebalance_modules/test/data/120_20_balanced_grouped.array new file mode 100644 index 000000000000..ae9d07953137 --- /dev/null +++ b/gpMgmt/bin/gprebalance_modules/test/data/120_20_balanced_grouped.array @@ -0,0 +1,262 @@ +# Coordinator +1|-1|p|p|s|u|cdw|cdw|5432|/data/coordinator +# SDW1 +2|0|p|p|s|u|sdw1|sdw1|7000|/data/primary0 +3|1|p|p|s|u|sdw1|sdw1|7001|/data/primary1 +4|2|p|p|s|u|sdw1|sdw1|7002|/data/primary2 +5|3|p|p|s|u|sdw1|sdw1|7003|/data/primary3 +6|4|p|p|s|u|sdw1|sdw1|7004|/data/primary4 +7|5|p|p|s|u|sdw1|sdw1|7005|/data/primary5 +236|114|m|m|s|u|sdw1|sdw1|8114|/data/mirror114 +237|115|m|m|s|u|sdw1|sdw1|8115|/data/mirror115 +238|116|m|m|s|u|sdw1|sdw1|8116|/data/mirror116 +239|117|m|m|s|u|sdw1|sdw1|8117|/data/mirror117 +240|118|m|m|s|u|sdw1|sdw1|8118|/data/mirror118 +241|119|m|m|s|u|sdw1|sdw1|8119|/data/mirror119 +# SDW10 +56|54|p|p|s|u|sdw10|sdw10|7054|/data/primary54 +57|55|p|p|s|u|sdw10|sdw10|7055|/data/primary55 +58|56|p|p|s|u|sdw10|sdw10|7056|/data/primary56 +59|57|p|p|s|u|sdw10|sdw10|7057|/data/primary57 +60|58|p|p|s|u|sdw10|sdw10|7058|/data/primary58 +61|59|p|p|s|u|sdw10|sdw10|7059|/data/primary59 +170|48|m|m|s|u|sdw10|sdw10|8048|/data/mirror48 +171|49|m|m|s|u|sdw10|sdw10|8049|/data/mirror49 +172|50|m|m|s|u|sdw10|sdw10|8050|/data/mirror50 +173|51|m|m|s|u|sdw10|sdw10|8051|/data/mirror51 +174|52|m|m|s|u|sdw10|sdw10|8052|/data/mirror52 +175|53|m|m|s|u|sdw10|sdw10|8053|/data/mirror53 +# SDW11 +62|60|p|p|s|u|sdw11|sdw11|7060|/data/primary60 +63|61|p|p|s|u|sdw11|sdw11|7061|/data/primary61 +64|62|p|p|s|u|sdw11|sdw11|7062|/data/primary62 +65|63|p|p|s|u|sdw11|sdw11|7063|/data/primary63 +66|64|p|p|s|u|sdw11|sdw11|7064|/data/primary64 +67|65|p|p|s|u|sdw11|sdw11|7065|/data/primary65 +176|54|m|m|s|u|sdw11|sdw11|8054|/data/mirror54 +177|55|m|m|s|u|sdw11|sdw11|8055|/data/mirror55 +178|56|m|m|s|u|sdw11|sdw11|8056|/data/mirror56 +179|57|m|m|s|u|sdw11|sdw11|8057|/data/mirror57 +180|58|m|m|s|u|sdw11|sdw11|8058|/data/mirror58 +181|59|m|m|s|u|sdw11|sdw11|8059|/data/mirror59 +# SDW12 +68|66|p|p|s|u|sdw12|sdw12|7066|/data/primary66 +69|67|p|p|s|u|sdw12|sdw12|7067|/data/primary67 +70|68|p|p|s|u|sdw12|sdw12|7068|/data/primary68 +71|69|p|p|s|u|sdw12|sdw12|7069|/data/primary69 +72|70|p|p|s|u|sdw12|sdw12|7070|/data/primary70 +73|71|p|p|s|u|sdw12|sdw12|7071|/data/primary71 +182|60|m|m|s|u|sdw12|sdw12|8060|/data/mirror60 +183|61|m|m|s|u|sdw12|sdw12|8061|/data/mirror61 +184|62|m|m|s|u|sdw12|sdw12|8062|/data/mirror62 +185|63|m|m|s|u|sdw12|sdw12|8063|/data/mirror63 +186|64|m|m|s|u|sdw12|sdw12|8064|/data/mirror64 +187|65|m|m|s|u|sdw12|sdw12|8065|/data/mirror65 +# SDW13 +74|72|p|p|s|u|sdw13|sdw13|7072|/data/primary72 +75|73|p|p|s|u|sdw13|sdw13|7073|/data/primary73 +76|74|p|p|s|u|sdw13|sdw13|7074|/data/primary74 +77|75|p|p|s|u|sdw13|sdw13|7075|/data/primary75 +78|76|p|p|s|u|sdw13|sdw13|7076|/data/primary76 +79|77|p|p|s|u|sdw13|sdw13|7077|/data/primary77 +188|66|m|m|s|u|sdw13|sdw13|8066|/data/mirror66 +189|67|m|m|s|u|sdw13|sdw13|8067|/data/mirror67 +190|68|m|m|s|u|sdw13|sdw13|8068|/data/mirror68 +191|69|m|m|s|u|sdw13|sdw13|8069|/data/mirror69 +192|70|m|m|s|u|sdw13|sdw13|8070|/data/mirror70 +193|71|m|m|s|u|sdw13|sdw13|8071|/data/mirror71 +# SDW14 +80|78|p|p|s|u|sdw14|sdw14|7078|/data/primary78 +81|79|p|p|s|u|sdw14|sdw14|7079|/data/primary79 +82|80|p|p|s|u|sdw14|sdw14|7080|/data/primary80 +83|81|p|p|s|u|sdw14|sdw14|7081|/data/primary81 +84|82|p|p|s|u|sdw14|sdw14|7082|/data/primary82 +85|83|p|p|s|u|sdw14|sdw14|7083|/data/primary83 +194|72|m|m|s|u|sdw14|sdw14|8072|/data/mirror72 +195|73|m|m|s|u|sdw14|sdw14|8073|/data/mirror73 +196|74|m|m|s|u|sdw14|sdw14|8074|/data/mirror74 +197|75|m|m|s|u|sdw14|sdw14|8075|/data/mirror75 +198|76|m|m|s|u|sdw14|sdw14|8076|/data/mirror76 +199|77|m|m|s|u|sdw14|sdw14|8077|/data/mirror77 +# SDW15 +86|84|p|p|s|u|sdw15|sdw15|7084|/data/primary84 +87|85|p|p|s|u|sdw15|sdw15|7085|/data/primary85 +88|86|p|p|s|u|sdw15|sdw15|7086|/data/primary86 +89|87|p|p|s|u|sdw15|sdw15|7087|/data/primary87 +90|88|p|p|s|u|sdw15|sdw15|7088|/data/primary88 +91|89|p|p|s|u|sdw15|sdw15|7089|/data/primary89 +200|78|m|m|s|u|sdw15|sdw15|8078|/data/mirror78 +201|79|m|m|s|u|sdw15|sdw15|8079|/data/mirror79 +202|80|m|m|s|u|sdw15|sdw15|8080|/data/mirror80 +203|81|m|m|s|u|sdw15|sdw15|8081|/data/mirror81 +204|82|m|m|s|u|sdw15|sdw15|8082|/data/mirror82 +205|83|m|m|s|u|sdw15|sdw15|8083|/data/mirror83 +# SDW16 +92|90|p|p|s|u|sdw16|sdw16|7090|/data/primary90 +93|91|p|p|s|u|sdw16|sdw16|7091|/data/primary91 +94|92|p|p|s|u|sdw16|sdw16|7092|/data/primary92 +95|93|p|p|s|u|sdw16|sdw16|7093|/data/primary93 +96|94|p|p|s|u|sdw16|sdw16|7094|/data/primary94 +97|95|p|p|s|u|sdw16|sdw16|7095|/data/primary95 +206|84|m|m|s|u|sdw16|sdw16|8084|/data/mirror84 +207|85|m|m|s|u|sdw16|sdw16|8085|/data/mirror85 +208|86|m|m|s|u|sdw16|sdw16|8086|/data/mirror86 +209|87|m|m|s|u|sdw16|sdw16|8087|/data/mirror87 +210|88|m|m|s|u|sdw16|sdw16|8088|/data/mirror88 +211|89|m|m|s|u|sdw16|sdw16|8089|/data/mirror89 +# SDW17 +98|96|p|p|s|u|sdw17|sdw17|7096|/data/primary96 +99|97|p|p|s|u|sdw17|sdw17|7097|/data/primary97 +100|98|p|p|s|u|sdw17|sdw17|7098|/data/primary98 +101|99|p|p|s|u|sdw17|sdw17|7099|/data/primary99 +102|100|p|p|s|u|sdw17|sdw17|7100|/data/primary100 +103|101|p|p|s|u|sdw17|sdw17|7101|/data/primary101 +212|90|m|m|s|u|sdw17|sdw17|8090|/data/mirror90 +213|91|m|m|s|u|sdw17|sdw17|8091|/data/mirror91 +214|92|m|m|s|u|sdw17|sdw17|8092|/data/mirror92 +215|93|m|m|s|u|sdw17|sdw17|8093|/data/mirror93 +216|94|m|m|s|u|sdw17|sdw17|8094|/data/mirror94 +217|95|m|m|s|u|sdw17|sdw17|8095|/data/mirror95 +# SDW18 +104|102|p|p|s|u|sdw18|sdw18|7102|/data/primary102 +105|103|p|p|s|u|sdw18|sdw18|7103|/data/primary103 +106|104|p|p|s|u|sdw18|sdw18|7104|/data/primary104 +107|105|p|p|s|u|sdw18|sdw18|7105|/data/primary105 +108|106|p|p|s|u|sdw18|sdw18|7106|/data/primary106 +109|107|p|p|s|u|sdw18|sdw18|7107|/data/primary107 +218|96|m|m|s|u|sdw18|sdw18|8096|/data/mirror96 +219|97|m|m|s|u|sdw18|sdw18|8097|/data/mirror97 +220|98|m|m|s|u|sdw18|sdw18|8098|/data/mirror98 +221|99|m|m|s|u|sdw18|sdw18|8099|/data/mirror99 +222|100|m|m|s|u|sdw18|sdw18|8100|/data/mirror100 +223|101|m|m|s|u|sdw18|sdw18|8101|/data/mirror101 +# SDW19 +110|108|p|p|s|u|sdw19|sdw19|7108|/data/primary108 +111|109|p|p|s|u|sdw19|sdw19|7109|/data/primary109 +112|110|p|p|s|u|sdw19|sdw19|7110|/data/primary110 +113|111|p|p|s|u|sdw19|sdw19|7111|/data/primary111 +114|112|p|p|s|u|sdw19|sdw19|7112|/data/primary112 +115|113|p|p|s|u|sdw19|sdw19|7113|/data/primary113 +224|102|m|m|s|u|sdw19|sdw19|8102|/data/mirror102 +225|103|m|m|s|u|sdw19|sdw19|8103|/data/mirror103 +226|104|m|m|s|u|sdw19|sdw19|8104|/data/mirror104 +227|105|m|m|s|u|sdw19|sdw19|8105|/data/mirror105 +228|106|m|m|s|u|sdw19|sdw19|8106|/data/mirror106 +229|107|m|m|s|u|sdw19|sdw19|8107|/data/mirror107 +# SDW2 +8|6|p|p|s|u|sdw2|sdw2|7006|/data/primary6 +9|7|p|p|s|u|sdw2|sdw2|7007|/data/primary7 +10|8|p|p|s|u|sdw2|sdw2|7008|/data/primary8 +11|9|p|p|s|u|sdw2|sdw2|7009|/data/primary9 +12|10|p|p|s|u|sdw2|sdw2|7010|/data/primary10 +13|11|p|p|s|u|sdw2|sdw2|7011|/data/primary11 +122|0|m|m|s|u|sdw2|sdw2|8000|/data/mirror0 +123|1|m|m|s|u|sdw2|sdw2|8001|/data/mirror1 +124|2|m|m|s|u|sdw2|sdw2|8002|/data/mirror2 +125|3|m|m|s|u|sdw2|sdw2|8003|/data/mirror3 +126|4|m|m|s|u|sdw2|sdw2|8004|/data/mirror4 +127|5|m|m|s|u|sdw2|sdw2|8005|/data/mirror5 +# SDW20 +116|114|p|p|s|u|sdw20|sdw20|7114|/data/primary114 +117|115|p|p|s|u|sdw20|sdw20|7115|/data/primary115 +118|116|p|p|s|u|sdw20|sdw20|7116|/data/primary116 +119|117|p|p|s|u|sdw20|sdw20|7117|/data/primary117 +120|118|p|p|s|u|sdw20|sdw20|7118|/data/primary118 +121|119|p|p|s|u|sdw20|sdw20|7119|/data/primary119 +230|108|m|m|s|u|sdw20|sdw20|8108|/data/mirror108 +231|109|m|m|s|u|sdw20|sdw20|8109|/data/mirror109 +232|110|m|m|s|u|sdw20|sdw20|8110|/data/mirror110 +233|111|m|m|s|u|sdw20|sdw20|8111|/data/mirror111 +234|112|m|m|s|u|sdw20|sdw20|8112|/data/mirror112 +235|113|m|m|s|u|sdw20|sdw20|8113|/data/mirror113 +# SDW3 +14|12|p|p|s|u|sdw3|sdw3|7012|/data/primary12 +15|13|p|p|s|u|sdw3|sdw3|7013|/data/primary13 +16|14|p|p|s|u|sdw3|sdw3|7014|/data/primary14 +17|15|p|p|s|u|sdw3|sdw3|7015|/data/primary15 +18|16|p|p|s|u|sdw3|sdw3|7016|/data/primary16 +19|17|p|p|s|u|sdw3|sdw3|7017|/data/primary17 +128|6|m|m|s|u|sdw3|sdw3|8006|/data/mirror6 +129|7|m|m|s|u|sdw3|sdw3|8007|/data/mirror7 +130|8|m|m|s|u|sdw3|sdw3|8008|/data/mirror8 +131|9|m|m|s|u|sdw3|sdw3|8009|/data/mirror9 +132|10|m|m|s|u|sdw3|sdw3|8010|/data/mirror10 +133|11|m|m|s|u|sdw3|sdw3|8011|/data/mirror11 +# SDW4 +20|18|p|p|s|u|sdw4|sdw4|7018|/data/primary18 +21|19|p|p|s|u|sdw4|sdw4|7019|/data/primary19 +22|20|p|p|s|u|sdw4|sdw4|7020|/data/primary20 +23|21|p|p|s|u|sdw4|sdw4|7021|/data/primary21 +24|22|p|p|s|u|sdw4|sdw4|7022|/data/primary22 +25|23|p|p|s|u|sdw4|sdw4|7023|/data/primary23 +134|12|m|m|s|u|sdw4|sdw4|8012|/data/mirror12 +135|13|m|m|s|u|sdw4|sdw4|8013|/data/mirror13 +136|14|m|m|s|u|sdw4|sdw4|8014|/data/mirror14 +137|15|m|m|s|u|sdw4|sdw4|8015|/data/mirror15 +138|16|m|m|s|u|sdw4|sdw4|8016|/data/mirror16 +139|17|m|m|s|u|sdw4|sdw4|8017|/data/mirror17 +# SDW5 +26|24|p|p|s|u|sdw5|sdw5|7024|/data/primary24 +27|25|p|p|s|u|sdw5|sdw5|7025|/data/primary25 +28|26|p|p|s|u|sdw5|sdw5|7026|/data/primary26 +29|27|p|p|s|u|sdw5|sdw5|7027|/data/primary27 +30|28|p|p|s|u|sdw5|sdw5|7028|/data/primary28 +31|29|p|p|s|u|sdw5|sdw5|7029|/data/primary29 +140|18|m|m|s|u|sdw5|sdw5|8018|/data/mirror18 +141|19|m|m|s|u|sdw5|sdw5|8019|/data/mirror19 +142|20|m|m|s|u|sdw5|sdw5|8020|/data/mirror20 +143|21|m|m|s|u|sdw5|sdw5|8021|/data/mirror21 +144|22|m|m|s|u|sdw5|sdw5|8022|/data/mirror22 +145|23|m|m|s|u|sdw5|sdw5|8023|/data/mirror23 +# SDW6 +32|30|p|p|s|u|sdw6|sdw6|7030|/data/primary30 +33|31|p|p|s|u|sdw6|sdw6|7031|/data/primary31 +34|32|p|p|s|u|sdw6|sdw6|7032|/data/primary32 +35|33|p|p|s|u|sdw6|sdw6|7033|/data/primary33 +36|34|p|p|s|u|sdw6|sdw6|7034|/data/primary34 +37|35|p|p|s|u|sdw6|sdw6|7035|/data/primary35 +146|24|m|m|s|u|sdw6|sdw6|8024|/data/mirror24 +147|25|m|m|s|u|sdw6|sdw6|8025|/data/mirror25 +148|26|m|m|s|u|sdw6|sdw6|8026|/data/mirror26 +149|27|m|m|s|u|sdw6|sdw6|8027|/data/mirror27 +150|28|m|m|s|u|sdw6|sdw6|8028|/data/mirror28 +151|29|m|m|s|u|sdw6|sdw6|8029|/data/mirror29 +# SDW7 +38|36|p|p|s|u|sdw7|sdw7|7036|/data/primary36 +39|37|p|p|s|u|sdw7|sdw7|7037|/data/primary37 +40|38|p|p|s|u|sdw7|sdw7|7038|/data/primary38 +41|39|p|p|s|u|sdw7|sdw7|7039|/data/primary39 +42|40|p|p|s|u|sdw7|sdw7|7040|/data/primary40 +43|41|p|p|s|u|sdw7|sdw7|7041|/data/primary41 +152|30|m|m|s|u|sdw7|sdw7|8030|/data/mirror30 +153|31|m|m|s|u|sdw7|sdw7|8031|/data/mirror31 +154|32|m|m|s|u|sdw7|sdw7|8032|/data/mirror32 +155|33|m|m|s|u|sdw7|sdw7|8033|/data/mirror33 +156|34|m|m|s|u|sdw7|sdw7|8034|/data/mirror34 +157|35|m|m|s|u|sdw7|sdw7|8035|/data/mirror35 +# SDW8 +44|42|p|p|s|u|sdw8|sdw8|7042|/data/primary42 +45|43|p|p|s|u|sdw8|sdw8|7043|/data/primary43 +46|44|p|p|s|u|sdw8|sdw8|7044|/data/primary44 +47|45|p|p|s|u|sdw8|sdw8|7045|/data/primary45 +48|46|p|p|s|u|sdw8|sdw8|7046|/data/primary46 +49|47|p|p|s|u|sdw8|sdw8|7047|/data/primary47 +158|36|m|m|s|u|sdw8|sdw8|8036|/data/mirror36 +159|37|m|m|s|u|sdw8|sdw8|8037|/data/mirror37 +160|38|m|m|s|u|sdw8|sdw8|8038|/data/mirror38 +161|39|m|m|s|u|sdw8|sdw8|8039|/data/mirror39 +162|40|m|m|s|u|sdw8|sdw8|8040|/data/mirror40 +163|41|m|m|s|u|sdw8|sdw8|8041|/data/mirror41 +# SDW9 +50|48|p|p|s|u|sdw9|sdw9|7048|/data/primary48 +51|49|p|p|s|u|sdw9|sdw9|7049|/data/primary49 +52|50|p|p|s|u|sdw9|sdw9|7050|/data/primary50 +53|51|p|p|s|u|sdw9|sdw9|7051|/data/primary51 +54|52|p|p|s|u|sdw9|sdw9|7052|/data/primary52 +55|53|p|p|s|u|sdw9|sdw9|7053|/data/primary53 +164|42|m|m|s|u|sdw9|sdw9|8042|/data/mirror42 +165|43|m|m|s|u|sdw9|sdw9|8043|/data/mirror43 +166|44|m|m|s|u|sdw9|sdw9|8044|/data/mirror44 +167|45|m|m|s|u|sdw9|sdw9|8045|/data/mirror45 +168|46|m|m|s|u|sdw9|sdw9|8046|/data/mirror46 +169|47|m|m|s|u|sdw9|sdw9|8047|/data/mirror47 diff --git a/gpMgmt/bin/gprebalance_modules/test/data/120_20_balanced_spread.array b/gpMgmt/bin/gprebalance_modules/test/data/120_20_balanced_spread.array new file mode 100644 index 000000000000..1148fa76016b --- /dev/null +++ b/gpMgmt/bin/gprebalance_modules/test/data/120_20_balanced_spread.array @@ -0,0 +1,262 @@ +# Coordinator +1|-1|p|p|s|u|cdw|cdw|5432|/data/coordinator +# SDW1 +2|0|p|p|s|u|sdw1|sdw1|7000|/data/primary0 +3|1|p|p|s|u|sdw1|sdw1|7001|/data/primary1 +4|2|p|p|s|u|sdw1|sdw1|7002|/data/primary2 +5|3|p|p|s|u|sdw1|sdw1|7003|/data/primary3 +6|4|p|p|s|u|sdw1|sdw1|7004|/data/primary4 +7|5|p|p|s|u|sdw1|sdw1|7005|/data/primary5 +211|89|m|m|s|u|sdw1|sdw1|8089|/data/mirror89 +216|94|m|m|s|u|sdw1|sdw1|8094|/data/mirror94 +221|99|m|m|s|u|sdw1|sdw1|8099|/data/mirror99 +226|104|m|m|s|u|sdw1|sdw1|8104|/data/mirror104 +231|109|m|m|s|u|sdw1|sdw1|8109|/data/mirror109 +236|114|m|m|s|u|sdw1|sdw1|8114|/data/mirror114 +# SDW10 +56|54|p|p|s|u|sdw10|sdw10|7054|/data/primary54 +57|55|p|p|s|u|sdw10|sdw10|7055|/data/primary55 +58|56|p|p|s|u|sdw10|sdw10|7056|/data/primary56 +59|57|p|p|s|u|sdw10|sdw10|7057|/data/primary57 +60|58|p|p|s|u|sdw10|sdw10|7058|/data/primary58 +61|59|p|p|s|u|sdw10|sdw10|7059|/data/primary59 +145|23|m|m|s|u|sdw10|sdw10|8023|/data/mirror23 +150|28|m|m|s|u|sdw10|sdw10|8028|/data/mirror28 +155|33|m|m|s|u|sdw10|sdw10|8033|/data/mirror33 +160|38|m|m|s|u|sdw10|sdw10|8038|/data/mirror38 +165|43|m|m|s|u|sdw10|sdw10|8043|/data/mirror43 +170|48|m|m|s|u|sdw10|sdw10|8048|/data/mirror48 +# SDW11 +62|60|p|p|s|u|sdw11|sdw11|7060|/data/primary60 +63|61|p|p|s|u|sdw11|sdw11|7061|/data/primary61 +64|62|p|p|s|u|sdw11|sdw11|7062|/data/primary62 +65|63|p|p|s|u|sdw11|sdw11|7063|/data/primary63 +66|64|p|p|s|u|sdw11|sdw11|7064|/data/primary64 +67|65|p|p|s|u|sdw11|sdw11|7065|/data/primary65 +151|29|m|m|s|u|sdw11|sdw11|8029|/data/mirror29 +156|34|m|m|s|u|sdw11|sdw11|8034|/data/mirror34 +161|39|m|m|s|u|sdw11|sdw11|8039|/data/mirror39 +166|44|m|m|s|u|sdw11|sdw11|8044|/data/mirror44 +171|49|m|m|s|u|sdw11|sdw11|8049|/data/mirror49 +176|54|m|m|s|u|sdw11|sdw11|8054|/data/mirror54 +# SDW12 +68|66|p|p|s|u|sdw12|sdw12|7066|/data/primary66 +69|67|p|p|s|u|sdw12|sdw12|7067|/data/primary67 +70|68|p|p|s|u|sdw12|sdw12|7068|/data/primary68 +71|69|p|p|s|u|sdw12|sdw12|7069|/data/primary69 +72|70|p|p|s|u|sdw12|sdw12|7070|/data/primary70 +73|71|p|p|s|u|sdw12|sdw12|7071|/data/primary71 +157|35|m|m|s|u|sdw12|sdw12|8035|/data/mirror35 +162|40|m|m|s|u|sdw12|sdw12|8040|/data/mirror40 +167|45|m|m|s|u|sdw12|sdw12|8045|/data/mirror45 +172|50|m|m|s|u|sdw12|sdw12|8050|/data/mirror50 +177|55|m|m|s|u|sdw12|sdw12|8055|/data/mirror55 +182|60|m|m|s|u|sdw12|sdw12|8060|/data/mirror60 +# SDW13 +74|72|p|p|s|u|sdw13|sdw13|7072|/data/primary72 +75|73|p|p|s|u|sdw13|sdw13|7073|/data/primary73 +76|74|p|p|s|u|sdw13|sdw13|7074|/data/primary74 +77|75|p|p|s|u|sdw13|sdw13|7075|/data/primary75 +78|76|p|p|s|u|sdw13|sdw13|7076|/data/primary76 +79|77|p|p|s|u|sdw13|sdw13|7077|/data/primary77 +163|41|m|m|s|u|sdw13|sdw13|8041|/data/mirror41 +168|46|m|m|s|u|sdw13|sdw13|8046|/data/mirror46 +173|51|m|m|s|u|sdw13|sdw13|8051|/data/mirror51 +178|56|m|m|s|u|sdw13|sdw13|8056|/data/mirror56 +183|61|m|m|s|u|sdw13|sdw13|8061|/data/mirror61 +188|66|m|m|s|u|sdw13|sdw13|8066|/data/mirror66 +# SDW14 +80|78|p|p|s|u|sdw14|sdw14|7078|/data/primary78 +81|79|p|p|s|u|sdw14|sdw14|7079|/data/primary79 +82|80|p|p|s|u|sdw14|sdw14|7080|/data/primary80 +83|81|p|p|s|u|sdw14|sdw14|7081|/data/primary81 +84|82|p|p|s|u|sdw14|sdw14|7082|/data/primary82 +85|83|p|p|s|u|sdw14|sdw14|7083|/data/primary83 +169|47|m|m|s|u|sdw14|sdw14|8047|/data/mirror47 +174|52|m|m|s|u|sdw14|sdw14|8052|/data/mirror52 +179|57|m|m|s|u|sdw14|sdw14|8057|/data/mirror57 +184|62|m|m|s|u|sdw14|sdw14|8062|/data/mirror62 +189|67|m|m|s|u|sdw14|sdw14|8067|/data/mirror67 +194|72|m|m|s|u|sdw14|sdw14|8072|/data/mirror72 +# SDW15 +86|84|p|p|s|u|sdw15|sdw15|7084|/data/primary84 +87|85|p|p|s|u|sdw15|sdw15|7085|/data/primary85 +88|86|p|p|s|u|sdw15|sdw15|7086|/data/primary86 +89|87|p|p|s|u|sdw15|sdw15|7087|/data/primary87 +90|88|p|p|s|u|sdw15|sdw15|7088|/data/primary88 +91|89|p|p|s|u|sdw15|sdw15|7089|/data/primary89 +175|53|m|m|s|u|sdw15|sdw15|8053|/data/mirror53 +180|58|m|m|s|u|sdw15|sdw15|8058|/data/mirror58 +185|63|m|m|s|u|sdw15|sdw15|8063|/data/mirror63 +190|68|m|m|s|u|sdw15|sdw15|8068|/data/mirror68 +195|73|m|m|s|u|sdw15|sdw15|8073|/data/mirror73 +200|78|m|m|s|u|sdw15|sdw15|8078|/data/mirror78 +# SDW16 +92|90|p|p|s|u|sdw16|sdw16|7090|/data/primary90 +93|91|p|p|s|u|sdw16|sdw16|7091|/data/primary91 +94|92|p|p|s|u|sdw16|sdw16|7092|/data/primary92 +95|93|p|p|s|u|sdw16|sdw16|7093|/data/primary93 +96|94|p|p|s|u|sdw16|sdw16|7094|/data/primary94 +97|95|p|p|s|u|sdw16|sdw16|7095|/data/primary95 +181|59|m|m|s|u|sdw16|sdw16|8059|/data/mirror59 +186|64|m|m|s|u|sdw16|sdw16|8064|/data/mirror64 +191|69|m|m|s|u|sdw16|sdw16|8069|/data/mirror69 +196|74|m|m|s|u|sdw16|sdw16|8074|/data/mirror74 +201|79|m|m|s|u|sdw16|sdw16|8079|/data/mirror79 +206|84|m|m|s|u|sdw16|sdw16|8084|/data/mirror84 +# SDW17 +98|96|p|p|s|u|sdw17|sdw17|7096|/data/primary96 +99|97|p|p|s|u|sdw17|sdw17|7097|/data/primary97 +100|98|p|p|s|u|sdw17|sdw17|7098|/data/primary98 +101|99|p|p|s|u|sdw17|sdw17|7099|/data/primary99 +102|100|p|p|s|u|sdw17|sdw17|7100|/data/primary100 +103|101|p|p|s|u|sdw17|sdw17|7101|/data/primary101 +187|65|m|m|s|u|sdw17|sdw17|8065|/data/mirror65 +192|70|m|m|s|u|sdw17|sdw17|8070|/data/mirror70 +197|75|m|m|s|u|sdw17|sdw17|8075|/data/mirror75 +202|80|m|m|s|u|sdw17|sdw17|8080|/data/mirror80 +207|85|m|m|s|u|sdw17|sdw17|8085|/data/mirror85 +212|90|m|m|s|u|sdw17|sdw17|8090|/data/mirror90 +# SDW18 +104|102|p|p|s|u|sdw18|sdw18|7102|/data/primary102 +105|103|p|p|s|u|sdw18|sdw18|7103|/data/primary103 +106|104|p|p|s|u|sdw18|sdw18|7104|/data/primary104 +107|105|p|p|s|u|sdw18|sdw18|7105|/data/primary105 +108|106|p|p|s|u|sdw18|sdw18|7106|/data/primary106 +109|107|p|p|s|u|sdw18|sdw18|7107|/data/primary107 +193|71|m|m|s|u|sdw18|sdw18|8071|/data/mirror71 +198|76|m|m|s|u|sdw18|sdw18|8076|/data/mirror76 +203|81|m|m|s|u|sdw18|sdw18|8081|/data/mirror81 +208|86|m|m|s|u|sdw18|sdw18|8086|/data/mirror86 +213|91|m|m|s|u|sdw18|sdw18|8091|/data/mirror91 +218|96|m|m|s|u|sdw18|sdw18|8096|/data/mirror96 +# SDW19 +110|108|p|p|s|u|sdw19|sdw19|7108|/data/primary108 +111|109|p|p|s|u|sdw19|sdw19|7109|/data/primary109 +112|110|p|p|s|u|sdw19|sdw19|7110|/data/primary110 +113|111|p|p|s|u|sdw19|sdw19|7111|/data/primary111 +114|112|p|p|s|u|sdw19|sdw19|7112|/data/primary112 +115|113|p|p|s|u|sdw19|sdw19|7113|/data/primary113 +199|77|m|m|s|u|sdw19|sdw19|8077|/data/mirror77 +204|82|m|m|s|u|sdw19|sdw19|8082|/data/mirror82 +209|87|m|m|s|u|sdw19|sdw19|8087|/data/mirror87 +214|92|m|m|s|u|sdw19|sdw19|8092|/data/mirror92 +219|97|m|m|s|u|sdw19|sdw19|8097|/data/mirror97 +224|102|m|m|s|u|sdw19|sdw19|8102|/data/mirror102 +# SDW2 +8|6|p|p|s|u|sdw2|sdw2|7006|/data/primary6 +9|7|p|p|s|u|sdw2|sdw2|7007|/data/primary7 +10|8|p|p|s|u|sdw2|sdw2|7008|/data/primary8 +11|9|p|p|s|u|sdw2|sdw2|7009|/data/primary9 +12|10|p|p|s|u|sdw2|sdw2|7010|/data/primary10 +13|11|p|p|s|u|sdw2|sdw2|7011|/data/primary11 +122|0|m|m|s|u|sdw2|sdw2|8000|/data/mirror0 +217|95|m|m|s|u|sdw2|sdw2|8095|/data/mirror95 +222|100|m|m|s|u|sdw2|sdw2|8100|/data/mirror100 +227|105|m|m|s|u|sdw2|sdw2|8105|/data/mirror105 +232|110|m|m|s|u|sdw2|sdw2|8110|/data/mirror110 +237|115|m|m|s|u|sdw2|sdw2|8115|/data/mirror115 +# SDW20 +116|114|p|p|s|u|sdw20|sdw20|7114|/data/primary114 +117|115|p|p|s|u|sdw20|sdw20|7115|/data/primary115 +118|116|p|p|s|u|sdw20|sdw20|7116|/data/primary116 +119|117|p|p|s|u|sdw20|sdw20|7117|/data/primary117 +120|118|p|p|s|u|sdw20|sdw20|7118|/data/primary118 +121|119|p|p|s|u|sdw20|sdw20|7119|/data/primary119 +205|83|m|m|s|u|sdw20|sdw20|8083|/data/mirror83 +210|88|m|m|s|u|sdw20|sdw20|8088|/data/mirror88 +215|93|m|m|s|u|sdw20|sdw20|8093|/data/mirror93 +220|98|m|m|s|u|sdw20|sdw20|8098|/data/mirror98 +225|103|m|m|s|u|sdw20|sdw20|8103|/data/mirror103 +230|108|m|m|s|u|sdw20|sdw20|8108|/data/mirror108 +# SDW3 +14|12|p|p|s|u|sdw3|sdw3|7012|/data/primary12 +15|13|p|p|s|u|sdw3|sdw3|7013|/data/primary13 +16|14|p|p|s|u|sdw3|sdw3|7014|/data/primary14 +17|15|p|p|s|u|sdw3|sdw3|7015|/data/primary15 +18|16|p|p|s|u|sdw3|sdw3|7016|/data/primary16 +19|17|p|p|s|u|sdw3|sdw3|7017|/data/primary17 +123|1|m|m|s|u|sdw3|sdw3|8001|/data/mirror1 +128|6|m|m|s|u|sdw3|sdw3|8006|/data/mirror6 +223|101|m|m|s|u|sdw3|sdw3|8101|/data/mirror101 +228|106|m|m|s|u|sdw3|sdw3|8106|/data/mirror106 +233|111|m|m|s|u|sdw3|sdw3|8111|/data/mirror111 +238|116|m|m|s|u|sdw3|sdw3|8116|/data/mirror116 +# SDW4 +20|18|p|p|s|u|sdw4|sdw4|7018|/data/primary18 +21|19|p|p|s|u|sdw4|sdw4|7019|/data/primary19 +22|20|p|p|s|u|sdw4|sdw4|7020|/data/primary20 +23|21|p|p|s|u|sdw4|sdw4|7021|/data/primary21 +24|22|p|p|s|u|sdw4|sdw4|7022|/data/primary22 +25|23|p|p|s|u|sdw4|sdw4|7023|/data/primary23 +124|2|m|m|s|u|sdw4|sdw4|8002|/data/mirror2 +129|7|m|m|s|u|sdw4|sdw4|8007|/data/mirror7 +134|12|m|m|s|u|sdw4|sdw4|8012|/data/mirror12 +229|107|m|m|s|u|sdw4|sdw4|8107|/data/mirror107 +234|112|m|m|s|u|sdw4|sdw4|8112|/data/mirror112 +239|117|m|m|s|u|sdw4|sdw4|8117|/data/mirror117 +# SDW5 +26|24|p|p|s|u|sdw5|sdw5|7024|/data/primary24 +27|25|p|p|s|u|sdw5|sdw5|7025|/data/primary25 +28|26|p|p|s|u|sdw5|sdw5|7026|/data/primary26 +29|27|p|p|s|u|sdw5|sdw5|7027|/data/primary27 +30|28|p|p|s|u|sdw5|sdw5|7028|/data/primary28 +31|29|p|p|s|u|sdw5|sdw5|7029|/data/primary29 +125|3|m|m|s|u|sdw5|sdw5|8003|/data/mirror3 +130|8|m|m|s|u|sdw5|sdw5|8008|/data/mirror8 +135|13|m|m|s|u|sdw5|sdw5|8013|/data/mirror13 +140|18|m|m|s|u|sdw5|sdw5|8018|/data/mirror18 +235|113|m|m|s|u|sdw5|sdw5|8113|/data/mirror113 +240|118|m|m|s|u|sdw5|sdw5|8118|/data/mirror118 +# SDW6 +32|30|p|p|s|u|sdw6|sdw6|7030|/data/primary30 +33|31|p|p|s|u|sdw6|sdw6|7031|/data/primary31 +34|32|p|p|s|u|sdw6|sdw6|7032|/data/primary32 +35|33|p|p|s|u|sdw6|sdw6|7033|/data/primary33 +36|34|p|p|s|u|sdw6|sdw6|7034|/data/primary34 +37|35|p|p|s|u|sdw6|sdw6|7035|/data/primary35 +126|4|m|m|s|u|sdw6|sdw6|8004|/data/mirror4 +131|9|m|m|s|u|sdw6|sdw6|8009|/data/mirror9 +136|14|m|m|s|u|sdw6|sdw6|8014|/data/mirror14 +141|19|m|m|s|u|sdw6|sdw6|8019|/data/mirror19 +146|24|m|m|s|u|sdw6|sdw6|8024|/data/mirror24 +241|119|m|m|s|u|sdw6|sdw6|8119|/data/mirror119 +# SDW7 +38|36|p|p|s|u|sdw7|sdw7|7036|/data/primary36 +39|37|p|p|s|u|sdw7|sdw7|7037|/data/primary37 +40|38|p|p|s|u|sdw7|sdw7|7038|/data/primary38 +41|39|p|p|s|u|sdw7|sdw7|7039|/data/primary39 +42|40|p|p|s|u|sdw7|sdw7|7040|/data/primary40 +43|41|p|p|s|u|sdw7|sdw7|7041|/data/primary41 +127|5|m|m|s|u|sdw7|sdw7|8005|/data/mirror5 +132|10|m|m|s|u|sdw7|sdw7|8010|/data/mirror10 +137|15|m|m|s|u|sdw7|sdw7|8015|/data/mirror15 +142|20|m|m|s|u|sdw7|sdw7|8020|/data/mirror20 +147|25|m|m|s|u|sdw7|sdw7|8025|/data/mirror25 +152|30|m|m|s|u|sdw7|sdw7|8030|/data/mirror30 +# SDW8 +44|42|p|p|s|u|sdw8|sdw8|7042|/data/primary42 +45|43|p|p|s|u|sdw8|sdw8|7043|/data/primary43 +46|44|p|p|s|u|sdw8|sdw8|7044|/data/primary44 +47|45|p|p|s|u|sdw8|sdw8|7045|/data/primary45 +48|46|p|p|s|u|sdw8|sdw8|7046|/data/primary46 +49|47|p|p|s|u|sdw8|sdw8|7047|/data/primary47 +133|11|m|m|s|u|sdw8|sdw8|8011|/data/mirror11 +138|16|m|m|s|u|sdw8|sdw8|8016|/data/mirror16 +143|21|m|m|s|u|sdw8|sdw8|8021|/data/mirror21 +148|26|m|m|s|u|sdw8|sdw8|8026|/data/mirror26 +153|31|m|m|s|u|sdw8|sdw8|8031|/data/mirror31 +158|36|m|m|s|u|sdw8|sdw8|8036|/data/mirror36 +# SDW9 +50|48|p|p|s|u|sdw9|sdw9|7048|/data/primary48 +51|49|p|p|s|u|sdw9|sdw9|7049|/data/primary49 +52|50|p|p|s|u|sdw9|sdw9|7050|/data/primary50 +53|51|p|p|s|u|sdw9|sdw9|7051|/data/primary51 +54|52|p|p|s|u|sdw9|sdw9|7052|/data/primary52 +55|53|p|p|s|u|sdw9|sdw9|7053|/data/primary53 +139|17|m|m|s|u|sdw9|sdw9|8017|/data/mirror17 +144|22|m|m|s|u|sdw9|sdw9|8022|/data/mirror22 +149|27|m|m|s|u|sdw9|sdw9|8027|/data/mirror27 +154|32|m|m|s|u|sdw9|sdw9|8032|/data/mirror32 +159|37|m|m|s|u|sdw9|sdw9|8037|/data/mirror37 +164|42|m|m|s|u|sdw9|sdw9|8042|/data/mirror42 diff --git a/gpMgmt/bin/gprebalance_modules/test/data/120_20_unbalanced_grouped.array b/gpMgmt/bin/gprebalance_modules/test/data/120_20_unbalanced_grouped.array new file mode 100644 index 000000000000..9a807e00dc9b --- /dev/null +++ b/gpMgmt/bin/gprebalance_modules/test/data/120_20_unbalanced_grouped.array @@ -0,0 +1,262 @@ +# Coordinator +1|-1|p|p|s|u|cdw|cdw|5432|/data/coordinator +# SDW1 +2|0|p|p|s|u|sdw1|sdw1|7000|/data/primary0 +3|1|p|p|s|u|sdw1|sdw1|7001|/data/primary1 +4|2|p|p|s|u|sdw1|sdw1|7002|/data/primary2 +5|3|p|p|s|u|sdw1|sdw1|7003|/data/primary3 +6|4|p|p|s|u|sdw1|sdw1|7004|/data/primary4 +139|17|m|m|s|u|sdw1|sdw1|8017|/data/mirror17 +140|18|m|m|s|u|sdw1|sdw1|8018|/data/mirror18 +141|19|m|m|s|u|sdw1|sdw1|8019|/data/mirror19 +142|20|m|m|s|u|sdw1|sdw1|8020|/data/mirror20 +143|21|m|m|s|u|sdw1|sdw1|8021|/data/mirror21 +144|22|m|m|s|u|sdw1|sdw1|8022|/data/mirror22 +# SDW10 +52|50|p|p|s|u|sdw10|sdw10|7050|/data/primary50 +53|51|p|p|s|u|sdw10|sdw10|7051|/data/primary51 +54|52|p|p|s|u|sdw10|sdw10|7052|/data/primary52 +55|53|p|p|s|u|sdw10|sdw10|7053|/data/primary53 +56|54|p|p|s|u|sdw10|sdw10|7054|/data/primary54 +57|55|p|p|s|u|sdw10|sdw10|7055|/data/primary55 +58|56|p|p|s|u|sdw10|sdw10|7056|/data/primary56 +# SDW11 +59|57|p|p|s|u|sdw11|sdw11|7057|/data/primary57 +60|58|p|p|s|u|sdw11|sdw11|7058|/data/primary58 +61|59|p|p|s|u|sdw11|sdw11|7059|/data/primary59 +62|60|p|p|s|u|sdw11|sdw11|7060|/data/primary60 +63|61|p|p|s|u|sdw11|sdw11|7061|/data/primary61 +64|62|p|p|s|u|sdw11|sdw11|7062|/data/primary62 +167|45|m|m|s|u|sdw11|sdw11|8045|/data/mirror45 +168|46|m|m|s|u|sdw11|sdw11|8046|/data/mirror46 +169|47|m|m|s|u|sdw11|sdw11|8047|/data/mirror47 +170|48|m|m|s|u|sdw11|sdw11|8048|/data/mirror48 +171|49|m|m|s|u|sdw11|sdw11|8049|/data/mirror49 +# SDW12 +65|63|p|p|s|u|sdw12|sdw12|7063|/data/primary63 +66|64|p|p|s|u|sdw12|sdw12|7064|/data/primary64 +67|65|p|p|s|u|sdw12|sdw12|7065|/data/primary65 +68|66|p|p|s|u|sdw12|sdw12|7066|/data/primary66 +189|67|m|m|s|u|sdw12|sdw12|8067|/data/mirror67 +190|68|m|m|s|u|sdw12|sdw12|8068|/data/mirror68 +191|69|m|m|s|u|sdw12|sdw12|8069|/data/mirror69 +192|70|m|m|s|u|sdw12|sdw12|8070|/data/mirror70 +193|71|m|m|s|u|sdw12|sdw12|8071|/data/mirror71 +# SDW13 +69|67|p|p|s|u|sdw13|sdw13|7067|/data/primary67 +70|68|p|p|s|u|sdw13|sdw13|7068|/data/primary68 +71|69|p|p|s|u|sdw13|sdw13|7069|/data/primary69 +72|70|p|p|s|u|sdw13|sdw13|7070|/data/primary70 +73|71|p|p|s|u|sdw13|sdw13|7071|/data/primary71 +172|50|m|m|s|u|sdw13|sdw13|8050|/data/mirror50 +173|51|m|m|s|u|sdw13|sdw13|8051|/data/mirror51 +174|52|m|m|s|u|sdw13|sdw13|8052|/data/mirror52 +175|53|m|m|s|u|sdw13|sdw13|8053|/data/mirror53 +176|54|m|m|s|u|sdw13|sdw13|8054|/data/mirror54 +177|55|m|m|s|u|sdw13|sdw13|8055|/data/mirror55 +178|56|m|m|s|u|sdw13|sdw13|8056|/data/mirror56 +# SDW14 +74|72|p|p|s|u|sdw14|sdw14|7072|/data/primary72 +75|73|p|p|s|u|sdw14|sdw14|7073|/data/primary73 +76|74|p|p|s|u|sdw14|sdw14|7074|/data/primary74 +77|75|p|p|s|u|sdw14|sdw14|7075|/data/primary75 +78|76|p|p|s|u|sdw14|sdw14|7076|/data/primary76 +79|77|p|p|s|u|sdw14|sdw14|7077|/data/primary77 +80|78|p|p|s|u|sdw14|sdw14|7078|/data/primary78 +185|63|m|m|s|u|sdw14|sdw14|8063|/data/mirror63 +186|64|m|m|s|u|sdw14|sdw14|8064|/data/mirror64 +187|65|m|m|s|u|sdw14|sdw14|8065|/data/mirror65 +188|66|m|m|s|u|sdw14|sdw14|8066|/data/mirror66 +# SDW15 +81|79|p|p|s|u|sdw15|sdw15|7079|/data/primary79 +82|80|p|p|s|u|sdw15|sdw15|7080|/data/primary80 +83|81|p|p|s|u|sdw15|sdw15|7081|/data/primary81 +84|82|p|p|s|u|sdw15|sdw15|7082|/data/primary82 +85|83|p|p|s|u|sdw15|sdw15|7083|/data/primary83 +86|84|p|p|s|u|sdw15|sdw15|7084|/data/primary84 +194|72|m|m|s|u|sdw15|sdw15|8072|/data/mirror72 +195|73|m|m|s|u|sdw15|sdw15|8073|/data/mirror73 +196|74|m|m|s|u|sdw15|sdw15|8074|/data/mirror74 +197|75|m|m|s|u|sdw15|sdw15|8075|/data/mirror75 +198|76|m|m|s|u|sdw15|sdw15|8076|/data/mirror76 +199|77|m|m|s|u|sdw15|sdw15|8077|/data/mirror77 +200|78|m|m|s|u|sdw15|sdw15|8078|/data/mirror78 +# SDW16 +87|85|p|p|s|u|sdw16|sdw16|7085|/data/primary85 +88|86|p|p|s|u|sdw16|sdw16|7086|/data/primary86 +89|87|p|p|s|u|sdw16|sdw16|7087|/data/primary87 +90|88|p|p|s|u|sdw16|sdw16|7088|/data/primary88 +91|89|p|p|s|u|sdw16|sdw16|7089|/data/primary89 +92|90|p|p|s|u|sdw16|sdw16|7090|/data/primary90 +93|91|p|p|s|u|sdw16|sdw16|7091|/data/primary91 +94|92|p|p|s|u|sdw16|sdw16|7092|/data/primary92 +201|79|m|m|s|u|sdw16|sdw16|8079|/data/mirror79 +202|80|m|m|s|u|sdw16|sdw16|8080|/data/mirror80 +203|81|m|m|s|u|sdw16|sdw16|8081|/data/mirror81 +204|82|m|m|s|u|sdw16|sdw16|8082|/data/mirror82 +205|83|m|m|s|u|sdw16|sdw16|8083|/data/mirror83 +206|84|m|m|s|u|sdw16|sdw16|8084|/data/mirror84 +# SDW17 +95|93|p|p|s|u|sdw17|sdw17|7093|/data/primary93 +96|94|p|p|s|u|sdw17|sdw17|7094|/data/primary94 +97|95|p|p|s|u|sdw17|sdw17|7095|/data/primary95 +98|96|p|p|s|u|sdw17|sdw17|7096|/data/primary96 +99|97|p|p|s|u|sdw17|sdw17|7097|/data/primary97 +100|98|p|p|s|u|sdw17|sdw17|7098|/data/primary98 +101|99|p|p|s|u|sdw17|sdw17|7099|/data/primary99 +207|85|m|m|s|u|sdw17|sdw17|8085|/data/mirror85 +208|86|m|m|s|u|sdw17|sdw17|8086|/data/mirror86 +209|87|m|m|s|u|sdw17|sdw17|8087|/data/mirror87 +210|88|m|m|s|u|sdw17|sdw17|8088|/data/mirror88 +211|89|m|m|s|u|sdw17|sdw17|8089|/data/mirror89 +212|90|m|m|s|u|sdw17|sdw17|8090|/data/mirror90 +213|91|m|m|s|u|sdw17|sdw17|8091|/data/mirror91 +214|92|m|m|s|u|sdw17|sdw17|8092|/data/mirror92 +# SDW18 +102|100|p|p|s|u|sdw18|sdw18|7100|/data/primary100 +103|101|p|p|s|u|sdw18|sdw18|7101|/data/primary101 +104|102|p|p|s|u|sdw18|sdw18|7102|/data/primary102 +105|103|p|p|s|u|sdw18|sdw18|7103|/data/primary103 +106|104|p|p|s|u|sdw18|sdw18|7104|/data/primary104 +107|105|p|p|s|u|sdw18|sdw18|7105|/data/primary105 +215|93|m|m|s|u|sdw18|sdw18|8093|/data/mirror93 +216|94|m|m|s|u|sdw18|sdw18|8094|/data/mirror94 +217|95|m|m|s|u|sdw18|sdw18|8095|/data/mirror95 +218|96|m|m|s|u|sdw18|sdw18|8096|/data/mirror96 +219|97|m|m|s|u|sdw18|sdw18|8097|/data/mirror97 +220|98|m|m|s|u|sdw18|sdw18|8098|/data/mirror98 +221|99|m|m|s|u|sdw18|sdw18|8099|/data/mirror99 +# SDW19 +108|106|p|p|s|u|sdw19|sdw19|7106|/data/primary106 +109|107|p|p|s|u|sdw19|sdw19|7107|/data/primary107 +110|108|p|p|s|u|sdw19|sdw19|7108|/data/primary108 +111|109|p|p|s|u|sdw19|sdw19|7109|/data/primary109 +112|110|p|p|s|u|sdw19|sdw19|7110|/data/primary110 +113|111|p|p|s|u|sdw19|sdw19|7111|/data/primary111 +114|112|p|p|s|u|sdw19|sdw19|7112|/data/primary112 +235|113|m|m|s|u|sdw19|sdw19|8113|/data/mirror113 +236|114|m|m|s|u|sdw19|sdw19|8114|/data/mirror114 +237|115|m|m|s|u|sdw19|sdw19|8115|/data/mirror115 +238|116|m|m|s|u|sdw19|sdw19|8116|/data/mirror116 +239|117|m|m|s|u|sdw19|sdw19|8117|/data/mirror117 +240|118|m|m|s|u|sdw19|sdw19|8118|/data/mirror118 +241|119|m|m|s|u|sdw19|sdw19|8119|/data/mirror119 +# SDW2 +7|5|p|p|s|u|sdw2|sdw2|7005|/data/primary5 +8|6|p|p|s|u|sdw2|sdw2|7006|/data/primary6 +9|7|p|p|s|u|sdw2|sdw2|7007|/data/primary7 +10|8|p|p|s|u|sdw2|sdw2|7008|/data/primary8 +11|9|p|p|s|u|sdw2|sdw2|7009|/data/primary9 +12|10|p|p|s|u|sdw2|sdw2|7010|/data/primary10 +122|0|m|m|s|u|sdw2|sdw2|8000|/data/mirror0 +123|1|m|m|s|u|sdw2|sdw2|8001|/data/mirror1 +124|2|m|m|s|u|sdw2|sdw2|8002|/data/mirror2 +125|3|m|m|s|u|sdw2|sdw2|8003|/data/mirror3 +126|4|m|m|s|u|sdw2|sdw2|8004|/data/mirror4 +228|106|m|m|s|u|sdw2|sdw2|8106|/data/mirror106 +229|107|m|m|s|u|sdw2|sdw2|8107|/data/mirror107 +230|108|m|m|s|u|sdw2|sdw2|8108|/data/mirror108 +231|109|m|m|s|u|sdw2|sdw2|8109|/data/mirror109 +232|110|m|m|s|u|sdw2|sdw2|8110|/data/mirror110 +233|111|m|m|s|u|sdw2|sdw2|8111|/data/mirror111 +234|112|m|m|s|u|sdw2|sdw2|8112|/data/mirror112 +# SDW20 +115|113|p|p|s|u|sdw20|sdw20|7113|/data/primary113 +116|114|p|p|s|u|sdw20|sdw20|7114|/data/primary114 +117|115|p|p|s|u|sdw20|sdw20|7115|/data/primary115 +118|116|p|p|s|u|sdw20|sdw20|7116|/data/primary116 +119|117|p|p|s|u|sdw20|sdw20|7117|/data/primary117 +120|118|p|p|s|u|sdw20|sdw20|7118|/data/primary118 +121|119|p|p|s|u|sdw20|sdw20|7119|/data/primary119 +222|100|m|m|s|u|sdw20|sdw20|8100|/data/mirror100 +223|101|m|m|s|u|sdw20|sdw20|8101|/data/mirror101 +224|102|m|m|s|u|sdw20|sdw20|8102|/data/mirror102 +225|103|m|m|s|u|sdw20|sdw20|8103|/data/mirror103 +226|104|m|m|s|u|sdw20|sdw20|8104|/data/mirror104 +227|105|m|m|s|u|sdw20|sdw20|8105|/data/mirror105 +# SDW3 +13|11|p|p|s|u|sdw3|sdw3|7011|/data/primary11 +14|12|p|p|s|u|sdw3|sdw3|7012|/data/primary12 +15|13|p|p|s|u|sdw3|sdw3|7013|/data/primary13 +16|14|p|p|s|u|sdw3|sdw3|7014|/data/primary14 +17|15|p|p|s|u|sdw3|sdw3|7015|/data/primary15 +18|16|p|p|s|u|sdw3|sdw3|7016|/data/primary16 +151|29|m|m|s|u|sdw3|sdw3|8029|/data/mirror29 +152|30|m|m|s|u|sdw3|sdw3|8030|/data/mirror30 +153|31|m|m|s|u|sdw3|sdw3|8031|/data/mirror31 +154|32|m|m|s|u|sdw3|sdw3|8032|/data/mirror32 +155|33|m|m|s|u|sdw3|sdw3|8033|/data/mirror33 +156|34|m|m|s|u|sdw3|sdw3|8034|/data/mirror34 +# SDW4 +19|17|p|p|s|u|sdw4|sdw4|7017|/data/primary17 +20|18|p|p|s|u|sdw4|sdw4|7018|/data/primary18 +21|19|p|p|s|u|sdw4|sdw4|7019|/data/primary19 +22|20|p|p|s|u|sdw4|sdw4|7020|/data/primary20 +23|21|p|p|s|u|sdw4|sdw4|7021|/data/primary21 +24|22|p|p|s|u|sdw4|sdw4|7022|/data/primary22 +127|5|m|m|s|u|sdw4|sdw4|8005|/data/mirror5 +128|6|m|m|s|u|sdw4|sdw4|8006|/data/mirror6 +129|7|m|m|s|u|sdw4|sdw4|8007|/data/mirror7 +130|8|m|m|s|u|sdw4|sdw4|8008|/data/mirror8 +131|9|m|m|s|u|sdw4|sdw4|8009|/data/mirror9 +132|10|m|m|s|u|sdw4|sdw4|8010|/data/mirror10 +# SDW5 +25|23|p|p|s|u|sdw5|sdw5|7023|/data/primary23 +26|24|p|p|s|u|sdw5|sdw5|7024|/data/primary24 +27|25|p|p|s|u|sdw5|sdw5|7025|/data/primary25 +28|26|p|p|s|u|sdw5|sdw5|7026|/data/primary26 +29|27|p|p|s|u|sdw5|sdw5|7027|/data/primary27 +30|28|p|p|s|u|sdw5|sdw5|7028|/data/primary28 +162|40|m|m|s|u|sdw5|sdw5|8040|/data/mirror40 +163|41|m|m|s|u|sdw5|sdw5|8041|/data/mirror41 +164|42|m|m|s|u|sdw5|sdw5|8042|/data/mirror42 +165|43|m|m|s|u|sdw5|sdw5|8043|/data/mirror43 +166|44|m|m|s|u|sdw5|sdw5|8044|/data/mirror44 +# SDW6 +31|29|p|p|s|u|sdw6|sdw6|7029|/data/primary29 +32|30|p|p|s|u|sdw6|sdw6|7030|/data/primary30 +33|31|p|p|s|u|sdw6|sdw6|7031|/data/primary31 +34|32|p|p|s|u|sdw6|sdw6|7032|/data/primary32 +35|33|p|p|s|u|sdw6|sdw6|7033|/data/primary33 +36|34|p|p|s|u|sdw6|sdw6|7034|/data/primary34 +133|11|m|m|s|u|sdw6|sdw6|8011|/data/mirror11 +134|12|m|m|s|u|sdw6|sdw6|8012|/data/mirror12 +135|13|m|m|s|u|sdw6|sdw6|8013|/data/mirror13 +136|14|m|m|s|u|sdw6|sdw6|8014|/data/mirror14 +137|15|m|m|s|u|sdw6|sdw6|8015|/data/mirror15 +138|16|m|m|s|u|sdw6|sdw6|8016|/data/mirror16 +# SDW7 +37|35|p|p|s|u|sdw7|sdw7|7035|/data/primary35 +38|36|p|p|s|u|sdw7|sdw7|7036|/data/primary36 +39|37|p|p|s|u|sdw7|sdw7|7037|/data/primary37 +40|38|p|p|s|u|sdw7|sdw7|7038|/data/primary38 +41|39|p|p|s|u|sdw7|sdw7|7039|/data/primary39 +145|23|m|m|s|u|sdw7|sdw7|8023|/data/mirror23 +146|24|m|m|s|u|sdw7|sdw7|8024|/data/mirror24 +147|25|m|m|s|u|sdw7|sdw7|8025|/data/mirror25 +148|26|m|m|s|u|sdw7|sdw7|8026|/data/mirror26 +149|27|m|m|s|u|sdw7|sdw7|8027|/data/mirror27 +150|28|m|m|s|u|sdw7|sdw7|8028|/data/mirror28 +# SDW8 +42|40|p|p|s|u|sdw8|sdw8|7040|/data/primary40 +43|41|p|p|s|u|sdw8|sdw8|7041|/data/primary41 +44|42|p|p|s|u|sdw8|sdw8|7042|/data/primary42 +45|43|p|p|s|u|sdw8|sdw8|7043|/data/primary43 +46|44|p|p|s|u|sdw8|sdw8|7044|/data/primary44 +179|57|m|m|s|u|sdw8|sdw8|8057|/data/mirror57 +180|58|m|m|s|u|sdw8|sdw8|8058|/data/mirror58 +181|59|m|m|s|u|sdw8|sdw8|8059|/data/mirror59 +182|60|m|m|s|u|sdw8|sdw8|8060|/data/mirror60 +183|61|m|m|s|u|sdw8|sdw8|8061|/data/mirror61 +184|62|m|m|s|u|sdw8|sdw8|8062|/data/mirror62 +# SDW9 +47|45|p|p|s|u|sdw9|sdw9|7045|/data/primary45 +48|46|p|p|s|u|sdw9|sdw9|7046|/data/primary46 +49|47|p|p|s|u|sdw9|sdw9|7047|/data/primary47 +50|48|p|p|s|u|sdw9|sdw9|7048|/data/primary48 +51|49|p|p|s|u|sdw9|sdw9|7049|/data/primary49 +157|35|m|m|s|u|sdw9|sdw9|8035|/data/mirror35 +158|36|m|m|s|u|sdw9|sdw9|8036|/data/mirror36 +159|37|m|m|s|u|sdw9|sdw9|8037|/data/mirror37 +160|38|m|m|s|u|sdw9|sdw9|8038|/data/mirror38 +161|39|m|m|s|u|sdw9|sdw9|8039|/data/mirror39 diff --git a/gpMgmt/bin/gprebalance_modules/test/data/120_20_unbalanced_spread.array b/gpMgmt/bin/gprebalance_modules/test/data/120_20_unbalanced_spread.array new file mode 100644 index 000000000000..16faef4b141c --- /dev/null +++ b/gpMgmt/bin/gprebalance_modules/test/data/120_20_unbalanced_spread.array @@ -0,0 +1,262 @@ +# Coordinator +1|-1|p|p|s|u|cdw|cdw|5432|/data/coordinator +# SDW1 +2|0|p|p|s|u|sdw1|sdw1|7000|/data/primary0 +3|1|p|p|s|u|sdw1|sdw1|7001|/data/primary1 +4|2|p|p|s|u|sdw1|sdw1|7002|/data/primary2 +5|3|p|p|s|u|sdw1|sdw1|7003|/data/primary3 +6|4|p|p|s|u|sdw1|sdw1|7004|/data/primary4 +7|5|p|p|s|u|sdw1|sdw1|7005|/data/primary5 +128|6|m|m|s|u|sdw1|sdw1|8006|/data/mirror6 +142|20|m|m|s|u|sdw1|sdw1|8020|/data/mirror20 +163|41|m|m|s|u|sdw1|sdw1|8041|/data/mirror41 +182|60|m|m|s|u|sdw1|sdw1|8060|/data/mirror60 +201|79|m|m|s|u|sdw1|sdw1|8079|/data/mirror79 +223|101|m|m|s|u|sdw1|sdw1|8101|/data/mirror101 +236|114|m|m|s|u|sdw1|sdw1|8114|/data/mirror114 +# SDW10 +56|54|p|p|s|u|sdw10|sdw10|7054|/data/primary54 +57|55|p|p|s|u|sdw10|sdw10|7055|/data/primary55 +58|56|p|p|s|u|sdw10|sdw10|7056|/data/primary56 +59|57|p|p|s|u|sdw10|sdw10|7057|/data/primary57 +60|58|p|p|s|u|sdw10|sdw10|7058|/data/primary58 +61|59|p|p|s|u|sdw10|sdw10|7059|/data/primary59 +131|9|m|m|s|u|sdw10|sdw10|8009|/data/mirror9 +152|30|m|m|s|u|sdw10|sdw10|8030|/data/mirror30 +171|49|m|m|s|u|sdw10|sdw10|8049|/data/mirror49 +192|70|m|m|s|u|sdw10|sdw10|8070|/data/mirror70 +208|86|m|m|s|u|sdw10|sdw10|8086|/data/mirror86 +231|109|m|m|s|u|sdw10|sdw10|8109|/data/mirror109 +# SDW11 +62|60|p|p|s|u|sdw11|sdw11|7060|/data/primary60 +63|61|p|p|s|u|sdw11|sdw11|7061|/data/primary61 +64|62|p|p|s|u|sdw11|sdw11|7062|/data/primary62 +65|63|p|p|s|u|sdw11|sdw11|7063|/data/primary63 +133|11|m|m|s|u|sdw11|sdw11|8011|/data/mirror11 +148|26|m|m|s|u|sdw11|sdw11|8026|/data/mirror26 +172|50|m|m|s|u|sdw11|sdw11|8050|/data/mirror50 +193|71|m|m|s|u|sdw11|sdw11|8071|/data/mirror71 +212|90|m|m|s|u|sdw11|sdw11|8090|/data/mirror90 +232|110|m|m|s|u|sdw11|sdw11|8110|/data/mirror110 +# SDW12 +66|64|p|p|s|u|sdw12|sdw12|7064|/data/primary64 +67|65|p|p|s|u|sdw12|sdw12|7065|/data/primary65 +68|66|p|p|s|u|sdw12|sdw12|7066|/data/primary66 +69|67|p|p|s|u|sdw12|sdw12|7067|/data/primary67 +70|68|p|p|s|u|sdw12|sdw12|7068|/data/primary68 +134|12|m|m|s|u|sdw12|sdw12|8012|/data/mirror12 +153|31|m|m|s|u|sdw12|sdw12|8031|/data/mirror31 +173|51|m|m|s|u|sdw12|sdw12|8051|/data/mirror51 +194|72|m|m|s|u|sdw12|sdw12|8072|/data/mirror72 +213|91|m|m|s|u|sdw12|sdw12|8091|/data/mirror91 +233|111|m|m|s|u|sdw12|sdw12|8111|/data/mirror111 +# SDW13 +71|69|p|p|s|u|sdw13|sdw13|7069|/data/primary69 +72|70|p|p|s|u|sdw13|sdw13|7070|/data/primary70 +73|71|p|p|s|u|sdw13|sdw13|7071|/data/primary71 +74|72|p|p|s|u|sdw13|sdw13|7072|/data/primary72 +75|73|p|p|s|u|sdw13|sdw13|7073|/data/primary73 +76|74|p|p|s|u|sdw13|sdw13|7074|/data/primary74 +77|75|p|p|s|u|sdw13|sdw13|7075|/data/primary75 +78|76|p|p|s|u|sdw13|sdw13|7076|/data/primary76 +136|14|m|m|s|u|sdw13|sdw13|8014|/data/mirror14 +154|32|m|m|s|u|sdw13|sdw13|8032|/data/mirror32 +168|46|m|m|s|u|sdw13|sdw13|8046|/data/mirror46 +199|77|m|m|s|u|sdw13|sdw13|8077|/data/mirror77 +214|92|m|m|s|u|sdw13|sdw13|8092|/data/mirror92 +234|112|m|m|s|u|sdw13|sdw13|8112|/data/mirror112 +# SDW14 +79|77|p|p|s|u|sdw14|sdw14|7077|/data/primary77 +80|78|p|p|s|u|sdw14|sdw14|7078|/data/primary78 +81|79|p|p|s|u|sdw14|sdw14|7079|/data/primary79 +82|80|p|p|s|u|sdw14|sdw14|7080|/data/primary80 +83|81|p|p|s|u|sdw14|sdw14|7081|/data/primary81 +84|82|p|p|s|u|sdw14|sdw14|7082|/data/primary82 +138|16|m|m|s|u|sdw14|sdw14|8016|/data/mirror16 +155|33|m|m|s|u|sdw14|sdw14|8033|/data/mirror33 +174|52|m|m|s|u|sdw14|sdw14|8052|/data/mirror52 +190|68|m|m|s|u|sdw14|sdw14|8068|/data/mirror68 +216|94|m|m|s|u|sdw14|sdw14|8094|/data/mirror94 +235|113|m|m|s|u|sdw14|sdw14|8113|/data/mirror113 +# SDW15 +85|83|p|p|s|u|sdw15|sdw15|7083|/data/primary83 +86|84|p|p|s|u|sdw15|sdw15|7084|/data/primary84 +87|85|p|p|s|u|sdw15|sdw15|7085|/data/primary85 +88|86|p|p|s|u|sdw15|sdw15|7086|/data/primary86 +89|87|p|p|s|u|sdw15|sdw15|7087|/data/primary87 +90|88|p|p|s|u|sdw15|sdw15|7088|/data/primary88 +91|89|p|p|s|u|sdw15|sdw15|7089|/data/primary89 +132|10|m|m|s|u|sdw15|sdw15|8010|/data/mirror10 +158|36|m|m|s|u|sdw15|sdw15|8036|/data/mirror36 +177|55|m|m|s|u|sdw15|sdw15|8055|/data/mirror55 +195|73|m|m|s|u|sdw15|sdw15|8073|/data/mirror73 +217|95|m|m|s|u|sdw15|sdw15|8095|/data/mirror95 +237|115|m|m|s|u|sdw15|sdw15|8115|/data/mirror115 +# SDW16 +92|90|p|p|s|u|sdw16|sdw16|7090|/data/primary90 +93|91|p|p|s|u|sdw16|sdw16|7091|/data/primary91 +94|92|p|p|s|u|sdw16|sdw16|7092|/data/primary92 +95|93|p|p|s|u|sdw16|sdw16|7093|/data/primary93 +96|94|p|p|s|u|sdw16|sdw16|7094|/data/primary94 +97|95|p|p|s|u|sdw16|sdw16|7095|/data/primary95 +98|96|p|p|s|u|sdw16|sdw16|7096|/data/primary96 +135|13|m|m|s|u|sdw16|sdw16|8013|/data/mirror13 +159|37|m|m|s|u|sdw16|sdw16|8037|/data/mirror37 +179|57|m|m|s|u|sdw16|sdw16|8057|/data/mirror57 +196|74|m|m|s|u|sdw16|sdw16|8074|/data/mirror74 +219|97|m|m|s|u|sdw16|sdw16|8097|/data/mirror97 +238|116|m|m|s|u|sdw16|sdw16|8116|/data/mirror116 +# SDW17 +99|97|p|p|s|u|sdw17|sdw17|7097|/data/primary97 +100|98|p|p|s|u|sdw17|sdw17|7098|/data/primary98 +101|99|p|p|s|u|sdw17|sdw17|7099|/data/primary99 +102|100|p|p|s|u|sdw17|sdw17|7100|/data/primary100 +103|101|p|p|s|u|sdw17|sdw17|7101|/data/primary101 +104|102|p|p|s|u|sdw17|sdw17|7102|/data/primary102 +139|17|m|m|s|u|sdw17|sdw17|8017|/data/mirror17 +161|39|m|m|s|u|sdw17|sdw17|8039|/data/mirror39 +180|58|m|m|s|u|sdw17|sdw17|8058|/data/mirror58 +198|76|m|m|s|u|sdw17|sdw17|8076|/data/mirror76 +218|96|m|m|s|u|sdw17|sdw17|8096|/data/mirror96 +240|118|m|m|s|u|sdw17|sdw17|8118|/data/mirror118 +# SDW18 +105|103|p|p|s|u|sdw18|sdw18|7103|/data/primary103 +106|104|p|p|s|u|sdw18|sdw18|7104|/data/primary104 +107|105|p|p|s|u|sdw18|sdw18|7105|/data/primary105 +108|106|p|p|s|u|sdw18|sdw18|7106|/data/primary106 +109|107|p|p|s|u|sdw18|sdw18|7107|/data/primary107 +110|108|p|p|s|u|sdw18|sdw18|7108|/data/primary108 +140|18|m|m|s|u|sdw18|sdw18|8018|/data/mirror18 +162|40|m|m|s|u|sdw18|sdw18|8040|/data/mirror40 +178|56|m|m|s|u|sdw18|sdw18|8056|/data/mirror56 +200|78|m|m|s|u|sdw18|sdw18|8078|/data/mirror78 +215|93|m|m|s|u|sdw18|sdw18|8093|/data/mirror93 +241|119|m|m|s|u|sdw18|sdw18|8119|/data/mirror119 +# SDW19 +111|109|p|p|s|u|sdw19|sdw19|7109|/data/primary109 +112|110|p|p|s|u|sdw19|sdw19|7110|/data/primary110 +113|111|p|p|s|u|sdw19|sdw19|7111|/data/primary111 +114|112|p|p|s|u|sdw19|sdw19|7112|/data/primary112 +115|113|p|p|s|u|sdw19|sdw19|7113|/data/primary113 +116|114|p|p|s|u|sdw19|sdw19|7114|/data/primary114 +141|19|m|m|s|u|sdw19|sdw19|8019|/data/mirror19 +160|38|m|m|s|u|sdw19|sdw19|8038|/data/mirror38 +181|59|m|m|s|u|sdw19|sdw19|8059|/data/mirror59 +202|80|m|m|s|u|sdw19|sdw19|8080|/data/mirror80 +220|98|m|m|s|u|sdw19|sdw19|8098|/data/mirror98 +239|117|m|m|s|u|sdw19|sdw19|8117|/data/mirror117 +# SDW2 +8|6|p|p|s|u|sdw2|sdw2|7006|/data/primary6 +9|7|p|p|s|u|sdw2|sdw2|7007|/data/primary7 +10|8|p|p|s|u|sdw2|sdw2|7008|/data/primary8 +11|9|p|p|s|u|sdw2|sdw2|7009|/data/primary9 +12|10|p|p|s|u|sdw2|sdw2|7010|/data/primary10 +122|0|m|m|s|u|sdw2|sdw2|8000|/data/mirror0 +143|21|m|m|s|u|sdw2|sdw2|8021|/data/mirror21 +157|35|m|m|s|u|sdw2|sdw2|8035|/data/mirror35 +183|61|m|m|s|u|sdw2|sdw2|8061|/data/mirror61 +205|83|m|m|s|u|sdw2|sdw2|8083|/data/mirror83 +221|99|m|m|s|u|sdw2|sdw2|8099|/data/mirror99 +# SDW20 +117|115|p|p|s|u|sdw20|sdw20|7115|/data/primary115 +118|116|p|p|s|u|sdw20|sdw20|7116|/data/primary116 +119|117|p|p|s|u|sdw20|sdw20|7117|/data/primary117 +120|118|p|p|s|u|sdw20|sdw20|7118|/data/primary118 +121|119|p|p|s|u|sdw20|sdw20|7119|/data/primary119 +137|15|m|m|s|u|sdw20|sdw20|8015|/data/mirror15 +156|34|m|m|s|u|sdw20|sdw20|8034|/data/mirror34 +175|53|m|m|s|u|sdw20|sdw20|8053|/data/mirror53 +203|81|m|m|s|u|sdw20|sdw20|8081|/data/mirror81 +222|100|m|m|s|u|sdw20|sdw20|8100|/data/mirror100 +# SDW3 +13|11|p|p|s|u|sdw3|sdw3|7011|/data/primary11 +14|12|p|p|s|u|sdw3|sdw3|7012|/data/primary12 +15|13|p|p|s|u|sdw3|sdw3|7013|/data/primary13 +16|14|p|p|s|u|sdw3|sdw3|7014|/data/primary14 +17|15|p|p|s|u|sdw3|sdw3|7015|/data/primary15 +18|16|p|p|s|u|sdw3|sdw3|7016|/data/primary16 +123|1|m|m|s|u|sdw3|sdw3|8001|/data/mirror1 +144|22|m|m|s|u|sdw3|sdw3|8022|/data/mirror22 +165|43|m|m|s|u|sdw3|sdw3|8043|/data/mirror43 +184|62|m|m|s|u|sdw3|sdw3|8062|/data/mirror62 +197|75|m|m|s|u|sdw3|sdw3|8075|/data/mirror75 +225|103|m|m|s|u|sdw3|sdw3|8103|/data/mirror103 +# SDW4 +19|17|p|p|s|u|sdw4|sdw4|7017|/data/primary17 +20|18|p|p|s|u|sdw4|sdw4|7018|/data/primary18 +21|19|p|p|s|u|sdw4|sdw4|7019|/data/primary19 +22|20|p|p|s|u|sdw4|sdw4|7020|/data/primary20 +23|21|p|p|s|u|sdw4|sdw4|7021|/data/primary21 +24|22|p|p|s|u|sdw4|sdw4|7022|/data/primary22 +124|2|m|m|s|u|sdw4|sdw4|8002|/data/mirror2 +145|23|m|m|s|u|sdw4|sdw4|8023|/data/mirror23 +164|42|m|m|s|u|sdw4|sdw4|8042|/data/mirror42 +185|63|m|m|s|u|sdw4|sdw4|8063|/data/mirror63 +207|85|m|m|s|u|sdw4|sdw4|8085|/data/mirror85 +226|104|m|m|s|u|sdw4|sdw4|8104|/data/mirror104 +# SDW5 +25|23|p|p|s|u|sdw5|sdw5|7023|/data/primary23 +26|24|p|p|s|u|sdw5|sdw5|7024|/data/primary24 +27|25|p|p|s|u|sdw5|sdw5|7025|/data/primary25 +28|26|p|p|s|u|sdw5|sdw5|7026|/data/primary26 +29|27|p|p|s|u|sdw5|sdw5|7027|/data/primary27 +30|28|p|p|s|u|sdw5|sdw5|7028|/data/primary28 +125|3|m|m|s|u|sdw5|sdw5|8003|/data/mirror3 +151|29|m|m|s|u|sdw5|sdw5|8029|/data/mirror29 +166|44|m|m|s|u|sdw5|sdw5|8044|/data/mirror44 +186|64|m|m|s|u|sdw5|sdw5|8064|/data/mirror64 +206|84|m|m|s|u|sdw5|sdw5|8084|/data/mirror84 +227|105|m|m|s|u|sdw5|sdw5|8105|/data/mirror105 +# SDW6 +31|29|p|p|s|u|sdw6|sdw6|7029|/data/primary29 +32|30|p|p|s|u|sdw6|sdw6|7030|/data/primary30 +33|31|p|p|s|u|sdw6|sdw6|7031|/data/primary31 +34|32|p|p|s|u|sdw6|sdw6|7032|/data/primary32 +35|33|p|p|s|u|sdw6|sdw6|7033|/data/primary33 +36|34|p|p|s|u|sdw6|sdw6|7034|/data/primary34 +37|35|p|p|s|u|sdw6|sdw6|7035|/data/primary35 +126|4|m|m|s|u|sdw6|sdw6|8004|/data/mirror4 +146|24|m|m|s|u|sdw6|sdw6|8024|/data/mirror24 +167|45|m|m|s|u|sdw6|sdw6|8045|/data/mirror45 +187|65|m|m|s|u|sdw6|sdw6|8065|/data/mirror65 +209|87|m|m|s|u|sdw6|sdw6|8087|/data/mirror87 +228|106|m|m|s|u|sdw6|sdw6|8106|/data/mirror106 +# SDW7 +38|36|p|p|s|u|sdw7|sdw7|7036|/data/primary36 +39|37|p|p|s|u|sdw7|sdw7|7037|/data/primary37 +40|38|p|p|s|u|sdw7|sdw7|7038|/data/primary38 +41|39|p|p|s|u|sdw7|sdw7|7039|/data/primary39 +42|40|p|p|s|u|sdw7|sdw7|7040|/data/primary40 +127|5|m|m|s|u|sdw7|sdw7|8005|/data/mirror5 +149|27|m|m|s|u|sdw7|sdw7|8027|/data/mirror27 +169|47|m|m|s|u|sdw7|sdw7|8047|/data/mirror47 +188|66|m|m|s|u|sdw7|sdw7|8066|/data/mirror66 +210|88|m|m|s|u|sdw7|sdw7|8088|/data/mirror88 +229|107|m|m|s|u|sdw7|sdw7|8107|/data/mirror107 +# SDW8 +43|41|p|p|s|u|sdw8|sdw8|7041|/data/primary41 +44|42|p|p|s|u|sdw8|sdw8|7042|/data/primary42 +45|43|p|p|s|u|sdw8|sdw8|7043|/data/primary43 +46|44|p|p|s|u|sdw8|sdw8|7044|/data/primary44 +47|45|p|p|s|u|sdw8|sdw8|7045|/data/primary45 +48|46|p|p|s|u|sdw8|sdw8|7046|/data/primary46 +49|47|p|p|s|u|sdw8|sdw8|7047|/data/primary47 +129|7|m|m|s|u|sdw8|sdw8|8007|/data/mirror7 +150|28|m|m|s|u|sdw8|sdw8|8028|/data/mirror28 +170|48|m|m|s|u|sdw8|sdw8|8048|/data/mirror48 +189|67|m|m|s|u|sdw8|sdw8|8067|/data/mirror67 +204|82|m|m|s|u|sdw8|sdw8|8082|/data/mirror82 +224|102|m|m|s|u|sdw8|sdw8|8102|/data/mirror102 +# SDW9 +50|48|p|p|s|u|sdw9|sdw9|7048|/data/primary48 +51|49|p|p|s|u|sdw9|sdw9|7049|/data/primary49 +52|50|p|p|s|u|sdw9|sdw9|7050|/data/primary50 +53|51|p|p|s|u|sdw9|sdw9|7051|/data/primary51 +54|52|p|p|s|u|sdw9|sdw9|7052|/data/primary52 +55|53|p|p|s|u|sdw9|sdw9|7053|/data/primary53 +130|8|m|m|s|u|sdw9|sdw9|8008|/data/mirror8 +147|25|m|m|s|u|sdw9|sdw9|8025|/data/mirror25 +176|54|m|m|s|u|sdw9|sdw9|8054|/data/mirror54 +191|69|m|m|s|u|sdw9|sdw9|8069|/data/mirror69 +211|89|m|m|s|u|sdw9|sdw9|8089|/data/mirror89 +230|108|m|m|s|u|sdw9|sdw9|8108|/data/mirror108 diff --git a/gpMgmt/bin/gprebalance_modules/test/data/35_7_balanced_grouped.array b/gpMgmt/bin/gprebalance_modules/test/data/35_7_balanced_grouped.array new file mode 100644 index 000000000000..844aeb214fc9 --- /dev/null +++ b/gpMgmt/bin/gprebalance_modules/test/data/35_7_balanced_grouped.array @@ -0,0 +1,79 @@ +# Coordinator +1|-1|p|p|s|u|cdw|cdw|5432|/data/coordinator +# SDW1 +2|0|p|p|s|u|sdw1|sdw1|7000|/data/primary0 +3|1|p|p|s|u|sdw1|sdw1|7001|/data/primary1 +4|2|p|p|s|u|sdw1|sdw1|7002|/data/primary2 +5|3|p|p|s|u|sdw1|sdw1|7003|/data/primary3 +6|4|p|p|s|u|sdw1|sdw1|7004|/data/primary4 +67|30|m|m|s|u|sdw1|sdw1|8030|/data/mirror30 +68|31|m|m|s|u|sdw1|sdw1|8031|/data/mirror31 +69|32|m|m|s|u|sdw1|sdw1|8032|/data/mirror32 +70|33|m|m|s|u|sdw1|sdw1|8033|/data/mirror33 +71|34|m|m|s|u|sdw1|sdw1|8034|/data/mirror34 +# SDW2 +7|5|p|p|s|u|sdw2|sdw2|7005|/data/primary5 +8|6|p|p|s|u|sdw2|sdw2|7006|/data/primary6 +9|7|p|p|s|u|sdw2|sdw2|7007|/data/primary7 +10|8|p|p|s|u|sdw2|sdw2|7008|/data/primary8 +11|9|p|p|s|u|sdw2|sdw2|7009|/data/primary9 +37|0|m|m|s|u|sdw2|sdw2|8000|/data/mirror0 +38|1|m|m|s|u|sdw2|sdw2|8001|/data/mirror1 +39|2|m|m|s|u|sdw2|sdw2|8002|/data/mirror2 +40|3|m|m|s|u|sdw2|sdw2|8003|/data/mirror3 +41|4|m|m|s|u|sdw2|sdw2|8004|/data/mirror4 +# SDW3 +12|10|p|p|s|u|sdw3|sdw3|7010|/data/primary10 +13|11|p|p|s|u|sdw3|sdw3|7011|/data/primary11 +14|12|p|p|s|u|sdw3|sdw3|7012|/data/primary12 +15|13|p|p|s|u|sdw3|sdw3|7013|/data/primary13 +16|14|p|p|s|u|sdw3|sdw3|7014|/data/primary14 +42|5|m|m|s|u|sdw3|sdw3|8005|/data/mirror5 +43|6|m|m|s|u|sdw3|sdw3|8006|/data/mirror6 +44|7|m|m|s|u|sdw3|sdw3|8007|/data/mirror7 +45|8|m|m|s|u|sdw3|sdw3|8008|/data/mirror8 +46|9|m|m|s|u|sdw3|sdw3|8009|/data/mirror9 +# SDW4 +17|15|p|p|s|u|sdw4|sdw4|7015|/data/primary15 +18|16|p|p|s|u|sdw4|sdw4|7016|/data/primary16 +19|17|p|p|s|u|sdw4|sdw4|7017|/data/primary17 +20|18|p|p|s|u|sdw4|sdw4|7018|/data/primary18 +21|19|p|p|s|u|sdw4|sdw4|7019|/data/primary19 +47|10|m|m|s|u|sdw4|sdw4|8010|/data/mirror10 +48|11|m|m|s|u|sdw4|sdw4|8011|/data/mirror11 +49|12|m|m|s|u|sdw4|sdw4|8012|/data/mirror12 +50|13|m|m|s|u|sdw4|sdw4|8013|/data/mirror13 +51|14|m|m|s|u|sdw4|sdw4|8014|/data/mirror14 +# SDW5 +22|20|p|p|s|u|sdw5|sdw5|7020|/data/primary20 +23|21|p|p|s|u|sdw5|sdw5|7021|/data/primary21 +24|22|p|p|s|u|sdw5|sdw5|7022|/data/primary22 +25|23|p|p|s|u|sdw5|sdw5|7023|/data/primary23 +26|24|p|p|s|u|sdw5|sdw5|7024|/data/primary24 +52|15|m|m|s|u|sdw5|sdw5|8015|/data/mirror15 +53|16|m|m|s|u|sdw5|sdw5|8016|/data/mirror16 +54|17|m|m|s|u|sdw5|sdw5|8017|/data/mirror17 +55|18|m|m|s|u|sdw5|sdw5|8018|/data/mirror18 +56|19|m|m|s|u|sdw5|sdw5|8019|/data/mirror19 +# SDW6 +27|25|p|p|s|u|sdw6|sdw6|7025|/data/primary25 +28|26|p|p|s|u|sdw6|sdw6|7026|/data/primary26 +29|27|p|p|s|u|sdw6|sdw6|7027|/data/primary27 +30|28|p|p|s|u|sdw6|sdw6|7028|/data/primary28 +31|29|p|p|s|u|sdw6|sdw6|7029|/data/primary29 +57|20|m|m|s|u|sdw6|sdw6|8020|/data/mirror20 +58|21|m|m|s|u|sdw6|sdw6|8021|/data/mirror21 +59|22|m|m|s|u|sdw6|sdw6|8022|/data/mirror22 +60|23|m|m|s|u|sdw6|sdw6|8023|/data/mirror23 +61|24|m|m|s|u|sdw6|sdw6|8024|/data/mirror24 +# SDW7 +32|30|p|p|s|u|sdw7|sdw7|7030|/data/primary30 +33|31|p|p|s|u|sdw7|sdw7|7031|/data/primary31 +34|32|p|p|s|u|sdw7|sdw7|7032|/data/primary32 +35|33|p|p|s|u|sdw7|sdw7|7033|/data/primary33 +36|34|p|p|s|u|sdw7|sdw7|7034|/data/primary34 +62|25|m|m|s|u|sdw7|sdw7|8025|/data/mirror25 +63|26|m|m|s|u|sdw7|sdw7|8026|/data/mirror26 +64|27|m|m|s|u|sdw7|sdw7|8027|/data/mirror27 +65|28|m|m|s|u|sdw7|sdw7|8028|/data/mirror28 +66|29|m|m|s|u|sdw7|sdw7|8029|/data/mirror29 diff --git a/gpMgmt/bin/gprebalance_modules/test/data/35_7_balanced_spread.array b/gpMgmt/bin/gprebalance_modules/test/data/35_7_balanced_spread.array new file mode 100644 index 000000000000..8a09d0130228 --- /dev/null +++ b/gpMgmt/bin/gprebalance_modules/test/data/35_7_balanced_spread.array @@ -0,0 +1,79 @@ +# Coordinator +1|-1|p|p|s|u|cdw|cdw|5432|/data/coordinator +# SDW1 +2|0|p|p|s|u|sdw1|sdw1|7000|/data/primary0 +3|1|p|p|s|u|sdw1|sdw1|7001|/data/primary1 +4|2|p|p|s|u|sdw1|sdw1|7002|/data/primary2 +5|3|p|p|s|u|sdw1|sdw1|7003|/data/primary3 +6|4|p|p|s|u|sdw1|sdw1|7004|/data/primary4 +51|14|m|m|s|u|sdw1|sdw1|8014|/data/mirror14 +55|18|m|m|s|u|sdw1|sdw1|8018|/data/mirror18 +59|22|m|m|s|u|sdw1|sdw1|8022|/data/mirror22 +63|26|m|m|s|u|sdw1|sdw1|8026|/data/mirror26 +67|30|m|m|s|u|sdw1|sdw1|8030|/data/mirror30 +# SDW2 +7|5|p|p|s|u|sdw2|sdw2|7005|/data/primary5 +8|6|p|p|s|u|sdw2|sdw2|7006|/data/primary6 +9|7|p|p|s|u|sdw2|sdw2|7007|/data/primary7 +10|8|p|p|s|u|sdw2|sdw2|7008|/data/primary8 +11|9|p|p|s|u|sdw2|sdw2|7009|/data/primary9 +37|0|m|m|s|u|sdw2|sdw2|8000|/data/mirror0 +56|19|m|m|s|u|sdw2|sdw2|8019|/data/mirror19 +60|23|m|m|s|u|sdw2|sdw2|8023|/data/mirror23 +64|27|m|m|s|u|sdw2|sdw2|8027|/data/mirror27 +68|31|m|m|s|u|sdw2|sdw2|8031|/data/mirror31 +# SDW3 +12|10|p|p|s|u|sdw3|sdw3|7010|/data/primary10 +13|11|p|p|s|u|sdw3|sdw3|7011|/data/primary11 +14|12|p|p|s|u|sdw3|sdw3|7012|/data/primary12 +15|13|p|p|s|u|sdw3|sdw3|7013|/data/primary13 +16|14|p|p|s|u|sdw3|sdw3|7014|/data/primary14 +38|1|m|m|s|u|sdw3|sdw3|8001|/data/mirror1 +42|5|m|m|s|u|sdw3|sdw3|8005|/data/mirror5 +61|24|m|m|s|u|sdw3|sdw3|8024|/data/mirror24 +65|28|m|m|s|u|sdw3|sdw3|8028|/data/mirror28 +69|32|m|m|s|u|sdw3|sdw3|8032|/data/mirror32 +# SDW4 +17|15|p|p|s|u|sdw4|sdw4|7015|/data/primary15 +18|16|p|p|s|u|sdw4|sdw4|7016|/data/primary16 +19|17|p|p|s|u|sdw4|sdw4|7017|/data/primary17 +20|18|p|p|s|u|sdw4|sdw4|7018|/data/primary18 +21|19|p|p|s|u|sdw4|sdw4|7019|/data/primary19 +39|2|m|m|s|u|sdw4|sdw4|8002|/data/mirror2 +43|6|m|m|s|u|sdw4|sdw4|8006|/data/mirror6 +47|10|m|m|s|u|sdw4|sdw4|8010|/data/mirror10 +66|29|m|m|s|u|sdw4|sdw4|8029|/data/mirror29 +70|33|m|m|s|u|sdw4|sdw4|8033|/data/mirror33 +# SDW5 +22|20|p|p|s|u|sdw5|sdw5|7020|/data/primary20 +23|21|p|p|s|u|sdw5|sdw5|7021|/data/primary21 +24|22|p|p|s|u|sdw5|sdw5|7022|/data/primary22 +25|23|p|p|s|u|sdw5|sdw5|7023|/data/primary23 +26|24|p|p|s|u|sdw5|sdw5|7024|/data/primary24 +40|3|m|m|s|u|sdw5|sdw5|8003|/data/mirror3 +44|7|m|m|s|u|sdw5|sdw5|8007|/data/mirror7 +48|11|m|m|s|u|sdw5|sdw5|8011|/data/mirror11 +52|15|m|m|s|u|sdw5|sdw5|8015|/data/mirror15 +71|34|m|m|s|u|sdw5|sdw5|8034|/data/mirror34 +# SDW6 +27|25|p|p|s|u|sdw6|sdw6|7025|/data/primary25 +28|26|p|p|s|u|sdw6|sdw6|7026|/data/primary26 +29|27|p|p|s|u|sdw6|sdw6|7027|/data/primary27 +30|28|p|p|s|u|sdw6|sdw6|7028|/data/primary28 +31|29|p|p|s|u|sdw6|sdw6|7029|/data/primary29 +41|4|m|m|s|u|sdw6|sdw6|8004|/data/mirror4 +45|8|m|m|s|u|sdw6|sdw6|8008|/data/mirror8 +49|12|m|m|s|u|sdw6|sdw6|8012|/data/mirror12 +53|16|m|m|s|u|sdw6|sdw6|8016|/data/mirror16 +57|20|m|m|s|u|sdw6|sdw6|8020|/data/mirror20 +# SDW7 +32|30|p|p|s|u|sdw7|sdw7|7030|/data/primary30 +33|31|p|p|s|u|sdw7|sdw7|7031|/data/primary31 +34|32|p|p|s|u|sdw7|sdw7|7032|/data/primary32 +35|33|p|p|s|u|sdw7|sdw7|7033|/data/primary33 +36|34|p|p|s|u|sdw7|sdw7|7034|/data/primary34 +46|9|m|m|s|u|sdw7|sdw7|8009|/data/mirror9 +50|13|m|m|s|u|sdw7|sdw7|8013|/data/mirror13 +54|17|m|m|s|u|sdw7|sdw7|8017|/data/mirror17 +58|21|m|m|s|u|sdw7|sdw7|8021|/data/mirror21 +62|25|m|m|s|u|sdw7|sdw7|8025|/data/mirror25 diff --git a/gpMgmt/bin/gprebalance_modules/test/data/40_5_unbalanced_grouped.array b/gpMgmt/bin/gprebalance_modules/test/data/40_5_unbalanced_grouped.array new file mode 100644 index 000000000000..9d21e2e84857 --- /dev/null +++ b/gpMgmt/bin/gprebalance_modules/test/data/40_5_unbalanced_grouped.array @@ -0,0 +1,87 @@ +# Coordinator +1|-1|p|p|s|u|cdw|cdw|5432|/data/coordinator +# SDW1 +2|0|p|p|s|u|sdw1|sdw1|7000|/data/primary0 +3|1|p|p|s|u|sdw1|sdw1|7001|/data/primary1 +4|2|p|p|s|u|sdw1|sdw1|7002|/data/primary2 +5|3|p|p|s|u|sdw1|sdw1|7003|/data/primary3 +6|4|p|p|s|u|sdw1|sdw1|7004|/data/primary4 +7|5|p|p|s|u|sdw1|sdw1|7005|/data/primary5 +8|6|p|p|s|u|sdw1|sdw1|7006|/data/primary6 +49|7|m|m|s|u|sdw1|sdw1|8007|/data/mirror7 +50|8|m|m|s|u|sdw1|sdw1|8008|/data/mirror8 +51|9|m|m|s|u|sdw1|sdw1|8009|/data/mirror9 +52|10|m|m|s|u|sdw1|sdw1|8010|/data/mirror10 +53|11|m|m|s|u|sdw1|sdw1|8011|/data/mirror11 +54|12|m|m|s|u|sdw1|sdw1|8012|/data/mirror12 +55|13|m|m|s|u|sdw1|sdw1|8013|/data/mirror13 +56|14|m|m|s|u|sdw1|sdw1|8014|/data/mirror14 +57|15|m|m|s|u|sdw1|sdw1|8015|/data/mirror15 +75|33|m|m|s|u|sdw1|sdw1|8033|/data/mirror33 +76|34|m|m|s|u|sdw1|sdw1|8034|/data/mirror34 +77|35|m|m|s|u|sdw1|sdw1|8035|/data/mirror35 +78|36|m|m|s|u|sdw1|sdw1|8036|/data/mirror36 +79|37|m|m|s|u|sdw1|sdw1|8037|/data/mirror37 +80|38|m|m|s|u|sdw1|sdw1|8038|/data/mirror38 +81|39|m|m|s|u|sdw1|sdw1|8039|/data/mirror39 +# SDW2 +9|7|p|p|s|u|sdw2|sdw2|7007|/data/primary7 +10|8|p|p|s|u|sdw2|sdw2|7008|/data/primary8 +11|9|p|p|s|u|sdw2|sdw2|7009|/data/primary9 +12|10|p|p|s|u|sdw2|sdw2|7010|/data/primary10 +13|11|p|p|s|u|sdw2|sdw2|7011|/data/primary11 +14|12|p|p|s|u|sdw2|sdw2|7012|/data/primary12 +15|13|p|p|s|u|sdw2|sdw2|7013|/data/primary13 +16|14|p|p|s|u|sdw2|sdw2|7014|/data/primary14 +17|15|p|p|s|u|sdw2|sdw2|7015|/data/primary15 +66|24|m|m|s|u|sdw2|sdw2|8024|/data/mirror24 +67|25|m|m|s|u|sdw2|sdw2|8025|/data/mirror25 +68|26|m|m|s|u|sdw2|sdw2|8026|/data/mirror26 +69|27|m|m|s|u|sdw2|sdw2|8027|/data/mirror27 +70|28|m|m|s|u|sdw2|sdw2|8028|/data/mirror28 +71|29|m|m|s|u|sdw2|sdw2|8029|/data/mirror29 +72|30|m|m|s|u|sdw2|sdw2|8030|/data/mirror30 +73|31|m|m|s|u|sdw2|sdw2|8031|/data/mirror31 +74|32|m|m|s|u|sdw2|sdw2|8032|/data/mirror32 +# SDW3 +18|16|p|p|s|u|sdw3|sdw3|7016|/data/primary16 +19|17|p|p|s|u|sdw3|sdw3|7017|/data/primary17 +20|18|p|p|s|u|sdw3|sdw3|7018|/data/primary18 +21|19|p|p|s|u|sdw3|sdw3|7019|/data/primary19 +22|20|p|p|s|u|sdw3|sdw3|7020|/data/primary20 +23|21|p|p|s|u|sdw3|sdw3|7021|/data/primary21 +24|22|p|p|s|u|sdw3|sdw3|7022|/data/primary22 +25|23|p|p|s|u|sdw3|sdw3|7023|/data/primary23 +# SDW4 +26|24|p|p|s|u|sdw4|sdw4|7024|/data/primary24 +27|25|p|p|s|u|sdw4|sdw4|7025|/data/primary25 +28|26|p|p|s|u|sdw4|sdw4|7026|/data/primary26 +29|27|p|p|s|u|sdw4|sdw4|7027|/data/primary27 +30|28|p|p|s|u|sdw4|sdw4|7028|/data/primary28 +31|29|p|p|s|u|sdw4|sdw4|7029|/data/primary29 +32|30|p|p|s|u|sdw4|sdw4|7030|/data/primary30 +33|31|p|p|s|u|sdw4|sdw4|7031|/data/primary31 +34|32|p|p|s|u|sdw4|sdw4|7032|/data/primary32 +42|0|m|m|s|u|sdw4|sdw4|8000|/data/mirror0 +43|1|m|m|s|u|sdw4|sdw4|8001|/data/mirror1 +44|2|m|m|s|u|sdw4|sdw4|8002|/data/mirror2 +45|3|m|m|s|u|sdw4|sdw4|8003|/data/mirror3 +46|4|m|m|s|u|sdw4|sdw4|8004|/data/mirror4 +47|5|m|m|s|u|sdw4|sdw4|8005|/data/mirror5 +48|6|m|m|s|u|sdw4|sdw4|8006|/data/mirror6 +58|16|m|m|s|u|sdw4|sdw4|8016|/data/mirror16 +59|17|m|m|s|u|sdw4|sdw4|8017|/data/mirror17 +60|18|m|m|s|u|sdw4|sdw4|8018|/data/mirror18 +61|19|m|m|s|u|sdw4|sdw4|8019|/data/mirror19 +62|20|m|m|s|u|sdw4|sdw4|8020|/data/mirror20 +63|21|m|m|s|u|sdw4|sdw4|8021|/data/mirror21 +64|22|m|m|s|u|sdw4|sdw4|8022|/data/mirror22 +65|23|m|m|s|u|sdw4|sdw4|8023|/data/mirror23 +# SDW5 +35|33|p|p|s|u|sdw5|sdw5|7033|/data/primary33 +36|34|p|p|s|u|sdw5|sdw5|7034|/data/primary34 +37|35|p|p|s|u|sdw5|sdw5|7035|/data/primary35 +38|36|p|p|s|u|sdw5|sdw5|7036|/data/primary36 +39|37|p|p|s|u|sdw5|sdw5|7037|/data/primary37 +40|38|p|p|s|u|sdw5|sdw5|7038|/data/primary38 +41|39|p|p|s|u|sdw5|sdw5|7039|/data/primary39 diff --git a/gpMgmt/bin/gprebalance_modules/test/data/40_5_unbalanced_spread.array b/gpMgmt/bin/gprebalance_modules/test/data/40_5_unbalanced_spread.array new file mode 100644 index 000000000000..fac7059b73a6 --- /dev/null +++ b/gpMgmt/bin/gprebalance_modules/test/data/40_5_unbalanced_spread.array @@ -0,0 +1,87 @@ +# Coordinator +1|-1|p|p|s|u|cdw|cdw|5432|/data/coordinator +# SDW1 +2|0|p|p|s|u|sdw1|sdw1|7000|/data/primary0 +3|1|p|p|s|u|sdw1|sdw1|7001|/data/primary1 +4|2|p|p|s|u|sdw1|sdw1|7002|/data/primary2 +5|3|p|p|s|u|sdw1|sdw1|7003|/data/primary3 +6|4|p|p|s|u|sdw1|sdw1|7004|/data/primary4 +7|5|p|p|s|u|sdw1|sdw1|7005|/data/primary5 +8|6|p|p|s|u|sdw1|sdw1|7006|/data/primary6 +9|7|p|p|s|u|sdw1|sdw1|7007|/data/primary7 +50|8|m|m|s|u|sdw1|sdw1|8008|/data/mirror8 +51|9|m|m|s|u|sdw1|sdw1|8009|/data/mirror9 +53|11|m|m|s|u|sdw1|sdw1|8011|/data/mirror11 +57|15|m|m|s|u|sdw1|sdw1|8015|/data/mirror15 +62|20|m|m|s|u|sdw1|sdw1|8020|/data/mirror20 +72|30|m|m|s|u|sdw1|sdw1|8030|/data/mirror30 +75|33|m|m|s|u|sdw1|sdw1|8033|/data/mirror33 +78|36|m|m|s|u|sdw1|sdw1|8036|/data/mirror36 +# SDW2 +10|8|p|p|s|u|sdw2|sdw2|7008|/data/primary8 +11|9|p|p|s|u|sdw2|sdw2|7009|/data/primary9 +12|10|p|p|s|u|sdw2|sdw2|7010|/data/primary10 +13|11|p|p|s|u|sdw2|sdw2|7011|/data/primary11 +14|12|p|p|s|u|sdw2|sdw2|7012|/data/primary12 +15|13|p|p|s|u|sdw2|sdw2|7013|/data/primary13 +16|14|p|p|s|u|sdw2|sdw2|7014|/data/primary14 +17|15|p|p|s|u|sdw2|sdw2|7015|/data/primary15 +18|16|p|p|s|u|sdw2|sdw2|7016|/data/primary16 +42|0|m|m|s|u|sdw2|sdw2|8000|/data/mirror0 +44|2|m|m|s|u|sdw2|sdw2|8002|/data/mirror2 +59|17|m|m|s|u|sdw2|sdw2|8017|/data/mirror17 +60|18|m|m|s|u|sdw2|sdw2|8018|/data/mirror18 +63|21|m|m|s|u|sdw2|sdw2|8021|/data/mirror21 +65|23|m|m|s|u|sdw2|sdw2|8023|/data/mirror23 +66|24|m|m|s|u|sdw2|sdw2|8024|/data/mirror24 +79|37|m|m|s|u|sdw2|sdw2|8037|/data/mirror37 +80|38|m|m|s|u|sdw2|sdw2|8038|/data/mirror38 +# SDW3 +19|17|p|p|s|u|sdw3|sdw3|7017|/data/primary17 +20|18|p|p|s|u|sdw3|sdw3|7018|/data/primary18 +21|19|p|p|s|u|sdw3|sdw3|7019|/data/primary19 +22|20|p|p|s|u|sdw3|sdw3|7020|/data/primary20 +23|21|p|p|s|u|sdw3|sdw3|7021|/data/primary21 +24|22|p|p|s|u|sdw3|sdw3|7022|/data/primary22 +25|23|p|p|s|u|sdw3|sdw3|7023|/data/primary23 +26|24|p|p|s|u|sdw3|sdw3|7024|/data/primary24 +27|25|p|p|s|u|sdw3|sdw3|7025|/data/primary25 +43|1|m|m|s|u|sdw3|sdw3|8001|/data/mirror1 +48|6|m|m|s|u|sdw3|sdw3|8006|/data/mirror6 +54|12|m|m|s|u|sdw3|sdw3|8012|/data/mirror12 +58|16|m|m|s|u|sdw3|sdw3|8016|/data/mirror16 +68|26|m|m|s|u|sdw3|sdw3|8026|/data/mirror26 +69|27|m|m|s|u|sdw3|sdw3|8027|/data/mirror27 +70|28|m|m|s|u|sdw3|sdw3|8028|/data/mirror28 +71|29|m|m|s|u|sdw3|sdw3|8029|/data/mirror29 +77|35|m|m|s|u|sdw3|sdw3|8035|/data/mirror35 +81|39|m|m|s|u|sdw3|sdw3|8039|/data/mirror39 +# SDW4 +28|26|p|p|s|u|sdw4|sdw4|7026|/data/primary26 +29|27|p|p|s|u|sdw4|sdw4|7027|/data/primary27 +30|28|p|p|s|u|sdw4|sdw4|7028|/data/primary28 +31|29|p|p|s|u|sdw4|sdw4|7029|/data/primary29 +32|30|p|p|s|u|sdw4|sdw4|7030|/data/primary30 +33|31|p|p|s|u|sdw4|sdw4|7031|/data/primary31 +45|3|m|m|s|u|sdw4|sdw4|8003|/data/mirror3 +47|5|m|m|s|u|sdw4|sdw4|8005|/data/mirror5 +52|10|m|m|s|u|sdw4|sdw4|8010|/data/mirror10 +56|14|m|m|s|u|sdw4|sdw4|8014|/data/mirror14 +67|25|m|m|s|u|sdw4|sdw4|8025|/data/mirror25 +74|32|m|m|s|u|sdw4|sdw4|8032|/data/mirror32 +76|34|m|m|s|u|sdw4|sdw4|8034|/data/mirror34 +# SDW5 +34|32|p|p|s|u|sdw5|sdw5|7032|/data/primary32 +35|33|p|p|s|u|sdw5|sdw5|7033|/data/primary33 +36|34|p|p|s|u|sdw5|sdw5|7034|/data/primary34 +37|35|p|p|s|u|sdw5|sdw5|7035|/data/primary35 +38|36|p|p|s|u|sdw5|sdw5|7036|/data/primary36 +39|37|p|p|s|u|sdw5|sdw5|7037|/data/primary37 +40|38|p|p|s|u|sdw5|sdw5|7038|/data/primary38 +41|39|p|p|s|u|sdw5|sdw5|7039|/data/primary39 +46|4|m|m|s|u|sdw5|sdw5|8004|/data/mirror4 +49|7|m|m|s|u|sdw5|sdw5|8007|/data/mirror7 +55|13|m|m|s|u|sdw5|sdw5|8013|/data/mirror13 +61|19|m|m|s|u|sdw5|sdw5|8019|/data/mirror19 +64|22|m|m|s|u|sdw5|sdw5|8022|/data/mirror22 +73|31|m|m|s|u|sdw5|sdw5|8031|/data/mirror31 diff --git a/gpMgmt/bin/gprebalance_modules/test/data/40_8_unbalanced_spread.array b/gpMgmt/bin/gprebalance_modules/test/data/40_8_unbalanced_spread.array new file mode 100644 index 000000000000..950ee7dbed27 --- /dev/null +++ b/gpMgmt/bin/gprebalance_modules/test/data/40_8_unbalanced_spread.array @@ -0,0 +1,90 @@ +# Coordinator +1|-1|p|p|s|u|cdw|cdw|5432|/data/coordinator +# SDW1 +2|0|p|p|s|u|sdw1|sdw1|7000|/data/primary0 +3|1|p|p|s|u|sdw1|sdw1|7001|/data/primary1 +4|2|p|p|s|u|sdw1|sdw1|7002|/data/primary2 +5|3|p|p|s|u|sdw1|sdw1|7003|/data/primary3 +6|4|p|p|s|u|sdw1|sdw1|7004|/data/primary4 +47|5|m|m|s|u|sdw1|sdw1|8005|/data/mirror5 +50|8|m|m|s|u|sdw1|sdw1|8008|/data/mirror8 +60|18|m|m|s|u|sdw1|sdw1|8018|/data/mirror18 +69|27|m|m|s|u|sdw1|sdw1|8027|/data/mirror27 +75|33|m|m|s|u|sdw1|sdw1|8033|/data/mirror33 +# SDW2 +7|5|p|p|s|u|sdw2|sdw2|7005|/data/primary5 +8|6|p|p|s|u|sdw2|sdw2|7006|/data/primary6 +9|7|p|p|s|u|sdw2|sdw2|7007|/data/primary7 +10|8|p|p|s|u|sdw2|sdw2|7008|/data/primary8 +11|9|p|p|s|u|sdw2|sdw2|7009|/data/primary9 +12|10|p|p|s|u|sdw2|sdw2|7010|/data/primary10 +42|0|m|m|s|u|sdw2|sdw2|8000|/data/mirror0 +53|11|m|m|s|u|sdw2|sdw2|8011|/data/mirror11 +56|14|m|m|s|u|sdw2|sdw2|8014|/data/mirror14 +70|28|m|m|s|u|sdw2|sdw2|8028|/data/mirror28 +72|30|m|m|s|u|sdw2|sdw2|8030|/data/mirror30 +# SDW3 +13|11|p|p|s|u|sdw3|sdw3|7011|/data/primary11 +14|12|p|p|s|u|sdw3|sdw3|7012|/data/primary12 +15|13|p|p|s|u|sdw3|sdw3|7013|/data/primary13 +16|14|p|p|s|u|sdw3|sdw3|7014|/data/primary14 +17|15|p|p|s|u|sdw3|sdw3|7015|/data/primary15 +18|16|p|p|s|u|sdw3|sdw3|7016|/data/primary16 +43|1|m|m|s|u|sdw3|sdw3|8001|/data/mirror1 +51|9|m|m|s|u|sdw3|sdw3|8009|/data/mirror9 +62|20|m|m|s|u|sdw3|sdw3|8020|/data/mirror20 +71|29|m|m|s|u|sdw3|sdw3|8029|/data/mirror29 +78|36|m|m|s|u|sdw3|sdw3|8036|/data/mirror36 +# SDW4 +19|17|p|p|s|u|sdw4|sdw4|7017|/data/primary17 +20|18|p|p|s|u|sdw4|sdw4|7018|/data/primary18 +21|19|p|p|s|u|sdw4|sdw4|7019|/data/primary19 +22|20|p|p|s|u|sdw4|sdw4|7020|/data/primary20 +46|4|m|m|s|u|sdw4|sdw4|8004|/data/mirror4 +52|10|m|m|s|u|sdw4|sdw4|8010|/data/mirror10 +63|21|m|m|s|u|sdw4|sdw4|8021|/data/mirror21 +66|24|m|m|s|u|sdw4|sdw4|8024|/data/mirror24 +68|26|m|m|s|u|sdw4|sdw4|8026|/data/mirror26 +81|39|m|m|s|u|sdw4|sdw4|8039|/data/mirror39 +# SDW5 +23|21|p|p|s|u|sdw5|sdw5|7021|/data/primary21 +24|22|p|p|s|u|sdw5|sdw5|7022|/data/primary22 +25|23|p|p|s|u|sdw5|sdw5|7023|/data/primary23 +26|24|p|p|s|u|sdw5|sdw5|7024|/data/primary24 +27|25|p|p|s|u|sdw5|sdw5|7025|/data/primary25 +28|26|p|p|s|u|sdw5|sdw5|7026|/data/primary26 +48|6|m|m|s|u|sdw5|sdw5|8006|/data/mirror6 +54|12|m|m|s|u|sdw5|sdw5|8012|/data/mirror12 +57|15|m|m|s|u|sdw5|sdw5|8015|/data/mirror15 +73|31|m|m|s|u|sdw5|sdw5|8031|/data/mirror31 +76|34|m|m|s|u|sdw5|sdw5|8034|/data/mirror34 +# SDW6 +29|27|p|p|s|u|sdw6|sdw6|7027|/data/primary27 +30|28|p|p|s|u|sdw6|sdw6|7028|/data/primary28 +31|29|p|p|s|u|sdw6|sdw6|7029|/data/primary29 +32|30|p|p|s|u|sdw6|sdw6|7030|/data/primary30 +33|31|p|p|s|u|sdw6|sdw6|7031|/data/primary31 +44|2|m|m|s|u|sdw6|sdw6|8002|/data/mirror2 +55|13|m|m|s|u|sdw6|sdw6|8013|/data/mirror13 +64|22|m|m|s|u|sdw6|sdw6|8022|/data/mirror22 +74|32|m|m|s|u|sdw6|sdw6|8032|/data/mirror32 +80|38|m|m|s|u|sdw6|sdw6|8038|/data/mirror38 +# SDW7 +34|32|p|p|s|u|sdw7|sdw7|7032|/data/primary32 +35|33|p|p|s|u|sdw7|sdw7|7033|/data/primary33 +36|34|p|p|s|u|sdw7|sdw7|7034|/data/primary34 +37|35|p|p|s|u|sdw7|sdw7|7035|/data/primary35 +49|7|m|m|s|u|sdw7|sdw7|8007|/data/mirror7 +58|16|m|m|s|u|sdw7|sdw7|8016|/data/mirror16 +61|19|m|m|s|u|sdw7|sdw7|8019|/data/mirror19 +67|25|m|m|s|u|sdw7|sdw7|8025|/data/mirror25 +79|37|m|m|s|u|sdw7|sdw7|8037|/data/mirror37 +# SDW8 +38|36|p|p|s|u|sdw8|sdw8|7036|/data/primary36 +39|37|p|p|s|u|sdw8|sdw8|7037|/data/primary37 +40|38|p|p|s|u|sdw8|sdw8|7038|/data/primary38 +41|39|p|p|s|u|sdw8|sdw8|7039|/data/primary39 +45|3|m|m|s|u|sdw8|sdw8|8003|/data/mirror3 +59|17|m|m|s|u|sdw8|sdw8|8017|/data/mirror17 +65|23|m|m|s|u|sdw8|sdw8|8023|/data/mirror23 +77|35|m|m|s|u|sdw8|sdw8|8035|/data/mirror35 diff --git a/gpMgmt/bin/gprebalance_modules/test/data/balanced_grouped_6.array b/gpMgmt/bin/gprebalance_modules/test/data/balanced_grouped_6.array new file mode 100644 index 000000000000..7afb7c748c5a --- /dev/null +++ b/gpMgmt/bin/gprebalance_modules/test/data/balanced_grouped_6.array @@ -0,0 +1,16 @@ +# Coordinator +1|-1|p|p|s|u|cdw|cdw|5432|/data/coordinator +# SDW1 +2|0|p|p|s|u|sdw1|sdw1|40000|/data/primary0 +3|1|p|p|s|u|sdw1|sdw1|40001|/data/primary1 +4|2|p|p|s|u|sdw1|sdw1|40002|/data/primary2 +11|3|m|m|s|u|sdw1|sdw1|50003|/data/mirror3 +12|4|m|m|s|u|sdw1|sdw1|50004|/data/mirror4 +13|5|m|m|s|u|sdw1|sdw1|50005|/data/mirror5 +# SDW2 +5|3|p|p|s|u|sdw2|sdw2|40003|/data/primary3 +6|4|p|p|s|u|sdw2|sdw2|40004|/data/primary4 +7|5|p|p|s|u|sdw2|sdw2|40005|/data/primary5 +8|0|m|m|s|u|sdw2|sdw2|50000|/data/mirror0 +9|1|m|m|s|u|sdw2|sdw2|50001|/data/mirror1 +10|2|m|m|s|u|sdw2|sdw2|50002|/data/mirror2 diff --git a/gpMgmt/bin/gprebalance_modules/test/data/balanced_spread_24.array b/gpMgmt/bin/gprebalance_modules/test/data/balanced_spread_24.array new file mode 100644 index 000000000000..32584e9be11f --- /dev/null +++ b/gpMgmt/bin/gprebalance_modules/test/data/balanced_spread_24.array @@ -0,0 +1,54 @@ +# Coordinator +1|-1|p|p|s|u|cdw|cdw|5432|/data/coordinator +# SDW1 +2|0|p|p|s|u|sdw1|sdw1|40000|/data/primary0 +3|1|p|p|s|u|sdw1|sdw1|40001|/data/primary1 +4|2|p|p|s|u|sdw1|sdw1|40002|/data/primary2 +5|3|p|p|s|u|sdw1|sdw1|40003|/data/primary3 +6|4|p|p|s|u|sdw1|sdw1|40004|/data/primary4 +7|5|p|p|s|u|sdw1|sdw1|40005|/data/primary5 +32|6|m|m|s|u|sdw1|sdw1|50006|/data/mirror6 +35|9|m|m|s|u|sdw1|sdw1|50009|/data/mirror9 +38|12|m|m|s|u|sdw1|sdw1|50012|/data/mirror12 +41|15|m|m|s|u|sdw1|sdw1|50015|/data/mirror15 +44|18|m|m|s|u|sdw1|sdw1|50018|/data/mirror18 +47|21|m|m|s|u|sdw1|sdw1|50021|/data/mirror21 +# SDW2 +8|6|p|p|s|u|sdw2|sdw2|40006|/data/primary6 +9|7|p|p|s|u|sdw2|sdw2|40007|/data/primary7 +10|8|p|p|s|u|sdw2|sdw2|40008|/data/primary8 +11|9|p|p|s|u|sdw2|sdw2|40009|/data/primary9 +12|10|p|p|s|u|sdw2|sdw2|40010|/data/primary10 +13|11|p|p|s|u|sdw2|sdw2|40011|/data/primary11 +26|0|m|m|s|u|sdw2|sdw2|50000|/data/mirror0 +29|3|m|m|s|u|sdw2|sdw2|50003|/data/mirror3 +39|13|m|m|s|u|sdw2|sdw2|50013|/data/mirror13 +42|16|m|m|s|u|sdw2|sdw2|50016|/data/mirror16 +45|19|m|m|s|u|sdw2|sdw2|50019|/data/mirror19 +48|22|m|m|s|u|sdw2|sdw2|50022|/data/mirror22 +# SDW3 +14|12|p|p|s|u|sdw3|sdw3|40012|/data/primary12 +15|13|p|p|s|u|sdw3|sdw3|40013|/data/primary13 +16|14|p|p|s|u|sdw3|sdw3|40014|/data/primary14 +17|15|p|p|s|u|sdw3|sdw3|40015|/data/primary15 +18|16|p|p|s|u|sdw3|sdw3|40016|/data/primary16 +19|17|p|p|s|u|sdw3|sdw3|40017|/data/primary17 +27|1|m|m|s|u|sdw3|sdw3|50001|/data/mirror1 +30|4|m|m|s|u|sdw3|sdw3|50004|/data/mirror4 +33|7|m|m|s|u|sdw3|sdw3|50007|/data/mirror7 +36|10|m|m|s|u|sdw3|sdw3|50010|/data/mirror10 +46|20|m|m|s|u|sdw3|sdw3|50020|/data/mirror20 +49|23|m|m|s|u|sdw3|sdw3|50023|/data/mirror23 +# SDW4 +20|18|p|p|s|u|sdw4|sdw4|40018|/data/primary18 +21|19|p|p|s|u|sdw4|sdw4|40019|/data/primary19 +22|20|p|p|s|u|sdw4|sdw4|40020|/data/primary20 +23|21|p|p|s|u|sdw4|sdw4|40021|/data/primary21 +24|22|p|p|s|u|sdw4|sdw4|40022|/data/primary22 +25|23|p|p|s|u|sdw4|sdw4|40023|/data/primary23 +28|2|m|m|s|u|sdw4|sdw4|50002|/data/mirror2 +31|5|m|m|s|u|sdw4|sdw4|50005|/data/mirror5 +34|8|m|m|s|u|sdw4|sdw4|50008|/data/mirror8 +37|11|m|m|s|u|sdw4|sdw4|50011|/data/mirror11 +40|14|m|m|s|u|sdw4|sdw4|50014|/data/mirror14 +43|17|m|m|s|u|sdw4|sdw4|50017|/data/mirror17 diff --git a/gpMgmt/bin/gprebalance_modules/test/data/config_generator.py b/gpMgmt/bin/gprebalance_modules/test/data/config_generator.py new file mode 100644 index 000000000000..8ca82d1fb7fc --- /dev/null +++ b/gpMgmt/bin/gprebalance_modules/test/data/config_generator.py @@ -0,0 +1,616 @@ +#!/usr/bin/env python3 +""" +Cluster Configuration Generator for Testing + +Generates various cluster configurations with different characteristics: +- Balanced vs Unbalanced load distribution +- Grouped vs Spread mirroring strategies +- Various cluster sizes (hosts, segments) +""" + +import random +from typing import List, Dict, Tuple, Optional +from dataclasses import dataclass +from enum import Enum +import argparse + +class MirroringStrategy(Enum): + GROUPED = "grouped" + SPREAD = "spread" + +@dataclass +class SegmentConfig: + """ + Single segment configuration + """ + dbid: int + content: int + role: str # 'p' or 'm' + preferred_role: str + mode: str = 's' + status: str = 'u' + hostname: str = '' + address: str = '' + port: int = 0 + datadir: str = '' + + def to_line(self) -> str: + """ + Convert to str + """ + return (f"{self.dbid}|{self.content}|{self.role}|{self.preferred_role}|" + f"{self.mode}|{self.status}|{self.hostname}|{self.address}|" + f"{self.port}|{self.datadir}") + +class ClusterConfigGenerator: + """ + Generate Greengage cluster configurations + """ + + def __init__(self, n_segments: int, n_hosts: int, strategy: MirroringStrategy, + base_primary_port: int = 7000, base_mirror_port: int = 8000): + self.n_segments = n_segments + self.n_hosts = n_hosts + self.strategy = strategy + self.base_primary_port = base_primary_port + self.base_mirror_port = base_mirror_port + self.next_dbid = 2 # Start after coordinator (dbid=1) + + def generate_balanced_grouped(self) -> List[SegmentConfig]: + """ + Generate balanced configuration with GROUPED mirroring. + + Pattern: Primaries and mirrors evenly distributed. + All mirrors for primaries on host_i go to host_((i+1) % n_hosts) + """ + configs = [] + segments_per_host = self.n_segments // self.n_hosts + + if self.n_segments % self.n_hosts != 0: + raise ValueError(f"Cannot evenly distribute {self.n_segments} segments " + f"across {self.n_hosts} hosts for balanced config") + + # Assign primaries + primary_assignment = {} + content_id = 0 + + for host_id in range(self.n_hosts): + for _ in range(segments_per_host): + primary_assignment[content_id] = host_id + content_id += 1 + + # Assign mirrors (grouped) + mirror_assignment = {} + for content_id, primary_host in primary_assignment.items(): + mirror_host = (primary_host + 1) % self.n_hosts + mirror_assignment[content_id] = mirror_host + + # Generate segment configs + configs.extend(self._create_segments(primary_assignment, mirror_assignment)) + + return configs + + def generate_balanced_spread(self) -> List[SegmentConfig]: + """ + Generate balanced configuration with SPREAD mirroring. + + Pattern: Primaries and mirrors evenly distributed. + Mirrors for primaries on one host are spread across different hosts. + """ + configs = [] + segments_per_host = self.n_segments // self.n_hosts + + if self.n_segments % self.n_hosts != 0: + raise ValueError(f"Cannot evenly distribute {self.n_segments} segments " + f"across {self.n_hosts} hosts for balanced config") + + # Assign primaries + primary_assignment = {} + content_id = 0 + + for host_id in range(self.n_hosts): + for _ in range(segments_per_host): + primary_assignment[content_id] = host_id + content_id += 1 + + # Assign mirrors using global offset pattern + mirror_assignment = {} + + for content_id in range(self.n_segments): + primary_host = primary_assignment[content_id] + + # Calculate local index within primary host's segments + local_idx = content_id % segments_per_host + + # Global offset ensures different mirrors for same local_idx across hosts + # Formula: mirror = (primary + local_idx + global_offset) % n_hosts + # where global_offset depends on which primary host we're on + + # Use: mirror = (primary + 1 + local_idx) % n_hosts + # If mirror == primary, shift by 1 again + mirror_host = (primary_host + 1 + local_idx) % self.n_hosts + + # Ensure no colocation + if mirror_host == primary_host: + mirror_host = (mirror_host + 1) % self.n_hosts + + mirror_assignment[content_id] = mirror_host + + configs.extend(self._create_segments(primary_assignment, mirror_assignment)) + + return configs + + def generate_unbalanced_grouped(self, skew_factor: float = 0.2) -> List[SegmentConfig]: + """ + Generate UNBALANCED configuration with GROUPED mirroring. + + Args: + skew_factor: 0.0 = perfectly balanced, 1.0 = maximum imbalance + Recommended: 0.1-0.3 for realistic scenarios + + Pattern: Some hosts have slightly more segments than others. + All mirrors for primaries on host_i still go to one host. + """ + configs = [] + + # Create SLIGHTLY skewed distribution (not extreme) + primary_assignment = self._create_realistic_skewed_distribution(skew_factor) + + # Assign mirrors (grouped) + mirror_assignment = {} + mirror_load = [0] * self.n_hosts + + for host_id in range(self.n_hosts): + # Get all primaries on this host + primary_contents = [c for c, h in primary_assignment.items() if h == host_id] + + if not primary_contents: + continue + + # Choose mirror host - prefer underloaded hosts for realism + available_hosts = [h for h in range(self.n_hosts) if h != host_id] + + # Bias towards least loaded mirrors (creates more realistic initial state) + available_hosts.sort(key=lambda h: mirror_load[h]) + + # Pick from top 3 least loaded (adds some randomness) + target_mirror_host = random.choice(available_hosts[:min(3, len(available_hosts))]) + + for content_id in primary_contents: + mirror_assignment[content_id] = target_mirror_host + mirror_load[target_mirror_host] += 1 + + configs.extend(self._create_segments(primary_assignment, mirror_assignment)) + + return configs + + def generate_unbalanced_spread(self, skew_factor: float = 0.2) -> List[SegmentConfig]: + """ + Generate UNBALANCED configuration with SPREAD mirroring. + + Args: + skew_factor: 0.0 = perfectly balanced, 1.0 = maximum imbalance + Recommended: 0.1-0.3 for realistic scenarios + """ + configs = [] + + # Create slightly skewed distribution + primary_assignment = self._create_realistic_skewed_distribution(skew_factor) + + # Assign mirrors (spread) with slight preference for balance + mirror_assignment = {} + mirror_load = [0] * self.n_hosts + + for host_id in range(self.n_hosts): + # Get primaries on this host + primary_contents = [c for c, h in primary_assignment.items() if h == host_id] + + if not primary_contents: + continue + + # Spread mirrors across OTHER hosts, biased towards less loaded + available_mirror_hosts = [h for h in range(self.n_hosts) if h != host_id] + + for idx, content_id in enumerate(sorted(primary_contents)): + # Sort by load each time (semi-balanced spread) + available_mirror_hosts.sort(key=lambda h: (mirror_load[h], h)) + + # Pick with some randomness: 70% least loaded, 30% random + if random.random() < 0.7: + mirror_host = available_mirror_hosts[0] # Least loaded + else: + mirror_host = available_mirror_hosts[idx % len(available_mirror_hosts)] # Round-robin + + mirror_assignment[content_id] = mirror_host + mirror_load[mirror_host] += 1 + + configs.extend(self._create_segments(primary_assignment, mirror_assignment)) + + return configs + + def _create_realistic_skewed_distribution(self, skew_factor: float) -> Dict[int, int]: + """ + Create realistic skewed distribution. + + Instead of extreme imbalance, this creates a distribution where: + - Most hosts are within 10-20% of target load + - A few hosts are overloaded/underloaded + - Skew_factor controls the magnitude + + Args: + skew_factor: 0.0 = perfectly balanced, 1.0 = severe imbalance + + Returns: + Dict[content_id] -> host_id + """ + target_per_host = self.n_segments / self.n_hosts + + # Calculate actual segments per host with skew + # Use normal distribution around target with controlled variance + segments_per_host = [] + + for host_id in range(self.n_hosts): + if skew_factor == 0.0: + # Perfectly balanced + count = int(target_per_host) + else: + # Add controlled noise + # skew_factor controls standard deviation + std_dev = skew_factor * target_per_host * 0.3 # Max 30% deviation + noise = random.gauss(0, std_dev) + count = int(target_per_host + noise) + + # Ensure minimum 50% of target (never empty unless skew=1.0) + min_count = int(target_per_host * max(0.5, 1.0 - skew_factor)) + max_count = int(target_per_host * (1.0 + skew_factor * 0.5)) + count = max(min_count, min(max_count, count)) + + segments_per_host.append(count) + + # Adjust to exact total + current_total = sum(segments_per_host) + diff = self.n_segments - current_total + + # Distribute difference to random hosts + while diff != 0: + host_id = random.randint(0, self.n_hosts - 1) + if diff > 0: + segments_per_host[host_id] += 1 + diff -= 1 + elif segments_per_host[host_id] > 1: # Never go to 0 + segments_per_host[host_id] -= 1 + diff += 1 + + # Create assignment + primary_assignment = {} + content_id = 0 + + for host_id in range(self.n_hosts): + for _ in range(segments_per_host[host_id]): + primary_assignment[content_id] = host_id + content_id += 1 + + return primary_assignment + + def _create_segments(self, primary_assignment: Dict[int, int], + mirror_assignment: Dict[int, int]) -> List[SegmentConfig]: + """Create SegmentConfig objects from assignments""" + configs = [] + + # Create primaries + for content_id in sorted(primary_assignment.keys()): + host_id = primary_assignment[content_id] + hostname = f"sdw{host_id + 1}" + + # Count how many primaries on this host before this one + port_offset = sum(1 for c, h in primary_assignment.items() + if h == host_id and c < content_id) + + config = SegmentConfig( + dbid=self.next_dbid, + content=content_id, + role='p', + preferred_role='p', + hostname=hostname, + address=hostname, + port=self.base_primary_port + content_id, + datadir=f"/data/primary{content_id}" + ) + self.next_dbid += 1 + configs.append(config) + + # Create mirrors + for content_id in sorted(mirror_assignment.keys()): + host_id = mirror_assignment[content_id] + hostname = f"sdw{host_id + 1}" + + config = SegmentConfig( + dbid=self.next_dbid, + content=content_id, + role='m', + preferred_role='m', + hostname=hostname, + address=hostname, + port=self.base_mirror_port + content_id, + datadir=f"/data/mirror{content_id}" + ) + self.next_dbid += 1 + configs.append(config) + + return configs + + def generate_coordinator(self) -> SegmentConfig: + """Generate coordinator segment""" + return SegmentConfig( + dbid=1, + content=-1, + role='p', + preferred_role='p', + hostname='cdw', + address='cdw', + port=5432, + datadir='/data/coordinator' + ) + + def generate_config(self, balanced: bool = True) -> List[SegmentConfig]: + """ + Generate complete configuration. + + Args: + balanced: If True, generate balanced config; otherwise unbalanced + """ + if balanced: + if self.strategy == MirroringStrategy.GROUPED: + return self.generate_balanced_grouped() + else: + return self.generate_balanced_spread() + else: + skew_factor = random.uniform(0.3, 0.8) + if self.strategy == MirroringStrategy.GROUPED: + return self.generate_unbalanced_grouped(skew_factor) + else: + return self.generate_unbalanced_spread(skew_factor) + +def write_config_file(filename: str, configs: List[SegmentConfig], + coordinator: SegmentConfig): + """Write configuration to file""" + with open(filename, 'w') as f: + # Write coordinator + f.write("# Coordinator\n") + f.write(coordinator.to_line() + "\n") + + # Group segments by host + segments_by_host = {} + for config in configs: + host = config.hostname + if host not in segments_by_host: + segments_by_host[host] = [] + segments_by_host[host].append(config) + + # Write segments by host + for hostname in sorted(segments_by_host.keys()): + f.write(f"# {hostname.upper()}\n") + + host_segments = segments_by_host[hostname] + # Sort by role (primaries first), then by content + host_segments.sort(key=lambda s: (s.role != 'p', s.content)) + + for seg in host_segments: + f.write(seg.to_line() + "\n") + +def print_statistics(configs: List[SegmentConfig], n_hosts: int): + """Print configuration statistics""" + print("\n" + "=" * 80) + print("CONFIGURATION STATISTICS".center(80)) + print("=" * 80) + + # Count segments per host + primaries_per_host = [0] * n_hosts + mirrors_per_host = [0] * n_hosts + + for config in configs: + host_id = int(config.hostname.replace('sdw', '')) - 1 + if config.role == 'p': + primaries_per_host[host_id] += 1 + else: + mirrors_per_host[host_id] += 1 + + print(f"\nTotal Segments: {len(configs) // 2}") + print(f"Total Hosts: {n_hosts}") + print(f"\nLoad Distribution:") + print(f"{'Host':<10} {'Primaries':<12} {'Mirrors':<12} {'Total':<10}") + print("-" * 50) + + for host_id in range(n_hosts): + hostname = f"sdw{host_id + 1}" + primaries = primaries_per_host[host_id] + mirrors = mirrors_per_host[host_id] + total = primaries + mirrors + print(f"{hostname:<10} {primaries:<12} {mirrors:<12} {total:<10}") + + print("-" * 50) + + # Calculate balance metrics + total_loads = [p + m for p, m in zip(primaries_per_host, mirrors_per_host)] + avg_load = sum(total_loads) / n_hosts + max_load = max(total_loads) + min_load = min(total_loads) + + print(f"\nBalance Metrics:") + print(f" Average load per host: {avg_load:.2f}") + print(f" Min load: {min_load}") + print(f" Max load: {max_load}") + print(f" Load deviation: {max_load - min_load}") + print(f" Balance ratio: {min_load / max_load * 100:.1f}%") + + # Check mirroring strategy + print(f"\nMirroring Analysis:") + + for host_id in range(n_hosts): + # Get primaries on this host + primary_contents = [c.content for c in configs + if c.role == 'p' and c.hostname == f"sdw{host_id + 1}"] + + if not primary_contents: + continue + + # Find where their mirrors are + mirror_hosts = [] + for content in primary_contents: + mirror = next(c for c in configs if c.role == 'm' and c.content == content) + mirror_hosts.append(mirror.hostname) + + unique_mirror_hosts = len(set(mirror_hosts)) + + if unique_mirror_hosts == 1: + strategy = "GROUPED" + elif unique_mirror_hosts == len(mirror_hosts): + strategy = "SPREAD (perfect)" + else: + strategy = f"SPREAD (partial - {unique_mirror_hosts} hosts)" + + print(f" sdw{host_id + 1}: {len(primary_contents)} primaries → " + f"mirrors on {unique_mirror_hosts} host(s) [{strategy}]") + + print("=" * 80 + "\n") + +def main(): + parser = argparse.ArgumentParser( + description='Generate Greengage cluster test configurations' + ) + parser.add_argument('-n', '--segments', type=int, required=True, + help='Number of segment pairs') + parser.add_argument('-m', '--hosts', type=int, required=True, + help='Number of hosts') + parser.add_argument('-s', '--strategy', choices=['grouped', 'spread'], + required=True, help='Mirroring strategy') + parser.add_argument('-b', '--balanced', action='store_true', + help='Generate balanced configuration') + parser.add_argument('-u', '--unbalanced', action='store_true', + help='Generate unbalanced configuration') + parser.add_argument('-o', '--output', type=str, default='cluster_config.txt', + help='Output filename') + parser.add_argument('--seed', type=int, default=None, + help='Random seed for reproducibility') + + args = parser.parse_args() + + # Set random seed + if args.seed is not None: + random.seed(args.seed) + + # Determine balance + if args.balanced and args.unbalanced: + print("Error: Cannot specify both --balanced and --unbalanced") + return + + balanced = args.balanced or not args.unbalanced + + # Create generator + strategy = MirroringStrategy.GROUPED if args.strategy == 'grouped' else MirroringStrategy.SPREAD + + generator = ClusterConfigGenerator( + n_segments=args.segments, + n_hosts=args.hosts, + strategy=strategy + ) + + # Generate configuration + print(f"Generating {'balanced' if balanced else 'unbalanced'} configuration...") + print(f" Segments: {args.segments}") + print(f" Hosts: {args.hosts}") + print(f" Strategy: {args.strategy.upper()}") + + try: + configs = generator.generate_config(balanced=balanced) + coordinator = generator.generate_coordinator() + + # Write to file + write_config_file(args.output, configs, coordinator) + print(f"\nConfiguration written to: {args.output}") + + # Print statistics + print_statistics(configs, args.hosts) + + except ValueError as e: + print(f"Error: {e}") + return + +# Batch generator for test suite +def generate_test_suite(): + """Generate a comprehensive test suite""" + + test_cases = [ + # Small balanced configs + {'n': 35, 'm': 7, 'strategy': 'grouped', 'balanced': True, 'name': '35_7_balanced_grouped'}, + {'n': 35, 'm': 7, 'strategy': 'spread', 'balanced': True, 'name': '35_7_balanced_spread'}, + + # Small unbalanced configs + {'n': 40, 'm': 5, 'strategy': 'grouped', 'balanced': False, 'name': '40_5_unbalanced_grouped'}, + {'n': 40, 'm': 5, 'strategy': 'spread', 'balanced': False, 'name': '40_5_unbalanced_spread'}, + + # Medium balanced configs + {'n': 120, 'm': 20, 'strategy': 'grouped', 'balanced': True, 'name': '120_20_balanced_grouped'}, + {'n': 120, 'm': 20, 'strategy': 'spread', 'balanced': True, 'name': '120_20_balanced_spread'}, + + # Medium unbalanced configs + {'n': 120, 'm': 20, 'strategy': 'grouped', 'balanced': False, 'name': '120_20_unbalanced_grouped'}, + {'n': 120, 'm': 20, 'strategy': 'spread', 'balanced': False, 'name': '120_20_unbalanced_spread'}, + + # Large balanced configs + {'n': 1000, 'm': 50, 'strategy': 'grouped', 'balanced': True, 'name': '1000_50_balanced_grouped'}, + {'n': 1000, 'm': 50, 'strategy': 'spread', 'balanced': True, 'name': '1000_50_balanced_spread'}, + + # Large unbalanced configs + {'n': 1000, 'm': 50, 'strategy': 'grouped', 'balanced': False, 'name': '1000_50_unbalanced_grouped'}, + {'n': 1000, 'm': 50, 'strategy': 'spread', 'balanced': False, 'name': '1000_50_unbalanced_spread'}, + ] + + print("=" * 80) + print("GENERATING TEST SUITE".center(80)) + print("=" * 80) + + for idx, test_case in enumerate(test_cases, 1): + print(f"\n[{idx}/{len(test_cases)}] Generating: {test_case['name']}") + + strategy = MirroringStrategy.GROUPED if test_case['strategy'] == 'grouped' else MirroringStrategy.SPREAD + + generator = ClusterConfigGenerator( + n_segments=test_case['n'], + n_hosts=test_case['m'], + strategy=strategy + ) + + try: + configs = generator.generate_config(balanced=test_case['balanced']) + coordinator = generator.generate_coordinator() + + filename = f"{test_case['name']}.array" + write_config_file(filename, configs, coordinator) + + print(f" ✓ Written to: {filename}") + print_statistics(configs, test_case['m']) + + except Exception as e: + print(f" ✗ Failed: {e}") + + print("\n" + "=" * 80) + print("TEST SUITE GENERATION COMPLETE".center(80)) + print("=" * 80) + +if __name__ == '__main__': + import sys + + if len(sys.argv) > 1 and sys.argv[1] == '--generate-suite': + generate_test_suite() + elif len(sys.argv) == 1: + # Interactive mode or show help + print("Usage:") + print(" Generate single config:") + print(" python config_generator.py -n 6 -m 3 -s grouped -b -o config.txt") + print("\n Generate test suite:") + print(" python config_generator.py --generate-suite") + print("\n For help:") + print(" python config_generator.py -h") + else: + main() diff --git a/gpMgmt/bin/gprebalance_modules/test/data/role_mismatch.array b/gpMgmt/bin/gprebalance_modules/test/data/role_mismatch.array new file mode 100644 index 000000000000..d082012d4e89 --- /dev/null +++ b/gpMgmt/bin/gprebalance_modules/test/data/role_mismatch.array @@ -0,0 +1,12 @@ +# Coordinator +1|-1|p|p|s|u|cdw|cdw|5432|/data/coordinator +# SDW1 +2|0|p|p|s|u|sdw1|sdw1|40000|/data/primary0 +3|1|p|m|s|u|sdw1|sdw1|40001|/data/primary1 +8|2|m|m|s|u|sdw1|sdw1|50002|/data/mirror2 +9|3|m|m|s|u|sdw1|sdw1|50003|/data/mirror3 +# SDW2 +4|2|p|p|s|u|sdw2|sdw2|40002|/data/primary2 +5|3|p|p|s|u|sdw2|sdw2|40003|/data/primary3 +6|0|m|m|s|u|sdw2|sdw2|50000|/data/mirror0 +7|1|m|p|s|u|sdw2|sdw2|50001|/data/mirror1 diff --git a/gpMgmt/bin/gprebalance_modules/test/data/seg_down.array b/gpMgmt/bin/gprebalance_modules/test/data/seg_down.array new file mode 100644 index 000000000000..debb30bb58b4 --- /dev/null +++ b/gpMgmt/bin/gprebalance_modules/test/data/seg_down.array @@ -0,0 +1,12 @@ +# Coordinator +1|-1|p|p|s|u|cdw|cdw|5432|/data/coordinator +# SDW1 +2|0|p|p|s|d|sdw1|sdw1|40000|/data/primary0 +3|1|p|p|s|u|sdw1|sdw1|40001|/data/primary1 +8|2|m|m|s|u|sdw1|sdw1|50002|/data/mirror2 +9|3|m|m|s|u|sdw1|sdw1|50003|/data/mirror3 +# SDW2 +4|2|p|p|s|u|sdw2|sdw2|40002|/data/primary2 +5|3|p|p|s|u|sdw2|sdw2|40003|/data/primary3 +6|0|m|m|s|u|sdw2|sdw2|50000|/data/mirror0 +7|1|m|m|s|u|sdw2|sdw2|50001|/data/mirror1 diff --git a/gpMgmt/bin/gprebalance_modules/test/data/unbalanced_9_ip.array b/gpMgmt/bin/gprebalance_modules/test/data/unbalanced_9_ip.array new file mode 100644 index 000000000000..94d8bf5b2a31 --- /dev/null +++ b/gpMgmt/bin/gprebalance_modules/test/data/unbalanced_9_ip.array @@ -0,0 +1,23 @@ +# Coordinator +1|-1|p|p|s|u|cdw|cdw|5432|/data/coordinator +# SDW1 +2|0|p|p|s|u|sdw1|172.20.0.6|7000|/data/primary0 +3|1|p|p|s|u|sdw1|172.20.0.6|7001|/data/primary1 +4|2|p|p|s|u|sdw1|172.20.0.6|7002|/data/primary2 +14|3|m|m|s|u|sdw1|172.20.0.6|8003|/data/mirror3 +15|4|m|m|s|u|sdw1|172.20.0.6|8004|/data/mirror4 +16|5|m|m|s|u|sdw1|172.20.0.6|8005|/data/mirror5 +17|6|m|m|s|u|sdw1|172.20.0.6|8006|/data/mirror6 +18|7|m|m|s|u|sdw1|172.20.0.6|8007|/data/mirror7 +19|8|m|m|s|u|sdw1|172.20.0.6|8008|/data/mirror8 +# SDW2 +5|3|p|p|s|u|sdw2|172.20.0.7|7003|/data/primary3 +6|4|p|p|s|u|sdw2|172.20.0.7|7004|/data/primary4 +# SDW3 +7|5|p|p|s|u|sdw3|172.20.0.8|7005|/data/primary5 +8|6|p|p|s|u|sdw3|172.20.0.8|7006|/data/primary6 +9|7|p|p|s|u|sdw3|172.20.0.8|7007|/data/primary7 +10|8|p|p|s|u|sdw3|172.20.0.8|7008|/data/primary8 +11|0|m|m|s|u|sdw3|172.20.0.8|8000|/data/mirror0 +12|1|m|m|s|u|sdw3|172.20.0.8|8001|/data/mirror1 +13|2|m|m|s|u|sdw3|172.20.0.8|8002|/data/mirror2 diff --git a/gpMgmt/bin/gprebalance_modules/test/test_unit_commons.py b/gpMgmt/bin/gprebalance_modules/test/test_unit_commons.py new file mode 100644 index 000000000000..5b75f0232200 --- /dev/null +++ b/gpMgmt/bin/gprebalance_modules/test/test_unit_commons.py @@ -0,0 +1,831 @@ +import os +import socket +import tempfile + +from gppylib.test.unit.gp_unittest import * +from mock import * +from gprebalance_modules.rebalance_commons import HostResolver, TemplateParser, ValidationError, is_ip_address + +class TestHostResolver(GpTestCase): + """ + Test suite for HostResolver class + """ + + def setUp(self): + self.resolver = HostResolver() + + def tearDown(self): + self.resolver = None + + def test_is_ip_address_valid_ipv4(self): + self.assertTrue(is_ip_address('192.168.1.1')) + self.assertTrue(is_ip_address('10.0.0.1')) + self.assertTrue(is_ip_address('255.255.255.255')) + self.assertTrue(is_ip_address('0.0.0.0')) + + def test_is_ip_address_valid_ipv6(self): + self.assertTrue(is_ip_address('2001:0db8:85a3::8a2e:0370:7334')) + self.assertTrue(is_ip_address('::1')) + self.assertTrue(is_ip_address('fe80::1')) + self.assertTrue(is_ip_address('::')) + + def test_is_ip_address_invalid(self): + self.assertFalse(is_ip_address('hostname')) + self.assertFalse(is_ip_address('not-an-ip')) + self.assertFalse(is_ip_address('999.999.999.999')) + self.assertFalse(is_ip_address('192.168.1')) + self.assertFalse(is_ip_address('')) + self.assertFalse(is_ip_address('192.168.1.1.1')) + + @patch('socket.getaddrinfo') + def test_resolve_hostname_success_ipv4(self, mock_getaddrinfo): + # Mock socket.getaddrinfo to return IPv4 address + mock_getaddrinfo.return_value = [ + (socket.AF_INET, socket.SOCK_STREAM, 6, '', ('192.168.1.10', 0)) + ] + + result = self.resolver.resolve_hostname('testhost') + + self.assertEqual(result, '192.168.1.10') + mock_getaddrinfo.assert_called_once_with( + 'testhost', None, socket.AF_UNSPEC, socket.SOCK_STREAM + ) + # Check caching - now returns a set + self.assertEqual(self.resolver._hostname_to_ips['testhost'], {'192.168.1.10'}) + # Check reverse mapping + self.assertEqual(self.resolver._ip_to_hostname['192.168.1.10'], 'testhost') + + @patch('socket.getaddrinfo') + def test_resolve_hostname_success_ipv6(self, mock_getaddrinfo): + mock_getaddrinfo.return_value = [ + (socket.AF_INET6, socket.SOCK_STREAM, 6, '', ('2001:db8::1', 0, 0, 0)) + ] + + result = self.resolver.resolve_hostname('testhost6') + + self.assertEqual(result, '2001:db8::1') + self.assertEqual(self.resolver._hostname_to_ips['testhost6'], {'2001:db8::1'}) + self.assertEqual(self.resolver._ip_to_hostname['2001:db8::1'], 'testhost6') + + @patch('socket.getaddrinfo') + def test_resolve_hostname_success_ipv6_with_scope(self, mock_getaddrinfo): + # Test IPv6 with scope ID normalization + mock_getaddrinfo.return_value = [ + (socket.AF_INET6, socket.SOCK_STREAM, 6, '', ('fe80::1%eth0', 0, 0, 0)) + ] + + result = self.resolver.resolve_hostname('testhost6') + + # Should normalize by removing scope ID + self.assertEqual(result, 'fe80::1') + self.assertEqual(self.resolver._hostname_to_ips['testhost6'], {'fe80::1'}) + + @patch('socket.getaddrinfo') + def test_resolve_hostname_multiple_ips(self, mock_getaddrinfo): + mock_getaddrinfo.return_value = [ + (socket.AF_INET, socket.SOCK_STREAM, 6, '', ('192.168.1.10', 0)), + (socket.AF_INET, socket.SOCK_STREAM, 6, '', ('10.0.0.10', 0)), + (socket.AF_INET6, socket.SOCK_STREAM, 6, '', ('fe80::1', 0, 0, 0)) + ] + + result = self.resolver.resolve_hostname('multihost') + + # Should return first IP when sorted + self.assertIn(result, {'10.0.0.10', '192.168.1.10', 'fe80::1'}) + # Should cache all IPs + self.assertEqual(self.resolver._hostname_to_ips['multihost'], + {'192.168.1.10', '10.0.0.10', 'fe80::1'}) + # All IPs should have reverse mappings + self.assertEqual(self.resolver._ip_to_hostname['192.168.1.10'], 'multihost') + self.assertEqual(self.resolver._ip_to_hostname['10.0.0.10'], 'multihost') + self.assertEqual(self.resolver._ip_to_hostname['fe80::1'], 'multihost') + + @patch('socket.getaddrinfo') + def test_resolve_hostname_cached(self, mock_getaddrinfo): + self.resolver._hostname_to_ips['cached'] = {'192.168.1.100'} + + result = self.resolver.resolve_hostname('cached') + + self.assertEqual(result, '192.168.1.100') + mock_getaddrinfo.assert_not_called() + + @patch('gprebalance_modules.rebalance_commons.Hostname') + def test_resolve_ip_success(self, mock_hostname_class): + # Mock the Hostname command + mock_cmd = MagicMock() + mock_cmd.was_successful.return_value = True + mock_cmd.get_hostname.return_value = 'testhost' + mock_hostname_class.return_value = mock_cmd + + result = self.resolver.resolve_ip('192.168.1.10') + + self.assertEqual(result, 'testhost') + mock_hostname_class.assert_called_once() + mock_cmd.run.assert_called_once() + # Check caching - reverse mapping + self.assertEqual(self.resolver._ip_to_hostname['192.168.1.10'], 'testhost') + # Check forward mapping was also created + self.assertIn('192.168.1.10', self.resolver._hostname_to_ips['testhost']) + + @patch('gprebalance_modules.rebalance_commons.Hostname') + def test_resolve_ip_cached(self, mock_hostname_class): + self.resolver._ip_to_hostname['192.168.1.100'] = 'cached' + + result = self.resolver.resolve_ip('192.168.1.100') + + self.assertEqual(result, 'cached') + mock_hostname_class.assert_not_called() + + @patch('gprebalance_modules.rebalance_commons.Hostname') + def test_resolve_ip_invalid_ip(self, mock_hostname_class): + result = self.resolver.resolve_ip('not-an-ip') + + self.assertIsNone(result) + mock_hostname_class.assert_not_called() + + @patch('gprebalance_modules.rebalance_commons.Hostname') + def test_resolve_ip_command_failure(self, mock_hostname_class): + mock_cmd = MagicMock() + mock_cmd.was_successful.return_value = False + mock_hostname_class.return_value = mock_cmd + + result = self.resolver.resolve_ip('192.168.1.99') + + self.assertIsNone(result) + + @patch('gprebalance_modules.rebalance_commons.Hostname') + def test_resolve_ip_exception(self, mock_hostname_class): + mock_cmd = MagicMock() + mock_cmd.run.side_effect = Exception('Resolution failed') + mock_hostname_class.return_value = mock_cmd + + result = self.resolver.resolve_ip('192.168.1.99') + + self.assertIsNone(result) + + @patch('gprebalance_modules.rebalance_commons.Hostname') + def test_resolve_ip_ipv6(self, mock_hostname_class): + mock_cmd = MagicMock() + mock_cmd.was_successful.return_value = True + mock_cmd.get_hostname.return_value = 'testhost6' + mock_hostname_class.return_value = mock_cmd + + result = self.resolver.resolve_ip('2001:db8::1') + + self.assertEqual(result, 'testhost6') + self.assertEqual(self.resolver._ip_to_hostname['2001:db8::1'], 'testhost6') + + def test_hosts_match_identical_hostnames(self): + self.assertTrue(self.resolver.hosts_match('testhost', 'testhost')) + + def test_hosts_match_identical_ips(self): + self.assertTrue(self.resolver.hosts_match('192.168.1.10', '192.168.1.10')) + + def test_hosts_match_ipv6_normalization(self): + # IPv6 addresses with scope IDs should match + self.assertTrue(self.resolver.hosts_match('fe80::1%eth0', 'fe80::1%eth1')) + self.assertTrue(self.resolver.hosts_match('fe80::1%eth0', 'fe80::1')) + + def test_hosts_match_different_ips(self): + self.assertFalse(self.resolver.hosts_match('192.168.1.10', '192.168.1.11')) + + @patch('socket.getaddrinfo') + def test_hosts_match_ip_to_hostname(self, mock_getaddrinfo): + mock_getaddrinfo.return_value = [ + (socket.AF_INET, socket.SOCK_STREAM, 6, '', ('192.168.1.10', 0)) + ] + + # IP matches hostname's resolved IP + self.assertTrue(self.resolver.hosts_match('192.168.1.10', 'testhost')) + # Should work in reverse too + self.assertTrue(self.resolver.hosts_match('testhost', '192.168.1.10')) + + @patch('socket.getaddrinfo') + def test_hosts_match_ip_to_hostname_multiple_ips(self, mock_getaddrinfo): + mock_getaddrinfo.return_value = [ + (socket.AF_INET, socket.SOCK_STREAM, 6, '', ('192.168.1.10', 0)), + (socket.AF_INET, socket.SOCK_STREAM, 6, '', ('10.0.0.10', 0)) + ] + + # Should match any of the IPs + self.assertTrue(self.resolver.hosts_match('192.168.1.10', 'testhost')) + self.assertTrue(self.resolver.hosts_match('10.0.0.10', 'testhost')) + self.assertFalse(self.resolver.hosts_match('192.168.1.99', 'testhost')) + + @patch('socket.getaddrinfo') + def test_hosts_match_ip_to_hostname_no_match(self, mock_getaddrinfo): + mock_getaddrinfo.return_value = [ + (socket.AF_INET, socket.SOCK_STREAM, 6, '', ('192.168.1.10', 0)) + ] + + # IP doesn't match hostname's resolved IP + self.assertFalse(self.resolver.hosts_match('192.168.1.99', 'testhost')) + + @patch('socket.getaddrinfo') + def test_hosts_match_hostname_to_hostname_shared_ip(self, mock_getaddrinfo): + # Two hostnames sharing an IP address + def getaddrinfo_side_effect(hostname, *args): + if hostname == 'host1': + return [(socket.AF_INET, socket.SOCK_STREAM, 6, '', ('192.168.1.10', 0))] + elif hostname == 'host2': + return [(socket.AF_INET, socket.SOCK_STREAM, 6, '', ('192.168.1.10', 0))] + return [] + + mock_getaddrinfo.side_effect = getaddrinfo_side_effect + + # Should match because they share an IP + self.assertTrue(self.resolver.hosts_match('host1', 'host2')) + + @patch('socket.getaddrinfo') + def test_hosts_match_hostname_to_hostname_no_shared_ip(self, mock_getaddrinfo): + def getaddrinfo_side_effect(hostname, *args): + if hostname == 'host1': + return [(socket.AF_INET, socket.SOCK_STREAM, 6, '', ('192.168.1.10', 0))] + elif hostname == 'host2': + return [(socket.AF_INET, socket.SOCK_STREAM, 6, '', ('192.168.1.11', 0))] + return [] + + mock_getaddrinfo.side_effect = getaddrinfo_side_effect + + # Should not match - no shared IPs + self.assertFalse(self.resolver.hosts_match('host1', 'host2')) + + def test_hosts_match_different_hostnames_no_resolution(self): + # Without resolution data, different hostnames don't match + self.assertFalse(self.resolver.hosts_match('host1', 'host2')) + + @patch('socket.getaddrinfo') + def test_hosts_match_resolution_failure(self, mock_getaddrinfo): + mock_getaddrinfo.side_effect = socket.gaierror('Resolution failed') + + # Should return False when resolution fails + self.assertFalse(self.resolver.hosts_match('192.168.1.10', 'unknown')) + + def test_hosts_match_ipv6(self): + self.assertTrue(self.resolver.hosts_match('2001:db8::1', '2001:db8::1')) + self.assertFalse(self.resolver.hosts_match('2001:db8::1', '2001:db8::2')) + + def test_find_matching_hostname_exact_match(self): + existing_hosts = ['host1', 'host2', 'host3'] + + result = self.resolver.find_matching_hostname('host2', existing_hosts) + + self.assertEqual(result, 'host2') + + @patch('socket.getaddrinfo') + def test_find_matching_hostname_ip_match(self, mock_getaddrinfo): + def getaddrinfo_side_effect(hostname, *args): + if hostname == 'host1': + return [(socket.AF_INET, socket.SOCK_STREAM, 6, '', ('192.168.1.10', 0))] + elif hostname == 'host2': + return [(socket.AF_INET, socket.SOCK_STREAM, 6, '', ('192.168.1.11', 0))] + return [] + + mock_getaddrinfo.side_effect = getaddrinfo_side_effect + + existing_hosts = ['host1', 'host2'] + + result = self.resolver.find_matching_hostname('192.168.1.10', existing_hosts) + + # Should match host1 + self.assertEqual(result, 'host1') + + def test_find_matching_hostname_no_match(self): + existing_hosts = ['host1', 'host2'] + + result = self.resolver.find_matching_hostname('nonexistent', existing_hosts) + + self.assertIsNone(result) + + def test_find_matching_hostname_empty_list(self): + result = self.resolver.find_matching_hostname('testhost', []) + + self.assertIsNone(result) + + def test_get_all_addresses(self): + self.resolver._hostname_to_ips['multihost'] = {'192.168.1.10', '10.0.0.10'} + + result = self.resolver.get_all_addresses('multihost') + + self.assertEqual(result, {'192.168.1.10', '10.0.0.10'}) + + def test_get_all_addresses_empty(self): + result = self.resolver.get_all_addresses('unknown') + + self.assertEqual(result, set()) + + def test_get_address_returns_first_sorted(self): + self.resolver._hostname_to_ips['multihost'] = {'192.168.1.10', '10.0.0.10', 'fe80::1'} + + result = self.resolver.get_address('multihost') + + # Should return first when sorted + sorted_ips = sorted(['192.168.1.10', '10.0.0.10', 'fe80::1']) + self.assertEqual(result, sorted_ips[0]) + + def test_get_hostname(self): + self.resolver._ip_to_hostname['192.168.1.10'] = 'testhost' + + result = self.resolver.get_hostname('192.168.1.10') + + self.assertEqual(result, 'testhost') + + def test_get_hostname_not_found(self): + result = self.resolver.get_hostname('192.168.1.99') + + # Should return IP itself if not found + self.assertEqual(result, '192.168.1.99') + + @patch('socket.getaddrinfo') + def test_caching_integration(self, mock_getaddrinfo): + mock_getaddrinfo.return_value = [ + (socket.AF_INET, socket.SOCK_STREAM, 6, '', ('192.168.1.10', 0)) + ] + + # First call should hit the network + result1 = self.resolver.resolve_hostname('testhost') + self.assertEqual(result1, '192.168.1.10') + self.assertEqual(mock_getaddrinfo.call_count, 1) + + # Second call should use cache + result2 = self.resolver.resolve_hostname('testhost') + self.assertEqual(result2, '192.168.1.10') + self.assertEqual(mock_getaddrinfo.call_count, 1) # Still 1, not called again + + # Third call to get from cache directly + result3 = self.resolver.get_address('testhost') + self.assertEqual(result3, '192.168.1.10') + + # get_all_addresses should also work + all_ips = self.resolver.get_all_addresses('testhost') + self.assertEqual(all_ips, {'192.168.1.10'}) + + @patch('socket.getaddrinfo') + def test_multiple_hosts(self, mock_getaddrinfo): + def getaddrinfo_side_effect(hostname, *args): + mapping = { + 'host1': [(socket.AF_INET, socket.SOCK_STREAM, 6, '', ('192.168.1.10', 0))], + 'host2': [(socket.AF_INET, socket.SOCK_STREAM, 6, '', ('192.168.1.11', 0))], + 'host3': [(socket.AF_INET, socket.SOCK_STREAM, 6, '', ('192.168.1.12', 0))] + } + return mapping.get(hostname, []) + + mock_getaddrinfo.side_effect = getaddrinfo_side_effect + + existing_hosts = ['host1', 'host2', 'host3'] + + # Test various scenarios + self.assertEqual(self.resolver.find_matching_hostname('host1', existing_hosts), 'host1') + self.assertEqual(self.resolver.find_matching_hostname('192.168.1.11', existing_hosts), 'host2') + self.assertIsNone(self.resolver.find_matching_hostname('192.168.1.99', existing_hosts)) + + # Test hosts_match + self.assertTrue(self.resolver.hosts_match('host1', '192.168.1.10')) + self.assertFalse(self.resolver.hosts_match('host1', '192.168.1.11')) + + # Test get_all_addresses + self.assertEqual(self.resolver.get_all_addresses('host2'), {'192.168.1.11'}) + + +class TestTemplateParser(GpTestCase): + """Test suite for TemplateParser""" + + def test_parse_basic_input(self): + """Test basic comma-separated input""" + primary, mirror = TemplateParser.parse_datadirs_input( + "/data/primary, /data/mirror" + ) + self.assertEqual(primary, "/data/primary/gpseg{content}") + self.assertEqual(mirror, "/data/mirror/gpseg{content}") + + def test_parse_quoted_input_double_quotes(self): + """Test input with double quotes""" + primary, mirror = TemplateParser.parse_datadirs_input( + '"/data/primary", "/data/mirror"' + ) + self.assertEqual(primary, "/data/primary/gpseg{content}") + self.assertEqual(mirror, "/data/mirror/gpseg{content}") + + def test_parse_quoted_input_single_quotes(self): + """Test input with single quotes""" + primary, mirror = TemplateParser.parse_datadirs_input( + "'/data/primary', '/data/mirror'" + ) + self.assertEqual(primary, "/data/primary/gpseg{content}") + self.assertEqual(mirror, "/data/mirror/gpseg{content}") + + def test_parse_with_content_placeholder(self): + """Test input that already has {content}""" + primary, mirror = TemplateParser.parse_datadirs_input( + "/data/primary/gpseg{content}, /data/mirror/gpseg{content}" + ) + self.assertEqual(primary, "/data/primary/gpseg{content}") + self.assertEqual(mirror, "/data/mirror/gpseg{content}") + + def test_parse_with_hostname_placeholder(self): + """Test input with {hostname} placeholder""" + primary, mirror = TemplateParser.parse_datadirs_input( + "/data/{hostname}/primary, /data/{hostname}/mirror" + ) + self.assertEqual(primary, "/data/{hostname}/primary/gpseg{content}") + self.assertEqual(mirror, "/data/{hostname}/mirror/gpseg{content}") + + def test_parse_with_both_placeholders(self): + """Test input with both {hostname} and {content} placeholders""" + primary, mirror = TemplateParser.parse_datadirs_input( + "/data/{hostname}/primary/gpseg{content}, /data/{hostname}/mirror/gpseg{content}" + ) + self.assertEqual(primary, "/data/{hostname}/primary/gpseg{content}") + self.assertEqual(mirror, "/data/{hostname}/mirror/gpseg{content}") + + def test_parse_with_trailing_slashes(self): + """Test input with trailing slashes""" + primary, mirror = TemplateParser.parse_datadirs_input( + "/data/primary/, /data/mirror/" + ) + self.assertEqual(primary, "/data/primary/gpseg{content}") + self.assertEqual(mirror, "/data/mirror/gpseg{content}") + + def test_parse_with_extra_whitespace(self): + """Test input with extra whitespace""" + primary, mirror = TemplateParser.parse_datadirs_input( + " /data/primary , /data/mirror " + ) + self.assertEqual(primary, "/data/primary/gpseg{content}") + self.assertEqual(mirror, "/data/mirror/gpseg{content}") + + def test_identical_templates_error(self): + """Test that identical templates raise error""" + with self.assertRaisesRegex(ValidationError, "cannot be identical"): + TemplateParser.parse_datadirs_input( + "/data/primary, /data/primary" + ) + + def test_identical_after_normalization_error(self): + """Test that templates identical after normalization raise error""" + with self.assertRaisesRegex(ValidationError, "cannot be identical"): + TemplateParser.parse_datadirs_input( + "/data/segments, /data/segments/" + ) + + def test_non_absolute_path_error(self): + """Test that relative paths raise error""" + with self.assertRaisesRegex(ValidationError, "must be absolute"): + TemplateParser.parse_datadirs_input( + "data/primary, data/mirror" + ) + + def test_non_absolute_second_path_error(self): + """Test that relative path in second position raises error""" + with self.assertRaisesRegex(ValidationError, "must be absolute"): + TemplateParser.parse_datadirs_input( + "/data/primary, data/mirror" + ) + + def test_empty_path_error(self): + """Test that empty paths raise error""" + with self.assertRaisesRegex(ValidationError, "cannot be empty"): + TemplateParser.parse_datadirs_input( + '"", /data/mirror' + ) + + def test_empty_after_quotes_error(self): + """Test that empty path after quote removal raises error""" + with self.assertRaisesRegex(ValidationError, "cannot be empty"): + TemplateParser.parse_datadirs_input( + ' "" , /data/mirror' + ) + + def test_invalid_placeholder_error(self): + """Test that invalid placeholders raise error""" + with self.assertRaisesRegex(ValidationError, "Invalid placeholder"): + TemplateParser.parse_datadirs_input( + "/data/{invalid}/primary, /data/mirror" + ) + + def test_multiple_invalid_placeholders_error(self): + """Test that multiple invalid placeholders are caught""" + with self.assertRaisesRegex(ValidationError, "Invalid placeholder"): + TemplateParser.parse_datadirs_input( + "/data/{foo}/{bar}/primary, /data/mirror" + ) + + def test_wrong_format_missing_second_dir_error(self): + """Test that missing second directory raises error""" + with self.assertRaisesRegex(ValidationError, "should have format"): + TemplateParser.parse_datadirs_input( + "/data/primary" + ) + + def test_wrong_format_too_many_dirs_error(self): + """Test that too many directories raise error""" + with self.assertRaisesRegex(ValidationError, "should have format"): + TemplateParser.parse_datadirs_input( + "/data/primary, /data/mirror, /data/extra" + ) + + def test_instantiate_template_both_placeholders(self): + """Test template instantiation with both placeholders""" + template = "/data/{hostname}/gpseg{content}" + result = TemplateParser.instantiate_template( + template, hostname="sdw1", content=0 + ) + self.assertEqual(result, "/data/sdw1/gpseg0") + + def test_instantiate_template_content_only(self): + """Test template instantiation with content only""" + template = "/data/primary/gpseg{content}" + result = TemplateParser.instantiate_template( + template, content=5 + ) + self.assertEqual(result, "/data/primary/gpseg5") + + def test_instantiate_template_hostname_only(self): + """Test template instantiation with hostname only""" + template = "/data/{hostname}/segments/gpseg{content}" + result = TemplateParser.instantiate_template( + template, hostname="sdw2" + ) + self.assertEqual(result, "/data/sdw2/segments/gpseg{content}") + + def test_instantiate_template_no_substitution(self): + """Test template instantiation with no substitution""" + template = "/data/{hostname}/gpseg{content}" + result = TemplateParser.instantiate_template(template) + self.assertEqual(result, "/data/{hostname}/gpseg{content}") + + def test_instantiate_template_large_content_id(self): + """Test template instantiation with large content ID""" + template = "/data/primary/gpseg{content}" + result = TemplateParser.instantiate_template( + template, content=12345 + ) + self.assertEqual(result, "/data/primary/gpseg12345") + + def test_extract_parent_directory_basic(self): + """Test parent directory extraction""" + result = TemplateParser.extract_parent_directory( + "/data/primary/gpseg0" + ) + self.assertEqual(result, "/data/primary") + + def test_extract_parent_directory_with_trailing_slash(self): + """Test parent directory extraction with trailing slash""" + result = TemplateParser.extract_parent_directory( + "/data/primary/gpseg0/" + ) + self.assertEqual(result, "/data/primary") + + def test_extract_parent_directory_deep_path(self): + """Test parent directory extraction with deep path""" + result = TemplateParser.extract_parent_directory( + "/mnt/disk1/data/segments/primary/gpseg0" + ) + self.assertEqual(result, "/mnt/disk1/data/segments/primary") + + def test_extract_parent_directory_custom_naming(self): + """Test parent directory extraction with custom segment naming""" + result = TemplateParser.extract_parent_directory( + "/data/primary/seg_data_0" + ) + self.assertEqual(result, "/data/primary") + + def test_parse_comma_in_double_quotes(self): + """Test paths with commas inside double quotes""" + primary, mirror = TemplateParser.parse_datadirs_input( + '"/di,r1/primary/", "/dir2/primary"' + ) + self.assertEqual(primary, "/di,r1/primary/gpseg{content}") + self.assertEqual(mirror, "/dir2/primary/gpseg{content}") + + def test_parse_comma_in_single_quotes(self): + """Test paths with commas inside single quotes""" + primary, mirror = TemplateParser.parse_datadirs_input( + "'/di,r1/primary/', '/dir2/primary'" + ) + self.assertEqual(primary, "/di,r1/primary/gpseg{content}") + self.assertEqual(mirror, "/dir2/primary/gpseg{content}") + + def test_parse_multiple_commas_in_quotes(self): + """Test paths with multiple commas inside quotes""" + primary, mirror = TemplateParser.parse_datadirs_input( + '"/path,with,many,commas/primary", "/simple/mirror"' + ) + self.assertEqual(primary, "/path,with,many,commas/primary/gpseg{content}") + self.assertEqual(mirror, "/simple/mirror/gpseg{content}") + +class TestTemplateParserFile(GpTestCase): + """Test suite for TemplateParser file parsing""" + + def setUp(self): + """Create temporary directory for test files""" + self.temp_dir = tempfile.mkdtemp() + + def tearDown(self): + """Clean up temporary directory""" + import shutil + shutil.rmtree(self.temp_dir) + + def _create_test_file(self, filename, content): + """Helper to create test file""" + filepath = os.path.join(self.temp_dir, filename) + with open(filepath, 'w') as f: + f.write(content) + return filepath + + def test_parse_file_basic(self): + """Test parsing from file with basic paths""" + filepath = self._create_test_file( + "datadirs.txt", + "/data/primary\n/data/mirror\n" + ) + + primary, mirror = TemplateParser.parse_datadirs_file(filepath) + self.assertEqual(primary, "/data/primary/gpseg{content}") + self.assertEqual(mirror, "/data/mirror/gpseg{content}") + + def test_parse_file_with_quotes(self): + """Test parsing file with quoted paths""" + filepath = self._create_test_file( + "datadirs.txt", + '"/data/primary"\n"/data/mirror"\n' + ) + + primary, mirror = TemplateParser.parse_datadirs_file(filepath) + self.assertEqual(primary, "/data/primary/gpseg{content}") + self.assertEqual(mirror, "/data/mirror/gpseg{content}") + + def test_parse_file_with_blank_lines(self): + """Test parsing file with blank lines""" + filepath = self._create_test_file( + "datadirs.txt", + "\n/data/primary\n\n/data/mirror\n\n" + ) + + primary, mirror = TemplateParser.parse_datadirs_file(filepath) + self.assertEqual(primary, "/data/primary/gpseg{content}") + self.assertEqual(mirror, "/data/mirror/gpseg{content}") + + def test_parse_file_with_placeholders(self): + """Test parsing file with template placeholders""" + filepath = self._create_test_file( + "datadirs.txt", + "/data/{hostname}/primary/gpseg{content}\n" + "/data/{hostname}/mirror/gpseg{content}\n" + ) + + primary, mirror = TemplateParser.parse_datadirs_file(filepath) + self.assertEqual(primary, "/data/{hostname}/primary/gpseg{content}") + self.assertEqual(mirror, "/data/{hostname}/mirror/gpseg{content}") + + def test_parse_file_with_whitespace(self): + """Test parsing file with extra whitespace""" + filepath = self._create_test_file( + "datadirs.txt", + " /data/primary \n /data/mirror \n" + ) + + primary, mirror = TemplateParser.parse_datadirs_file(filepath) + self.assertEqual(primary, "/data/primary/gpseg{content}") + self.assertEqual(mirror, "/data/mirror/gpseg{content}") + + def test_parse_file_wrong_line_count_one_line(self): + """Test file with only one line""" + filepath = self._create_test_file( + "datadirs.txt", + "/data/primary\n" + ) + + with self.assertRaisesRegex(ValidationError, "exactly 2 lines"): + TemplateParser.parse_datadirs_file(filepath) + + def test_parse_file_wrong_line_count_three_lines(self): + """Test file with three lines""" + filepath = self._create_test_file( + "datadirs.txt", + "/data/primary\n/data/mirror\n/data/extra\n" + ) + + with self.assertRaisesRegex(ValidationError, "exactly 2 lines"): + TemplateParser.parse_datadirs_file(filepath) + + def test_parse_file_empty_file(self): + """Test empty file""" + filepath = self._create_test_file("datadirs.txt", "") + + with self.assertRaisesRegex(ValidationError, "exactly 2 lines"): + TemplateParser.parse_datadirs_file(filepath) + + def test_parse_file_only_blank_lines(self): + """Test file with only blank lines""" + filepath = self._create_test_file( + "datadirs.txt", + "\n\n\n" + ) + + with self.assertRaisesRegex(ValidationError, "exactly 2 lines"): + TemplateParser.parse_datadirs_file(filepath) + + def test_parse_file_not_exists(self): + """Test non-existent file""" + with self.assertRaises(FileNotFoundError): + TemplateParser.parse_datadirs_file("/nonexistent/file.txt") + + def test_parse_file_identical_paths(self): + """Test file with identical paths""" + filepath = self._create_test_file( + "datadirs.txt", + "/data/segments\n/data/segments\n" + ) + + with self.assertRaisesRegex(ValidationError, "cannot be identical"): + TemplateParser.parse_datadirs_file(filepath) + + def test_parse_file_invalid_placeholder(self): + """Test file with invalid placeholder""" + filepath = self._create_test_file( + "datadirs.txt", + "/data/{invalid}/primary\n/data/mirror\n" + ) + + with self.assertRaisesRegex(ValidationError, "Invalid placeholder"): + TemplateParser.parse_datadirs_file(filepath) + + def test_parse_file_non_absolute_path(self): + """Test file with relative path""" + filepath = self._create_test_file( + "datadirs.txt", + "data/primary\n/data/mirror\n" + ) + + with self.assertRaisesRegex(ValidationError, "must be absolute"): + TemplateParser.parse_datadirs_file(filepath) + +class TestTemplateParserEdgeCases(unittest.TestCase): + """Test edge cases and corner scenarios""" + + def test_parse_windows_style_path_error(self): + """Test that Windows-style paths are rejected""" + with self.assertRaisesRegex(ValidationError, "must be absolute"): + TemplateParser.parse_datadirs_input( + "C:\\data\\primary, C:\\data\\mirror" + ) + + def test_parse_mixed_quote_types(self): + """Test mixed quote types (one double, one single)""" + primary, mirror = TemplateParser.parse_datadirs_input( + '"/data/primary", \'/data/mirror\'' + ) + self.assertEqual(primary, "/data/primary/gpseg{content}") + self.assertEqual(mirror, "/data/mirror/gpseg{content}") + + def test_parse_only_first_quoted(self): + """Test only first path quoted""" + primary, mirror = TemplateParser.parse_datadirs_input( + '"/data/primary", /data/mirror' + ) + self.assertEqual(primary, "/data/primary/gpseg{content}") + self.assertEqual(mirror, "/data/mirror/gpseg{content}") + + def test_parse_special_characters_in_path(self): + """Test paths with special characters""" + primary, mirror = TemplateParser.parse_datadirs_input( + "/data/primary-data_01, /data/mirror-data_01" + ) + self.assertEqual(primary, "/data/primary-data_01/gpseg{content}") + self.assertEqual(mirror, "/data/mirror-data_01/gpseg{content}") + + def test_parse_very_long_path(self): + """Test very long paths""" + long_path = "/data/" + "a" * 200 + primary, mirror = TemplateParser.parse_datadirs_input( + f"{long_path}/primary, {long_path}/mirror" + ) + self.assertEqual(primary, f"{long_path}/primary/gpseg{{content}}") + self.assertEqual(mirror, f"{long_path}/mirror/gpseg{{content}}") + + def test_instantiate_template_zero_content(self): + """Test instantiation with content=0""" + template = "/data/primary/gpseg{content}" + result = TemplateParser.instantiate_template(template, content=0) + self.assertEqual(result, "/data/primary/gpseg0") + + def test_instantiate_multiple_hostname_placeholders(self): + """Test instantiation with multiple hostname placeholders""" + template = "/data/{hostname}/primary/{hostname}/gpseg{content}" + result = TemplateParser.instantiate_template( + template, hostname="sdw1", content=0 + ) + self.assertEqual(result, "/data/sdw1/primary/sdw1/gpseg0") + + def test_collision_same_parent_different_suffix(self): + """Test that templates with same parent but different suffixes are valid""" + # This should NOT raise an error - they resolve to different paths + primary, mirror = TemplateParser.parse_datadirs_input( + "/data/gpseg{content}, /data/gpmirror{content}" + ) + self.assertEqual(primary, "/data/gpseg{content}") + self.assertEqual(mirror, "/data/gpmirror{content}") + +if __name__ == '__main__': + run_tests() diff --git a/gpMgmt/bin/gprebalance_modules/test/test_unit_ggrebalance.py b/gpMgmt/bin/gprebalance_modules/test/test_unit_ggrebalance.py new file mode 100644 index 000000000000..560e17f223ac --- /dev/null +++ b/gpMgmt/bin/gprebalance_modules/test/test_unit_ggrebalance.py @@ -0,0 +1,266 @@ +import os +import imp +import sys +import socket + +from gppylib.test.unit.gp_unittest import * +from mock import * +from gppylib.db.dbconn import DbURL +from gppylib.gplog import * +from gppylib.system.configurationInterface import GpConfigurationProvider +from gppylib.system.environment import GpCoordinatorEnvironment + +from gprebalance_modules.test.config import initGparrayFromFile +from gprebalance_modules.planner import Planner +from gprebalance_modules.rebalance_commons import Host, HostStatus, ValidationError + +def rebalance_only(numsegs): + def inner(func): + def wrapper(self, *args, **kwargs): + self.options.target_segment_count = numsegs + ret = func(self, *args, **kwargs) + self.options.target_segment_count = None + return ret + return wrapper + return inner + +def check_query(conn, query): + if "SELECT COUNT(1) FROM pg_namespace WHERE nspname =" in query: + return [0] + return None + +class TestRebalanceUtilityCLI(GpTestCase): + def setUp(self): + gprebalance_file = os.path.abspath( + os.path.dirname(__file__) + "/../../ggrebalance") + self.subject = imp.load_source('ggrebalance', gprebalance_file) + self.old_sys_argv = sys.argv + sys.argv = [] + self.options, self.args, self.parser = self.subject.parseargs() + + self.subject.logger = Mock( + spec=['log', 'warn', 'info', 'debug', 'error', 'warning', 'fatal', 'exception']) + + self.subject.check_running_gputils = Mock(return_value=False) + + self.apply_patches([ + patch('builtins.open', mock_open(), create=True), + patch('builtins.input'), + patch('ggrebalance.dbconn.DbURL', return_value=Mock(), spec=DbURL), + patch('ggrebalance.dbconn.connect', return_value=Mock()), + patch('ggrebalance.GpCoordinatorEnvironment', + return_value=Mock(), spec=GpCoordinatorEnvironment), + patch('ggrebalance.configurationInterface.getConfigurationProvider'), + patch('os.path.exists', return_value=Mock()), + patch('ggrebalance.get_default_logger', + return_value=self.subject.logger), + ]) + + self.getConfigProviderFunctionMock = self.get_mock_from_apply_patch( + "getConfigurationProvider") + self.gpCoordinatorEnvironmentMock = self.get_mock_from_apply_patch( + "GpCoordinatorEnvironment") + self.previous_coordinator_data_directory = os.getenv( + 'COORDINATOR_DATA_DIRECTORY', '') + os.environ["COORDINATOR_DATA_DIRECTORY"] = '/tmp/dirdoesnotexist' + os.environ["GPHOME"] = '/tmp/dirdoesnotexist' + configProviderMock = Mock(spec=GpConfigurationProvider) + self.getConfigProviderFunctionMock.return_value = configProviderMock + configProviderMock.initializeProvider.return_value = configProviderMock + self.gpCoordinatorEnvironmentMock.return_value.getCoordinatorPort.return_value = 123456 + + def tearDown(self): + os.environ['COORDINATOR_DATA_DIRECTORY'] = self.previous_coordinator_data_directory + sys.argv = self.old_sys_argv + super(TestRebalanceUtilityCLI, self).tearDown() + + @patch('ggrebalance.GpArray.initFromCatalog', + return_value=initGparrayFromFile("balanced_grouped_6")) + @patch('os.path.exists', side_effect=lambda path: path not in ['/tmp/dirdoesnotexist/gparraydump']) + @patch('gppylib.db.dbconn.queryRow', side_effect=check_query) + @patch('ggrebalance.check_down_segments') + @rebalance_only(numsegs = 6) + def test_already_balanced_grouped(self, mockCatalog, mockOsPath, mockCursor, mock_down): + with self.assertRaises(SystemExit): + self.subject.main(self.options, self.args, self.parser) + self.subject.logger.info.assert_any_call( + "Cluster is already balanced, no segment moves will be held.") + + @patch('ggrebalance.GpArray.initFromCatalog', + return_value=initGparrayFromFile("seg_down")) + @patch('os.path.exists', side_effect=lambda path: path not in ['/tmp/dirdoesnotexist/gparraydump']) + @patch('gppylib.db.dbconn.queryRow', side_effect=check_query) + @patch('ggrebalance.check_down_segments') + @rebalance_only(numsegs = 4) + def test_segment_down(self, mockCatalog, mockOsPath, mockCursor, mock_down): + with self.assertRaises(SystemExit): + self.subject.main(self.options, self.args, self.parser) + self.subject.logger.error.assert_any_call( + "ggrebalance failed: Some segments in 'down' status. ggrebalance can't proceed further \n\nExiting...") + + @patch('ggrebalance.GpArray.initFromCatalog', + return_value=initGparrayFromFile("balanced_spread_24")) + @patch('os.path.exists', side_effect=lambda path: path not in ['/tmp/dirdoesnotexist/gparraydump']) + @patch('gppylib.db.dbconn.queryRow', side_effect=check_query) + @patch('ggrebalance.check_down_segments') + @rebalance_only(numsegs = 24) + def test_invalid_target_datadir(self, mockCatalog, mockOsPath, mockCursor, mock_down): + self.options.target_hosts = "sdw1, sdw2, sdw3" + self.options.target_datadirs = '/data/primary/gpseg{content}' + with self.assertRaises(SystemExit): + self.subject.main(self.options, self.args, self.parser) + self.subject.logger.error.assert_any_call( + 'ggrebalance failed: --target-datadirs should have format: ' + '"/data/primary/gpseg{content}, /data/mirror/gpseg{content}". ' + 'Available templated parameters: {hostname}, {content} \n\nExiting...') + + @patch('ggrebalance.GpArray.initFromCatalog', + return_value=initGparrayFromFile("role_mismatch")) + @patch('os.path.exists', side_effect=lambda path: path not in ['/tmp/dirdoesnotexist/gparraydump']) + @patch('gppylib.db.dbconn.queryRow', side_effect=check_query) + @patch('ggrebalance.check_down_segments') + @rebalance_only(numsegs = 4) + def test_role_mistmatch(self, mockCatalog, mockOsPath, mockCursor, mock_down): + with self.assertRaises(SystemExit): + self.subject.main(self.options, self.args, self.parser) + self.subject.logger.error.assert_any_call( + "ggrebalance failed: Current role does not match preferred role for several segments. \n\nExiting...") + + +class TestHostsOptions(GpTestCase): + + def setUp(self): + self.options = Mock() + self.options.target_hosts = None + self.options.add_hosts = None + self.options.remove_hosts = None + self.options.target_datadirs = None + self.options.target_hosts_file = None + self.options.add_hosts_file = None + self.options.remove_hosts_file = None + self.options.target_datadirs_file = None + + def test_target_hosts_invalid_chars(self): + gparray = initGparrayFromFile('balanced_grouped_6') + self.options.target_hosts = "sdw1, sdw2, asfsa@" + with self.assertRaises(ValidationError) as ctx: + planner = Planner(logger=Mock(), + dburl=Mock(), + gpArray=gparray, + options=self.options) + self.assertIn("contains invalid characters", str(ctx.exception)) + + def test_target_hosts_duplicates(self): + gparray = initGparrayFromFile('balanced_grouped_6') + self.options.target_hosts = "sdw1, sdw2, sdw2" + with self.assertRaises(ValidationError) as ctx: + planner = Planner(logger=Mock(), + dburl=Mock(), + gpArray=gparray, + options=self.options) + self.assertIn("Duplicate host", str(ctx.exception)) + + def test_target_hosts_ip_and_name(self): + gparray = initGparrayFromFile('balanced_grouped_6') + self.options.target_hosts = "sdw1, sdw2, 192.168.0.15" + with self.assertRaises(ValidationError) as ctx: + planner = Planner(logger=Mock(), + dburl=Mock(), + gpArray=gparray, + options=self.options) + self.assertIn("must not contain IP adress and hostname simultaniously", str(ctx.exception)) + + @patch('gprebalance_modules.rebalance_commons.HostResolver.find_matching_hostname', return_value='sdw1') + def test_add_hosts_existing(self, MockResolver): + gparray = initGparrayFromFile('balanced_grouped_6') + self.options.add_hosts = "sdw1" + with self.assertRaises(ValidationError) as ctx: + planner = Planner(logger=Mock(), + dburl=Mock(), + gpArray=gparray, + options=self.options) + self.assertIn("--add-hosts: Host 'sdw1' already exists in cluster as 'sdw1'", str(ctx.exception)) + + @patch('gprebalance_modules.planner.HostResolver.resolve_ip', return_value='sdw1') + @patch('gprebalance_modules.rebalance_commons.HostResolver.find_matching_hostname', return_value='sdw1') + def test_add_hosts_existing_ip(self, MockResolver, mock_ip): + gparray = initGparrayFromFile('balanced_grouped_6') + self.options.add_hosts = "172.20.0.6" + with self.assertRaises(ValidationError) as ctx: + planner = Planner(logger=Mock(), + dburl=Mock(), + gpArray=gparray, + options=self.options) + self.assertIn("--add-hosts: Host '172.20.0.6' already exists in cluster as 'sdw1'", str(ctx.exception)) + + @patch('gprebalance_modules.rebalance_commons.HostResolver.find_matching_hostname', return_value=None) + def test_remove_hosts_unexisting(self, MockResolver): + gparray = initGparrayFromFile('balanced_grouped_6') + self.options.remove_hosts = "sdw3" + with self.assertRaises(ValidationError) as ctx: + planner = Planner(logger=Mock(), + dburl=Mock(), + gpArray=gparray, + options=self.options) + self.assertIn("--remove-hosts: Host 'sdw3' does not exist in cluster", str(ctx.exception)) + + @patch('gprebalance_modules.planner.HostResolver.resolve_ip', return_value='sdw3') + @patch('gprebalance_modules.rebalance_commons.HostResolver.find_matching_hostname', return_value=None) + def test_remove_hosts_unexisting_ip(self, MockResolver, mock_ip): + gparray = initGparrayFromFile('balanced_grouped_6') + self.options.remove_hosts = "172.20.0.9" + with self.assertRaises(ValidationError) as ctx: + planner = Planner(logger=Mock(), + dburl=Mock(), + gpArray=gparray, + options=self.options) + self.assertIn("--remove-hosts: Host '172.20.0.9' does not exist in cluster", str(ctx.exception)) + + @patch('socket.getaddrinfo') + def test_target_hosts_positive(self, mock_getaddrinfo): + gparray = initGparrayFromFile('balanced_grouped_6') + self.options.target_hosts = "sdw1, sdw2" + + mock_getaddrinfo.side_effect = [ + [(socket.AF_INET, socket.SOCK_STREAM, 6, '', ('192.168.1.10', 0))], + [(socket.AF_INET, socket.SOCK_STREAM, 6, '', ('192.168.1.11', 0))] + ] + planner = Planner(logger=Mock(), + dburl=Mock(), + gpArray=gparray, + options=self.options) + self.assertTrue(planner.target_hosts == [Host('sdw1', '192.168.1.10'), Host('sdw2', '192.168.1.11')]) + + @patch('socket.getaddrinfo') + def test_target_hosts_positive_ip(self, mock_getaddrinfo): + gparray = initGparrayFromFile('unbalanced_9_ip') + self.options.target_hosts = "sdw1, sdw2" + mock_getaddrinfo.side_effect = [ + [(socket.AF_INET, socket.SOCK_STREAM, 6, '', ('172.20.0.6', 0))], + [(socket.AF_INET, socket.SOCK_STREAM, 6, '', ('172.20.0.7', 0))], + [(socket.AF_INET, socket.SOCK_STREAM, 6, '', ('172.20.0.8', 0))] + ] + planner = Planner(logger=Mock(), + dburl=Mock(), + gpArray=gparray, + options=self.options) + self.assertTrue(planner.target_hosts == [Host('sdw1', '172.20.0.6'), Host('sdw3', '172.20.0.8'), Host('sdw2', '172.20.0.7')]) + self.assertTrue(planner.target_hosts[1].status == HostStatus.DECOMMISSIONED) + + @patch('gprebalance_modules.planner.HostResolver') + def test_target_hosts_ip(self, mockHostResolver): + gparray = initGparrayFromFile('unbalanced_9_ip') + self.options.target_hosts = "172.20.0.7, 172.20.0.8, 172.20.0.9" + mockHostResolver.return_value.resolve_hostname.side_effect = ['172.20.0.6', '172.20.0.7', '172.20.0.8', '172.20.0.9'] + mockHostResolver.return_value.resolve_ip.side_effect = ['sdw2', 'sdw3', 'sdw4'] + mockHostResolver.return_value.get_address.side_effect = ['172.20.0.6', '172.20.0.7', '172.20.0.8', '172.20.0.9'] + planner = Planner(logger=Mock(), + dburl=Mock(), + gpArray=gparray, + options=self.options) + self.assertEqual(len(planner.target_hosts), 4) + self.assertTrue(planner.target_hosts[3].status == HostStatus.NEW) + self.assertTrue(planner.target_hosts[0].status == HostStatus.DECOMMISSIONED) + +if __name__ == '__main__': + run_tests() diff --git a/gpMgmt/bin/gprebalance_modules/test/test_unit_resources.py b/gpMgmt/bin/gprebalance_modules/test/test_unit_resources.py new file mode 100644 index 000000000000..fd12f321aca8 --- /dev/null +++ b/gpMgmt/bin/gprebalance_modules/test/test_unit_resources.py @@ -0,0 +1,1621 @@ +from gppylib.test.unit.gp_unittest import * +from mock import * + +from gprebalance_modules.planner import ResourceEstimator, ResourceError, LogicalMove, Planner, PortAllocator, PlanningError +from gppylib.db.dbconn import DbURL +from gppylib import gparray +from gprebalance_modules.test.config import initGparrayFromFile +from gprebalance_modules.rebalance_commons import ( + SegmentSize, + DiskSpaceChecker, + DiskSpaceInfo, + Host, + HostStatus +) + +def check_query(conn, query): + if "SELECT COUNT(1) FROM pg_namespace WHERE nspname =" in query: + return [0] + return None + +class TestDiskSpaceChecker(GpTestCase): + """Test cases for DiskSpaceChecker""" + + def setUp(self): + self.logger = Mock() + self.checker = DiskSpaceChecker(self.logger, batch_size=4) + + @patch('gprebalance_modules.rebalance_commons.WorkerPool') + @patch('gprebalance_modules.rebalance_commons.DiskUsage') + def test_get_disk_usage_success(self, mock_disk_usage_class, mock_pool_class): + """Test successful disk usage retrieval""" + # Setup mock pool + mock_pool = Mock() + mock_pool_class.return_value = mock_pool + + # Create mock commands + cmd1 = Mock() + cmd1.was_successful.return_value = True + cmd1.directory = '/data1/primary/gpseg0' + cmd1.kbytes_used.return_value = 1048576 + + cmd2 = Mock() + cmd2.was_successful.return_value = True + cmd2.directory = '/data1/primary/gpseg1' + cmd2.kbytes_used.return_value = 2097152 + + mock_pool.getCompletedItems.return_value = [cmd1, cmd2] + + # Execute + directories = ['/data1/primary/gpseg0', '/data1/primary/gpseg1'] + result = self.checker.get_disk_usage('sdw1', directories) + + # Verify results + self.assertEqual(len(result), 2) + self.assertEqual(result['/data1/primary/gpseg0'], 1048576) + self.assertEqual(result['/data1/primary/gpseg1'], 2097152) + + # Verify pool usage + self.assertEqual(mock_pool.addCommand.call_count, 2) + mock_pool.join.assert_called_once() + mock_pool.haltWork.assert_called_once() + mock_pool.joinWorkers.assert_called_once() + + @patch('gprebalance_modules.rebalance_commons.WorkerPool') + def test_get_disk_usage_empty_list(self, mock_pool_class): + """Test disk usage with empty directory list""" + result = self.checker.get_disk_usage('sdw1', []) + + self.assertEqual(result, {}) + mock_pool_class.assert_not_called() + + @patch('gprebalance_modules.rebalance_commons.WorkerPool') + @patch('gprebalance_modules.rebalance_commons.DiskUsage') + def test_get_disk_usage_command_failure(self, mock_disk_usage_class, mock_pool_class): + """Test disk usage command failure""" + mock_pool = Mock() + mock_pool_class.return_value = mock_pool + + cmd = Mock() + cmd.was_successful.return_value = False + cmd.get_results.return_value.stderr = "Permission denied" + + mock_pool.getCompletedItems.return_value = [cmd] + + with self.assertRaises(Exception) as context: + self.checker.get_disk_usage('sdw1', ['/data1/primary/gpseg0']) + + self.assertIn("Unable to check disk usage", str(context.exception)) + + @patch('gprebalance_modules.rebalance_commons.WorkerPool') + @patch('gprebalance_modules.rebalance_commons.DiskFree') + @patch('gprebalance_modules.rebalance_commons.pickle') + @patch('gprebalance_modules.rebalance_commons.base64') + def test_get_available_space_success(self, mock_base64, mock_pickle, + mock_disk_free_class, mock_pool_class): + """Test successful available space retrieval""" + from gppylib.operations.validate_disk_space import FileSystem + + # Setup mock pool + mock_pool = Mock() + mock_pool_class.return_value = mock_pool + + # Mock FileSystem objects + fs1 = Mock(spec=FileSystem) + fs1.name = '/dev/sdb1' + fs1.disk_free = 10485760 + fs1.directories = ['/data1/primary/gpseg0'] + + fs2 = Mock(spec=FileSystem) + fs2.name = '/dev/sdb1' + fs2.disk_free = 10485760 + fs2.directories = ['/data1/primary/gpseg1'] + + mock_pickle.loads.return_value = [fs1, fs2] + mock_base64.urlsafe_b64decode.return_value = b'pickled_data' + + cmd = Mock() + cmd.was_successful.return_value = True + cmd.get_results.return_value.stdout = 'encoded_data' + + mock_pool.getCompletedItems.return_value = [cmd] + + # Execute + directories = ['/data1/primary/gpseg0', '/data1/primary/gpseg1'] + result = self.checker.get_available_space('sdw1', directories) + + # Verify + self.assertEqual(len(result), 2) + self.assertIn('/data1/primary/gpseg0', result) + self.assertIn('/data1/primary/gpseg1', result) + + self.assertEqual(result['/data1/primary/gpseg0'].filesystem, '/dev/sdb1') + self.assertEqual(result['/data1/primary/gpseg0'].available_kb, 10485760) + self.assertEqual(result['/data1/primary/gpseg0'].available_gb, 10.0) + + @patch('gprebalance_modules.rebalance_commons.DiskFree') + def test_get_available_space_command_failure(self, mock_disk_free_class): + """Test available space command failure""" + + cmd = Mock() + mock_disk_free_class.return_value = cmd + cmd.was_successful.return_value = False + cmd.get_results.return_value.stderr = "No such file or directory" + + with self.assertRaises(Exception) as context: + self.checker.get_available_space('sdw1', ['/data1/primary/gpseg0']) + + self.assertIn("Failed to check disk free", str(context.exception)) + +class TestResourceEstimator(GpTestCase): + """Test cases for ResourceEstimator using real GpArray configuration""" + + def setUp(self): + """Set up test fixtures""" + self.logger = Mock() + self.logger.info = Mock() + self.logger.debug = Mock() + self.logger.warning = Mock() + self.logger.error = Mock() + + self.conn = Mock() + self.dburl = Mock(spec=DbURL) + + # Load real GpArray configuration + self.gparray = initGparrayFromFile('unbalanced_9_ip') + + # Options for planner + self.options = Mock() + self.options.target_segment_count = 9 + self.options.target_hosts = None + self.options.add_hosts = None + self.options.remove_hosts = None + self.options.target_datadirs = None + self.options.target_hosts_file = None + self.options.add_hosts_file = None + self.options.remove_hosts_file = None + self.options.target_datadirs_file = None + self.options.mirror_mode = 'grouped' + self.options.skip_rebalance = False + self.options.skip_resource_estimation = False + self.options.batch_size = 16 + + @patch('gprebalance_modules.planner.dbconn') + def test_estimate_segment_sizes_from_unbalanced_cluster(self, mock_dbconn): + """Test segment size estimation from unbalanced cluster""" + # Create a realistic move: move primary seg0 from sdw1 to sdw2 + seg0 = None + for seg in self.gparray.getDbList(): + if seg.content == 0 and seg.isSegmentPrimary(): + seg0 = seg + break + + self.assertIsNotNone(seg0) + self.assertEqual(seg0.hostname, 'sdw1') + self.assertEqual(seg0.address, '172.20.0.6') + + moves = [ + LogicalMove( + seg=seg0, + srcHost=Host('sdw1', '172.20.0.6', status=HostStatus.ACTIVE), + dstHost=Host('sdw2', '172.20.0.7', status=HostStatus.ACTIVE), + target_datadir='/data/primary0', + target_port=7000, + segment_size=None + ) + ] + + estimator = ResourceEstimator(self.logger, self.conn, self.gparray) + + # Mock disk checker to return segment size + estimator.disk_checker.get_disk_usage = Mock(return_value={ + '/data/primary0': 2097152 # 2GB + }) + + # Mock tablespace query (no tablespaces) + mock_dbconn.query.return_value = [] + + estimator._estimate_segment_sizes(moves) + + # Verify segment size was set + self.assertIsNotNone(moves[0].segment_size) + self.assertEqual(moves[0].segment_size.datadir_size_kb, 2097152) + self.assertEqual(moves[0].segment_size.total_size_kb, 2097152) + + # Verify disk usage was called with correct parameters + estimator.disk_checker.get_disk_usage.assert_called_once_with( + '172.20.0.6', + ['/data/primary0'] + ) + + @patch('gprebalance_modules.planner.dbconn') + def test_estimate_multiple_segments_same_host(self, mock_dbconn): + """ + Test estimating multiple segments from same source host + """ + # Get primaries from sdw1: seg0, seg1, seg2 + segs_from_sdw1 = [] + for seg in self.gparray.getSegDbList(): + if seg.getSegmentHostName() == 'sdw1' and seg.isSegmentPrimary(): + segs_from_sdw1.append(seg) + + self.assertEqual(len(segs_from_sdw1), 3) + + # Create moves for all three segments + moves = [ + LogicalMove( + seg=segs_from_sdw1[0], + srcHost=Host('sdw1', '172.20.0.6', status=HostStatus.ACTIVE), + dstHost=Host('sdw2', '172.20.0.7', status=HostStatus.ACTIVE), + target_datadir='/data/primary0', + target_port=7000, + segment_size=None + ), + LogicalMove( + seg=segs_from_sdw1[1], + srcHost=Host('sdw1', '172.20.0.6', status=HostStatus.ACTIVE), + dstHost=Host('sdw2', '172.20.0.7', status=HostStatus.ACTIVE), + target_datadir='/data/primary1', + target_port=7001, + segment_size=None + ), + LogicalMove( + seg=segs_from_sdw1[2], + srcHost=Host('sdw1', '172.20.0.6', status=HostStatus.ACTIVE), + dstHost=Host('sdw2', '172.20.0.7', status=HostStatus.ACTIVE), + target_datadir='/data/primary2', + target_port=7002, + segment_size=None + ) + ] + estimator = ResourceEstimator(self.logger, self.conn, self.gparray) + + # Mock disk checker to return sizes for all segments + estimator.disk_checker.get_disk_usage = Mock(return_value={ + '/data/primary0': 1048576, # 1GB + '/data/primary1': 2097152, # 2GB + '/data/primary2': 1572864 # 1.5GB + }) + + # Mock tablespace query + mock_dbconn.query.return_value = [] + + estimator._estimate_segment_sizes(moves) + + # Verify all segment sizes were set + self.assertEqual(moves[0].segment_size.datadir_size_kb, 1048576) + self.assertEqual(moves[1].segment_size.datadir_size_kb, 2097152) + self.assertEqual(moves[2].segment_size.datadir_size_kb, 1572864) + + # Verify single call to get_disk_usage with all directories + estimator.disk_checker.get_disk_usage.assert_called_once() + call_args = estimator.disk_checker.get_disk_usage.call_args[0] + self.assertEqual(call_args[0], '172.20.0.6') + self.assertEqual(set(call_args[1]), {'/data/primary0', '/data/primary1', '/data/primary2'}) + + @patch('gprebalance_modules.planner.dbconn') + def test_estimate_with_tablespaces(self, mock_dbconn): + # Get primary seg0 from sdw1 + seg0 = None + for seg in self.gparray.getDbList(): + if seg.content == 0 and seg.isSegmentPrimary(): + seg0 = seg + break + + moves = [ + LogicalMove( + seg=seg0, + srcHost=Host('sdw1', '172.20.0.6', status=HostStatus.ACTIVE), + dstHost=Host('sdw2', '172.20.0.7', status=HostStatus.ACTIVE), + target_datadir='/data/primary0', + target_port=7000, + segment_size=None + ) + ] + + estimator = ResourceEstimator(self.logger, self.conn, self.gparray) + + # Mock disk checker for datadir and tablespaces + call_count = [0] + def mock_disk_usage_side_effect(host, dirs): + call_count[0] += 1 + if call_count[0] == 1: + # First call: datadir + return {'/data/primary0': 2097152} # 2GB + else: + # Second call: tablespaces + return { + '/tablespace1/2': 524288, # 512MB + '/tablespace2/2': 1048576 # 1GB + } + + estimator.disk_checker.get_disk_usage = Mock(side_effect=mock_disk_usage_side_effect) + + # Mock tablespace query to return tablespace locations + mock_dbconn.query.return_value = [ + (2, '/tablespace1/2'), + (2, '/tablespace2/2') + ] + + estimator._estimate_segment_sizes(moves) + + # Verify segment size includes tablespaces + self.assertIsNotNone(moves[0].segment_size) + self.assertEqual(moves[0].segment_size.datadir_size_kb, 2097152) + self.assertIsNotNone(moves[0].segment_size.tablespace_usage) + self.assertEqual(moves[0].segment_size.tablespace_usage['/tablespace1/2'], 524288) + self.assertEqual(moves[0].segment_size.tablespace_usage['/tablespace2/2'], 1048576) + + # Total should be datadir + tablespaces + expected_total = 2097152 + 524288 + 1048576 + self.assertEqual(moves[0].segment_size.total_size_kb, expected_total) + + def test_validate_target_space_sufficient(self): + """Test validation passes when sufficient space is available""" + # Get primary seg0 + seg0 = None + for seg in self.gparray.getDbList(): + if seg.content == 0 and seg.isSegmentPrimary(): + seg0 = seg + break + + moves = [ + LogicalMove( + seg=seg0, + srcHost=Host('sdw1', '172.20.0.6', status=HostStatus.ACTIVE), + dstHost=Host('sdw2', '172.20.0.7', status=HostStatus.ACTIVE), + target_datadir='/data/primary0', + target_port=7000, + segment_size=SegmentSize(datadir_size_kb=2097152) # 2GB + ) + ] + + estimator = ResourceEstimator(self.logger, self.conn, self.gparray) + + # Mock disk checker - 20GB available + estimator.disk_checker.check_batch_available_space = Mock(return_value={ + '172.20.0.7': { + '/data/primary0': DiskSpaceInfo( + filesystem='/dev/sdb1', + available_kb=20971520, # 20GB + directory='/data/primary0' + ) + } + }) + + # Should not raise exception + try: + estimator._validate_and_build_allocations(moves) + except ResourceError: + self.fail("ResourceError raised when space is sufficient") + + # Verify disk space check was called + estimator.disk_checker.check_batch_available_space.assert_called_once() + call_args = estimator.disk_checker.check_batch_available_space.call_args[0][0] + self.assertIn('172.20.0.7', call_args) + self.assertIn('/data/primary0', call_args['172.20.0.7']) + + def test_validate_target_space_insufficient(self): + """Test validation fails when insufficient space for datadir""" + # Get primary seg0 + seg0 = None + for seg in self.gparray.getDbList(): + if seg.content == 0 and seg.isSegmentPrimary(): + seg0 = seg + break + + moves = [ + LogicalMove( + seg=seg0, + srcHost=Host('sdw1', '172.20.0.6', status=HostStatus.ACTIVE), + dstHost=Host('sdw2', '172.20.0.7', status=HostStatus.ACTIVE), + target_datadir='/data/primary0', + target_port=7000, + segment_size=SegmentSize(datadir_size_kb=10485760) # 10GB + ) + ] + + estimator = ResourceEstimator(self.logger, self.conn, self.gparray) + + # Mock disk checker - only 2GB available + estimator.disk_checker.check_batch_available_space = Mock(return_value={ + '172.20.0.7': { + '/data/primary0': DiskSpaceInfo( + filesystem='/dev/sdb1', + available_kb=2097152, # 2GB available + directory='/data/primary0' + ) + } + }) + + # Should raise ResourceError + with self.assertRaises(ResourceError) as context: + estimator._validate_and_build_allocations(moves) + + error_msg = str(context.exception) + self.assertIn("Insufficient disk space", error_msg) + self.assertIn("sdw2", error_msg) + self.assertIn("/data/primary0", error_msg) + + def test_validate_multiple_moves_same_filesystem_insufficient(self): + """Test validation aggregates space requirements on same filesystem""" + # Get primaries from sdw1 + segs_from_sdw1 = [] + for seg in self.gparray.getDbList(): + if seg.hostname == 'sdw1' and seg.isSegmentPrimary() and seg.content < 2: + segs_from_sdw1.append(seg) + + moves = [ + LogicalMove( + seg=segs_from_sdw1[0], + srcHost=Host('sdw1', '172.20.0.6', status=HostStatus.ACTIVE), + dstHost=Host('sdw2', '172.20.0.7', status=HostStatus.ACTIVE), + target_datadir='/data/primary0', + target_port=7000, + segment_size=SegmentSize(datadir_size_kb=5242880) # 5GB + ), + LogicalMove( + seg=segs_from_sdw1[1], + srcHost=Host('sdw1', '172.20.0.6', status=HostStatus.ACTIVE), + dstHost=Host('sdw2', '172.20.0.7', status=HostStatus.ACTIVE), + target_datadir='/data/primary1', + target_port=7001, + segment_size=SegmentSize(datadir_size_kb=5242880) # 5GB + ) + ] + + estimator = ResourceEstimator(self.logger, self.conn, self.gparray) + + # Total needed: (5GB + 5GB) * 1.1 = 11GB + # Available: only 8GB (insufficient) + estimator.disk_checker.check_batch_available_space = Mock(return_value={ + '172.20.0.7': { + '/data/primary0': DiskSpaceInfo( + filesystem='/dev/sdb1', + available_kb=8388608, # 8GB + directory='/data/primary0' + ), + '/data/primary1': DiskSpaceInfo( + filesystem='/dev/sdb1', # Same filesystem + available_kb=8388608, # 8GB (same) + directory='/data/primary1' + ) + } + }) + + with self.assertRaises(ResourceError) as context: + estimator._validate_and_build_allocations(moves) + + error_msg = str(context.exception) + self.assertIn("Insufficient disk space", error_msg) + # Both directories should be mentioned + self.assertIn("/data/primary0", error_msg) + self.assertIn("/data/primary1", error_msg) + # Should show aggregated requirement (~11GB) vs available (8GB) + self.assertIn("11", error_msg) + self.assertIn("8", error_msg) + + def test_validate_multiple_moves_same_filesystem_sufficient(self): + """Test validation passes when aggregated space is sufficient""" + # Get primaries from sdw1 + segs_from_sdw1 = [] + for seg in self.gparray.getDbList(): + if seg.hostname == 'sdw1' and seg.isSegmentPrimary() and seg.content < 2: + segs_from_sdw1.append(seg) + + moves = [ + LogicalMove( + seg=segs_from_sdw1[0], + srcHost=Host('sdw1', '172.20.0.6', status=HostStatus.ACTIVE), + dstHost=Host('sdw2', '172.20.0.7', status=HostStatus.ACTIVE), + target_datadir='/data/primary0', + target_port=7000, + segment_size=SegmentSize(datadir_size_kb=5242880) # 5GB + ), + LogicalMove( + seg=segs_from_sdw1[1], + srcHost=Host('sdw1', '172.20.0.6', status=HostStatus.ACTIVE), + dstHost=Host('sdw2', '172.20.0.7', status=HostStatus.ACTIVE), + target_datadir='/data/primary1', + target_port=7001, + segment_size=SegmentSize(datadir_size_kb=5242880) # 5GB + ) + ] + + estimator = ResourceEstimator(self.logger, self.conn, self.gparray) + + # Total needed: (5GB + 5GB) * 1.1 = 11GB + # Available: 15GB (sufficient) + estimator.disk_checker.check_batch_available_space = Mock(return_value={ + '172.20.0.7': { + '/data/primary0': DiskSpaceInfo( + filesystem='/dev/sdb1', + available_kb=15728640, # 15GB + directory='/data/primary0' + ), + '/data/primary1': DiskSpaceInfo( + filesystem='/dev/sdb1', # Same filesystem + available_kb=15728640, # 15GB (same) + directory='/data/primary1' + ) + } + }) + + # Should not raise + try: + estimator._validate_and_build_allocations(moves) + except ResourceError: + self.fail("ResourceError raised when space is sufficient") + + def test_validate_target_space_no_space_info(self): + """Test validation fails when no space info is available""" + seg0 = None + for seg in self.gparray.getDbList(): + if seg.content == 0 and seg.isSegmentPrimary(): + seg0 = seg + break + + moves = [ + LogicalMove( + seg=seg0, + srcHost=Host('sdw1', '172.20.0.6', status=HostStatus.ACTIVE), + dstHost=Host('sdw2', '172.20.0.7', status=HostStatus.ACTIVE), + target_datadir='/data/primary0', + target_port=7000, + segment_size=SegmentSize(datadir_size_kb=2097152) + ) + ] + + estimator = ResourceEstimator(self.logger, self.conn, self.gparray) + + # Return empty space info + estimator.disk_checker.check_batch_available_space = Mock(return_value={}) + + with self.assertRaises(ResourceError) as context: + estimator._validate_and_build_allocations(moves) + + self.assertIn("No disk space information for sdw2:/data/primary0", str(context.exception)) + + def test_validate_tablespace_space_sufficient(self): + """Test validation succeeds when tablespace has sufficient space on separate filesystem""" + seg0 = None + for seg in self.gparray.getDbList(): + if seg.content == 0 and seg.isSegmentPrimary(): + seg0 = seg + break + + moves = [ + LogicalMove( + seg=seg0, + srcHost=Host('sdw1', '172.20.0.6', status=HostStatus.ACTIVE), + dstHost=Host('sdw2', '172.20.0.7', status=HostStatus.ACTIVE), + target_datadir='/data/primary0', + target_port=7000, + segment_size=SegmentSize( + datadir_size_kb=2097152, # 2GB + tablespace_usage={'/tablespace1/2': 1048576} # 1GB + ) + ) + ] + + estimator = ResourceEstimator(self.logger, self.conn, self.gparray) + + # Mock sufficient space for both datadir and tablespace on different filesystems + estimator.disk_checker.check_batch_available_space = Mock(return_value={ + '172.20.0.7': { + '/data/primary0': DiskSpaceInfo( + filesystem='/dev/sdb1', + available_kb=20971520, # 20GB + directory='/data/primary0' + ), + '/tablespace1': DiskSpaceInfo( + filesystem='/dev/sdc1', # Different filesystem + available_kb=10485760, # 10GB + directory='/tablespace1' + ) + } + }) + + # Should not raise exception + try: + estimator._validate_and_build_allocations(moves) + except ResourceError: + self.fail("ResourceError raised when space is sufficient") + + def test_validate_tablespace_space_insufficient(self): + """Test validation fails when tablespace has insufficient space""" + seg0 = None + for seg in self.gparray.getDbList(): + if seg.content == 0 and seg.isSegmentPrimary(): + seg0 = seg + break + + moves = [ + LogicalMove( + seg=seg0, + srcHost=Host('sdw1', '172.20.0.6', status=HostStatus.ACTIVE), + dstHost=Host('sdw2', '172.20.0.7', status=HostStatus.ACTIVE), + target_datadir='/data/primary0', + target_port=7000, + segment_size=SegmentSize( + datadir_size_kb=2097152, # 2GB + tablespace_usage={'/tablespace1/2': 10485760} # 10GB + ) + ) + ] + + estimator = ResourceEstimator(self.logger, self.conn, self.gparray) + + # Mock sufficient space for datadir but insufficient for tablespace + estimator.disk_checker.check_batch_available_space = Mock(return_value={ + '172.20.0.7': { + '/data/primary0': DiskSpaceInfo( + filesystem='/dev/sdb1', + available_kb=20971520, # 20GB + directory='/data/primary0' + ), + '/tablespace1': DiskSpaceInfo( + filesystem='/dev/sdc1', + available_kb=1048576, # Only 1GB available + directory='/tablespace1' + ) + } + }) + + with self.assertRaises(ResourceError) as context: + estimator._validate_and_build_allocations(moves) + + error_msg = str(context.exception) + self.assertIn("Insufficient disk space", error_msg) + self.assertIn("/tablespace1", error_msg) + + def test_validate_multiple_tablespaces_different_filesystems(self): + """Test validation with multiple tablespaces on different filesystems""" + seg0 = None + for seg in self.gparray.getDbList(): + if seg.content == 0 and seg.isSegmentPrimary(): + seg0 = seg + break + + moves = [ + LogicalMove( + seg=seg0, + srcHost=Host('sdw1', '172.20.0.6', status=HostStatus.ACTIVE), + dstHost=Host('sdw2', '172.20.0.7', status=HostStatus.ACTIVE), + target_datadir='/data/primary0', + target_port=7000, + segment_size=SegmentSize( + datadir_size_kb=2097152, # 2GB + tablespace_usage={ + '/tablespace1/2': 3145728, # 3GB + '/tablespace2/2': 4194304 # 4GB + } + ) + ) + ] + + estimator = ResourceEstimator(self.logger, self.conn, self.gparray) + + # Mock space - datadir OK, tablespace1 OK, tablespace2 insufficient + estimator.disk_checker.check_batch_available_space = Mock(return_value={ + '172.20.0.7': { + '/data/primary0': DiskSpaceInfo( + filesystem='/dev/sdb1', + available_kb=20971520, # 20GB + directory='/data/primary0' + ), + '/tablespace1': DiskSpaceInfo( + filesystem='/dev/sdc1', + available_kb=10485760, # 10GB - sufficient + directory='/tablespace1' + ), + '/tablespace2': DiskSpaceInfo( + filesystem='/dev/sdd1', + available_kb=2097152, # Only 2GB - insufficient + directory='/tablespace2' + ) + } + }) + + with self.assertRaises(ResourceError) as context: + estimator._validate_and_build_allocations(moves) + + error_msg = str(context.exception) + self.assertIn("Insufficient disk space", error_msg) + # Only tablespace2 should be in error + self.assertIn("/tablespace2", error_msg) + + def test_validate_tablespace_same_filesystem_as_datadir_insufficient(self): + """Test when tablespace and datadir share the same filesystem - insufficient space""" + seg0 = None + for seg in self.gparray.getDbList(): + if seg.content == 0 and seg.isSegmentPrimary(): + seg0 = seg + break + + moves = [ + LogicalMove( + seg=seg0, + srcHost=Host('sdw1', '172.20.0.6', status=HostStatus.ACTIVE), + dstHost=Host('sdw2', '172.20.0.7', status=HostStatus.ACTIVE), + target_datadir='/data/primary0', + target_port=7000, + segment_size=SegmentSize( + datadir_size_kb=5242880, # 5GB + tablespace_usage={'/data/tablespace1/2': 5242880} # 5GB + ) + ) + ] + + estimator = ResourceEstimator(self.logger, self.conn, self.gparray) + + # Both datadir and tablespace on same filesystem /dev/sdb1 + # Total needed: (5GB + 5GB) * 1.1 = 11GB + # Available: only 8GB (insufficient) + estimator.disk_checker.check_batch_available_space = Mock(return_value={ + '172.20.0.7': { + '/data/primary0': DiskSpaceInfo( + filesystem='/dev/sdb1', + available_kb=8388608, # 8GB + directory='/data/primary0' + ), + '/data/tablespace1': DiskSpaceInfo( + filesystem='/dev/sdb1', # Same filesystem! + available_kb=8388608, # 8GB (same as above - shared filesystem!) + directory='/data/tablespace1' + ) + } + }) + + with self.assertRaises(ResourceError) as context: + estimator._validate_and_build_allocations(moves) + + error_msg = str(context.exception) + self.assertIn("Insufficient disk space", error_msg) + # Should show BOTH directories since they're on the same filesystem + self.assertIn("/data/primary0", error_msg) + self.assertIn("/data/tablespace1", error_msg) + # Should show it needs ~11GB but only has 8GB + self.assertIn("11", error_msg) + self.assertIn("8", error_msg) + + def test_validate_tablespace_same_filesystem_as_datadir_sufficient(self): + """Test when tablespace and datadir share filesystem with sufficient space""" + seg0 = None + for seg in self.gparray.getDbList(): + if seg.content == 0 and seg.isSegmentPrimary(): + seg0 = seg + break + + moves = [ + LogicalMove( + seg=seg0, + srcHost=Host('sdw1', '172.20.0.6', status=HostStatus.ACTIVE), + dstHost=Host('sdw2', '172.20.0.7', status=HostStatus.ACTIVE), + target_datadir='/data/primary0', + target_port=7000, + segment_size=SegmentSize( + datadir_size_kb=5242880, # 5GB + tablespace_usage={'/data/tablespace1/2': 5242880} # 5GB + ) + ) + ] + + estimator = ResourceEstimator(self.logger, self.conn, self.gparray) + + # Both on same filesystem, need 11GB, have 20GB - should pass + estimator.disk_checker.check_batch_available_space = Mock(return_value={ + '172.20.0.7': { + '/data/primary0': DiskSpaceInfo( + filesystem='/dev/sdb1', + available_kb=20971520, # 20GB + directory='/data/primary0' + ), + '/data/tablespace1': DiskSpaceInfo( + filesystem='/dev/sdb1', # Same filesystem + available_kb=20971520, # 20GB (same) + directory='/data/tablespace1' + ) + } + }) + + # Should not raise exception + try: + estimator._validate_and_build_allocations(moves) + except ResourceError: + self.fail("ResourceError raised when space is sufficient") + + def test_validate_complex_overlapping_filesystems(self): + """Test multiple segments with overlapping filesystem usage""" + segs = [seg for seg in self.gparray.getDbList() + if seg.isSegmentPrimary() and seg.content < 2] + + moves = [ + # Seg 0: datadir on /dev/sdb1, tablespace on /dev/sdc1 + LogicalMove( + seg=segs[0], + srcHost=Host('sdw1', '172.20.0.6', status=HostStatus.ACTIVE), + dstHost=Host('sdw2', '172.20.0.7', status=HostStatus.ACTIVE), + target_datadir='/data/primary0', + target_port=7000, + segment_size=SegmentSize( + datadir_size_kb=3145728, # 3GB + tablespace_usage={'/tablespace1/2': 2097152} # 2GB + ) + ), + # Seg 1: datadir on /dev/sdb1 (SAME as seg0 datadir), + # tablespace on /dev/sdb1 (SAME filesystem!) + LogicalMove( + seg=segs[1], + srcHost=Host('sdw1', '172.20.0.6', status=HostStatus.ACTIVE), + dstHost=Host('sdw2', '172.20.0.7', status=HostStatus.ACTIVE), + target_datadir='/data/primary1', + target_port=7001, + segment_size=SegmentSize( + datadir_size_kb=4194304, # 4GB + tablespace_usage={'/data/tblspace2/3': 3145728} # 3GB + ) + ) + ] + + estimator = ResourceEstimator(self.logger, self.conn, self.gparray) + + # /dev/sdb1: needs (3GB + 4GB + 3GB) * 1.1 = 11GB, has 12GB - PASS + # /dev/sdc1: needs 2GB * 1.1 = 2.2GB, has 5GB - PASS + estimator.disk_checker.check_batch_available_space = Mock(return_value={ + '172.20.0.7': { + '/data/primary0': DiskSpaceInfo( + filesystem='/dev/sdb1', + available_kb=12582912, # 12GB + directory='/data/primary0' + ), + '/data/primary1': DiskSpaceInfo( + filesystem='/dev/sdb1', # Same as primary0 + available_kb=12582912, + directory='/data/primary1' + ), + '/data/tblspace2': DiskSpaceInfo( + filesystem='/dev/sdb1', # Same filesystem! + available_kb=12582912, + directory='/data/tblspace2' + ), + '/tablespace1': DiskSpaceInfo( + filesystem='/dev/sdc1', # Different filesystem + available_kb=5242880, # 5GB + directory='/tablespace1' + ) + } + }) + + # Should not raise + try: + estimator._validate_and_build_allocations(moves) + except ResourceError: + self.fail("ResourceError raised when space is sufficient") + + def test_validate_complex_overlapping_filesystems_insufficient(self): + """Test multiple segments with overlapping filesystems - insufficient space""" + segs = [seg for seg in self.gparray.getDbList() + if seg.isSegmentPrimary() and seg.content < 2] + + moves = [ + # Seg 0: datadir on /dev/sdb1, tablespace on /dev/sdc1 + LogicalMove( + seg=segs[0], + srcHost=Host('sdw1', '172.20.0.6', status=HostStatus.ACTIVE), + dstHost=Host('sdw2', '172.20.0.7', status=HostStatus.ACTIVE), + target_datadir='/data/primary0', + target_port=7000, + segment_size=SegmentSize( + datadir_size_kb=3145728, # 3GB + tablespace_usage={'/tablespace1/2': 2097152} # 2GB + ) + ), + # Seg 1: datadir on /dev/sdb1, tablespace on /dev/sdb1 (SAME filesystem!) + LogicalMove( + seg=segs[1], + srcHost=Host('sdw1', '172.20.0.6', status=HostStatus.ACTIVE), + dstHost=Host('sdw2', '172.20.0.7', status=HostStatus.ACTIVE), + target_datadir='/data/primary1', + target_port=7001, + segment_size=SegmentSize( + datadir_size_kb=4194304, # 4GB + tablespace_usage={'/data/tblspace2/3': 3145728} # 3GB + ) + ) + ] + + estimator = ResourceEstimator(self.logger, self.conn, self.gparray) + + # /dev/sdb1: needs (3GB + 4GB + 3GB) * 1.1 = 11GB, has only 9GB - FAIL + # /dev/sdc1: needs 2GB * 1.1 = 2.2GB, has 5GB - PASS + estimator.disk_checker.check_batch_available_space = Mock(return_value={ + '172.20.0.7': { + '/data/primary0': DiskSpaceInfo( + filesystem='/dev/sdb1', + available_kb=9437184, # 9GB - insufficient! + directory='/data/primary0' + ), + '/data/primary1': DiskSpaceInfo( + filesystem='/dev/sdb1', # Same as primary0 + available_kb=9437184, + directory='/data/primary1' + ), + '/data/tblspace2': DiskSpaceInfo( + filesystem='/dev/sdb1', # Same filesystem! + available_kb=9437184, + directory='/data/tblspace2' + ), + '/tablespace1': DiskSpaceInfo( + filesystem='/dev/sdc1', # Different filesystem + available_kb=5242880, # 5GB - sufficient + directory='/tablespace1' + ) + } + }) + + with self.assertRaises(ResourceError) as context: + estimator._validate_and_build_allocations(moves) + + error_msg = str(context.exception) + self.assertIn("Insufficient disk space", error_msg) + # Should show all three directories on /dev/sdb1 + self.assertIn("/data/primary0", error_msg) + self.assertIn("/data/primary1", error_msg) + self.assertIn("/data/tblspace2", error_msg) + # Should show aggregated requirement (~11GB) vs available (9GB) + self.assertIn("11", error_msg) + self.assertIn("9", error_msg) + + @patch('gprebalance_modules.planner.PortIsAvailable') + @patch('gprebalance_modules.planner.DiskSpaceChecker') + @patch('gprebalance_modules.planner.HostResolver.resolve_hostname') + @patch('gprebalance_modules.planner.HostResolver.get_address') + @patch('gprebalance_modules.planner.GreedySolver') + @patch('gprebalance_modules.rebalance_schema.dbconn.queryRow', side_effect=check_query) + @patch('gprebalance_modules.planner.dbconn') + def test_planner_with_resource_estimation(self, mock_dbconn, mock_schema, mock_solver, + mock_get_address, mock_resolve, mock_disk_check, mock_port): + """Test Planner integration with resource estimation""" + # Setup resolver mocks + mock_resolve.return_value = None + def address_side_effect(hostname): + addr_map = { + 'sdw1': '172.20.0.6', + 'sdw2': '172.20.0.7', + 'sdw3': '172.20.0.8' + } + return addr_map.get(hostname, hostname) + mock_get_address.side_effect = address_side_effect + + # Setup solver mock + mock_solver_instance = Mock() + mock_solver.return_value = mock_solver_instance + + # Mock solution: move segments to balance cluster + solution = {0: (0, 1), + 1: (0, 1), + 2: (0, 1), + 3: (2, 0), + 4: (2, 0), + 5: (1, 2), + 6: (1, 2), + 7: (1, 2), + 8: (2, 0)} + + mock_solver_instance.solve.return_value = (solution, {}) + + self.options.target_datadirs="/data/primary{content}, /data/mirror{content}" + + mock_port.return_value._is_port_available.return_value = True + # Create planner + planner = Planner( + logger=self.logger, + dburl=self.dburl, + gpArray=self.gparray, + options=self.options + ) + + # Mock disk usage - all segments are 2GB + mock_disk_check.return_value.get_disk_usage.return_value = { + '/data/primary0': 2097152, + '/data/primary1': 2097152, + '/data/primary2': 2097152, + '/data/mirror3': 2097152, + '/data/mirror4': 2097152, + '/data/mirror5': 2097152, + '/data/mirror6': 2097152, + '/data/mirror7': 2097152, + '/data/mirror8': 2097152, + } + + # Mock available space - on all targets where we move segments to + mock_disk_check.return_value.check_batch_available_space.return_value = { + '172.20.0.7': { + '/data/primary8': DiskSpaceInfo('/dev/sdb1', 52428800, '/data/primary8'), + '/data/mirror5': DiskSpaceInfo('/dev/sdb1', 52428800, '/data/mirror5'), + '/data/mirror6': DiskSpaceInfo('/dev/sdb1', 52428800, '/data/mirror6'), + '/data/mirror7': DiskSpaceInfo('/dev/sdb1', 52428800, '/data/mirror7'), + }, + '172.20.0.8': { + '/data/mirror0': DiskSpaceInfo('/dev/sdc1', 52428800, '/data/mirror0'), + '/data/mirror1': DiskSpaceInfo('/dev/sdc1', 52428800, '/data/mirror1'), + '/data/mirror2': DiskSpaceInfo('/dev/sdc1', 52428800, '/data/mirror2'), + }, + } + + mock_dbconn.connect.return_value = self.conn + mock_dbconn.query.return_value = [] + + # Execute planning + plan = planner.plan() + + # Verify moves were created + self.assertIsNotNone(plan.getMoves()) + + # Verify resource estimation was performed + for move in plan.getMoves(): + self.assertIsNotNone(move.segment_size, + f"Segment size not set for move: {move}") + + @patch('gprebalance_modules.planner.dbconn') + @patch('gprebalance_modules.rebalance_schema.dbconn.queryRow', side_effect=check_query) + @patch('gprebalance_modules.planner.HostResolver.resolve_hostname') + @patch('gprebalance_modules.planner.HostResolver.get_address') + def test_planner_skips_resource_estimation_when_requested(self, mock_get_address, + mock_resolve, mock_schema, mock_conn): + """Test Planner skips resource estimation when skip_resource_estimation=True""" + mock_resolve.return_value = None + def address_side_effect(hostname): + addr_map = { + 'sdw1': '172.20.0.6', + 'sdw2': '172.20.0.7', + 'sdw3': '172.20.0.8' + } + return addr_map.get(hostname, hostname) + mock_get_address.side_effect = address_side_effect + + # Enable skip flag + self.options.skip_resource_estimation = True + + planner = Planner( + logger=self.logger, + dburl=self.dburl, + gpArray=self.gparray, + options=self.options + ).plan() + + # Verify warning was logged + self.logger.warning.assert_any_call("Skipping resource estimation") + +class TestPortAllocator(GpTestCase): + """ + Unit tests for PortAllocator class + """ + + def setUp(self): + self.logger = Mock() + + def _create_mock_segment(self, dbid, content, hostname, port, role='p', preferred_role='p'): + """ + Helper to create mock segment + """ + seg = Mock(spec=gparray.Segment) + seg.dbid = dbid + seg.content = content + seg.hostname = hostname + seg.datadir = f'/data{content}/seg{content}' + seg.port = port + seg.role = role + seg.preferred_role = preferred_role + + seg.getSegmentDbId.return_value = dbid + seg.getSegmentContentId.return_value = content + seg.getSegmentHostName.return_value = hostname + seg.getSegmentDataDirectory.return_value = seg.datadir + seg.getSegmentPort.return_value = port + seg.isSegmentPrimary.return_value = (role == 'p') + seg.isSegmentMirror.return_value = (role == 'm') + + return seg + + def _create_mock_gparray(self, segments): + """ + Helper to create mock GpArray + """ + array = Mock(spec=gparray.GpArray) + array.getSegDbList.return_value = segments + return array + + def test_initialization_simple(self): + """ + Test basic initialization with simple segment configuration + """ + segments = [ + self._create_mock_segment(1, 0, 'host1', 6000, 'p'), + self._create_mock_segment(2, 0, 'host2', 6000, 'm'), + self._create_mock_segment(3, 1, 'host1', 6001, 'p'), + self._create_mock_segment(4, 1, 'host2', 6001, 'm'), + ] + + gparray_mock = self._create_mock_gparray(segments) + allocator = PortAllocator(gparray_mock, self.logger) + + # Verify existing ports tracked + self.assertIn(6000, allocator.existing_ports_by_host['host1']) + self.assertIn(6001, allocator.existing_ports_by_host['host1']) + self.assertIn(6000, allocator.existing_ports_by_host['host2']) + self.assertIn(6001, allocator.existing_ports_by_host['host2']) + + # Verify base ports detected + self.assertEqual(allocator.base_ports_by_host['host1'], (6000, None)) + self.assertEqual(allocator.base_ports_by_host['host2'], (None, 6000)) + + def test_initialization_with_primaries_and_mirrors(self): + """ + Test initialization with both primaries and mirrors on same host + """ + segments = [ + self._create_mock_segment(1, 0, 'host1', 6000, 'p'), + self._create_mock_segment(2, 1, 'host1', 6001, 'p'), + self._create_mock_segment(3, 0, 'host2', 7000, 'm'), + self._create_mock_segment(4, 1, 'host2', 7001, 'm'), + ] + + gparray_mock = self._create_mock_gparray(segments) + allocator = PortAllocator(gparray_mock, self.logger) + + # Host1 has primaries + primary_ports, mirror_ports = allocator.existing_ports_by_role['host1'] + self.assertEqual(primary_ports, {6000, 6001}) + self.assertEqual(mirror_ports, set()) + + # Host2 has mirrors + primary_ports, mirror_ports = allocator.existing_ports_by_role['host2'] + self.assertEqual(primary_ports, set()) + self.assertEqual(mirror_ports, {7000, 7001}) + + # Base ports + self.assertEqual(allocator.base_ports_by_host['host1'], (6000, None)) + self.assertEqual(allocator.base_ports_by_host['host2'], (None, 7000)) + + def test_allocate_port_existing_host_port_available(self): + """ + Test allocating port on existing host when current port is available + """ + segments = [ + self._create_mock_segment(1, 0, 'host1', 6000, 'p'), + self._create_mock_segment(2, 1, 'host1', 6001, 'p'), + ] + + gparray_mock = self._create_mock_gparray(segments) + allocator = PortAllocator(gparray_mock, self.logger, verify_ports=False) + + host = Host(hostname='host2', address='10.0.0.2', status=HostStatus.ACTIVE) + + # Allocate port 7000 on host2 (doesn't exist in cluster yet) + allocated_port = allocator.allocate_port(host, current_port=7000, is_mirror=False) + + self.assertEqual(allocated_port, 7000) + self.assertIn(7000, allocator.planned_ports_by_host['host2']) + + def test_allocate_port_existing_host_port_conflict(self): + """ + Test allocating port on existing host when current port conflicts + """ + segments = [ + self._create_mock_segment(1, 0, 'host1', 6000, 'p'), + self._create_mock_segment(2, 1, 'host1', 6001, 'p'), + ] + + gparray_mock = self._create_mock_gparray(segments) + allocator = PortAllocator(gparray_mock, self.logger, verify_ports=False) + + host = Host(hostname='host1', address='10.0.0.1', status=HostStatus.ACTIVE) + + # Try to allocate port 6000 which is already used + allocated_port = allocator.allocate_port(host, current_port=6000, is_mirror=False) + + # Should get next available port + self.assertNotEqual(allocated_port, 6000) + self.assertGreaterEqual(allocated_port, 6000) + self.assertIn(allocated_port, allocator.planned_ports_by_host['host1']) + + def test_allocate_port_new_host_first_primary(self): + """ + Test allocating first primary port on new host + """ + segments = [ + self._create_mock_segment(1, 0, 'host1', 6000, 'p'), + ] + + gparray_mock = self._create_mock_gparray(segments) + allocator = PortAllocator(gparray_mock, self.logger, verify_ports=False) + + new_host = Host(hostname='host2', address='10.0.0.2', status=HostStatus.NEW) + + # Allocate first primary on new host + allocated_port = allocator.allocate_port(new_host, current_port=6000, is_mirror=False) + + self.assertEqual(allocated_port, 6000) + self.assertEqual(allocator.base_ports_by_host['host2'], (6000, None)) + self.assertIn(6000, allocator.planned_ports_by_host['host2']) + + def test_allocate_port_new_host_first_mirror(self): + """ + Test allocating first mirror port on new host + """ + segments = [ + self._create_mock_segment(1, 0, 'host1', 6000, 'p'), + ] + + gparray_mock = self._create_mock_gparray(segments) + allocator = PortAllocator(gparray_mock, self.logger, verify_ports=False) + + new_host = Host(hostname='host2', address='10.0.0.2', status=HostStatus.NEW) + + # Allocate first mirror on new host + allocated_port = allocator.allocate_port(new_host, current_port=7000, is_mirror=True) + + self.assertEqual(allocated_port, 7000) + self.assertEqual(allocator.base_ports_by_host['host2'], (None, 7000)) + self.assertIn(7000, allocator.planned_ports_by_host['host2']) + + def test_allocate_port_new_host_subsequent_primaries(self): + """ + Test allocating subsequent primary ports on new host follow pattern + """ + segments = [ + self._create_mock_segment(1, 0, 'host1', 6000, 'p'), + ] + + gparray_mock = self._create_mock_gparray(segments) + allocator = PortAllocator(gparray_mock, self.logger, verify_ports=False) + + new_host = Host(hostname='host2', address='10.0.0.2', status=HostStatus.NEW) + + # Allocate first primary + port1 = allocator.allocate_port(new_host, current_port=6000, is_mirror=False) + self.assertEqual(port1, 6000) + + # Allocate second primary - should follow pattern + port2 = allocator.allocate_port(new_host, current_port=6001, is_mirror=False) + self.assertEqual(port2, 6001) + + # Allocate third primary + port3 = allocator.allocate_port(new_host, current_port=6002, is_mirror=False) + self.assertEqual(port3, 6002) + + def test_allocate_port_new_host_mirrors_separate_from_primaries(self): + """ + Test that mirrors and primaries maintain separate port ranges on new host + """ + segments = [] + + gparray_mock = self._create_mock_gparray(segments) + allocator = PortAllocator(gparray_mock, self.logger, verify_ports=False) + + new_host = Host(hostname='host1', address='10.0.0.1', status=HostStatus.NEW) + + # Allocate primaries starting at 6000 + port_p1 = allocator.allocate_port(new_host, current_port=6000, is_mirror=False) + port_p2 = allocator.allocate_port(new_host, current_port=6001, is_mirror=False) + + # Allocate mirrors starting at 7000 + port_m1 = allocator.allocate_port(new_host, current_port=7000, is_mirror=True) + port_m2 = allocator.allocate_port(new_host, current_port=7001, is_mirror=True) + + self.assertEqual(port_p1, 6000) + self.assertEqual(port_p2, 6001) + self.assertEqual(port_m1, 7000) + self.assertEqual(port_m2, 7001) + + # Verify base ports + self.assertEqual(allocator.base_ports_by_host['host1'], (6000, 7000)) + + def test_is_port_available(self): + """ + Test port availability checking + """ + segments = [ + self._create_mock_segment(1, 0, 'host1', 6000, 'p'), + ] + + gparray_mock = self._create_mock_gparray(segments) + allocator = PortAllocator(gparray_mock, self.logger) + + # Port 6000 is used + self.assertFalse(allocator._is_port_available('host1', 6000)) + + # Port 6001 is free + self.assertTrue(allocator._is_port_available('host1', 6001)) + + # Mark 6001 as planned + allocator.planned_ports_by_host['host1'].add(6001) + self.assertFalse(allocator._is_port_available('host1', 6001)) + + @patch('gprebalance_modules.planner.PortIsAvailable') + def test_check_port_on_host_available(self, mock_port_is_available): + """Test actual port verification on host when port is available""" + segments = [] + gparray_mock = self._create_mock_gparray(segments) + allocator = PortAllocator(gparray_mock, self.logger, verify_ports=True) + + # Mock PortIsAvailable command + mock_cmd = Mock() + mock_cmd.is_port_available.return_value = True + mock_port_is_available.return_value = mock_cmd + + host = Host(hostname='host1', address='10.0.0.1', status=HostStatus.NEW) + result = allocator._check_port_on_host(host, 6000) + + self.assertTrue(result) + mock_cmd.run.assert_called_once() + + @patch('gprebalance_modules.planner.PortIsAvailable') + def test_check_port_on_host_in_use(self, mock_port_is_available): + """Test actual port verification when port is in use""" + segments = [] + gparray_mock = self._create_mock_gparray(segments) + allocator = PortAllocator(gparray_mock, self.logger, verify_ports=True) + + # Mock PortIsAvailable command + mock_cmd = Mock() + mock_cmd.is_port_available.return_value = False + mock_port_is_available.return_value = mock_cmd + + host = Host(hostname='host1', address='10.0.0.1', status=HostStatus.NEW) + result = allocator._check_port_on_host(host, 6000) + + self.assertFalse(result) + + @patch('gprebalance_modules.planner.PortIsAvailable') + def test_check_port_on_host_verification_disabled(self, mock_port_is_available): + """Test that verification is skipped when disabled""" + segments = [] + gparray_mock = self._create_mock_gparray(segments) + allocator = PortAllocator(gparray_mock, self.logger, verify_ports=False) + + host = Host(hostname='host1', address='10.0.0.1', status=HostStatus.NEW) + result = allocator._check_port_on_host(host, 6000) + + # Should return True without checking + self.assertTrue(result) + mock_port_is_available.assert_not_called() + + @patch('gprebalance_modules.planner.PortIsAvailable') + def test_verify_and_allocate_port_preferred_available(self, mock_port_is_available): + """Test verify_and_allocate when preferred port is available""" + segments = [] + gparray_mock = self._create_mock_gparray(segments) + allocator = PortAllocator(gparray_mock, self.logger, verify_ports=True) + + # Mock port as available + mock_cmd = Mock() + mock_cmd.is_port_available.return_value = True + mock_port_is_available.return_value = mock_cmd + + host = Host(hostname='host1', address='10.0.0.1', status=HostStatus.NEW) + allocated = allocator._verify_and_allocate_port(host, 6000) + + self.assertEqual(allocated, 6000) + + @patch('gprebalance_modules.planner.PortIsAvailable') + def test_verify_and_allocate_port_preferred_in_use(self, mock_port_is_available): + """ + Test verify_and_allocate when preferred port is in use + """ + segments = [] + gparray_mock = self._create_mock_gparray(segments) + allocator = PortAllocator(gparray_mock, self.logger, verify_ports=True) + + # Mock first port as in use, second as available + def mock_is_available_side_effect(): + call_count = [0] + def side_effect(): + call_count[0] += 1 + return call_count[0] > 1 + return side_effect + + mock_cmd = Mock() + mock_cmd.is_port_available.side_effect = mock_is_available_side_effect() + mock_port_is_available.return_value = mock_cmd + + host = Host(hostname='host1', address='10.0.0.1', status=HostStatus.NEW) + allocated = allocator._verify_and_allocate_port(host, 6000) + + # Should get next port + self.assertEqual(allocated, 6001) + self.assertIn(6000, allocator.existing_ports_by_host['host1']) # Marked as used + + def test_find_next_available_port_with_base(self): + """Test finding next available port with established base""" + segments = [ + self._create_mock_segment(1, 0, 'host1', 6000, 'p'), + self._create_mock_segment(2, 1, 'host1', 6001, 'p'), + ] + + gparray_mock = self._create_mock_gparray(segments) + allocator = PortAllocator(gparray_mock, self.logger, verify_ports=False) + + host = Host(hostname='host1', address='10.0.0.1', status=HostStatus.ACTIVE) + + # Find next primary port (base is 6000) + next_port = allocator._find_next_available_port(host, 6000, is_mirror=False) + + # Should skip 6000, 6001 and return 6002 + self.assertEqual(next_port, 6002) + + def test_find_verified_port_max_attempts(self): + """Test that finding port fails after max attempts""" + segments = [] + gparray_mock = self._create_mock_gparray(segments) + allocator = PortAllocator(gparray_mock, self.logger, verify_ports=False) + + host = Host(hostname='host1', address='10.0.0.1', status=HostStatus.NEW) + + # Fill up many ports + for i in range(1000): + allocator.existing_ports_by_host['host1'].add(6000 + i) + + # Should raise error after max attempts + with self.assertRaises(PlanningError) as ctx: + allocator._find_verified_port(host, 6000) + + self.assertIn("Cannot find available port", str(ctx.exception)) + self.assertIn("after 1000 attempts", str(ctx.exception)) + + def test_complex_scenario_mixed_roles(self): + """ + Test complex scenario with primaries and mirrors on same host + """ + segments = [ + # Host1: primaries on 6000-6002 + self._create_mock_segment(1, 0, 'host1', 6000, 'p'), + self._create_mock_segment(2, 1, 'host1', 6001, 'p'), + self._create_mock_segment(3, 2, 'host1', 6002, 'p'), + # Host1: mirrors on 7000-7001 + self._create_mock_segment(4, 3, 'host1', 7000, 'm'), + self._create_mock_segment(5, 4, 'host1', 7001, 'm'), + ] + + gparray_mock = self._create_mock_gparray(segments) + allocator = PortAllocator(gparray_mock, self.logger, verify_ports=False) + + host1 = Host(hostname='host1', address='10.0.0.1', status=HostStatus.ACTIVE) + + # Allocate new primary (should get 6003) + port_p = allocator.allocate_port(host1, current_port=6003, is_mirror=False) + self.assertEqual(port_p, 6003) + + # Allocate new mirror (should get 7002) + port_m = allocator.allocate_port(host1, current_port=7002, is_mirror=True) + self.assertEqual(port_m, 7002) + + # Verify role separation maintained + primary_ports, mirror_ports = allocator.existing_ports_by_role['host1'] + self.assertIn(6003, primary_ports) + self.assertIn(7002, mirror_ports) + + def test_empty_gparray(self): + """Test initialization with empty gparray""" + segments = [] + gparray_mock = self._create_mock_gparray(segments) + allocator = PortAllocator(gparray_mock, self.logger) + + self.assertEqual(len(allocator.existing_ports_by_host), 0) + self.assertEqual(len(allocator.base_ports_by_host), 0) + + # Should still be able to allocate on new host + new_host = Host(hostname='host1', address='10.0.0.1', status=HostStatus.NEW) + port = allocator.allocate_port(new_host, current_port=6000, is_mirror=False) + self.assertEqual(port, 6000) + +class TestPortAllocatorIntegration(GpTestCase): + """Integration tests simulating realistic rebalance scenarios""" + + def setUp(self): + self.logger = Mock() + + def _create_segment(self, dbid, content, hostname, port, role='p'): + """Helper to create mock segment""" + seg = Mock(spec=gparray.Segment) + seg.dbid = dbid + seg.content = content + seg.hostname = hostname + seg.datadir = f'/data{content}/seg{content}' + seg.port = port + seg.role = role + seg.preferred_role = role + + seg.getSegmentDbId.return_value = dbid + seg.getSegmentContentId.return_value = content + seg.getSegmentHostName.return_value = hostname + seg.getSegmentDataDirectory.return_value = seg.datadir + seg.getSegmentPort.return_value = port + seg.isSegmentPrimary.return_value = (role == 'p') + seg.isSegmentMirror.return_value = (role == 'm') + + return seg + + def test_expansion_scenario(self): + """Test port allocation during cluster expansion (add new host)""" + # Initial: 2 hosts with 2 primaries each + segments = [ + self._create_segment(1, 0, 'host1', 6000, 'p'), + self._create_segment(2, 1, 'host1', 6001, 'p'), + self._create_segment(3, 0, 'host2', 7000, 'm'), + self._create_segment(4, 1, 'host2', 7001, 'm'), + ] + + gparray_mock = Mock(spec=gparray.GpArray) + gparray_mock.getSegDbList.return_value = segments + + allocator = PortAllocator(gparray_mock, self.logger, verify_ports=False) + + # Add new host3 - move some mirrors there + new_host = Host(hostname='host3', address='10.0.0.3', status=HostStatus.NEW) + + # Move mirror of content 0 to new host + port1 = allocator.allocate_port(new_host, current_port=7000, is_mirror=True) + self.assertEqual(port1, 7000) # Establishes base + + # Move mirror of content 1 to new host + port2 = allocator.allocate_port(new_host, current_port=7001, is_mirror=True) + self.assertEqual(port2, 7001) # Follows pattern + + def test_shrink_scenario(self): + """Test port allocation during cluster shrink (remove host)""" + # Initial: 3 hosts + segments = [ + self._create_segment(1, 0, 'host1', 6000, 'p'), + self._create_segment(2, 1, 'host2', 6000, 'p'), + self._create_segment(3, 2, 'host3', 6000, 'p'), + self._create_segment(4, 0, 'host2', 7000, 'm'), + self._create_segment(5, 1, 'host3', 7000, 'm'), + self._create_segment(6, 2, 'host1', 7000, 'm'), + ] + + gparray_mock = Mock(spec=gparray.GpArray) + gparray_mock.getSegDbList.return_value = segments + + allocator = PortAllocator(gparray_mock, self.logger, verify_ports=False) + + # Remove host3 - redistribute its segments + host1 = Host(hostname='host1', address='10.0.0.1', status=HostStatus.ACTIVE) + host2 = Host(hostname='host2', address='10.0.0.2', status=HostStatus.ACTIVE) + + # Move content 2 primary from host3 to host1 + port_p = allocator.allocate_port(host1, current_port=6000, is_mirror=False) + self.assertEqual(port_p, 6001) # 6000 in use, get next + + # Move content 2 mirror from host1 to host2 + port_m = allocator.allocate_port(host2, current_port=7000, is_mirror=True) + self.assertEqual(port_m, 7001) # 7000 in use, get next + + def test_rebalance_after_failure(self): + """Test rebalance scenario after segment failure and recovery""" + # Scenario: segment recovered with different port, need to rebalance + segments = [ + self._create_segment(1, 0, 'host1', 6000, 'p'), + self._create_segment(2, 1, 'host1', 6001, 'p'), + self._create_segment(3, 2, 'host1', 6002, 'p'), + self._create_segment(4, 0, 'host2', 7000, 'm'), + self._create_segment(5, 1, 'host2', 7001, 'm'), + self._create_segment(6, 2, 'host2', 7999, 'm'), # Recovered with odd port + ] + + gparray_mock = Mock(spec=gparray.GpArray) + gparray_mock.getSegDbList.return_value = segments + + allocator = PortAllocator(gparray_mock, self.logger, verify_ports=False) + + # Rebalance: move content 2 mirror back to proper port + host2 = Host(hostname='host2', address='10.0.0.2', status=HostStatus.ACTIVE) + + # Try to use 7002 (following pattern) + port = allocator.allocate_port(host2, current_port=7002, is_mirror=True) + self.assertEqual(port, 7002) # Should be available + +if __name__ == '__main__': + run_tests() diff --git a/gpMgmt/bin/gprebalance_modules/test/test_unit_solver.py b/gpMgmt/bin/gprebalance_modules/test/test_unit_solver.py new file mode 100644 index 000000000000..2cc1cd3cbe83 --- /dev/null +++ b/gpMgmt/bin/gprebalance_modules/test/test_unit_solver.py @@ -0,0 +1,502 @@ +import random +from collections import defaultdict + +from gppylib.test.unit.gp_unittest import * +from ..solver import GreedySolver, Solution, SolverConfig +from .config import getEncoding + + +class TestGreedySolver(GpTestCase): + + def setUp(self): + random.seed(42) + + def _validate_solution_allassign(self, solution: Solution, + solver: GreedySolver) -> bool: + if len(solution) != solver.n_segments: + return False + + return True + + def _validate_solution_host_ids(self, solution: Solution, + solver: GreedySolver) -> bool: + for seg_id, (primary, mirror) in solution.items(): + if not (0 <= primary < solver.n_hosts_target): + return False + if not (0 <= mirror < solver.n_hosts_target): + return False + + return True + + def _validate_solution_nocolocation(self, solution: Solution, + solver: GreedySolver) -> bool: + for seg_id, (primary, mirror) in solution.items(): + if primary == mirror: + return False + return True + + def _validate_solution_balance(self, solution: Solution, + solver: GreedySolver) -> bool: + load = [0] * solver.n_hosts_target + for seg_id, (primary, mirror) in solution.items(): + load[primary] += 1 + load[mirror] += 1 + + for host, host_load in enumerate(load): + if host_load != solver.target_load: + return False + + return True + + def _validate_strategy(self, solution: Solution, + solver: GreedySolver): + primary_to_mirrors = defaultdict(list) + for seg_id, (primary, mirror) in solution.items(): + primary_to_mirrors[primary].append(mirror) + + for primary, mirrors in primary_to_mirrors.items(): + if solver.strategy == 'grouped': + unique_mirrors = set(mirrors) + if len(unique_mirrors) > 1: + return False + elif solver.strategy == 'spread': + unique_mirrors = set(mirrors) + if len(mirrors) != len(unique_mirrors): + return False + return True + + def _validate_solition(self, solution: Solution, + solver: GreedySolver): + self.assertTrue(self._validate_solution_allassign(solution, solver), + f"Missing segments. Expected {solver.n_segments}, got {len(solution)}") + + self.assertTrue(self._validate_solution_host_ids(solution, solver), + "Some segments has impossible host assignment") + + self.assertTrue(self._validate_solution_nocolocation(solution, solver), + "Some segment has primary==mirror assignment") + + self.assertTrue(self._validate_solution_balance(solution, solver), + f"One of hosts has load different from expected {solver.target_load}") + + self.assertTrue(self._validate_strategy(solution, solver), + "Grouped mirroring strategy is violated") + + def perform_run(self, run_improve:bool, cost:int): + conf = self.encoding[0] + solver = GreedySolver(conf, run_improve=run_improve) + solution, actual_cost = solver.solve() + self.assertEqual(actual_cost, cost) + self._validate_solition(solution, solver) + + @getEncoding('35_7_balanced_grouped', 'grouped', None, None, None) + def test_validity_small_grouped_balanced(self): + self.perform_run(False, 0) + + @getEncoding('35_7_balanced_spread', 'spread', None, None, None) + def test_validity_small_spread_balanced(self): + self.perform_run(False, 0) + + @getEncoding('40_5_unbalanced_grouped', 'grouped', None, None, None) + def test_validity_small_grouped_unbalanced(self): + self.perform_run(False, 19) + + @getEncoding('40_5_unbalanced_grouped', 'grouped', None, None, None) + def test_validity_small_grouped_unbalanced_with_improve(self): + self.perform_run(True, 18) + + @getEncoding('40_5_unbalanced_spread', 'spread', None, None, None) + def test_validity_small_spread_neg(self): + conf = self.encoding[0] + + with self.assertRaises(ValueError, msg='Cannot follow spread mirroring strategy') as cm: + solver = GreedySolver(conf, run_improve=False) + + @getEncoding('40_8_unbalanced_spread', 'spread', None, None, None) + def test_validity_small_spread_unbalanced(self): + self.perform_run(False, 10) + + @getEncoding('40_8_unbalanced_spread', 'spread', None, None, None) + def test_validity_small_spread_unbalanced_with_improve(self): + self.perform_run(True, 6) + + @getEncoding('120_20_unbalanced_spread', 'spread', None, None, None) + def test_validity_medium_spread_unbalanced(self): + self.perform_run(False, 10) + + @getEncoding('120_20_unbalanced_spread', 'spread', None, None, None) + def test_validity_medium_spread_unbalanced_with_improve(self): + self.perform_run(True, 7) + + @getEncoding('1000_50_unbalanced_spread', 'spread', None, None, None) + def test_validity_large_spread_unbalanced(self): + self.perform_run(False, 124) + + @getEncoding('1000_50_unbalanced_spread', 'spread', None, None, None) + def test_validity_large_spread_unbalanced_with_improve(self): + # in more or less standart configurations with lightly skewed + # distribution greedy initial solution is pretty-well generated. + # it's expected that ALNS may not bring any impovements. + self.perform_run(True, 124) + + @getEncoding('1000_50_unbalanced_grouped', 'grouped', None, None, None) + def test_validity_large_grouped_unbalanced(self): + self.perform_run(False, 140) + + @getEncoding('1000_50_unbalanced_grouped', 'grouped', None, None, None) + def test_validity_large_grouped_unbalanced_with_improve(self): + # in standart configurations with lightly skewed + # distribution greedy initial solution is pretty-well generated. + # it's expected that ALNS may not bring any impovements. + self.perform_run(True, 140) + + @getEncoding('120_20_unbalanced_grouped', 'spread', None, None, None) + def test_strategy_change_medium_grouped_unbalanced(self): + self.perform_run(False, 101) + + @getEncoding('120_20_unbalanced_grouped', 'spread', None, None, None) + def test_strategy_change_medium_grouped_unbalanced_with_improve(self): + self.perform_run(True, 100) + + @getEncoding('120_20_unbalanced_spread', 'grouped', None, None, None) + def test_strategy_change_medium_spread_unbalanced(self): + self.perform_run(False, 106) + + @getEncoding('120_20_unbalanced_spread', 'grouped', None, None, None) + def test_strategy_change_medium_spread_unbalanced_with_improve(self): + self.perform_run(True, 102) + + @getEncoding('120_20_unbalanced_grouped', 'grouped', target_hosts=None, + add_hosts=None, remove_hosts="sdw13, sdw14, sdw15, sdw16, sdw17, sdw18, sdw19, sdw20") + def test_decomission_hosts(self): + self.perform_run(False, 116) + + @getEncoding('120_20_unbalanced_grouped', 'grouped', target_hosts=None, + add_hosts=None, remove_hosts="sdw13, sdw14, sdw15, sdw16, sdw17, sdw18, sdw19, sdw20") + def test_decomission_hosts_with_improve(self): + self.perform_run(True, 109) + + @getEncoding('1000_50_unbalanced_grouped', 'grouped', target_hosts=None, + add_hosts=None, remove_hosts=",".join(["sdw" + str(i) for i in range(20, 30)])) + def test_decomission_hosts_large(self): + self.perform_run(False, 470) + + @getEncoding('1000_50_unbalanced_grouped', 'grouped', target_hosts=None, + add_hosts=None, remove_hosts=",".join(["sdw" + str(i) for i in range(20, 30)])) + def test_decomission_hosts_large_with_improve(self): + self.perform_run(True, 457) + + @getEncoding('1000_50_balanced_grouped', 'grouped', target_hosts=None, + add_hosts=",".join(["sdw" + str(i) for i in range(51, 101)]), + remove_hosts=None) + def test_new_hosts_balanced(self): + self.perform_run(False, 1000) + + @getEncoding('1000_50_balanced_grouped', 'grouped', target_hosts=None, + add_hosts=",".join(["sdw" + str(i) for i in range(51, 101)]), + remove_hosts=None) + def test_new_hosts_balanced_with_improve(self): + self.perform_run(True, 1000) + + @getEncoding('120_20_unbalanced_grouped', 'grouped', target_hosts= + "sdw1, sdw2, sdw3, sdw4, sdw5, sdw21, sdw22, sdw23, sdw24, sdw25, sdw12, sdw13", + add_hosts=None, remove_hosts=None) + def test_target_hosts(self): + self.perform_run(False, 175) + + @getEncoding('120_20_unbalanced_grouped', 'grouped', target_hosts= + "sdw1, sdw2, sdw3, sdw4, sdw5, sdw21, sdw22, sdw23, sdw24, sdw25, sdw12, sdw13", + add_hosts=None, remove_hosts=None) + def test_target_hosts_with_improve(self): + self.perform_run(True, 160) + +class TestGreedySolverFunc(GpTestCase): + """ + Test GreedySolver class. + """ + + def test_init_single_host_error(self): + config = SolverConfig( + n_segments=4, + n_hosts_target=1, # Invalid! + n_hosts_initial=2, + initial_primary_mapping=[0, 0, 1, 1], + initial_mirror_mapping=[1, 1, 0, 0], + strategy='grouped' + ) + with self.assertRaises(ValueError) as ctx: + GreedySolver(config, run_improve=False) + self.assertIn("Cannot balance to single host", str(ctx.exception)) + + def test_init_uneven_distribution_error(self): + config = SolverConfig( + n_segments=13, + n_hosts_target=3, + n_hosts_initial=3, + initial_primary_mapping=[0]*13, + initial_mirror_mapping=[1]*13, + strategy='grouped' + ) + with self.assertRaises(ValueError) as ctx: + GreedySolver(config, run_improve=False) + self.assertIn("Cannot evenly distribute", str(ctx.exception)) + + def test_init_spread_impossible_error(self): + config = SolverConfig( + n_segments=12, # 12/3 = 4 per host, but spread needs max 2 (3-1) + n_hosts_target=3, + n_hosts_initial=3, + initial_primary_mapping=[0]*12, + initial_mirror_mapping=[1]*12, + strategy='spread' + ) + with self.assertRaises(ValueError) as ctx: + GreedySolver(config, run_improve=False) + self.assertIn("Cannot follow spread mirroring strategy", str(ctx.exception)) + + def test_balance_primaries_keep_on_original(self): + config = SolverConfig( + n_segments=6, + n_hosts_target=3, + n_hosts_initial=3, + initial_primary_mapping=[0, 0, 1, 1, 2, 2], # Already balanced + initial_mirror_mapping=[1, 1, 2, 2, 0, 0], + strategy='grouped' + ) + solver = GreedySolver(config, run_improve=False) + primary = solver._balance_primaries() + + # Should keep original placements + self.assertEqual(primary, [0, 0, 1, 1, 2, 2]) + + def test_balance_primaries_move_from_decommissioned(self): + """ + Test that _balance_primaries keeps valid placements. + """ + config_grouped = SolverConfig( + n_segments=12, + n_hosts_target=3, + n_hosts_initial=4, + initial_primary_mapping=[0, 1, 2, 0, 1, 2, 0, 1, 2, 3, 3, 3], + initial_mirror_mapping=[1, 2, 0, 1, 2, 0, 1, 2, 0, 0, 1, 2], + strategy='grouped' + ) + + solver = GreedySolver(config_grouped, run_improve=False) + primary_mapping = solver._balance_primaries() + + # Check all segments assigned + self.assertEqual(len(primary_mapping), 12) + self.assertTrue(all( p in [0, 1, 2] + for p in primary_mapping)) + + def test_balance_primaries_move_from_overloaded(self): + config = SolverConfig( + n_segments=12, + n_hosts_target=3, + n_hosts_initial=3, + initial_primary_mapping=[0]*8 + [1]*2 + [2]*2, # Host 0 overloaded + initial_mirror_mapping=[1]*8 + [2]*2 + [0]*2, + strategy='grouped' + ) + solver = GreedySolver(config, run_improve=False) + primary = solver._balance_primaries() + + # Should balance to 4 per host + primary_count = defaultdict(int) + for p in primary: + primary_count[p] += 1 + + for h in range(3): + self.assertEqual(primary_count[h], 4) + + def test_assign_mirrors_grouped_strategy_balanced(self): + config = SolverConfig( + n_segments=12, + n_hosts_target=3, + n_hosts_initial=3, + initial_primary_mapping=[0]*4 + [1]*4 + [2]*4, + initial_mirror_mapping=[1]*4 + [2]*4 + [0]*4, + strategy='grouped' + ) + solver = GreedySolver(config, run_improve=False) + primary = solver._balance_primaries() + mirror = solver._assign_mirror_hosts(primary) + + self.assertEqual(primary, config.initial_primary_mapping) + self.assertEqual(mirror, config.initial_mirror_mapping) + + def test_assign_mirrors_spread_strategy_balanced(self): + config = SolverConfig( + n_segments=8, + n_hosts_target=4, + n_hosts_initial=4, + initial_primary_mapping=[0, 0, 1, 1, 2, 2, 3, 3], + initial_mirror_mapping=[1, 2, 2, 3, 3, 0, 0, 1], + strategy='spread' + ) + solver = GreedySolver(config, run_improve=False) + primary = solver._balance_primaries() + mirror = solver._assign_mirror_hosts(primary) + + self.assertEqual(primary, config.initial_primary_mapping) + self.assertEqual(mirror, config.initial_mirror_mapping) + + def test_select_group_mirror_priority1_most_used(self): + """ most-used original mirror with capacity""" + config = SolverConfig( + n_segments=12, + n_hosts_target=3, + n_hosts_initial=3, + initial_primary_mapping=[0, 0, 0, 0, 0, 2, 2, 2, 2, 1, 1, 1], + initial_mirror_mapping=[1, 1, 1, 2, 2, 1, 1, 2, 2, 0, 0, 0], + strategy='grouped' + ) + solver = GreedySolver(config, run_improve=False) + + mirror_mapping = [-1] * 12 + mirror_load = [0] * 3 + groups = {0: [0, 1, 2, 3], 1: [4, 5, 6, 7], 2: [8, 9, 10, 11]} + phost_to_mhost = {} + + preferences = solver._compute_mirror_preferences(groups) + + best = solver._select_group_mirror( + p_host=0, + mirror_mapping=mirror_mapping, + mirror_load=mirror_load, + phost_to_mhost=phost_to_mhost, + groups=groups, + preferences=preferences + ) + + # Should select most-preferred mirror (host 1) + self.assertEqual(best, 1) + + def test_select_group_mirror_priority2_least_loaded(self): + config = SolverConfig( + n_segments=12, + n_hosts_target=3, + n_hosts_initial=4, # Host 3 decommissioned + initial_primary_mapping=[0]*4 + [1]*4 + [2]*4, + initial_mirror_mapping=[3]*12, # All on decommissioned host + strategy='grouped' + ) + solver = GreedySolver(config, run_improve=False) + + mirror_mapping = [-1] * 12 + mirror_load = [2, 0, 1] # Host 1 is least loaded + groups = {0: [0, 1, 2, 3], 1: [4, 5, 6, 7], 2: [8, 9, 10, 11]} + phost_to_mhost = {} + + preferences = solver._compute_mirror_preferences(groups) + + best = solver._select_group_mirror( + p_host=0, + mirror_mapping=mirror_mapping, + mirror_load=mirror_load, + phost_to_mhost=phost_to_mhost, + groups=groups, + preferences=preferences + ) + + # Should select least loaded (host 1) + self.assertEqual(best, 1) + + def test_select_group_mirror_priority3_deadlock_swap(self): + config = SolverConfig( + n_segments=8, + n_hosts_target=4, + n_hosts_initial=4, + initial_primary_mapping=[0]*2 + [1]*2 + [2]*2 + [3]*2,# does not matter + initial_mirror_mapping=[1]*4 + [0]*4,# does not matter + strategy='grouped' + ) + solver = GreedySolver(config, run_improve=False) + + # Simulate deadlock: all mirror hosts are full, except the host 0. + # Need swap in case of such mapping where primary group at host 0 + # has no options to put the mirrors + mirror_mapping = [2, 2, 3, 3, 1, 1, -1, -1] + mirror_load = [0, 2, 2, 2] # Both full + groups = {0: [6, 7], 1: [0, 1], 2: [2, 3], 3: [4, 5]} + phost_to_mhost = {1: 2, 2: 3, 3: 1} + + best = solver._swap_to_resolve_deadlock( + blocked_p_host=0, + mirror_load=mirror_load, + phost_to_mhost=phost_to_mhost, + groups=groups, + mirror_mapping=mirror_mapping + ) + + # This tests the deadlock branch is reached + self.assertIsNotNone(best) + + def test_spread_use_original_mirror(self): + config = SolverConfig( + n_segments=8, + n_hosts_target=4, + n_hosts_initial=4, + initial_primary_mapping=[0, 0, 1, 1, 2, 2, 3, 3], + initial_mirror_mapping=[1, 2, 2, 3, 3, 0, 0, 1], # Valid originals + strategy='spread' + ) + solver = GreedySolver(config, run_improve=False) + primary_mapping = solver._balance_primaries() + mirror_mapping = solver._assign_mirror_hosts(primary_mapping) + + self.assertEqual(primary_mapping, config.initial_primary_mapping) + self.assertEqual(mirror_mapping, config.initial_mirror_mapping) + + def test_spread_find_alternative(self): + config = SolverConfig( + n_segments=8, + n_hosts_target=4, + n_hosts_initial=5, + initial_primary_mapping=[0, 0, 1, 1, 2, 2, 3, 3], + initial_mirror_mapping=[4, 4, 4, 4, 4, 4, 4, 4], # All on decommissioned + strategy='spread' + ) + solver = GreedySolver(config, run_improve=False) + primary_mapping = solver._balance_primaries() + mirror_mapping = solver._assign_mirror_hosts(primary_mapping) + + # Should assign to valid hosts + for m in mirror_mapping: + self.assertLess(m, 4) + + def test_spread_deadlock_swap_found(self): + config = SolverConfig( + n_segments=8, + n_hosts_target=4, + n_hosts_initial=5, + initial_primary_mapping=[0, 0, 1, 1, 2, 2, 3, 3], + initial_mirror_mapping=[4, 4, 4, 4, 4, 4, 4, 4],# All on decommissioned + strategy='spread' + ) + solver = GreedySolver(config, run_improve=False) + + primary_mapping = [0, 0, 1, 1, 2, 2, 3, 3] + mirror_mapping = [1, 2, 2, 1, 3, 0, 0, -1] + mirror_load = [2, 2, 2, 1] + primary_host_to_mirror_hosts = { + 0: {1, 2}, + 1: {1, 2}, + 2: {3, 0}, + 3: {0} + } + + result = solver._resolve_spread_deadlock_for_segment( + seg=7, + primary_mapping=primary_mapping, + mirror_mapping=mirror_mapping, + mirror_load=mirror_load, + used_in_group=primary_host_to_mirror_hosts + ) + + # Must find a swap + self.assertEqual(1, result) + +if __name__ == '__main__': + run_tests() diff --git a/gpMgmt/test/behave/mgmt_utils/environment.py b/gpMgmt/test/behave/mgmt_utils/environment.py index 43acb7c40306..90e1f06cd948 100644 --- a/gpMgmt/test/behave/mgmt_utils/environment.py +++ b/gpMgmt/test/behave/mgmt_utils/environment.py @@ -22,7 +22,7 @@ def before_feature(context, feature): # we should be able to run gpexpand without having a cluster initialized tags_to_skip = ['gpexpand', 'gpaddmirrors', 'gpstate', 'gpmovemirrors', 'gpconfig', 'gpssh-exkeys', 'gpstop', 'gpinitsystem', 'cross_subnet', - 'gplogfilter'] + 'gplogfilter', 'ggrebalance_basics', 'ggrebalance_shrink', 'ggrebalance_rebalance', 'ggrebalance_misc_options'] if set(context.feature.tags).intersection(tags_to_skip): return @@ -125,7 +125,7 @@ def before_scenario(context, scenario): tags_to_skip = ['gpexpand', 'gpaddmirrors', 'gpstate', 'gpmovemirrors', 'gpconfig', 'gpssh-exkeys', 'gpstop', 'gpinitsystem', 'cross_subnet', - 'gplogfilter'] + 'gplogfilter', 'ggrebalance_basics', 'ggrebalance_shrink', 'ggrebalance_rebalance', 'ggrebalance_misc_options'] if set(context.feature.tags).intersection(tags_to_skip): return @@ -158,7 +158,7 @@ def after_scenario(context, scenario): # NOTE: gpconfig after_scenario cleanup is in the step `the gpconfig context is setup` tags_to_skip = ['gpexpand', 'gpaddmirrors', 'gpinitstandby', 'gpconfig', 'gpstop', 'gpinitsystem', 'cross_subnet', - 'gplogfilter'] + 'gplogfilter', 'ggrebalance_basics', 'ggrebalance_shrink', 'ggrebalance_rebalance', 'ggrebalance_misc_options'] if set(context.feature.tags).intersection(tags_to_skip): return diff --git a/gpMgmt/test/behave/mgmt_utils/ggrebalance_basics.feature b/gpMgmt/test/behave/mgmt_utils/ggrebalance_basics.feature new file mode 100755 index 000000000000..48d7f40339ca --- /dev/null +++ b/gpMgmt/test/behave/mgmt_utils/ggrebalance_basics.feature @@ -0,0 +1,98 @@ +@ggrebalance_basics +Feature: ggrebalance behave tests + + Scenario Outline: test 1. validate incompatible option combinations + Given a standard local demo cluster is running + And the environment variable "COORDINATOR_DATA_DIRECTORY" is set from output of "echo $(dirname $(pwd))/gpAux/gpdemo/datadirs/qddir/demoDataDir-1" + And coordinator data directory is updated + When the user runs "ggrebalance " + Then ggrebalance should return a return code of 1 + And ggrebalance should print "" to stdout + Given + Then + And coordinator data directory is updated + + Examples: Mutually exclusive options + | options | error_message | run_status | restore_env| + | --target-hosts sdw1,sdw2 --target-hosts-file /tmp/hosts.txt | Can't use together options '--target-hosts' and '--target-hosts-file' | stub |stub| + | --target-hosts sdw1,sdw2 --add-hosts sdw3 | Can't use together options '--target-hosts' and '--add-hosts' | stub |stub| + | --target-hosts sdw1,sdw2 --remove-hosts sdw3 | Can't use together options '--target-hosts' and '--remove-hosts' | stub |stub| + | --add-hosts sdw3 --add-hosts-file /tmp/add.txt | Can't use together options '--add-hosts' and '--add-hosts-file' | stub |stub| + | --remove-hosts sdw3 --remove-hosts-file /tmp/rm.txt | Can't use together options '--remove-hosts' and '--remove-hosts-file' | stub |stub| + | --add-hosts sdw3 --remove-hosts sdw3 | Can't use together options '--add-hosts' and '--remove-hosts' | stub |stub| + | --target-datadirs '/data/p/gpseg{content},/data/m/gpseg{content}' --target-datadirs-file /tmp/datadirs.txt | Can't use together options '--target-datadirs' and '--target-datadirs-file' | stub |stub| + | --mirror-mode grouped --skip-rebalance | Can't use together options '--skip-rebalance' and '--mirror-mode' | stub |stub| + | -m spread --skip-rebalance | Can't use together options '--skip-rebalance' and '--mirror-mode' | stub |stub| + | --skip-rebalance --inplace-swap-roles | Can't use together options '--skip-rebalance' and '--inplace-swap-roles' | stub |stub| + | -c --target-hosts sdw1,sdw2 | Can't use together options '--clean-required' and '--target-hosts' | stub |stub| + | -c --add-hosts sdw3 | Can't use together options '--clean-required' and '--add-hosts' | stub |stub| + | -c --target-datadirs '/data/p/gpseg{content},/data/m/gpseg{content}' | Can't use together options '--clean-required' and '--target-datadirs' | stub |stub| + | -c -x 2 | Can't use together options '--clean-required' and '--target-segment-count' | stub |stub| + | -c --mirror-mode grouped | Can't use together options '--clean-required' and '--mirror-mode' | stub |stub| + | -c --skip-rebalance | Can't use together options '--clean-required' and '--skip-rebalance' | stub |stub| + | -c --show-plan | Can't use together options '--clean-required' and '--show-plan' | stub |stub| + | -c --analyze | Can't use together options '--clean-required' and '--analyze' | stub |stub| + | -c --replay-lag 1 | Can't use together options '--clean-required' and '--replay-lag' | stub |stub| + | -c --hba-hostnames | Can't use together options '--clean-required' and '--hba-hostnames' | stub |stub| + | -c --inplace-swap-roles | Can't use together options '--clean-required' and '--inplace-swap-roles' | stub |stub| + | -c --skip-resource-estimation | Can't use together options '--clean-required' and '--skip-resource-estimation' | stub |stub| + | -r --target-hosts sdw1,sdw2 | Can't use together options '--rollback-required' and '--target-hosts' | stub |stub| + | -r --add-hosts sdw3 | Can't use together options '--rollback-required' and '--add-hosts' | stub |stub| + | -r --target-datadirs '/data/p/gpseg{content},/data/m/gpseg{content}' | Can't use together options '--rollback-required' and '--target-datadirs' | stub |stub| + | -r -x 2 | Can't use together options '--rollback-required' and '--target-segment-count' | stub |stub| + | -r --mirror-mode grouped | Can't use together options '--rollback-required' and '--mirror-mode' | stub |stub| + | -r --skip-rebalance | Can't use together options '--rollback-required' and '--skip-rebalance' | stub |stub| + | -r --show-plan | Can't use together options '--rollback-required' and '--show-plan' | stub |stub| + | -r --inplace-swap-roles | Can't use together options '--rollback-required' and '--inplace-swap-roles' | stub |stub| + | -r --skip-resource-estimation | Can't use together options '--rollback-required' and '--skip-resource-estimation' |the database is not running |"COORDINATOR_DATA_DIRECTORY" environment variable should be restored| + + Scenario: test 2. ggrebalance simple scenarios + Given the database is not running + And a working directory of the test as '/data/gpdata/ggrebalance' + And a cluster is created with mirrors on "cdw" and "sdw1, sdw2" + And segment information for content 1 is saved in context + And segment information for content 2 is saved in context + And segment information for content 3 is saved in context + And all files in gpAdminLogs directory are deleted + When the user runs "ggrebalance -x 4 --skip-rebalance" + Then ggrebalance should return a return code of 0 + And ggrebalance should print "Skipping rebalance" to logfile with latest timestamp + When the user runs "ggrebalance -c" + Then ggrebalance should return a return code of 0 + When the user runs "ggrebalance -c" + Then ggrebalance should return a return code of 0 + And ggrebalance should print "Rebalance schema doesn't exist. Cleanup is not required." to logfile with latest timestamp + When the user runs "ggrebalance -r" + Then ggrebalance should return a return code of 0 + And ggrebalance should print "Rebalance schema doesn't exist. Can't perform rollback." to logfile with latest timestamp + When the user runs "ggrebalance -r" + Then ggrebalance should return a return code of 0 + And ggrebalance should print "Rebalance schema doesn't exist. Can't perform rollback." to logfile with latest timestamp + When the user runs "ggrebalance -x 2 --skip-rebalance" + Then ggrebalance should return a return code of 0 + And ggrebalance should print "Shrink is complete" to logfile with latest timestamp + And verify no segment running for saved segment information + When the user runs "ggrebalance -x 1 --skip-rebalance" + Then ggrebalance should return a return code of 0 + And ggrebalance should print "Previous run was completed successfully. Please execute cleanup before a new run." to logfile with latest timestamp + When the user runs "ggrebalance -r" + Then ggrebalance should return a return code of 0 + And ggrebalance should print "Previous run was completed successfully. Can't perform rollback." to logfile with latest timestamp + When the user runs "ggrebalance -c" + Then ggrebalance should return a return code of 0 + And ggrebalance should print "Cleanup is complete" to logfile with latest timestamp + + Scenario: test 3. check cleanup after the target segment count was updated + Given the database is not running + And a working directory of the test as '/data/gpdata/ggrebalance' + And a cluster is created with mirrors on "cdw" and "sdw1" + And all files in gpAdminLogs directory are deleted + And set fault inject "on_enter_STATE_BACKUP_CATALOG_AND_UPDATE_TARGET_SEGMENT_COUNT_STARTED_end" + When the user runs "ggrebalance -x 1 --skip-rebalance" + Then ggrebalance should return a return code of 1 + And ggrebalance should print "ggrebalance failed" to logfile with latest timestamp + And unset fault inject + When the user runs "ggrebalance -c -y" + Then ggrebalance should return a return code of 0 + And ggrebalance should print "Reset numsegments to default is done." to logfile with latest timestamp + And ggrebalance should print "Cleanup is complete" to logfile with latest timestamp diff --git a/gpMgmt/test/behave/mgmt_utils/ggrebalance_misc_options.feature b/gpMgmt/test/behave/mgmt_utils/ggrebalance_misc_options.feature new file mode 100755 index 000000000000..2ce9f0f7b1f8 --- /dev/null +++ b/gpMgmt/test/behave/mgmt_utils/ggrebalance_misc_options.feature @@ -0,0 +1,480 @@ +@ggrebalance_misc_options +Feature: ggrebalance behave tests (misc options scenarios) + + Scenario: test 1. Check if cluster has no mirroring. + Given the database is not running + And a working directory of the test as '/data/gpdata/ggrebalance' + And a cluster is created with no mirrors on "cdw" and "sdw1, sdw2, sdw3" + And all files in gpAdminLogs directory are deleted + When the user runs "ggrebalance -x 6 --remove-hosts sdw3 -d '/home/gpadmin/gpdb_src/gpAux/gpdemo/datadirs/dbfast, /home/gpadmin/gpdb_src/gpAux/gpdemo/datadirs/dbfast_mirror'" + Then ggrebalance should return a return code of 1 + And ggrebalance should print "Cluster has mirroring disabled. Can't proceed with rebalance" to logfile with latest timestamp + + Scenario: test 2. Check if cluster can't be rebalanced to a balanced state with given parameters. + Given the database is not running + And a working directory of the test as '/data/gpdata/ggrebalance' + And a cluster is created with mirrors on "cdw" and "sdw1, sdw2, sdw3" + And all files in gpAdminLogs directory are deleted + When the user runs "ggrebalance -x 5 --remove-hosts sdw3 -d '/home/gpadmin/gpdb_src/gpAux/gpdemo/datadirs/dbfast, /home/gpadmin/gpdb_src/gpAux/gpdemo/datadirs/dbfast_mirror'" + Then ggrebalance should return a return code of 1 + And ggrebalance should print "Cannot evenly distribute 5 segments across 2 hosts." to logfile with latest timestamp + When the user runs "ggrebalance -x 6 --remove-hosts 'sdw2, sdw3' -d '/home/gpadmin/gpdb_src/gpAux/gpdemo/datadirs/dbfast, /home/gpadmin/gpdb_src/gpAux/gpdemo/datadirs/dbfast_mirror'" + Then ggrebalance should return a return code of 1 + And ggrebalance should print "Cannot perform rebalance at 1 host" to logfile with latest timestamp + + Scenario: test 3. Check rebalance with 'grouped' mirror configuration. + Given the database is not running + And a working directory of the test as '/data/gpdata/ggrebalance' + And a cluster is created with mirrors on "cdw" and "sdw1, sdw2, sdw3", with 3 segments on each + And database "test_db_1" exists + And schema "test_schema_1" exists in "test_db_1" + And there is a "heap" table "test_schema_1.test_table_1" in "test_db_1" with "100" rows + And there is a "ao" table "test_schema_1.test_table_2" in "test_db_1" with "100" rows + And database "test_db_2" exists + And schema "test_schema_2" exists in "test_db_2" + And there is a "heap" table "test_schema_2.test_table_1" in "test_db_2" with "100" rows + And there is a "ao" table "test_schema_2.test_table_2" in "test_db_2" with "100" rows + And all files in gpAdminLogs directory are deleted + When the user runs "ggrebalance -x 6 --mirror-mode grouped -d '/home/gpadmin/gpdb_src/gpAux/gpdemo/datadirs/dbfast, /home/gpadmin/gpdb_src/gpAux/gpdemo/datadirs/dbfast_mirror'" + Then ggrebalance should return a return code of 0 + And ggrebalance should print "Rebalance is complete" to logfile with latest timestamp + And verify that mirror segments are in "group" configuration + And distribution information from table "test_schema_1.test_table_1" with data in "test_db_1" is equal to segment count = 6, row count = 100 + And distribution information from table "test_schema_1.test_table_2" with data in "test_db_1" is equal to segment count = 6, row count = 100 + And distribution information from table "test_schema_2.test_table_1" with data in "test_db_2" is equal to segment count = 6, row count = 100 + And distribution information from table "test_schema_2.test_table_2" with data in "test_db_2" is equal to segment count = 6, row count = 100 + + Scenario: test 4. Check rebalance with 'spread' mirror configuration. + Given the database is not running + And a working directory of the test as '/data/gpdata/ggrebalance' + And a cluster is created with mirrors on "cdw" and "sdw1, sdw2, sdw3", with 3 segments on each + And database "test_db_1" exists + And schema "test_schema_1" exists in "test_db_1" + And there is a "heap" table "test_schema_1.test_table_1" in "test_db_1" with "100" rows + And there is a "ao" table "test_schema_1.test_table_2" in "test_db_1" with "100" rows + And database "test_db_2" exists + And schema "test_schema_2" exists in "test_db_2" + And there is a "heap" table "test_schema_2.test_table_1" in "test_db_2" with "100" rows + And there is a "ao" table "test_schema_2.test_table_2" in "test_db_2" with "100" rows + And all files in gpAdminLogs directory are deleted + When the user runs "ggrebalance -x 6 --mirror-mode spread -d '/home/gpadmin/gpdb_src/gpAux/gpdemo/datadirs/dbfast, /home/gpadmin/gpdb_src/gpAux/gpdemo/datadirs/dbfast_mirror'" + Then ggrebalance should return a return code of 0 + And ggrebalance should print "Rebalance is complete" to logfile with latest timestamp + And verify that mirror segments are in "spread" configuration + And distribution information from table "test_schema_1.test_table_1" with data in "test_db_1" is equal to segment count = 6, row count = 100 + And distribution information from table "test_schema_1.test_table_2" with data in "test_db_1" is equal to segment count = 6, row count = 100 + And distribution information from table "test_schema_2.test_table_1" with data in "test_db_2" is equal to segment count = 6, row count = 100 + And distribution information from table "test_schema_2.test_table_2" with data in "test_db_2" is equal to segment count = 6, row count = 100 + + Scenario: test 5. Check rebalance against coordinator-only mode. + Given the database is not running + And the user runs "gpstart -ma" + And "gpstart -ma" should return a return code of 0 + And all files in gpAdminLogs directory are deleted + When the user runs "ggrebalance -x 1" + Then ggrebalance should return a return code of 1 + And ggrebalance should print "System was started in single node mode - only utility mode connections are allowed" to logfile with latest timestamp + + Scenario: test 6. Check rebalance with '--target-hosts'. + Given the database is not running + And a working directory of the test as '/data/gpdata/ggrebalance' + And a cluster is created with mirrors on "cdw" and "sdw1", with 4 segments on each + And database "test_db_1" exists + And schema "test_schema_1" exists in "test_db_1" + And there is a "heap" table "test_schema_1.test_table_1" in "test_db_1" with "100" rows + And there is a "ao" table "test_schema_1.test_table_2" in "test_db_1" with "100" rows + And database "test_db_2" exists + And schema "test_schema_2" exists in "test_db_2" + And there is a "heap" table "test_schema_2.test_table_1" in "test_db_2" with "100" rows + And there is a "ao" table "test_schema_2.test_table_2" in "test_db_2" with "100" rows + And all files in gpAdminLogs directory are deleted + When the user runs "ggrebalance -x 4 --target-hosts 'sdw2, sdw3' -d '/home/gpadmin/gpdb_src/gpAux/gpdemo/datadirs/dbfast, /home/gpadmin/gpdb_src/gpAux/gpdemo/datadirs/dbfast_mirror'" + Then ggrebalance should return a return code of 0 + And ggrebalance should print "Rebalance is complete" to logfile with latest timestamp + And the cluster configuration has 0 segments where "hostname='sdw1' and content > -1 and role = 'p' and status = 'u'" + And the cluster configuration has 0 segments where "hostname='sdw1' and content > -1 and role = 'm' and status = 'u'" + And the cluster configuration has 2 segments where "hostname='sdw2' and content > -1 and role = 'p' and status = 'u'" + And the cluster configuration has 2 segments where "hostname='sdw2' and content > -1 and role = 'm' and status = 'u'" + And the cluster configuration has 2 segments where "hostname='sdw3' and content > -1 and role = 'p' and status = 'u'" + And the cluster configuration has 2 segments where "hostname='sdw3' and content > -1 and role = 'm' and status = 'u'" + And distribution information from table "test_schema_1.test_table_1" with data in "test_db_1" is equal to segment count = 4, row count = 100 + And distribution information from table "test_schema_1.test_table_2" with data in "test_db_1" is equal to segment count = 4, row count = 100 + And distribution information from table "test_schema_2.test_table_1" with data in "test_db_2" is equal to segment count = 4, row count = 100 + And distribution information from table "test_schema_2.test_table_2" with data in "test_db_2" is equal to segment count = 4, row count = 100 + + Scenario: test 7. Check rebalance with '--target-hosts-file'. + Given the database is not running + And a working directory of the test as '/data/gpdata/ggrebalance' + And a cluster is created with mirrors on "cdw" and "sdw1", with 4 segments on each + And database "test_db_1" exists + And schema "test_schema_1" exists in "test_db_1" + And there is a "heap" table "test_schema_1.test_table_1" in "test_db_1" with "100" rows + And there is a "ao" table "test_schema_1.test_table_2" in "test_db_1" with "100" rows + And database "test_db_2" exists + And schema "test_schema_2" exists in "test_db_2" + And there is a "heap" table "test_schema_2.test_table_1" in "test_db_2" with "100" rows + And there is a "ao" table "test_schema_2.test_table_2" in "test_db_2" with "100" rows + And all files in gpAdminLogs directory are deleted + When there is a file "/tmp/ggrebalance_target_hosts" with hosts " " + And the user runs "ggrebalance -x 4 --target-hosts-file /tmp/ggrebalance_target_hosts -d '/home/gpadmin/gpdb_src/gpAux/gpdemo/datadirs/dbfast, /home/gpadmin/gpdb_src/gpAux/gpdemo/datadirs/dbfast_mirror'" + Then ggrebalance should return a return code of 1 + And ggrebalance should print "Empty '/tmp/ggrebalance_target_hosts' file" to logfile with latest timestamp + And the temporary file "/tmp/ggrebalance_target_hosts" is removed + And all files in gpAdminLogs directory are deleted + When the user runs "ggrebalance -x 4 --target-hosts-file /tmp/ggrebalance_non_existing_file -d '/home/gpadmin/gpdb_src/gpAux/gpdemo/datadirs/dbfast, /home/gpadmin/gpdb_src/gpAux/gpdemo/datadirs/dbfast_mirror'" + Then ggrebalance should return a return code of 1 + And ggrebalance should print "No such file or directory: '/tmp/ggrebalance_non_existing_file'" to logfile with latest timestamp + And all files in gpAdminLogs directory are deleted + When there is a file "/tmp/ggrebalance_target_hosts" with hosts "sdw2|sdw3" + And the user runs "ggrebalance -x 4 --target-hosts-file /tmp/ggrebalance_target_hosts -d '/home/gpadmin/gpdb_src/gpAux/gpdemo/datadirs/dbfast, /home/gpadmin/gpdb_src/gpAux/gpdemo/datadirs/dbfast_mirror'" + Then ggrebalance should return a return code of 0 + And ggrebalance should print "Rebalance is complete" to logfile with latest timestamp + And the cluster configuration has 0 segments where "hostname='sdw1' and content > -1 and role = 'p' and status = 'u'" + And the cluster configuration has 0 segments where "hostname='sdw1' and content > -1 and role = 'm' and status = 'u'" + And the cluster configuration has 2 segments where "hostname='sdw2' and content > -1 and role = 'p' and status = 'u'" + And the cluster configuration has 2 segments where "hostname='sdw2' and content > -1 and role = 'm' and status = 'u'" + And the cluster configuration has 2 segments where "hostname='sdw3' and content > -1 and role = 'p' and status = 'u'" + And the cluster configuration has 2 segments where "hostname='sdw3' and content > -1 and role = 'm' and status = 'u'" + And distribution information from table "test_schema_1.test_table_1" with data in "test_db_1" is equal to segment count = 4, row count = 100 + And distribution information from table "test_schema_1.test_table_2" with data in "test_db_1" is equal to segment count = 4, row count = 100 + And distribution information from table "test_schema_2.test_table_1" with data in "test_db_2" is equal to segment count = 4, row count = 100 + And distribution information from table "test_schema_2.test_table_2" with data in "test_db_2" is equal to segment count = 4, row count = 100 + And the temporary file "/tmp/ggrebalance_target_hosts" is removed + + Scenario: test 8. Check rebalance with '--add-hosts-file'. + Given the database is not running + And a working directory of the test as '/data/gpdata/ggrebalance' + And a cluster is created with mirrors on "cdw" and "sdw1", with 3 segments on each + And database "test_db_1" exists + And schema "test_schema_1" exists in "test_db_1" + And there is a "heap" table "test_schema_1.test_table_1" in "test_db_1" with "100" rows + And there is a "ao" table "test_schema_1.test_table_2" in "test_db_1" with "100" rows + And database "test_db_2" exists + And schema "test_schema_2" exists in "test_db_2" + And there is a "heap" table "test_schema_2.test_table_1" in "test_db_2" with "100" rows + And there is a "ao" table "test_schema_2.test_table_2" in "test_db_2" with "100" rows + And all files in gpAdminLogs directory are deleted + And there is a file "/tmp/ggrebalance_add_hosts" with hosts "sdw2|sdw3" + When the user runs "ggrebalance -x 3 --add-hosts-file /tmp/ggrebalance_add_hosts -d '/home/gpadmin/gpdb_src/gpAux/gpdemo/datadirs/dbfast, /home/gpadmin/gpdb_src/gpAux/gpdemo/datadirs/dbfast_mirror'" + Then ggrebalance should return a return code of 0 + And ggrebalance should print "Rebalance is complete" to logfile with latest timestamp + And the cluster configuration has 1 segments where "hostname='sdw1' and content > -1 and role = 'p' and status = 'u'" + And the cluster configuration has 1 segments where "hostname='sdw1' and content > -1 and role = 'm' and status = 'u'" + And the cluster configuration has 1 segments where "hostname='sdw2' and content > -1 and role = 'p' and status = 'u'" + And the cluster configuration has 1 segments where "hostname='sdw2' and content > -1 and role = 'm' and status = 'u'" + And the cluster configuration has 1 segments where "hostname='sdw3' and content > -1 and role = 'p' and status = 'u'" + And the cluster configuration has 1 segments where "hostname='sdw3' and content > -1 and role = 'm' and status = 'u'" + And distribution information from table "test_schema_1.test_table_1" with data in "test_db_1" is equal to segment count = 3, row count = 100 + And distribution information from table "test_schema_1.test_table_2" with data in "test_db_1" is equal to segment count = 3, row count = 100 + And distribution information from table "test_schema_2.test_table_1" with data in "test_db_2" is equal to segment count = 3, row count = 100 + And distribution information from table "test_schema_2.test_table_2" with data in "test_db_2" is equal to segment count = 3, row count = 100 + And the temporary file "/tmp/ggrebalance_add_hosts" is removed + + Scenario: test 9. Check rebalance with '--remove-hosts-file'. + Given the database is not running + And a working directory of the test as '/data/gpdata/ggrebalance' + And a cluster is created with mirrors on "cdw" and "sdw1, sdw2, sdw3", with 2 segments on each + And database "test_db_1" exists + And schema "test_schema_1" exists in "test_db_1" + And there is a "heap" table "test_schema_1.test_table_1" in "test_db_1" with "100" rows + And there is a "ao" table "test_schema_1.test_table_2" in "test_db_1" with "100" rows + And database "test_db_2" exists + And schema "test_schema_2" exists in "test_db_2" + And there is a "heap" table "test_schema_2.test_table_1" in "test_db_2" with "100" rows + And there is a "ao" table "test_schema_2.test_table_2" in "test_db_2" with "100" rows + And all files in gpAdminLogs directory are deleted + And there is a file "/tmp/ggrebalance_remove_hosts" with hosts "sdw3" + When the user runs "ggrebalance -x 4 --remove-hosts-file /tmp/ggrebalance_remove_hosts -d '/home/gpadmin/gpdb_src/gpAux/gpdemo/datadirs/dbfast, /home/gpadmin/gpdb_src/gpAux/gpdemo/datadirs/dbfast_mirror'" + Then ggrebalance should return a return code of 0 + And ggrebalance should print "Rebalance is complete" to logfile with latest timestamp + And the cluster configuration has 2 segments where "hostname='sdw1' and content > -1 and role = 'p' and status = 'u'" + And the cluster configuration has 2 segments where "hostname='sdw1' and content > -1 and role = 'm' and status = 'u'" + And the cluster configuration has 2 segments where "hostname='sdw2' and content > -1 and role = 'p' and status = 'u'" + And the cluster configuration has 2 segments where "hostname='sdw2' and content > -1 and role = 'm' and status = 'u'" + And the cluster configuration has 0 segments where "hostname='sdw3' and content > -1 and role = 'p' and status = 'u'" + And the cluster configuration has 0 segments where "hostname='sdw3' and content > -1 and role = 'm' and status = 'u'" + And distribution information from table "test_schema_1.test_table_1" with data in "test_db_1" is equal to segment count = 4, row count = 100 + And distribution information from table "test_schema_1.test_table_2" with data in "test_db_1" is equal to segment count = 4, row count = 100 + And distribution information from table "test_schema_2.test_table_1" with data in "test_db_2" is equal to segment count = 4, row count = 100 + And distribution information from table "test_schema_2.test_table_2" with data in "test_db_2" is equal to segment count = 4, row count = 100 + And the temporary file "/tmp/ggrebalance_remove_hosts" is removed + + Scenario: test 10. Check rebalance with '--target-datadirs' plain paths (without template substitution). + Given the database is not running + And a working directory of the test as '/data/gpdata/ggrebalance' + And a cluster is created with mirrors on "cdw" and "sdw1, sdw2, sdw3", with 2 segments on each + And database "test_db_1" exists + And schema "test_schema_1" exists in "test_db_1" + And there is a "heap" table "test_schema_1.test_table_1" in "test_db_1" with "100" rows + And there is a "ao" table "test_schema_1.test_table_2" in "test_db_1" with "100" rows + And database "test_db_2" exists + And schema "test_schema_2" exists in "test_db_2" + And there is a "heap" table "test_schema_2.test_table_1" in "test_db_2" with "100" rows + And there is a "ao" table "test_schema_2.test_table_2" in "test_db_2" with "100" rows + And all files in gpAdminLogs directory are deleted + When the user runs "ggrebalance -x 6 --remove-hosts sdw3 --target-datadirs '/home/gpadmin/gpdb_src/gpAux/gpdemo/datadirs_new/dbfast, /home/gpadmin/gpdb_src/gpAux/gpdemo/datadirs_new/dbfast_mirror'" + Then ggrebalance should return a return code of 0 + And ggrebalance should print "Rebalance is complete" to logfile with latest timestamp + And the cluster configuration has some segments where "datadir like '/home/gpadmin/gpdb\_src/gpAux/gpdemo/datadirs\_new/dbfast/gpseg_'" + And the cluster configuration has some segments where "datadir like '/home/gpadmin/gpdb\_src/gpAux/gpdemo/datadirs\_new/dbfast_mirror/gpseg_'" + And distribution information from table "test_schema_1.test_table_1" with data in "test_db_1" is equal to segment count = 6, row count = 100 + And distribution information from table "test_schema_1.test_table_2" with data in "test_db_1" is equal to segment count = 6, row count = 100 + And distribution information from table "test_schema_2.test_table_1" with data in "test_db_2" is equal to segment count = 6, row count = 100 + And distribution information from table "test_schema_2.test_table_2" with data in "test_db_2" is equal to segment count = 6, row count = 100 + + Scenario: test 11. Check rebalance with '--target-datadirs' {content} and {hostname} substitution. + Given the database is not running + And a working directory of the test as '/data/gpdata/ggrebalance' + And a cluster is created with mirrors on "cdw" and "sdw1, sdw2, sdw3", with 2 segments on each + And database "test_db_1" exists + And schema "test_schema_1" exists in "test_db_1" + And there is a "heap" table "test_schema_1.test_table_1" in "test_db_1" with "100" rows + And there is a "ao" table "test_schema_1.test_table_2" in "test_db_1" with "100" rows + And database "test_db_2" exists + And schema "test_schema_2" exists in "test_db_2" + And there is a "heap" table "test_schema_2.test_table_1" in "test_db_2" with "100" rows + And there is a "ao" table "test_schema_2.test_table_2" in "test_db_2" with "100" rows + And all files in gpAdminLogs directory are deleted + When the user runs "ggrebalance -x 6 --remove-hosts sdw3 --target-datadirs '/home/gpadmin/gpdb_src/gpAux/gpdemo/datadirs_new/dbfast/new_seg{content}_on_host_{hostname}, /home/gpadmin/gpdb_src/gpAux/gpdemo/datadirs_new/dbfast_mirror/new_seg{content}_on_host_{hostname}'" + Then ggrebalance should return a return code of 0 + And ggrebalance should print "Rebalance is complete" to logfile with latest timestamp + And the cluster configuration has some segments where "datadir like '/home/gpadmin/gpdb\_src/gpAux/gpdemo/datadirs\_new/dbfast/new\_seg_\_on\_host\_sdw_'" + And the cluster configuration has some segments where "datadir like '/home/gpadmin/gpdb\_src/gpAux/gpdemo/datadirs\_new/dbfast\_mirror/new\_seg_\_on\_host\_sdw_'" + And distribution information from table "test_schema_1.test_table_1" with data in "test_db_1" is equal to segment count = 6, row count = 100 + And distribution information from table "test_schema_1.test_table_2" with data in "test_db_1" is equal to segment count = 6, row count = 100 + And distribution information from table "test_schema_2.test_table_1" with data in "test_db_2" is equal to segment count = 6, row count = 100 + And distribution information from table "test_schema_2.test_table_2" with data in "test_db_2" is equal to segment count = 6, row count = 100 + + Scenario: test 12. Check rebalance with '--target-datadirs-file' {content} and {hostname} substitution. + Given the database is not running + And a working directory of the test as '/data/gpdata/ggrebalance' + And a cluster is created with mirrors on "cdw" and "sdw1, sdw2, sdw3", with 2 segments on each + And database "test_db_1" exists + And schema "test_schema_1" exists in "test_db_1" + And there is a "heap" table "test_schema_1.test_table_1" in "test_db_1" with "100" rows + And there is a "ao" table "test_schema_1.test_table_2" in "test_db_1" with "100" rows + And database "test_db_2" exists + And schema "test_schema_2" exists in "test_db_2" + And there is a "heap" table "test_schema_2.test_table_1" in "test_db_2" with "100" rows + And there is a "ao" table "test_schema_2.test_table_2" in "test_db_2" with "100" rows + And all files in gpAdminLogs directory are deleted + And there is a file "/tmp/ggrebalance_target_datadirs" with hosts "/home/gpadmin/gpdb_src/gpAux/gpdemo/datadirs_{hostname}/dbfast/new_seg{content}|/home/gpadmin/gpdb_src/gpAux/gpdemo/datadirs_{hostname}/dbfast_mirror/new_seg{content}" + When the user runs "ggrebalance -x 6 --remove-hosts sdw3 --target-datadirs-file /tmp/ggrebalance_target_datadirs" + Then ggrebalance should return a return code of 0 + And ggrebalance should print "Rebalance is complete" to logfile with latest timestamp + And the cluster configuration has some segments where "datadir like '/home/gpadmin/gpdb\_src/gpAux/gpdemo/datadirs\_sdw_/dbfast/new\_seg_'" + And the cluster configuration has some segments where "datadir like '/home/gpadmin/gpdb\_src/gpAux/gpdemo/datadirs\_sdw_/dbfast\_mirror/new\_seg_'" + And distribution information from table "test_schema_1.test_table_1" with data in "test_db_1" is equal to segment count = 6, row count = 100 + And distribution information from table "test_schema_1.test_table_2" with data in "test_db_1" is equal to segment count = 6, row count = 100 + And distribution information from table "test_schema_2.test_table_1" with data in "test_db_2" is equal to segment count = 6, row count = 100 + And distribution information from table "test_schema_2.test_table_2" with data in "test_db_2" is equal to segment count = 6, row count = 100 + And the temporary file "/tmp/ggrebalance_target_datadirs" is removed + + Scenario: test 13. Check ggrebalance launch when pid file (or any other mark) for other tool exists. + Given the database is not running + And a working directory of the test as '/data/gpdata/ggrebalance' + And a cluster is created with mirrors on "cdw" and "sdw1, sdw2, sdw3" + And all files in gpAdminLogs directory are deleted + When we run a sample background script to generate a pid on "coordinator" segment + And a sample ggrebalance.pid file is created using the background pid in the coordinator_data_directory + And the user runs "ggrebalance -x 6 --remove-hosts sdw3 -d '/home/gpadmin/gpdb_src/gpAux/gpdemo/datadirs/dbfast, /home/gpadmin/gpdb_src/gpAux/gpdemo/datadirs/dbfast_mirror'" + Then ggrebalance should return a return code of 1 + And ggrebalance should print "ggrebalance is already running." to logfile with latest timestamp + And the background pid is killed on "coordinator" segment + And a sample ggrebalance.pid file is removed from the coordinator_data_directory + And all files in gpAdminLogs directory are deleted + When we run a sample background script to generate a pid on "coordinator" segment + And a sample gpexpand.pid file is created using the background pid in the coordinator_data_directory + And the user runs "ggrebalance -x 6 --remove-hosts sdw3 -d '/home/gpadmin/gpdb_src/gpAux/gpdemo/datadirs/dbfast, /home/gpadmin/gpdb_src/gpAux/gpdemo/datadirs/dbfast_mirror'" + Then ggrebalance should return a return code of 1 + And ggrebalance should print "gpexpand is already running." to logfile with latest timestamp + And the background pid is killed on "coordinator" segment + And a sample gpexpand.pid file is removed from the coordinator_data_directory + And all files in gpAdminLogs directory are deleted + When schema "gpexpand" exists in "postgres" + And the user runs "ggrebalance -x 6 --remove-hosts sdw3 -d '/home/gpadmin/gpdb_src/gpAux/gpdemo/datadirs/dbfast, /home/gpadmin/gpdb_src/gpAux/gpdemo/datadirs/dbfast_mirror'" + Then ggrebalance should return a return code of 1 + And ggrebalance should print "gpexpand schema exists. Assuming gpexpand is already running." to logfile with latest timestamp + And schema "gpexpand" is removed in "postgres" + And all files in gpAdminLogs directory are deleted + When we run a sample background script to generate a pid on "coordinator" segment + And a sample gpexpand.status file is created using the background pid in the coordinator_data_directory + And the user runs "ggrebalance -x 6 --remove-hosts sdw3 -d '/home/gpadmin/gpdb_src/gpAux/gpdemo/datadirs/dbfast, /home/gpadmin/gpdb_src/gpAux/gpdemo/datadirs/dbfast_mirror'" + Then ggrebalance should return a return code of 1 + And ggrebalance should print "gpexpand.status file exists. Assuming gpexpand is already running." to logfile with latest timestamp + And the background pid is killed on "coordinator" segment + And a sample gpexpand.status file is removed from the coordinator_data_directory + And all files in gpAdminLogs directory are deleted + When we run a sample background script to generate a pid on "coordinator" segment + And a sample gprecoverseg.lock directory is created using the background pid in coordinator_data_directory + And the user runs "ggrebalance -x 6 --remove-hosts sdw3 -d '/home/gpadmin/gpdb_src/gpAux/gpdemo/datadirs/dbfast, /home/gpadmin/gpdb_src/gpAux/gpdemo/datadirs/dbfast_mirror'" + Then ggrebalance should return a return code of 1 + And ggrebalance should print "gprecoverseg is already running." to logfile with latest timestamp + And the background pid is killed on "coordinator" segment + And the gprecoverseg lock directory is removed + And all files in gpAdminLogs directory are deleted + + Scenario: test 14. Check ggrebalance if pg_basebackup is already running + Given the database is not running + And a working directory of the test as '/data/gpdata/ggrebalance' + And a cluster is created with mirrors on "cdw" and "sdw1, sdw2, sdw3", with 1 segments on each + And all the segments are running + And the segments are synchronized + And all files in gpAdminLogs directory are deleted + And the information of contents 0,1,2 is saved + And user immediately stops all mirror processes for content 0,1,2 + And user can start transactions + And the user suspend the walsender on the primary on content 0 + And the user asynchronously runs "gprecoverseg -aF" and the process is saved + And the user just waits until recovery_progress.file is created in gpAdminLogs + And user waits until gp_stat_replication table has no pg_basebackup entries for content 1,2 + And an FTS probe is triggered + And the user waits until mirror on content 1,2 is up + And verify that mirror on content 0 is down + And the gprecoverseg lock directory is removed + And user immediately stops all mirror processes for content 1,2 + And the user waits until mirror on content 1,2 is down + When the user runs "ggrebalance" + Then ggrebalance should return a return code of 1 + And ggrebalance should print "Segments {0} have running pg_basebackup." to logfile with latest timestamp + And the user reset the walsender on the primary on content 0 + And the user waits until saved async process is completed + And verify that mirror on content 0 is up + + Scenario: test 15. Check '--log-dir' option. + Given the database is not running + And a working directory of the test as '/data/gpdata/ggrebalance' + And a cluster is created with mirrors on "cdw" and "sdw1, sdw2, sdw3" + And all files in gpAdminLogs directory are deleted + When the user runs "ggrebalance -l "/tmp/ggrebalance_logs" -x 6 --remove-hosts sdw3 -d '/home/gpadmin/gpdb_src/gpAux/gpdemo/datadirs/dbfast, /home/gpadmin/gpdb_src/gpAux/gpdemo/datadirs/dbfast_mirror'" + Then ggrebalance should return a return code of 0 + And gpAdminLogs directory has no "ggrebalance*" files + And gpAdminLogs directory has no "gpmovemirrors*" files + And gpAdminLogs directory has no "gprecoverseg*" files + And "/tmp/ggrebalance_logs" directory has "ggrebalance*" files + And "/tmp/ggrebalance_logs" directory has "gpmovemirrors*" files + And "/tmp/ggrebalance_logs" directory has "gprecoverseg*" files + And all files in "/tmp/ggrebalance_logs" directory are deleted + When the user runs "ggrebalance -c" + And all files in gpAdminLogs directory are deleted + When the user runs "ggrebalance --log-dir "/tmp/ggrebalance logs" -x 6 --add-hosts sdw3 -d '/home/gpadmin/gpdb_src/gpAux/gpdemo/datadirs/dbfast, /home/gpadmin/gpdb_src/gpAux/gpdemo/datadirs/dbfast_mirror'" + Then ggrebalance should return a return code of 0 + And gpAdminLogs directory has no "ggrebalance*" files + And gpAdminLogs directory has no "gpmovemirrors*" files + And gpAdminLogs directory has no "gprecoverseg*" files + And "/tmp/ggrebalance logs" directory has "ggrebalance*" files + And "/tmp/ggrebalance logs" directory has "gpmovemirrors*" files + And "/tmp/ggrebalance logs" directory has "gprecoverseg*" files + And all files in "/tmp/ggrebalance logs" directory are deleted + + Scenario: test 16.1. Check without '--analyze' option. + Given the database is not running + And a working directory of the test as '/data/gpdata/ggrebalance' + And a cluster is created with mirrors on "cdw" and "sdw1, sdw2, sdw3" + And database "test_db_1" exists + And schema "test_schema_1" exists in "test_db_1" + And there is a "heap" table "test_schema_1.test_table_1" in "test_db_1" with "100" rows + And there is a "ao" table "test_schema_1.test_table_2" in "test_db_1" with "100" rows + And all files in gpAdminLogs directory are deleted + When execute following sql in db "test_db_1" and store result in the context + """ + SELECT COUNT(1) AS not_analyzed_tables_cnt FROM pg_stat_all_tables WHERE last_analyze IS NULL AND relname IN ('test_table_1', 'test_table_2'); + """ + Then validate that following rows are in the stored rows + | not_analyzed_tables_cnt | + | 2 | + When execute following sql in db "test_db_1" and store result in the context + """ + SELECT COUNT(1) AS analyzed_tables_cnt FROM pg_stat_all_tables WHERE last_analyze IS NOT NULL AND relname IN ('test_table_1', 'test_table_2'); + """ + Then validate that following rows are in the stored rows + | analyzed_tables_cnt | + | 0 | + When the user runs "ggrebalance -x 3 -d '/home/gpadmin/gpdb_src/gpAux/gpdemo/datadirs/dbfast, /home/gpadmin/gpdb_src/gpAux/gpdemo/datadirs/dbfast_mirror'" + Then ggrebalance should return a return code of 0 + And ggrebalance should print "Rebalance is complete" to logfile with latest timestamp + When execute following sql in db "test_db_1" and store result in the context + """ + SELECT COUNT(1) AS not_analyzed_tables_cnt FROM pg_stat_all_tables WHERE last_analyze IS NULL AND relname IN ('test_table_1', 'test_table_2'); + """ + Then validate that following rows are in the stored rows + | not_analyzed_tables_cnt | + | 2 | + When execute following sql in db "test_db_1" and store result in the context + """ + SELECT COUNT(1) AS analyzed_tables_cnt FROM pg_stat_all_tables WHERE last_analyze IS NOT NULL AND relname IN ('test_table_1', 'test_table_2'); + """ + Then validate that following rows are in the stored rows + | analyzed_tables_cnt | + | 0 | + + Scenario: test 16.2. Check with '--analyze' option. + Given the database is not running + And a working directory of the test as '/data/gpdata/ggrebalance' + And a cluster is created with mirrors on "cdw" and "sdw1, sdw2, sdw3" + And database "test_db_1" exists + And schema "test_schema_1" exists in "test_db_1" + And there is a "heap" table "test_schema_1.test_table_1" in "test_db_1" with "100" rows + And there is a "ao" table "test_schema_1.test_table_2" in "test_db_1" with "100" rows + And all files in gpAdminLogs directory are deleted + When execute following sql in db "test_db_1" and store result in the context + """ + SELECT COUNT(1) AS not_analyzed_tables_cnt FROM pg_stat_all_tables WHERE last_analyze IS NULL AND relname IN ('test_table_1', 'test_table_2'); + """ + Then validate that following rows are in the stored rows + | not_analyzed_tables_cnt | + | 2 | + When execute following sql in db "test_db_1" and store result in the context + """ + SELECT COUNT(1) AS analyzed_tables_cnt FROM pg_stat_all_tables WHERE last_analyze IS NOT NULL AND relname IN ('test_table_1', 'test_table_2'); + """ + Then validate that following rows are in the stored rows + | analyzed_tables_cnt | + | 0 | + When the user runs "ggrebalance --analyze -x 3 -d '/home/gpadmin/gpdb_src/gpAux/gpdemo/datadirs/dbfast, /home/gpadmin/gpdb_src/gpAux/gpdemo/datadirs/dbfast_mirror'" + Then ggrebalance should return a return code of 0 + And ggrebalance should print "Rebalance is complete" to logfile with latest timestamp + When execute following sql in db "test_db_1" and store result in the context + """ + SELECT COUNT(1) AS not_analyzed_tables_cnt FROM pg_stat_all_tables WHERE last_analyze IS NULL AND relname IN ('test_table_1', 'test_table_2'); + """ + Then validate that following rows are in the stored rows + | not_analyzed_tables_cnt | + | 0 | + When execute following sql in db "test_db_1" and store result in the context + """ + SELECT COUNT(1) AS analyzed_tables_cnt FROM pg_stat_all_tables WHERE last_analyze IS NOT NULL AND relname IN ('test_table_1', 'test_table_2'); + """ + Then validate that following rows are in the stored rows + | analyzed_tables_cnt | + | 2 | + + Scenario: test 17. Check '--replay-lag' option. + Given the database is not running + And a working directory of the test as '/data/gpdata/ggrebalance' + And a cluster is created with mirrors on "cdw" and "sdw1, sdw2, sdw3" + And all files in gpAdminLogs directory are deleted + When the user runs "ggrebalance --replay-lag 0 -x 6 --remove-hosts sdw3 -d '/home/gpadmin/gpdb_src/gpAux/gpdemo/datadirs/dbfast, /home/gpadmin/gpdb_src/gpAux/gpdemo/datadirs/dbfast_mirror'" + Then ggrebalance should return a return code of 1 + And gprecoverseg should print "0 bytes of wal is still to be replayed on mirror with dbid.*, let mirror catchup on replay then trigger rebalance" regex to logfile + And all files in gpAdminLogs directory are deleted + + Scenario: test 18-1. Check '--hba-hostnames' option. + Given the database is not running + And a working directory of the test as '/data/gpdata/ggrebalance' + And a cluster is created with mirrors on "cdw" and "sdw1" + And all files in gpAdminLogs directory are deleted + When the user runs "ggrebalance --hba-hostnames -x 2 --add-hosts sdw2 -d '/home/gpadmin/gpdb_src/gpAux/gpdemo/datadirs/dbfast, /home/gpadmin/gpdb_src/gpAux/gpdemo/datadirs/dbfast_mirror'" + Then ggrebalance should return a return code of 0 + And ggrebalance should print "Rebalance is complete" to logfile with latest timestamp + And pg_hba file "/data/gpdata/ggrebalance/data/primary/gpseg0/pg_hba.conf" on host "sdw1" contains entries for "sdw2" + + Scenario: test 18-2. Check without '--hba-hostnames' option. + Given the database is not running + And a working directory of the test as '/data/gpdata/ggrebalance' + And a cluster is created with mirrors on "cdw" and "sdw1" + And all files in gpAdminLogs directory are deleted + When the user runs "ggrebalance -x 2 --add-hosts sdw2 -d '/home/gpadmin/gpdb_src/gpAux/gpdemo/datadirs/dbfast, /home/gpadmin/gpdb_src/gpAux/gpdemo/datadirs/dbfast_mirror'" + Then ggrebalance should return a return code of 0 + And ggrebalance should print "Rebalance is complete" to logfile with latest timestamp + And pg_hba file "/data/gpdata/ggrebalance/data/primary/gpseg0/pg_hba.conf" on host "sdw1" contains only cidr addresses diff --git a/gpMgmt/test/behave/mgmt_utils/ggrebalance_rebalance.feature b/gpMgmt/test/behave/mgmt_utils/ggrebalance_rebalance.feature new file mode 100755 index 000000000000..ac09d8017971 --- /dev/null +++ b/gpMgmt/test/behave/mgmt_utils/ggrebalance_rebalance.feature @@ -0,0 +1,380 @@ +@ggrebalance_rebalance +Feature: ggrebalance behave tests (rebalance scenarios) + + Scenario Outline: test 1. rebalance - check scenario, when we remove/add a host and rebalance the cluster (with different parallel size). + Given the database is not running + And a working directory of the test as '/data/gpdata/ggrebalance' + And a cluster is created with mirrors on "cdw" and "sdw1, sdw2, sdw3" + And database "test_db_1" exists + And schema "test_schema_1" exists in "test_db_1" + And there is a "heap" table "test_schema_1.test_table_1" in "test_db_1" with "100" rows + And there is a "ao" table "test_schema_1.test_table_2" in "test_db_1" with "100" rows + And database "test_db_2" exists + And schema "test_schema_2" exists in "test_db_2" + And there is a "heap" table "test_schema_2.test_table_1" in "test_db_2" with "100" rows + And there is a "ao" table "test_schema_2.test_table_2" in "test_db_2" with "100" rows + And all files in gpAdminLogs directory are deleted + When the user runs "ggrebalance --parallel -x 6 --remove-hosts sdw3 -d '/home/gpadmin/gpdb_src/gpAux/gpdemo/datadirs/dbfast, /home/gpadmin/gpdb_src/gpAux/gpdemo/datadirs/dbfast_mirror'" + Then ggrebalance should return a return code of 0 + And ggrebalance should print "Rebalance is complete" to logfile with latest timestamp + And the cluster configuration has 3 segments where "hostname='sdw1' and content > -1 and role = 'p' and status = 'u'" + And the cluster configuration has 3 segments where "hostname='sdw1' and content > -1 and role = 'm' and status = 'u'" + And the cluster configuration has 3 segments where "hostname='sdw2' and content > -1 and role = 'p' and status = 'u'" + And the cluster configuration has 3 segments where "hostname='sdw2' and content > -1 and role = 'm' and status = 'u'" + And the cluster configuration has 0 segments where "hostname='sdw3' and content > -1 and role = 'p' and status = 'u'" + And the cluster configuration has 0 segments where "hostname='sdw3' and content > -1 and role = 'm' and status = 'u'" + And distribution information from table "test_schema_1.test_table_1" with data in "test_db_1" is equal to segment count = 6, row count = 100 + And distribution information from table "test_schema_1.test_table_2" with data in "test_db_1" is equal to segment count = 6, row count = 100 + And distribution information from table "test_schema_2.test_table_1" with data in "test_db_2" is equal to segment count = 6, row count = 100 + And distribution information from table "test_schema_2.test_table_2" with data in "test_db_2" is equal to segment count = 6, row count = 100 + When there is a "heap" table "test_schema_1.test_table_3" in "test_db_1" with "100" rows + Then distribution information from table "test_schema_1.test_table_3" with data in "test_db_1" is equal to segment count = 6, row count = 100 + And all files in gpAdminLogs directory are deleted + When the user runs "ggrebalance -x 6" + Then ggrebalance should return a return code of 0 + And ggrebalance should print "Previous run was completed successfully. Please execute cleanup before a new run" to logfile with latest timestamp + And all files in gpAdminLogs directory are deleted + When the user runs "ggrebalance -c" + Then ggrebalance should return a return code of 0 + And all files in gpAdminLogs directory are deleted + When the user runs "ggrebalance -x 6" + Then ggrebalance should return a return code of 0 + And ggrebalance should print "Cluster is already balanced, no segment moves will be held." to logfile with latest timestamp + And all files in gpAdminLogs directory are deleted + When the user runs "ggrebalance -c" + Then ggrebalance should return a return code of 0 + And all files in gpAdminLogs directory are deleted + When the user runs "ggrebalance --parallel -x 6 --add-hosts sdw3 -d '/home/gpadmin/gpdb_src/gpAux/gpdemo/datadirs/dbfast, /home/gpadmin/gpdb_src/gpAux/gpdemo/datadirs/dbfast_mirror'" + Then ggrebalance should return a return code of 0 + And ggrebalance should print "Rebalance is complete" to logfile with latest timestamp + And the cluster configuration has 2 segments where "hostname='sdw1' and content > -1 and role = 'p' and status = 'u'" + And the cluster configuration has 2 segments where "hostname='sdw1' and content > -1 and role = 'm' and status = 'u'" + And the cluster configuration has 2 segments where "hostname='sdw2' and content > -1 and role = 'p' and status = 'u'" + And the cluster configuration has 2 segments where "hostname='sdw2' and content > -1 and role = 'm' and status = 'u'" + And the cluster configuration has 2 segments where "hostname='sdw3' and content > -1 and role = 'p' and status = 'u'" + And the cluster configuration has 2 segments where "hostname='sdw3' and content > -1 and role = 'm' and status = 'u'" + And distribution information from table "test_schema_1.test_table_1" with data in "test_db_1" is equal to segment count = 6, row count = 100 + And distribution information from table "test_schema_1.test_table_2" with data in "test_db_1" is equal to segment count = 6, row count = 100 + And distribution information from table "test_schema_2.test_table_1" with data in "test_db_2" is equal to segment count = 6, row count = 100 + And distribution information from table "test_schema_2.test_table_2" with data in "test_db_2" is equal to segment count = 6, row count = 100 + When there is a "heap" table "test_schema_1.test_table_3" in "test_db_1" with "100" rows + Then distribution information from table "test_schema_1.test_table_3" with data in "test_db_1" is equal to segment count = 6, row count = 100 + And all files in gpAdminLogs directory are deleted + When the user runs "ggrebalance -x 6" + Then ggrebalance should return a return code of 0 + And ggrebalance should print "Previous run was completed successfully. Please execute cleanup before a new run" to logfile with latest timestamp + And all files in gpAdminLogs directory are deleted + When the user runs "ggrebalance -c" + Then ggrebalance should return a return code of 0 + And all files in gpAdminLogs directory are deleted + When the user runs "ggrebalance -x 6" + Then ggrebalance should return a return code of 0 + And ggrebalance should print "Cluster is already balanced, no segment moves will be held." to logfile with latest timestamp + + Examples: + | parallel_size | + | 1 | + | 16 | + | 64 | + | 96 | + + Scenario: test 2. rebalance - check rebalance after shrink. + Given the database is not running + And a working directory of the test as '/data/gpdata/ggrebalance' + And a cluster is created with mirrors on "cdw" and "sdw1, sdw2, sdw3" + And database "test_db_1" exists + And schema "test_schema_1" exists in "test_db_1" + And there is a "heap" table "test_schema_1.test_table_1" in "test_db_1" with "100" rows + And there is a "ao" table "test_schema_1.test_table_2" in "test_db_1" with "100" rows + And database "test_db_2" exists + And schema "test_schema_2" exists in "test_db_2" + And there is a "heap" table "test_schema_2.test_table_1" in "test_db_2" with "100" rows + And there is a "ao" table "test_schema_2.test_table_2" in "test_db_2" with "100" rows + And all files in gpAdminLogs directory are deleted + When the user runs "ggrebalance -x 3 -d '/home/gpadmin/gpdb_src/gpAux/gpdemo/datadirs/dbfast, /home/gpadmin/gpdb_src/gpAux/gpdemo/datadirs/dbfast_mirror'" + Then ggrebalance should return a return code of 0 + And ggrebalance should print "Rebalance is complete" to logfile with latest timestamp + And the cluster configuration has 1 segments where "hostname='sdw1' and content > -1 and role = 'p' and status = 'u'" + And the cluster configuration has 1 segments where "hostname='sdw1' and content > -1 and role = 'm' and status = 'u'" + And the cluster configuration has 1 segments where "hostname='sdw2' and content > -1 and role = 'p' and status = 'u'" + And the cluster configuration has 1 segments where "hostname='sdw2' and content > -1 and role = 'm' and status = 'u'" + And the cluster configuration has 1 segments where "hostname='sdw3' and content > -1 and role = 'p' and status = 'u'" + And the cluster configuration has 1 segments where "hostname='sdw3' and content > -1 and role = 'm' and status = 'u'" + And distribution information from table "test_schema_1.test_table_1" with data in "test_db_1" is equal to segment count = 3, row count = 100 + And distribution information from table "test_schema_1.test_table_2" with data in "test_db_1" is equal to segment count = 3, row count = 100 + And distribution information from table "test_schema_2.test_table_1" with data in "test_db_2" is equal to segment count = 3, row count = 100 + And distribution information from table "test_schema_2.test_table_2" with data in "test_db_2" is equal to segment count = 3, row count = 100 + When there is a "heap" table "test_schema_1.test_table_3" in "test_db_1" with "100" rows + Then distribution information from table "test_schema_1.test_table_3" with data in "test_db_1" is equal to segment count = 3, row count = 100 + And all files in gpAdminLogs directory are deleted + When the user runs "ggrebalance -c" + Then ggrebalance should return a return code of 0 + And all files in gpAdminLogs directory are deleted + When the user runs "ggrebalance -x 3" + Then ggrebalance should return a return code of 0 + And ggrebalance should print "Cluster is already balanced, no segment moves will be held." to logfile with latest timestamp + + Scenario Outline: 3. rebalance - interrupt and continue. + Given the database is not running + And a working directory of the test as '/data/gpdata/ggrebalance' + And a cluster is created with mirrors on "cdw" and "sdw1, sdw2, sdw3" + And database "test_db_1" exists + And schema "test_schema_1" exists in "test_db_1" + And there is a "heap" table "test_schema_1.test_table_1" in "test_db_1" with "100" rows + And there is a "ao" table "test_schema_1.test_table_2" in "test_db_1" with "100" rows + And database "test_db_2" exists + And schema "test_schema_2" exists in "test_db_2" + And there is a "heap" table "test_schema_2.test_table_1" in "test_db_2" with "100" rows + And there is a "ao" table "test_schema_2.test_table_2" in "test_db_2" with "100" rows + And all files in gpAdminLogs directory are deleted + And set fault inject "" + And set fault inject delay ms + When the user runs "ggrebalance -x 6 --remove-hosts sdw3 -d '/home/gpadmin/gpdb_src/gpAux/gpdemo/datadirs/dbfast, /home/gpadmin/gpdb_src/gpAux/gpdemo/datadirs/dbfast_mirror'" + Then ggrebalance should return a return code of 1 + And ggrebalance should print "ggrebalance failed" to logfile with latest timestamp + And unset fault inject + And unset fault inject delay + And all files in gpAdminLogs directory are deleted + And the gprecoverseg lock directory is removed + When the user runs "ggrebalance" + Then ggrebalance should return a return code of 0 + And ggrebalance should print "Rebalance is complete" to logfile with latest timestamp + And the cluster configuration has 3 segments where "hostname='sdw1' and content > -1 and role = 'p' and status = 'u'" + And the cluster configuration has 3 segments where "hostname='sdw1' and content > -1 and role = 'm' and status = 'u'" + And the cluster configuration has 3 segments where "hostname='sdw2' and content > -1 and role = 'p' and status = 'u'" + And the cluster configuration has 3 segments where "hostname='sdw2' and content > -1 and role = 'm' and status = 'u'" + And the cluster configuration has 0 segments where "hostname='sdw3' and content > -1 and role = 'p' and status = 'u'" + And the cluster configuration has 0 segments where "hostname='sdw3' and content > -1 and role = 'm' and status = 'u'" + And distribution information from table "test_schema_1.test_table_1" with data in "test_db_1" is equal to segment count = 6, row count = 100 + And distribution information from table "test_schema_1.test_table_2" with data in "test_db_1" is equal to segment count = 6, row count = 100 + And distribution information from table "test_schema_2.test_table_1" with data in "test_db_2" is equal to segment count = 6, row count = 100 + And distribution information from table "test_schema_2.test_table_2" with data in "test_db_2" is equal to segment count = 6, row count = 100 + When there is a "heap" table "test_schema_1.test_table_3" in "test_db_1" with "100" rows + Then distribution information from table "test_schema_1.test_table_3" with data in "test_db_1" is equal to segment count = 6, row count = 100 + + Examples: + | fault_name | fault_delay_ms | + | on_enter_STATE_REBALANCE_STARTED_begin | 0 | + | on_enter_STATE_REBALANCE_STARTED_end | 0 | + | on_enter_STATE_REBALANCE_PREPARE_MOVES_STARTED_begin | 0 | + | on_enter_STATE_REBALANCE_PREPARE_MOVES_STARTED_end | 0 | + | on_enter_STATE_REBALANCE_PREPARE_MOVES_DONE_begin | 0 | + | on_enter_STATE_REBALANCE_PREPARE_MOVES_DONE_end | 0 | + | on_enter_STATE_REBALANCE_EXECUTION_STARTED_begin | 0 | + | on_enter_STATE_REBALANCE_EXECUTION_STARTED_end | 0 | + | on_enter_STATE_REBALANCE_MOVES_SUCCEEDED_begin | 0 | + | on_enter_STATE_REBALANCE_MOVES_SUCCEEDED_end | 0 | + | on_enter_STATE_REBALANCE_EXECUTION_AWAITING_SWITCHOVER_APPROVE_STARTED_begin | 0 | + | on_enter_STATE_REBALANCE_EXECUTION_AWAITING_SWITCHOVER_APPROVE_STARTED_end | 0 | + | on_enter_STATE_REBALANCE_EXECUTION_AWAITING_SWITCHOVER_APPROVE_DONE_begin | 0 | + | on_enter_STATE_REBALANCE_EXECUTION_AWAITING_SWITCHOVER_APPROVE_DONE_end | 0 | + | on_enter_STATE_REBALANCE_EXECUTION_DONE_begin | 0 | + | on_enter_STATE_REBALANCE_EXECUTION_DONE_end | 0 | + | FAULT_BEFORE_GPRECOVERSEG_PRIMARY_TO_MIRROR | 0 | + | FAULT_BEFORE_GPRECOVERSEG_MIRROR_TO_PRIMARY | 0 | + | on_enter_STATE_REBALANCE_DONE_begin | 0 | + | on_enter_STATE_REBALANCE_DONE_end | 0 | + | FAULT_BEFORE_GPRECOVERSEG_PRIMARY_TO_MIRROR | 1500 | + | FAULT_BEFORE_GPRECOVERSEG_PRIMARY_TO_MIRROR | 3000 | + | FAULT_BEFORE_GPRECOVERSEG_MIRROR_TO_PRIMARY | 1500 | + | FAULT_BEFORE_GPRECOVERSEG_MIRROR_TO_PRIMARY | 3000 | + | on_enter_STATE_REBALANCE_EXECUTION_STARTED_begin | 3000 | + + Scenario: 4. rebalance - check rebalance after interrupted shrink. + Given the database is not running + And a working directory of the test as '/data/gpdata/ggrebalance' + And a cluster is created with mirrors on "cdw" and "sdw1, sdw2, sdw3" + And database "test_db_1" exists + And schema "test_schema_1" exists in "test_db_1" + And there is a "heap" table "test_schema_1.test_table_1" in "test_db_1" with "100" rows + And there is a "ao" table "test_schema_1.test_table_2" in "test_db_1" with "100" rows + And database "test_db_2" exists + And schema "test_schema_2" exists in "test_db_2" + And there is a "heap" table "test_schema_2.test_table_1" in "test_db_2" with "100" rows + And there is a "ao" table "test_schema_2.test_table_2" in "test_db_2" with "100" rows + And all files in gpAdminLogs directory are deleted + And set fault inject "fault_rebalance_table_test_db_2.test_schema_2.test_table_1" + When the user runs "ggrebalance -x 4 --remove-hosts sdw3 -d '/home/gpadmin/gpdb_src/gpAux/gpdemo/datadirs/dbfast, /home/gpadmin/gpdb_src/gpAux/gpdemo/datadirs/dbfast_mirror'" + Then ggrebalance should return a return code of 1 + And ggrebalance should print "ggrebalance failed" to logfile with latest timestamp + And unset fault inject + And all files in gpAdminLogs directory are deleted + When the user runs "ggrebalance" + Then ggrebalance should return a return code of 0 + And ggrebalance should print "Continue interrupted shrink operation..." to logfile with latest timestamp + And ggrebalance should print "Rebalance is complete" to logfile with latest timestamp + And the cluster configuration has 2 segments where "hostname='sdw1' and content > -1 and role = 'p' and status = 'u'" + And the cluster configuration has 2 segments where "hostname='sdw1' and content > -1 and role = 'm' and status = 'u'" + And the cluster configuration has 2 segments where "hostname='sdw2' and content > -1 and role = 'p' and status = 'u'" + And the cluster configuration has 2 segments where "hostname='sdw2' and content > -1 and role = 'm' and status = 'u'" + And the cluster configuration has 0 segments where "hostname='sdw3' and content > -1 and role = 'p' and status = 'u'" + And the cluster configuration has 0 segments where "hostname='sdw3' and content > -1 and role = 'm' and status = 'u'" + And distribution information from table "test_schema_1.test_table_1" with data in "test_db_1" is equal to segment count = 4, row count = 100 + And distribution information from table "test_schema_1.test_table_2" with data in "test_db_1" is equal to segment count = 4, row count = 100 + And distribution information from table "test_schema_2.test_table_1" with data in "test_db_2" is equal to segment count = 4, row count = 100 + And distribution information from table "test_schema_2.test_table_2" with data in "test_db_2" is equal to segment count = 4, row count = 100 + When there is a "heap" table "test_schema_1.test_table_3" in "test_db_1" with "100" rows + Then distribution information from table "test_schema_1.test_table_3" with data in "test_db_1" is equal to segment count = 4, row count = 100 + + Scenario: 5. rebalance - check interrupted rebalance after shrink. + Given the database is not running + And a working directory of the test as '/data/gpdata/ggrebalance' + And a cluster is created with mirrors on "cdw" and "sdw1, sdw2, sdw3" + And database "test_db_1" exists + And schema "test_schema_1" exists in "test_db_1" + And there is a "heap" table "test_schema_1.test_table_1" in "test_db_1" with "100" rows + And there is a "ao" table "test_schema_1.test_table_2" in "test_db_1" with "100" rows + And database "test_db_2" exists + And schema "test_schema_2" exists in "test_db_2" + And there is a "heap" table "test_schema_2.test_table_1" in "test_db_2" with "100" rows + And there is a "ao" table "test_schema_2.test_table_2" in "test_db_2" with "100" rows + And all files in gpAdminLogs directory are deleted + And set fault inject "on_enter_STATE_REBALANCE_EXECUTION_STARTED_begin" + When the user runs "ggrebalance -x 4 --remove-hosts sdw3 -d '/home/gpadmin/gpdb_src/gpAux/gpdemo/datadirs/dbfast, /home/gpadmin/gpdb_src/gpAux/gpdemo/datadirs/dbfast_mirror'" + Then ggrebalance should return a return code of 1 + And ggrebalance should print "ggrebalance failed" to logfile with latest timestamp + And unset fault inject + And all files in gpAdminLogs directory are deleted + When the user runs "ggrebalance" + Then ggrebalance should return a return code of 0 + And ggrebalance should print "Continue interrupted rebalance operation..." to logfile with latest timestamp + And ggrebalance should print "Rebalance is complete" to logfile with latest timestamp + And the cluster configuration has 2 segments where "hostname='sdw1' and content > -1 and role = 'p' and status = 'u'" + And the cluster configuration has 2 segments where "hostname='sdw1' and content > -1 and role = 'm' and status = 'u'" + And the cluster configuration has 2 segments where "hostname='sdw2' and content > -1 and role = 'p' and status = 'u'" + And the cluster configuration has 2 segments where "hostname='sdw2' and content > -1 and role = 'm' and status = 'u'" + And the cluster configuration has 0 segments where "hostname='sdw3' and content > -1 and role = 'p' and status = 'u'" + And the cluster configuration has 0 segments where "hostname='sdw3' and content > -1 and role = 'm' and status = 'u'" + And distribution information from table "test_schema_1.test_table_1" with data in "test_db_1" is equal to segment count = 4, row count = 100 + And distribution information from table "test_schema_1.test_table_2" with data in "test_db_1" is equal to segment count = 4, row count = 100 + And distribution information from table "test_schema_2.test_table_1" with data in "test_db_2" is equal to segment count = 4, row count = 100 + And distribution information from table "test_schema_2.test_table_2" with data in "test_db_2" is equal to segment count = 4, row count = 100 + When there is a "heap" table "test_schema_1.test_table_3" in "test_db_1" with "100" rows + Then distribution information from table "test_schema_1.test_table_3" with data in "test_db_1" is equal to segment count = 4, row count = 100 + + Scenario Outline: 6. rebalance - case when mirror and primary swap their hosts. + Given the database is not running + And a working directory of the test as '/data/gpdata/ggrebalance' + And the user runs command "gpssh -h sdw1 -h sdw2 -h sdw3 -e 'mkdir -p /data/gpdata/ggrebalance/primary'" + And the user runs command "gpssh -h sdw1 -h sdw2 -h sdw3 -e 'mkdir -p /data/gpdata/ggrebalance/mirror'" + And the temporary file "/tmp/rebalance_s6" is created with content + """ + TRUSTED_SHELL=ssh + ENCODING=UNICODE + SEG_PREFIX=gpseg + HEAP_CHECKSUM=on + HBA_HOSTNAMES=0 + QD_PRIMARY_ARRAY=cdw~cdw~7000~/data/gpdata/ggrebalance/gpseg-1~1~-1~0 + declare -a PRIMARY_ARRAY=( + sdw1~sdw1~7002~/data/gpdata/ggrebalance/primary/gpseg0~2~0~11100 + sdw1~sdw1~7003~/data/gpdata/ggrebalance/primary/gpseg1~3~1~11110 + sdw1~sdw1~7004~/data/gpdata/ggrebalance/primary/gpseg2~4~2~11220 + sdw2~sdw2~7003~/data/gpdata/ggrebalance/primary/gpseg3~5~3~11350 + sdw3~sdw3~7004~/data/gpdata/ggrebalance/primary/gpseg4~6~4~11360 + sdw3~sdw3~7005~/data/gpdata/ggrebalance/primary/gpseg5~7~5~11370 + ) + declare -a MIRROR_ARRAY=( + sdw3~sdw3~7050~/data/gpdata/ggrebalance/mirror/gpseg0~8~0~51130 + sdw3~sdw3~7051~/data/gpdata/ggrebalance/mirror/gpseg1~9~1~51140 + sdw2~sdw2~7052~/data/gpdata/ggrebalance/mirror/gpseg2~10~2~51160 + sdw1~sdw1~7053~/data/gpdata/ggrebalance/mirror/gpseg3~11~3~51160 + sdw2~sdw2~7054~/data/gpdata/ggrebalance/mirror/gpseg4~12~4~51200 + sdw2~sdw2~7055~/data/gpdata/ggrebalance/mirror/gpseg5~13~5~51136 + ) + """ + When initialize a cluster using "/tmp/rebalance_s6" + Then the temporary file "/tmp/rebalance_s6" is removed + Given the environment variable "COORDINATOR_DATA_DIRECTORY" is set to "/data/gpdata/ggrebalance/gpseg-1" + And coordinator data directory is updated + And database "test_db_1" exists + And schema "test_schema_1" exists in "test_db_1" + And there is a "heap" table "test_schema_1.test_table_1" in "test_db_1" with "100" rows + And there is a "ao" table "test_schema_1.test_table_2" in "test_db_1" with "100" rows + And database "test_db_2" exists + And schema "test_schema_2" exists in "test_db_2" + And there is a "heap" table "test_schema_2.test_table_1" in "test_db_2" with "100" rows + And there is a "ao" table "test_schema_2.test_table_2" in "test_db_2" with "100" rows + And all files in gpAdminLogs directory are deleted + When the user runs + Then ggrebalance should return a return code of 0 + And ggrebalance should print "Rebalance is complete" to logfile with latest timestamp + And the cluster configuration has 1 segments where "hostname='sdw1' and content = 2 and role = 'm' and status = 'u'" + And the cluster configuration has 1 segments where "hostname='sdw2' and content = 2 and role = 'p' and status = 'u'" + And distribution information from table "test_schema_1.test_table_1" with data in "test_db_1" is equal to segment count = 6, row count = 100 + And distribution information from table "test_schema_1.test_table_2" with data in "test_db_1" is equal to segment count = 6, row count = 100 + And distribution information from table "test_schema_2.test_table_1" with data in "test_db_2" is equal to segment count = 6, row count = 100 + And distribution information from table "test_schema_2.test_table_2" with data in "test_db_2" is equal to segment count = 6, row count = 100 + When there is a "heap" table "test_schema_1.test_table_3" in "test_db_1" with "100" rows + Then distribution information from table "test_schema_1.test_table_3" with data in "test_db_1" is equal to segment count = 6, row count = 100 + Given the database is not running + Given the user runs command "gpssh -h sdw1 -h sdw2 -h sdw3 -e 'rm -rf /data/gpdata/ggrebalance/primary'" + And the user runs command "gpssh -h sdw1 -h sdw2 -h sdw3 -e 'rm -rf /data/gpdata/ggrebalance/mirror'" + Then + And coordinator data directory is updated + + Examples: inplace swap and using 3rd host + | command | restore_env| + |"ggrebalance -x 6 -d '/data/gpdata/ggrebalance/primary, /data/gpdata/ggrebalance/mirror'"|stub| + |"ggrebalance -x 6 -d '/data/gpdata/ggrebalance/primary, /data/gpdata/ggrebalance/mirror' --inplace-swap-roles"|"COORDINATOR_DATA_DIRECTORY" environment variable should be restored| + + Scenario: 7. rebalance - case with multiple swaps. + Given the database is not running + And a working directory of the test as '/data/gpdata/ggrebalance' + And the user runs command "gpssh -h sdw1 -h sdw2 -h sdw3 -e 'mkdir -p /data/gpdata/ggrebalance/primary'" + And the user runs command "gpssh -h sdw1 -h sdw2 -h sdw3 -e 'mkdir -p /data/gpdata/ggrebalance/mirror'" + And the temporary file "/tmp/rebalance_s6" is created with content + """ + TRUSTED_SHELL=ssh + ENCODING=UNICODE + SEG_PREFIX=gpseg + HEAP_CHECKSUM=on + HBA_HOSTNAMES=0 + QD_PRIMARY_ARRAY=cdw~cdw~7000~/data/gpdata/ggrebalance/gpseg-1~1~-1~0 + declare -a PRIMARY_ARRAY=( + sdw1~sdw1~7002~/data/gpdata/ggrebalance/primary/gpseg0~2~0~11100 + sdw1~sdw1~7003~/data/gpdata/ggrebalance/primary/gpseg1~3~1~11110 + sdw1~sdw1~7004~/data/gpdata/ggrebalance/primary/gpseg2~4~2~11220 + sdw1~sdw1~7005~/data/gpdata/ggrebalance/primary/gpseg3~5~3~11350 + sdw2~sdw2~7006~/data/gpdata/ggrebalance/primary/gpseg4~6~4~11360 + sdw2~sdw2~7007~/data/gpdata/ggrebalance/primary/gpseg5~7~5~11370 + ) + declare -a MIRROR_ARRAY=( + sdw2~sdw2~7050~/data/gpdata/ggrebalance/mirror/gpseg0~8~0~51130 + sdw2~sdw2~7051~/data/gpdata/ggrebalance/mirror/gpseg1~9~1~51140 + sdw3~sdw3~7052~/data/gpdata/ggrebalance/mirror/gpseg2~10~2~51160 + sdw3~sdw3~7053~/data/gpdata/ggrebalance/mirror/gpseg3~11~3~51160 + sdw3~sdw3~7054~/data/gpdata/ggrebalance/mirror/gpseg4~12~4~51200 + sdw3~sdw3~7055~/data/gpdata/ggrebalance/mirror/gpseg5~13~5~51136 + ) + """ + When initialize a cluster using "/tmp/rebalance_s6" + Then the temporary file "/tmp/rebalance_s6" is removed + Given the environment variable "COORDINATOR_DATA_DIRECTORY" is set to "/data/gpdata/ggrebalance/gpseg-1" + And coordinator data directory is updated + And database "test_db_1" exists + And schema "test_schema_1" exists in "test_db_1" + And there is a "heap" table "test_schema_1.test_table_1" in "test_db_1" with "100" rows + And there is a "ao" table "test_schema_1.test_table_2" in "test_db_1" with "100" rows + And database "test_db_2" exists + And schema "test_schema_2" exists in "test_db_2" + And there is a "heap" table "test_schema_2.test_table_1" in "test_db_2" with "100" rows + And there is a "ao" table "test_schema_2.test_table_2" in "test_db_2" with "100" rows + And all files in gpAdminLogs directory are deleted + When the user runs "ggrebalance -x 6 -d '/data/gpdata/ggrebalance/primary, /data/gpdata/ggrebalance/mirror'" + Then ggrebalance should return a return code of 0 + And ggrebalance should print "Rebalance is complete" to logfile with latest timestamp + And the cluster configuration has 1 segments where "hostname='sdw1' and content = 2 and role = 'm' and status = 'u'" + And the cluster configuration has 1 segments where "hostname='sdw3' and content = 2 and role = 'p' and status = 'u'" + And the cluster configuration has 1 segments where "hostname='sdw1' and content = 3 and role = 'm' and status = 'u'" + And the cluster configuration has 1 segments where "hostname='sdw3' and content = 3 and role = 'p' and status = 'u'" + And distribution information from table "test_schema_1.test_table_1" with data in "test_db_1" is equal to segment count = 6, row count = 100 + And distribution information from table "test_schema_1.test_table_2" with data in "test_db_1" is equal to segment count = 6, row count = 100 + And distribution information from table "test_schema_2.test_table_1" with data in "test_db_2" is equal to segment count = 6, row count = 100 + And distribution information from table "test_schema_2.test_table_2" with data in "test_db_2" is equal to segment count = 6, row count = 100 + When there is a "heap" table "test_schema_1.test_table_3" in "test_db_1" with "100" rows + Then distribution information from table "test_schema_1.test_table_3" with data in "test_db_1" is equal to segment count = 6, row count = 100 + Given the database is not running + Given the user runs command "gpssh -h sdw1 -h sdw2 -h sdw3 -e 'rm -rf /data/gpdata/ggrebalance/primary'" + And the user runs command "gpssh -h sdw1 -h sdw2 -h sdw3 -e 'rm -rf /data/gpdata/ggrebalance/mirror'" + Then "COORDINATOR_DATA_DIRECTORY" environment variable should be restored + And coordinator data directory is updated diff --git a/gpMgmt/test/behave/mgmt_utils/ggrebalance_shrink.feature b/gpMgmt/test/behave/mgmt_utils/ggrebalance_shrink.feature new file mode 100755 index 000000000000..48241783d44f --- /dev/null +++ b/gpMgmt/test/behave/mgmt_utils/ggrebalance_shrink.feature @@ -0,0 +1,804 @@ +@ggrebalance_shrink +Feature: ggrebalance behave tests + + Scenario: test 1.1. shrink - check continue after interrupted state, if interruption is done before the rebalance schema creation + Given the database is not running + And a working directory of the test as '/data/gpdata/ggrebalance' + And a cluster is created with mirrors on "cdw" and "sdw1" + And segment information for content 1 is saved in context + And all files in gpAdminLogs directory are deleted + And set fault inject "on_enter_STATE_SETUP_SCHEMA_STARTED_begin" + And database "test_db_1" exists + And schema "test_schema_1" exists in "test_db_1" + And there is a "heap" table "test_schema_1.test_table_1" in "test_db_1" with "100" rows + And there is a "ao" table "test_schema_1.test_table_2" in "test_db_1" with "100" rows + And there is a "heap" partition table "test_schema_1.part_test_table_1" in "test_db_1" with "100" rows + And there is a "ao" partition table "test_schema_1.part_test_table_2" in "test_db_1" with "100" rows + And there is an unlogged "heap" table "test_schema_1.unlogged_test_table_1" in "test_db_1" with "100" rows + And a materialized view "test_schema_1.mv_test_table_1" exists on table "test_schema_1.test_table_1" + And database "gptest" exists + And the user create a writable external table with name "ext_test" + And database "test_db_2" exists + And schema "test_schema_2" exists in "test_db_2" + And there is a "heap" table "test_schema_2.test_table_1" in "test_db_2" with "100" rows + And there is a "ao" table "test_schema_2.test_table_2" in "test_db_2" with "100" rows + When the user runs "ggrebalance -x 1 --skip-rebalance" + Then ggrebalance should return a return code of 1 + And ggrebalance should print "ggrebalance failed" to logfile with latest timestamp + And unset fault inject + When the user runs "ggrebalance" + Then ggrebalance should return a return code of 1 + And ggrebalance should print "Rebalance schema doesn't exists and no shrink plan is supplied. Please specify shrink plan." to logfile with latest timestamp + When the user runs "ggrebalance -x 1 --skip-rebalance" + Then ggrebalance should return a return code of 0 + And ggrebalance should print "Shrink is complete" to logfile with latest timestamp + And verify no segment running for saved segment information + And distribution information from table "test_schema_1.test_table_1" with data in "test_db_1" is equal to segment count = 1, row count = 100 + And distribution information from table "test_schema_1.test_table_2" with data in "test_db_1" is equal to segment count = 1, row count = 100 + And distribution information from table "test_schema_1.part_test_table_1" with data in "test_db_1" is equal to segment count = 1, row count = 100 + And distribution information from table "test_schema_1.part_test_table_2" with data in "test_db_1" is equal to segment count = 1, row count = 100 + And distribution information from table "test_schema_1.unlogged_test_table_1" with data in "test_db_1" is equal to segment count = 1, row count = 100 + And distribution information from table "test_schema_1.mv_test_table_1" with data in "test_db_1" is equal to segment count = 1, row count = 100 + And the numsegments of table "ext_test" is 1 + And distribution information from table "test_schema_2.test_table_1" with data in "test_db_2" is equal to segment count = 1, row count = 100 + And distribution information from table "test_schema_2.test_table_2" with data in "test_db_2" is equal to segment count = 1, row count = 100 + When there is a "heap" table "test_schema_1.test_table_3" in "test_db_1" with "100" rows + Then distribution information from table "test_schema_1.test_table_3" with data in "test_db_1" is equal to segment count = 1, row count = 100 + + Scenario Outline: test 1.2. shrink - check continue after interrupted state + Given the database is not running + And a working directory of the test as '/data/gpdata/ggrebalance' + And a cluster is created with mirrors on "cdw" and "sdw1" + And segment information for content 1 is saved in context + And all files in gpAdminLogs directory are deleted + And set fault inject "" + And database "test_db_1" exists + And schema "test_schema_1" exists in "test_db_1" + And there is a "heap" table "test_schema_1.test_table_1" in "test_db_1" with "100" rows + And there is a "ao" table "test_schema_1.test_table_2" in "test_db_1" with "100" rows + And there is a "heap" partition table "test_schema_1.part_test_table_1" in "test_db_1" with "100" rows + And there is a "ao" partition table "test_schema_1.part_test_table_2" in "test_db_1" with "100" rows + And there is an unlogged "heap" table "test_schema_1.unlogged_test_table_1" in "test_db_1" with "100" rows + And a materialized view "test_schema_1.mv_test_table_1" exists on table "test_schema_1.test_table_1" + And the user connects to "gptest" with named connection "test_connection" + And the user executes "CREATE TEMP TABLE temp_table(a int);" with named connection "test_connection" + And database "gptest" exists + And the user create a writable external table with name "ext_test" + And database "test_db_2" exists + And schema "test_schema_2" exists in "test_db_2" + And there is a "heap" table "test_schema_2.test_table_1" in "test_db_2" with "100" rows + And there is a "ao" table "test_schema_2.test_table_2" in "test_db_2" with "100" rows + When the user runs "ggrebalance -x 1 --parallel 1 --batch-size 1 --skip-rebalance" + Then ggrebalance should return a return code of 1 + And ggrebalance should print "ggrebalance failed" to logfile with latest timestamp + And unset fault inject + And the user drops the named connection "test_connection" + When execute following sql in db "postgres" and store result in the context + """ + select count(1) as temp_tables_for_redistribute from ggrebalance.table_rebalance_status_detail where schema_name LIKE 'pg\_temp\_%'; + """ + Then validate that following rows are in the stored rows + | temp_tables_for_redistribute | + | 0 | + When the user runs "ggrebalance -x 1 --parallel 1 --batch-size 1 --skip-rebalance" + Then ggrebalance should return a return code of 1 + And ggrebalance should print "Can't start a new operation, because the previous one was interrupted" to logfile with latest timestamp + When the user runs "ggrebalance -x 1 --parallel 1 --batch-size 1 --skip-rebalance" + Then ggrebalance should return a return code of 1 + And ggrebalance should print "Can't start a new operation, because the previous one was interrupted" to logfile with latest timestamp + When the user runs "ggrebalance --parallel 1 --batch-size 1" + Then ggrebalance should return a return code of 0 + And ggrebalance should print "Shrink is complete" to logfile with latest timestamp + And verify no segment running for saved segment information + And distribution information from table "test_schema_1.test_table_1" with data in "test_db_1" is equal to segment count = 1, row count = 100 + And distribution information from table "test_schema_1.test_table_2" with data in "test_db_1" is equal to segment count = 1, row count = 100 + And distribution information from table "test_schema_1.part_test_table_1" with data in "test_db_1" is equal to segment count = 1, row count = 100 + And distribution information from table "test_schema_1.part_test_table_2" with data in "test_db_1" is equal to segment count = 1, row count = 100 + And distribution information from table "test_schema_1.unlogged_test_table_1" with data in "test_db_1" is equal to segment count = 1, row count = 100 + And distribution information from table "test_schema_1.mv_test_table_1" with data in "test_db_1" is equal to segment count = 1, row count = 100 + And the numsegments of table "ext_test" is 1 + And distribution information from table "test_schema_2.test_table_1" with data in "test_db_2" is equal to segment count = 1, row count = 100 + And distribution information from table "test_schema_2.test_table_2" with data in "test_db_2" is equal to segment count = 1, row count = 100 + When there is a "heap" table "test_schema_1.test_table_3" in "test_db_1" with "100" rows + Then distribution information from table "test_schema_1.test_table_3" with data in "test_db_1" is equal to segment count = 1, row count = 100 + + Examples: + | fault_name | + | on_enter_STATE_SETUP_SCHEMA_STARTED_end | + | on_enter_STATE_SETUP_SCHEMA_DONE_begin | + | on_enter_STATE_SETUP_SCHEMA_DONE_end | + | on_enter_STATE_BACKUP_CATALOG_AND_UPDATE_TARGET_SEGMENT_COUNT_STARTED_begin | + | on_enter_STATE_BACKUP_CATALOG_AND_UPDATE_TARGET_SEGMENT_COUNT_STARTED_end | + | on_enter_STATE_BACKUP_CATALOG_AND_UPDATE_TARGET_SEGMENT_COUNT_DONE_begin | + | on_enter_STATE_BACKUP_CATALOG_AND_UPDATE_TARGET_SEGMENT_COUNT_DONE_end | + | on_enter_STATE_PREPARE_SHRINK_SCHEMA_STARTED_begin | + | on_enter_STATE_PREPARE_SHRINK_SCHEMA_STARTED_end | + | on_enter_STATE_PREPARE_SHRINK_SCHEMA_DONE_begin | + | on_enter_STATE_PREPARE_SHRINK_SCHEMA_DONE_end | + | on_enter_STATE_SHRINK_TABLES_STARTED_begin | + | on_enter_STATE_SHRINK_TABLES_STARTED_end | + | on_enter_STATE_SHRINK_TABLES_DONE_begin | + | on_enter_STATE_SHRINK_TABLES_DONE_end | + | on_enter_STATE_SHRINK_CATALOG_STARTED_begin | + | on_enter_STATE_SHRINK_CATALOG_STARTED_end | + | on_enter_STATE_SHRINK_CATALOG_DONE_begin | + | on_enter_STATE_SHRINK_CATALOG_DONE_end | + | on_enter_STATE_SHRINK_SEGMENTS_STOP_STARTED_begin | + | on_enter_STATE_SHRINK_SEGMENTS_STOP_STARTED_end | + | fault_rebalance_table_test_db_2.test_schema_2.test_table_1 | + | fault_segment_stop_dbid_3 | + + Scenario Outline: test 1.3. test shrink continue after cluster restart + Given the database is not running + And a working directory of the test as '/data/gpdata/ggrebalance' + And a cluster is created with mirrors on "cdw" and "sdw1" + And segment information for content 1 is saved in context + And all files in gpAdminLogs directory are deleted + And database "test_db_1" exists + And schema "test_schema_1" exists in "test_db_1" + And there is a "heap" table "test_schema_1.test_table_1" in "test_db_1" with "100" rows + And there is a "ao" table "test_schema_1.test_table_2" in "test_db_1" with "100" rows + And there is a "heap" partition table "test_schema_1.part_test_table_1" in "test_db_1" with "100" rows + And there is a "ao" partition table "test_schema_1.part_test_table_2" in "test_db_1" with "100" rows + And there is an unlogged "heap" table "test_schema_1.unlogged_test_table_1" in "test_db_1" with "100" rows + And a materialized view "test_schema_1.mv_test_table_1" exists on table "test_schema_1.test_table_1" + And database "gptest" exists + And the user create a writable external table with name "ext_test" + And database "test_db_2" exists + And schema "test_schema_2" exists in "test_db_2" + And there is a "heap" table "test_schema_2.test_table_1" in "test_db_2" with "100" rows + And there is a "ao" table "test_schema_2.test_table_2" in "test_db_2" with "100" rows + When set fault inject "" + And the user runs "ggrebalance -x 1 --skip-rebalance" + Then ggrebalance should return a return code of 1 + And ggrebalance should print "ggrebalance failed" to logfile with latest timestamp + And unset fault inject + When the user runs "gpstop -arf" + Then gpstart should return a return code of 0 + When there is a "heap" table "test_schema_2.test_table_3" in "test_db_2" with data + And the user runs "ggrebalance" + Then ggrebalance should return a return code of 0 + And ggrebalance should print "Shrink is complete" to logfile with latest timestamp + And verify no segment running for saved segment information + And distribution information from table "test_schema_1.test_table_1" with data in "test_db_1" is equal to segment count = 1, row count = 100 + And distribution information from table "test_schema_1.test_table_2" with data in "test_db_1" is equal to segment count = 1, row count = 100 + And distribution information from table "test_schema_1.part_test_table_1" with data in "test_db_1" is equal to segment count = 1, row count = 100 + And distribution information from table "test_schema_1.part_test_table_2" with data in "test_db_1" is equal to segment count = 1, row count = 100 + And distribution information from table "test_schema_1.unlogged_test_table_1" with data in "test_db_1" is equal to segment count = 1, row count = 100 + And distribution information from table "test_schema_1.mv_test_table_1" with data in "test_db_1" is equal to segment count = 1, row count = 100 + And the numsegments of table "ext_test" is 1 + And distribution information from table "test_schema_2.test_table_2" with data in "test_db_2" is equal to segment count = 1, row count = 100 + And distribution information from table "test_schema_2.test_table_3" with data in "test_db_2" is equal to segment count = 1, row count = 1094 + + Examples: + | fault_name | + | on_enter_STATE_BACKUP_CATALOG_AND_UPDATE_TARGET_SEGMENT_COUNT_DONE_begin | + | on_enter_STATE_PREPARE_SHRINK_SCHEMA_DONE_begin | + | on_enter_STATE_SHRINK_TABLES_DONE_begin | + | on_enter_STATE_SHRINK_TABLES_DONE_end | + | on_enter_STATE_SHRINK_CATALOG_STARTED_begin | + | on_enter_STATE_SHRINK_CATALOG_STARTED_end | + + Scenario: test 2.1. shrink - check rollback after interrupted state, if interruption is done before the rebalance schema creation + Given the database is not running + And a working directory of the test as '/data/gpdata/ggrebalance' + And a cluster is created with mirrors on "cdw" and "sdw1" + And all files in gpAdminLogs directory are deleted + And set fault inject "on_enter_STATE_SETUP_SCHEMA_STARTED_begin" + And database "test_db_1" exists + And schema "test_schema_1" exists in "test_db_1" + And there is a "heap" table "test_schema_1.test_table_1" in "test_db_1" with "100" rows + And there is a "ao" table "test_schema_1.test_table_2" in "test_db_1" with "100" rows + And there is a "heap" partition table "test_schema_1.part_test_table_1" in "test_db_1" with "100" rows + And there is a "ao" partition table "test_schema_1.part_test_table_2" in "test_db_1" with "100" rows + And there is an unlogged "heap" table "test_schema_1.unlogged_test_table_1" in "test_db_1" with "100" rows + And a materialized view "test_schema_1.mv_test_table_1" exists on table "test_schema_1.test_table_1" + And database "gptest" exists + And the user create a writable external table with name "ext_test" + And database "test_db_2" exists + And schema "test_schema_2" exists in "test_db_2" + And there is a "heap" table "test_schema_2.test_table_1" in "test_db_2" with "100" rows + And there is a "ao" table "test_schema_2.test_table_2" in "test_db_2" with "100" rows + When the user runs "ggrebalance -x 1 --skip-rebalance" + Then ggrebalance should return a return code of 1 + And ggrebalance should print "ggrebalance failed" to logfile with latest timestamp + And unset fault inject + When the user runs "ggrebalance -r" + Then ggrebalance should return a return code of 0 + And ggrebalance should print "Rebalance schema doesn't exist. Can't perform rollback." to logfile with latest timestamp + When the user runs "ggrebalance -c" + Then ggrebalance should return a return code of 0 + And distribution information from table "test_schema_1.test_table_1" with data in "test_db_1" is equal to segment count = 2, row count = 100 + And distribution information from table "test_schema_1.test_table_2" with data in "test_db_1" is equal to segment count = 2, row count = 100 + And distribution information from table "test_schema_1.part_test_table_1" with data in "test_db_1" is equal to segment count = 2, row count = 100 + And distribution information from table "test_schema_1.part_test_table_2" with data in "test_db_1" is equal to segment count = 2, row count = 100 + And distribution information from table "test_schema_1.unlogged_test_table_1" with data in "test_db_1" is equal to segment count = 2, row count = 100 + And distribution information from table "test_schema_1.mv_test_table_1" with data in "test_db_1" is equal to segment count = 2, row count = 100 + And the numsegments of table "ext_test" is 2 + And distribution information from table "test_schema_2.test_table_1" with data in "test_db_2" is equal to segment count = 2, row count = 100 + And distribution information from table "test_schema_2.test_table_2" with data in "test_db_2" is equal to segment count = 2, row count = 100 + When there is a "heap" table "test_schema_1.test_table_3" in "test_db_1" with "100" rows + Then distribution information from table "test_schema_1.test_table_3" with data in "test_db_1" is equal to segment count = 2, row count = 100 + + Scenario Outline: test 2.2. shrink - check rollback after interrupted state + Given the database is not running + And a working directory of the test as '/data/gpdata/ggrebalance' + And a cluster is created with mirrors on "cdw" and "sdw1" + And all files in gpAdminLogs directory are deleted + And set fault inject "" + And database "test_db_1" exists + And schema "test_schema_1" exists in "test_db_1" + And there is a "heap" table "test_schema_1.test_table_1" in "test_db_1" with "100" rows + And there is a "ao" table "test_schema_1.test_table_2" in "test_db_1" with "100" rows + And there is a "heap" partition table "test_schema_1.part_test_table_1" in "test_db_1" with "100" rows + And there is a "ao" partition table "test_schema_1.part_test_table_2" in "test_db_1" with "100" rows + And there is an unlogged "heap" table "test_schema_1.unlogged_test_table_1" in "test_db_1" with "100" rows + And a materialized view "test_schema_1.mv_test_table_1" exists on table "test_schema_1.test_table_1" + And database "gptest" exists + And the user create a writable external table with name "ext_test" + And database "test_db_2" exists + And schema "test_schema_2" exists in "test_db_2" + And there is a "heap" table "test_schema_2.test_table_1" in "test_db_2" with "100" rows + And there is a "ao" table "test_schema_2.test_table_2" in "test_db_2" with "100" rows + When the user runs "ggrebalance -x 1 --parallel 1 --batch-size 1 --skip-rebalance" + Then ggrebalance should return a return code of 1 + And ggrebalance should print "ggrebalance failed" to logfile with latest timestamp + And unset fault inject + When the user runs "ggrebalance -r" + Then ggrebalance should return a return code of 0 + And ggrebalance should print "Rollback is complete" to logfile with latest timestamp + And distribution information from table "test_schema_1.test_table_1" with data in "test_db_1" is equal to segment count = 2, row count = 100 + And distribution information from table "test_schema_1.test_table_2" with data in "test_db_1" is equal to segment count = 2, row count = 100 + And distribution information from table "test_schema_1.part_test_table_1" with data in "test_db_1" is equal to segment count = 2, row count = 100 + And distribution information from table "test_schema_1.part_test_table_2" with data in "test_db_1" is equal to segment count = 2, row count = 100 + And distribution information from table "test_schema_1.unlogged_test_table_1" with data in "test_db_1" is equal to segment count = 2, row count = 100 + And distribution information from table "test_schema_1.mv_test_table_1" with data in "test_db_1" is equal to segment count = 2, row count = 100 + And the numsegments of table "ext_test" is 2 + And distribution information from table "test_schema_2.test_table_1" with data in "test_db_2" is equal to segment count = 2, row count = 100 + And distribution information from table "test_schema_2.test_table_2" with data in "test_db_2" is equal to segment count = 2, row count = 100 + When there is a "heap" table "test_schema_1.test_table_3" in "test_db_1" with "100" rows + Then distribution information from table "test_schema_1.test_table_3" with data in "test_db_1" is equal to segment count = 2, row count = 100 + + Examples: + | fault_name | + | on_enter_STATE_SETUP_SCHEMA_STARTED_end | + | on_enter_STATE_SETUP_SCHEMA_DONE_begin | + | on_enter_STATE_SETUP_SCHEMA_DONE_end | + | on_enter_STATE_BACKUP_CATALOG_AND_UPDATE_TARGET_SEGMENT_COUNT_STARTED_begin | + | on_enter_STATE_BACKUP_CATALOG_AND_UPDATE_TARGET_SEGMENT_COUNT_STARTED_end | + | on_enter_STATE_BACKUP_CATALOG_AND_UPDATE_TARGET_SEGMENT_COUNT_DONE_begin | + | on_enter_STATE_BACKUP_CATALOG_AND_UPDATE_TARGET_SEGMENT_COUNT_DONE_end | + | on_enter_STATE_PREPARE_SHRINK_SCHEMA_STARTED_begin | + | on_enter_STATE_PREPARE_SHRINK_SCHEMA_STARTED_end | + | on_enter_STATE_PREPARE_SHRINK_SCHEMA_DONE_begin | + | on_enter_STATE_PREPARE_SHRINK_SCHEMA_DONE_end | + | on_enter_STATE_SHRINK_TABLES_STARTED_begin | + | on_enter_STATE_SHRINK_TABLES_STARTED_end | + | on_enter_STATE_SHRINK_TABLES_DONE_begin | + | on_enter_STATE_SHRINK_TABLES_DONE_end | + | on_enter_STATE_SHRINK_CATALOG_STARTED_begin | + | fault_rebalance_table_test_db_2.test_schema_2.test_table_1 | + + Scenario Outline: test 2.3. shrink - check rollback after interrupted state (interruption is done after the point of no return). Rollback fails. So just continue shrink. + Given the database is not running + And a working directory of the test as '/data/gpdata/ggrebalance' + And a cluster is created with mirrors on "cdw" and "sdw1" + And segment information for content 1 is saved in context + And all files in gpAdminLogs directory are deleted + And set fault inject "" + And database "test_db_1" exists + And schema "test_schema_1" exists in "test_db_1" + And there is a "heap" table "test_schema_1.test_table_1" in "test_db_1" with "100" rows + And there is a "ao" table "test_schema_1.test_table_2" in "test_db_1" with "100" rows + And there is a "heap" partition table "test_schema_1.part_test_table_1" in "test_db_1" with "100" rows + And there is a "ao" partition table "test_schema_1.part_test_table_2" in "test_db_1" with "100" rows + And there is an unlogged "heap" table "test_schema_1.unlogged_test_table_1" in "test_db_1" with "100" rows + And a materialized view "test_schema_1.mv_test_table_1" exists on table "test_schema_1.test_table_1" + And database "gptest" exists + And the user create a writable external table with name "ext_test" + And database "test_db_2" exists + And schema "test_schema_2" exists in "test_db_2" + And there is a "heap" table "test_schema_2.test_table_1" in "test_db_2" with "100" rows + And there is a "ao" table "test_schema_2.test_table_2" in "test_db_2" with "100" rows + When the user runs "ggrebalance -x 1 --skip-rebalance" + Then ggrebalance should return a return code of 1 + And ggrebalance should print "ggrebalance failed" to logfile with latest timestamp + And unset fault inject + When the user runs "ggrebalance -r" + Then ggrebalance should return a return code of 0 + And ggrebalance should print "Can't perform rollback as the catalog is already updated" to logfile with latest timestamp + When the user runs "ggrebalance" + Then ggrebalance should return a return code of 0 + And ggrebalance should print "Shrink is complete" to logfile with latest timestamp + And verify no segment running for saved segment information + And distribution information from table "test_schema_1.test_table_1" with data in "test_db_1" is equal to segment count = 1, row count = 100 + And distribution information from table "test_schema_1.test_table_2" with data in "test_db_1" is equal to segment count = 1, row count = 100 + And distribution information from table "test_schema_1.part_test_table_1" with data in "test_db_1" is equal to segment count = 1, row count = 100 + And distribution information from table "test_schema_1.part_test_table_2" with data in "test_db_1" is equal to segment count = 1, row count = 100 + And distribution information from table "test_schema_1.unlogged_test_table_1" with data in "test_db_1" is equal to segment count = 1, row count = 100 + And distribution information from table "test_schema_1.mv_test_table_1" with data in "test_db_1" is equal to segment count = 1, row count = 100 + And the numsegments of table "ext_test" is 1 + And distribution information from table "test_schema_2.test_table_1" with data in "test_db_2" is equal to segment count = 1, row count = 100 + And distribution information from table "test_schema_2.test_table_2" with data in "test_db_2" is equal to segment count = 1, row count = 100 + When there is a "heap" table "test_schema_1.test_table_3" in "test_db_1" with "100" rows + Then distribution information from table "test_schema_1.test_table_3" with data in "test_db_1" is equal to segment count = 1, row count = 100 + + Examples: + | fault_name | + | on_enter_STATE_SHRINK_CATALOG_STARTED_end | + | on_enter_STATE_SHRINK_CATALOG_DONE_begin | + | on_enter_STATE_SHRINK_CATALOG_DONE_end | + | on_enter_STATE_SHRINK_SEGMENTS_STOP_STARTED_begin | + | on_enter_STATE_SHRINK_SEGMENTS_STOP_STARTED_end | + | fault_segment_stop_dbid_3 | + + Scenario Outline: test 2.4. shrink - check rollback after interrupted state and cluster restart + Given the database is not running + And a working directory of the test as '/data/gpdata/ggrebalance' + And a cluster is created with mirrors on "cdw" and "sdw1" + And all files in gpAdminLogs directory are deleted + And database "test_db_1" exists + And schema "test_schema_1" exists in "test_db_1" + And there is a "heap" table "test_schema_1.test_table_1" in "test_db_1" with "100" rows + And there is a "ao" table "test_schema_1.test_table_2" in "test_db_1" with "100" rows + And there is a "heap" partition table "test_schema_1.part_test_table_1" in "test_db_1" with "100" rows + And there is a "ao" partition table "test_schema_1.part_test_table_2" in "test_db_1" with "100" rows + And there is an unlogged "heap" table "test_schema_1.unlogged_test_table_1" in "test_db_1" with "100" rows + And a materialized view "test_schema_1.mv_test_table_1" exists on table "test_schema_1.test_table_1" + And database "gptest" exists + And the user create a writable external table with name "ext_test" + And database "test_db_2" exists + And schema "test_schema_2" exists in "test_db_2" + And there is a "heap" table "test_schema_2.test_table_1" in "test_db_2" with "100" rows + And there is a "ao" table "test_schema_2.test_table_2" in "test_db_2" with "100" rows + When set fault inject "" + And the user runs "ggrebalance -x 1 --skip-rebalance" + Then ggrebalance should return a return code of 1 + And ggrebalance should print "ggrebalance failed" to logfile with latest timestamp + And unset fault inject + And there is a "heap" table "test_schema_2.test_table_3" in "test_db_2" with "200" rows + When the user runs "gpstop -arf" + Then gpstart should return a return code of 0 + When there is a "heap" table "test_schema_2.test_table_4" in "test_db_2" with "300" rows + And the user runs "ggrebalance -r" + Then ggrebalance should return a return code of 0 + And ggrebalance should print "Rollback is complete" to logfile with latest timestamp + And distribution information from table "test_schema_1.test_table_1" with data in "test_db_1" is equal to segment count = 2, row count = 100 + And distribution information from table "test_schema_1.test_table_2" with data in "test_db_1" is equal to segment count = 2, row count = 100 + And distribution information from table "test_schema_1.part_test_table_1" with data in "test_db_1" is equal to segment count = 2, row count = 100 + And distribution information from table "test_schema_1.part_test_table_2" with data in "test_db_1" is equal to segment count = 2, row count = 100 + And distribution information from table "test_schema_1.unlogged_test_table_1" with data in "test_db_1" is equal to segment count = 2, row count = 100 + And distribution information from table "test_schema_1.mv_test_table_1" with data in "test_db_1" is equal to segment count = 2, row count = 100 + And the numsegments of table "ext_test" is 2 + And distribution information from table "test_schema_2.test_table_1" with data in "test_db_2" is equal to segment count = 2, row count = 100 + And distribution information from table "test_schema_2.test_table_2" with data in "test_db_2" is equal to segment count = 2, row count = 100 + And distribution information from table "test_schema_2.test_table_3" with data in "test_db_2" is equal to segment count = 2, row count = 200 + And distribution information from table "test_schema_2.test_table_4" with data in "test_db_2" is equal to segment count = 2, row count = 300 + Examples: + | fault_name | + | on_enter_STATE_SETUP_SCHEMA_STARTED_end | + | on_enter_STATE_SETUP_SCHEMA_DONE_begin | + | on_enter_STATE_SETUP_SCHEMA_DONE_end | + | on_enter_STATE_BACKUP_CATALOG_AND_UPDATE_TARGET_SEGMENT_COUNT_STARTED_begin | + | on_enter_STATE_BACKUP_CATALOG_AND_UPDATE_TARGET_SEGMENT_COUNT_STARTED_end | + | on_enter_STATE_BACKUP_CATALOG_AND_UPDATE_TARGET_SEGMENT_COUNT_DONE_begin | + | on_enter_STATE_BACKUP_CATALOG_AND_UPDATE_TARGET_SEGMENT_COUNT_DONE_end | + | on_enter_STATE_PREPARE_SHRINK_SCHEMA_STARTED_begin | + | on_enter_STATE_PREPARE_SHRINK_SCHEMA_STARTED_end | + | on_enter_STATE_PREPARE_SHRINK_SCHEMA_DONE_begin | + | on_enter_STATE_PREPARE_SHRINK_SCHEMA_DONE_end | + | on_enter_STATE_SHRINK_TABLES_STARTED_begin | + | on_enter_STATE_SHRINK_TABLES_STARTED_end | + | on_enter_STATE_SHRINK_TABLES_DONE_begin | + | on_enter_STATE_SHRINK_TABLES_DONE_end | + | on_enter_STATE_SHRINK_CATALOG_STARTED_begin | + | fault_rebalance_table_test_db_2.test_schema_2.test_table_1 | + + Scenario Outline: test 3.1. shrink - check continue after interrupted rollback state. In this case we fail in rollback too early, and normal shrink will be complete. + Given the database is not running + And a working directory of the test as '/data/gpdata/ggrebalance' + And a cluster is created with mirrors on "cdw" and "sdw1" + And segment information for content 1 is saved in context + And all files in gpAdminLogs directory are deleted + And set fault inject "" + And database "test_db_1" exists + And schema "test_schema_1" exists in "test_db_1" + And there is a "heap" table "test_schema_1.test_table_1" in "test_db_1" with "100" rows + And there is a "ao" table "test_schema_1.test_table_2" in "test_db_1" with "100" rows + And there is a "heap" partition table "test_schema_1.part_test_table_1" in "test_db_1" with "100" rows + And there is a "ao" partition table "test_schema_1.part_test_table_2" in "test_db_1" with "100" rows + And there is an unlogged "heap" table "test_schema_1.unlogged_test_table_1" in "test_db_1" with "100" rows + And a materialized view "test_schema_1.mv_test_table_1" exists on table "test_schema_1.test_table_1" + And database "gptest" exists + And the user create a writable external table with name "ext_test" + And database "test_db_2" exists + And schema "test_schema_2" exists in "test_db_2" + And there is a "heap" table "test_schema_2.test_table_1" in "test_db_2" with "100" rows + And there is a "ao" table "test_schema_2.test_table_2" in "test_db_2" with "100" rows + When the user runs "ggrebalance -x 1 --skip-rebalance" + Then ggrebalance should return a return code of 1 + And ggrebalance should print "ggrebalance failed" to logfile with latest timestamp + And unset fault inject + And set fault inject "" + When the user runs "ggrebalance -r" + Then ggrebalance should return a return code of 1 + And ggrebalance should print "ggrebalance failed" to logfile with latest timestamp + And unset fault inject + When the user runs "ggrebalance" + Then ggrebalance should return a return code of 0 + And ggrebalance should print "Shrink is complete" to logfile with latest timestamp + And verify no segment running for saved segment information + And distribution information from table "test_schema_1.test_table_1" with data in "test_db_1" is equal to segment count = 1, row count = 100 + And distribution information from table "test_schema_1.test_table_2" with data in "test_db_1" is equal to segment count = 1, row count = 100 + And distribution information from table "test_schema_1.part_test_table_1" with data in "test_db_1" is equal to segment count = 1, row count = 100 + And distribution information from table "test_schema_1.part_test_table_2" with data in "test_db_1" is equal to segment count = 1, row count = 100 + And distribution information from table "test_schema_1.unlogged_test_table_1" with data in "test_db_1" is equal to segment count = 1, row count = 100 + And distribution information from table "test_schema_1.mv_test_table_1" with data in "test_db_1" is equal to segment count = 1, row count = 100 + And the numsegments of table "ext_test" is 1 + And distribution information from table "test_schema_2.test_table_1" with data in "test_db_2" is equal to segment count = 1, row count = 100 + And distribution information from table "test_schema_2.test_table_2" with data in "test_db_2" is equal to segment count = 1, row count = 100 + When there is a "heap" table "test_schema_1.test_table_3" in "test_db_1" with "100" rows + Then distribution information from table "test_schema_1.test_table_3" with data in "test_db_1" is equal to segment count = 1, row count = 100 + + Examples: + | fault_name_shrink | fault_name_rollback | + | on_enter_STATE_PREPARE_SHRINK_SCHEMA_STARTED_end | on_enter_STATE_SHRINK_ROLLBACK_RESTORE_TARGET_SEGMENT_COUNT_START_begin | + | fault_rebalance_table_test_db_2.test_schema_2.test_table_1 | on_enter_STATE_SHRINK_ROLLBACK_RESTORE_TARGET_SEGMENT_COUNT_START_begin | + + Scenario Outline: test 3.2. shrink - check continue after interrupted rollback state + Given the database is not running + And a working directory of the test as '/data/gpdata/ggrebalance' + And a cluster is created with mirrors on "cdw" and "sdw1" + And all files in gpAdminLogs directory are deleted + And set fault inject "" + And database "test_db_1" exists + And schema "test_schema_1" exists in "test_db_1" + And there is a "heap" table "test_schema_1.test_table_1" in "test_db_1" with "100" rows + And there is a "ao" table "test_schema_1.test_table_2" in "test_db_1" with "100" rows + And there is a "heap" partition table "test_schema_1.part_test_table_1" in "test_db_1" with "100" rows + And there is a "ao" partition table "test_schema_1.part_test_table_2" in "test_db_1" with "100" rows + And there is an unlogged "heap" table "test_schema_1.unlogged_test_table_1" in "test_db_1" with "100" rows + And a materialized view "test_schema_1.mv_test_table_1" exists on table "test_schema_1.test_table_1" + And database "gptest" exists + And the user create a writable external table with name "ext_test" + And database "test_db_2" exists + And schema "test_schema_2" exists in "test_db_2" + And there is a "heap" table "test_schema_2.test_table_1" in "test_db_2" with "100" rows + And there is a "ao" table "test_schema_2.test_table_2" in "test_db_2" with "100" rows + When the user runs "ggrebalance -x 1 --skip-rebalance" + Then ggrebalance should return a return code of 1 + And ggrebalance should print "ggrebalance failed" to logfile with latest timestamp + And unset fault inject + And set fault inject "" + When the user runs "ggrebalance -r" + Then ggrebalance should return a return code of 1 + And ggrebalance should print "ggrebalance failed" to logfile with latest timestamp + And unset fault inject + When the user runs "ggrebalance" + Then ggrebalance should return a return code of 0 + And ggrebalance should print "Rollback is complete" to logfile with latest timestamp + And distribution information from table "test_schema_1.test_table_1" with data in "test_db_1" is equal to segment count = 2, row count = 100 + And distribution information from table "test_schema_1.test_table_2" with data in "test_db_1" is equal to segment count = 2, row count = 100 + And distribution information from table "test_schema_1.part_test_table_1" with data in "test_db_1" is equal to segment count = 2, row count = 100 + And distribution information from table "test_schema_1.part_test_table_2" with data in "test_db_1" is equal to segment count = 2, row count = 100 + And distribution information from table "test_schema_1.unlogged_test_table_1" with data in "test_db_1" is equal to segment count = 2, row count = 100 + And distribution information from table "test_schema_1.mv_test_table_1" with data in "test_db_1" is equal to segment count = 2, row count = 100 + And the numsegments of table "ext_test" is 2 + And distribution information from table "test_schema_2.test_table_1" with data in "test_db_2" is equal to segment count = 2, row count = 100 + And distribution information from table "test_schema_2.test_table_2" with data in "test_db_2" is equal to segment count = 2, row count = 100 + When there is a "heap" table "test_schema_1.test_table_3" in "test_db_1" with "100" rows + Then distribution information from table "test_schema_1.test_table_3" with data in "test_db_1" is equal to segment count = 2, row count = 100 + + Examples: + | fault_name_shrink | fault_name_rollback | + | on_enter_STATE_PREPARE_SHRINK_SCHEMA_STARTED_end | on_enter_STATE_SHRINK_ROLLBACK_RESTORE_TARGET_SEGMENT_COUNT_START_end | + | on_enter_STATE_PREPARE_SHRINK_SCHEMA_STARTED_end | on_enter_STATE_SHRINK_ROLLBACK_RESTORE_TARGET_SEGMENT_COUNT_DONE_begin | + | on_enter_STATE_PREPARE_SHRINK_SCHEMA_STARTED_end | on_enter_STATE_SHRINK_ROLLBACK_RESTORE_TARGET_SEGMENT_COUNT_DONE_end | + | on_enter_STATE_PREPARE_SHRINK_SCHEMA_STARTED_end | on_enter_STATE_SHRINK_ROLLBACK_PREPARE_SCHEMA_START_begin | + | on_enter_STATE_PREPARE_SHRINK_SCHEMA_STARTED_end | on_enter_STATE_SHRINK_ROLLBACK_PREPARE_SCHEMA_START_end | + | on_enter_STATE_PREPARE_SHRINK_SCHEMA_STARTED_end | on_enter_STATE_SHRINK_ROLLBACK_PREPARE_SCHEMA_DONE_begin | + | on_enter_STATE_PREPARE_SHRINK_SCHEMA_STARTED_end | on_enter_STATE_SHRINK_ROLLBACK_PREPARE_SCHEMA_DONE_end | + | on_enter_STATE_PREPARE_SHRINK_SCHEMA_STARTED_end | on_enter_STATE_SHRINK_ROLLBACK_SHRINKED_TABLES_START_begin | + | on_enter_STATE_PREPARE_SHRINK_SCHEMA_STARTED_end | on_enter_STATE_SHRINK_ROLLBACK_SHRINKED_TABLES_START_end | + | on_enter_STATE_PREPARE_SHRINK_SCHEMA_STARTED_end | on_enter_STATE_SHRINK_ROLLBACK_SHRINKED_TABLES_DONE_begin | + | on_enter_STATE_PREPARE_SHRINK_SCHEMA_STARTED_end | on_enter_STATE_SHRINK_ROLLBACK_SHRINKED_TABLES_DONE_end | + | on_enter_STATE_PREPARE_SHRINK_SCHEMA_STARTED_end | on_enter_STATE_SHRINK_ROLLBACK_DROP_SCHEMA_START_begin | + | fault_rebalance_table_test_db_2.test_schema_2.test_table_1 | on_enter_STATE_SHRINK_ROLLBACK_RESTORE_TARGET_SEGMENT_COUNT_START_end | + | fault_rebalance_table_test_db_2.test_schema_2.test_table_1 | on_enter_STATE_SHRINK_ROLLBACK_RESTORE_TARGET_SEGMENT_COUNT_DONE_begin | + | fault_rebalance_table_test_db_2.test_schema_2.test_table_1 | on_enter_STATE_SHRINK_ROLLBACK_RESTORE_TARGET_SEGMENT_COUNT_DONE_end | + | fault_rebalance_table_test_db_2.test_schema_2.test_table_1 | on_enter_STATE_SHRINK_ROLLBACK_PREPARE_SCHEMA_START_begin | + | fault_rebalance_table_test_db_2.test_schema_2.test_table_1 | on_enter_STATE_SHRINK_ROLLBACK_PREPARE_SCHEMA_START_end | + | fault_rebalance_table_test_db_2.test_schema_2.test_table_1 | on_enter_STATE_SHRINK_ROLLBACK_PREPARE_SCHEMA_DONE_begin | + | fault_rebalance_table_test_db_2.test_schema_2.test_table_1 | on_enter_STATE_SHRINK_ROLLBACK_PREPARE_SCHEMA_DONE_end | + | fault_rebalance_table_test_db_2.test_schema_2.test_table_1 | on_enter_STATE_SHRINK_ROLLBACK_SHRINKED_TABLES_START_begin | + | fault_rebalance_table_test_db_2.test_schema_2.test_table_1 | on_enter_STATE_SHRINK_ROLLBACK_SHRINKED_TABLES_START_end | + | fault_rebalance_table_test_db_2.test_schema_2.test_table_1 | on_enter_STATE_SHRINK_ROLLBACK_SHRINKED_TABLES_DONE_begin | + | fault_rebalance_table_test_db_2.test_schema_2.test_table_1 | on_enter_STATE_SHRINK_ROLLBACK_SHRINKED_TABLES_DONE_end | + | fault_rebalance_table_test_db_2.test_schema_2.test_table_1 | on_enter_STATE_SHRINK_ROLLBACK_DROP_SCHEMA_START_begin | + | on_enter_STATE_SHRINK_TABLES_DONE_begin | fault_rebalance_table_test_db_2.test_schema_2.test_table_1 | + + Scenario Outline: test 3.3. shrink - check continue after interrupted rollback state (interruption is done after rebalance schema is dropped) + Given the database is not running + And a working directory of the test as '/data/gpdata/ggrebalance' + And a cluster is created with mirrors on "cdw" and "sdw1" + And all files in gpAdminLogs directory are deleted + And set fault inject "" + And database "test_db_1" exists + And schema "test_schema_1" exists in "test_db_1" + And there is a "heap" table "test_schema_1.test_table_1" in "test_db_1" with "100" rows + And there is a "ao" table "test_schema_1.test_table_2" in "test_db_1" with "100" rows + And there is a "heap" partition table "test_schema_1.part_test_table_1" in "test_db_1" with "100" rows + And there is a "ao" partition table "test_schema_1.part_test_table_2" in "test_db_1" with "100" rows + And there is an unlogged "heap" table "test_schema_1.unlogged_test_table_1" in "test_db_1" with "100" rows + And a materialized view "test_schema_1.mv_test_table_1" exists on table "test_schema_1.test_table_1" + And database "gptest" exists + And the user create a writable external table with name "ext_test" + And database "test_db_2" exists + And schema "test_schema_2" exists in "test_db_2" + And there is a "heap" table "test_schema_2.test_table_1" in "test_db_2" with "100" rows + And there is a "ao" table "test_schema_2.test_table_2" in "test_db_2" with "100" rows + When the user runs "ggrebalance -x 1 --skip-rebalance" + Then ggrebalance should return a return code of 1 + And ggrebalance should print "ggrebalance failed" to logfile with latest timestamp + And unset fault inject + And set fault inject "" + When the user runs "ggrebalance -r" + Then ggrebalance should return a return code of 1 + And ggrebalance should print "ggrebalance failed" to logfile with latest timestamp + And unset fault inject + When the user runs "ggrebalance" + Then ggrebalance should return a return code of 1 + And ggrebalance should print "Rebalance schema doesn't exists and no shrink plan is supplied. Please specify shrink plan." to logfile with latest timestamp + And distribution information from table "test_schema_1.test_table_1" with data in "test_db_1" is equal to segment count = 2, row count = 100 + And distribution information from table "test_schema_1.test_table_2" with data in "test_db_1" is equal to segment count = 2, row count = 100 + And distribution information from table "test_schema_1.part_test_table_1" with data in "test_db_1" is equal to segment count = 2, row count = 100 + And distribution information from table "test_schema_1.part_test_table_2" with data in "test_db_1" is equal to segment count = 2, row count = 100 + And distribution information from table "test_schema_1.unlogged_test_table_1" with data in "test_db_1" is equal to segment count = 2, row count = 100 + And distribution information from table "test_schema_1.mv_test_table_1" with data in "test_db_1" is equal to segment count = 2, row count = 100 + And the numsegments of table "ext_test" is 2 + And distribution information from table "test_schema_2.test_table_1" with data in "test_db_2" is equal to segment count = 2, row count = 100 + And distribution information from table "test_schema_2.test_table_2" with data in "test_db_2" is equal to segment count = 2, row count = 100 + When there is a "heap" table "test_schema_1.test_table_3" in "test_db_1" with "100" rows + Then distribution information from table "test_schema_1.test_table_3" with data in "test_db_1" is equal to segment count = 2, row count = 100 + + Examples: + | fault_name_shrink | fault_name_rollback | + | on_enter_STATE_PREPARE_SHRINK_SCHEMA_STARTED_end | on_enter_STATE_SHRINK_ROLLBACK_DROP_SCHEMA_START_end | + | on_enter_STATE_PREPARE_SHRINK_SCHEMA_STARTED_end | on_enter_STATE_SHRINK_ROLLBACK_DROP_SCHEMA_DONE_begin | + | on_enter_STATE_PREPARE_SHRINK_SCHEMA_STARTED_end | on_enter_STATE_SHRINK_ROLLBACK_DROP_SCHEMA_DONE_end | + | fault_rebalance_table_test_db_2.test_schema_2.test_table_1 | on_enter_STATE_SHRINK_ROLLBACK_DROP_SCHEMA_START_end | + | fault_rebalance_table_test_db_2.test_schema_2.test_table_1 | on_enter_STATE_SHRINK_ROLLBACK_DROP_SCHEMA_DONE_begin | + | fault_rebalance_table_test_db_2.test_schema_2.test_table_1 | on_enter_STATE_SHRINK_ROLLBACK_DROP_SCHEMA_DONE_end | + + Scenario Outline: test 3.4. shrink - check continue after interrupted rollback state and cluster restart + Given the database is not running + And a working directory of the test as '/data/gpdata/ggrebalance' + And a cluster is created with mirrors on "cdw" and "sdw1" + And all files in gpAdminLogs directory are deleted + And set fault inject "" + And database "test_db_1" exists + And schema "test_schema_1" exists in "test_db_1" + And there is a "heap" table "test_schema_1.test_table_1" in "test_db_1" with "100" rows + And there is a "ao" table "test_schema_1.test_table_2" in "test_db_1" with "100" rows + And there is a "heap" partition table "test_schema_1.part_test_table_1" in "test_db_1" with "100" rows + And there is a "ao" partition table "test_schema_1.part_test_table_2" in "test_db_1" with "100" rows + And there is an unlogged "heap" table "test_schema_1.unlogged_test_table_1" in "test_db_1" with "100" rows + And a materialized view "test_schema_1.mv_test_table_1" exists on table "test_schema_1.test_table_1" + And database "gptest" exists + And the user create a writable external table with name "ext_test" + And database "test_db_2" exists + And schema "test_schema_2" exists in "test_db_2" + And there is a "heap" table "test_schema_2.test_table_1" in "test_db_2" with "100" rows + And there is a "ao" table "test_schema_2.test_table_2" in "test_db_2" with "100" rows + When the user runs "ggrebalance -x 1 --skip-rebalance" + Then ggrebalance should return a return code of 1 + And ggrebalance should print "ggrebalance failed" to logfile with latest timestamp + And unset fault inject + And set fault inject "" + When the user runs "ggrebalance -r" + Then ggrebalance should return a return code of 1 + And ggrebalance should print "ggrebalance failed" to logfile with latest timestamp + And unset fault inject + And there is a "heap" table "test_schema_2.test_table_3" in "test_db_2" with "200" rows + When the user runs "gpstop -arf" + Then gpstart should return a return code of 0 + When there is a "heap" table "test_schema_2.test_table_4" in "test_db_2" with "300" rows + When the user runs "ggrebalance" + Then ggrebalance should return a return code of 0 + And ggrebalance should print "Rollback is complete" to logfile with latest timestamp + And distribution information from table "test_schema_1.test_table_1" with data in "test_db_1" is equal to segment count = 2, row count = 100 + And distribution information from table "test_schema_1.test_table_2" with data in "test_db_1" is equal to segment count = 2, row count = 100 + And distribution information from table "test_schema_1.part_test_table_1" with data in "test_db_1" is equal to segment count = 2, row count = 100 + And distribution information from table "test_schema_1.part_test_table_2" with data in "test_db_1" is equal to segment count = 2, row count = 100 + And distribution information from table "test_schema_1.unlogged_test_table_1" with data in "test_db_1" is equal to segment count = 2, row count = 100 + And distribution information from table "test_schema_1.mv_test_table_1" with data in "test_db_1" is equal to segment count = 2, row count = 100 + And the numsegments of table "ext_test" is 2 + And distribution information from table "test_schema_2.test_table_1" with data in "test_db_2" is equal to segment count = 2, row count = 100 + And distribution information from table "test_schema_2.test_table_2" with data in "test_db_2" is equal to segment count = 2, row count = 100 + And distribution information from table "test_schema_2.test_table_3" with data in "test_db_2" is equal to segment count = 2, row count = 200 + And distribution information from table "test_schema_2.test_table_4" with data in "test_db_2" is equal to segment count = 2, row count = 300 + When there is a "heap" table "test_schema_1.test_table_3" in "test_db_1" with "100" rows + Then distribution information from table "test_schema_1.test_table_3" with data in "test_db_1" is equal to segment count = 2, row count = 100 + + Examples: + | fault_name_shrink | fault_name_rollback | + | on_enter_STATE_PREPARE_SHRINK_SCHEMA_STARTED_end | on_enter_STATE_SHRINK_ROLLBACK_RESTORE_TARGET_SEGMENT_COUNT_START_end | + | on_enter_STATE_PREPARE_SHRINK_SCHEMA_STARTED_end | on_enter_STATE_SHRINK_ROLLBACK_RESTORE_TARGET_SEGMENT_COUNT_DONE_begin | + | on_enter_STATE_PREPARE_SHRINK_SCHEMA_STARTED_end | on_enter_STATE_SHRINK_ROLLBACK_RESTORE_TARGET_SEGMENT_COUNT_DONE_end | + | on_enter_STATE_PREPARE_SHRINK_SCHEMA_STARTED_end | on_enter_STATE_SHRINK_ROLLBACK_PREPARE_SCHEMA_START_begin | + | on_enter_STATE_PREPARE_SHRINK_SCHEMA_STARTED_end | on_enter_STATE_SHRINK_ROLLBACK_PREPARE_SCHEMA_START_end | + | on_enter_STATE_PREPARE_SHRINK_SCHEMA_STARTED_end | on_enter_STATE_SHRINK_ROLLBACK_PREPARE_SCHEMA_DONE_begin | + | on_enter_STATE_PREPARE_SHRINK_SCHEMA_STARTED_end | on_enter_STATE_SHRINK_ROLLBACK_PREPARE_SCHEMA_DONE_end | + | on_enter_STATE_PREPARE_SHRINK_SCHEMA_STARTED_end | on_enter_STATE_SHRINK_ROLLBACK_SHRINKED_TABLES_START_begin | + | on_enter_STATE_PREPARE_SHRINK_SCHEMA_STARTED_end | on_enter_STATE_SHRINK_ROLLBACK_SHRINKED_TABLES_START_end | + | on_enter_STATE_PREPARE_SHRINK_SCHEMA_STARTED_end | on_enter_STATE_SHRINK_ROLLBACK_SHRINKED_TABLES_DONE_begin | + | on_enter_STATE_PREPARE_SHRINK_SCHEMA_STARTED_end | on_enter_STATE_SHRINK_ROLLBACK_SHRINKED_TABLES_DONE_end | + | on_enter_STATE_PREPARE_SHRINK_SCHEMA_STARTED_end | on_enter_STATE_SHRINK_ROLLBACK_DROP_SCHEMA_START_begin | + | fault_rebalance_table_test_db_2.test_schema_2.test_table_1 | on_enter_STATE_SHRINK_ROLLBACK_RESTORE_TARGET_SEGMENT_COUNT_START_end | + | fault_rebalance_table_test_db_2.test_schema_2.test_table_1 | on_enter_STATE_SHRINK_ROLLBACK_RESTORE_TARGET_SEGMENT_COUNT_DONE_begin | + | fault_rebalance_table_test_db_2.test_schema_2.test_table_1 | on_enter_STATE_SHRINK_ROLLBACK_RESTORE_TARGET_SEGMENT_COUNT_DONE_end | + | fault_rebalance_table_test_db_2.test_schema_2.test_table_1 | on_enter_STATE_SHRINK_ROLLBACK_PREPARE_SCHEMA_START_begin | + | fault_rebalance_table_test_db_2.test_schema_2.test_table_1 | on_enter_STATE_SHRINK_ROLLBACK_PREPARE_SCHEMA_START_end | + | fault_rebalance_table_test_db_2.test_schema_2.test_table_1 | on_enter_STATE_SHRINK_ROLLBACK_PREPARE_SCHEMA_DONE_begin | + | fault_rebalance_table_test_db_2.test_schema_2.test_table_1 | on_enter_STATE_SHRINK_ROLLBACK_PREPARE_SCHEMA_DONE_end | + | fault_rebalance_table_test_db_2.test_schema_2.test_table_1 | on_enter_STATE_SHRINK_ROLLBACK_SHRINKED_TABLES_START_begin | + | fault_rebalance_table_test_db_2.test_schema_2.test_table_1 | on_enter_STATE_SHRINK_ROLLBACK_SHRINKED_TABLES_START_end | + | fault_rebalance_table_test_db_2.test_schema_2.test_table_1 | on_enter_STATE_SHRINK_ROLLBACK_SHRINKED_TABLES_DONE_begin | + | fault_rebalance_table_test_db_2.test_schema_2.test_table_1 | on_enter_STATE_SHRINK_ROLLBACK_SHRINKED_TABLES_DONE_end | + | fault_rebalance_table_test_db_2.test_schema_2.test_table_1 | on_enter_STATE_SHRINK_ROLLBACK_DROP_SCHEMA_START_begin | + + Scenario: test 4. test shrink continue, when a table planned for rebalance was dropped + Given the database is not running + And a working directory of the test as '/data/gpdata/ggrebalance' + And a cluster is created with mirrors on "cdw" and "sdw1" + And segment information for content 1 is saved in context + And all files in gpAdminLogs directory are deleted + And database "test_db_1" exists + And schema "test_schema_1" exists in "test_db_1" + And there is a "heap" table "test_schema_1.test_table_1" in "test_db_1" with "100" rows + And there is a "ao" table "test_schema_1.test_table_2" in "test_db_1" with "100" rows + And there is a "heap" partition table "test_schema_1.part_test_table_1" in "test_db_1" with "100" rows + And there is a "ao" partition table "test_schema_1.part_test_table_2" in "test_db_1" with "100" rows + And there is an unlogged "heap" table "test_schema_1.unlogged_test_table_1" in "test_db_1" with "100" rows + And a materialized view "test_schema_1.mv_test_table_1" exists on table "test_schema_1.test_table_1" + And database "gptest" exists + And the user create a writable external table with name "ext_test" + And database "test_db_2" exists + And schema "test_schema_2" exists in "test_db_2" + And there is a "heap" table "test_schema_2.test_table_1" in "test_db_2" with "100" rows + And there is a "ao" table "test_schema_2.test_table_2" in "test_db_2" with "100" rows + When set fault inject "fault_rebalance_table_test_db_2.test_schema_2.test_table_1" + And the user runs "ggrebalance -x 1 --skip-rebalance" + Then ggrebalance should return a return code of 1 + And ggrebalance should print "ggrebalance failed" to logfile with latest timestamp + And unset fault inject + And table "test_schema_2.test_table_1" is dropped in "test_db_2" + When the user runs "ggrebalance" + Then ggrebalance should return a return code of 0 + And ggrebalance should print "Shrink is complete" to logfile with latest timestamp + And verify no segment running for saved segment information + And distribution information from table "test_schema_1.test_table_2" with data in "test_db_1" is equal to segment count = 1, row count = 100 + And distribution information from table "test_schema_1.part_test_table_1" with data in "test_db_1" is equal to segment count = 1, row count = 100 + And distribution information from table "test_schema_1.part_test_table_2" with data in "test_db_1" is equal to segment count = 1, row count = 100 + And distribution information from table "test_schema_1.unlogged_test_table_1" with data in "test_db_1" is equal to segment count = 1, row count = 100 + And distribution information from table "test_schema_1.mv_test_table_1" with data in "test_db_1" is equal to segment count = 1, row count = 100 + And the numsegments of table "ext_test" is 1 + And distribution information from table "test_schema_2.test_table_2" with data in "test_db_2" is equal to segment count = 1, row count = 100 + + Scenario: test 4.1. test shrink continue, when a mat view planned for rebalance was dropped + Given the database is not running + And a working directory of the test as '/data/gpdata/ggrebalance' + And a cluster is created with mirrors on "cdw" and "sdw1" + And segment information for content 1 is saved in context + And all files in gpAdminLogs directory are deleted + And database "test_db_1" exists + And schema "test_schema_1" exists in "test_db_1" + And there is a "heap" table "test_schema_1.test_table_1" in "test_db_1" with "100" rows + And there is a "ao" table "test_schema_1.test_table_2" in "test_db_1" with "100" rows + And there is a "heap" partition table "test_schema_1.part_test_table_1" in "test_db_1" with "100" rows + And there is a "ao" partition table "test_schema_1.part_test_table_2" in "test_db_1" with "100" rows + And there is an unlogged "heap" table "test_schema_1.unlogged_test_table_1" in "test_db_1" with "100" rows + And a materialized view "test_schema_1.mv_test_table_1" exists on table "test_schema_1.test_table_1" + And database "gptest" exists + And the user create a writable external table with name "ext_test" + And database "test_db_2" exists + And schema "test_schema_2" exists in "test_db_2" + And there is a "heap" table "test_schema_2.test_table_1" in "test_db_2" with "100" rows + And there is a "ao" table "test_schema_2.test_table_2" in "test_db_2" with "100" rows + When set fault inject "fault_rebalance_table_test_db_1.test_schema_1.mv_test_table_1" + And the user runs "ggrebalance -x 1 --skip-rebalance" + Then ggrebalance should return a return code of 1 + And ggrebalance should print "ggrebalance failed" to logfile with latest timestamp + And unset fault inject + And materialized view "test_schema_1.mv_test_table_1" is dropped in "test_db_1" + When the user runs "ggrebalance" + Then ggrebalance should return a return code of 0 + And ggrebalance should print "Shrink is complete" to logfile with latest timestamp + And verify no segment running for saved segment information + And distribution information from table "test_schema_1.test_table_2" with data in "test_db_1" is equal to segment count = 1, row count = 100 + And distribution information from table "test_schema_1.part_test_table_1" with data in "test_db_1" is equal to segment count = 1, row count = 100 + And distribution information from table "test_schema_1.part_test_table_2" with data in "test_db_1" is equal to segment count = 1, row count = 100 + And distribution information from table "test_schema_1.unlogged_test_table_1" with data in "test_db_1" is equal to segment count = 1, row count = 100 + And the numsegments of table "ext_test" is 1 + And distribution information from table "test_schema_2.test_table_1" with data in "test_db_2" is equal to segment count = 1, row count = 100 + And distribution information from table "test_schema_2.test_table_2" with data in "test_db_2" is equal to segment count = 1, row count = 100 + + Scenario: test 5. test shrink continue, when a db with the table planned for rebalance was dropped + Given the database is not running + And a working directory of the test as '/data/gpdata/ggrebalance' + And a cluster is created with mirrors on "cdw" and "sdw1" + And segment information for content 1 is saved in context + And all files in gpAdminLogs directory are deleted + And database "test_db_1" exists + And schema "test_schema_1" exists in "test_db_1" + And there is a "heap" table "test_schema_1.test_table_1" in "test_db_1" with "100" rows + And there is a "ao" table "test_schema_1.test_table_2" in "test_db_1" with "100" rows + And there is a "heap" partition table "test_schema_1.part_test_table_1" in "test_db_1" with "100" rows + And there is a "ao" partition table "test_schema_1.part_test_table_2" in "test_db_1" with "100" rows + And there is an unlogged "heap" table "test_schema_1.unlogged_test_table_1" in "test_db_1" with "100" rows + And a materialized view "test_schema_1.mv_test_table_1" exists on table "test_schema_1.test_table_1" + And database "gptest" exists + And the user create a writable external table with name "ext_test" + And database "test_db_2" exists + And schema "test_schema_2" exists in "test_db_2" + And there is a "heap" table "test_schema_2.test_table_1" in "test_db_2" with "100" rows + And there is a "ao" table "test_schema_2.test_table_2" in "test_db_2" with "100" rows + When set fault inject "fault_rebalance_table_test_db_2.test_schema_2.test_table_1" + And the user runs "ggrebalance -x 1 --skip-rebalance" + Then ggrebalance should return a return code of 1 + And ggrebalance should print "ggrebalance failed" to logfile with latest timestamp + And unset fault inject + And the database "test_db_2" does not exist + When the user runs "ggrebalance" + Then ggrebalance should return a return code of 0 + And ggrebalance should print "Shrink is complete" to logfile with latest timestamp + And verify no segment running for saved segment information + And distribution information from table "test_schema_1.test_table_2" with data in "test_db_1" is equal to segment count = 1, row count = 100 + And distribution information from table "test_schema_1.part_test_table_1" with data in "test_db_1" is equal to segment count = 1, row count = 100 + And distribution information from table "test_schema_1.part_test_table_2" with data in "test_db_1" is equal to segment count = 1, row count = 100 + And distribution information from table "test_schema_1.unlogged_test_table_1" with data in "test_db_1" is equal to segment count = 1, row count = 100 + And distribution information from table "test_schema_1.mv_test_table_1" with data in "test_db_1" is equal to segment count = 1, row count = 100 + And the numsegments of table "ext_test" is 1 + + Scenario: test 6. test shrink, when a table, planned for rebalance, is dropped in a parallel transaction, committed after the start of table redistribution + Given the database is not running + And a working directory of the test as '/data/gpdata/ggrebalance' + And a cluster is created with mirrors on "cdw" and "sdw1" + And segment information for content 1 is saved in context + And all files in gpAdminLogs directory are deleted + And database "test_db_1" exists + And schema "test_schema_1" exists in "test_db_1" + And there is a "heap" table "test_schema_1.test_table_1" in "test_db_1" with "100" rows + And there is a "ao" table "test_schema_1.test_table_2" in "test_db_1" with "100" rows + And there is a "heap" partition table "test_schema_1.part_test_table_1" in "test_db_1" with "100" rows + And there is a "ao" partition table "test_schema_1.part_test_table_2" in "test_db_1" with "100" rows + And there is an unlogged "heap" table "test_schema_1.unlogged_test_table_1" in "test_db_1" with "100" rows + And a materialized view "test_schema_1.mv_test_table_1" exists on table "test_schema_1.test_table_1" + And database "gptest" exists + And there is a "heap" table "test_table_1" in "gptest" with "100" rows + And the user create a writable external table with name "ext_test" + And database "test_db_2" exists + And schema "test_schema_2" exists in "test_db_2" + And there is a "heap" table "test_schema_2.test_table_1" in "test_db_2" with "100" rows + And there is a "ao" table "test_schema_2.test_table_2" in "test_db_2" with "100" rows + And set fault inject "on_enter_STATE_PREPARE_SHRINK_SCHEMA_STARTED_begin" + And set fault inject type to suspend + When the user asynchronously runs "ggrebalance -x 1 --skip-rebalance" and the process is saved + And the user waits till ggrebalance prints "Updated target segment count to 1" in the logs (with timeout of "60" sec) + And the user connects to "gptest" with named connection "test_connection" + And the user executes "BEGIN; DROP TABLE test_table_1;" with named connection "test_connection" + And unset fault inject + And the user waits till ggrebalance prints "Start table rebalance for \"gptest\".\"public\".\"test_table_1\" to 1 segments" in the logs (with timeout of "60" sec) + And waiting "5" seconds + And the user executes "COMMIT;" with named connection "test_connection" + And the user drops the named connection "test_connection" + Then the async process finished with a return code of 0 + And ggrebalance should print "Shrink is complete" to logfile with latest timestamp + And verify no segment running for saved segment information + And distribution information from table "test_schema_1.test_table_2" with data in "test_db_1" is equal to segment count = 1, row count = 100 + And distribution information from table "test_schema_1.part_test_table_1" with data in "test_db_1" is equal to segment count = 1, row count = 100 + And distribution information from table "test_schema_1.part_test_table_2" with data in "test_db_1" is equal to segment count = 1, row count = 100 + And distribution information from table "test_schema_1.unlogged_test_table_1" with data in "test_db_1" is equal to segment count = 1, row count = 100 + And distribution information from table "test_schema_1.mv_test_table_1" with data in "test_db_1" is equal to segment count = 1, row count = 100 + And distribution information from table "test_schema_2.test_table_2" with data in "test_db_2" is equal to segment count = 1, row count = 100 + And the numsegments of table "ext_test" is 1 diff --git a/gpMgmt/test/behave/mgmt_utils/gpaddmirrors.feature b/gpMgmt/test/behave/mgmt_utils/gpaddmirrors.feature index 9817183b5792..e446aedb6769 100644 --- a/gpMgmt/test/behave/mgmt_utils/gpaddmirrors.feature +++ b/gpMgmt/test/behave/mgmt_utils/gpaddmirrors.feature @@ -493,9 +493,9 @@ Feature: Tests for gpaddmirrors And the segments are synchronized When user stops all primary processes And user can start transactions - Then verify that there is a "heap" table "public.heap_table" in "gptest" with "202" rows - Then verify that there is a "ao" table "public.ao_table" in "gptest" with "202" rows - Then verify that there is a "co" table "public.co_table" in "gptest" with "202" rows + Then verify that there is a "heap" table "public.heap_table" in "gptest" with "100" rows + Then verify that there is a "ao" table "public.ao_table" in "gptest" with "100" rows + Then verify that there is a "co" table "public.co_table" in "gptest" with "100" rows And the user runs "gpstop -aqM fast" @concourse_cluster diff --git a/gpMgmt/test/behave/mgmt_utils/gprebalance.feature b/gpMgmt/test/behave/mgmt_utils/gprebalance.feature new file mode 100644 index 000000000000..0e4872e3e103 --- /dev/null +++ b/gpMgmt/test/behave/mgmt_utils/gprebalance.feature @@ -0,0 +1,15 @@ +@gprebalance +Feature: Tests for gprebalance + Scenario: gprebalance balances cluster after expansion on fixed hosts + Given the database is not running + And a working directory of the test as '/data/gpdata/gpexpand' + And a temporary directory under "/data/gpdata/gpexpand/expandedData" to expand into + And a cluster is created with mirrors on "cdw" and "sdw1,sdw2" + And the cluster is setup for an expansion on hosts "cdw,sdw1,sdw2" + And the number of segments have been saved + When the user runs gpexpand with segment imbalance for a two-node cluster with mirrors + Then verify that the cluster has 4 new segments + When the user runs "gprebalance -s" + Then gprebalance should return a return code of 0 + When the user runs "gprebalance -s" + Then gprebalance should print "[INFO]:-Cluster is already balanced" escaped to stdout diff --git a/gpMgmt/test/behave/mgmt_utils/steps/analyzedb_mgmt_utils.py b/gpMgmt/test/behave/mgmt_utils/steps/analyzedb_mgmt_utils.py index f7bf6f1bdfa7..43c2a172e982 100644 --- a/gpMgmt/test/behave/mgmt_utils/steps/analyzedb_mgmt_utils.py +++ b/gpMgmt/test/behave/mgmt_utils/steps/analyzedb_mgmt_utils.py @@ -94,13 +94,13 @@ def impl(context, view_name, table_name, schema_name): @given('a view "{view_name}" exists on table "{table_name}"') def impl(context, view_name, table_name): with closing(dbconn.connect(dbconn.DbURL(dbname=context.dbname))) as conn: - create_view_on_table(context.conn, view_name, table_name) + create_view_on_table(conn, view_name, table_name) @given('a materialized view "{view_name}" exists on table "{table_name}"') def impl(context, view_name, table_name): with closing(dbconn.connect(dbconn.DbURL(dbname=context.dbname))) as conn: - create_materialized_view_on_table_in_schema(context.conn, viewname=view_name, + create_materialized_view_on_table_in_schema(conn, viewname=view_name, tablename=table_name) diff --git a/gpMgmt/test/behave/mgmt_utils/steps/gpstate_utils.py b/gpMgmt/test/behave/mgmt_utils/steps/gpstate_utils.py index 271b1433d7de..e9e1db548523 100644 --- a/gpMgmt/test/behave/mgmt_utils/steps/gpstate_utils.py +++ b/gpMgmt/test/behave/mgmt_utils/steps/gpstate_utils.py @@ -19,6 +19,7 @@ def impl(context): fp.write("full:5: 1164848/1371715 kB (84%), 0/1 tablespace (...t1/demoDataDir0/base/16384/40962)\n") fp.write("incremental:6: 1/1371875 kB (1%)") +@when('a sample {lock_file} directory is created using the background pid in coordinator_data_directory') @then('a sample {lock_file} directory is created using the background pid in coordinator_data_directory') @given('a sample {lock_file} directory is created using the background pid in coordinator_data_directory') def impl(context, lock_file): diff --git a/gpMgmt/test/behave/mgmt_utils/steps/mgmt_utils.py b/gpMgmt/test/behave/mgmt_utils/steps/mgmt_utils.py index 378e82c7aa9c..c93909102d57 100644 --- a/gpMgmt/test/behave/mgmt_utils/steps/mgmt_utils.py +++ b/gpMgmt/test/behave/mgmt_utils/steps/mgmt_utils.py @@ -39,6 +39,7 @@ from gppylib import pgconf from gppylib.commands.gp import get_coordinatordatadir from gppylib.parseutils import canonicalize_address +from gppylib import fault_injection coordinator_data_dir = gp.get_coordinatordatadir() if coordinator_data_dir is None: @@ -156,6 +157,8 @@ def impl(context, query, db, contentids): @given('the user connects to "{dbname}" with named connection "{cname}"') +@when('the user connects to "{dbname}" with named connection "{cname}"') +@then('the user connects to "{dbname}" with named connection "{cname}"') def impl(context, dbname, cname): if not hasattr(context, 'named_conns'): context.named_conns = {} @@ -197,11 +200,14 @@ def impl(conetxt, tabname): @given('the user executes "{sql}" with named connection "{cname}"') +@when('the user executes "{sql}" with named connection "{cname}"') +@then('the user executes "{sql}" with named connection "{cname}"') def impl(context, cname, sql): conn = context.named_conns[cname] dbconn.execSQL(conn, sql) +@when('the user drops the named connection "{cname}"') @then('the user drops the named connection "{cname}"') def impl(context, cname): if cname in context.named_conns: @@ -660,6 +666,31 @@ def impl(context, kill_process_name, log_msg, logfile_name): "fi; done" % (log_msg, logfile_name, kill_process_name) run_async_command(context, command) +@given('the user waits till {process_name} prints "{log_msg}" in the logs (with timeout of "{timeout}" sec)') +@when('the user waits till {process_name} prints "{log_msg}" in the logs (with timeout of "{timeout}" sec)') +@then('the user waits till {process_name} prints "{log_msg}" in the logs (with timeout of "{timeout}" sec)') +def impl(context, process_name, log_msg, timeout): + poll_period = 0.1 + max_iteration_cnt = int(int(timeout) / poll_period) + command = f""" + ITERATION=0 + MAX_ITERATION_CNT={max_iteration_cnt} + while sleep {poll_period}; do + if grep -E --quiet '{log_msg}' ~/gpAdminLogs/{process_name}*log ; + then break 2; + fi; + + ITERATION=$((ITERATION + 1)) + if [ $ITERATION -ge $MAX_ITERATION_CNT ]; then + echo "Timeout after {timeout} seconds waiting for '{log_msg}' in {process_name} logs" >&2 + exit 1 + fi + done + """ + rc, _, error = run_cmd(command) + if rc: + raise Exception(error) + @given('the user asynchronously sets up to end {process_name} process with {signal_name}') @when('the user asynchronously sets up to end {process_name} process with {signal_name}') @then('the user asynchronously sets up to end {process_name} process with {signal_name}') @@ -963,6 +994,7 @@ def impl(context, tname, dbname, nrows): check_row_count(context, tname, dbname, int(nrows)) @given('schema "{schema_list}" exists in "{dbname}"') +@when('schema "{schema_list}" exists in "{dbname}"') @then('schema "{schema_list}" exists in "{dbname}"') def impl(context, schema_list, dbname): schemas = [s.strip() for s in schema_list.split(',')] @@ -970,6 +1002,13 @@ def impl(context, schema_list, dbname): drop_schema_if_exists(context, s.strip(), dbname) create_schema(context, s.strip(), dbname) +@given('schema "{schema_list}" is removed in "{dbname}"') +@then('schema "{schema_list}" is removed in "{dbname}"') +def impl(context, schema_list, dbname): + schemas = [s.strip() for s in schema_list.split(',')] + for s in schemas: + drop_schema_if_exists(context, s.strip(), dbname) + @then('the temporary file "{filename}" is removed') def impl(context, filename): @@ -977,19 +1016,25 @@ def impl(context, filename): os.remove(filename) -def create_table_file_locally(context, filename, table_list, location=os.getcwd()): - tables = table_list.split('|') +def create_value_list_file_locally(context, filename, value_list, location=os.getcwd()): + values = value_list.split('|') file_path = os.path.join(location, filename) with open(file_path, 'w') as fp: - for t in tables: + for t in values: fp.write(t + '\n') context.filename = file_path -@given('there is a file "{filename}" with tables "{table_list}"') -@then('there is a file "{filename}" with tables "{table_list}"') -def impl(context, filename, table_list): - create_table_file_locally(context, filename, table_list) +@given('there is a file "{filename}" with tables "{list}"') +@then('there is a file "{filename}" with tables "{list}"') +@given('there is a file "{filename}" with hosts "{list}"') +@when('there is a file "{filename}" with hosts "{list}"') +@then('there is a file "{filename}" with hosts "{list}"') +@given('there is a file "{filename}" with datadirs "{list}"') +@when('there is a file "{filename}" with datadirs "{list}"') +@then('there is a file "{filename}" with datadirs "{list}"') +def impl(context, filename, list): + create_value_list_file_locally(context, filename, list) @given('the row "{row_values}" is inserted into "{table}" in "{dbname}"') @@ -1547,6 +1592,11 @@ def get_opened_files(filename, pidfile): def impl(context, tablename, dbname): drop_table_if_exists(context, table_name=tablename, dbname=dbname) +@when('materialized view "{viewname}" is dropped in "{dbname}"') +@then('materialized view "{viewname}" is dropped in "{dbname}"') +@given('materialized view "{viewname}" is dropped in "{dbname}"') +def impl(context, viewname, dbname): + drop_materialized_view_if_exists(context, view_name=viewname, dbname=dbname) @given('all the segments are running') @when('all the segments are running') @@ -1603,6 +1653,18 @@ def impl(context, filter): Given the user runs psql with "-c 'BEGIN; CREATE TEMP TABLE tempt(a int); COMMIT'" against database "postgres" ''') +@given('the cluster configuration has {segment_count} segments where "{filter}"') +@when('the cluster configuration has {segment_count} segments where "{filter}"') +@then('the cluster configuration has {segment_count} segments where "{filter}"') +def impl(context, segment_count, filter): + sql = "SELECT count(*) FROM gp_segment_configuration WHERE %s" % filter + with closing(dbconn.connect(dbconn.DbURL(), unsetSearchPath=False)) as conn: + row = dbconn.queryRow(conn, sql) + if segment_count == 'some': + if int(row[0]) == 0: + raise Exception(f"Expected some segments, but got 0") + elif int(row[0]) != int(segment_count): + raise Exception(f"Expected {segment_count} segments, but got {row[0]}") @given('the cluster configuration is saved for "{when}"') @then('the cluster configuration is saved for "{when}"') @@ -1672,6 +1734,30 @@ def impl(context, seg): cmd.run(validateAfter=True) +@given('a sample {lock_file} file is created using the background pid in the coordinator_data_directory') +@when('a sample {lock_file} file is created using the background pid in the coordinator_data_directory') +@then('a sample {lock_file} file is created using the background pid in the coordinator_data_directory') +def impl(context, lock_file): + if 'bg_pid' in context: + bg_pid = context.bg_pid + if not unix.check_pid(bg_pid): + raise Exception("The background process with PID {} is not running.".format(bg_pid)) + else: + bg_pid = "" + + utility_pidfile = os.path.join(get_coordinatordatadir(), lock_file) + + with open(utility_pidfile, 'w') as f: + f.write(bg_pid) + +@given('a sample {lock_file} file is removed from the coordinator_data_directory') +@when('a sample {lock_file} file is removed from the coordinator_data_directory') +@then('a sample {lock_file} file is removed from the coordinator_data_directory') +def impl(context, lock_file): + utility_pidfile = os.path.join(get_coordinatordatadir(), lock_file) + if os.path.exists(utility_pidfile): + os.remove(utility_pidfile) + @when('{process} is killed on mirror with content {contentids}') @then('{process} is killed on mirror with content {contentids}') def impl(context, process, contentids): @@ -2283,9 +2369,16 @@ def impl(context): ''') @given('there is a "{tabletype}" table "{tablename}" in "{dbname}" with "{numrows}" rows') +@then('there is a "{tabletype}" table "{tablename}" in "{dbname}" with "{numrows}" rows') +@when('there is a "{tabletype}" table "{tablename}" in "{dbname}" with "{numrows}" rows') def impl(context, tabletype, tablename, dbname, numrows): populate_regular_table_data(context, tabletype, tablename, dbname, compression_type=None, with_data=True, rowcount=int(numrows)) +@given('there is an unlogged "{tabletype}" table "{tablename}" in "{dbname}" with "{numrows}" rows') +@then('there is an unlogged "{tabletype}" table "{tablename}" in "{dbname}" with "{numrows}" rows') +@when('there is an unlogged "{tabletype}" table "{tablename}" in "{dbname}" with "{numrows}" rows') +def impl(context, tabletype, tablename, dbname, numrows): + populate_regular_table_data(context, tabletype, tablename, dbname, compression_type=None, with_data=True, rowcount=int(numrows), unlogged=True) @given('there is a "{tabletype}" table "{tablename}" in "{dbname}" with data') @then('there is a "{tabletype}" table "{tablename}" in "{dbname}" with data') @@ -2306,6 +2399,12 @@ def impl(context, tabletype, tablename, dbname): def impl(context, tabletype, table_name, dbname): create_partition(context, tablename=table_name, storage_type=tabletype, dbname=dbname, with_data=True) +@given('there is a "{tabletype}" partition table "{table_name}" in "{dbname}" with "{numrows}" rows') +@then('there is a "{tabletype}" partition table "{table_name}" in "{dbname}" with "{numrows}" rows') +@when('there is a "{tabletype}" partition table "{table_name}" in "{dbname}" with "{numrows}" rows') +def impl(context, tabletype, table_name, dbname, numrows): + create_partition(context, tablename=table_name, storage_type=tabletype, dbname=dbname, with_data=True, rowcount=int(numrows)) + @given('there is a view without columns in "{dbname}"') @then('there is a view without columns in "{dbname}"') @when('there is a view without columns in "{dbname}"') @@ -2614,7 +2713,15 @@ def impl(context, location): @when('all files in gpAdminLogs directory are deleted') @then('all files in gpAdminLogs directory are deleted') def impl(context): - log_dir = _get_gpAdminLogs_directory() + remove_all_logfiles(_get_gpAdminLogs_directory()) + +@given('all files in "{log_dir}" directory are deleted') +@when('all files in "{log_dir}" directory are deleted') +@then('all files in "{log_dir}" directory are deleted') +def impl(context, log_dir): + remove_all_logfiles(log_dir) + +def remove_all_logfiles(log_dir): files_found = glob.glob('%s/*' % (log_dir)) for file in files_found: os.remove(file) @@ -2655,7 +2762,15 @@ def impl(context): @then('gpAdminLogs directory {has} "{expected_file}" files') def impl(context, has, expected_file): - log_dir = _get_gpAdminLogs_directory() + check_logs_directory(_get_gpAdminLogs_directory(), has, expected_file) + + +@then('"{log_dir}" directory {has} "{expected_file}" files') +def impl(context, log_dir, has, expected_file): + check_logs_directory(log_dir, has, expected_file) + + +def check_logs_directory(log_dir, has, expected_file): files_found = glob.glob('%s/%s' % (log_dir, expected_file)) if files_found and (has == 'has no'): raise Exception("expected no %s files in %s, but found %s" % (expected_file, log_dir, files_found)) @@ -2843,7 +2958,7 @@ def _create_working_directory(context, working_directory, mode=''): os.mkdir(context.working_directory) -def _create_cluster(context, coordinator_host, segment_host_list, hba_hostnames='0', with_mirrors=False, mirroring_configuration='group'): +def _create_cluster(context, coordinator_host, segment_host_list, hba_hostnames='0', with_mirrors=False, mirroring_configuration='group', number_of_segments=2): if segment_host_list == "": segment_host_list = [] else: @@ -2865,7 +2980,10 @@ def _create_cluster(context, coordinator_host, segment_host_list, hba_hostnames= except: pass - testcluster = TestCluster(hosts=[coordinator_host]+segment_host_list, base_dir=context.working_directory,hba_hostnames=hba_hostnames) + testcluster = TestCluster(hosts=[coordinator_host]+segment_host_list, + base_dir=context.working_directory, + hba_hostnames=hba_hostnames, + number_of_segments=number_of_segments) testcluster.reset_cluster() testcluster.create_cluster(with_mirrors=with_mirrors, mirroring_configuration=mirroring_configuration) context.gpexpand_mirrors_enabled = with_mirrors @@ -2890,6 +3008,10 @@ def impl(context, coordinator_host, segment_host_list): def impl(context, mirroring_configuration, coordinator_host, segment_host_list): _create_cluster(context, coordinator_host, segment_host_list, with_mirrors=True, mirroring_configuration=mirroring_configuration) +@given('a cluster is created with mirrors on "{coordinator_host}" and "{segment_host_list}", with {number_of_segments} segments on each') +def impl(context, coordinator_host, segment_host_list, number_of_segments): + _create_cluster(context, coordinator_host, segment_host_list, with_mirrors=True, mirroring_configuration='group', number_of_segments=int(number_of_segments)) + @given('the user runs gpexpand interview to add {num_of_segments} new segment and {num_of_hosts} new host "{hostnames}"') @when('the user runs gpexpand interview to add {num_of_segments} new segment and {num_of_hosts} new host "{hostnames}"') def impl(context, num_of_segments, num_of_hosts, hostnames): @@ -3029,6 +3151,24 @@ def impl(context): gpexpand = Gpexpand(context, working_directory=context.working_directory) gpexpand.initialize_segments() +@given('the user runs gpexpand with segment imbalance for a two-node cluster with mirrors') +@when('the user runs gpexpand with segment imbalance for a two-node cluster with mirrors') +def impl(context): + inputfile_contents = """ +sdw2|sdw2|20502|/tmp/gpexpand_behave/two_nodes/data/primary/gpseg4|10|4|p +sdw2|sdw2|21502|/tmp/gpexpand_behave/two_nodes/data/mirror/gpseg4|11|4|m +sdw2|sdw2|20503|/tmp/gpexpand_behave/two_nodes/data/primary/gpseg5|12|5|p +sdw2|sdw2|21503|/tmp/gpexpand_behave/two_nodes/data/mirror/gpseg5|13|5|m""" + timestamp = datetime.now().strftime("%Y%m%d_%H%M%S") + inputfile_name = "%s/gpexpand_inputfile_%s" % (context.working_directory, timestamp) + with open(inputfile_name, 'w') as fd: + fd.write(inputfile_contents) + + gpexpand = Gpexpand(context, working_directory=context.working_directory) + ret_code, std_err, std_out = gpexpand.initialize_segments() + if ret_code != 0: + raise Exception("gpexpand exited with return code: %d.\nstderr=%s\nstdout=%s" % (ret_code, std_err, std_out)) + @given('the coordinator pid has been saved') def impl(context): data_dir = os.path.join(context.working_directory, @@ -3463,6 +3603,20 @@ def impl(context, table, dbname): context.pre_redistribution_row_count = _get_row_count_per_segment(table, dbname) context.pre_redistribution_dist_policy = _get_dist_policy_per_partition(table, dbname) +@then('distribution information from table "{table}" with data in "{dbname}" is equal to segment count = {seg_cnt}, row count = {row_cnt}') +def impl(context, table, dbname, seg_cnt, row_cnt): + with closing(dbconn.connect(dbconn.DbURL(dbname=dbname), unsetSearchPath=False)) as conn: + query = "SELECT count(1) FROM (SELECT gp_segment_id FROM %s GROUP BY gp_segment_id) t" % table + cursor = dbconn.query(conn, query) + table_segments = cursor.fetchone()[0] + query = "SELECT count(1) FROM %s" % table + cursor = dbconn.query(conn, query) + table_rows = cursor.fetchone()[0] + if int(table_segments) != int(seg_cnt): + raise Exception("Expected table %s in db %s to be distributed on %s segments, but it is distributed on %s segments" % (table, dbname, seg_cnt, table_segments)) + if int(table_rows) != int(row_cnt): + raise Exception("Expected table %s in db %s to have %s rows, but got %s rows" % (table, dbname, row_cnt, table_rows)) + @then('distribution information from table "{table}" with data in "{dbname}" is verified against saved data') def impl(context, table, dbname): pre_distribution_row_count = context.pre_redistribution_row_count @@ -4126,6 +4280,29 @@ def impl(context): raise Exception("Postgres process {0} not killed on {1}.".format(pid, host)) +@given('segment information for content {content} is saved in context') +@when('segment information for content {content} is saved in context') +@then('segment information for content {content} is saved in context') +def impl(context, content): + segment_information = [] + segs = GpArray.initFromCatalog(dbconn.DbURL()).getDbList() + for seg in segs: + if int(content) == int(seg.getSegmentContentId()): + segment_information.append(seg) + + context.segment_information = segment_information + + +@given('verify no segment running for saved segment information') +@when('verify no segment running for saved segment information') +@then('verify no segment running for saved segment information') +def impl(context): + for seg in context.segment_information: + segment_check = gp.SegmentIsShutDown('', seg.getSegmentDataDirectory(), REMOTE, seg.getSegmentHostName()) + segment_check.run() + if not segment_check.is_shutdown(): + raise Exception(f'Segment is up (dbid {seg.getSegmentDbId()})') + @then('the database segments are in execute mode') def impl(context): # Get all primary segments details @@ -4391,3 +4568,65 @@ def step_impl(context): @given(u'the cluster is running in IC proxy mode with new proxy address {address}') def step_impl(context, address): set_ic_proxy_and_address(context, address) + +@given('set fault inject "{fault}"') +@then('set fault inject "{fault}"') +@when('set fault inject "{fault}"') +def impl(context, fault): + os.environ[fault_injection.GPMGMT_FAULT_POINT] = fault + +@given('unset fault inject') +@then('unset fault inject') +@when('unset fault inject') +def impl(context): + os.environ[fault_injection.GPMGMT_FAULT_POINT] = "" + os.environ[fault_injection.GPMGMT_FAULT_TYPE] = "" + os.environ[fault_injection.GPMGMT_FAULT_FILE_FLAG] = "" + if hasattr(context, 'fault_flag_filename') and os.path.exists(context.fault_flag_filename): + os.remove(context.fault_flag_filename) + +@given('set fault inject delay {delay} ms') +@then('set fault inject delay {delay} ms') +@when('set fault inject delay {delay} ms') +def impl(context, delay): + os.environ[fault_injection.GPMGMT_FAULT_DELAY_MS] = delay + +@given('set fault inject type to suspend') +@then('set fault inject type to suspend') +@when('set fault inject type to suspend') +def impl(context): + os.environ[fault_injection.GPMGMT_FAULT_TYPE] = fault_injection.GPMGMT_FAULT_TYPE_SYSPEND + context.fault_flag_filename = "/tmp/ggrebalance_fault_suspend_flag" + with open(context.fault_flag_filename, "w"): + pass + os.environ[fault_injection.GPMGMT_FAULT_FILE_FLAG] = context.fault_flag_filename + +@given('unset fault inject delay') +@then('unset fault inject delay') +@when('unset fault inject delay') +def impl(context): + os.environ[fault_injection.GPMGMT_FAULT_DELAY_MS] = "" + +@given('stub') +@then('stub') +@when('stub') +def impl(context): + pass + +@given('the temporary file "{filename}" is created with content') +@then('the temporary file "{filename}" is created with content') +@when('the temporary file "{filename}" is created with content') +def impl(context, filename): + with open(filename, 'w') as f: + f.write(context.text + '\n') + +@given('the environment variable "{var}" is set from output of "{command}"') +def impl(context, var, command): + run_command(context, command) + context.execute_steps(f'Given the environment variable "{var}" is set to "{context.stdout_message.rstrip()}"') + +@given('coordinator data directory is updated') +@then('coordinator data directory is updated') +def impl(context): + global coordinator_data_dir + coordinator_data_dir = os.environ.get('COORDINATOR_DATA_DIRECTORY') diff --git a/gpMgmt/test/behave_utils/cluster_setup.py b/gpMgmt/test/behave_utils/cluster_setup.py index bf88e04571f6..ec5e7a35c030 100755 --- a/gpMgmt/test/behave_utils/cluster_setup.py +++ b/gpMgmt/test/behave_utils/cluster_setup.py @@ -21,7 +21,7 @@ def run(self, validate=True): class TestCluster: - def __init__(self, hosts = None, base_dir = '/tmp/default_gpinitsystem', hba_hostnames='0'): + def __init__(self, hosts = None, base_dir = '/tmp/default_gpinitsystem', hba_hostnames='0', number_of_segments = 2): """ hosts: lists of cluster hosts. coordinator host will be assumed to be the first element. base_dir: cluster directory @@ -55,7 +55,7 @@ def __init__(self, hosts = None, base_dir = '/tmp/default_gpinitsystem', hba_hos # Test metadata # Whether to do gpinitsystem or not self.mirror_enabled = False - self.number_of_segments = 2 + self.number_of_segments = number_of_segments self.number_of_hosts = len(self.hosts)-1 self.number_of_expansion_hosts = 0 diff --git a/gpMgmt/test/behave_utils/utils.py b/gpMgmt/test/behave_utils/utils.py index 5a94dcc75eb2..53cac4169ffd 100644 --- a/gpMgmt/test/behave_utils/utils.py +++ b/gpMgmt/test/behave_utils/utils.py @@ -358,6 +358,10 @@ def drop_table(context, table_name, dbname, host=None, port=0, user=None): if check_table_exists(context, table_name=table_name, dbname=dbname, host=host, port=port, user=user): raise Exception('Unable to successfully drop the table %s' % table_name) +def drop_materialized_view_if_exists(context, view_name, dbname, host=None, port=0, user=None): + SQL = 'drop materialized view if exists %s' % view_name + with closing(dbconn.connect(dbconn.DbURL(hostname=host, port=port, username=user, dbname=dbname), unsetSearchPath=False)) as conn: + dbconn.execSQL(conn, SQL) def check_schema_exists(context, schema_name, dbname): schema_check_sql = "select * from pg_namespace where nspname='%s';" % schema_name @@ -430,11 +434,14 @@ def create_external_partition(context, tablename, dbname, port, filename): def create_partition(context, tablename, storage_type, dbname, compression_type=None, partition=True, rowcount=1094, - with_data=True, with_desc=False, host=None, port=0, user=None): + with_data=True, with_desc=False, host=None, port=0, user=None, unlogged=False): interval = '1 year' table_definition = 'Column1 int, Column2 varchar(20), Column3 date' - create_table_str = "Create table " + tablename + "(" + table_definition + ")" + create_table_str = "Create table " + if unlogged: + create_table_str = "Create unlogged table " + create_table_str = create_table_str + tablename + "(" + table_definition + ")" storage_type_dict = {'ao': 'row', 'co': 'column'} part_table = " Distributed Randomly Partition by list(Column2) \ @@ -472,9 +479,7 @@ def create_partition(context, tablename, storage_type, dbname, compression_type= # same data size as populate partition, but different values def populate_partition(tablename, start_date, dbname, data_offset, rowcount=1094, host=None, port=0, user=None): - insert_sql_str = "insert into %s select i+%d, 'backup', i + date '%s' from generate_series(0,%d) as i" % ( - tablename, data_offset, start_date, rowcount) - insert_sql_str += "; insert into %s select i+%d, 'restore', i + date '%s' from generate_series(0,%d) as i" % ( + insert_sql_str = "insert into %s select i+%d, 'backup', i + date '%s' from generate_series(1,%d) as i" % ( tablename, data_offset, start_date, rowcount) with closing(dbconn.connect(dbconn.DbURL(hostname=host, port=port, username=user, dbname=dbname), unsetSearchPath=False)) as conn: @@ -734,11 +739,11 @@ def validate_local_path(path): def populate_regular_table_data(context, tabletype, table_name, dbname, compression_type=None, rowcount=1094, - with_data=False, with_desc=False, host=None, port=0, user=None): + with_data=False, with_desc=False, host=None, port=0, user=None, unlogged=False): create_database_if_not_exists(context, dbname, host=host, port=port, user=user) drop_table_if_exists(context, table_name=table_name, dbname=dbname, host=host, port=port, user=user) create_partition(context, table_name, tabletype, dbname, compression_type=compression_type, partition=False, - rowcount=rowcount, with_data=with_data, with_desc=with_desc, host=host, port=port, user=user) + rowcount=rowcount, with_data=with_data, with_desc=with_desc, host=host, port=port, user=user, unlogged=unlogged) def is_process_running(proc_name, host=None): diff --git a/gpcontrib/gp_debug_numsegments/gp_debug_numsegments.c b/gpcontrib/gp_debug_numsegments/gp_debug_numsegments.c index 736101ac53b0..98b4830b272e 100644 --- a/gpcontrib/gp_debug_numsegments/gp_debug_numsegments.c +++ b/gpcontrib/gp_debug_numsegments/gp_debug_numsegments.c @@ -16,6 +16,8 @@ #include "utils/builtins.h" #include "cdb/cdbutil.h" #include "catalog/pg_type.h" +#include "port/atomics.h" +#include "utils/gpexpand.h" #ifdef PG_MODULE_MAGIC @@ -53,6 +55,13 @@ gp_debug_set_create_table_default_numsegments(PG_FUNCTION_ARGS) Assert(1 == PG_NARGS()); + uint32 gp_rebalance_numsegs = pg_atomic_read_u32(gp_create_table_rebalance_numsegments); + if (gp_rebalance_numsegs != GP_DEFAULT_NUMSEGMENTS_SHARED_UNSET) + ereport(ERROR, + (errcode(ERRCODE_INSUFFICIENT_PRIVILEGE), + errmsg("gprebalance in progress, all relations will be created at %u segments", + gp_rebalance_numsegs))); + argtypeoid = get_fn_expr_argtype(fcinfo->flinfo, 0); /* Accept argument in integer / text format */ @@ -138,6 +147,13 @@ gp_debug_get_create_table_default_numsegments(PG_FUNCTION_ARGS) { char buf[64]; const char *result = NULL; + uint32 gp_rebalance_numsegs = pg_atomic_read_u32(gp_create_table_rebalance_numsegments); + + if (gp_rebalance_numsegs != GP_DEFAULT_NUMSEGMENTS_SHARED_UNSET) + ereport(ERROR, + (errcode(ERRCODE_INSUFFICIENT_PRIVILEGE), + errmsg("gprebalance in progress, all relations will be created at %u segments", + gp_rebalance_numsegs))); switch (gp_create_table_default_numsegments) { diff --git a/gpcontrib/gp_toolkit/Makefile b/gpcontrib/gp_toolkit/Makefile index 82fc7f7dee99..852b3d93ba82 100644 --- a/gpcontrib/gp_toolkit/Makefile +++ b/gpcontrib/gp_toolkit/Makefile @@ -1,10 +1,10 @@ EXTENSION = gp_toolkit DATA = gp_toolkit--1.1--1.2.sql gp_toolkit--1.0--1.1.sql gp_toolkit--1.0.sql \ gp_toolkit--1.2--1.3.sql gp_toolkit--1.3.sql gp_toolkit--1.3--1.4.sql \ - gp_toolkit--1.4--1.5.sql gp_toolkit--1.5--1.6.sql + gp_toolkit--1.4--1.5.sql gp_toolkit--1.5--1.6.sql gp_toolkit--1.6--1.7.sql MODULE_big = gp_toolkit ifeq ($(shell uname -s), Linux) -OBJS = resgroup.o gp_partition_maint.o +OBJS = resgroup.o gp_partition_maint.o gp_rebalance_numsegments.o else OBJS = resgroup-dummy.o gp_partition_maint.o endif diff --git a/gpcontrib/gp_toolkit/gp_rebalance_numsegments.c b/gpcontrib/gp_toolkit/gp_rebalance_numsegments.c new file mode 100644 index 000000000000..96dba865886a --- /dev/null +++ b/gpcontrib/gp_toolkit/gp_rebalance_numsegments.c @@ -0,0 +1,99 @@ +#include "postgres.h" + +#include "fmgr.h" +#include "utils/builtins.h" +#include "cdb/cdbutil.h" +#include "catalog/pg_type.h" +#include "storage/lock.h" +#include "utils/gpexpand.h" + +static void +errorOutOnLock(void) +{ + LOCKTAG gp_expand_locktag = + { + /* FIXME: how to fill the locktag? */ + .locktag_field1 = 0xdead, + .locktag_field2 = 0xdead, + .locktag_field3 = 0xdead, + .locktag_field4 = 0xdd, + .locktag_type = LOCKTAG_USERLOCK, + .locktag_lockmethodid = USER_LOCKMETHOD, + }; + LockAcquireResult acquired = LockAcquire(&gp_expand_locktag, AccessExclusiveLock, false, true); + + if (acquired != LOCKACQUIRE_ALREADY_HELD) + ereport(ERROR, + (errcode(ERRCODE_INSUFFICIENT_PRIVILEGE), + errmsg("gprebalance not in progress, function can be used only under catalog lock"))); +} + +/* + * Get the rebalance numsegments when creating a table. + */ +PG_FUNCTION_INFO_V1(gp_get_rebalance_numsegments); +Datum +gp_get_rebalance_numsegments(PG_FUNCTION_ARGS) +{ + errorOutOnLock(); + + uint32 gp_rebalance_numsegs = pg_atomic_read_u32(gp_create_table_rebalance_numsegments); + + PG_RETURN_UINT32(gp_rebalance_numsegs); +} + +/* + * Set the numsegments when creating tables during gprebalance run. + * + * For integer argument the valid range is [1, gp_num_contents_in_cluster]. + */ +PG_FUNCTION_INFO_V1(gp_set_rebalance_numsegments); +Datum +gp_set_rebalance_numsegments(PG_FUNCTION_ARGS) +{ + int numsegments = -1; + + Assert(1 == PG_NARGS()); + + errorOutOnLock(); + + numsegments = PG_GETARG_INT32(0); + + if (numsegments < 1 || numsegments > getgpsegmentCount()) + ereport(ERROR, + (errcode(ERRCODE_NUMERIC_VALUE_OUT_OF_RANGE), + errmsg("invalid integer value for default numsegments: %d", + numsegments), + errhint("Valid range: [1, %d (gp_num_contents_in_cluster)]", + getgpsegmentCount()))); + + pg_atomic_write_u32(gp_create_table_rebalance_numsegments, numsegments); + + return gp_get_rebalance_numsegments(fcinfo); +} + +/* + * Reset the rebalance numsegments when creating a table. + */ +PG_FUNCTION_INFO_V1(gp_reset_rebalance_numsegments); +Datum +gp_reset_rebalance_numsegments(PG_FUNCTION_ARGS) +{ + errorOutOnLock(); + + pg_atomic_write_u32(gp_create_table_rebalance_numsegments, GP_DEFAULT_NUMSEGMENTS_SHARED_UNSET); + + PG_RETURN_VOID(); +} + +/* + * Check if the rebalance numsegments is set. + */ +PG_FUNCTION_INFO_V1(gp_rebalance_numsegments_is_set); +Datum +gp_rebalance_numsegments_is_set(PG_FUNCTION_ARGS) +{ + PG_RETURN_BOOL( + pg_atomic_read_u32(gp_create_table_rebalance_numsegments) + != GP_DEFAULT_NUMSEGMENTS_SHARED_UNSET); +} diff --git a/gpcontrib/gp_toolkit/gp_toolkit--1.6--1.7.sql b/gpcontrib/gp_toolkit/gp_toolkit--1.6--1.7.sql new file mode 100644 index 000000000000..3e0c21b413fa --- /dev/null +++ b/gpcontrib/gp_toolkit/gp_toolkit--1.6--1.7.sql @@ -0,0 +1,33 @@ +/* gpcontrib/gp_toolkit/gp_toolkit--1.6--1.7.sql */ + +-- complain if script is sourced in psql, rather than via ALTER EXTENSION +\echo Use "ALTER EXTENSION gp_toolkit UPDATE TO '1.7" to load this file. \quit + +-- This function set the numsegments when creating tables during rebalance operation. +-- This form accepts an integer argument: [1, gp_num_contents_in_cluster]. +CREATE FUNCTION gp_toolkit.gp_set_rebalance_numsegments(integer) RETURNS int +AS '$libdir/gp_toolkit','gp_set_rebalance_numsegments' +LANGUAGE C STRICT; + +REVOKE ALL ON FUNCTION gp_toolkit.gp_set_rebalance_numsegments(integer) FROM public; + +-- This function reset the default numsegments. +-- This function resets numsegments directly to default value INT_MAX +CREATE FUNCTION gp_toolkit.gp_reset_rebalance_numsegments() RETURNS void +AS '$libdir/gp_toolkit','gp_reset_rebalance_numsegments' +LANGUAGE C STRICT; + +REVOKE ALL ON FUNCTION gp_toolkit.gp_reset_rebalance_numsegments() FROM public; + +-- This function get the default numsegments when creating tables. +CREATE FUNCTION gp_toolkit.gp_get_rebalance_numsegments() RETURNS int +AS '$libdir/gp_toolkit','gp_get_rebalance_numsegments' +LANGUAGE C STRICT; + +REVOKE ALL ON FUNCTION gp_toolkit.gp_get_rebalance_numsegments() FROM public; + +CREATE FUNCTION gp_toolkit.gp_rebalance_numsegments_is_set() RETURNS int +AS '$libdir/gp_toolkit','gp_rebalance_numsegments_is_set' +LANGUAGE C STRICT; + +REVOKE ALL ON FUNCTION gp_toolkit.gp_rebalance_numsegments_is_set() FROM public; diff --git a/gpcontrib/gp_toolkit/gp_toolkit.control b/gpcontrib/gp_toolkit/gp_toolkit.control index 0d9b47e7e6d6..c8c0a4bacad4 100644 --- a/gpcontrib/gp_toolkit/gp_toolkit.control +++ b/gpcontrib/gp_toolkit/gp_toolkit.control @@ -1,5 +1,5 @@ # gp_toolkit extension comment = 'various GPDB administrative views/functions' -default_version = '1.6' +default_version = '1.7' schema = gp_toolkit diff --git a/src/backend/cdb/cdbvars.c b/src/backend/cdb/cdbvars.c index d29a66ab5e2a..d59727eb18c4 100644 --- a/src/backend/cdb/cdbvars.c +++ b/src/backend/cdb/cdbvars.c @@ -334,6 +334,8 @@ bool log_autostats = true; /* GUC to toggle JIT instrumentation output for EXPLAIN */ bool gp_explain_jit = true; +int gp_segment_number_for_table_shrink = 0; + /* -------------------------------------------------------------------------------------------------- * Server debugging */ diff --git a/src/backend/commands/tablecmds.c b/src/backend/commands/tablecmds.c index 5e198508dac6..f4c4412e640a 100644 --- a/src/backend/commands/tablecmds.c +++ b/src/backend/commands/tablecmds.c @@ -78,6 +78,7 @@ #include "commands/vacuum.h" #include "executor/executor.h" #include "executor/instrument.h" +#include "executor/spi.h" #include "foreign/foreign.h" #include "miscadmin.h" #include "nodes/makefuncs.h" @@ -507,9 +508,14 @@ static void RangeVarCallbackForAlterRelation(const RangeVar *rv, Oid relid, Oid oldrelid, void *arg); static void ATExecExpandTable(List **wqueue, Relation rel, AlterTableCmd *cmd); + +static void ATExecRebalanceTable(List **wqueue, Relation rel, AlterTableCmd *cmd); + static void ATRepackTable(Relation origTable, AlteredTableInfo *tab); static void ATExecExpandPartitionTablePrepare(Relation rel); -static void ATExecExpandTableCTAS(AlterTableCmd *rootCmd, Relation rel, AlterTableCmd *cmd); +static void ATExecRebalanceTableCTAS(AlterTableCmd *rootCmd, + Relation rel, AlterTableCmd *cmd, + int targetNumSegments); static void ATExecSetDistributedBy(Relation rel, Node *node, AlterTableCmd *cmd); @@ -4800,6 +4806,7 @@ AlterTableGetLockLevel(List *cmds) case AT_ExpandTable: case AT_ExpandPartitionTablePrepare: case AT_SetDistributedBy: + case AT_Rebalance: cmd_lockmode = AccessExclusiveLock; break; case AT_RepackTable: /* GPDB: REPACK TABLE */ @@ -5293,6 +5300,59 @@ ATPrepCmd(List **wqueue, Relation rel, AlterTableCmd *cmd, ATSimpleRecursion(wqueue, rel, cmd, recurse, lockmode); pass = AT_PASS_MISC; break; + case AT_Rebalance: + ATSimplePermissions(rel, ATT_TABLE | ATT_FOREIGN_TABLE | ATT_MATVIEW); + + Assert(IsA(cmd->def, Integer)); + int targetNumSegments = intVal(cmd->def); + + if (targetNumSegments <= 0) + { + targetNumSegments = GP_POLICY_DEFAULT_NUMSEGMENTS(); + + elog(NOTICE, + "REBALANCE to current default target number of segments %d", + targetNumSegments); + + pfree(cmd->def); + cmd->def = (Node *) makeInteger(targetNumSegments); + } + + if (!recursing) + { + if (Gp_role == GP_ROLE_DISPATCH && + rel->rd_cdbpolicy->numsegments < targetNumSegments && + targetNumSegments != getgpsegmentCount()) + { + targetNumSegments = getgpsegmentCount(); + elog(NOTICE, + "REBALANCE currently supports expand only to total cluster " + "segments number, so target number of segments is forced to %d", + targetNumSegments); + } + + if (Gp_role == GP_ROLE_DISPATCH && + rel->rd_cdbpolicy->numsegments == targetNumSegments) + ereport(ERROR, + (errcode(ERRCODE_WRONG_OBJECT_TYPE), + errmsg("cannot rebalance table \"%s\"", + RelationGetRelationName(rel)), + errdetail("table has already been rebalanced"))); + + if (rel->rd_rel->relispartition) + ereport(ERROR, + (errcode(ERRCODE_WRONG_OBJECT_TYPE), + errmsg("cannot rebalance leaf or interior partition \"%s\"", + RelationGetRelationName(rel)), + errdetail("Root/leaf/interior partitions need to have same numsegments"), + errhint("Call ALTER TABLE REBALANCE on the root table instead"))); + + } + + ATSimpleRecursion(wqueue, rel, cmd, recurse, lockmode); + pass = AT_PASS_MISC; + break; + case AT_RepackTable: /* GPDB: REPACK TABLE */ ATSimplePermissions(rel, ATT_TABLE); ATSimpleRecursion(wqueue, rel, cmd, recurse, lockmode); @@ -5844,6 +5904,9 @@ ATExecCmd(List **wqueue, AlteredTableInfo *tab, Relation rel, case AT_ExpandTable: /* EXPAND TABLE */ ATExecExpandTable(wqueue, rel, cmd); break; + case AT_Rebalance: /* REBALANCE */ + ATExecRebalanceTable(wqueue, rel, cmd); + break; case AT_RepackTable: for (int i = 0; i < AT_NUM_PASSES; ++i) { @@ -17452,7 +17515,7 @@ ATExecExpandTable(List **wqueue, Relation rel, AlterTableCmd *cmd) } else { - ATExecExpandTableCTAS(rootCmd, rel, cmd); + ATExecRebalanceTableCTAS(rootCmd, rel, cmd, getgpsegmentCount()); } /* Update numsegments to cluster size */ @@ -17460,6 +17523,177 @@ ATExecExpandTable(List **wqueue, Relation rel, AlterTableCmd *cmd) GpPolicyReplace(relid, newPolicy); } +/* + * Move data as needed from the excessive segments into the reduced number of + * segments. + * + * We do it only for hashed or randomly distributed tables. There is no need for + * the replicated tables. + * + * Moving is done by inserting values from the excessive segments. The plan for + * the query is forced to treat the table as randonly distributed and force + * redistribute motion for the insert action. Distribution policy for the insert + * action is adjusted at the planning stage to consider only the target number + * of segments. + */ +static void +ATExecShrinkTable(Relation rel, GpPolicy *policy) +{ + if (Gp_role == GP_ROLE_DISPATCH && GpPolicyIsPartitioned(policy)) + { + volatile bool connected = false; + StringInfoData sqlstmtInsert; + initStringInfo(&sqlstmtInsert); + char *nsp = get_namespace_name(RelationGetNamespace(rel)); + char *relname = RelationGetRelationName(rel); + + /* + * 'gp_dist_random' will cause fallback to Postgres planner, + * so no need to tweak 'optimizer' value. + */ + appendStringInfo(&sqlstmtInsert, + "insert into %s.%s select * " + "from gp_dist_random('%s.%s') " + "where gp_segment_id >= %d", + nsp, relname, + nsp, relname, + policy->numsegments); + + gp_segment_number_for_table_shrink = policy->numsegments; + + PG_TRY(); + { + if (SPI_OK_CONNECT != SPI_connect()) + ereport(ERROR, + (errcode(ERRCODE_INTERNAL_ERROR), + errmsg("unable shrink table"), + errdetail("SPI_connect failed in %s.", __func__))); + + connected = true; + + if (SPI_execute(sqlstmtInsert.data, false, 0) <= 0) + ereport(ERROR, + (errcode(ERRCODE_INTERNAL_ERROR), + errmsg("unable shrink table"), + errdetail("SPI_execute failed in %s.", __func__))); + + connected = false; + SPI_finish(); + } + + /* Clean up in case of error. */ + PG_CATCH(); + { + if (connected) + SPI_finish(); + + pfree(sqlstmtInsert.data); + + gp_segment_number_for_table_shrink = 0; + + /* Carry on with error handling. */ + PG_RE_THROW(); + } + PG_END_TRY(); + + pfree(sqlstmtInsert.data); + + gp_segment_number_for_table_shrink = 0; + } +} + +/* + * ALTER TABLE REBALANCE + * + * In case the target number of segments is more than the number of segments in + * the table's distribution policy, we invoke the expand functionality. + * Otherwise we shrink the table and update table's "numsegments" value to the + * target number of segments. + */ +static void +ATExecRebalanceTable(List **wqueue, Relation rel, AlterTableCmd *cmd) +{ + MemoryContext oldContext; + Oid relid = RelationGetRelid(rel); + GpPolicy *newPolicy; + GpPolicy *policy = rel->rd_cdbpolicy; + int targetNumSegments = intVal(cmd->def); + + if (Gp_role == GP_ROLE_UTILITY) + ereport(ERROR, + (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), + errmsg("REBALANCE not supported in utility mode"))); + + /* Permissions checks */ + if (!pg_class_ownercheck(relid, GetUserId())) + aclcheck_error(ACLCHECK_NOT_OWNER, OBJECT_TABLE, + RelationGetRelationName(rel)); + + if (IsSystemRelation(rel)) + ereport(ERROR, + (errcode(ERRCODE_INSUFFICIENT_PRIVILEGE), + errmsg("permission denied: \"%s\" is a system catalog", + RelationGetRelationName(rel)))); + + if (targetNumSegments > policy->numsegments) + { + ATExecExpandTable(wqueue, rel, cmd); + return; + } + + oldContext = MemoryContextSwitchTo(GetMemoryChunkContext(rel)); + newPolicy = GpPolicyCopy(policy); + MemoryContextSwitchTo(oldContext); + newPolicy->numsegments = targetNumSegments; + + if (rel->rd_rel->relkind == RELKIND_PARTITIONED_TABLE) + { + /* + * Nothing to do on a partitioned table. But we better recurse to the + * child partitions. + */ + } + else if (rel->rd_rel->relkind == RELKIND_FOREIGN_TABLE) + { + if (rel_is_external_table(relid)) + { + ExtTableEntry *ext = GetExtTableEntry(relid); + + if (!ext->iswritable) + { + /* + * Skip expanding readable external table, since data is not + * located inside gpdb + */ + return; + } + } + else + { + /* Skip expanding foreign table, since data is not located inside gpdb */ + return; + } + } + else if (rel->rd_rel->relkind == RELKIND_MATVIEW) + { + /* + * We can't insert data directly to an existing matview, + * therefore the approach from ATExecShrinkTable() is not suitable, + * and we use CTAS method for matviews. + */ + AlteredTableInfo *tab = linitial(*wqueue); + AlterTableCmd *rootCmd = + (AlterTableCmd *)linitial(tab->subcmds[AT_PASS_MISC]); + ATExecRebalanceTableCTAS(rootCmd, rel, cmd, targetNumSegments); + } + else + { + ATExecShrinkTable(rel, newPolicy); + } + + GpPolicyReplace(relid, newPolicy); +} + /* * ALTER TABLE xxx EXPAND PARTITION PREPARE * @@ -17553,7 +17787,8 @@ ATExecExpandPartitionTablePrepare(Relation rel) } static void -ATExecExpandTableCTAS(AlterTableCmd *rootCmd, Relation rel, AlterTableCmd *cmd) +ATExecRebalanceTableCTAS(AlterTableCmd *rootCmd, Relation rel, + AlterTableCmd *cmd, int targetNumSegments) { RangeVar *tmprv; Oid tmprelid; @@ -17592,7 +17827,7 @@ ATExecExpandTableCTAS(AlterTableCmd *rootCmd, Relation rel, AlterTableCmd *cmd) /* Step (b) - build CTAS */ distby = make_distributedby_for_rel(rel); - distby->numsegments = getgpsegmentCount(); + distby->numsegments = targetNumSegments; queryDesc = build_ctas_with_dist(rel, distby, untransformRelOptions(get_rel_opts(rel)), diff --git a/src/backend/gpopt/gpdbwrappers.cpp b/src/backend/gpopt/gpdbwrappers.cpp index cc0a1c1b8040..ab2090c022df 100644 --- a/src/backend/gpopt/gpdbwrappers.cpp +++ b/src/backend/gpopt/gpdbwrappers.cpp @@ -1094,6 +1094,17 @@ gpdb::GetGPSegmentCount(void) return 0; } +int +gpdb::GetGPTargetSegmentCount(void) +{ + GP_WRAP_START; + { + return GP_POLICY_DEFAULT_NUMSEGMENTS(); + } + GP_WRAP_END; + return 0; +} + bool gpdb::HeapAttIsNull(HeapTuple tup, int attno) { diff --git a/src/backend/gpopt/translate/CTranslatorDXLToPlStmt.cpp b/src/backend/gpopt/translate/CTranslatorDXLToPlStmt.cpp index e8c514060e79..a71cdad3aacf 100644 --- a/src/backend/gpopt/translate/CTranslatorDXLToPlStmt.cpp +++ b/src/backend/gpopt/translate/CTranslatorDXLToPlStmt.cpp @@ -6304,6 +6304,13 @@ CTranslatorDXLToPlStmt::TranslateDXLPhyCtasToDistrPolicy( num_of_distr_cols_alloc = num_of_distr_cols; } + if (gpdb::GetGPTargetSegmentCount() != gpdb::GetGPSegmentCount()) + { + // GPORCA does not support partially distributed tables yet + GPOS_RAISE(gpdxl::ExmaMD, gpdxl::ExmiDXLInvalidAttributeValue, + GPOS_WSZ_LIT("Partially Distributed Data")); + } + // always set numsegments to ALL for CTAS GpPolicy *distr_policy = gpdb::MakeGpPolicy(POLICYTYPE_PARTITIONED, num_of_distr_cols_alloc, diff --git a/src/backend/optimizer/util/pathnode.c b/src/backend/optimizer/util/pathnode.c index 490c3679bb0c..faa9c8a228f5 100644 --- a/src/backend/optimizer/util/pathnode.c +++ b/src/backend/optimizer/util/pathnode.c @@ -5406,6 +5406,15 @@ adjust_modifytable_subpaths(PlannerInfo *root, CmdType operation, targetPolicy = GpPolicyFetch(rte->relid); targetPolicyType = targetPolicy->ptype; + /* + * Limit the number of segments to the target value that we want + * to achieve by the shrink operation. + */ + if (targetPolicyType == POLICYTYPE_PARTITIONED && + gp_segment_number_for_table_shrink > 0) + targetPolicy->numsegments = + Min(targetPolicy->numsegments, gp_segment_number_for_table_shrink); + numsegments = Max(targetPolicy->numsegments, numsegments); if (targetPolicyType == POLICYTYPE_PARTITIONED) diff --git a/src/backend/parser/gram.y b/src/backend/parser/gram.y index 9e94f28411fc..3197a1c2ced0 100644 --- a/src/backend/parser/gram.y +++ b/src/backend/parser/gram.y @@ -837,7 +837,7 @@ static void check_expressions_in_partition_key(PartitionSpec *spec, core_yyscan_ QUEUE RANDOMLY READABLE READS REJECT_P REPACK REPLICATED - RESOURCE ROOTPARTITION + RESOURCE ROOTPARTITION REBALANCE SCATTER SEGMENT SEGMENTS SPLIT SUBPARTITION @@ -3332,6 +3332,22 @@ alter_table_cmd: n->subtype = AT_ExpandPartitionTablePrepare; $$ = (Node *)n; } + /* ALTER TABLE REBALANCE*/ + | REBALANCE + { + AlterTableCmd *n = makeNode(AlterTableCmd); + n->subtype = AT_Rebalance; + n->def = (Node *) makeInteger(0); + $$ = (Node *)n; + } + /* ALTER TABLE REBALANCE */ + | REBALANCE SignedIconst + { + AlterTableCmd *n = makeNode(AlterTableCmd); + n->subtype = AT_Rebalance; + n->def = (Node *) makeInteger($2); + $$ = (Node *)n; + } /* ALTER TABLE OF */ | OF any_name { @@ -18313,6 +18329,7 @@ unreserved_keyword: | READABLE | READS | REASSIGN + | REBALANCE | RECHECK | RECURSIVE | REF diff --git a/src/backend/storage/ipc/ipci.c b/src/backend/storage/ipc/ipci.c index bd71b5df69ca..8e95fc556e34 100644 --- a/src/backend/storage/ipc/ipci.c +++ b/src/backend/storage/ipc/ipci.c @@ -227,6 +227,9 @@ CreateSharedMemoryAndSemaphores(int port) /* size of expand version */ size = add_size(size, GpExpandVersionShmemSize()); + /* size of rebalance numsegments variable */ + size = add_size(size, GpRebalanceNumsegmentsShmemSize()); + /* size of token and endpoint shared memory */ size = add_size(size, EndpointShmemSize()); @@ -385,6 +388,8 @@ CreateSharedMemoryAndSemaphores(int port) GpExpandVersionShmemInit(); + GpRebalanceNumsegmentsShmemInit(); + #ifdef EXEC_BACKEND /* diff --git a/src/backend/utils/misc/gpexpand.c b/src/backend/utils/misc/gpexpand.c index 2cd826516087..c937c2551274 100644 --- a/src/backend/utils/misc/gpexpand.c +++ b/src/backend/utils/misc/gpexpand.c @@ -29,9 +29,13 @@ #include "utils/rel.h" #include "utils/relcache.h" #include "utils/gpexpand.h" +#include "port/atomics.h" +#include "catalog/gp_distribution_policy.h" static volatile int *gp_expand_version; +volatile pg_atomic_uint32 *gp_create_table_rebalance_numsegments; + /* * Catalog lock. */ @@ -157,3 +161,25 @@ gp_expand_protect_catalog_changes(Relation relation) "catalog changes are disallowed", oldVersion, newVersion))); } +int +GpRebalanceNumsegmentsShmemSize(void) +{ + return sizeof(*gp_create_table_rebalance_numsegments); +} + +void +GpRebalanceNumsegmentsShmemInit(void) +{ + if (IsUnderPostmaster) + return; + + /* only postmaster initialize it */ + gp_create_table_rebalance_numsegments = (volatile pg_atomic_uint32 *)ShmemAlloc(GpRebalanceNumsegmentsShmemSize()); + pg_atomic_init_u32(gp_create_table_rebalance_numsegments, GP_DEFAULT_NUMSEGMENTS_SHARED_UNSET); +} + +uint32 +getRebalanceNumsegments(void) +{ + return pg_atomic_read_u32(gp_create_table_rebalance_numsegments); +} diff --git a/src/bin/psql/tab-complete.c b/src/bin/psql/tab-complete.c index c8fa2e5122b9..0edcf6c1b8b5 100644 --- a/src/bin/psql/tab-complete.c +++ b/src/bin/psql/tab-complete.c @@ -1883,7 +1883,7 @@ psql_completion(const char *text, int start, int end) "ENABLE", "INHERIT", "NO INHERIT", "RENAME", "RESET", "OWNER TO", "SET", "VALIDATE CONSTRAINT", "REPLICA IDENTITY", "ATTACH PARTITION", - "DETACH PARTITION"); + "DETACH PARTITION", "REBALANCE"); /* ALTER TABLE xxx ENABLE */ else if (Matches("ALTER", "TABLE", MatchAny, "ENABLE")) COMPLETE_WITH("ALWAYS", "REPLICA", "ROW LEVEL SECURITY", "RULE", diff --git a/src/include/catalog/gp_distribution_policy.h b/src/include/catalog/gp_distribution_policy.h index fdb864e68137..76ba7839935b 100644 --- a/src/include/catalog/gp_distribution_policy.h +++ b/src/include/catalog/gp_distribution_policy.h @@ -59,6 +59,10 @@ typedef FormData_gp_distribution_policy *Form_gp_distribution_policy; * gp_create_table_default_numsegments. */ #define GP_POLICY_DEFAULT_NUMSEGMENTS() \ +( (getRebalanceNumsegments() != GP_DEFAULT_NUMSEGMENTS_SHARED_UNSET) ? \ + getRebalanceNumsegments() : GP_POLICY_ORIGINAL_BEHAVIOR() ) + +#define GP_POLICY_ORIGINAL_BEHAVIOR() \ ( gp_create_table_default_numsegments == GP_DEFAULT_NUMSEGMENTS_FULL ? getgpsegmentCount() \ : gp_create_table_default_numsegments == GP_DEFAULT_NUMSEGMENTS_RANDOM ? (1 + random() % getgpsegmentCount()) \ : gp_create_table_default_numsegments == GP_DEFAULT_NUMSEGMENTS_MINIMAL ? 1 \ @@ -67,12 +71,14 @@ typedef FormData_gp_distribution_policy *Form_gp_distribution_policy; /* * The the default numsegments policies when creating a table. * + * - UNSET: shared value used during rebalance is not set, default variable is used * - FULL: all the segments; * - RANDOM: pick a random set of segments each time; * - MINIMAL: the minimal set of segments; */ enum { + GP_DEFAULT_NUMSEGMENTS_SHARED_UNSET = 0x7fffffff, GP_DEFAULT_NUMSEGMENTS_FULL = -1, GP_DEFAULT_NUMSEGMENTS_RANDOM = -2, GP_DEFAULT_NUMSEGMENTS_MINIMAL = -3, @@ -113,7 +119,6 @@ typedef struct GpPolicy * Global Variables */ extern int gp_create_table_default_numsegments; - /* * GpPolicyCopy -- Return a copy of a GpPolicy object. * @@ -167,4 +172,6 @@ extern GpPolicy *createHashPartitionedPolicy(List *keys, List *opclasses, int nu extern bool IsReplicatedTable(Oid relid); +extern uint32 getRebalanceNumsegments(void); + #endif /* GP_DISTRIBUTION_POLICY_H */ diff --git a/src/include/cdb/cdbvars.h b/src/include/cdb/cdbvars.h index 4c0a8826773d..9518af9e23d7 100644 --- a/src/include/cdb/cdbvars.h +++ b/src/include/cdb/cdbvars.h @@ -433,6 +433,8 @@ extern int gp_udpic_network_disable_ipv6; */ extern uint32 gp_interconnect_id; +extern int gp_segment_number_for_table_shrink; + /* -------------------------------------------------------------------------------------------------- * Logging */ diff --git a/src/include/gpopt/gpdbwrappers.h b/src/include/gpopt/gpdbwrappers.h index c68e0100aa75..9cd2ccfb1460 100644 --- a/src/include/gpopt/gpdbwrappers.h +++ b/src/include/gpopt/gpdbwrappers.h @@ -292,9 +292,12 @@ Oid GetTypeRelid(Oid typid); // name of the type with the given oid char *GetTypeName(Oid typid); -// number of GP segments +// total number of GP segments int GetGPSegmentCount(void); +// target number of GP segments to use +int GetGPTargetSegmentCount(void); + // heap attribute is null bool HeapAttIsNull(HeapTuple tup, int attnum); diff --git a/src/include/nodes/parsenodes.h b/src/include/nodes/parsenodes.h index 04d67c653f04..1d58ed11f655 100644 --- a/src/include/nodes/parsenodes.h +++ b/src/include/nodes/parsenodes.h @@ -2019,7 +2019,9 @@ typedef enum AlterTableType AT_PartTruncate, /* Truncate */ /* kept at end for ABI hygiene */ - AT_RepackTable /* REPACK TABLE */ + AT_RepackTable, /* REPACK TABLE */ + + AT_Rebalance /* REBALANCE */ } AlterTableType; typedef struct ReplicaIdentityStmt diff --git a/src/include/parser/kwlist.h b/src/include/parser/kwlist.h index cdb394701bc8..6f59aad2940a 100644 --- a/src/include/parser/kwlist.h +++ b/src/include/parser/kwlist.h @@ -372,6 +372,7 @@ PG_KEYWORD("readable", READABLE, UNRESERVED_KEYWORD) /* GPDB */ PG_KEYWORD("reads", READS, UNRESERVED_KEYWORD) PG_KEYWORD("real", REAL, COL_NAME_KEYWORD) PG_KEYWORD("reassign", REASSIGN, UNRESERVED_KEYWORD) +PG_KEYWORD("rebalance", REBALANCE, UNRESERVED_KEYWORD) /* GPDB */ PG_KEYWORD("recheck", RECHECK, UNRESERVED_KEYWORD) PG_KEYWORD("recursive", RECURSIVE, UNRESERVED_KEYWORD) PG_KEYWORD("ref", REF, UNRESERVED_KEYWORD) diff --git a/src/include/utils/gpexpand.h b/src/include/utils/gpexpand.h index 8614c3434077..cd8f8d4bb44b 100644 --- a/src/include/utils/gpexpand.h +++ b/src/include/utils/gpexpand.h @@ -24,6 +24,10 @@ extern void gp_expand_protect_catalog_changes(Relation relation); extern Datum gp_expand_bump_version(PG_FUNCTION_ARGS); +extern volatile pg_atomic_uint32 *gp_create_table_rebalance_numsegments; + +extern void GpRebalanceNumsegmentsShmemInit(void); +extern int GpRebalanceNumsegmentsShmemSize(void); #endif /* GPEXPAND_H */ diff --git a/src/test/isolation2/expected/alter_blocks_for_update_and_viceversa.out b/src/test/isolation2/expected/alter_blocks_for_update_and_viceversa.out index 7f9e1a7004a0..0c9abaef9e62 100644 --- a/src/test/isolation2/expected/alter_blocks_for_update_and_viceversa.out +++ b/src/test/isolation2/expected/alter_blocks_for_update_and_viceversa.out @@ -36,3 +36,69 @@ ALTER TABLE COMMIT 1<: <... completed> UPDATE 1 + +-- Check ALTER REBALANCE +1: DROP TABLE alter_block; +DROP TABLE +1: CREATE TABLE alter_block(a int, b int) DISTRIBUTED BY (a); +CREATE TABLE +1: INSERT INTO alter_block SELECT i, i FROM generate_series(1, 20)i; +INSERT 0 20 + + +-- Validate SELECT blocks the ALTER +1: BEGIN; +BEGIN +2: BEGIN; +BEGIN +1: SELECT count(1), gp_segment_id FROM alter_block GROUP BY gp_segment_id ORDER BY gp_segment_id; + count | gp_segment_id +-------+--------------- + 8 | 0 + 4 | 1 + 8 | 2 +(3 rows) +2&: ALTER TABLE alter_block REBALANCE 2; +1: COMMIT; +COMMIT +2<: <... completed> +ALTER TABLE +2: ROLLBACK; +ROLLBACK + +-- Validate UPDATE blocks the ALTER +1: BEGIN; +BEGIN +2: BEGIN; +BEGIN +1: UPDATE alter_block SET b = b + 1 WHERE a = 1; +UPDATE 1 +2&: ALTER TABLE alter_block REBALANCE 2; +1: COMMIT; +COMMIT +2<: <... completed> +ALTER TABLE +2: ROLLBACK; +ROLLBACK + +-- Validate ALTER blocks the SELECT +1: BEGIN; +BEGIN +2: BEGIN; +BEGIN +2: ALTER TABLE alter_block REBALANCE 2; +ALTER TABLE +1&: SELECT count(1), gp_segment_id FROM alter_block GROUP BY gp_segment_id ORDER BY gp_segment_id; +2: COMMIT; +COMMIT +1<: <... completed> + count | gp_segment_id +-------+--------------- + 12 | 0 + 8 | 1 +(2 rows) +1: COMMIT; +COMMIT + +1: DROP TABLE alter_block; +DROP TABLE diff --git a/src/test/isolation2/expected/rebalance_numsegments.out b/src/test/isolation2/expected/rebalance_numsegments.out new file mode 100644 index 000000000000..db00c1a2115b --- /dev/null +++ b/src/test/isolation2/expected/rebalance_numsegments.out @@ -0,0 +1,84 @@ +-- Test setting numsegments value for table creation during rebalance operation +-- Test error throwing when catalog is not locked by gprebalance +select gp_toolkit.gp_set_rebalance_numsegments(2); +ERROR: gprebalance not in progress, function can be used only under catalog lock +select gp_toolkit.gp_get_rebalance_numsegments(); +ERROR: gprebalance not in progress, function can be used only under catalog lock +select gp_toolkit.gp_reset_rebalance_numsegments(); +ERROR: gprebalance not in progress, function can be used only under catalog lock +-- Test the numsegment value setting mechanics +1: begin; +BEGIN +1: select gp_expand_lock_catalog(); + gp_expand_lock_catalog +------------------------ + +(1 row) +2: create table t_reb (i int) distributed by (i); +ERROR: gpexpand in progress, catalog changes are disallowed. +1: select gp_toolkit.gp_get_rebalance_numsegments(); + gp_get_rebalance_numsegments +------------------------------ + 2147483647 +(1 row) +1: select gp_toolkit.gp_set_rebalance_numsegments(2); + gp_set_rebalance_numsegments +------------------------------ + 2 +(1 row) +1: select gp_toolkit.gp_get_rebalance_numsegments(); + gp_get_rebalance_numsegments +------------------------------ + 2 +(1 row) +1: select gp_toolkit.gp_reset_rebalance_numsegments(); + gp_reset_rebalance_numsegments +-------------------------------- + +(1 row) +1: select gp_toolkit.gp_get_rebalance_numsegments(); + gp_get_rebalance_numsegments +------------------------------ + 2147483647 +(1 row) +1: select gp_toolkit.gp_set_rebalance_numsegments(2); + gp_set_rebalance_numsegments +------------------------------ + 2 +(1 row) +1: commit; +COMMIT +-- Test that newly created tables are using updated number of segmetns +1: create table t_reb (i int) distributed by (i); +CREATE TABLE +1: select numsegments from gp_distribution_policy where localoid = 't_reb'::regclass; + numsegments +------------- + 2 +(1 row) +1: drop table t_reb; +DROP TABLE + +1: begin; +BEGIN +1: select gp_expand_lock_catalog(); + gp_expand_lock_catalog +------------------------ + +(1 row) +1: select gp_toolkit.gp_reset_rebalance_numsegments(); + gp_reset_rebalance_numsegments +-------------------------------- + +(1 row) +1: create table t_reb (i int) distributed by (i); +CREATE TABLE +1: select numsegments from gp_distribution_policy where localoid = 't_reb'::regclass; + numsegments +------------- + 3 +(1 row) +1: rollback; +ROLLBACK +1q: ... +2q: ... diff --git a/src/test/isolation2/isolation2_schedule b/src/test/isolation2/isolation2_schedule index 45afebf07d92..3ebfb5eab1b1 100644 --- a/src/test/isolation2/isolation2_schedule +++ b/src/test/isolation2/isolation2_schedule @@ -373,3 +373,4 @@ test: brin_heap # Intensive tests for BRIN test: uao/brin_chain_row uao/brin_chain_column +test: rebalance_numsegments diff --git a/src/test/isolation2/sql/alter_blocks_for_update_and_viceversa.sql b/src/test/isolation2/sql/alter_blocks_for_update_and_viceversa.sql index b22000f13c30..f89a077abb28 100644 --- a/src/test/isolation2/sql/alter_blocks_for_update_and_viceversa.sql +++ b/src/test/isolation2/sql/alter_blocks_for_update_and_viceversa.sql @@ -17,3 +17,38 @@ 2: SELECT wait_event_type FROM pg_stat_activity where query like 'UPDATE alter_block SET %'; 2: COMMIT; 1<: + +-- Check ALTER REBALANCE +1: DROP TABLE alter_block; +1: CREATE TABLE alter_block(a int, b int) DISTRIBUTED BY (a); +1: INSERT INTO alter_block SELECT i, i FROM generate_series(1, 20)i; + + +-- Validate SELECT blocks the ALTER +1: BEGIN; +2: BEGIN; +1: SELECT count(1), gp_segment_id FROM alter_block GROUP BY gp_segment_id ORDER BY gp_segment_id; +2&: ALTER TABLE alter_block REBALANCE 2; +1: COMMIT; +2<: +2: ROLLBACK; + +-- Validate UPDATE blocks the ALTER +1: BEGIN; +2: BEGIN; +1: UPDATE alter_block SET b = b + 1 WHERE a = 1; +2&: ALTER TABLE alter_block REBALANCE 2; +1: COMMIT; +2<: +2: ROLLBACK; + +-- Validate ALTER blocks the SELECT +1: BEGIN; +2: BEGIN; +2: ALTER TABLE alter_block REBALANCE 2; +1&: SELECT count(1), gp_segment_id FROM alter_block GROUP BY gp_segment_id ORDER BY gp_segment_id; +2: COMMIT; +1<: +1: COMMIT; + +1: DROP TABLE alter_block; diff --git a/src/test/isolation2/sql/rebalance_numsegments.sql b/src/test/isolation2/sql/rebalance_numsegments.sql new file mode 100644 index 000000000000..fb8c75a13d36 --- /dev/null +++ b/src/test/isolation2/sql/rebalance_numsegments.sql @@ -0,0 +1,29 @@ +-- Test setting numsegments value for table creation during rebalance operation +-- Test error throwing when catalog is not locked by gprebalance +select gp_toolkit.gp_set_rebalance_numsegments(2); +select gp_toolkit.gp_get_rebalance_numsegments(); +select gp_toolkit.gp_reset_rebalance_numsegments(); +-- Test the numsegment value setting mechanics +1: begin; +1: select gp_expand_lock_catalog(); +2: create table t_reb (i int) distributed by (i); +1: select gp_toolkit.gp_get_rebalance_numsegments(); +1: select gp_toolkit.gp_set_rebalance_numsegments(2); +1: select gp_toolkit.gp_get_rebalance_numsegments(); +1: select gp_toolkit.gp_reset_rebalance_numsegments(); +1: select gp_toolkit.gp_get_rebalance_numsegments(); +1: select gp_toolkit.gp_set_rebalance_numsegments(2); +1: commit; +-- Test that newly created tables are using updated number of segmetns +1: create table t_reb (i int) distributed by (i); +1: select numsegments from gp_distribution_policy where localoid = 't_reb'::regclass; +1: drop table t_reb; + +1: begin; +1: select gp_expand_lock_catalog(); +1: select gp_toolkit.gp_reset_rebalance_numsegments(); +1: create table t_reb (i int) distributed by (i); +1: select numsegments from gp_distribution_policy where localoid = 't_reb'::regclass; +1: rollback; +1q: +2q: diff --git a/src/test/regress/expected/alter_rebalance.out b/src/test/regress/expected/alter_rebalance.out new file mode 100644 index 000000000000..24a6a1c892a2 --- /dev/null +++ b/src/test/regress/expected/alter_rebalance.out @@ -0,0 +1,3619 @@ +-- Check 'ALTER TABLE ... REBALANCE' command and 'gp_target_numsegments' GUC +-- Create hashed distributed tables +create table table_distr_hashed(a int) distributed by (a); +insert into table_distr_hashed select generate_series(1, 20); +create table table_distr_hashed_ao_row(a int) with (appendonly=true, orientation=row) distributed by (a); +insert into table_distr_hashed_ao_row select generate_series(1, 20); +create table table_distr_hashed_ao_col(a int) with (appendonly=true, orientation=column) distributed by (a); +insert into table_distr_hashed_ao_col select generate_series(1, 20); +-- Create randomly distributed tables +create table table_distr_random(a int) distributed randomly; +insert into table_distr_random select generate_series(1, 20); +create table table_distr_random_ao_row(a int) with (appendonly=true, orientation=row) distributed randomly; +insert into table_distr_random_ao_row select generate_series(1, 20); +create table table_distr_random_ao_col(a int) with (appendonly=true, orientation=column) distributed randomly; +insert into table_distr_random_ao_col select generate_series(1, 20); +-- Create replicated distributed tables +create table table_distr_replicated(a int) distributed replicated; +insert into table_distr_replicated select generate_series(1, 20); +create table table_distr_replicated_ao_row(a int) with (appendonly=true, orientation=row) distributed replicated; +insert into table_distr_replicated_ao_row select generate_series(1, 20); +create table table_distr_replicated_ao_col(a int) with (appendonly=true, orientation=column) distributed replicated; +insert into table_distr_replicated_ao_col select generate_series(1, 20); +-- Create part tables +create table part_range_table_distr_hashed (a int, b date) distributed by (a) +partition by range (b) ( + start (date '2023-01-01') inclusive + end (date '2024-01-01') exclusive + every (interval '1 month'), + default partition other_vals +); +insert into part_range_table_distr_hashed select i, '2023-01-02' from generate_series(1, 20)i; +insert into part_range_table_distr_hashed select i, '2023-05-02' from generate_series(1, 20)i; +insert into part_range_table_distr_hashed select i, '2020-05-02' from generate_series(1, 20)i; +create table part_range_table_distr_random (a int, b date) distributed randomly +partition by range (b) ( + start (date '2023-01-01') inclusive + end (date '2024-01-01') exclusive + every (interval '1 month'), + default partition other_vals +); +insert into part_range_table_distr_random select i, '2023-01-02' from generate_series(1, 20)i; +insert into part_range_table_distr_random select i, '2023-05-02' from generate_series(1, 20)i; +insert into part_range_table_distr_random select i, '2020-05-02' from generate_series(1, 20)i; +create table part_list_table_distr_hashed(a int, b text) distributed by (a) +partition by list (b) ( + partition part1 values ('test1'), + partition part2 values ('test2'), + default partition other_vals +); +insert into part_list_table_distr_hashed select i, 'test1' from generate_series(1, 20)i; +insert into part_list_table_distr_hashed select i, 'test2' from generate_series(1, 20)i; +insert into part_list_table_distr_hashed select i, 'test3' from generate_series(1, 20)i; +create table part_list_table_distr_random(a int, b text) distributed randomly +partition by list (b) ( + partition part1 values ('test1'), + partition part2 values ('test2'), + default partition other_vals +); +insert into part_list_table_distr_random select i, 'test1' from generate_series(1, 20)i; +insert into part_list_table_distr_random select i, 'test2' from generate_series(1, 20)i; +insert into part_list_table_distr_random select i, 'test3' from generate_series(1, 20)i; +create table multi_part_table_distr_hashed(a int, b date, c text) distributed by (a) +partition by range (b) +subpartition by list (c) subpartition template +( + subpartition subpart1 values ('test1'), + subpartition subpart2 values ('test2') +) +( + partition part1 start (date '2023-01-01'), + partition part2 start (date '2023-02-01'), + partition part3 start (date '2023-03-01') end (date '2024-01-01') +); +insert into multi_part_table_distr_hashed select i, '2023-01-05', 'test1' from generate_series(1, 20)i; +insert into multi_part_table_distr_hashed select i, '2023-02-05', 'test2' from generate_series(1, 20)i; +insert into multi_part_table_distr_hashed select i, '2023-03-05', 'test1' from generate_series(1, 20)i; +-- Now check shrink of the created tables into 2 segments +begin; +select gp_expand_lock_catalog(); + gp_expand_lock_catalog +------------------------ + +(1 row) + +select gp_toolkit.gp_set_rebalance_numsegments(2); + gp_set_rebalance_numsegments +------------------------------ + 2 +(1 row) + +end; +alter table table_distr_hashed rebalance; +NOTICE: REBALANCE to current default target number of segments 2 +alter table table_distr_hashed_ao_row rebalance; +NOTICE: REBALANCE to current default target number of segments 2 +alter table table_distr_hashed_ao_col rebalance; +NOTICE: REBALANCE to current default target number of segments 2 +alter table table_distr_random rebalance; +NOTICE: REBALANCE to current default target number of segments 2 +alter table table_distr_random_ao_row rebalance; +NOTICE: REBALANCE to current default target number of segments 2 +alter table table_distr_random_ao_col rebalance; +NOTICE: REBALANCE to current default target number of segments 2 +alter table table_distr_replicated rebalance; +NOTICE: REBALANCE to current default target number of segments 2 +alter table table_distr_replicated_ao_row rebalance; +NOTICE: REBALANCE to current default target number of segments 2 +alter table table_distr_replicated_ao_col rebalance; +NOTICE: REBALANCE to current default target number of segments 2 +alter table part_range_table_distr_hashed rebalance; +NOTICE: REBALANCE to current default target number of segments 2 +alter table part_range_table_distr_random rebalance; +NOTICE: REBALANCE to current default target number of segments 2 +alter table part_list_table_distr_hashed rebalance; +NOTICE: REBALANCE to current default target number of segments 2 +alter table part_list_table_distr_random rebalance; +NOTICE: REBALANCE to current default target number of segments 2 +alter table multi_part_table_distr_hashed rebalance; +NOTICE: REBALANCE to current default target number of segments 2 +-- Verify that data is presented only on segments #0 and #1 +select a, gp_segment_id from table_distr_hashed order by a; + a | gp_segment_id +----+--------------- + 1 | 1 + 2 | 0 + 3 | 0 + 4 | 0 + 5 | 1 + 6 | 0 + 7 | 0 + 8 | 0 + 9 | 0 + 10 | 0 + 11 | 1 + 12 | 1 + 13 | 0 + 14 | 1 + 15 | 1 + 16 | 0 + 17 | 1 + 18 | 0 + 19 | 0 + 20 | 1 +(20 rows) + +select a, gp_segment_id from table_distr_hashed_ao_row order by a; + a | gp_segment_id +----+--------------- + 1 | 1 + 2 | 0 + 3 | 0 + 4 | 0 + 5 | 1 + 6 | 0 + 7 | 0 + 8 | 0 + 9 | 0 + 10 | 0 + 11 | 1 + 12 | 1 + 13 | 0 + 14 | 1 + 15 | 1 + 16 | 0 + 17 | 1 + 18 | 0 + 19 | 0 + 20 | 1 +(20 rows) + +select a, gp_segment_id from table_distr_hashed_ao_col order by a; + a | gp_segment_id +----+--------------- + 1 | 1 + 2 | 0 + 3 | 0 + 4 | 0 + 5 | 1 + 6 | 0 + 7 | 0 + 8 | 0 + 9 | 0 + 10 | 0 + 11 | 1 + 12 | 1 + 13 | 0 + 14 | 1 + 15 | 1 + 16 | 0 + 17 | 1 + 18 | 0 + 19 | 0 + 20 | 1 +(20 rows) + +select a, (gp_segment_id < 2) as correct_segment_id from table_distr_random order by a; + a | correct_segment_id +----+-------------------- + 1 | t + 2 | t + 3 | t + 4 | t + 5 | t + 6 | t + 7 | t + 8 | t + 9 | t + 10 | t + 11 | t + 12 | t + 13 | t + 14 | t + 15 | t + 16 | t + 17 | t + 18 | t + 19 | t + 20 | t +(20 rows) + +select a, (gp_segment_id < 2) as correct_segment_id from table_distr_random_ao_row order by a; + a | correct_segment_id +----+-------------------- + 1 | t + 2 | t + 3 | t + 4 | t + 5 | t + 6 | t + 7 | t + 8 | t + 9 | t + 10 | t + 11 | t + 12 | t + 13 | t + 14 | t + 15 | t + 16 | t + 17 | t + 18 | t + 19 | t + 20 | t +(20 rows) + +select a, (gp_segment_id < 2) as correct_segment_id from table_distr_random_ao_col order by a; + a | correct_segment_id +----+-------------------- + 1 | t + 2 | t + 3 | t + 4 | t + 5 | t + 6 | t + 7 | t + 8 | t + 9 | t + 10 | t + 11 | t + 12 | t + 13 | t + 14 | t + 15 | t + 16 | t + 17 | t + 18 | t + 19 | t + 20 | t +(20 rows) + +select a from table_distr_replicated order by a; + a +---- + 1 + 2 + 3 + 4 + 5 + 6 + 7 + 8 + 9 + 10 + 11 + 12 + 13 + 14 + 15 + 16 + 17 + 18 + 19 + 20 +(20 rows) + +select a from table_distr_replicated_ao_row order by a; + a +---- + 1 + 2 + 3 + 4 + 5 + 6 + 7 + 8 + 9 + 10 + 11 + 12 + 13 + 14 + 15 + 16 + 17 + 18 + 19 + 20 +(20 rows) + +select a from table_distr_replicated_ao_col order by a; + a +---- + 1 + 2 + 3 + 4 + 5 + 6 + 7 + 8 + 9 + 10 + 11 + 12 + 13 + 14 + 15 + 16 + 17 + 18 + 19 + 20 +(20 rows) + +select *, gp_segment_id from part_range_table_distr_hashed order by a, b; + a | b | gp_segment_id +----+------------+--------------- + 1 | 05-02-2020 | 1 + 1 | 01-02-2023 | 1 + 1 | 05-02-2023 | 1 + 2 | 05-02-2020 | 0 + 2 | 01-02-2023 | 0 + 2 | 05-02-2023 | 0 + 3 | 05-02-2020 | 0 + 3 | 01-02-2023 | 0 + 3 | 05-02-2023 | 0 + 4 | 05-02-2020 | 0 + 4 | 01-02-2023 | 0 + 4 | 05-02-2023 | 0 + 5 | 05-02-2020 | 1 + 5 | 01-02-2023 | 1 + 5 | 05-02-2023 | 1 + 6 | 05-02-2020 | 0 + 6 | 01-02-2023 | 0 + 6 | 05-02-2023 | 0 + 7 | 05-02-2020 | 0 + 7 | 01-02-2023 | 0 + 7 | 05-02-2023 | 0 + 8 | 05-02-2020 | 0 + 8 | 01-02-2023 | 0 + 8 | 05-02-2023 | 0 + 9 | 05-02-2020 | 0 + 9 | 01-02-2023 | 0 + 9 | 05-02-2023 | 0 + 10 | 05-02-2020 | 0 + 10 | 01-02-2023 | 0 + 10 | 05-02-2023 | 0 + 11 | 05-02-2020 | 1 + 11 | 01-02-2023 | 1 + 11 | 05-02-2023 | 1 + 12 | 05-02-2020 | 1 + 12 | 01-02-2023 | 1 + 12 | 05-02-2023 | 1 + 13 | 05-02-2020 | 0 + 13 | 01-02-2023 | 0 + 13 | 05-02-2023 | 0 + 14 | 05-02-2020 | 1 + 14 | 01-02-2023 | 1 + 14 | 05-02-2023 | 1 + 15 | 05-02-2020 | 1 + 15 | 01-02-2023 | 1 + 15 | 05-02-2023 | 1 + 16 | 05-02-2020 | 0 + 16 | 01-02-2023 | 0 + 16 | 05-02-2023 | 0 + 17 | 05-02-2020 | 1 + 17 | 01-02-2023 | 1 + 17 | 05-02-2023 | 1 + 18 | 05-02-2020 | 0 + 18 | 01-02-2023 | 0 + 18 | 05-02-2023 | 0 + 19 | 05-02-2020 | 0 + 19 | 01-02-2023 | 0 + 19 | 05-02-2023 | 0 + 20 | 05-02-2020 | 1 + 20 | 01-02-2023 | 1 + 20 | 05-02-2023 | 1 +(60 rows) + +select *, (gp_segment_id < 2) as correct_segment_id from part_range_table_distr_random order by a, b; + a | b | correct_segment_id +----+------------+-------------------- + 1 | 05-02-2020 | t + 1 | 01-02-2023 | t + 1 | 05-02-2023 | t + 2 | 05-02-2020 | t + 2 | 01-02-2023 | t + 2 | 05-02-2023 | t + 3 | 05-02-2020 | t + 3 | 01-02-2023 | t + 3 | 05-02-2023 | t + 4 | 05-02-2020 | t + 4 | 01-02-2023 | t + 4 | 05-02-2023 | t + 5 | 05-02-2020 | t + 5 | 01-02-2023 | t + 5 | 05-02-2023 | t + 6 | 05-02-2020 | t + 6 | 01-02-2023 | t + 6 | 05-02-2023 | t + 7 | 05-02-2020 | t + 7 | 01-02-2023 | t + 7 | 05-02-2023 | t + 8 | 05-02-2020 | t + 8 | 01-02-2023 | t + 8 | 05-02-2023 | t + 9 | 05-02-2020 | t + 9 | 01-02-2023 | t + 9 | 05-02-2023 | t + 10 | 05-02-2020 | t + 10 | 01-02-2023 | t + 10 | 05-02-2023 | t + 11 | 05-02-2020 | t + 11 | 01-02-2023 | t + 11 | 05-02-2023 | t + 12 | 05-02-2020 | t + 12 | 01-02-2023 | t + 12 | 05-02-2023 | t + 13 | 05-02-2020 | t + 13 | 01-02-2023 | t + 13 | 05-02-2023 | t + 14 | 05-02-2020 | t + 14 | 01-02-2023 | t + 14 | 05-02-2023 | t + 15 | 05-02-2020 | t + 15 | 01-02-2023 | t + 15 | 05-02-2023 | t + 16 | 05-02-2020 | t + 16 | 01-02-2023 | t + 16 | 05-02-2023 | t + 17 | 05-02-2020 | t + 17 | 01-02-2023 | t + 17 | 05-02-2023 | t + 18 | 05-02-2020 | t + 18 | 01-02-2023 | t + 18 | 05-02-2023 | t + 19 | 05-02-2020 | t + 19 | 01-02-2023 | t + 19 | 05-02-2023 | t + 20 | 05-02-2020 | t + 20 | 01-02-2023 | t + 20 | 05-02-2023 | t +(60 rows) + +select *, gp_segment_id from part_list_table_distr_hashed order by a, b; + a | b | gp_segment_id +----+-------+--------------- + 1 | test1 | 1 + 1 | test2 | 1 + 1 | test3 | 1 + 2 | test1 | 0 + 2 | test2 | 0 + 2 | test3 | 0 + 3 | test1 | 0 + 3 | test2 | 0 + 3 | test3 | 0 + 4 | test1 | 0 + 4 | test2 | 0 + 4 | test3 | 0 + 5 | test1 | 1 + 5 | test2 | 1 + 5 | test3 | 1 + 6 | test1 | 0 + 6 | test2 | 0 + 6 | test3 | 0 + 7 | test1 | 0 + 7 | test2 | 0 + 7 | test3 | 0 + 8 | test1 | 0 + 8 | test2 | 0 + 8 | test3 | 0 + 9 | test1 | 0 + 9 | test2 | 0 + 9 | test3 | 0 + 10 | test1 | 0 + 10 | test2 | 0 + 10 | test3 | 0 + 11 | test1 | 1 + 11 | test2 | 1 + 11 | test3 | 1 + 12 | test1 | 1 + 12 | test2 | 1 + 12 | test3 | 1 + 13 | test1 | 0 + 13 | test2 | 0 + 13 | test3 | 0 + 14 | test1 | 1 + 14 | test2 | 1 + 14 | test3 | 1 + 15 | test1 | 1 + 15 | test2 | 1 + 15 | test3 | 1 + 16 | test1 | 0 + 16 | test2 | 0 + 16 | test3 | 0 + 17 | test1 | 1 + 17 | test2 | 1 + 17 | test3 | 1 + 18 | test1 | 0 + 18 | test2 | 0 + 18 | test3 | 0 + 19 | test1 | 0 + 19 | test2 | 0 + 19 | test3 | 0 + 20 | test1 | 1 + 20 | test2 | 1 + 20 | test3 | 1 +(60 rows) + +select *, (gp_segment_id < 2) as correct_segment_id from part_list_table_distr_random order by a, b; + a | b | correct_segment_id +----+-------+-------------------- + 1 | test1 | t + 1 | test2 | t + 1 | test3 | t + 2 | test1 | t + 2 | test2 | t + 2 | test3 | t + 3 | test1 | t + 3 | test2 | t + 3 | test3 | t + 4 | test1 | t + 4 | test2 | t + 4 | test3 | t + 5 | test1 | t + 5 | test2 | t + 5 | test3 | t + 6 | test1 | t + 6 | test2 | t + 6 | test3 | t + 7 | test1 | t + 7 | test2 | t + 7 | test3 | t + 8 | test1 | t + 8 | test2 | t + 8 | test3 | t + 9 | test1 | t + 9 | test2 | t + 9 | test3 | t + 10 | test1 | t + 10 | test2 | t + 10 | test3 | t + 11 | test1 | t + 11 | test2 | t + 11 | test3 | t + 12 | test1 | t + 12 | test2 | t + 12 | test3 | t + 13 | test1 | t + 13 | test2 | t + 13 | test3 | t + 14 | test1 | t + 14 | test2 | t + 14 | test3 | t + 15 | test1 | t + 15 | test2 | t + 15 | test3 | t + 16 | test1 | t + 16 | test2 | t + 16 | test3 | t + 17 | test1 | t + 17 | test2 | t + 17 | test3 | t + 18 | test1 | t + 18 | test2 | t + 18 | test3 | t + 19 | test1 | t + 19 | test2 | t + 19 | test3 | t + 20 | test1 | t + 20 | test2 | t + 20 | test3 | t +(60 rows) + +select *, gp_segment_id from multi_part_table_distr_hashed order by a, b, c; + a | b | c | gp_segment_id +----+------------+-------+--------------- + 1 | 01-05-2023 | test1 | 1 + 1 | 02-05-2023 | test2 | 1 + 1 | 03-05-2023 | test1 | 1 + 2 | 01-05-2023 | test1 | 0 + 2 | 02-05-2023 | test2 | 0 + 2 | 03-05-2023 | test1 | 0 + 3 | 01-05-2023 | test1 | 0 + 3 | 02-05-2023 | test2 | 0 + 3 | 03-05-2023 | test1 | 0 + 4 | 01-05-2023 | test1 | 0 + 4 | 02-05-2023 | test2 | 0 + 4 | 03-05-2023 | test1 | 0 + 5 | 01-05-2023 | test1 | 1 + 5 | 02-05-2023 | test2 | 1 + 5 | 03-05-2023 | test1 | 1 + 6 | 01-05-2023 | test1 | 0 + 6 | 02-05-2023 | test2 | 0 + 6 | 03-05-2023 | test1 | 0 + 7 | 01-05-2023 | test1 | 0 + 7 | 02-05-2023 | test2 | 0 + 7 | 03-05-2023 | test1 | 0 + 8 | 01-05-2023 | test1 | 0 + 8 | 02-05-2023 | test2 | 0 + 8 | 03-05-2023 | test1 | 0 + 9 | 01-05-2023 | test1 | 0 + 9 | 02-05-2023 | test2 | 0 + 9 | 03-05-2023 | test1 | 0 + 10 | 01-05-2023 | test1 | 0 + 10 | 02-05-2023 | test2 | 0 + 10 | 03-05-2023 | test1 | 0 + 11 | 01-05-2023 | test1 | 1 + 11 | 02-05-2023 | test2 | 1 + 11 | 03-05-2023 | test1 | 1 + 12 | 01-05-2023 | test1 | 1 + 12 | 02-05-2023 | test2 | 1 + 12 | 03-05-2023 | test1 | 1 + 13 | 01-05-2023 | test1 | 0 + 13 | 02-05-2023 | test2 | 0 + 13 | 03-05-2023 | test1 | 0 + 14 | 01-05-2023 | test1 | 1 + 14 | 02-05-2023 | test2 | 1 + 14 | 03-05-2023 | test1 | 1 + 15 | 01-05-2023 | test1 | 1 + 15 | 02-05-2023 | test2 | 1 + 15 | 03-05-2023 | test1 | 1 + 16 | 01-05-2023 | test1 | 0 + 16 | 02-05-2023 | test2 | 0 + 16 | 03-05-2023 | test1 | 0 + 17 | 01-05-2023 | test1 | 1 + 17 | 02-05-2023 | test2 | 1 + 17 | 03-05-2023 | test1 | 1 + 18 | 01-05-2023 | test1 | 0 + 18 | 02-05-2023 | test2 | 0 + 18 | 03-05-2023 | test1 | 0 + 19 | 01-05-2023 | test1 | 0 + 19 | 02-05-2023 | test2 | 0 + 19 | 03-05-2023 | test1 | 0 + 20 | 01-05-2023 | test1 | 1 + 20 | 02-05-2023 | test2 | 1 + 20 | 03-05-2023 | test1 | 1 +(60 rows) + +-- Check that new data is added only to reduced set of segments +begin; +select gp_expand_lock_catalog(); + gp_expand_lock_catalog +------------------------ + +(1 row) + +select gp_toolkit.gp_reset_rebalance_numsegments(); + gp_reset_rebalance_numsegments +-------------------------------- + +(1 row) + +end; +insert into table_distr_hashed select generate_series(21, 40); +insert into table_distr_hashed_ao_row select generate_series(21, 40); +insert into table_distr_hashed_ao_col select generate_series(21, 40); +insert into table_distr_random select generate_series(21, 40); +insert into table_distr_random_ao_row select generate_series(21, 40); +insert into table_distr_random_ao_col select generate_series(21, 40); +insert into table_distr_replicated select generate_series(21, 40); +insert into table_distr_replicated_ao_row select generate_series(21, 40); +insert into table_distr_replicated_ao_col select generate_series(21, 40); +insert into part_range_table_distr_hashed select i, '2023-01-02' from generate_series(21, 40)i; +insert into part_range_table_distr_hashed select i, '2023-05-02' from generate_series(21, 40)i; +insert into part_range_table_distr_hashed select i, '2020-05-02' from generate_series(21, 40)i; +insert into part_range_table_distr_random select i, '2023-01-02' from generate_series(21, 40)i; +insert into part_range_table_distr_random select i, '2023-05-02' from generate_series(21, 40)i; +insert into part_range_table_distr_random select i, '2020-05-02' from generate_series(21, 40)i; +insert into part_list_table_distr_hashed select i, 'test1' from generate_series(21, 40)i; +insert into part_list_table_distr_hashed select i, 'test2' from generate_series(21, 40)i; +insert into part_list_table_distr_hashed select i, 'test3' from generate_series(21, 40)i; +insert into part_list_table_distr_random select i, 'test1' from generate_series(21, 40)i; +insert into part_list_table_distr_random select i, 'test2' from generate_series(21, 40)i; +insert into part_list_table_distr_random select i, 'test3' from generate_series(21, 40)i; +insert into multi_part_table_distr_hashed select i, '2023-01-05', 'test1' from generate_series(21, 40)i; +insert into multi_part_table_distr_hashed select i, '2023-02-05', 'test2' from generate_series(21, 40)i; +insert into multi_part_table_distr_hashed select i, '2023-03-05', 'test1' from generate_series(21, 40)i; +select a, gp_segment_id from table_distr_hashed order by a; + a | gp_segment_id +----+--------------- + 1 | 1 + 2 | 0 + 3 | 0 + 4 | 0 + 5 | 1 + 6 | 0 + 7 | 0 + 8 | 0 + 9 | 0 + 10 | 0 + 11 | 1 + 12 | 1 + 13 | 0 + 14 | 1 + 15 | 1 + 16 | 0 + 17 | 1 + 18 | 0 + 19 | 0 + 20 | 1 + 21 | 0 + 22 | 0 + 23 | 1 + 24 | 0 + 25 | 1 + 26 | 1 + 27 | 0 + 28 | 0 + 29 | 0 + 30 | 1 + 31 | 1 + 32 | 0 + 33 | 0 + 34 | 0 + 35 | 1 + 36 | 1 + 37 | 0 + 38 | 1 + 39 | 0 + 40 | 1 +(40 rows) + +select a, gp_segment_id from table_distr_hashed_ao_row order by a; + a | gp_segment_id +----+--------------- + 1 | 1 + 2 | 0 + 3 | 0 + 4 | 0 + 5 | 1 + 6 | 0 + 7 | 0 + 8 | 0 + 9 | 0 + 10 | 0 + 11 | 1 + 12 | 1 + 13 | 0 + 14 | 1 + 15 | 1 + 16 | 0 + 17 | 1 + 18 | 0 + 19 | 0 + 20 | 1 + 21 | 0 + 22 | 0 + 23 | 1 + 24 | 0 + 25 | 1 + 26 | 1 + 27 | 0 + 28 | 0 + 29 | 0 + 30 | 1 + 31 | 1 + 32 | 0 + 33 | 0 + 34 | 0 + 35 | 1 + 36 | 1 + 37 | 0 + 38 | 1 + 39 | 0 + 40 | 1 +(40 rows) + +select a, gp_segment_id from table_distr_hashed_ao_col order by a; + a | gp_segment_id +----+--------------- + 1 | 1 + 2 | 0 + 3 | 0 + 4 | 0 + 5 | 1 + 6 | 0 + 7 | 0 + 8 | 0 + 9 | 0 + 10 | 0 + 11 | 1 + 12 | 1 + 13 | 0 + 14 | 1 + 15 | 1 + 16 | 0 + 17 | 1 + 18 | 0 + 19 | 0 + 20 | 1 + 21 | 0 + 22 | 0 + 23 | 1 + 24 | 0 + 25 | 1 + 26 | 1 + 27 | 0 + 28 | 0 + 29 | 0 + 30 | 1 + 31 | 1 + 32 | 0 + 33 | 0 + 34 | 0 + 35 | 1 + 36 | 1 + 37 | 0 + 38 | 1 + 39 | 0 + 40 | 1 +(40 rows) + +select a, (gp_segment_id < 2) as correct_segment_id from table_distr_random order by a; + a | correct_segment_id +----+-------------------- + 1 | t + 2 | t + 3 | t + 4 | t + 5 | t + 6 | t + 7 | t + 8 | t + 9 | t + 10 | t + 11 | t + 12 | t + 13 | t + 14 | t + 15 | t + 16 | t + 17 | t + 18 | t + 19 | t + 20 | t + 21 | t + 22 | t + 23 | t + 24 | t + 25 | t + 26 | t + 27 | t + 28 | t + 29 | t + 30 | t + 31 | t + 32 | t + 33 | t + 34 | t + 35 | t + 36 | t + 37 | t + 38 | t + 39 | t + 40 | t +(40 rows) + +select a, (gp_segment_id < 2) as correct_segment_id from table_distr_random_ao_row order by a; + a | correct_segment_id +----+-------------------- + 1 | t + 2 | t + 3 | t + 4 | t + 5 | t + 6 | t + 7 | t + 8 | t + 9 | t + 10 | t + 11 | t + 12 | t + 13 | t + 14 | t + 15 | t + 16 | t + 17 | t + 18 | t + 19 | t + 20 | t + 21 | t + 22 | t + 23 | t + 24 | t + 25 | t + 26 | t + 27 | t + 28 | t + 29 | t + 30 | t + 31 | t + 32 | t + 33 | t + 34 | t + 35 | t + 36 | t + 37 | t + 38 | t + 39 | t + 40 | t +(40 rows) + +select a, (gp_segment_id < 2) as correct_segment_id from table_distr_random_ao_col order by a; + a | correct_segment_id +----+-------------------- + 1 | t + 2 | t + 3 | t + 4 | t + 5 | t + 6 | t + 7 | t + 8 | t + 9 | t + 10 | t + 11 | t + 12 | t + 13 | t + 14 | t + 15 | t + 16 | t + 17 | t + 18 | t + 19 | t + 20 | t + 21 | t + 22 | t + 23 | t + 24 | t + 25 | t + 26 | t + 27 | t + 28 | t + 29 | t + 30 | t + 31 | t + 32 | t + 33 | t + 34 | t + 35 | t + 36 | t + 37 | t + 38 | t + 39 | t + 40 | t +(40 rows) + +select a from table_distr_replicated order by a; + a +---- + 1 + 2 + 3 + 4 + 5 + 6 + 7 + 8 + 9 + 10 + 11 + 12 + 13 + 14 + 15 + 16 + 17 + 18 + 19 + 20 + 21 + 22 + 23 + 24 + 25 + 26 + 27 + 28 + 29 + 30 + 31 + 32 + 33 + 34 + 35 + 36 + 37 + 38 + 39 + 40 +(40 rows) + +select a from table_distr_replicated_ao_row order by a; + a +---- + 1 + 2 + 3 + 4 + 5 + 6 + 7 + 8 + 9 + 10 + 11 + 12 + 13 + 14 + 15 + 16 + 17 + 18 + 19 + 20 + 21 + 22 + 23 + 24 + 25 + 26 + 27 + 28 + 29 + 30 + 31 + 32 + 33 + 34 + 35 + 36 + 37 + 38 + 39 + 40 +(40 rows) + +select a from table_distr_replicated_ao_col order by a; + a +---- + 1 + 2 + 3 + 4 + 5 + 6 + 7 + 8 + 9 + 10 + 11 + 12 + 13 + 14 + 15 + 16 + 17 + 18 + 19 + 20 + 21 + 22 + 23 + 24 + 25 + 26 + 27 + 28 + 29 + 30 + 31 + 32 + 33 + 34 + 35 + 36 + 37 + 38 + 39 + 40 +(40 rows) + +select *, gp_segment_id from part_range_table_distr_hashed order by a, b; + a | b | gp_segment_id +----+------------+--------------- + 1 | 05-02-2020 | 1 + 1 | 01-02-2023 | 1 + 1 | 05-02-2023 | 1 + 2 | 05-02-2020 | 0 + 2 | 01-02-2023 | 0 + 2 | 05-02-2023 | 0 + 3 | 05-02-2020 | 0 + 3 | 01-02-2023 | 0 + 3 | 05-02-2023 | 0 + 4 | 05-02-2020 | 0 + 4 | 01-02-2023 | 0 + 4 | 05-02-2023 | 0 + 5 | 05-02-2020 | 1 + 5 | 01-02-2023 | 1 + 5 | 05-02-2023 | 1 + 6 | 05-02-2020 | 0 + 6 | 01-02-2023 | 0 + 6 | 05-02-2023 | 0 + 7 | 05-02-2020 | 0 + 7 | 01-02-2023 | 0 + 7 | 05-02-2023 | 0 + 8 | 05-02-2020 | 0 + 8 | 01-02-2023 | 0 + 8 | 05-02-2023 | 0 + 9 | 05-02-2020 | 0 + 9 | 01-02-2023 | 0 + 9 | 05-02-2023 | 0 + 10 | 05-02-2020 | 0 + 10 | 01-02-2023 | 0 + 10 | 05-02-2023 | 0 + 11 | 05-02-2020 | 1 + 11 | 01-02-2023 | 1 + 11 | 05-02-2023 | 1 + 12 | 05-02-2020 | 1 + 12 | 01-02-2023 | 1 + 12 | 05-02-2023 | 1 + 13 | 05-02-2020 | 0 + 13 | 01-02-2023 | 0 + 13 | 05-02-2023 | 0 + 14 | 05-02-2020 | 1 + 14 | 01-02-2023 | 1 + 14 | 05-02-2023 | 1 + 15 | 05-02-2020 | 1 + 15 | 01-02-2023 | 1 + 15 | 05-02-2023 | 1 + 16 | 05-02-2020 | 0 + 16 | 01-02-2023 | 0 + 16 | 05-02-2023 | 0 + 17 | 05-02-2020 | 1 + 17 | 01-02-2023 | 1 + 17 | 05-02-2023 | 1 + 18 | 05-02-2020 | 0 + 18 | 01-02-2023 | 0 + 18 | 05-02-2023 | 0 + 19 | 05-02-2020 | 0 + 19 | 01-02-2023 | 0 + 19 | 05-02-2023 | 0 + 20 | 05-02-2020 | 1 + 20 | 01-02-2023 | 1 + 20 | 05-02-2023 | 1 + 21 | 05-02-2020 | 0 + 21 | 01-02-2023 | 0 + 21 | 05-02-2023 | 0 + 22 | 05-02-2020 | 0 + 22 | 01-02-2023 | 0 + 22 | 05-02-2023 | 0 + 23 | 05-02-2020 | 1 + 23 | 01-02-2023 | 1 + 23 | 05-02-2023 | 1 + 24 | 05-02-2020 | 0 + 24 | 01-02-2023 | 0 + 24 | 05-02-2023 | 0 + 25 | 05-02-2020 | 1 + 25 | 01-02-2023 | 1 + 25 | 05-02-2023 | 1 + 26 | 05-02-2020 | 1 + 26 | 01-02-2023 | 1 + 26 | 05-02-2023 | 1 + 27 | 05-02-2020 | 0 + 27 | 01-02-2023 | 0 + 27 | 05-02-2023 | 0 + 28 | 05-02-2020 | 0 + 28 | 01-02-2023 | 0 + 28 | 05-02-2023 | 0 + 29 | 05-02-2020 | 0 + 29 | 01-02-2023 | 0 + 29 | 05-02-2023 | 0 + 30 | 05-02-2020 | 1 + 30 | 01-02-2023 | 1 + 30 | 05-02-2023 | 1 + 31 | 05-02-2020 | 1 + 31 | 01-02-2023 | 1 + 31 | 05-02-2023 | 1 + 32 | 05-02-2020 | 0 + 32 | 01-02-2023 | 0 + 32 | 05-02-2023 | 0 + 33 | 05-02-2020 | 0 + 33 | 01-02-2023 | 0 + 33 | 05-02-2023 | 0 + 34 | 05-02-2020 | 0 + 34 | 01-02-2023 | 0 + 34 | 05-02-2023 | 0 + 35 | 05-02-2020 | 1 + 35 | 01-02-2023 | 1 + 35 | 05-02-2023 | 1 + 36 | 05-02-2020 | 1 + 36 | 01-02-2023 | 1 + 36 | 05-02-2023 | 1 + 37 | 05-02-2020 | 0 + 37 | 01-02-2023 | 0 + 37 | 05-02-2023 | 0 + 38 | 05-02-2020 | 1 + 38 | 01-02-2023 | 1 + 38 | 05-02-2023 | 1 + 39 | 05-02-2020 | 0 + 39 | 01-02-2023 | 0 + 39 | 05-02-2023 | 0 + 40 | 05-02-2020 | 1 + 40 | 01-02-2023 | 1 + 40 | 05-02-2023 | 1 +(120 rows) + +select *, (gp_segment_id < 2) as correct_segment_id from part_range_table_distr_random order by a, b; + a | b | correct_segment_id +----+------------+-------------------- + 1 | 05-02-2020 | t + 1 | 01-02-2023 | t + 1 | 05-02-2023 | t + 2 | 05-02-2020 | t + 2 | 01-02-2023 | t + 2 | 05-02-2023 | t + 3 | 05-02-2020 | t + 3 | 01-02-2023 | t + 3 | 05-02-2023 | t + 4 | 05-02-2020 | t + 4 | 01-02-2023 | t + 4 | 05-02-2023 | t + 5 | 05-02-2020 | t + 5 | 01-02-2023 | t + 5 | 05-02-2023 | t + 6 | 05-02-2020 | t + 6 | 01-02-2023 | t + 6 | 05-02-2023 | t + 7 | 05-02-2020 | t + 7 | 01-02-2023 | t + 7 | 05-02-2023 | t + 8 | 05-02-2020 | t + 8 | 01-02-2023 | t + 8 | 05-02-2023 | t + 9 | 05-02-2020 | t + 9 | 01-02-2023 | t + 9 | 05-02-2023 | t + 10 | 05-02-2020 | t + 10 | 01-02-2023 | t + 10 | 05-02-2023 | t + 11 | 05-02-2020 | t + 11 | 01-02-2023 | t + 11 | 05-02-2023 | t + 12 | 05-02-2020 | t + 12 | 01-02-2023 | t + 12 | 05-02-2023 | t + 13 | 05-02-2020 | t + 13 | 01-02-2023 | t + 13 | 05-02-2023 | t + 14 | 05-02-2020 | t + 14 | 01-02-2023 | t + 14 | 05-02-2023 | t + 15 | 05-02-2020 | t + 15 | 01-02-2023 | t + 15 | 05-02-2023 | t + 16 | 05-02-2020 | t + 16 | 01-02-2023 | t + 16 | 05-02-2023 | t + 17 | 05-02-2020 | t + 17 | 01-02-2023 | t + 17 | 05-02-2023 | t + 18 | 05-02-2020 | t + 18 | 01-02-2023 | t + 18 | 05-02-2023 | t + 19 | 05-02-2020 | t + 19 | 01-02-2023 | t + 19 | 05-02-2023 | t + 20 | 05-02-2020 | t + 20 | 01-02-2023 | t + 20 | 05-02-2023 | t + 21 | 05-02-2020 | t + 21 | 01-02-2023 | t + 21 | 05-02-2023 | t + 22 | 05-02-2020 | t + 22 | 01-02-2023 | t + 22 | 05-02-2023 | t + 23 | 05-02-2020 | t + 23 | 01-02-2023 | t + 23 | 05-02-2023 | t + 24 | 05-02-2020 | t + 24 | 01-02-2023 | t + 24 | 05-02-2023 | t + 25 | 05-02-2020 | t + 25 | 01-02-2023 | t + 25 | 05-02-2023 | t + 26 | 05-02-2020 | t + 26 | 01-02-2023 | t + 26 | 05-02-2023 | t + 27 | 05-02-2020 | t + 27 | 01-02-2023 | t + 27 | 05-02-2023 | t + 28 | 05-02-2020 | t + 28 | 01-02-2023 | t + 28 | 05-02-2023 | t + 29 | 05-02-2020 | t + 29 | 01-02-2023 | t + 29 | 05-02-2023 | t + 30 | 05-02-2020 | t + 30 | 01-02-2023 | t + 30 | 05-02-2023 | t + 31 | 05-02-2020 | t + 31 | 01-02-2023 | t + 31 | 05-02-2023 | t + 32 | 05-02-2020 | t + 32 | 01-02-2023 | t + 32 | 05-02-2023 | t + 33 | 05-02-2020 | t + 33 | 01-02-2023 | t + 33 | 05-02-2023 | t + 34 | 05-02-2020 | t + 34 | 01-02-2023 | t + 34 | 05-02-2023 | t + 35 | 05-02-2020 | t + 35 | 01-02-2023 | t + 35 | 05-02-2023 | t + 36 | 05-02-2020 | t + 36 | 01-02-2023 | t + 36 | 05-02-2023 | t + 37 | 05-02-2020 | t + 37 | 01-02-2023 | t + 37 | 05-02-2023 | t + 38 | 05-02-2020 | t + 38 | 01-02-2023 | t + 38 | 05-02-2023 | t + 39 | 05-02-2020 | t + 39 | 01-02-2023 | t + 39 | 05-02-2023 | t + 40 | 05-02-2020 | t + 40 | 01-02-2023 | t + 40 | 05-02-2023 | t +(120 rows) + +select *, gp_segment_id from part_list_table_distr_hashed order by a, b; + a | b | gp_segment_id +----+-------+--------------- + 1 | test1 | 1 + 1 | test2 | 1 + 1 | test3 | 1 + 2 | test1 | 0 + 2 | test2 | 0 + 2 | test3 | 0 + 3 | test1 | 0 + 3 | test2 | 0 + 3 | test3 | 0 + 4 | test1 | 0 + 4 | test2 | 0 + 4 | test3 | 0 + 5 | test1 | 1 + 5 | test2 | 1 + 5 | test3 | 1 + 6 | test1 | 0 + 6 | test2 | 0 + 6 | test3 | 0 + 7 | test1 | 0 + 7 | test2 | 0 + 7 | test3 | 0 + 8 | test1 | 0 + 8 | test2 | 0 + 8 | test3 | 0 + 9 | test1 | 0 + 9 | test2 | 0 + 9 | test3 | 0 + 10 | test1 | 0 + 10 | test2 | 0 + 10 | test3 | 0 + 11 | test1 | 1 + 11 | test2 | 1 + 11 | test3 | 1 + 12 | test1 | 1 + 12 | test2 | 1 + 12 | test3 | 1 + 13 | test1 | 0 + 13 | test2 | 0 + 13 | test3 | 0 + 14 | test1 | 1 + 14 | test2 | 1 + 14 | test3 | 1 + 15 | test1 | 1 + 15 | test2 | 1 + 15 | test3 | 1 + 16 | test1 | 0 + 16 | test2 | 0 + 16 | test3 | 0 + 17 | test1 | 1 + 17 | test2 | 1 + 17 | test3 | 1 + 18 | test1 | 0 + 18 | test2 | 0 + 18 | test3 | 0 + 19 | test1 | 0 + 19 | test2 | 0 + 19 | test3 | 0 + 20 | test1 | 1 + 20 | test2 | 1 + 20 | test3 | 1 + 21 | test1 | 0 + 21 | test2 | 0 + 21 | test3 | 0 + 22 | test1 | 0 + 22 | test2 | 0 + 22 | test3 | 0 + 23 | test1 | 1 + 23 | test2 | 1 + 23 | test3 | 1 + 24 | test1 | 0 + 24 | test2 | 0 + 24 | test3 | 0 + 25 | test1 | 1 + 25 | test2 | 1 + 25 | test3 | 1 + 26 | test1 | 1 + 26 | test2 | 1 + 26 | test3 | 1 + 27 | test1 | 0 + 27 | test2 | 0 + 27 | test3 | 0 + 28 | test1 | 0 + 28 | test2 | 0 + 28 | test3 | 0 + 29 | test1 | 0 + 29 | test2 | 0 + 29 | test3 | 0 + 30 | test1 | 1 + 30 | test2 | 1 + 30 | test3 | 1 + 31 | test1 | 1 + 31 | test2 | 1 + 31 | test3 | 1 + 32 | test1 | 0 + 32 | test2 | 0 + 32 | test3 | 0 + 33 | test1 | 0 + 33 | test2 | 0 + 33 | test3 | 0 + 34 | test1 | 0 + 34 | test2 | 0 + 34 | test3 | 0 + 35 | test1 | 1 + 35 | test2 | 1 + 35 | test3 | 1 + 36 | test1 | 1 + 36 | test2 | 1 + 36 | test3 | 1 + 37 | test1 | 0 + 37 | test2 | 0 + 37 | test3 | 0 + 38 | test1 | 1 + 38 | test2 | 1 + 38 | test3 | 1 + 39 | test1 | 0 + 39 | test2 | 0 + 39 | test3 | 0 + 40 | test1 | 1 + 40 | test2 | 1 + 40 | test3 | 1 +(120 rows) + +select *, (gp_segment_id < 2) as correct_segment_id from part_list_table_distr_random order by a, b; + a | b | correct_segment_id +----+-------+-------------------- + 1 | test1 | t + 1 | test2 | t + 1 | test3 | t + 2 | test1 | t + 2 | test2 | t + 2 | test3 | t + 3 | test1 | t + 3 | test2 | t + 3 | test3 | t + 4 | test1 | t + 4 | test2 | t + 4 | test3 | t + 5 | test1 | t + 5 | test2 | t + 5 | test3 | t + 6 | test1 | t + 6 | test2 | t + 6 | test3 | t + 7 | test1 | t + 7 | test2 | t + 7 | test3 | t + 8 | test1 | t + 8 | test2 | t + 8 | test3 | t + 9 | test1 | t + 9 | test2 | t + 9 | test3 | t + 10 | test1 | t + 10 | test2 | t + 10 | test3 | t + 11 | test1 | t + 11 | test2 | t + 11 | test3 | t + 12 | test1 | t + 12 | test2 | t + 12 | test3 | t + 13 | test1 | t + 13 | test2 | t + 13 | test3 | t + 14 | test1 | t + 14 | test2 | t + 14 | test3 | t + 15 | test1 | t + 15 | test2 | t + 15 | test3 | t + 16 | test1 | t + 16 | test2 | t + 16 | test3 | t + 17 | test1 | t + 17 | test2 | t + 17 | test3 | t + 18 | test1 | t + 18 | test2 | t + 18 | test3 | t + 19 | test1 | t + 19 | test2 | t + 19 | test3 | t + 20 | test1 | t + 20 | test2 | t + 20 | test3 | t + 21 | test1 | t + 21 | test2 | t + 21 | test3 | t + 22 | test1 | t + 22 | test2 | t + 22 | test3 | t + 23 | test1 | t + 23 | test2 | t + 23 | test3 | t + 24 | test1 | t + 24 | test2 | t + 24 | test3 | t + 25 | test1 | t + 25 | test2 | t + 25 | test3 | t + 26 | test1 | t + 26 | test2 | t + 26 | test3 | t + 27 | test1 | t + 27 | test2 | t + 27 | test3 | t + 28 | test1 | t + 28 | test2 | t + 28 | test3 | t + 29 | test1 | t + 29 | test2 | t + 29 | test3 | t + 30 | test1 | t + 30 | test2 | t + 30 | test3 | t + 31 | test1 | t + 31 | test2 | t + 31 | test3 | t + 32 | test1 | t + 32 | test2 | t + 32 | test3 | t + 33 | test1 | t + 33 | test2 | t + 33 | test3 | t + 34 | test1 | t + 34 | test2 | t + 34 | test3 | t + 35 | test1 | t + 35 | test2 | t + 35 | test3 | t + 36 | test1 | t + 36 | test2 | t + 36 | test3 | t + 37 | test1 | t + 37 | test2 | t + 37 | test3 | t + 38 | test1 | t + 38 | test2 | t + 38 | test3 | t + 39 | test1 | t + 39 | test2 | t + 39 | test3 | t + 40 | test1 | t + 40 | test2 | t + 40 | test3 | t +(120 rows) + +select *, gp_segment_id from multi_part_table_distr_hashed order by a, b, c; + a | b | c | gp_segment_id +----+------------+-------+--------------- + 1 | 01-05-2023 | test1 | 1 + 1 | 02-05-2023 | test2 | 1 + 1 | 03-05-2023 | test1 | 1 + 2 | 01-05-2023 | test1 | 0 + 2 | 02-05-2023 | test2 | 0 + 2 | 03-05-2023 | test1 | 0 + 3 | 01-05-2023 | test1 | 0 + 3 | 02-05-2023 | test2 | 0 + 3 | 03-05-2023 | test1 | 0 + 4 | 01-05-2023 | test1 | 0 + 4 | 02-05-2023 | test2 | 0 + 4 | 03-05-2023 | test1 | 0 + 5 | 01-05-2023 | test1 | 1 + 5 | 02-05-2023 | test2 | 1 + 5 | 03-05-2023 | test1 | 1 + 6 | 01-05-2023 | test1 | 0 + 6 | 02-05-2023 | test2 | 0 + 6 | 03-05-2023 | test1 | 0 + 7 | 01-05-2023 | test1 | 0 + 7 | 02-05-2023 | test2 | 0 + 7 | 03-05-2023 | test1 | 0 + 8 | 01-05-2023 | test1 | 0 + 8 | 02-05-2023 | test2 | 0 + 8 | 03-05-2023 | test1 | 0 + 9 | 01-05-2023 | test1 | 0 + 9 | 02-05-2023 | test2 | 0 + 9 | 03-05-2023 | test1 | 0 + 10 | 01-05-2023 | test1 | 0 + 10 | 02-05-2023 | test2 | 0 + 10 | 03-05-2023 | test1 | 0 + 11 | 01-05-2023 | test1 | 1 + 11 | 02-05-2023 | test2 | 1 + 11 | 03-05-2023 | test1 | 1 + 12 | 01-05-2023 | test1 | 1 + 12 | 02-05-2023 | test2 | 1 + 12 | 03-05-2023 | test1 | 1 + 13 | 01-05-2023 | test1 | 0 + 13 | 02-05-2023 | test2 | 0 + 13 | 03-05-2023 | test1 | 0 + 14 | 01-05-2023 | test1 | 1 + 14 | 02-05-2023 | test2 | 1 + 14 | 03-05-2023 | test1 | 1 + 15 | 01-05-2023 | test1 | 1 + 15 | 02-05-2023 | test2 | 1 + 15 | 03-05-2023 | test1 | 1 + 16 | 01-05-2023 | test1 | 0 + 16 | 02-05-2023 | test2 | 0 + 16 | 03-05-2023 | test1 | 0 + 17 | 01-05-2023 | test1 | 1 + 17 | 02-05-2023 | test2 | 1 + 17 | 03-05-2023 | test1 | 1 + 18 | 01-05-2023 | test1 | 0 + 18 | 02-05-2023 | test2 | 0 + 18 | 03-05-2023 | test1 | 0 + 19 | 01-05-2023 | test1 | 0 + 19 | 02-05-2023 | test2 | 0 + 19 | 03-05-2023 | test1 | 0 + 20 | 01-05-2023 | test1 | 1 + 20 | 02-05-2023 | test2 | 1 + 20 | 03-05-2023 | test1 | 1 + 21 | 01-05-2023 | test1 | 0 + 21 | 02-05-2023 | test2 | 0 + 21 | 03-05-2023 | test1 | 0 + 22 | 01-05-2023 | test1 | 0 + 22 | 02-05-2023 | test2 | 0 + 22 | 03-05-2023 | test1 | 0 + 23 | 01-05-2023 | test1 | 1 + 23 | 02-05-2023 | test2 | 1 + 23 | 03-05-2023 | test1 | 1 + 24 | 01-05-2023 | test1 | 0 + 24 | 02-05-2023 | test2 | 0 + 24 | 03-05-2023 | test1 | 0 + 25 | 01-05-2023 | test1 | 1 + 25 | 02-05-2023 | test2 | 1 + 25 | 03-05-2023 | test1 | 1 + 26 | 01-05-2023 | test1 | 1 + 26 | 02-05-2023 | test2 | 1 + 26 | 03-05-2023 | test1 | 1 + 27 | 01-05-2023 | test1 | 0 + 27 | 02-05-2023 | test2 | 0 + 27 | 03-05-2023 | test1 | 0 + 28 | 01-05-2023 | test1 | 0 + 28 | 02-05-2023 | test2 | 0 + 28 | 03-05-2023 | test1 | 0 + 29 | 01-05-2023 | test1 | 0 + 29 | 02-05-2023 | test2 | 0 + 29 | 03-05-2023 | test1 | 0 + 30 | 01-05-2023 | test1 | 1 + 30 | 02-05-2023 | test2 | 1 + 30 | 03-05-2023 | test1 | 1 + 31 | 01-05-2023 | test1 | 1 + 31 | 02-05-2023 | test2 | 1 + 31 | 03-05-2023 | test1 | 1 + 32 | 01-05-2023 | test1 | 0 + 32 | 02-05-2023 | test2 | 0 + 32 | 03-05-2023 | test1 | 0 + 33 | 01-05-2023 | test1 | 0 + 33 | 02-05-2023 | test2 | 0 + 33 | 03-05-2023 | test1 | 0 + 34 | 01-05-2023 | test1 | 0 + 34 | 02-05-2023 | test2 | 0 + 34 | 03-05-2023 | test1 | 0 + 35 | 01-05-2023 | test1 | 1 + 35 | 02-05-2023 | test2 | 1 + 35 | 03-05-2023 | test1 | 1 + 36 | 01-05-2023 | test1 | 1 + 36 | 02-05-2023 | test2 | 1 + 36 | 03-05-2023 | test1 | 1 + 37 | 01-05-2023 | test1 | 0 + 37 | 02-05-2023 | test2 | 0 + 37 | 03-05-2023 | test1 | 0 + 38 | 01-05-2023 | test1 | 1 + 38 | 02-05-2023 | test2 | 1 + 38 | 03-05-2023 | test1 | 1 + 39 | 01-05-2023 | test1 | 0 + 39 | 02-05-2023 | test2 | 0 + 39 | 03-05-2023 | test1 | 0 + 40 | 01-05-2023 | test1 | 1 + 40 | 02-05-2023 | test2 | 1 + 40 | 03-05-2023 | test1 | 1 +(120 rows) + +-- And do some cleanup +drop table table_distr_hashed; +drop table table_distr_hashed_ao_row; +drop table table_distr_hashed_ao_col; +drop table table_distr_random; +drop table table_distr_random_ao_row; +drop table table_distr_random_ao_col; +drop table table_distr_replicated; +drop table table_distr_replicated_ao_row; +drop table table_distr_replicated_ao_col; +drop table part_range_table_distr_hashed; +drop table part_range_table_distr_random; +drop table part_list_table_distr_hashed; +drop table part_list_table_distr_random; +drop table multi_part_table_distr_hashed; +-- Check that all newly created tables have data only on segments #0 and #1 +begin; +select gp_expand_lock_catalog(); + gp_expand_lock_catalog +------------------------ + +(1 row) + +select gp_toolkit.gp_set_rebalance_numsegments(2); + gp_set_rebalance_numsegments +------------------------------ + 2 +(1 row) + +end; +create table new_table_distr_hashed(a int) distributed by (a); +insert into new_table_distr_hashed select generate_series(1, 20); +create table new_table_distr_hashed_ao_row(a int) with (appendonly=true, orientation=row) distributed by (a); +insert into new_table_distr_hashed_ao_row select generate_series(1, 20); +create table new_table_distr_hashed_ao_col(a int) with (appendonly=true, orientation=column) distributed by (a); +insert into new_table_distr_hashed_ao_col select generate_series(1, 20); +create table new_table_distr_random(a int) distributed randomly; +insert into new_table_distr_random select generate_series(1, 20); +create table new_table_distr_random_ao_row(a int) with (appendonly=true, orientation=row) distributed randomly; +insert into new_table_distr_random_ao_row select generate_series(1, 20); +create table new_table_distr_random_ao_col(a int) with (appendonly=true, orientation=column) distributed randomly; +insert into new_table_distr_random_ao_col select generate_series(1, 20); +create table new_table_distr_replicated(a int) distributed replicated; +insert into new_table_distr_replicated select generate_series(1, 20); +create table new_table_distr_replicated_ao_row(a int) with (appendonly=true, orientation=row) distributed replicated; +insert into new_table_distr_replicated_ao_row select generate_series(1, 20); +create table new_table_distr_replicated_ao_col(a int) with (appendonly=true, orientation=column) distributed replicated; +insert into new_table_distr_replicated_ao_col select generate_series(1, 20); +create table new_part_range_table_distr_hashed (a int, b date) distributed by (a) +partition by range (b) ( + start (date '2023-01-01') inclusive + end (date '2024-01-01') exclusive + every (interval '1 month'), + default partition other_vals +); +insert into new_part_range_table_distr_hashed select i, '2023-01-02' from generate_series(1, 20)i; +insert into new_part_range_table_distr_hashed select i, '2023-05-02' from generate_series(1, 20)i; +insert into new_part_range_table_distr_hashed select i, '2020-05-02' from generate_series(1, 20)i; +create table new_part_range_table_distr_random (a int, b date) distributed randomly +partition by range (b) ( + start (date '2023-01-01') inclusive + end (date '2024-01-01') exclusive + every (interval '1 month'), + default partition other_vals +); +insert into new_part_range_table_distr_random select i, '2023-01-02' from generate_series(1, 20)i; +insert into new_part_range_table_distr_random select i, '2023-05-02' from generate_series(1, 20)i; +insert into new_part_range_table_distr_random select i, '2020-05-02' from generate_series(1, 20)i; +create table new_part_list_table_distr_hashed(a int, b text) distributed by (a) +partition by list (b) ( + partition part1 values ('test1'), + partition part2 values ('test2'), + default partition other_vals +); +insert into new_part_list_table_distr_hashed select i, 'test1' from generate_series(1, 20)i; +insert into new_part_list_table_distr_hashed select i, 'test2' from generate_series(1, 20)i; +insert into new_part_list_table_distr_hashed select i, 'test3' from generate_series(1, 20)i; +create table new_part_list_table_distr_random(a int, b text) distributed randomly +partition by list (b) ( + partition part1 values ('test1'), + partition part2 values ('test2'), + default partition other_vals +); +insert into new_part_list_table_distr_random select i, 'test1' from generate_series(1, 20)i; +insert into new_part_list_table_distr_random select i, 'test2' from generate_series(1, 20)i; +insert into new_part_list_table_distr_random select i, 'test3' from generate_series(1, 20)i; +create table new_multi_part_table_distr_hashed(a int, b date, c text) distributed by (a) +partition by range (b) +subpartition by list (c) subpartition template +( + subpartition subpart1 values ('test1'), + subpartition subpart2 values ('test2') +) +( + partition part1 start (date '2023-01-01'), + partition part2 start (date '2023-02-01'), + partition part3 start (date '2023-03-01') end (date '2024-01-01') +); +insert into new_multi_part_table_distr_hashed select i, '2023-01-05', 'test1' from generate_series(1, 20)i; +insert into new_multi_part_table_distr_hashed select i, '2023-02-05', 'test2' from generate_series(1, 20)i; +insert into new_multi_part_table_distr_hashed select i, '2023-03-05', 'test1' from generate_series(1, 20)i; +-- Also check CTAS statement +create table new_table_ctas as select a from generate_series(1, 20)a distributed by(a); +select * into new_table_into from generate_series(1, 20)a; +select a, gp_segment_id from new_table_distr_hashed order by a; + a | gp_segment_id +----+--------------- + 1 | 1 + 2 | 0 + 3 | 0 + 4 | 0 + 5 | 1 + 6 | 0 + 7 | 0 + 8 | 0 + 9 | 0 + 10 | 0 + 11 | 1 + 12 | 1 + 13 | 0 + 14 | 1 + 15 | 1 + 16 | 0 + 17 | 1 + 18 | 0 + 19 | 0 + 20 | 1 +(20 rows) + +select a, gp_segment_id from new_table_distr_hashed_ao_row order by a; + a | gp_segment_id +----+--------------- + 1 | 1 + 2 | 0 + 3 | 0 + 4 | 0 + 5 | 1 + 6 | 0 + 7 | 0 + 8 | 0 + 9 | 0 + 10 | 0 + 11 | 1 + 12 | 1 + 13 | 0 + 14 | 1 + 15 | 1 + 16 | 0 + 17 | 1 + 18 | 0 + 19 | 0 + 20 | 1 +(20 rows) + +select a, gp_segment_id from new_table_distr_hashed_ao_col order by a; + a | gp_segment_id +----+--------------- + 1 | 1 + 2 | 0 + 3 | 0 + 4 | 0 + 5 | 1 + 6 | 0 + 7 | 0 + 8 | 0 + 9 | 0 + 10 | 0 + 11 | 1 + 12 | 1 + 13 | 0 + 14 | 1 + 15 | 1 + 16 | 0 + 17 | 1 + 18 | 0 + 19 | 0 + 20 | 1 +(20 rows) + +select a, (gp_segment_id < 2) as correct_segment_id from new_table_distr_random order by a; + a | correct_segment_id +----+-------------------- + 1 | t + 2 | t + 3 | t + 4 | t + 5 | t + 6 | t + 7 | t + 8 | t + 9 | t + 10 | t + 11 | t + 12 | t + 13 | t + 14 | t + 15 | t + 16 | t + 17 | t + 18 | t + 19 | t + 20 | t +(20 rows) + +select a, (gp_segment_id < 2) as correct_segment_id from new_table_distr_random_ao_row order by a; + a | correct_segment_id +----+-------------------- + 1 | t + 2 | t + 3 | t + 4 | t + 5 | t + 6 | t + 7 | t + 8 | t + 9 | t + 10 | t + 11 | t + 12 | t + 13 | t + 14 | t + 15 | t + 16 | t + 17 | t + 18 | t + 19 | t + 20 | t +(20 rows) + +select a, (gp_segment_id < 2) as correct_segment_id from new_table_distr_random_ao_col order by a; + a | correct_segment_id +----+-------------------- + 1 | t + 2 | t + 3 | t + 4 | t + 5 | t + 6 | t + 7 | t + 8 | t + 9 | t + 10 | t + 11 | t + 12 | t + 13 | t + 14 | t + 15 | t + 16 | t + 17 | t + 18 | t + 19 | t + 20 | t +(20 rows) + +select a from new_table_distr_replicated order by a; + a +---- + 1 + 2 + 3 + 4 + 5 + 6 + 7 + 8 + 9 + 10 + 11 + 12 + 13 + 14 + 15 + 16 + 17 + 18 + 19 + 20 +(20 rows) + +select a from new_table_distr_replicated_ao_row order by a; + a +---- + 1 + 2 + 3 + 4 + 5 + 6 + 7 + 8 + 9 + 10 + 11 + 12 + 13 + 14 + 15 + 16 + 17 + 18 + 19 + 20 +(20 rows) + +select a from new_table_distr_replicated_ao_col order by a; + a +---- + 1 + 2 + 3 + 4 + 5 + 6 + 7 + 8 + 9 + 10 + 11 + 12 + 13 + 14 + 15 + 16 + 17 + 18 + 19 + 20 +(20 rows) + +select *, gp_segment_id from new_part_range_table_distr_hashed order by a, b; + a | b | gp_segment_id +----+------------+--------------- + 1 | 05-02-2020 | 1 + 1 | 01-02-2023 | 1 + 1 | 05-02-2023 | 1 + 2 | 05-02-2020 | 0 + 2 | 01-02-2023 | 0 + 2 | 05-02-2023 | 0 + 3 | 05-02-2020 | 0 + 3 | 01-02-2023 | 0 + 3 | 05-02-2023 | 0 + 4 | 05-02-2020 | 0 + 4 | 01-02-2023 | 0 + 4 | 05-02-2023 | 0 + 5 | 05-02-2020 | 1 + 5 | 01-02-2023 | 1 + 5 | 05-02-2023 | 1 + 6 | 05-02-2020 | 0 + 6 | 01-02-2023 | 0 + 6 | 05-02-2023 | 0 + 7 | 05-02-2020 | 0 + 7 | 01-02-2023 | 0 + 7 | 05-02-2023 | 0 + 8 | 05-02-2020 | 0 + 8 | 01-02-2023 | 0 + 8 | 05-02-2023 | 0 + 9 | 05-02-2020 | 0 + 9 | 01-02-2023 | 0 + 9 | 05-02-2023 | 0 + 10 | 05-02-2020 | 0 + 10 | 01-02-2023 | 0 + 10 | 05-02-2023 | 0 + 11 | 05-02-2020 | 1 + 11 | 01-02-2023 | 1 + 11 | 05-02-2023 | 1 + 12 | 05-02-2020 | 1 + 12 | 01-02-2023 | 1 + 12 | 05-02-2023 | 1 + 13 | 05-02-2020 | 0 + 13 | 01-02-2023 | 0 + 13 | 05-02-2023 | 0 + 14 | 05-02-2020 | 1 + 14 | 01-02-2023 | 1 + 14 | 05-02-2023 | 1 + 15 | 05-02-2020 | 1 + 15 | 01-02-2023 | 1 + 15 | 05-02-2023 | 1 + 16 | 05-02-2020 | 0 + 16 | 01-02-2023 | 0 + 16 | 05-02-2023 | 0 + 17 | 05-02-2020 | 1 + 17 | 01-02-2023 | 1 + 17 | 05-02-2023 | 1 + 18 | 05-02-2020 | 0 + 18 | 01-02-2023 | 0 + 18 | 05-02-2023 | 0 + 19 | 05-02-2020 | 0 + 19 | 01-02-2023 | 0 + 19 | 05-02-2023 | 0 + 20 | 05-02-2020 | 1 + 20 | 01-02-2023 | 1 + 20 | 05-02-2023 | 1 +(60 rows) + +select *, (gp_segment_id < 2) as correct_segment_id from new_part_range_table_distr_random order by a, b; + a | b | correct_segment_id +----+------------+-------------------- + 1 | 05-02-2020 | t + 1 | 01-02-2023 | t + 1 | 05-02-2023 | t + 2 | 05-02-2020 | t + 2 | 01-02-2023 | t + 2 | 05-02-2023 | t + 3 | 05-02-2020 | t + 3 | 01-02-2023 | t + 3 | 05-02-2023 | t + 4 | 05-02-2020 | t + 4 | 01-02-2023 | t + 4 | 05-02-2023 | t + 5 | 05-02-2020 | t + 5 | 01-02-2023 | t + 5 | 05-02-2023 | t + 6 | 05-02-2020 | t + 6 | 01-02-2023 | t + 6 | 05-02-2023 | t + 7 | 05-02-2020 | t + 7 | 01-02-2023 | t + 7 | 05-02-2023 | t + 8 | 05-02-2020 | t + 8 | 01-02-2023 | t + 8 | 05-02-2023 | t + 9 | 05-02-2020 | t + 9 | 01-02-2023 | t + 9 | 05-02-2023 | t + 10 | 05-02-2020 | t + 10 | 01-02-2023 | t + 10 | 05-02-2023 | t + 11 | 05-02-2020 | t + 11 | 01-02-2023 | t + 11 | 05-02-2023 | t + 12 | 05-02-2020 | t + 12 | 01-02-2023 | t + 12 | 05-02-2023 | t + 13 | 05-02-2020 | t + 13 | 01-02-2023 | t + 13 | 05-02-2023 | t + 14 | 05-02-2020 | t + 14 | 01-02-2023 | t + 14 | 05-02-2023 | t + 15 | 05-02-2020 | t + 15 | 01-02-2023 | t + 15 | 05-02-2023 | t + 16 | 05-02-2020 | t + 16 | 01-02-2023 | t + 16 | 05-02-2023 | t + 17 | 05-02-2020 | t + 17 | 01-02-2023 | t + 17 | 05-02-2023 | t + 18 | 05-02-2020 | t + 18 | 01-02-2023 | t + 18 | 05-02-2023 | t + 19 | 05-02-2020 | t + 19 | 01-02-2023 | t + 19 | 05-02-2023 | t + 20 | 05-02-2020 | t + 20 | 01-02-2023 | t + 20 | 05-02-2023 | t +(60 rows) + +select *, gp_segment_id from new_part_list_table_distr_hashed order by a, b; + a | b | gp_segment_id +----+-------+--------------- + 1 | test1 | 1 + 1 | test2 | 1 + 1 | test3 | 1 + 2 | test1 | 0 + 2 | test2 | 0 + 2 | test3 | 0 + 3 | test1 | 0 + 3 | test2 | 0 + 3 | test3 | 0 + 4 | test1 | 0 + 4 | test2 | 0 + 4 | test3 | 0 + 5 | test1 | 1 + 5 | test2 | 1 + 5 | test3 | 1 + 6 | test1 | 0 + 6 | test2 | 0 + 6 | test3 | 0 + 7 | test1 | 0 + 7 | test2 | 0 + 7 | test3 | 0 + 8 | test1 | 0 + 8 | test2 | 0 + 8 | test3 | 0 + 9 | test1 | 0 + 9 | test2 | 0 + 9 | test3 | 0 + 10 | test1 | 0 + 10 | test2 | 0 + 10 | test3 | 0 + 11 | test1 | 1 + 11 | test2 | 1 + 11 | test3 | 1 + 12 | test1 | 1 + 12 | test2 | 1 + 12 | test3 | 1 + 13 | test1 | 0 + 13 | test2 | 0 + 13 | test3 | 0 + 14 | test1 | 1 + 14 | test2 | 1 + 14 | test3 | 1 + 15 | test1 | 1 + 15 | test2 | 1 + 15 | test3 | 1 + 16 | test1 | 0 + 16 | test2 | 0 + 16 | test3 | 0 + 17 | test1 | 1 + 17 | test2 | 1 + 17 | test3 | 1 + 18 | test1 | 0 + 18 | test2 | 0 + 18 | test3 | 0 + 19 | test1 | 0 + 19 | test2 | 0 + 19 | test3 | 0 + 20 | test1 | 1 + 20 | test2 | 1 + 20 | test3 | 1 +(60 rows) + +select *, (gp_segment_id < 2) as correct_segment_id from new_part_list_table_distr_random order by a, b; + a | b | correct_segment_id +----+-------+-------------------- + 1 | test1 | t + 1 | test2 | t + 1 | test3 | t + 2 | test1 | t + 2 | test2 | t + 2 | test3 | t + 3 | test1 | t + 3 | test2 | t + 3 | test3 | t + 4 | test1 | t + 4 | test2 | t + 4 | test3 | t + 5 | test1 | t + 5 | test2 | t + 5 | test3 | t + 6 | test1 | t + 6 | test2 | t + 6 | test3 | t + 7 | test1 | t + 7 | test2 | t + 7 | test3 | t + 8 | test1 | t + 8 | test2 | t + 8 | test3 | t + 9 | test1 | t + 9 | test2 | t + 9 | test3 | t + 10 | test1 | t + 10 | test2 | t + 10 | test3 | t + 11 | test1 | t + 11 | test2 | t + 11 | test3 | t + 12 | test1 | t + 12 | test2 | t + 12 | test3 | t + 13 | test1 | t + 13 | test2 | t + 13 | test3 | t + 14 | test1 | t + 14 | test2 | t + 14 | test3 | t + 15 | test1 | t + 15 | test2 | t + 15 | test3 | t + 16 | test1 | t + 16 | test2 | t + 16 | test3 | t + 17 | test1 | t + 17 | test2 | t + 17 | test3 | t + 18 | test1 | t + 18 | test2 | t + 18 | test3 | t + 19 | test1 | t + 19 | test2 | t + 19 | test3 | t + 20 | test1 | t + 20 | test2 | t + 20 | test3 | t +(60 rows) + +select *, gp_segment_id from new_multi_part_table_distr_hashed order by a, b, c; + a | b | c | gp_segment_id +----+------------+-------+--------------- + 1 | 01-05-2023 | test1 | 1 + 1 | 02-05-2023 | test2 | 1 + 1 | 03-05-2023 | test1 | 1 + 2 | 01-05-2023 | test1 | 0 + 2 | 02-05-2023 | test2 | 0 + 2 | 03-05-2023 | test1 | 0 + 3 | 01-05-2023 | test1 | 0 + 3 | 02-05-2023 | test2 | 0 + 3 | 03-05-2023 | test1 | 0 + 4 | 01-05-2023 | test1 | 0 + 4 | 02-05-2023 | test2 | 0 + 4 | 03-05-2023 | test1 | 0 + 5 | 01-05-2023 | test1 | 1 + 5 | 02-05-2023 | test2 | 1 + 5 | 03-05-2023 | test1 | 1 + 6 | 01-05-2023 | test1 | 0 + 6 | 02-05-2023 | test2 | 0 + 6 | 03-05-2023 | test1 | 0 + 7 | 01-05-2023 | test1 | 0 + 7 | 02-05-2023 | test2 | 0 + 7 | 03-05-2023 | test1 | 0 + 8 | 01-05-2023 | test1 | 0 + 8 | 02-05-2023 | test2 | 0 + 8 | 03-05-2023 | test1 | 0 + 9 | 01-05-2023 | test1 | 0 + 9 | 02-05-2023 | test2 | 0 + 9 | 03-05-2023 | test1 | 0 + 10 | 01-05-2023 | test1 | 0 + 10 | 02-05-2023 | test2 | 0 + 10 | 03-05-2023 | test1 | 0 + 11 | 01-05-2023 | test1 | 1 + 11 | 02-05-2023 | test2 | 1 + 11 | 03-05-2023 | test1 | 1 + 12 | 01-05-2023 | test1 | 1 + 12 | 02-05-2023 | test2 | 1 + 12 | 03-05-2023 | test1 | 1 + 13 | 01-05-2023 | test1 | 0 + 13 | 02-05-2023 | test2 | 0 + 13 | 03-05-2023 | test1 | 0 + 14 | 01-05-2023 | test1 | 1 + 14 | 02-05-2023 | test2 | 1 + 14 | 03-05-2023 | test1 | 1 + 15 | 01-05-2023 | test1 | 1 + 15 | 02-05-2023 | test2 | 1 + 15 | 03-05-2023 | test1 | 1 + 16 | 01-05-2023 | test1 | 0 + 16 | 02-05-2023 | test2 | 0 + 16 | 03-05-2023 | test1 | 0 + 17 | 01-05-2023 | test1 | 1 + 17 | 02-05-2023 | test2 | 1 + 17 | 03-05-2023 | test1 | 1 + 18 | 01-05-2023 | test1 | 0 + 18 | 02-05-2023 | test2 | 0 + 18 | 03-05-2023 | test1 | 0 + 19 | 01-05-2023 | test1 | 0 + 19 | 02-05-2023 | test2 | 0 + 19 | 03-05-2023 | test1 | 0 + 20 | 01-05-2023 | test1 | 1 + 20 | 02-05-2023 | test2 | 1 + 20 | 03-05-2023 | test1 | 1 +(60 rows) + +select a, gp_segment_id from new_table_ctas order by a; + a | gp_segment_id +----+--------------- + 1 | 1 + 2 | 0 + 3 | 0 + 4 | 0 + 5 | 1 + 6 | 0 + 7 | 0 + 8 | 0 + 9 | 0 + 10 | 0 + 11 | 1 + 12 | 1 + 13 | 0 + 14 | 1 + 15 | 1 + 16 | 0 + 17 | 1 + 18 | 0 + 19 | 0 + 20 | 1 +(20 rows) + +select *, (gp_segment_id < 2) as correct_segment_id from new_table_into order by a; + a | correct_segment_id +----+-------------------- + 1 | t + 2 | t + 3 | t + 4 | t + 5 | t + 6 | t + 7 | t + 8 | t + 9 | t + 10 | t + 11 | t + 12 | t + 13 | t + 14 | t + 15 | t + 16 | t + 17 | t + 18 | t + 19 | t + 20 | t +(20 rows) + +-- Validate the insertion works fine with the new tables +-- after 'gp_target_numsegments' reset +begin; +select gp_expand_lock_catalog(); + gp_expand_lock_catalog +------------------------ + +(1 row) + +select gp_toolkit.gp_reset_rebalance_numsegments(); + gp_reset_rebalance_numsegments +-------------------------------- + +(1 row) + +end; +insert into new_table_ctas select generate_series(21, 40); +insert into new_table_into select generate_series(21, 40); +insert into new_table_distr_hashed select generate_series(21, 40); +insert into new_table_distr_hashed_ao_row select generate_series(21, 40); +insert into new_table_distr_hashed_ao_col select generate_series(21, 40); +insert into new_table_distr_random select generate_series(21, 40); +insert into new_table_distr_random_ao_row select generate_series(21, 40); +insert into new_table_distr_random_ao_col select generate_series(21, 40); +insert into new_part_range_table_distr_hashed select i, '2023-01-02' from generate_series(21, 40)i; +insert into new_part_range_table_distr_hashed select i, '2023-05-02' from generate_series(21, 40)i; +insert into new_part_range_table_distr_hashed select i, '2020-05-02' from generate_series(21, 40)i; +insert into new_part_range_table_distr_random select i, '2023-01-02' from generate_series(21, 40)i; +insert into new_part_range_table_distr_random select i, '2023-05-02' from generate_series(21, 40)i; +insert into new_part_range_table_distr_random select i, '2020-05-02' from generate_series(21, 40)i; +insert into new_part_list_table_distr_hashed select i, 'test1' from generate_series(21, 40)i; +insert into new_part_list_table_distr_hashed select i, 'test2' from generate_series(21, 40)i; +insert into new_part_list_table_distr_hashed select i, 'test3' from generate_series(21, 40)i; +insert into new_part_list_table_distr_random select i, 'test1' from generate_series(21, 40)i; +insert into new_part_list_table_distr_random select i, 'test2' from generate_series(21, 40)i; +insert into new_part_list_table_distr_random select i, 'test3' from generate_series(21, 40)i; +insert into new_multi_part_table_distr_hashed select i, '2023-01-05', 'test1' from generate_series(21, 40)i; +insert into new_multi_part_table_distr_hashed select i, '2023-02-05', 'test2' from generate_series(21, 40)i; +insert into new_multi_part_table_distr_hashed select i, '2023-03-05', 'test1' from generate_series(21, 40)i; +select a, gp_segment_id from new_table_ctas order by a; + a | gp_segment_id +----+--------------- + 1 | 1 + 2 | 0 + 3 | 0 + 4 | 0 + 5 | 1 + 6 | 0 + 7 | 0 + 8 | 0 + 9 | 0 + 10 | 0 + 11 | 1 + 12 | 1 + 13 | 0 + 14 | 1 + 15 | 1 + 16 | 0 + 17 | 1 + 18 | 0 + 19 | 0 + 20 | 1 + 21 | 0 + 22 | 0 + 23 | 1 + 24 | 0 + 25 | 1 + 26 | 1 + 27 | 0 + 28 | 0 + 29 | 0 + 30 | 1 + 31 | 1 + 32 | 0 + 33 | 0 + 34 | 0 + 35 | 1 + 36 | 1 + 37 | 0 + 38 | 1 + 39 | 0 + 40 | 1 +(40 rows) + +select *, (gp_segment_id < 2) as correct_segment_id from new_table_into order by a; + a | correct_segment_id +----+-------------------- + 1 | t + 2 | t + 3 | t + 4 | t + 5 | t + 6 | t + 7 | t + 8 | t + 9 | t + 10 | t + 11 | t + 12 | t + 13 | t + 14 | t + 15 | t + 16 | t + 17 | t + 18 | t + 19 | t + 20 | t + 21 | t + 22 | t + 23 | t + 24 | t + 25 | t + 26 | t + 27 | t + 28 | t + 29 | t + 30 | t + 31 | t + 32 | t + 33 | t + 34 | t + 35 | t + 36 | t + 37 | t + 38 | t + 39 | t + 40 | t +(40 rows) + +select a, gp_segment_id from new_table_distr_hashed order by a; + a | gp_segment_id +----+--------------- + 1 | 1 + 2 | 0 + 3 | 0 + 4 | 0 + 5 | 1 + 6 | 0 + 7 | 0 + 8 | 0 + 9 | 0 + 10 | 0 + 11 | 1 + 12 | 1 + 13 | 0 + 14 | 1 + 15 | 1 + 16 | 0 + 17 | 1 + 18 | 0 + 19 | 0 + 20 | 1 + 21 | 0 + 22 | 0 + 23 | 1 + 24 | 0 + 25 | 1 + 26 | 1 + 27 | 0 + 28 | 0 + 29 | 0 + 30 | 1 + 31 | 1 + 32 | 0 + 33 | 0 + 34 | 0 + 35 | 1 + 36 | 1 + 37 | 0 + 38 | 1 + 39 | 0 + 40 | 1 +(40 rows) + +select a, gp_segment_id from new_table_distr_hashed_ao_row order by a; + a | gp_segment_id +----+--------------- + 1 | 1 + 2 | 0 + 3 | 0 + 4 | 0 + 5 | 1 + 6 | 0 + 7 | 0 + 8 | 0 + 9 | 0 + 10 | 0 + 11 | 1 + 12 | 1 + 13 | 0 + 14 | 1 + 15 | 1 + 16 | 0 + 17 | 1 + 18 | 0 + 19 | 0 + 20 | 1 + 21 | 0 + 22 | 0 + 23 | 1 + 24 | 0 + 25 | 1 + 26 | 1 + 27 | 0 + 28 | 0 + 29 | 0 + 30 | 1 + 31 | 1 + 32 | 0 + 33 | 0 + 34 | 0 + 35 | 1 + 36 | 1 + 37 | 0 + 38 | 1 + 39 | 0 + 40 | 1 +(40 rows) + +select a, gp_segment_id from new_table_distr_hashed_ao_col order by a; + a | gp_segment_id +----+--------------- + 1 | 1 + 2 | 0 + 3 | 0 + 4 | 0 + 5 | 1 + 6 | 0 + 7 | 0 + 8 | 0 + 9 | 0 + 10 | 0 + 11 | 1 + 12 | 1 + 13 | 0 + 14 | 1 + 15 | 1 + 16 | 0 + 17 | 1 + 18 | 0 + 19 | 0 + 20 | 1 + 21 | 0 + 22 | 0 + 23 | 1 + 24 | 0 + 25 | 1 + 26 | 1 + 27 | 0 + 28 | 0 + 29 | 0 + 30 | 1 + 31 | 1 + 32 | 0 + 33 | 0 + 34 | 0 + 35 | 1 + 36 | 1 + 37 | 0 + 38 | 1 + 39 | 0 + 40 | 1 +(40 rows) + +select a, (gp_segment_id < 2) as correct_segment_id from new_table_distr_random order by a; + a | correct_segment_id +----+-------------------- + 1 | t + 2 | t + 3 | t + 4 | t + 5 | t + 6 | t + 7 | t + 8 | t + 9 | t + 10 | t + 11 | t + 12 | t + 13 | t + 14 | t + 15 | t + 16 | t + 17 | t + 18 | t + 19 | t + 20 | t + 21 | t + 22 | t + 23 | t + 24 | t + 25 | t + 26 | t + 27 | t + 28 | t + 29 | t + 30 | t + 31 | t + 32 | t + 33 | t + 34 | t + 35 | t + 36 | t + 37 | t + 38 | t + 39 | t + 40 | t +(40 rows) + +select a, (gp_segment_id < 2) as correct_segment_id from new_table_distr_random_ao_row order by a; + a | correct_segment_id +----+-------------------- + 1 | t + 2 | t + 3 | t + 4 | t + 5 | t + 6 | t + 7 | t + 8 | t + 9 | t + 10 | t + 11 | t + 12 | t + 13 | t + 14 | t + 15 | t + 16 | t + 17 | t + 18 | t + 19 | t + 20 | t + 21 | t + 22 | t + 23 | t + 24 | t + 25 | t + 26 | t + 27 | t + 28 | t + 29 | t + 30 | t + 31 | t + 32 | t + 33 | t + 34 | t + 35 | t + 36 | t + 37 | t + 38 | t + 39 | t + 40 | t +(40 rows) + +select a, (gp_segment_id < 2) as correct_segment_id from new_table_distr_random_ao_col order by a; + a | correct_segment_id +----+-------------------- + 1 | t + 2 | t + 3 | t + 4 | t + 5 | t + 6 | t + 7 | t + 8 | t + 9 | t + 10 | t + 11 | t + 12 | t + 13 | t + 14 | t + 15 | t + 16 | t + 17 | t + 18 | t + 19 | t + 20 | t + 21 | t + 22 | t + 23 | t + 24 | t + 25 | t + 26 | t + 27 | t + 28 | t + 29 | t + 30 | t + 31 | t + 32 | t + 33 | t + 34 | t + 35 | t + 36 | t + 37 | t + 38 | t + 39 | t + 40 | t +(40 rows) + +select *, gp_segment_id from new_part_range_table_distr_hashed order by a, b; + a | b | gp_segment_id +----+------------+--------------- + 1 | 05-02-2020 | 1 + 1 | 01-02-2023 | 1 + 1 | 05-02-2023 | 1 + 2 | 05-02-2020 | 0 + 2 | 01-02-2023 | 0 + 2 | 05-02-2023 | 0 + 3 | 05-02-2020 | 0 + 3 | 01-02-2023 | 0 + 3 | 05-02-2023 | 0 + 4 | 05-02-2020 | 0 + 4 | 01-02-2023 | 0 + 4 | 05-02-2023 | 0 + 5 | 05-02-2020 | 1 + 5 | 01-02-2023 | 1 + 5 | 05-02-2023 | 1 + 6 | 05-02-2020 | 0 + 6 | 01-02-2023 | 0 + 6 | 05-02-2023 | 0 + 7 | 05-02-2020 | 0 + 7 | 01-02-2023 | 0 + 7 | 05-02-2023 | 0 + 8 | 05-02-2020 | 0 + 8 | 01-02-2023 | 0 + 8 | 05-02-2023 | 0 + 9 | 05-02-2020 | 0 + 9 | 01-02-2023 | 0 + 9 | 05-02-2023 | 0 + 10 | 05-02-2020 | 0 + 10 | 01-02-2023 | 0 + 10 | 05-02-2023 | 0 + 11 | 05-02-2020 | 1 + 11 | 01-02-2023 | 1 + 11 | 05-02-2023 | 1 + 12 | 05-02-2020 | 1 + 12 | 01-02-2023 | 1 + 12 | 05-02-2023 | 1 + 13 | 05-02-2020 | 0 + 13 | 01-02-2023 | 0 + 13 | 05-02-2023 | 0 + 14 | 05-02-2020 | 1 + 14 | 01-02-2023 | 1 + 14 | 05-02-2023 | 1 + 15 | 05-02-2020 | 1 + 15 | 01-02-2023 | 1 + 15 | 05-02-2023 | 1 + 16 | 05-02-2020 | 0 + 16 | 01-02-2023 | 0 + 16 | 05-02-2023 | 0 + 17 | 05-02-2020 | 1 + 17 | 01-02-2023 | 1 + 17 | 05-02-2023 | 1 + 18 | 05-02-2020 | 0 + 18 | 01-02-2023 | 0 + 18 | 05-02-2023 | 0 + 19 | 05-02-2020 | 0 + 19 | 01-02-2023 | 0 + 19 | 05-02-2023 | 0 + 20 | 05-02-2020 | 1 + 20 | 01-02-2023 | 1 + 20 | 05-02-2023 | 1 + 21 | 05-02-2020 | 0 + 21 | 01-02-2023 | 0 + 21 | 05-02-2023 | 0 + 22 | 05-02-2020 | 0 + 22 | 01-02-2023 | 0 + 22 | 05-02-2023 | 0 + 23 | 05-02-2020 | 1 + 23 | 01-02-2023 | 1 + 23 | 05-02-2023 | 1 + 24 | 05-02-2020 | 0 + 24 | 01-02-2023 | 0 + 24 | 05-02-2023 | 0 + 25 | 05-02-2020 | 1 + 25 | 01-02-2023 | 1 + 25 | 05-02-2023 | 1 + 26 | 05-02-2020 | 1 + 26 | 01-02-2023 | 1 + 26 | 05-02-2023 | 1 + 27 | 05-02-2020 | 0 + 27 | 01-02-2023 | 0 + 27 | 05-02-2023 | 0 + 28 | 05-02-2020 | 0 + 28 | 01-02-2023 | 0 + 28 | 05-02-2023 | 0 + 29 | 05-02-2020 | 0 + 29 | 01-02-2023 | 0 + 29 | 05-02-2023 | 0 + 30 | 05-02-2020 | 1 + 30 | 01-02-2023 | 1 + 30 | 05-02-2023 | 1 + 31 | 05-02-2020 | 1 + 31 | 01-02-2023 | 1 + 31 | 05-02-2023 | 1 + 32 | 05-02-2020 | 0 + 32 | 01-02-2023 | 0 + 32 | 05-02-2023 | 0 + 33 | 05-02-2020 | 0 + 33 | 01-02-2023 | 0 + 33 | 05-02-2023 | 0 + 34 | 05-02-2020 | 0 + 34 | 01-02-2023 | 0 + 34 | 05-02-2023 | 0 + 35 | 05-02-2020 | 1 + 35 | 01-02-2023 | 1 + 35 | 05-02-2023 | 1 + 36 | 05-02-2020 | 1 + 36 | 01-02-2023 | 1 + 36 | 05-02-2023 | 1 + 37 | 05-02-2020 | 0 + 37 | 01-02-2023 | 0 + 37 | 05-02-2023 | 0 + 38 | 05-02-2020 | 1 + 38 | 01-02-2023 | 1 + 38 | 05-02-2023 | 1 + 39 | 05-02-2020 | 0 + 39 | 01-02-2023 | 0 + 39 | 05-02-2023 | 0 + 40 | 05-02-2020 | 1 + 40 | 01-02-2023 | 1 + 40 | 05-02-2023 | 1 +(120 rows) + +select *, (gp_segment_id < 2) as correct_segment_id from new_part_range_table_distr_random order by a, b; + a | b | correct_segment_id +----+------------+-------------------- + 1 | 05-02-2020 | t + 1 | 01-02-2023 | t + 1 | 05-02-2023 | t + 2 | 05-02-2020 | t + 2 | 01-02-2023 | t + 2 | 05-02-2023 | t + 3 | 05-02-2020 | t + 3 | 01-02-2023 | t + 3 | 05-02-2023 | t + 4 | 05-02-2020 | t + 4 | 01-02-2023 | t + 4 | 05-02-2023 | t + 5 | 05-02-2020 | t + 5 | 01-02-2023 | t + 5 | 05-02-2023 | t + 6 | 05-02-2020 | t + 6 | 01-02-2023 | t + 6 | 05-02-2023 | t + 7 | 05-02-2020 | t + 7 | 01-02-2023 | t + 7 | 05-02-2023 | t + 8 | 05-02-2020 | t + 8 | 01-02-2023 | t + 8 | 05-02-2023 | t + 9 | 05-02-2020 | t + 9 | 01-02-2023 | t + 9 | 05-02-2023 | t + 10 | 05-02-2020 | t + 10 | 01-02-2023 | t + 10 | 05-02-2023 | t + 11 | 05-02-2020 | t + 11 | 01-02-2023 | t + 11 | 05-02-2023 | t + 12 | 05-02-2020 | t + 12 | 01-02-2023 | t + 12 | 05-02-2023 | t + 13 | 05-02-2020 | t + 13 | 01-02-2023 | t + 13 | 05-02-2023 | t + 14 | 05-02-2020 | t + 14 | 01-02-2023 | t + 14 | 05-02-2023 | t + 15 | 05-02-2020 | t + 15 | 01-02-2023 | t + 15 | 05-02-2023 | t + 16 | 05-02-2020 | t + 16 | 01-02-2023 | t + 16 | 05-02-2023 | t + 17 | 05-02-2020 | t + 17 | 01-02-2023 | t + 17 | 05-02-2023 | t + 18 | 05-02-2020 | t + 18 | 01-02-2023 | t + 18 | 05-02-2023 | t + 19 | 05-02-2020 | t + 19 | 01-02-2023 | t + 19 | 05-02-2023 | t + 20 | 05-02-2020 | t + 20 | 01-02-2023 | t + 20 | 05-02-2023 | t + 21 | 05-02-2020 | t + 21 | 01-02-2023 | t + 21 | 05-02-2023 | t + 22 | 05-02-2020 | t + 22 | 01-02-2023 | t + 22 | 05-02-2023 | t + 23 | 05-02-2020 | t + 23 | 01-02-2023 | t + 23 | 05-02-2023 | t + 24 | 05-02-2020 | t + 24 | 01-02-2023 | t + 24 | 05-02-2023 | t + 25 | 05-02-2020 | t + 25 | 01-02-2023 | t + 25 | 05-02-2023 | t + 26 | 05-02-2020 | t + 26 | 01-02-2023 | t + 26 | 05-02-2023 | t + 27 | 05-02-2020 | t + 27 | 01-02-2023 | t + 27 | 05-02-2023 | t + 28 | 05-02-2020 | t + 28 | 01-02-2023 | t + 28 | 05-02-2023 | t + 29 | 05-02-2020 | t + 29 | 01-02-2023 | t + 29 | 05-02-2023 | t + 30 | 05-02-2020 | t + 30 | 01-02-2023 | t + 30 | 05-02-2023 | t + 31 | 05-02-2020 | t + 31 | 01-02-2023 | t + 31 | 05-02-2023 | t + 32 | 05-02-2020 | t + 32 | 01-02-2023 | t + 32 | 05-02-2023 | t + 33 | 05-02-2020 | t + 33 | 01-02-2023 | t + 33 | 05-02-2023 | t + 34 | 05-02-2020 | t + 34 | 01-02-2023 | t + 34 | 05-02-2023 | t + 35 | 05-02-2020 | t + 35 | 01-02-2023 | t + 35 | 05-02-2023 | t + 36 | 05-02-2020 | t + 36 | 01-02-2023 | t + 36 | 05-02-2023 | t + 37 | 05-02-2020 | t + 37 | 01-02-2023 | t + 37 | 05-02-2023 | t + 38 | 05-02-2020 | t + 38 | 01-02-2023 | t + 38 | 05-02-2023 | t + 39 | 05-02-2020 | t + 39 | 01-02-2023 | t + 39 | 05-02-2023 | t + 40 | 05-02-2020 | t + 40 | 01-02-2023 | t + 40 | 05-02-2023 | t +(120 rows) + +select *, gp_segment_id from new_part_list_table_distr_hashed order by a, b; + a | b | gp_segment_id +----+-------+--------------- + 1 | test1 | 1 + 1 | test2 | 1 + 1 | test3 | 1 + 2 | test1 | 0 + 2 | test2 | 0 + 2 | test3 | 0 + 3 | test1 | 0 + 3 | test2 | 0 + 3 | test3 | 0 + 4 | test1 | 0 + 4 | test2 | 0 + 4 | test3 | 0 + 5 | test1 | 1 + 5 | test2 | 1 + 5 | test3 | 1 + 6 | test1 | 0 + 6 | test2 | 0 + 6 | test3 | 0 + 7 | test1 | 0 + 7 | test2 | 0 + 7 | test3 | 0 + 8 | test1 | 0 + 8 | test2 | 0 + 8 | test3 | 0 + 9 | test1 | 0 + 9 | test2 | 0 + 9 | test3 | 0 + 10 | test1 | 0 + 10 | test2 | 0 + 10 | test3 | 0 + 11 | test1 | 1 + 11 | test2 | 1 + 11 | test3 | 1 + 12 | test1 | 1 + 12 | test2 | 1 + 12 | test3 | 1 + 13 | test1 | 0 + 13 | test2 | 0 + 13 | test3 | 0 + 14 | test1 | 1 + 14 | test2 | 1 + 14 | test3 | 1 + 15 | test1 | 1 + 15 | test2 | 1 + 15 | test3 | 1 + 16 | test1 | 0 + 16 | test2 | 0 + 16 | test3 | 0 + 17 | test1 | 1 + 17 | test2 | 1 + 17 | test3 | 1 + 18 | test1 | 0 + 18 | test2 | 0 + 18 | test3 | 0 + 19 | test1 | 0 + 19 | test2 | 0 + 19 | test3 | 0 + 20 | test1 | 1 + 20 | test2 | 1 + 20 | test3 | 1 + 21 | test1 | 0 + 21 | test2 | 0 + 21 | test3 | 0 + 22 | test1 | 0 + 22 | test2 | 0 + 22 | test3 | 0 + 23 | test1 | 1 + 23 | test2 | 1 + 23 | test3 | 1 + 24 | test1 | 0 + 24 | test2 | 0 + 24 | test3 | 0 + 25 | test1 | 1 + 25 | test2 | 1 + 25 | test3 | 1 + 26 | test1 | 1 + 26 | test2 | 1 + 26 | test3 | 1 + 27 | test1 | 0 + 27 | test2 | 0 + 27 | test3 | 0 + 28 | test1 | 0 + 28 | test2 | 0 + 28 | test3 | 0 + 29 | test1 | 0 + 29 | test2 | 0 + 29 | test3 | 0 + 30 | test1 | 1 + 30 | test2 | 1 + 30 | test3 | 1 + 31 | test1 | 1 + 31 | test2 | 1 + 31 | test3 | 1 + 32 | test1 | 0 + 32 | test2 | 0 + 32 | test3 | 0 + 33 | test1 | 0 + 33 | test2 | 0 + 33 | test3 | 0 + 34 | test1 | 0 + 34 | test2 | 0 + 34 | test3 | 0 + 35 | test1 | 1 + 35 | test2 | 1 + 35 | test3 | 1 + 36 | test1 | 1 + 36 | test2 | 1 + 36 | test3 | 1 + 37 | test1 | 0 + 37 | test2 | 0 + 37 | test3 | 0 + 38 | test1 | 1 + 38 | test2 | 1 + 38 | test3 | 1 + 39 | test1 | 0 + 39 | test2 | 0 + 39 | test3 | 0 + 40 | test1 | 1 + 40 | test2 | 1 + 40 | test3 | 1 +(120 rows) + +select *, (gp_segment_id < 2) as correct_segment_id from new_part_list_table_distr_random order by a, b; + a | b | correct_segment_id +----+-------+-------------------- + 1 | test1 | t + 1 | test2 | t + 1 | test3 | t + 2 | test1 | t + 2 | test2 | t + 2 | test3 | t + 3 | test1 | t + 3 | test2 | t + 3 | test3 | t + 4 | test1 | t + 4 | test2 | t + 4 | test3 | t + 5 | test1 | t + 5 | test2 | t + 5 | test3 | t + 6 | test1 | t + 6 | test2 | t + 6 | test3 | t + 7 | test1 | t + 7 | test2 | t + 7 | test3 | t + 8 | test1 | t + 8 | test2 | t + 8 | test3 | t + 9 | test1 | t + 9 | test2 | t + 9 | test3 | t + 10 | test1 | t + 10 | test2 | t + 10 | test3 | t + 11 | test1 | t + 11 | test2 | t + 11 | test3 | t + 12 | test1 | t + 12 | test2 | t + 12 | test3 | t + 13 | test1 | t + 13 | test2 | t + 13 | test3 | t + 14 | test1 | t + 14 | test2 | t + 14 | test3 | t + 15 | test1 | t + 15 | test2 | t + 15 | test3 | t + 16 | test1 | t + 16 | test2 | t + 16 | test3 | t + 17 | test1 | t + 17 | test2 | t + 17 | test3 | t + 18 | test1 | t + 18 | test2 | t + 18 | test3 | t + 19 | test1 | t + 19 | test2 | t + 19 | test3 | t + 20 | test1 | t + 20 | test2 | t + 20 | test3 | t + 21 | test1 | t + 21 | test2 | t + 21 | test3 | t + 22 | test1 | t + 22 | test2 | t + 22 | test3 | t + 23 | test1 | t + 23 | test2 | t + 23 | test3 | t + 24 | test1 | t + 24 | test2 | t + 24 | test3 | t + 25 | test1 | t + 25 | test2 | t + 25 | test3 | t + 26 | test1 | t + 26 | test2 | t + 26 | test3 | t + 27 | test1 | t + 27 | test2 | t + 27 | test3 | t + 28 | test1 | t + 28 | test2 | t + 28 | test3 | t + 29 | test1 | t + 29 | test2 | t + 29 | test3 | t + 30 | test1 | t + 30 | test2 | t + 30 | test3 | t + 31 | test1 | t + 31 | test2 | t + 31 | test3 | t + 32 | test1 | t + 32 | test2 | t + 32 | test3 | t + 33 | test1 | t + 33 | test2 | t + 33 | test3 | t + 34 | test1 | t + 34 | test2 | t + 34 | test3 | t + 35 | test1 | t + 35 | test2 | t + 35 | test3 | t + 36 | test1 | t + 36 | test2 | t + 36 | test3 | t + 37 | test1 | t + 37 | test2 | t + 37 | test3 | t + 38 | test1 | t + 38 | test2 | t + 38 | test3 | t + 39 | test1 | t + 39 | test2 | t + 39 | test3 | t + 40 | test1 | t + 40 | test2 | t + 40 | test3 | t +(120 rows) + +select *, gp_segment_id from new_multi_part_table_distr_hashed order by a, b, c; + a | b | c | gp_segment_id +----+------------+-------+--------------- + 1 | 01-05-2023 | test1 | 1 + 1 | 02-05-2023 | test2 | 1 + 1 | 03-05-2023 | test1 | 1 + 2 | 01-05-2023 | test1 | 0 + 2 | 02-05-2023 | test2 | 0 + 2 | 03-05-2023 | test1 | 0 + 3 | 01-05-2023 | test1 | 0 + 3 | 02-05-2023 | test2 | 0 + 3 | 03-05-2023 | test1 | 0 + 4 | 01-05-2023 | test1 | 0 + 4 | 02-05-2023 | test2 | 0 + 4 | 03-05-2023 | test1 | 0 + 5 | 01-05-2023 | test1 | 1 + 5 | 02-05-2023 | test2 | 1 + 5 | 03-05-2023 | test1 | 1 + 6 | 01-05-2023 | test1 | 0 + 6 | 02-05-2023 | test2 | 0 + 6 | 03-05-2023 | test1 | 0 + 7 | 01-05-2023 | test1 | 0 + 7 | 02-05-2023 | test2 | 0 + 7 | 03-05-2023 | test1 | 0 + 8 | 01-05-2023 | test1 | 0 + 8 | 02-05-2023 | test2 | 0 + 8 | 03-05-2023 | test1 | 0 + 9 | 01-05-2023 | test1 | 0 + 9 | 02-05-2023 | test2 | 0 + 9 | 03-05-2023 | test1 | 0 + 10 | 01-05-2023 | test1 | 0 + 10 | 02-05-2023 | test2 | 0 + 10 | 03-05-2023 | test1 | 0 + 11 | 01-05-2023 | test1 | 1 + 11 | 02-05-2023 | test2 | 1 + 11 | 03-05-2023 | test1 | 1 + 12 | 01-05-2023 | test1 | 1 + 12 | 02-05-2023 | test2 | 1 + 12 | 03-05-2023 | test1 | 1 + 13 | 01-05-2023 | test1 | 0 + 13 | 02-05-2023 | test2 | 0 + 13 | 03-05-2023 | test1 | 0 + 14 | 01-05-2023 | test1 | 1 + 14 | 02-05-2023 | test2 | 1 + 14 | 03-05-2023 | test1 | 1 + 15 | 01-05-2023 | test1 | 1 + 15 | 02-05-2023 | test2 | 1 + 15 | 03-05-2023 | test1 | 1 + 16 | 01-05-2023 | test1 | 0 + 16 | 02-05-2023 | test2 | 0 + 16 | 03-05-2023 | test1 | 0 + 17 | 01-05-2023 | test1 | 1 + 17 | 02-05-2023 | test2 | 1 + 17 | 03-05-2023 | test1 | 1 + 18 | 01-05-2023 | test1 | 0 + 18 | 02-05-2023 | test2 | 0 + 18 | 03-05-2023 | test1 | 0 + 19 | 01-05-2023 | test1 | 0 + 19 | 02-05-2023 | test2 | 0 + 19 | 03-05-2023 | test1 | 0 + 20 | 01-05-2023 | test1 | 1 + 20 | 02-05-2023 | test2 | 1 + 20 | 03-05-2023 | test1 | 1 + 21 | 01-05-2023 | test1 | 0 + 21 | 02-05-2023 | test2 | 0 + 21 | 03-05-2023 | test1 | 0 + 22 | 01-05-2023 | test1 | 0 + 22 | 02-05-2023 | test2 | 0 + 22 | 03-05-2023 | test1 | 0 + 23 | 01-05-2023 | test1 | 1 + 23 | 02-05-2023 | test2 | 1 + 23 | 03-05-2023 | test1 | 1 + 24 | 01-05-2023 | test1 | 0 + 24 | 02-05-2023 | test2 | 0 + 24 | 03-05-2023 | test1 | 0 + 25 | 01-05-2023 | test1 | 1 + 25 | 02-05-2023 | test2 | 1 + 25 | 03-05-2023 | test1 | 1 + 26 | 01-05-2023 | test1 | 1 + 26 | 02-05-2023 | test2 | 1 + 26 | 03-05-2023 | test1 | 1 + 27 | 01-05-2023 | test1 | 0 + 27 | 02-05-2023 | test2 | 0 + 27 | 03-05-2023 | test1 | 0 + 28 | 01-05-2023 | test1 | 0 + 28 | 02-05-2023 | test2 | 0 + 28 | 03-05-2023 | test1 | 0 + 29 | 01-05-2023 | test1 | 0 + 29 | 02-05-2023 | test2 | 0 + 29 | 03-05-2023 | test1 | 0 + 30 | 01-05-2023 | test1 | 1 + 30 | 02-05-2023 | test2 | 1 + 30 | 03-05-2023 | test1 | 1 + 31 | 01-05-2023 | test1 | 1 + 31 | 02-05-2023 | test2 | 1 + 31 | 03-05-2023 | test1 | 1 + 32 | 01-05-2023 | test1 | 0 + 32 | 02-05-2023 | test2 | 0 + 32 | 03-05-2023 | test1 | 0 + 33 | 01-05-2023 | test1 | 0 + 33 | 02-05-2023 | test2 | 0 + 33 | 03-05-2023 | test1 | 0 + 34 | 01-05-2023 | test1 | 0 + 34 | 02-05-2023 | test2 | 0 + 34 | 03-05-2023 | test1 | 0 + 35 | 01-05-2023 | test1 | 1 + 35 | 02-05-2023 | test2 | 1 + 35 | 03-05-2023 | test1 | 1 + 36 | 01-05-2023 | test1 | 1 + 36 | 02-05-2023 | test2 | 1 + 36 | 03-05-2023 | test1 | 1 + 37 | 01-05-2023 | test1 | 0 + 37 | 02-05-2023 | test2 | 0 + 37 | 03-05-2023 | test1 | 0 + 38 | 01-05-2023 | test1 | 1 + 38 | 02-05-2023 | test2 | 1 + 38 | 03-05-2023 | test1 | 1 + 39 | 01-05-2023 | test1 | 0 + 39 | 02-05-2023 | test2 | 0 + 39 | 03-05-2023 | test1 | 0 + 40 | 01-05-2023 | test1 | 1 + 40 | 02-05-2023 | test2 | 1 + 40 | 03-05-2023 | test1 | 1 +(120 rows) + +-- And do some cleanup +drop table new_table_distr_hashed; +drop table new_table_distr_hashed_ao_row; +drop table new_table_distr_hashed_ao_col; +drop table new_table_distr_random; +drop table new_table_distr_random_ao_row; +drop table new_table_distr_random_ao_col; +drop table new_table_distr_replicated; +drop table new_table_distr_replicated_ao_row; +drop table new_table_distr_replicated_ao_col; +drop table new_part_range_table_distr_hashed; +drop table new_part_range_table_distr_random; +drop table new_part_list_table_distr_hashed; +drop table new_part_list_table_distr_random; +drop table new_multi_part_table_distr_hashed; +drop table new_table_ctas; +drop table new_table_into; +-- Check rollback of alter rebalance operation +create table table_distr_hashed(a int) distributed by (a); +insert into table_distr_hashed select generate_series(1, 20); +select count(1), gp_segment_id from table_distr_hashed group by gp_segment_id order by gp_segment_id; + count | gp_segment_id +-------+--------------- + 8 | 0 + 4 | 1 + 8 | 2 +(3 rows) + +begin; +alter table table_distr_hashed rebalance 2; +select count(1), gp_segment_id from table_distr_hashed group by gp_segment_id order by gp_segment_id; + count | gp_segment_id +-------+--------------- + 12 | 0 + 8 | 1 +(2 rows) + +rollback; +select count(1), gp_segment_id from table_distr_hashed group by gp_segment_id order by gp_segment_id; + count | gp_segment_id +-------+--------------- + 8 | 0 + 4 | 1 + 8 | 2 +(3 rows) + +drop table table_distr_hashed; +-- Check rebalance with parameter +create table table_distr_hashed(a int) distributed by (a); +insert into table_distr_hashed select generate_series(1, 20); +select count(1), gp_segment_id from table_distr_hashed group by gp_segment_id order by gp_segment_id; + count | gp_segment_id +-------+--------------- + 8 | 0 + 4 | 1 + 8 | 2 +(3 rows) + +-- Shrink to 2 segments +alter table table_distr_hashed rebalance 2; +select count(1), gp_segment_id from table_distr_hashed group by gp_segment_id order by gp_segment_id; + count | gp_segment_id +-------+--------------- + 12 | 0 + 8 | 1 +(2 rows) + +-- Shrink to 1 segment +alter table table_distr_hashed rebalance 1; +select count(1), gp_segment_id from table_distr_hashed group by gp_segment_id order by gp_segment_id; + count | gp_segment_id +-------+--------------- + 20 | 0 +(1 row) + +-- Expand back to 3 segments +alter table table_distr_hashed rebalance 3; +select count(1), gp_segment_id from table_distr_hashed group by gp_segment_id order by gp_segment_id; + count | gp_segment_id +-------+--------------- + 8 | 0 + 4 | 1 + 8 | 2 +(3 rows) + +-- Try to expand to 4 segments - should do nothing +alter table table_distr_hashed rebalance 3; +ERROR: cannot rebalance table "table_distr_hashed" +DETAIL: table has already been rebalanced +drop table table_distr_hashed; +-- Check rebalance of materialized view +create table test_table(a int) distributed by (a); +insert into test_table select generate_series(1, 10); +create materialized view mv_test_table as select a from test_table distributed by (a); +alter table test_table rebalance 1; +alter materialized view mv_test_table rebalance 1; +select count(1), gp_segment_id from test_table group by gp_segment_id; + count | gp_segment_id +-------+--------------- + 10 | 0 +(1 row) + +select count(1), gp_segment_id from mv_test_table group by gp_segment_id; + count | gp_segment_id +-------+--------------- + 10 | 0 +(1 row) + +drop materialized view mv_test_table; +drop table test_table; +-- Check rollback of the rebalance of materialized view +create table test_table(a int) distributed by (a); +insert into test_table select generate_series(1, 10); +create materialized view mv_test_table as select a from test_table distributed by (a); +select count(1), gp_segment_id from test_table group by gp_segment_id order by gp_segment_id; + count | gp_segment_id +-------+--------------- + 5 | 0 + 1 | 1 + 4 | 2 +(3 rows) + +select count(1), gp_segment_id from mv_test_table group by gp_segment_id order by gp_segment_id; + count | gp_segment_id +-------+--------------- + 5 | 0 + 1 | 1 + 4 | 2 +(3 rows) + +begin; +alter table test_table rebalance 1; +alter materialized view mv_test_table rebalance 1; +rollback; +select count(1), gp_segment_id from test_table group by gp_segment_id order by gp_segment_id; + count | gp_segment_id +-------+--------------- + 5 | 0 + 1 | 1 + 4 | 2 +(3 rows) + +select count(1), gp_segment_id from mv_test_table group by gp_segment_id order by gp_segment_id; + count | gp_segment_id +-------+--------------- + 5 | 0 + 1 | 1 + 4 | 2 +(3 rows) + +drop materialized view mv_test_table; +drop table test_table; diff --git a/src/test/regress/greenplum_schedule b/src/test/regress/greenplum_schedule index 41df7af9f115..bf5f1b59229f 100755 --- a/src/test/regress/greenplum_schedule +++ b/src/test/regress/greenplum_schedule @@ -363,4 +363,5 @@ test: hba_conf test: gp_query_id +test: alter_rebalance # end of tests diff --git a/src/test/regress/sql/alter_rebalance.sql b/src/test/regress/sql/alter_rebalance.sql new file mode 100644 index 000000000000..09c9555f5b06 --- /dev/null +++ b/src/test/regress/sql/alter_rebalance.sql @@ -0,0 +1,491 @@ +-- Check 'ALTER TABLE ... REBALANCE' command and 'gp_target_numsegments' GUC + +-- Create hashed distributed tables +create table table_distr_hashed(a int) distributed by (a); +insert into table_distr_hashed select generate_series(1, 20); + +create table table_distr_hashed_ao_row(a int) with (appendonly=true, orientation=row) distributed by (a); +insert into table_distr_hashed_ao_row select generate_series(1, 20); + +create table table_distr_hashed_ao_col(a int) with (appendonly=true, orientation=column) distributed by (a); +insert into table_distr_hashed_ao_col select generate_series(1, 20); + +-- Create randomly distributed tables +create table table_distr_random(a int) distributed randomly; +insert into table_distr_random select generate_series(1, 20); + +create table table_distr_random_ao_row(a int) with (appendonly=true, orientation=row) distributed randomly; +insert into table_distr_random_ao_row select generate_series(1, 20); + +create table table_distr_random_ao_col(a int) with (appendonly=true, orientation=column) distributed randomly; +insert into table_distr_random_ao_col select generate_series(1, 20); + +-- Create replicated distributed tables +create table table_distr_replicated(a int) distributed replicated; +insert into table_distr_replicated select generate_series(1, 20); + +create table table_distr_replicated_ao_row(a int) with (appendonly=true, orientation=row) distributed replicated; +insert into table_distr_replicated_ao_row select generate_series(1, 20); + +create table table_distr_replicated_ao_col(a int) with (appendonly=true, orientation=column) distributed replicated; +insert into table_distr_replicated_ao_col select generate_series(1, 20); + +-- Create part tables +create table part_range_table_distr_hashed (a int, b date) distributed by (a) +partition by range (b) ( + start (date '2023-01-01') inclusive + end (date '2024-01-01') exclusive + every (interval '1 month'), + default partition other_vals +); +insert into part_range_table_distr_hashed select i, '2023-01-02' from generate_series(1, 20)i; +insert into part_range_table_distr_hashed select i, '2023-05-02' from generate_series(1, 20)i; +insert into part_range_table_distr_hashed select i, '2020-05-02' from generate_series(1, 20)i; + +create table part_range_table_distr_random (a int, b date) distributed randomly +partition by range (b) ( + start (date '2023-01-01') inclusive + end (date '2024-01-01') exclusive + every (interval '1 month'), + default partition other_vals +); +insert into part_range_table_distr_random select i, '2023-01-02' from generate_series(1, 20)i; +insert into part_range_table_distr_random select i, '2023-05-02' from generate_series(1, 20)i; +insert into part_range_table_distr_random select i, '2020-05-02' from generate_series(1, 20)i; + +create table part_list_table_distr_hashed(a int, b text) distributed by (a) +partition by list (b) ( + partition part1 values ('test1'), + partition part2 values ('test2'), + default partition other_vals +); +insert into part_list_table_distr_hashed select i, 'test1' from generate_series(1, 20)i; +insert into part_list_table_distr_hashed select i, 'test2' from generate_series(1, 20)i; +insert into part_list_table_distr_hashed select i, 'test3' from generate_series(1, 20)i; + +create table part_list_table_distr_random(a int, b text) distributed randomly +partition by list (b) ( + partition part1 values ('test1'), + partition part2 values ('test2'), + default partition other_vals +); +insert into part_list_table_distr_random select i, 'test1' from generate_series(1, 20)i; +insert into part_list_table_distr_random select i, 'test2' from generate_series(1, 20)i; +insert into part_list_table_distr_random select i, 'test3' from generate_series(1, 20)i; + +create table multi_part_table_distr_hashed(a int, b date, c text) distributed by (a) +partition by range (b) +subpartition by list (c) subpartition template +( + subpartition subpart1 values ('test1'), + subpartition subpart2 values ('test2') +) +( + partition part1 start (date '2023-01-01'), + partition part2 start (date '2023-02-01'), + partition part3 start (date '2023-03-01') end (date '2024-01-01') +); +insert into multi_part_table_distr_hashed select i, '2023-01-05', 'test1' from generate_series(1, 20)i; +insert into multi_part_table_distr_hashed select i, '2023-02-05', 'test2' from generate_series(1, 20)i; +insert into multi_part_table_distr_hashed select i, '2023-03-05', 'test1' from generate_series(1, 20)i; + +-- Now check shrink of the created tables into 2 segments +begin; +select gp_expand_lock_catalog(); +select gp_toolkit.gp_set_rebalance_numsegments(2); +end; + +alter table table_distr_hashed rebalance; +alter table table_distr_hashed_ao_row rebalance; +alter table table_distr_hashed_ao_col rebalance; + +alter table table_distr_random rebalance; +alter table table_distr_random_ao_row rebalance; +alter table table_distr_random_ao_col rebalance; + +alter table table_distr_replicated rebalance; +alter table table_distr_replicated_ao_row rebalance; +alter table table_distr_replicated_ao_col rebalance; + +alter table part_range_table_distr_hashed rebalance; +alter table part_range_table_distr_random rebalance; + +alter table part_list_table_distr_hashed rebalance; +alter table part_list_table_distr_random rebalance; + +alter table multi_part_table_distr_hashed rebalance; + +-- Verify that data is presented only on segments #0 and #1 +select a, gp_segment_id from table_distr_hashed order by a; +select a, gp_segment_id from table_distr_hashed_ao_row order by a; +select a, gp_segment_id from table_distr_hashed_ao_col order by a; + +select a, (gp_segment_id < 2) as correct_segment_id from table_distr_random order by a; +select a, (gp_segment_id < 2) as correct_segment_id from table_distr_random_ao_row order by a; +select a, (gp_segment_id < 2) as correct_segment_id from table_distr_random_ao_col order by a; + +select a from table_distr_replicated order by a; +select a from table_distr_replicated_ao_row order by a; +select a from table_distr_replicated_ao_col order by a; + +select *, gp_segment_id from part_range_table_distr_hashed order by a, b; +select *, (gp_segment_id < 2) as correct_segment_id from part_range_table_distr_random order by a, b; + +select *, gp_segment_id from part_list_table_distr_hashed order by a, b; +select *, (gp_segment_id < 2) as correct_segment_id from part_list_table_distr_random order by a, b; + +select *, gp_segment_id from multi_part_table_distr_hashed order by a, b, c; + +-- Check that new data is added only to reduced set of segments +begin; +select gp_expand_lock_catalog(); +select gp_toolkit.gp_reset_rebalance_numsegments(); +end; + +insert into table_distr_hashed select generate_series(21, 40); +insert into table_distr_hashed_ao_row select generate_series(21, 40); +insert into table_distr_hashed_ao_col select generate_series(21, 40); + +insert into table_distr_random select generate_series(21, 40); +insert into table_distr_random_ao_row select generate_series(21, 40); +insert into table_distr_random_ao_col select generate_series(21, 40); + +insert into table_distr_replicated select generate_series(21, 40); +insert into table_distr_replicated_ao_row select generate_series(21, 40); +insert into table_distr_replicated_ao_col select generate_series(21, 40); + +insert into part_range_table_distr_hashed select i, '2023-01-02' from generate_series(21, 40)i; +insert into part_range_table_distr_hashed select i, '2023-05-02' from generate_series(21, 40)i; +insert into part_range_table_distr_hashed select i, '2020-05-02' from generate_series(21, 40)i; + +insert into part_range_table_distr_random select i, '2023-01-02' from generate_series(21, 40)i; +insert into part_range_table_distr_random select i, '2023-05-02' from generate_series(21, 40)i; +insert into part_range_table_distr_random select i, '2020-05-02' from generate_series(21, 40)i; + +insert into part_list_table_distr_hashed select i, 'test1' from generate_series(21, 40)i; +insert into part_list_table_distr_hashed select i, 'test2' from generate_series(21, 40)i; +insert into part_list_table_distr_hashed select i, 'test3' from generate_series(21, 40)i; + +insert into part_list_table_distr_random select i, 'test1' from generate_series(21, 40)i; +insert into part_list_table_distr_random select i, 'test2' from generate_series(21, 40)i; +insert into part_list_table_distr_random select i, 'test3' from generate_series(21, 40)i; + +insert into multi_part_table_distr_hashed select i, '2023-01-05', 'test1' from generate_series(21, 40)i; +insert into multi_part_table_distr_hashed select i, '2023-02-05', 'test2' from generate_series(21, 40)i; +insert into multi_part_table_distr_hashed select i, '2023-03-05', 'test1' from generate_series(21, 40)i; + +select a, gp_segment_id from table_distr_hashed order by a; +select a, gp_segment_id from table_distr_hashed_ao_row order by a; +select a, gp_segment_id from table_distr_hashed_ao_col order by a; + +select a, (gp_segment_id < 2) as correct_segment_id from table_distr_random order by a; +select a, (gp_segment_id < 2) as correct_segment_id from table_distr_random_ao_row order by a; +select a, (gp_segment_id < 2) as correct_segment_id from table_distr_random_ao_col order by a; + +select a from table_distr_replicated order by a; +select a from table_distr_replicated_ao_row order by a; +select a from table_distr_replicated_ao_col order by a; + +select *, gp_segment_id from part_range_table_distr_hashed order by a, b; +select *, (gp_segment_id < 2) as correct_segment_id from part_range_table_distr_random order by a, b; + +select *, gp_segment_id from part_list_table_distr_hashed order by a, b; +select *, (gp_segment_id < 2) as correct_segment_id from part_list_table_distr_random order by a, b; + +select *, gp_segment_id from multi_part_table_distr_hashed order by a, b, c; + +-- And do some cleanup +drop table table_distr_hashed; +drop table table_distr_hashed_ao_row; +drop table table_distr_hashed_ao_col; + +drop table table_distr_random; +drop table table_distr_random_ao_row; +drop table table_distr_random_ao_col; + +drop table table_distr_replicated; +drop table table_distr_replicated_ao_row; +drop table table_distr_replicated_ao_col; + +drop table part_range_table_distr_hashed; +drop table part_range_table_distr_random; + +drop table part_list_table_distr_hashed; +drop table part_list_table_distr_random; + +drop table multi_part_table_distr_hashed; + +-- Check that all newly created tables have data only on segments #0 and #1 +begin; +select gp_expand_lock_catalog(); +select gp_toolkit.gp_set_rebalance_numsegments(2); +end; + +create table new_table_distr_hashed(a int) distributed by (a); +insert into new_table_distr_hashed select generate_series(1, 20); + +create table new_table_distr_hashed_ao_row(a int) with (appendonly=true, orientation=row) distributed by (a); +insert into new_table_distr_hashed_ao_row select generate_series(1, 20); + +create table new_table_distr_hashed_ao_col(a int) with (appendonly=true, orientation=column) distributed by (a); +insert into new_table_distr_hashed_ao_col select generate_series(1, 20); + +create table new_table_distr_random(a int) distributed randomly; +insert into new_table_distr_random select generate_series(1, 20); + +create table new_table_distr_random_ao_row(a int) with (appendonly=true, orientation=row) distributed randomly; +insert into new_table_distr_random_ao_row select generate_series(1, 20); + +create table new_table_distr_random_ao_col(a int) with (appendonly=true, orientation=column) distributed randomly; +insert into new_table_distr_random_ao_col select generate_series(1, 20); + +create table new_table_distr_replicated(a int) distributed replicated; +insert into new_table_distr_replicated select generate_series(1, 20); + +create table new_table_distr_replicated_ao_row(a int) with (appendonly=true, orientation=row) distributed replicated; +insert into new_table_distr_replicated_ao_row select generate_series(1, 20); + +create table new_table_distr_replicated_ao_col(a int) with (appendonly=true, orientation=column) distributed replicated; +insert into new_table_distr_replicated_ao_col select generate_series(1, 20); + +create table new_part_range_table_distr_hashed (a int, b date) distributed by (a) +partition by range (b) ( + start (date '2023-01-01') inclusive + end (date '2024-01-01') exclusive + every (interval '1 month'), + default partition other_vals +); +insert into new_part_range_table_distr_hashed select i, '2023-01-02' from generate_series(1, 20)i; +insert into new_part_range_table_distr_hashed select i, '2023-05-02' from generate_series(1, 20)i; +insert into new_part_range_table_distr_hashed select i, '2020-05-02' from generate_series(1, 20)i; + +create table new_part_range_table_distr_random (a int, b date) distributed randomly +partition by range (b) ( + start (date '2023-01-01') inclusive + end (date '2024-01-01') exclusive + every (interval '1 month'), + default partition other_vals +); +insert into new_part_range_table_distr_random select i, '2023-01-02' from generate_series(1, 20)i; +insert into new_part_range_table_distr_random select i, '2023-05-02' from generate_series(1, 20)i; +insert into new_part_range_table_distr_random select i, '2020-05-02' from generate_series(1, 20)i; + +create table new_part_list_table_distr_hashed(a int, b text) distributed by (a) +partition by list (b) ( + partition part1 values ('test1'), + partition part2 values ('test2'), + default partition other_vals +); +insert into new_part_list_table_distr_hashed select i, 'test1' from generate_series(1, 20)i; +insert into new_part_list_table_distr_hashed select i, 'test2' from generate_series(1, 20)i; +insert into new_part_list_table_distr_hashed select i, 'test3' from generate_series(1, 20)i; + +create table new_part_list_table_distr_random(a int, b text) distributed randomly +partition by list (b) ( + partition part1 values ('test1'), + partition part2 values ('test2'), + default partition other_vals +); +insert into new_part_list_table_distr_random select i, 'test1' from generate_series(1, 20)i; +insert into new_part_list_table_distr_random select i, 'test2' from generate_series(1, 20)i; +insert into new_part_list_table_distr_random select i, 'test3' from generate_series(1, 20)i; + +create table new_multi_part_table_distr_hashed(a int, b date, c text) distributed by (a) +partition by range (b) +subpartition by list (c) subpartition template +( + subpartition subpart1 values ('test1'), + subpartition subpart2 values ('test2') +) +( + partition part1 start (date '2023-01-01'), + partition part2 start (date '2023-02-01'), + partition part3 start (date '2023-03-01') end (date '2024-01-01') +); +insert into new_multi_part_table_distr_hashed select i, '2023-01-05', 'test1' from generate_series(1, 20)i; +insert into new_multi_part_table_distr_hashed select i, '2023-02-05', 'test2' from generate_series(1, 20)i; +insert into new_multi_part_table_distr_hashed select i, '2023-03-05', 'test1' from generate_series(1, 20)i; + +-- Also check CTAS statement +create table new_table_ctas as select a from generate_series(1, 20)a distributed by(a); +select * into new_table_into from generate_series(1, 20)a; + +select a, gp_segment_id from new_table_distr_hashed order by a; +select a, gp_segment_id from new_table_distr_hashed_ao_row order by a; +select a, gp_segment_id from new_table_distr_hashed_ao_col order by a; + +select a, (gp_segment_id < 2) as correct_segment_id from new_table_distr_random order by a; +select a, (gp_segment_id < 2) as correct_segment_id from new_table_distr_random_ao_row order by a; +select a, (gp_segment_id < 2) as correct_segment_id from new_table_distr_random_ao_col order by a; + +select a from new_table_distr_replicated order by a; +select a from new_table_distr_replicated_ao_row order by a; +select a from new_table_distr_replicated_ao_col order by a; + +select *, gp_segment_id from new_part_range_table_distr_hashed order by a, b; +select *, (gp_segment_id < 2) as correct_segment_id from new_part_range_table_distr_random order by a, b; + +select *, gp_segment_id from new_part_list_table_distr_hashed order by a, b; +select *, (gp_segment_id < 2) as correct_segment_id from new_part_list_table_distr_random order by a, b; + +select *, gp_segment_id from new_multi_part_table_distr_hashed order by a, b, c; + +select a, gp_segment_id from new_table_ctas order by a; +select *, (gp_segment_id < 2) as correct_segment_id from new_table_into order by a; + +-- Validate the insertion works fine with the new tables +-- after 'gp_target_numsegments' reset +begin; +select gp_expand_lock_catalog(); +select gp_toolkit.gp_reset_rebalance_numsegments(); +end; + +insert into new_table_ctas select generate_series(21, 40); +insert into new_table_into select generate_series(21, 40); + +insert into new_table_distr_hashed select generate_series(21, 40); +insert into new_table_distr_hashed_ao_row select generate_series(21, 40); +insert into new_table_distr_hashed_ao_col select generate_series(21, 40); + +insert into new_table_distr_random select generate_series(21, 40); +insert into new_table_distr_random_ao_row select generate_series(21, 40); +insert into new_table_distr_random_ao_col select generate_series(21, 40); + +insert into new_part_range_table_distr_hashed select i, '2023-01-02' from generate_series(21, 40)i; +insert into new_part_range_table_distr_hashed select i, '2023-05-02' from generate_series(21, 40)i; +insert into new_part_range_table_distr_hashed select i, '2020-05-02' from generate_series(21, 40)i; + +insert into new_part_range_table_distr_random select i, '2023-01-02' from generate_series(21, 40)i; +insert into new_part_range_table_distr_random select i, '2023-05-02' from generate_series(21, 40)i; +insert into new_part_range_table_distr_random select i, '2020-05-02' from generate_series(21, 40)i; + +insert into new_part_list_table_distr_hashed select i, 'test1' from generate_series(21, 40)i; +insert into new_part_list_table_distr_hashed select i, 'test2' from generate_series(21, 40)i; +insert into new_part_list_table_distr_hashed select i, 'test3' from generate_series(21, 40)i; + +insert into new_part_list_table_distr_random select i, 'test1' from generate_series(21, 40)i; +insert into new_part_list_table_distr_random select i, 'test2' from generate_series(21, 40)i; +insert into new_part_list_table_distr_random select i, 'test3' from generate_series(21, 40)i; + +insert into new_multi_part_table_distr_hashed select i, '2023-01-05', 'test1' from generate_series(21, 40)i; +insert into new_multi_part_table_distr_hashed select i, '2023-02-05', 'test2' from generate_series(21, 40)i; +insert into new_multi_part_table_distr_hashed select i, '2023-03-05', 'test1' from generate_series(21, 40)i; + +select a, gp_segment_id from new_table_ctas order by a; +select *, (gp_segment_id < 2) as correct_segment_id from new_table_into order by a; + +select a, gp_segment_id from new_table_distr_hashed order by a; +select a, gp_segment_id from new_table_distr_hashed_ao_row order by a; +select a, gp_segment_id from new_table_distr_hashed_ao_col order by a; + +select a, (gp_segment_id < 2) as correct_segment_id from new_table_distr_random order by a; +select a, (gp_segment_id < 2) as correct_segment_id from new_table_distr_random_ao_row order by a; +select a, (gp_segment_id < 2) as correct_segment_id from new_table_distr_random_ao_col order by a; + +select *, gp_segment_id from new_part_range_table_distr_hashed order by a, b; +select *, (gp_segment_id < 2) as correct_segment_id from new_part_range_table_distr_random order by a, b; + +select *, gp_segment_id from new_part_list_table_distr_hashed order by a, b; +select *, (gp_segment_id < 2) as correct_segment_id from new_part_list_table_distr_random order by a, b; + +select *, gp_segment_id from new_multi_part_table_distr_hashed order by a, b, c; + +-- And do some cleanup +drop table new_table_distr_hashed; +drop table new_table_distr_hashed_ao_row; +drop table new_table_distr_hashed_ao_col; + +drop table new_table_distr_random; +drop table new_table_distr_random_ao_row; +drop table new_table_distr_random_ao_col; + +drop table new_table_distr_replicated; +drop table new_table_distr_replicated_ao_row; +drop table new_table_distr_replicated_ao_col; + +drop table new_part_range_table_distr_hashed; +drop table new_part_range_table_distr_random; + +drop table new_part_list_table_distr_hashed; +drop table new_part_list_table_distr_random; + +drop table new_multi_part_table_distr_hashed; + +drop table new_table_ctas; +drop table new_table_into; + +-- Check rollback of alter rebalance operation +create table table_distr_hashed(a int) distributed by (a); +insert into table_distr_hashed select generate_series(1, 20); + +select count(1), gp_segment_id from table_distr_hashed group by gp_segment_id order by gp_segment_id; + +begin; +alter table table_distr_hashed rebalance 2; +select count(1), gp_segment_id from table_distr_hashed group by gp_segment_id order by gp_segment_id; +rollback; +select count(1), gp_segment_id from table_distr_hashed group by gp_segment_id order by gp_segment_id; + +drop table table_distr_hashed; + +-- Check rebalance with parameter +create table table_distr_hashed(a int) distributed by (a); +insert into table_distr_hashed select generate_series(1, 20); +select count(1), gp_segment_id from table_distr_hashed group by gp_segment_id order by gp_segment_id; + +-- Shrink to 2 segments +alter table table_distr_hashed rebalance 2; +select count(1), gp_segment_id from table_distr_hashed group by gp_segment_id order by gp_segment_id; + +-- Shrink to 1 segment +alter table table_distr_hashed rebalance 1; +select count(1), gp_segment_id from table_distr_hashed group by gp_segment_id order by gp_segment_id; + +-- Expand back to 3 segments +alter table table_distr_hashed rebalance 3; +select count(1), gp_segment_id from table_distr_hashed group by gp_segment_id order by gp_segment_id; + +-- Try to expand to 4 segments - should do nothing +alter table table_distr_hashed rebalance 3; + +drop table table_distr_hashed; + +-- Check rebalance of materialized view +-- start_ignore +drop materialized view if exists mv_test_table; +drop table if exists test_table; +-- end_ignore + +create table test_table(a int) distributed by (a); +insert into test_table select generate_series(1, 10); + +create materialized view mv_test_table as select a from test_table distributed by (a); + +alter table test_table rebalance 1; +alter materialized view mv_test_table rebalance 1; + +select count(1), gp_segment_id from test_table group by gp_segment_id; +select count(1), gp_segment_id from mv_test_table group by gp_segment_id; + +drop materialized view mv_test_table; +drop table test_table; + +-- Check rollback of the rebalance of materialized view +create table test_table(a int) distributed by (a); +insert into test_table select generate_series(1, 10); + +create materialized view mv_test_table as select a from test_table distributed by (a); + +select count(1), gp_segment_id from test_table group by gp_segment_id order by gp_segment_id; +select count(1), gp_segment_id from mv_test_table group by gp_segment_id order by gp_segment_id; + +begin; +alter table test_table rebalance 1; +alter materialized view mv_test_table rebalance 1; +rollback; + +select count(1), gp_segment_id from test_table group by gp_segment_id order by gp_segment_id; +select count(1), gp_segment_id from mv_test_table group by gp_segment_id order by gp_segment_id; + +drop materialized view mv_test_table; +drop table test_table;