From 5cbce01b9aae00c6404d0ae32b3aa1741b65cec2 Mon Sep 17 00:00:00 2001
From: Michael Scranton <mscranton@commercehub.com>
Date: Wed, 12 Oct 2022 16:32:25 -0400
Subject: [PATCH 01/62] dwconnect - tuning, hashed sk, scd bypass

---
 postgres/lib/dwconnect.js | 347 ++++++++++++++++++++++----------------
 1 file changed, 199 insertions(+), 148 deletions(-)

diff --git a/postgres/lib/dwconnect.js b/postgres/lib/dwconnect.js
index fb943e20..613fadee 100644
--- a/postgres/lib/dwconnect.js
+++ b/postgres/lib/dwconnect.js
@@ -27,6 +27,13 @@ module.exports = function (config, columnConfig) {
 		useSurrogateDateKeys: true,
 	}, columnConfig || {});
 
+	
+
+	if(config.hashedSurrogateKeys && config.useSlowlyChangingDimensions) {
+		logger.error(`Unsupported configuration, useSlowlyChangingDimensions:${useSlowlyChangingDimensions} hashedSurrogateKeys:${hashedSurrogateKeys}.`);
+		process.exit();
+	};
+
 	client.getDimensionColumn = columnConfig.dimColumnTransform;
 	client.columnConfig = columnConfig;
 
@@ -118,15 +125,23 @@ module.exports = function (config, columnConfig) {
 		schema[qualifiedStagingTable] = schema[qualifiedTable];
 
 		let tasks = [];
-
+		
 		let deleteHandler = deletesSetup(qualifiedTable, schema[qualifiedTable], columnConfig._deleted, true);
 
-		tempTables.push(qualifiedStagingTable);
+		tempTables.push(qualifiedStagingTable); //Looks like we don't actually need this due to the next line
 		tasks.push(done => client.query(`drop table if exists ${qualifiedStagingTable}`, done));
-		tasks.push(done => client.query(`drop table if exists ${qualifiedStagingTable}_changes`, done));
-		tasks.push(done => client.query(`create table ${qualifiedStagingTable} (like ${qualifiedTable})`, done));
-		tasks.push(done => client.query(`create index ${stagingTable}_id on ${qualifiedStagingTable} (${ids.join(', ')})`, done));
-		// tasks.push(done => ls.pipe(stream, client.streamToTable(qualifiedStagingTable), done));
+		if(config.version !== 'redshift') {
+			tasks.push(done => client.query(`create table ${qualifiedStagingTable} (like ${qualifiedTable})`, done));
+			tasks.push(done => client.query(`create index ${stagingTable}_id on ${qualifiedStagingTable} (${ids.join(', ')})`, done));
+		} else {
+			tasks.push(done => 
+				client.query(`create table ${qualifiedStagingTable} 
+							  diststyle all 
+							  as select *
+								 from ${qualifiedTable}
+								 limit 0;`, done));
+		};
+
 		tasks.push(done => {
 			ls.pipe(stream, ls.through((obj, done, push) => {
 				if (obj.__leo_delete__) {
@@ -181,14 +196,18 @@ module.exports = function (config, columnConfig) {
 					// Now insert any we were missing
 					tasks.push(done => {
 						connection.query(`INSERT INTO ${qualifiedTable} (${columns.join(',')},${columnConfig._deleted},${columnConfig._auditdate})
-								SELECT ${columns.map(column => `coalesce(staging.${column}, prev.${column})`)}, coalesce(prev.${columnConfig._deleted}, false), ${dwClient.auditdate} as ${columnConfig._auditdate}
+								SELECT ${columns.map(column => `staging.${column}`)},
+										false AS ${columnConfig._deleted},
+										${dwClient.auditdate} as ${columnConfig._auditdate}
 								FROM ${qualifiedStagingTable} staging
-								LEFT JOIN ${qualifiedTable} as prev on ${ids.map(id => `prev.${id} = staging.${id}`).join(' and ')}
-								WHERE prev.${ids[0]} is null	
+								WHERE NOT EXISTS (
+									SELECT *
+									FROM   ${qualifiedTable} as prev
+									WHERE ${ids.map(id => `prev.${id} = staging.${id}`).join(' and ')}
+								)	
 							`, done);
 					});
-					// tasks.push(done => connection.query(`drop table ${stagingTbl}`, done));
-
+					
 					async.series(tasks, err => {
 						if (!err) {
 							connection.query(`commit`, e => {
@@ -227,13 +246,22 @@ module.exports = function (config, columnConfig) {
 		let tasks = [];
 		let deleteHandler = deletesSetup(qualifiedTable, schema[qualifiedTable], columnConfig._enddate, dwClient.auditdate, `${columnConfig._current} = true`);
 
-		tempTables.push(qualifiedStagingTable);
+		// Prepare staging tables
 		tasks.push(done => client.query(`drop table if exists ${qualifiedStagingTable}`, done));
 		tasks.push(done => client.query(`drop table if exists ${qualifiedStagingTable}_changes`, done));
-		tasks.push(done => client.query(`create table ${qualifiedStagingTable} (like ${qualifiedTable})`, done));
-		tasks.push(done => client.query(`create index ${stagingTbl}_id on ${qualifiedStagingTable} (${nk.join(', ')})`, done));
+		if(config.version !== 'redshift') {
+			tasks.push(done => client.query(`create table ${qualifiedStagingTable} (like ${qualifiedTable})`, done));
+			tasks.push(done => client.query(`create index ${stagingTbl}_id on ${qualifiedStagingTable} (${nk.join(', ')})`, done));
+		} else {
+			tasks.push(done => 
+				client.query(`create table ${qualifiedStagingTable} 
+							  diststyle all 
+							  as select *
+								 from ${qualifiedTable}
+								 limit 0;`, done));
+		};
 		tasks.push(done => client.query(`alter table ${qualifiedStagingTable} drop column ${sk}`, done));
-		// tasks.push(done => ls.pipe(stream, client.streamToTable(qualifiedStagingTable), done));
+		
 		tasks.push(done => {
 			ls.pipe(stream, ls.through((obj, done, push) => {
 				if (obj.__leo_delete__) {
@@ -262,7 +290,6 @@ module.exports = function (config, columnConfig) {
 						return callback(err);
 					}
 
-					// let scd0 = scds[0] || []; // Not Used
 					let scd2 = scds[2] || [];
 					let scd3 = scds[3] || [];
 					let scd6 = Object.keys(scds[6] || {});
@@ -278,88 +305,102 @@ module.exports = function (config, columnConfig) {
 
 					let scdSQL = [];
 
-					// if (!scd2.length && !scd3.length && !scd6.length) {
-					//	scdSQL.push(`1 as runSCD1`);
-					// } else
-					if (scd1.length) {
-						scdSQL.push(`CASE WHEN md5(${scd1.map(f => 'md5(coalesce(s.' + f + '::text,\'\'))').join(' || ')}) = md5(${scd1.map(f => 'md5(coalesce(d.' + f + '::text,\'\'))').join(' || ')}) THEN 0 WHEN d.${nk[0]} is null then 0 ELSE 1 END as runSCD1`);
-					} else {
-						scdSQL.push(`0 as runSCD1`);
-					}
-					if (scd2.length) {
-						scdSQL.push(`CASE WHEN d.${nk[0]} is null then 1 WHEN md5(${scd2.map(f => 'md5(coalesce(s.' + f + '::text,\'\'))').join(' || ')}) = md5(${scd2.map(f => 'md5(coalesce(d.' + f + '::text,\'\'))').join(' || ')}) THEN 0 ELSE 1 END as runSCD2`);
-					} else {
-						scdSQL.push(`CASE WHEN d.${nk[0]} is null then 1 ELSE 0 END as runSCD2`);
-					}
-					if (scd3.length) {
-						scdSQL.push(`CASE WHEN md5(${scd3.map(f => 'md5(coalesce(s.' + f + '::text,\'\'))').join(' || ')}) = md5(${scd3.map(f => 'md5(coalesce(d.' + f + '::text,\'\'))').join(' || ')}) THEN 0 WHEN d.${nk[0]} is null then 0 ELSE 1 END as runSCD3`);
-					} else {
-						scdSQL.push(`0 as runSCD3`);
-					}
-					if (scd6.length) {
-						scdSQL.push(`CASE WHEN md5(${scd6.map(f => 'md5(coalesce(s.' + f + '::text,\'\'))').join(' || ')}) = md5(${scd6.map(f => 'md5(coalesce(d.' + f + '::text,\'\'))').join(' || ')}) THEN 0 WHEN d.${nk[0]} is null then 0 ELSE 1 END as runSCD6`);
-					} else {
-						scdSQL.push(`0 as runSCD6`);
-					}
+					if(!config.bypassSlowlyChangingDimensions) {
+						if (scd1.length) {
+							scdSQL.push(`CASE WHEN md5(${scd1.map(f => 'md5(coalesce(s.' + f + '::text,\'\'))').join(' || ')}) = md5(${scd1.map(f => 'md5(coalesce(d.' + f + '::text,\'\'))').join(' || ')}) THEN 0 WHEN d.${nk[0]} is null then 0 ELSE 1 END as runSCD1`);
+						} else {
+							scdSQL.push(`0 as runSCD1`);
+						}
+						if (scd2.length) {
+							scdSQL.push(`CASE WHEN d.${nk[0]} is null then 1 WHEN md5(${scd2.map(f => 'md5(coalesce(s.' + f + '::text,\'\'))').join(' || ')}) = md5(${scd2.map(f => 'md5(coalesce(d.' + f + '::text,\'\'))').join(' || ')}) THEN 0 ELSE 1 END as runSCD2`);
+						} else {
+							scdSQL.push(`CASE WHEN d.${nk[0]} is null then 1 ELSE 0 END as runSCD2`);
+						}
+						if (scd3.length) {
+							scdSQL.push(`CASE WHEN md5(${scd3.map(f => 'md5(coalesce(s.' + f + '::text,\'\'))').join(' || ')}) = md5(${scd3.map(f => 'md5(coalesce(d.' + f + '::text,\'\'))').join(' || ')}) THEN 0 WHEN d.${nk[0]} is null then 0 ELSE 1 END as runSCD3`);
+						} else {
+							scdSQL.push(`0 as runSCD3`);
+						}
+						if (scd6.length) {
+							scdSQL.push(`CASE WHEN md5(${scd6.map(f => 'md5(coalesce(s.' + f + '::text,\'\'))').join(' || ')}) = md5(${scd6.map(f => 'md5(coalesce(d.' + f + '::text,\'\'))').join(' || ')}) THEN 0 WHEN d.${nk[0]} is null then 0 ELSE 1 END as runSCD6`);
+						} else {
+							scdSQL.push(`0 as runSCD6`);
+						}
+					};
 
-					// let's figure out which SCDs needs to happen
-					tempTables.push(`${qualifiedStagingTable}_changes`);
 					connection.query(`create table ${qualifiedStagingTable}_changes as 
-				select ${nk.map(id => `s.${id}`).join(', ')}, d.${nk[0]} is null as isNew,
-					${scdSQL.join(',\n')}
-					FROM ${qualifiedStagingTable} s
-					LEFT JOIN ${qualifiedTable} d on ${nk.map(id => `d.${id} = s.${id}`).join(' and ')} and d.${columnConfig._current}`, (err) => {
+									  	select ${nk.map(id => `s.${id}`).join(', ')}, 
+											   d.${nk[0]} is null as isNew
+										${!config.bypassSlowlyChangingDimensions ? `,${scdSQL.join(',\n')}` : ``}
+										FROM ${qualifiedStagingTable} s
+										LEFT JOIN ${qualifiedTable} d on ${nk.map(id => `d.${id} = s.${id}`).join(' and ')} and d.${columnConfig._current}`, (err) => {
 						if (err) {
 							logger.error(err);
 							process.exit();
 						}
+
 						let tasks = [];
 						let rowId = null;
 						let totalRecords = 0;
 						tasks.push(done => connection.query(`analyze ${qualifiedStagingTable}_changes`, done));
-						tasks.push(done => {
-							connection.query(`select max(${sk}) as maxid from ${qualifiedTable}`, (err, results) => {
-								if (err) {
-									return done(err);
-								}
-								rowId = results[0].maxid || 10000;
-								totalRecords = (rowId - 10000);
-								done();
-							});
-						});
 
 						// The following code relies on the fact that now() will return the same time during all transaction events
 						tasks.push(done => connection.query(`Begin Transaction`, done));
 
-						tasks.push(done => {
-							let fields = [sk].concat(allColumns).concat([columnConfig._auditdate, columnConfig._startdate, columnConfig._enddate, columnConfig._current]);
-							connection.query(`INSERT INTO ${qualifiedTable} (${fields.join(',')})
-								SELECT row_number() over () + ${rowId}, ${allColumns.map(column => `coalesce(staging.${column}, prev.${column})`)}, ${dwClient.auditdate} as ${columnConfig._auditdate}, case when changes.isNew then '1900-01-01 00:00:00' else now() END as ${columnConfig._startdate}, '9999-01-01 00:00:00' as ${columnConfig._enddate}, true as ${columnConfig._current}
-								FROM ${qualifiedStagingTable}_changes changes  
-								JOIN ${qualifiedStagingTable} staging on ${nk.map(id => `staging.${id} = changes.${id}`).join(' and ')}
-								LEFT JOIN ${qualifiedTable} as prev on ${nk.map(id => `prev.${id} = changes.${id}`).join(' and ')} and prev.${columnConfig._current}
-								WHERE (changes.runSCD2 =1 OR changes.runSCD6=1)		
-								`, done);
-						});
-
+						let fields = [sk].concat(allColumns).concat([columnConfig._auditdate, columnConfig._startdate, columnConfig._enddate, columnConfig._current]);
+						
+						if(!config.bypassSlowlyChangingDimensions) {
+							tasks.push(done => {
+								let fields = [sk].concat(allColumns).concat([columnConfig._auditdate, columnConfig._startdate, columnConfig._enddate, columnConfig._current]);
+								connection.query(`INSERT INTO ${qualifiedTable} (${fields.join(',')})
+									SELECT row_number() over () + ${rowId}, ${allColumns.map(column => `coalesce(staging.${column}, prev.${column})`)}, ${dwClient.auditdate} as ${columnConfig._auditdate}, case when changes.isNew then '1900-01-01 00:00:00' else now() END as ${columnConfig._startdate}, '9999-01-01 00:00:00' as ${columnConfig._enddate}, true as ${columnConfig._current}
+									FROM ${qualifiedStagingTable}_changes changes  
+									JOIN ${qualifiedStagingTable} staging on ${nk.map(id => `staging.${id} = changes.${id}`).join(' and ')}
+									LEFT JOIN ${qualifiedTable} as prev on ${nk.map(id => `prev.${id} = changes.${id}`).join(' and ')} and prev.${columnConfig._current}
+									WHERE (changes.runSCD2 =1 OR changes.runSCD6=1)		
+									`, done);
+							});
+						} else if (config.hashedSurrogateKeys) {
+							tasks.push(done => {
+								connection.query(`INSERT INTO ${qualifiedTable} (${fields.join(',')})
+									SELECT farmFingerPrint64(${nk.map(id => `staging.${id}`).join('-')}), ${allColumns.map(column => `coalesce(staging.${column}, prev.${column})`)}, ${dwClient.auditdate} as ${columnConfig._auditdate}, case when changes.isNew then '1900-01-01 00:00:00' else now() END as ${columnConfig._startdate}, '9999-01-01 00:00:00' as ${columnConfig._enddate}, true as ${columnConfig._current}
+									FROM ${qualifiedStagingTable}_changes changes  
+									JOIN ${qualifiedStagingTable} staging on ${nk.map(id => `staging.${id} = changes.${id}`).join(' and ')}
+									LEFT JOIN ${qualifiedTable} as prev on ${nk.map(id => `prev.${id} = changes.${id}`).join(' and ')} and prev.${columnConfig._current}
+									WHERE isNew = true		
+									`, done);
+							});
+						};
+						
 						// This needs to be done last
-						tasks.push(done => {
-							// RUN SCD1 / SCD6 columns  (where we update the old records)
-							let columns = scd1.map(column => `"${column}" = coalesce(staging."${column}", prev."${column}")`).concat(scd6.map(column => `"current_${column}" = coalesce(staging."${column}", prev."${column}")`));
-							columns.push(`"${columnConfig._enddate}" = case when changes.runSCD2 =1 then now() else prev."${columnConfig._enddate}" END`);
-							columns.push(`"${columnConfig._current}" = case when changes.runSCD2 =1 then false else prev."${columnConfig._current}" END`);
-							columns.push(`"${columnConfig._auditdate}" = ${dwClient.auditdate}`);
-							connection.query(`update ${qualifiedTable} as prev
-										set  ${columns.join(', ')}
-										FROM ${qualifiedStagingTable}_changes changes
-										JOIN ${qualifiedStagingTable} staging on ${nk.map(id => `staging.${id} = changes.${id}`).join(' and ')}
-										where ${nk.map(id => `prev.${id} = changes.${id}`).join(' and ')} and prev.${columnConfig._startdate} != now() and changes.isNew = false /*Need to make sure we are only updating the ones not just inserted through SCD2 otherwise we run into issues with multiple rows having .${columnConfig._current}*/
-											and (changes.runSCD1=1 OR  changes.runSCD6=1 OR changes.runSCD2=1)
-										`, done);
-						});
+						if(!config.bypassSlowlyChangingDimensions) {
+							tasks.push(done => {
+								// RUN SCD1 / SCD6 columns  (where we update the old records)
+								let columns = scd1.map(column => `"${column}" = coalesce(staging."${column}", prev."${column}")`).concat(scd6.map(column => `"current_${column}" = coalesce(staging."${column}", prev."${column}")`));
+								columns.push(`"${columnConfig._enddate}" = case when changes.runSCD2 =1 then now() else prev."${columnConfig._enddate}" END`);
+								columns.push(`"${columnConfig._current}" = case when changes.runSCD2 =1 then false else prev."${columnConfig._current}" END`);
+								columns.push(`"${columnConfig._auditdate}" = ${dwClient.auditdate}`);
+								connection.query(`update ${qualifiedTable} as prev
+											set  ${columns.join(', ')}
+											FROM ${qualifiedStagingTable}_changes changes
+											JOIN ${qualifiedStagingTable} staging on ${nk.map(id => `staging.${id} = changes.${id}`).join(' and ')}
+											where ${nk.map(id => `prev.${id} = changes.${id}`).join(' and ')} and prev.${columnConfig._startdate} != now() and changes.isNew = false /*Need to make sure we are only updating the ones not just inserted through SCD2 otherwise we run into issues with multiple rows having .${columnConfig._current}*/
+												and (changes.runSCD1=1 OR  changes.runSCD6=1 OR changes.runSCD2=1)
+											`, done);
+							});
+						} else if(config.hashedSurrogateKeys) {
+							tasks.push(done => {
+								let columns = scd1.map(column => `"${column}" = coalesce(staging."${column}", prev."${column}")`).concat(scd6.map(column => `"current_${column}" = coalesce(staging."${column}", prev."${column}")`));
+								columns.push(`"${columnConfig._auditdate}" = ${dwClient.auditdate}`);
+								connection.query(`update ${qualifiedTable} as prev
+											set  ${columns.join(', ')}
+											FROM ${qualifiedStagingTable}_changes changes
+											JOIN ${qualifiedStagingTable} staging on ${nk.map(id => `staging.${id} = changes.${id}`).join(' and ')}
+											WHERE ${nk.map(id => `prev.${id} = changes.${id}`).join(' and ')} and prev.${columnConfig._startdate} != now() and changes.isNew = false
+											`, done);
+							});
+						};
 
-						// tasks.push(done => connection.query(`drop table ${qualifiedStagingTable}_changes`, done));
-						// tasks.push(done => connection.query(`drop table ${qualifiedStagingTable}`, done));
 						async.series(tasks, err => {
 							if (!err) {
 								connection.query(`commit`, e => {
@@ -382,62 +423,66 @@ module.exports = function (config, columnConfig) {
 	};
 
 	client.insertMissingDimensions = function (usedTables, tableConfig, tableSks, tableNks, callback) {
-		let unions = {};
-		let isDate = {
-			d_date: true,
-			d_datetime: true,
-			d_time: true,
-			date: true,
-			datetime: true,
-			dim_date: true,
-			dim_datetime: true,
-			dim_time: true,
-			time: true,
-		};
-		Object.keys(usedTables).map(table => {
-			Object.keys(tableConfig[table].structure).map(column => {
-				let field = tableConfig[table].structure[column];
-				if (field.dimension && !isDate[field.dimension]) {
-					if (!(unions[field.dimension])) {
-						unions[field.dimension] = [];
-					}
-					if (typeof tableNks[field.dimension] === 'undefined') {
-						throw new Error(`${field.dimension} not found in tableNks`);
+		if(config.hashedSurrogateKeys) {
+			callback(null);
+		} else {
+			let unions = {};
+			let isDate = {
+				d_date: true,
+				d_datetime: true,
+				d_time: true,
+				date: true,
+				datetime: true,
+				dim_date: true,
+				dim_datetime: true,
+				dim_time: true,
+				time: true,
+			};
+			Object.keys(usedTables).map(table => {
+				Object.keys(tableConfig[table].structure).map(column => {
+					let field = tableConfig[table].structure[column];
+					if (field.dimension && !isDate[field.dimension]) {
+						if (!(unions[field.dimension])) {
+							unions[field.dimension] = [];
+						}
+						if (typeof tableNks[field.dimension] === 'undefined') {
+							throw new Error(`${field.dimension} not found in tableNks`);
+						}
+						let dimTableNk = tableNks[field.dimension][0];
+						unions[field.dimension].push(`select ${table}.${column} as id from ${table} left join ${field.dimension} on ${field.dimension}.${dimTableNk} = ${table}.${column} where ${field.dimension}.${dimTableNk} is null and ${table}.${columnConfig._auditdate} = ${dwClient.auditdate}`);
 					}
-					let dimTableNk = tableNks[field.dimension][0];
-					unions[field.dimension].push(`select ${table}.${column} as id from ${table} left join ${field.dimension} on ${field.dimension}.${dimTableNk} = ${table}.${column} where ${field.dimension}.${dimTableNk} is null and ${table}.${columnConfig._auditdate} = ${dwClient.auditdate}`);
-				}
+				});
 			});
-		});
-		let missingDimTasks = Object.keys(unions).map(table => {
-			let sk = tableSks[table];
-			let nk = tableNks[table][0];
-			return (callback) => {
-				let done = (err, data) => {
-					trx && trx.release();
-					callback(err, data);
-				};
-				let trx;
-				client.connect().then(transaction => {
-					trx = transaction;
-					transaction.query(`select max(${sk}) as maxid from ${table}`, (err, results) => {
-						if (err) {
-							return done(err);
-						}
-						let rowId = results[0].maxid || 10000;
-						let _auditdate = dwClient.auditdate; // results[0]._auditdate ? `'${results[0]._auditdate.replace(/^\d* */,"")}'` : "now()";
-						let unionQuery = unions[table].join('\nUNION\n');
-						transaction.query(`insert into ${table} (${sk}, ${nk}, ${columnConfig._auditdate}, ${columnConfig._startdate}, ${columnConfig._enddate}, ${columnConfig._current}) select row_number() over () + ${rowId}, sub.id, max(${_auditdate})::timestamp, '1900-01-01 00:00:00', '9999-01-01 00:00:00', true from (${unionQuery}) as sub where sub.id is not null group by sub.id`, (err) => {
-							done(err);
+			let missingDimTasks = Object.keys(unions).map(table => {
+				let sk = tableSks[table];
+				let nk = tableNks[table][0];
+				return (callback) => {
+					let done = (err, data) => {
+						trx && trx.release();
+						callback(err, data);
+					};
+					let trx;
+					client.connect().then(transaction => {
+						trx = transaction;
+						transaction.query(`select max(${sk}) as maxid from ${table}`, (err, results) => {
+							if (err) {
+								return done(err);
+							}
+							let rowId = results[0].maxid || 10000;
+							let _auditdate = dwClient.auditdate; // results[0]._auditdate ? `'${results[0]._auditdate.replace(/^\d* */,"")}'` : "now()";
+							let unionQuery = unions[table].join('\nUNION\n');
+							transaction.query(`insert into ${table} (${sk}, ${nk}, ${columnConfig._auditdate}, ${columnConfig._startdate}, ${columnConfig._enddate}, ${columnConfig._current}) select row_number() over () + ${rowId}, sub.id, max(${_auditdate})::timestamp, '1900-01-01 00:00:00', '9999-01-01 00:00:00', true from (${unionQuery}) as sub where sub.id is not null group by sub.id`, (err) => {
+								done(err);
+							});
 						});
-					});
-				}).catch(done);
-			};
-		});
-		async.parallelLimit(missingDimTasks, 10, (missingDimError) => {
-			logger.info(`Missing Dimensions ${!missingDimError && 'Inserted'} ----------------------------`, missingDimError || '');
-			callback(missingDimError);
-		});
+					}).catch(done);
+				};
+			});
+			async.parallelLimit(missingDimTasks, 10, (missingDimError) => {
+				logger.info(`Missing Dimensions ${!missingDimError && 'Inserted'} ----------------------------`, missingDimError || '');
+				callback(missingDimError);
+			});
+		};
 	};
 
 	client.linkDimensions = function (table, links, nk, callback, tableStatus) {
@@ -449,7 +494,7 @@ module.exports = function (config, columnConfig) {
 
 			// Only run analyze on the table if this is the first load
 			if (tableStatus === 'First Load') {
-				tasks.push(done => client.query(`analyze ${table}`, done));
+				tasks.push(done => client.query(`ANALYZE ${table}`, done));
 			}
 			tasks.push(done => {
 				let joinTables = links.map(link => {
@@ -461,26 +506,32 @@ module.exports = function (config, columnConfig) {
 					} else if (columnConfig.useSurrogateDateKeys && (link.table === 'd_time' || link.table === 'time' || link.table === 'dim_time')) {
 						sets.push(`${link.destination}_time = coalesce(EXTRACT(EPOCH from t.${link.source}::time) + 10000, 1)`);
 					} else {
-						sets.push(`${link.destination} = coalesce(${link.join_id}_join_table.${link.sk}, 1)`);
-						var joinOn = `${link.join_id}_join_table.${link.on} = t.${link.source}`;
-						if (Array.isArray(link.source)) {
-							joinOn = link.source.map((v, i) => `${link.join_id}_join_table.${link.on[i]} = t.${v}`).join(' AND ');
-						}
-						return `LEFT JOIN ${link.table} ${link.join_id}_join_table
-                            on ${joinOn} 
-                                and t.${link.link_date} >= ${link.join_id}_join_table.${columnConfig._startdate}
-                                and (t.${link.link_date} <= ${link.join_id}_join_table.${columnConfig._enddate} or ${link.join_id}_join_table.${columnConfig._current})`;
+						if(config.hashedSurrogateKeys) {
+							sets.push(`${link.destination} = coalesce(farmFingerPrint64(t.${link.source}), 1)`);
+							return ``;
+						} else {
+							var joinOn = `${link.join_id}_join_table.${link.on} = t.${link.source}`;
+
+							if (Array.isArray(link.source)) {
+								joinOn = link.source.map((v, i) => `${link.join_id}_join_table.${link.on[i]} = t.${v}`).join(' AND ');
+							}
+							return `LEFT JOIN ${link.table} ${link.join_id}_join_table
+								ON ${joinOn} 
+									AND t.${link.link_date} >= ${link.join_id}_join_table.${columnConfig._startdate}
+									AND (t.${link.link_date} <= ${link.join_id}_join_table.${columnConfig._enddate} or ${link.join_id}_join_table.${columnConfig._current})`;
+						};
 					}
 				});
 
 				if (sets.length) {
-					client.query(`Update ${table} dm
+					client.query(`UPDATE ${table} dm
                         SET  ${sets.join(', ')}, ${columnConfig._auditdate} = ${linkAuditdate}
                         FROM ${table} t
-                        ${joinTables.join('\n')}
-                        where ${nk.map(id => `dm.${id} = t.${id}`).join(' and ')}
+						${joinTables.join('\n')}
+                        WHERE ${nk.map(id => `dm.${id} = t.${id}`).join(' AND ')}
 							AND dm.${columnConfig._auditdate} = ${dwClient.auditdate} AND t.${columnConfig._auditdate} = ${dwClient.auditdate}
                     `, done);
+					// removed join logic from above ${joinTables.join('\n')}
 				} else {
 					done();
 				}

From e32db11c2ce16d29d3d451a2a4f1561a4d4b5ede Mon Sep 17 00:00:00 2001
From: Michael Scranton <mscranton@commercehub.com>
Date: Tue, 18 Oct 2022 15:45:17 -0400
Subject: [PATCH 02/62] s3 load, sysdate

---
 postgres/lib/connect.js   | 168 +++++++++++++++++++++++++++++++++++++-
 postgres/lib/dwconnect.js | 133 ++++++++++++++++++++----------
 2 files changed, 258 insertions(+), 43 deletions(-)

diff --git a/postgres/lib/connect.js b/postgres/lib/connect.js
index 9815bda4..9d629cb4 100644
--- a/postgres/lib/connect.js
+++ b/postgres/lib/connect.js
@@ -571,8 +571,172 @@ function create (pool, parentCache) {
 				records: opts.records,
 			});
 		},
-		streamToTableFromS3: () => {
-			// opts = Object.assign({}, opts || {});
+		streamToTableFromS3: (table, config) => {
+			const ts = table.split('.');
+			let schema = 'public';
+			let shortTable = table;
+			if (ts.length > 1) {
+				schema = ts[0];
+				shortTable = ts[1];
+			}
+
+			let columns = [];
+			let stream;
+			let myClient = null;
+			let pending = null;
+			let ended = false;
+			let csvopts = { delimiter: '|' };
+			let keepS3Files = config.keepS3Files != null ? config.keepS3Files : false;
+
+			// Get prefix for S3 file path
+			let s3prefix =
+					config.s3prefix ||
+					process.env.AWS_LAMBDA_FUNCTION_NAME ||
+					'dw_redshift_ingest';
+			s3prefix = s3prefix.replace(/^\/*(.*?)\/*$/, '$1'); // Remove leading and trailing '/'
+
+			// clean audit date to use in S3 file path
+			let cleanAuditDate = client.auditdate
+				.replace(/'/g, '')
+				.replace(/:/g, '-');
+			let s3FileName = `files/${s3prefix}/${cleanAuditDate}/${table}.csv`;
+
+			client.connect().then(
+				c => {
+					client
+						.describeTable(shortTable, schema)
+						.then(result => {
+							if (ended) {
+								c.release(true);
+								return;
+							}
+							columns = result.map(f => f.column_name);
+							myClient = c;
+
+							stream = ls.toS3(leo.configuration.resources.LeoS3, s3FileName);
+							// copyFrom uses `end` but s3 `finish` so pipe finish to end
+							stream.on('finish', () => stream.emit('end'));
+
+							stream.on('error', function(err) {
+								logger.error(`COPY error: ${err.where}`, err);
+								process.exit();
+							});
+							if (pending) {
+								pending();
+							}
+						})
+						.catch(err => {
+							logger.error(err);
+						});
+				},
+				err => {
+					logger.error(err);
+				}
+			);
+
+			let count = 0;
+
+			function nonNull(v) {
+				if (v === '' || v === null || v === undefined) {
+					return '\\N';
+				} else if (typeof v === 'string' && v.search(/\r/) !== -1) {
+					return v.replace(/\r\n?/g, '\n');
+				} else {
+					return v;
+				}
+			}
+
+			
+			return ls.pipeline(
+				csv.createWriteStream({
+					...csvopts,
+					headers: false,
+					transform: (row, done) => {
+						if (!myClient) {
+							pending = () => {
+								done(
+									null,
+									columns.map(f => nonNull(row[f]))
+								);
+							};
+						} else {
+							done(
+								null,
+								columns.map(f => nonNull(row[f]))
+							);
+						}
+					},
+				}),
+				ls.write(
+					(r, done) => {
+						count++;
+						if (count % 10000 === 0) {
+							logger.info(table + ': ' + count);
+						}
+						if (!stream.write(r)) {
+							stream.once('drain', done);
+						} else {
+							done(null);
+						}
+					},
+					done => {
+						ended = true;
+						logger.debug(table + ': stream done');
+						if (stream) {
+							stream.on('end', err => {
+								logger.debug(table + ': stream ended', err || '');
+
+								// wrap done callback to release the connection
+								function innerDone(err) {
+									myClient.release(true);
+									logger.debug(table + ': stream client released', err || '');
+									done(err);
+								}
+
+								if (err) {
+									innerDone(err);
+								} else {
+									// Once the S3 file is complete run copy to load the staging table
+									let f = columns.map(f => `"${f}"`);
+									let file = `s3://${leo.configuration.s3}/${s3FileName}`;
+									let manifest = '';
+									let role = dbconfig.loaderRole;
+									myClient.query(
+										`copy ${table} (${f}) from '${file}' ${manifest} ${role ? `credentials 'aws_iam_role=${role}'` : ''
+										} NULL AS '\\\\N' format csv DELIMITER '|' ACCEPTINVCHARS TRUNCATECOLUMNS ACCEPTANYDATE TIMEFORMAT 'YYYY-MM-DD HH:MI:SS' COMPUPDATE OFF`,
+										copyErr => {
+											if (keepS3Files) {
+												innerDone(copyErr);
+											} else {
+												// Delete the S3 files when done
+												ls.s3.deleteObject(
+													{
+														Bucket: leo.configuration.s3,
+														Key: s3FileName,
+													},
+													deleteError => {
+														if (deleteError) {
+															logger.info(
+																'file failed to delete:',
+																s3FileName,
+																deleteError
+															);
+														}
+														innerDone(copyErr);
+													}
+												);
+											}
+										}
+									);
+								} 
+							});
+							stream.end();
+						} else {
+							done();
+						}
+					}
+				)
+			);
 		},
 	};
 
diff --git a/postgres/lib/dwconnect.js b/postgres/lib/dwconnect.js
index 613fadee..cc02db6a 100644
--- a/postgres/lib/dwconnect.js
+++ b/postgres/lib/dwconnect.js
@@ -142,24 +142,45 @@ module.exports = function (config, columnConfig) {
 								 limit 0;`, done));
 		};
 
-		tasks.push(done => {
-			ls.pipe(stream, ls.through((obj, done, push) => {
-				if (obj.__leo_delete__) {
-					if (obj.__leo_delete__ === 'id') {
-						push(obj);
+		if (config.version !== 'redshift') {
+			tasks.push(done => {
+				ls.pipe(stream, ls.through((obj, done, push) => {
+					if (obj.__leo_delete__) {
+						if (obj.__leo_delete__ === 'id') {
+							push(obj);
+						}
+						deleteHandler.add(obj, done);
+					} else {
+						done(null, obj);
 					}
-					deleteHandler.add(obj, done);
-				} else {
-					done(null, obj);
-				}
-			}), client.streamToTable(qualifiedStagingTable), (err) => {
-				if (err) {
-					return done(err);
-				} else {
-					deleteHandler.flush(done);
-				}
+				}), client.streamToTable(qualifiedStagingTable), (err) => {
+					if (err) {
+						return done(err);
+					} else {
+						deleteHandler.flush(done);
+					}
+				});
 			});
-		});
+		} else {
+			tasks.push(done => {
+				ls.pipe(stream, ls.through((obj, done, push) => {
+					if (obj.__leo_delete__) {
+						if (obj.__leo_delete__ === 'id') {
+							push(obj);
+						}
+						deleteHandler.add(obj, done);
+					} else {
+						done(null, obj);
+					}
+				}), client.streamToTableFromS3(qualifiedStagingTable, config), (err) => {
+					if (err) {
+						return done(err);
+					} else {
+						deleteHandler.flush(done);
+					}
+				});
+			});
+		};
 
 		tasks.push(done => client.query(`analyze ${qualifiedStagingTable}`, done));
 
@@ -174,7 +195,7 @@ module.exports = function (config, columnConfig) {
 
 					let tasks = [];
 					let totalRecords = 0;
-					// The following code relies on the fact that now() will return the same time during all transaction events
+					// The following code relies on the fact that now()/sysdate will return the same time during all transaction events
 					tasks.push(done => connection.query(`Begin Transaction`, done));
 					tasks.push(done => {
 						connection.query(`select 1 as total from ${qualifiedTable} limit 1`, (err, results) => {
@@ -262,24 +283,45 @@ module.exports = function (config, columnConfig) {
 		};
 		tasks.push(done => client.query(`alter table ${qualifiedStagingTable} drop column ${sk}`, done));
 		
-		tasks.push(done => {
-			ls.pipe(stream, ls.through((obj, done, push) => {
-				if (obj.__leo_delete__) {
-					if (obj.__leo_delete__ === 'id') {
-						push(obj);
+		if (config.version !== 'redshift') {
+			tasks.push(done => {
+				ls.pipe(stream, ls.through((obj, done, push) => {
+					if (obj.__leo_delete__) {
+						if (obj.__leo_delete__ === 'id') {
+							push(obj);
+						}
+						deleteHandler.add(obj, done);
+					} else {
+						done(null, obj);
 					}
-					deleteHandler.add(obj, done);
-				} else {
-					done(null, obj);
-				}
-			}), client.streamToTable(qualifiedStagingTable), (err) => {
-				if (err) {
-					return done(err);
-				} else {
-					deleteHandler.flush(done);
-				}
+				}), client.streamToTable(qualifiedStagingTable), (err) => {
+					if (err) {
+						return done(err);
+					} else {
+						deleteHandler.flush(done);
+					}
+				});
 			});
-		});
+		} else {
+			tasks.push(done => {
+				ls.pipe(stream, ls.through((obj, done, push) => {
+					if (obj.__leo_delete__) {
+						if (obj.__leo_delete__ === 'id') {
+							push(obj);
+						}
+						deleteHandler.add(obj, done);
+					} else {
+						done(null, obj);
+					}
+				}), client.streamToTableFromS3(qualifiedStagingTable, config.s3prefix, config.keepS3Filess), (err) => {
+					if (err) {
+						return done(err);
+					} else {
+						deleteHandler.flush(done);
+					}
+				});
+			});
+		};
 
 		tasks.push(done => client.query(`analyze ${qualifiedStagingTable}`, done));
 
@@ -344,7 +386,7 @@ module.exports = function (config, columnConfig) {
 						let totalRecords = 0;
 						tasks.push(done => connection.query(`analyze ${qualifiedStagingTable}_changes`, done));
 
-						// The following code relies on the fact that now() will return the same time during all transaction events
+						// The following code relies on the fact that now()/sysdate will return the same time during all transaction events
 						tasks.push(done => connection.query(`Begin Transaction`, done));
 
 						let fields = [sk].concat(allColumns).concat([columnConfig._auditdate, columnConfig._startdate, columnConfig._enddate, columnConfig._current]);
@@ -353,7 +395,12 @@ module.exports = function (config, columnConfig) {
 							tasks.push(done => {
 								let fields = [sk].concat(allColumns).concat([columnConfig._auditdate, columnConfig._startdate, columnConfig._enddate, columnConfig._current]);
 								connection.query(`INSERT INTO ${qualifiedTable} (${fields.join(',')})
-									SELECT row_number() over () + ${rowId}, ${allColumns.map(column => `coalesce(staging.${column}, prev.${column})`)}, ${dwClient.auditdate} as ${columnConfig._auditdate}, case when changes.isNew then '1900-01-01 00:00:00' else now() END as ${columnConfig._startdate}, '9999-01-01 00:00:00' as ${columnConfig._enddate}, true as ${columnConfig._current}
+									SELECT row_number() over () + ${rowId}, 
+										   ${allColumns.map(column => `coalesce(staging.${column}, prev.${column})`)}, 
+										   ${dwClient.auditdate} as ${columnConfig._auditdate}, 
+										   case when changes.isNew then '1900-01-01 00:00:00' else ${config.verion === 'redshift' ? 'sysdate' : 'now()'} END as ${columnConfig._startdate}, 
+										   '9999-01-01 00:00:00' as ${columnConfig._enddate}, 
+										   true as ${columnConfig._current}
 									FROM ${qualifiedStagingTable}_changes changes  
 									JOIN ${qualifiedStagingTable} staging on ${nk.map(id => `staging.${id} = changes.${id}`).join(' and ')}
 									LEFT JOIN ${qualifiedTable} as prev on ${nk.map(id => `prev.${id} = changes.${id}`).join(' and ')} and prev.${columnConfig._current}
@@ -363,7 +410,11 @@ module.exports = function (config, columnConfig) {
 						} else if (config.hashedSurrogateKeys) {
 							tasks.push(done => {
 								connection.query(`INSERT INTO ${qualifiedTable} (${fields.join(',')})
-									SELECT farmFingerPrint64(${nk.map(id => `staging.${id}`).join('-')}), ${allColumns.map(column => `coalesce(staging.${column}, prev.${column})`)}, ${dwClient.auditdate} as ${columnConfig._auditdate}, case when changes.isNew then '1900-01-01 00:00:00' else now() END as ${columnConfig._startdate}, '9999-01-01 00:00:00' as ${columnConfig._enddate}, true as ${columnConfig._current}
+									SELECT farmFingerPrint64(${nk.map(id => `staging.${id}`).join('-')}), 
+										   ${allColumns.map(column => `coalesce(staging.${column}, prev.${column})`)}, 
+										   ${dwClient.auditdate} as ${columnConfig._auditdate}, 
+										   case when changes.isNew then '1900-01-01 00:00:00' else ${config.verion === 'redshift' ? 'sysdate' : 'now()'} END as ${columnConfig._startdate}, 
+										   '9999-01-01 00:00:00' as ${columnConfig._enddate}, true as ${columnConfig._current}
 									FROM ${qualifiedStagingTable}_changes changes  
 									JOIN ${qualifiedStagingTable} staging on ${nk.map(id => `staging.${id} = changes.${id}`).join(' and ')}
 									LEFT JOIN ${qualifiedTable} as prev on ${nk.map(id => `prev.${id} = changes.${id}`).join(' and ')} and prev.${columnConfig._current}
@@ -377,14 +428,14 @@ module.exports = function (config, columnConfig) {
 							tasks.push(done => {
 								// RUN SCD1 / SCD6 columns  (where we update the old records)
 								let columns = scd1.map(column => `"${column}" = coalesce(staging."${column}", prev."${column}")`).concat(scd6.map(column => `"current_${column}" = coalesce(staging."${column}", prev."${column}")`));
-								columns.push(`"${columnConfig._enddate}" = case when changes.runSCD2 =1 then now() else prev."${columnConfig._enddate}" END`);
+								columns.push(`"${columnConfig._enddate}" = case when changes.runSCD2 =1 then ${config.verion === 'redshift' ? 'sysdate' : 'now()'} else prev."${columnConfig._enddate}" END`);
 								columns.push(`"${columnConfig._current}" = case when changes.runSCD2 =1 then false else prev."${columnConfig._current}" END`);
 								columns.push(`"${columnConfig._auditdate}" = ${dwClient.auditdate}`);
 								connection.query(`update ${qualifiedTable} as prev
 											set  ${columns.join(', ')}
 											FROM ${qualifiedStagingTable}_changes changes
 											JOIN ${qualifiedStagingTable} staging on ${nk.map(id => `staging.${id} = changes.${id}`).join(' and ')}
-											where ${nk.map(id => `prev.${id} = changes.${id}`).join(' and ')} and prev.${columnConfig._startdate} != now() and changes.isNew = false /*Need to make sure we are only updating the ones not just inserted through SCD2 otherwise we run into issues with multiple rows having .${columnConfig._current}*/
+											where ${nk.map(id => `prev.${id} = changes.${id}`).join(' and ')} and prev.${columnConfig._startdate} != ${config.verion === 'redshift' ? 'sysdate' : 'now()'} and changes.isNew = false /*Need to make sure we are only updating the ones not just inserted through SCD2 otherwise we run into issues with multiple rows having .${columnConfig._current}*/
 												and (changes.runSCD1=1 OR  changes.runSCD6=1 OR changes.runSCD2=1)
 											`, done);
 							});
@@ -396,7 +447,7 @@ module.exports = function (config, columnConfig) {
 											set  ${columns.join(', ')}
 											FROM ${qualifiedStagingTable}_changes changes
 											JOIN ${qualifiedStagingTable} staging on ${nk.map(id => `staging.${id} = changes.${id}`).join(' and ')}
-											WHERE ${nk.map(id => `prev.${id} = changes.${id}`).join(' and ')} and prev.${columnConfig._startdate} != now() and changes.isNew = false
+											WHERE ${nk.map(id => `prev.${id} = changes.${id}`).join(' and ')} and prev.${columnConfig._startdate} != ${config.verion === 'redshift' ? 'sysdate' : 'now()'} and changes.isNew = false
 											`, done);
 							});
 						};
@@ -469,7 +520,7 @@ module.exports = function (config, columnConfig) {
 								return done(err);
 							}
 							let rowId = results[0].maxid || 10000;
-							let _auditdate = dwClient.auditdate; // results[0]._auditdate ? `'${results[0]._auditdate.replace(/^\d* */,"")}'` : "now()";
+							let _auditdate = dwClient.auditdate;
 							let unionQuery = unions[table].join('\nUNION\n');
 							transaction.query(`insert into ${table} (${sk}, ${nk}, ${columnConfig._auditdate}, ${columnConfig._startdate}, ${columnConfig._enddate}, ${columnConfig._current}) select row_number() over () + ${rowId}, sub.id, max(${_auditdate})::timestamp, '1900-01-01 00:00:00', '9999-01-01 00:00:00', true from (${unionQuery}) as sub where sub.id is not null group by sub.id`, (err) => {
 								done(err);
@@ -714,7 +765,7 @@ module.exports = function (config, columnConfig) {
 			// Add empty row to new dim
 			defaults = defaults.concat([{
 				column: columnConfig._auditdate,
-				value: 'now()',
+				value: config.verion === 'redshift' ? 'sysdate' : 'now()',
 			}, {
 				column: columnConfig._startdate,
 				value: client.escapeValueNoToLower('1900-01-01 00:00:00'),

From b6b5b0320049f0747572b2c39ff79647a9da16d8 Mon Sep 17 00:00:00 2001
From: Michael Scranton <mscranton@commercehub.com>
Date: Thu, 20 Oct 2022 11:11:49 -0400
Subject: [PATCH 03/62] bump version

---
 postgres/lib/connect.js   |  8 +++++---
 postgres/lib/dwconnect.js | 14 +++++++-------
 postgres/package.json     |  2 +-
 3 files changed, 13 insertions(+), 11 deletions(-)

diff --git a/postgres/lib/connect.js b/postgres/lib/connect.js
index 9d629cb4..b32c2181 100644
--- a/postgres/lib/connect.js
+++ b/postgres/lib/connect.js
@@ -11,14 +11,16 @@ var copyFrom = require('pg-copy-streams').from;
 var copyTo = require('pg-copy-streams').to;
 let csv = require('fast-csv');
 
+// need to confirm there is no issue adding these dependencies
+const leo = require('leo-sdk');
+const ls = leo.streams;
+
 require('pg').types.setTypeParser(1114, (val) => {
 	val += 'Z';
 	logger.debug(val);
 	return moment(val).unix() + '  ' + moment(val).utc().format();
 });
 
-const ls = require('leo-sdk').streams;
-
 let queryCount = 0;
 module.exports = function (config) {
 	const pool = new Pool(Object.assign({
@@ -700,7 +702,7 @@ function create (pool, parentCache) {
 									let f = columns.map(f => `"${f}"`);
 									let file = `s3://${leo.configuration.s3}/${s3FileName}`;
 									let manifest = '';
-									let role = dbconfig.loaderRole;
+									let role = config.loaderRole;
 									myClient.query(
 										`copy ${table} (${f}) from '${file}' ${manifest} ${role ? `credentials 'aws_iam_role=${role}'` : ''
 										} NULL AS '\\\\N' format csv DELIMITER '|' ACCEPTINVCHARS TRUNCATECOLUMNS ACCEPTANYDATE TIMEFORMAT 'YYYY-MM-DD HH:MI:SS' COMPUPDATE OFF`,
diff --git a/postgres/lib/dwconnect.js b/postgres/lib/dwconnect.js
index cc02db6a..ad0abca7 100644
--- a/postgres/lib/dwconnect.js
+++ b/postgres/lib/dwconnect.js
@@ -313,7 +313,7 @@ module.exports = function (config, columnConfig) {
 					} else {
 						done(null, obj);
 					}
-				}), client.streamToTableFromS3(qualifiedStagingTable, config.s3prefix, config.keepS3Filess), (err) => {
+				}), client.streamToTableFromS3(qualifiedStagingTable, config), (err) => {
 					if (err) {
 						return done(err);
 					} else {
@@ -398,7 +398,7 @@ module.exports = function (config, columnConfig) {
 									SELECT row_number() over () + ${rowId}, 
 										   ${allColumns.map(column => `coalesce(staging.${column}, prev.${column})`)}, 
 										   ${dwClient.auditdate} as ${columnConfig._auditdate}, 
-										   case when changes.isNew then '1900-01-01 00:00:00' else ${config.verion === 'redshift' ? 'sysdate' : 'now()'} END as ${columnConfig._startdate}, 
+										   case when changes.isNew then '1900-01-01 00:00:00' else ${config.version === 'redshift' ? 'sysdate' : 'now()'} END as ${columnConfig._startdate}, 
 										   '9999-01-01 00:00:00' as ${columnConfig._enddate}, 
 										   true as ${columnConfig._current}
 									FROM ${qualifiedStagingTable}_changes changes  
@@ -413,7 +413,7 @@ module.exports = function (config, columnConfig) {
 									SELECT farmFingerPrint64(${nk.map(id => `staging.${id}`).join('-')}), 
 										   ${allColumns.map(column => `coalesce(staging.${column}, prev.${column})`)}, 
 										   ${dwClient.auditdate} as ${columnConfig._auditdate}, 
-										   case when changes.isNew then '1900-01-01 00:00:00' else ${config.verion === 'redshift' ? 'sysdate' : 'now()'} END as ${columnConfig._startdate}, 
+										   case when changes.isNew then '1900-01-01 00:00:00' else ${config.version === 'redshift' ? 'sysdate' : 'now()'} END as ${columnConfig._startdate}, 
 										   '9999-01-01 00:00:00' as ${columnConfig._enddate}, true as ${columnConfig._current}
 									FROM ${qualifiedStagingTable}_changes changes  
 									JOIN ${qualifiedStagingTable} staging on ${nk.map(id => `staging.${id} = changes.${id}`).join(' and ')}
@@ -428,14 +428,14 @@ module.exports = function (config, columnConfig) {
 							tasks.push(done => {
 								// RUN SCD1 / SCD6 columns  (where we update the old records)
 								let columns = scd1.map(column => `"${column}" = coalesce(staging."${column}", prev."${column}")`).concat(scd6.map(column => `"current_${column}" = coalesce(staging."${column}", prev."${column}")`));
-								columns.push(`"${columnConfig._enddate}" = case when changes.runSCD2 =1 then ${config.verion === 'redshift' ? 'sysdate' : 'now()'} else prev."${columnConfig._enddate}" END`);
+								columns.push(`"${columnConfig._enddate}" = case when changes.runSCD2 =1 then ${config.version === 'redshift' ? 'sysdate' : 'now()'} else prev."${columnConfig._enddate}" END`);
 								columns.push(`"${columnConfig._current}" = case when changes.runSCD2 =1 then false else prev."${columnConfig._current}" END`);
 								columns.push(`"${columnConfig._auditdate}" = ${dwClient.auditdate}`);
 								connection.query(`update ${qualifiedTable} as prev
 											set  ${columns.join(', ')}
 											FROM ${qualifiedStagingTable}_changes changes
 											JOIN ${qualifiedStagingTable} staging on ${nk.map(id => `staging.${id} = changes.${id}`).join(' and ')}
-											where ${nk.map(id => `prev.${id} = changes.${id}`).join(' and ')} and prev.${columnConfig._startdate} != ${config.verion === 'redshift' ? 'sysdate' : 'now()'} and changes.isNew = false /*Need to make sure we are only updating the ones not just inserted through SCD2 otherwise we run into issues with multiple rows having .${columnConfig._current}*/
+											where ${nk.map(id => `prev.${id} = changes.${id}`).join(' and ')} and prev.${columnConfig._startdate} != ${config.version === 'redshift' ? 'sysdate' : 'now()'} and changes.isNew = false /*Need to make sure we are only updating the ones not just inserted through SCD2 otherwise we run into issues with multiple rows having .${columnConfig._current}*/
 												and (changes.runSCD1=1 OR  changes.runSCD6=1 OR changes.runSCD2=1)
 											`, done);
 							});
@@ -447,7 +447,7 @@ module.exports = function (config, columnConfig) {
 											set  ${columns.join(', ')}
 											FROM ${qualifiedStagingTable}_changes changes
 											JOIN ${qualifiedStagingTable} staging on ${nk.map(id => `staging.${id} = changes.${id}`).join(' and ')}
-											WHERE ${nk.map(id => `prev.${id} = changes.${id}`).join(' and ')} and prev.${columnConfig._startdate} != ${config.verion === 'redshift' ? 'sysdate' : 'now()'} and changes.isNew = false
+											WHERE ${nk.map(id => `prev.${id} = changes.${id}`).join(' and ')} and prev.${columnConfig._startdate} != ${config.version === 'redshift' ? 'sysdate' : 'now()'} and changes.isNew = false
 											`, done);
 							});
 						};
@@ -765,7 +765,7 @@ module.exports = function (config, columnConfig) {
 			// Add empty row to new dim
 			defaults = defaults.concat([{
 				column: columnConfig._auditdate,
-				value: config.verion === 'redshift' ? 'sysdate' : 'now()',
+				value: config.version === 'redshift' ? 'sysdate' : 'now()',
 			}, {
 				column: columnConfig._startdate,
 				value: client.escapeValueNoToLower('1900-01-01 00:00:00'),
diff --git a/postgres/package.json b/postgres/package.json
index 1624be5c..941f9362 100644
--- a/postgres/package.json
+++ b/postgres/package.json
@@ -1,6 +1,6 @@
 {
 	"name": "leo-connector-postgres",
-	"version": "4.0.3",
+	"version": "4.0.5",
 	"description": "A Postgres database connector for use with Leo Platform",
 	"repository": {
 		"type": "git",

From 421867ccb4a8a8ff50df682cb7f13a8267ac92f9 Mon Sep 17 00:00:00 2001
From: Michael Scranton <mscranton@commercehub.com>
Date: Thu, 20 Oct 2022 12:35:00 -0400
Subject: [PATCH 04/62] add back code removed erroneously

---
 postgres/lib/dwconnect.js | 59 ++++++++++++++++++++++++---------------
 1 file changed, 36 insertions(+), 23 deletions(-)

diff --git a/postgres/lib/dwconnect.js b/postgres/lib/dwconnect.js
index ad0abca7..ca399314 100644
--- a/postgres/lib/dwconnect.js
+++ b/postgres/lib/dwconnect.js
@@ -27,9 +27,9 @@ module.exports = function (config, columnConfig) {
 		useSurrogateDateKeys: true,
 	}, columnConfig || {});
 
-	
 
-	if(config.hashedSurrogateKeys && config.useSlowlyChangingDimensions) {
+
+	if (config.hashedSurrogateKeys && config.useSlowlyChangingDimensions) {
 		logger.error(`Unsupported configuration, useSlowlyChangingDimensions:${useSlowlyChangingDimensions} hashedSurrogateKeys:${hashedSurrogateKeys}.`);
 		process.exit();
 	};
@@ -37,7 +37,7 @@ module.exports = function (config, columnConfig) {
 	client.getDimensionColumn = columnConfig.dimColumnTransform;
 	client.columnConfig = columnConfig;
 
-	function deletesSetup (qualifiedTable, schema, field, value, where = '') {
+	function deletesSetup(qualifiedTable, schema, field, value, where = '') {
 		let colLookup = {};
 		schema.map(col => {
 			colLookup[col.column_name] = true;
@@ -48,7 +48,7 @@ module.exports = function (config, columnConfig) {
 			where = `and ${where}`;
 		}
 
-		function tryFlushDelete (done, force = false) {
+		function tryFlushDelete(done, force = false) {
 			if (force || toDeleteCount >= 1000) {
 				let deleteTasks = Object.keys(toDelete).map(col => {
 					return deleteDone => client.query(`update ${qualifiedTable} set ${field} = ${value} where ${col} in (${toDelete[col].join(',')}) ${where}`, deleteDone);
@@ -125,16 +125,16 @@ module.exports = function (config, columnConfig) {
 		schema[qualifiedStagingTable] = schema[qualifiedTable];
 
 		let tasks = [];
-		
+
 		let deleteHandler = deletesSetup(qualifiedTable, schema[qualifiedTable], columnConfig._deleted, true);
 
-		tempTables.push(qualifiedStagingTable); //Looks like we don't actually need this due to the next line
+		tempTables.push(qualifiedStagingTable);
 		tasks.push(done => client.query(`drop table if exists ${qualifiedStagingTable}`, done));
-		if(config.version !== 'redshift') {
+		if (config.version !== 'redshift') {
 			tasks.push(done => client.query(`create table ${qualifiedStagingTable} (like ${qualifiedTable})`, done));
 			tasks.push(done => client.query(`create index ${stagingTable}_id on ${qualifiedStagingTable} (${ids.join(', ')})`, done));
 		} else {
-			tasks.push(done => 
+			tasks.push(done =>
 				client.query(`create table ${qualifiedStagingTable} 
 							  diststyle all 
 							  as select *
@@ -228,7 +228,7 @@ module.exports = function (config, columnConfig) {
 								)	
 							`, done);
 					});
-					
+
 					async.series(tasks, err => {
 						if (!err) {
 							connection.query(`commit`, e => {
@@ -267,14 +267,16 @@ module.exports = function (config, columnConfig) {
 		let tasks = [];
 		let deleteHandler = deletesSetup(qualifiedTable, schema[qualifiedTable], columnConfig._enddate, dwClient.auditdate, `${columnConfig._current} = true`);
 
+		tempTables.push(qualifiedStagingTable);
+
 		// Prepare staging tables
 		tasks.push(done => client.query(`drop table if exists ${qualifiedStagingTable}`, done));
 		tasks.push(done => client.query(`drop table if exists ${qualifiedStagingTable}_changes`, done));
-		if(config.version !== 'redshift') {
+		if (config.version !== 'redshift') {
 			tasks.push(done => client.query(`create table ${qualifiedStagingTable} (like ${qualifiedTable})`, done));
 			tasks.push(done => client.query(`create index ${stagingTbl}_id on ${qualifiedStagingTable} (${nk.join(', ')})`, done));
 		} else {
-			tasks.push(done => 
+			tasks.push(done =>
 				client.query(`create table ${qualifiedStagingTable} 
 							  diststyle all 
 							  as select *
@@ -282,7 +284,7 @@ module.exports = function (config, columnConfig) {
 								 limit 0;`, done));
 		};
 		tasks.push(done => client.query(`alter table ${qualifiedStagingTable} drop column ${sk}`, done));
-		
+
 		if (config.version !== 'redshift') {
 			tasks.push(done => {
 				ls.pipe(stream, ls.through((obj, done, push) => {
@@ -347,7 +349,7 @@ module.exports = function (config, columnConfig) {
 
 					let scdSQL = [];
 
-					if(!config.bypassSlowlyChangingDimensions) {
+					if (!config.bypassSlowlyChangingDimensions) {
 						if (scd1.length) {
 							scdSQL.push(`CASE WHEN md5(${scd1.map(f => 'md5(coalesce(s.' + f + '::text,\'\'))').join(' || ')}) = md5(${scd1.map(f => 'md5(coalesce(d.' + f + '::text,\'\'))').join(' || ')}) THEN 0 WHEN d.${nk[0]} is null then 0 ELSE 1 END as runSCD1`);
 						} else {
@@ -370,6 +372,7 @@ module.exports = function (config, columnConfig) {
 						}
 					};
 
+					tempTables.push(`${qualifiedStagingTable}_changes`);
 					connection.query(`create table ${qualifiedStagingTable}_changes as 
 									  	select ${nk.map(id => `s.${id}`).join(', ')}, 
 											   d.${nk[0]} is null as isNew
@@ -390,10 +393,20 @@ module.exports = function (config, columnConfig) {
 						tasks.push(done => connection.query(`Begin Transaction`, done));
 
 						let fields = [sk].concat(allColumns).concat([columnConfig._auditdate, columnConfig._startdate, columnConfig._enddate, columnConfig._current]);
-						
-						if(!config.bypassSlowlyChangingDimensions) {
+
+						if (!config.bypassSlowlyChangingDimensions) {
+							tasks.push(done => {
+								connection.query(`select max(${sk}) as maxid from ${qualifiedTable}`, (err, results) => {
+									if (err) {
+										return done(err);
+									}
+									rowId = results[0].maxid || 10000;
+									totalRecords = (rowId - 10000);
+									done();
+								});
+							});
+
 							tasks.push(done => {
-								let fields = [sk].concat(allColumns).concat([columnConfig._auditdate, columnConfig._startdate, columnConfig._enddate, columnConfig._current]);
 								connection.query(`INSERT INTO ${qualifiedTable} (${fields.join(',')})
 									SELECT row_number() over () + ${rowId}, 
 										   ${allColumns.map(column => `coalesce(staging.${column}, prev.${column})`)}, 
@@ -422,9 +435,9 @@ module.exports = function (config, columnConfig) {
 									`, done);
 							});
 						};
-						
+
 						// This needs to be done last
-						if(!config.bypassSlowlyChangingDimensions) {
+						if (!config.bypassSlowlyChangingDimensions) {
 							tasks.push(done => {
 								// RUN SCD1 / SCD6 columns  (where we update the old records)
 								let columns = scd1.map(column => `"${column}" = coalesce(staging."${column}", prev."${column}")`).concat(scd6.map(column => `"current_${column}" = coalesce(staging."${column}", prev."${column}")`));
@@ -439,7 +452,7 @@ module.exports = function (config, columnConfig) {
 												and (changes.runSCD1=1 OR  changes.runSCD6=1 OR changes.runSCD2=1)
 											`, done);
 							});
-						} else if(config.hashedSurrogateKeys) {
+						} else if (config.hashedSurrogateKeys) {
 							tasks.push(done => {
 								let columns = scd1.map(column => `"${column}" = coalesce(staging."${column}", prev."${column}")`).concat(scd6.map(column => `"current_${column}" = coalesce(staging."${column}", prev."${column}")`));
 								columns.push(`"${columnConfig._auditdate}" = ${dwClient.auditdate}`);
@@ -474,7 +487,7 @@ module.exports = function (config, columnConfig) {
 	};
 
 	client.insertMissingDimensions = function (usedTables, tableConfig, tableSks, tableNks, callback) {
-		if(config.hashedSurrogateKeys) {
+		if (config.hashedSurrogateKeys) {
 			callback(null);
 		} else {
 			let unions = {};
@@ -557,7 +570,7 @@ module.exports = function (config, columnConfig) {
 					} else if (columnConfig.useSurrogateDateKeys && (link.table === 'd_time' || link.table === 'time' || link.table === 'dim_time')) {
 						sets.push(`${link.destination}_time = coalesce(EXTRACT(EPOCH from t.${link.source}::time) + 10000, 1)`);
 					} else {
-						if(config.hashedSurrogateKeys) {
+						if (config.hashedSurrogateKeys) {
 							sets.push(`${link.destination} = coalesce(farmFingerPrint64(t.${link.source}), 1)`);
 							return ``;
 						} else {
@@ -567,7 +580,7 @@ module.exports = function (config, columnConfig) {
 								joinOn = link.source.map((v, i) => `${link.join_id}_join_table.${link.on[i]} = t.${v}`).join(' AND ');
 							}
 							return `LEFT JOIN ${link.table} ${link.join_id}_join_table
-								ON ${joinOn} 
+									ON ${joinOn} 
 									AND t.${link.link_date} >= ${link.join_id}_join_table.${columnConfig._startdate}
 									AND (t.${link.link_date} <= ${link.join_id}_join_table.${columnConfig._enddate} or ${link.join_id}_join_table.${columnConfig._current})`;
 						};
@@ -578,7 +591,7 @@ module.exports = function (config, columnConfig) {
 					client.query(`UPDATE ${table} dm
                         SET  ${sets.join(', ')}, ${columnConfig._auditdate} = ${linkAuditdate}
                         FROM ${table} t
-						${joinTables.join('\n')}
+						${config.hashedSurrogateKeys ? '' : joinTables.join('\n')}
                         WHERE ${nk.map(id => `dm.${id} = t.${id}`).join(' AND ')}
 							AND dm.${columnConfig._auditdate} = ${dwClient.auditdate} AND t.${columnConfig._auditdate} = ${dwClient.auditdate}
                     `, done);

From 1badf24eaf2797e62222968c9e08dcaa47924ece Mon Sep 17 00:00:00 2001
From: Michael Scranton <mscranton@commercehub.com>
Date: Mon, 24 Oct 2022 12:06:23 -0400
Subject: [PATCH 05/62] add back incorrectly removed code

---
 postgres/lib/dwconnect.js | 1 +
 1 file changed, 1 insertion(+)

diff --git a/postgres/lib/dwconnect.js b/postgres/lib/dwconnect.js
index ca399314..789bfa05 100644
--- a/postgres/lib/dwconnect.js
+++ b/postgres/lib/dwconnect.js
@@ -574,6 +574,7 @@ module.exports = function (config, columnConfig) {
 							sets.push(`${link.destination} = coalesce(farmFingerPrint64(t.${link.source}), 1)`);
 							return ``;
 						} else {
+							sets.push(`${link.destination} = coalesce(${link.join_id}_join_table.${link.sk}, 1)`);
 							var joinOn = `${link.join_id}_join_table.${link.on} = t.${link.source}`;
 
 							if (Array.isArray(link.source)) {

From b46e3a0b046d68741e071a8b2ca97626ad2dd5ab Mon Sep 17 00:00:00 2001
From: Michael Scranton <mscranton@commercehub.com>
Date: Tue, 25 Oct 2022 11:05:16 -0400
Subject: [PATCH 06/62] bug fixes, comments

---
 postgres/lib/dwconnect.js | 25 +++++++++++++------------
 1 file changed, 13 insertions(+), 12 deletions(-)

diff --git a/postgres/lib/dwconnect.js b/postgres/lib/dwconnect.js
index 789bfa05..5afb313d 100644
--- a/postgres/lib/dwconnect.js
+++ b/postgres/lib/dwconnect.js
@@ -28,9 +28,9 @@ module.exports = function (config, columnConfig) {
 	}, columnConfig || {});
 
 
-
-	if (config.hashedSurrogateKeys && config.useSlowlyChangingDimensions) {
-		logger.error(`Unsupported configuration, useSlowlyChangingDimensions:${useSlowlyChangingDimensions} hashedSurrogateKeys:${hashedSurrogateKeys}.`);
+	// Control flow for both of these configurations set to true has not been added. An error will be thrown until that is supported.
+	if (config.hashedSurrogateKeys && config.bypassSlowlyChangingDimensions) {
+		logger.error(`Unsupported configuration, bypassSlowlyChangingDimensions:${bypassSlowlyChangingDimensions} hashedSurrogateKeys:${hashedSurrogateKeys}.`);
 		process.exit();
 	};
 
@@ -277,6 +277,7 @@ module.exports = function (config, columnConfig) {
 			tasks.push(done => client.query(`create index ${stagingTbl}_id on ${qualifiedStagingTable} (${nk.join(', ')})`, done));
 		} else {
 			tasks.push(done =>
+				// Create staging table with DISTSTYLE ALL to prevent cross talk
 				client.query(`create table ${qualifiedStagingTable} 
 							  diststyle all 
 							  as select *
@@ -394,7 +395,7 @@ module.exports = function (config, columnConfig) {
 
 						let fields = [sk].concat(allColumns).concat([columnConfig._auditdate, columnConfig._startdate, columnConfig._enddate, columnConfig._current]);
 
-						if (!config.bypassSlowlyChangingDimensions) {
+						if (!config.hashedSurrogateKeys) {
 							tasks.push(done => {
 								connection.query(`select max(${sk}) as maxid from ${qualifiedTable}`, (err, results) => {
 									if (err) {
@@ -423,24 +424,24 @@ module.exports = function (config, columnConfig) {
 						} else if (config.hashedSurrogateKeys) {
 							tasks.push(done => {
 								connection.query(`INSERT INTO ${qualifiedTable} (${fields.join(',')})
-									SELECT farmFingerPrint64(${nk.map(id => `staging.${id}`).join('-')}), 
-										   ${allColumns.map(column => `coalesce(staging.${column}, prev.${column})`)}, 
+									SELECT farmFingerPrint64(${nk.map(id => `staging.${id}`).join(`|| '-' ||`)}),
+										   ${allColumns.map(column => `coalesce(staging.${column}, prev.${column})`)}, // Don't think we need the coalesce anymore
 										   ${dwClient.auditdate} as ${columnConfig._auditdate}, 
 										   case when changes.isNew then '1900-01-01 00:00:00' else ${config.version === 'redshift' ? 'sysdate' : 'now()'} END as ${columnConfig._startdate}, 
 										   '9999-01-01 00:00:00' as ${columnConfig._enddate}, true as ${columnConfig._current}
 									FROM ${qualifiedStagingTable}_changes changes  
 									JOIN ${qualifiedStagingTable} staging on ${nk.map(id => `staging.${id} = changes.${id}`).join(' and ')}
-									LEFT JOIN ${qualifiedTable} as prev on ${nk.map(id => `prev.${id} = changes.${id}`).join(' and ')} and prev.${columnConfig._current}
+									LEFT JOIN ${qualifiedTable} as prev on ${nk.map(id => `prev.${id} = changes.${id}`).join(' and ')} and prev.${columnConfig._current} // validate nothing will set _current to false, if so get rid of this join
 									WHERE isNew = true		
 									`, done);
 							});
-						};
+						}; // Final else is needed to support hashed surrogate keys and slowly changing dimensions
 
 						// This needs to be done last
+						let columns = scd1.map(column => `"${column}" = coalesce(staging."${column}", prev."${column}")`).concat(scd6.map(column => `"current_${column}" = coalesce(staging."${column}", prev."${column}")`));
 						if (!config.bypassSlowlyChangingDimensions) {
 							tasks.push(done => {
 								// RUN SCD1 / SCD6 columns  (where we update the old records)
-								let columns = scd1.map(column => `"${column}" = coalesce(staging."${column}", prev."${column}")`).concat(scd6.map(column => `"current_${column}" = coalesce(staging."${column}", prev."${column}")`));
 								columns.push(`"${columnConfig._enddate}" = case when changes.runSCD2 =1 then ${config.version === 'redshift' ? 'sysdate' : 'now()'} else prev."${columnConfig._enddate}" END`);
 								columns.push(`"${columnConfig._current}" = case when changes.runSCD2 =1 then false else prev."${columnConfig._current}" END`);
 								columns.push(`"${columnConfig._auditdate}" = ${dwClient.auditdate}`);
@@ -448,18 +449,19 @@ module.exports = function (config, columnConfig) {
 											set  ${columns.join(', ')}
 											FROM ${qualifiedStagingTable}_changes changes
 											JOIN ${qualifiedStagingTable} staging on ${nk.map(id => `staging.${id} = changes.${id}`).join(' and ')}
-											where ${nk.map(id => `prev.${id} = changes.${id}`).join(' and ')} and prev.${columnConfig._startdate} != ${config.version === 'redshift' ? 'sysdate' : 'now()'} and changes.isNew = false /*Need to make sure we are only updating the ones not just inserted through SCD2 otherwise we run into issues with multiple rows having .${columnConfig._current}*/
+											where ${nk.map(id => `prev.${id} = changes.${id}`).join(' and ')} and prev.${columnConfig._startdate} != ${config.version === 'redshift' ? 'sysdate' : 'now()'} and changes.isNew = false 
+											/*Need to make sure we are only updating the ones not just inserted through SCD2 otherwise we run into issues with multiple rows having .${columnConfig._current}*/
 												and (changes.runSCD1=1 OR  changes.runSCD6=1 OR changes.runSCD2=1)
 											`, done);
 							});
 						} else if (config.hashedSurrogateKeys) {
 							tasks.push(done => {
-								let columns = scd1.map(column => `"${column}" = coalesce(staging."${column}", prev."${column}")`).concat(scd6.map(column => `"current_${column}" = coalesce(staging."${column}", prev."${column}")`));
 								columns.push(`"${columnConfig._auditdate}" = ${dwClient.auditdate}`);
 								connection.query(`update ${qualifiedTable} as prev
 											set  ${columns.join(', ')}
 											FROM ${qualifiedStagingTable}_changes changes
 											JOIN ${qualifiedStagingTable} staging on ${nk.map(id => `staging.${id} = changes.${id}`).join(' and ')}
+																												// might be able to remove the _startdate dependency since we aren't using slowly changing dimensions
 											WHERE ${nk.map(id => `prev.${id} = changes.${id}`).join(' and ')} and prev.${columnConfig._startdate} != ${config.version === 'redshift' ? 'sysdate' : 'now()'} and changes.isNew = false
 											`, done);
 							});
@@ -596,7 +598,6 @@ module.exports = function (config, columnConfig) {
                         WHERE ${nk.map(id => `dm.${id} = t.${id}`).join(' AND ')}
 							AND dm.${columnConfig._auditdate} = ${dwClient.auditdate} AND t.${columnConfig._auditdate} = ${dwClient.auditdate}
                     `, done);
-					// removed join logic from above ${joinTables.join('\n')}
 				} else {
 					done();
 				}

From a3dd99e7944216fb3858bf9c1425de5b9735d0f5 Mon Sep 17 00:00:00 2001
From: Michael Scranton <mscranton@commercehub.com>
Date: Tue, 25 Oct 2022 11:06:11 -0400
Subject: [PATCH 07/62] GitHub action publish-postgres

---
 .github/workflows/publish-postgres.yml | 31 ++++++++++++++++++++++++++
 1 file changed, 31 insertions(+)
 create mode 100644 .github/workflows/publish-postgres.yml

diff --git a/.github/workflows/publish-postgres.yml b/.github/workflows/publish-postgres.yml
new file mode 100644
index 00000000..dd5f0ed1
--- /dev/null
+++ b/.github/workflows/publish-postgres.yml
@@ -0,0 +1,31 @@
+# This workflow will do a clean install of node dependencies, cache/restore them, build the source code and run tests across different versions of node
+# For more information see: https://help.github.com/actions/language-and-framework-guides/using-nodejs-with-github-actions
+
+name: Publish-Postgres
+
+on: 
+  release:
+    types: [published]
+    # branches: [ $default-branch ]
+
+jobs:
+  publish-new-release:
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v2
+    - name: Node 12
+      uses: actions/setup-node@v2
+      with:
+        node-version: '12.x'
+        cache: 'npm'
+        registry-url: 'https://npm.pkg.github.com'
+    - run: cd postgres
+    - run: npm ci
+    - run: npm run build --if-present
+    # - run: npm run test #Nothing is implemented here currently so this is omitted
+    # - run: git config user.email "githubactions@commercehub.com" #skipping this for now since we don't have a servie account for LeoPlatform yet
+    - run: git config user.name "GitHub-Actions"
+    - run: npm version ${{ github.event.release.tag_name }}
+    - run: npm publish
+      env:
+        NODE_AUTH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
\ No newline at end of file

From bea9996a3e8aca8dacc771c1554e9c22990df8af Mon Sep 17 00:00:00 2001
From: Michael Scranton <mscranton@commercehub.com>
Date: Tue, 25 Oct 2022 11:24:35 -0400
Subject: [PATCH 08/62] move cd to correct spot (I think)

---
 .github/workflows/publish-postgres.yml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/publish-postgres.yml b/.github/workflows/publish-postgres.yml
index dd5f0ed1..3df64334 100644
--- a/.github/workflows/publish-postgres.yml
+++ b/.github/workflows/publish-postgres.yml
@@ -12,6 +12,7 @@ jobs:
   publish-new-release:
     runs-on: ubuntu-latest
     steps:
+    - run: cd postgres
     - uses: actions/checkout@v2
     - name: Node 12
       uses: actions/setup-node@v2
@@ -19,7 +20,6 @@ jobs:
         node-version: '12.x'
         cache: 'npm'
         registry-url: 'https://npm.pkg.github.com'
-    - run: cd postgres
     - run: npm ci
     - run: npm run build --if-present
     # - run: npm run test #Nothing is implemented here currently so this is omitted
@@ -28,4 +28,4 @@ jobs:
     - run: npm version ${{ github.event.release.tag_name }}
     - run: npm publish
       env:
-        NODE_AUTH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
\ No newline at end of file
+        NODE_AUTH_TOKEN: ${{ secrets.GITHUB_TOKEN }}

From 395b6d2f5e2d04c509c5fc12a690e07e390a535c Mon Sep 17 00:00:00 2001
From: Michael Scranton <mscranton@commercehub.com>
Date: Tue, 25 Oct 2022 13:26:54 -0400
Subject: [PATCH 09/62] trying workflow things

---
 .github/workflows/publish-postgres.yml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/publish-postgres.yml b/.github/workflows/publish-postgres.yml
index 3df64334..0e02480b 100644
--- a/.github/workflows/publish-postgres.yml
+++ b/.github/workflows/publish-postgres.yml
@@ -18,8 +18,8 @@ jobs:
       uses: actions/setup-node@v2
       with:
         node-version: '12.x'
-        cache: 'npm'
-        registry-url: 'https://npm.pkg.github.com'
+        # cache: 'npm'
+        # registry-url: 'https://npm.pkg.github.com'
     - run: npm ci
     - run: npm run build --if-present
     # - run: npm run test #Nothing is implemented here currently so this is omitted

From 2840aea0225ef6c3bfafa30f21fc0bbf4e66b338 Mon Sep 17 00:00:00 2001
From: Michael Scranton <mscranton@commercehub.com>
Date: Tue, 25 Oct 2022 13:40:27 -0400
Subject: [PATCH 10/62] trying workflow things

---
 .github/workflows/publish-postgres.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/publish-postgres.yml b/.github/workflows/publish-postgres.yml
index 0e02480b..d105cada 100644
--- a/.github/workflows/publish-postgres.yml
+++ b/.github/workflows/publish-postgres.yml
@@ -12,8 +12,8 @@ jobs:
   publish-new-release:
     runs-on: ubuntu-latest
     steps:
-    - run: cd postgres
     - uses: actions/checkout@v2
+    - run: cd postgres
     - name: Node 12
       uses: actions/setup-node@v2
       with:

From 2cfc1093bf46554fced6bc5eedf879fda8a1e3ff Mon Sep 17 00:00:00 2001
From: Michael Scranton <mscranton@commercehub.com>
Date: Tue, 25 Oct 2022 13:58:01 -0400
Subject: [PATCH 11/62] trying workflow things

---
 .github/workflows/publish-postgres.yml |   4 +-
 postgres/package-lock.json             | 246 ++++++++++++++++++++++++-
 postgres/package.json                  |   4 +-
 3 files changed, 248 insertions(+), 6 deletions(-)

diff --git a/.github/workflows/publish-postgres.yml b/.github/workflows/publish-postgres.yml
index d105cada..98c51ff3 100644
--- a/.github/workflows/publish-postgres.yml
+++ b/.github/workflows/publish-postgres.yml
@@ -20,7 +20,9 @@ jobs:
         node-version: '12.x'
         # cache: 'npm'
         # registry-url: 'https://npm.pkg.github.com'
-    - run: npm ci
+    - run: |
+        pwd
+        npm ci
     - run: npm run build --if-present
     # - run: npm run test #Nothing is implemented here currently so this is omitted
     # - run: git config user.email "githubactions@commercehub.com" #skipping this for now since we don't have a servie account for LeoPlatform yet
diff --git a/postgres/package-lock.json b/postgres/package-lock.json
index 25396e1d..c557dff0 100644
--- a/postgres/package-lock.json
+++ b/postgres/package-lock.json
@@ -1,8 +1,7 @@
 {
-	"name": "leo-connector-postgres",
-	"version": "4.0.2",
-	"lockfileVersion": 1,
+	"name": "connector-postgres",
 	"requires": true,
+	"lockfileVersion": 1,
 	"dependencies": {
 		"@babel/code-frame": {
 			"version": "7.8.3",
@@ -30,6 +29,81 @@
 				"js-tokens": "^4.0.0"
 			}
 		},
+		"@dsco/layer": {
+			"version": "1.0.10",
+			"resolved": "https://registry.npmjs.org/@dsco/layer/-/layer-1.0.10.tgz",
+			"integrity": "sha512-WKeCqbggSSc9uMi+LH/LRk9ibv6FNf0npF8Q7BNKwdSD/f8xbi3FvnNJKuWUvJvK1j/iYJb36ajC+EUWU27gdQ==",
+			"dev": true
+		},
+		"@dsco/layer-leo": {
+			"version": "2.1.29",
+			"resolved": "https://registry.npmjs.org/@dsco/layer-leo/-/layer-leo-2.1.29.tgz",
+			"integrity": "sha512-eFGVSCsrlUVWA8Mh9x43KjxAC/HZOj9+5lXMgS/NGQBfn1sbO2OsUDg2IHYNMHBba5Lql5RwUYqqL6faMjqYJQ==",
+			"dev": true,
+			"requires": {
+				"@dsco/layer": "1.0.10",
+				"leo-auth": "3.0.1",
+				"leo-aws": "2.0.2-beta-x",
+				"leo-cache": "1.0.1",
+				"leo-config": "1.1.0",
+				"leo-connector-common": "4.0.8-rc",
+				"leo-connector-entity-table": "3.0.7-rc",
+				"leo-logger": "1.0.1",
+				"leo-sdk": "5.0.16",
+				"leo-streams": "2.0.0",
+				"moment": "2.24.0",
+				"moment-business-days": "1.1.3",
+				"moment-timezone": "0.5.26"
+			},
+			"dependencies": {
+				"leo-aws": {
+					"version": "2.0.2-beta-x",
+					"resolved": "https://registry.npmjs.org/leo-aws/-/leo-aws-2.0.2-beta-x.tgz",
+					"integrity": "sha512-vHMAuuRMr8pPEqFLAQDgr6SmdJG8ncE7YTMqzyH7AsAUkQluj1fjjZ+W7a0CsmLJz5UARhSVyFc3zCiqGcrgiw==",
+					"dev": true,
+					"requires": {
+						"async": "^2.6.1",
+						"backoff": "^2.5.0",
+						"leo-config": "1.1.0",
+						"leo-logger": "1.0.1",
+						"leo-streams": "2.0.0",
+						"lodash": "^4.17.14",
+						"lodash.merge": "^4.6.2"
+					}
+				},
+				"leo-connector-common": {
+					"version": "4.0.8-rc",
+					"resolved": "https://registry.npmjs.org/leo-connector-common/-/leo-connector-common-4.0.8-rc.tgz",
+					"integrity": "sha512-f9uJVkgDtI+EqJCF/c9BqP/236NlflId0uK3Wq0LwNNISESakDXDyxemb9oTgvaMfXWW7VOHFOSylthdFSLgmg==",
+					"dev": true,
+					"requires": {
+						"async": "2.6.3",
+						"leo-aws": "2.0.1",
+						"leo-logger": "1.0.1",
+						"leo-streams": "2.0.0",
+						"moment": "2.24.0",
+						"sqlstring": "2.3.1"
+					},
+					"dependencies": {
+						"leo-aws": {
+							"version": "2.0.1",
+							"resolved": "https://registry.npmjs.org/leo-aws/-/leo-aws-2.0.1.tgz",
+							"integrity": "sha512-40vq8BQklV96sZvN7fDRRmpKmY7p1CyZLIh1G4IkkokScw2QrqkhxOSs5H+A6xb7HBnt+IUvpgOXhwYe0+uOcQ==",
+							"dev": true,
+							"requires": {
+								"async": "^2.6.1",
+								"backoff": "^2.5.0",
+								"leo-config": "1.1.0",
+								"leo-logger": "1.0.1",
+								"leo-streams": "2.0.0",
+								"lodash": "^4.17.14",
+								"lodash.merge": "^4.6.2"
+							}
+						}
+					}
+				}
+			}
+		},
 		"@sinonjs/commons": {
 			"version": "1.7.1",
 			"resolved": "https://registry.npmjs.org/@sinonjs/commons/-/commons-1.7.1.tgz",
@@ -66,6 +140,12 @@
 			"integrity": "sha512-+iTbntw2IZPb/anVDbypzfQa+ay64MW0Zo8aJ8gZPWMMK6/OubMVb6lUPMagqjOPnmtauXnFCACVl3O7ogjeqQ==",
 			"dev": true
 		},
+		"abbrev": {
+			"version": "1.1.1",
+			"resolved": "https://registry.npmjs.org/abbrev/-/abbrev-1.1.1.tgz",
+			"integrity": "sha512-nne9/IiQ/hzIhY6pdDnbBtz7DjPTKrY00P/zvPSm5pOFkl6xuGrGnXn/VtTNNfNtAfZ9/1RtehkszU9qcTii0Q==",
+			"dev": true
+		},
 		"acorn": {
 			"version": "6.4.1",
 			"resolved": "https://registry.npmjs.org/acorn/-/acorn-6.4.1.tgz",
@@ -285,6 +365,16 @@
 			"integrity": "sha1-2Klr13/Wjfd5OnMDajug1UBdR3s=",
 			"dev": true
 		},
+		"config-chain": {
+			"version": "1.1.13",
+			"resolved": "https://registry.npmjs.org/config-chain/-/config-chain-1.1.13.tgz",
+			"integrity": "sha512-qj+f8APARXHrM0hraqXYb2/bOVSV4PvJQlNZ/DVj0QrmNM2q2euizkeuVckQ57J+W0mRH6Hvi+k50M4Jul2VRQ==",
+			"dev": true,
+			"requires": {
+				"ini": "^1.3.4",
+				"proto-list": "~1.2.1"
+			}
+		},
 		"core-util-is": {
 			"version": "1.0.3",
 			"resolved": "https://registry.npmjs.org/core-util-is/-/core-util-is-1.0.3.tgz",
@@ -381,6 +471,32 @@
 				"stream-shift": "^1.0.0"
 			}
 		},
+		"editorconfig": {
+			"version": "0.15.3",
+			"resolved": "https://registry.npmjs.org/editorconfig/-/editorconfig-0.15.3.tgz",
+			"integrity": "sha512-M9wIMFx96vq0R4F+gRpY3o2exzb8hEj/n9S8unZtHSvYjibBp/iMufSzvmOcV/laG0ZtuTVGtiJggPOSW2r93g==",
+			"dev": true,
+			"requires": {
+				"commander": "^2.19.0",
+				"lru-cache": "^4.1.5",
+				"semver": "^5.6.0",
+				"sigmund": "^1.0.1"
+			},
+			"dependencies": {
+				"commander": {
+					"version": "2.20.3",
+					"resolved": "https://registry.npmjs.org/commander/-/commander-2.20.3.tgz",
+					"integrity": "sha512-GpVkmM8vF2vQUkj2LvZmD35JxeJOLCwJ9cUkugyk2nuhbv3+mJvpLYYt+0+USMxE+oj+ey/lJEnhZw75x/OMcQ==",
+					"dev": true
+				},
+				"semver": {
+					"version": "5.7.1",
+					"resolved": "https://registry.npmjs.org/semver/-/semver-5.7.1.tgz",
+					"integrity": "sha512-sauaDf/PZdVgrLTNYHRtpXa1iRiKcaebiKQ1BJdpQlWH2lCvexQdX55snPFyK7QzpudqbCI0qXFfOasHdyNDGQ==",
+					"dev": true
+				}
+			}
+		},
 		"emoji-regex": {
 			"version": "7.0.3",
 			"resolved": "https://registry.npmjs.org/emoji-regex/-/emoji-regex-7.0.3.tgz",
@@ -845,6 +961,19 @@
 			"integrity": "sha1-6PvzdNxVb/iUehDcsFctYz8s+hA=",
 			"dev": true
 		},
+		"js-beautify": {
+			"version": "1.10.2",
+			"resolved": "https://registry.npmjs.org/js-beautify/-/js-beautify-1.10.2.tgz",
+			"integrity": "sha512-ZtBYyNUYJIsBWERnQP0rPN9KjkrDfJcMjuVGcvXOUJrD1zmOGwhRwQ4msG+HJ+Ni/FA7+sRQEMYVzdTQDvnzvQ==",
+			"dev": true,
+			"requires": {
+				"config-chain": "^1.1.12",
+				"editorconfig": "^0.15.3",
+				"glob": "^7.1.3",
+				"mkdirp": "~0.5.1",
+				"nopt": "~4.0.1"
+			}
+		},
 		"js-tokens": {
 			"version": "4.0.0",
 			"resolved": "https://registry.npmjs.org/js-tokens/-/js-tokens-4.0.0.tgz",
@@ -879,6 +1008,17 @@
 			"integrity": "sha512-ApcjaOdVTJ7y4r08xI5wIqpvwS48Q0PBG4DJROcEkH1f8MdAiNFyFxz3xoL0LWAVwjrwPYZdVHHxhRHcx/uGLA==",
 			"dev": true
 		},
+		"leo-auth": {
+			"version": "3.0.1",
+			"resolved": "https://registry.npmjs.org/leo-auth/-/leo-auth-3.0.1.tgz",
+			"integrity": "sha512-l+vzMxH6PIzjRdJCRVvHX3P4JGAybe71LFz2fg5dQfOcvbqhbTnPMNbfIo4qUcfdHPeFG/ZrZ6wxQJVMzVopEA==",
+			"dev": true,
+			"requires": {
+				"leo-aws": "2.0.1",
+				"leo-config": "1.1.0",
+				"netmask": "^1.0.6"
+			}
+		},
 		"leo-aws": {
 			"version": "2.0.1",
 			"resolved": "https://registry.npmjs.org/leo-aws/-/leo-aws-2.0.1.tgz",
@@ -893,6 +1033,12 @@
 				"lodash.merge": "^4.6.2"
 			}
 		},
+		"leo-cache": {
+			"version": "1.0.1",
+			"resolved": "https://registry.npmjs.org/leo-cache/-/leo-cache-1.0.1.tgz",
+			"integrity": "sha512-61QSKr+GrDIbcmDDtngm64a+ra0Xdqe/4wYKW5aP2DcmtVumaNV9rnfI5vUINeVnkwBcXMXNQBht2jw0yBOy7w==",
+			"dev": true
+		},
 		"leo-config": {
 			"version": "1.1.0",
 			"resolved": "https://registry.npmjs.org/leo-config/-/leo-config-1.1.0.tgz",
@@ -915,6 +1061,19 @@
 				"sqlstring": "2.3.1"
 			}
 		},
+		"leo-connector-entity-table": {
+			"version": "3.0.7-rc",
+			"resolved": "https://registry.npmjs.org/leo-connector-entity-table/-/leo-connector-entity-table-3.0.7-rc.tgz",
+			"integrity": "sha512-5Ws+LLynNTOePkB5bcwaqzRVVyBYgQo2eDx7+ZobP3CR2dq4SYGD6YtmVLAHxnNZLdvtuLaxfxsa3iWiTRbFbQ==",
+			"dev": true,
+			"requires": {
+				"async": "2.6.3",
+				"backoff": "2.5.0",
+				"js-beautify": "1.10.2",
+				"leo-aws": "2.0.1",
+				"leo-logger": "1.0.1"
+			}
+		},
 		"leo-logger": {
 			"version": "1.0.1",
 			"resolved": "https://registry.npmjs.org/leo-logger/-/leo-logger-1.0.1.tgz",
@@ -1045,6 +1204,16 @@
 			"integrity": "sha512-gKO5uExCXvSm6zbF562EvM+rd1kQDnB9AZBbiQVzf1ZmdDpxUSvpnAaVOP83N/31mRK8Ml8/VE8DMvsAZQ+7wg==",
 			"dev": true
 		},
+		"lru-cache": {
+			"version": "4.1.5",
+			"resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-4.1.5.tgz",
+			"integrity": "sha512-sWZlbEP2OsHNkXrMl5GYk/jKk70MBng6UU4YI/qGDYbgf6YbP4EvmqISbXCoJiRKs+1bSpFHVgQxvJ17F2li5g==",
+			"dev": true,
+			"requires": {
+				"pseudomap": "^1.0.2",
+				"yallist": "^2.1.2"
+			}
+		},
 		"map-stream": {
 			"version": "0.0.7",
 			"resolved": "https://registry.npmjs.org/map-stream/-/map-stream-0.0.7.tgz",
@@ -1159,6 +1328,21 @@
 			"resolved": "https://registry.npmjs.org/moment/-/moment-2.24.0.tgz",
 			"integrity": "sha512-bV7f+6l2QigeBBZSM/6yTNq4P2fNpSWj/0e7jQcy87A8e7o2nAfP/34/2ky5Vw4B9S446EtIhodAzkFCcR4dQg=="
 		},
+		"moment-business-days": {
+			"version": "1.1.3",
+			"resolved": "https://registry.npmjs.org/moment-business-days/-/moment-business-days-1.1.3.tgz",
+			"integrity": "sha512-VfpcLQxVWJ9ymq4ljaqPdzoco01kOJblX9sbfOCYguBvqCvVPYDJQTdez49a2gtSoSK8AjL8JfvLvmoQlrBMiQ==",
+			"dev": true
+		},
+		"moment-timezone": {
+			"version": "0.5.26",
+			"resolved": "https://registry.npmjs.org/moment-timezone/-/moment-timezone-0.5.26.tgz",
+			"integrity": "sha512-sFP4cgEKTCymBBKgoxZjYzlSovC20Y6J7y3nanDc5RoBIXKlZhoYwBoZGe3flwU6A372AcRwScH8KiwV6zjy1g==",
+			"dev": true,
+			"requires": {
+				"moment": ">= 2.9.0"
+			}
+		},
 		"ms": {
 			"version": "2.1.2",
 			"resolved": "https://registry.npmjs.org/ms/-/ms-2.1.2.tgz",
@@ -1177,6 +1361,12 @@
 			"integrity": "sha1-Sr6/7tdUHywnrPspvbvRXI1bpPc=",
 			"dev": true
 		},
+		"netmask": {
+			"version": "1.0.6",
+			"resolved": "https://registry.npmjs.org/netmask/-/netmask-1.0.6.tgz",
+			"integrity": "sha512-3DWDqAtIiPSkBXZyYEjwebfK56nrlQfRGt642fu8RPaL+ePu750+HCMHxjJCG3iEHq/0aeMvX6KIzlv7nuhfrA==",
+			"dev": true
+		},
 		"nice-try": {
 			"version": "1.0.5",
 			"resolved": "https://registry.npmjs.org/nice-try/-/nice-try-1.0.5.tgz",
@@ -1207,6 +1397,16 @@
 				}
 			}
 		},
+		"nopt": {
+			"version": "4.0.3",
+			"resolved": "https://registry.npmjs.org/nopt/-/nopt-4.0.3.tgz",
+			"integrity": "sha512-CvaGwVMztSMJLOeXPrez7fyfObdZqNUK1cPAEzLHrTybIua9pMdmmPR5YwtfNftIOMv3DPUhFaxsZMNTQO20Kg==",
+			"dev": true,
+			"requires": {
+				"abbrev": "1",
+				"osenv": "^0.1.4"
+			}
+		},
 		"object-extended": {
 			"version": "0.0.7",
 			"resolved": "https://registry.npmjs.org/object-extended/-/object-extended-0.0.7.tgz",
@@ -1248,12 +1448,28 @@
 				"word-wrap": "~1.2.3"
 			}
 		},
+		"os-homedir": {
+			"version": "1.0.2",
+			"resolved": "https://registry.npmjs.org/os-homedir/-/os-homedir-1.0.2.tgz",
+			"integrity": "sha512-B5JU3cabzk8c67mRRd3ECmROafjYMXbuzlwtqdM8IbS8ktlTix8aFGb2bAGKrSRIlnfKwovGUUr72JUPyOb6kQ==",
+			"dev": true
+		},
 		"os-tmpdir": {
 			"version": "1.0.2",
 			"resolved": "https://registry.npmjs.org/os-tmpdir/-/os-tmpdir-1.0.2.tgz",
 			"integrity": "sha1-u+Z0BseaqFxc/sdm/lc0VV36EnQ=",
 			"dev": true
 		},
+		"osenv": {
+			"version": "0.1.5",
+			"resolved": "https://registry.npmjs.org/osenv/-/osenv-0.1.5.tgz",
+			"integrity": "sha512-0CWcCECdMVc2Rw3U5w9ZjqX6ga6ubk1xDVKxtBQPK7wis/0F2r9T6k4ydGYhecl7YUBxBVxhL5oisPsNxAPe2g==",
+			"dev": true,
+			"requires": {
+				"os-homedir": "^1.0.0",
+				"os-tmpdir": "^1.0.0"
+			}
+		},
 		"packet-reader": {
 			"version": "1.0.0",
 			"resolved": "https://registry.npmjs.org/packet-reader/-/packet-reader-1.0.0.tgz",
@@ -1421,6 +1637,18 @@
 			"integrity": "sha512-7PiHtLll5LdnKIMw100I+8xJXR5gW2QwWYkT6iJva0bXitZKa/XMrSbdmg3r2Xnaidz9Qumd0VPaMrZlF9V9sA==",
 			"dev": true
 		},
+		"proto-list": {
+			"version": "1.2.4",
+			"resolved": "https://registry.npmjs.org/proto-list/-/proto-list-1.2.4.tgz",
+			"integrity": "sha512-vtK/94akxsTMhe0/cbfpR+syPuszcuwhqVjJq26CuNDgFGj682oRBXOP5MJpv2r7JtE8MsiepGIqvvOTBwn2vA==",
+			"dev": true
+		},
+		"pseudomap": {
+			"version": "1.0.2",
+			"resolved": "https://registry.npmjs.org/pseudomap/-/pseudomap-1.0.2.tgz",
+			"integrity": "sha512-b/YwNhb8lk1Zz2+bXXpS/LK9OisiZZ1SNsSLxN1x2OXVEhW2Ckr/7mWE5vrC1ZTiJlD9g19jWszTmJsB+oEpFQ==",
+			"dev": true
+		},
 		"pump": {
 			"version": "3.0.0",
 			"resolved": "https://registry.npmjs.org/pump/-/pump-3.0.0.tgz",
@@ -1550,6 +1778,12 @@
 			"integrity": "sha1-2kL0l0DAtC2yypcoVxyxkMmO/qM=",
 			"dev": true
 		},
+		"sigmund": {
+			"version": "1.0.1",
+			"resolved": "https://registry.npmjs.org/sigmund/-/sigmund-1.0.1.tgz",
+			"integrity": "sha512-fCvEXfh6NWpm+YSuY2bpXb/VIihqWA6hLsgboC+0nl71Q7N7o2eaCW8mJa/NLvQhs6jpd3VZV4UiUQlV6+lc8g==",
+			"dev": true
+		},
 		"signal-exit": {
 			"version": "3.0.2",
 			"resolved": "https://registry.npmjs.org/signal-exit/-/signal-exit-3.0.2.tgz",
@@ -1818,6 +2052,12 @@
 			"version": "4.0.2",
 			"resolved": "https://registry.npmjs.org/xtend/-/xtend-4.0.2.tgz",
 			"integrity": "sha512-LKYU1iAXJXUgAXn9URjiu+MWhyUXHsvfp7mcuYm9dSUKK0/CjtrUwFAxD82/mCWbtLsGjFIad0wIsod4zrTAEQ=="
+		},
+		"yallist": {
+			"version": "2.1.2",
+			"resolved": "https://registry.npmjs.org/yallist/-/yallist-2.1.2.tgz",
+			"integrity": "sha512-ncTzHV7NvsQZkYe1DW7cbDLm0YpzHmZF5r/iyP3ZnQtMiJ+pjzisCiMNI+Sj+xQF5pXhSHxSB3uDbsBTzY/c2A==",
+			"dev": true
 		}
 	}
 }
diff --git a/postgres/package.json b/postgres/package.json
index 941f9362..14f1b305 100644
--- a/postgres/package.json
+++ b/postgres/package.json
@@ -1,6 +1,5 @@
 {
-	"name": "leo-connector-postgres",
-	"version": "4.0.5",
+	"name": "connector-postgres",
 	"description": "A Postgres database connector for use with Leo Platform",
 	"repository": {
 		"type": "git",
@@ -25,6 +24,7 @@
 		"pg-format": "1.0.4"
 	},
 	"devDependencies": {
+		"@dsco/layer-leo": "2.1.29",
 		"chai": "^4.2.0",
 		"eslint": "^5.13.0",
 		"mocha": "^5.2.0",

From 1e1334095b76d3f9a22619bbafff7cf1993dba1f Mon Sep 17 00:00:00 2001
From: Michael Scranton <mscranton@commercehub.com>
Date: Tue, 25 Oct 2022 14:01:26 -0400
Subject: [PATCH 12/62] trying workflow things

---
 .github/workflows/publish-postgres.yml | 1 +
 1 file changed, 1 insertion(+)

diff --git a/.github/workflows/publish-postgres.yml b/.github/workflows/publish-postgres.yml
index 98c51ff3..5001f2ed 100644
--- a/.github/workflows/publish-postgres.yml
+++ b/.github/workflows/publish-postgres.yml
@@ -22,6 +22,7 @@ jobs:
         # registry-url: 'https://npm.pkg.github.com'
     - run: |
         pwd
+        ls
         npm ci
     - run: npm run build --if-present
     # - run: npm run test #Nothing is implemented here currently so this is omitted

From f0a69770694fefa7065b491ac79b1085ce8e19fa Mon Sep 17 00:00:00 2001
From: Michael Scranton <mscranton@commercehub.com>
Date: Tue, 25 Oct 2022 14:05:04 -0400
Subject: [PATCH 13/62] trying workflow things

---
 .github/workflows/publish-postgres.yml | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/.github/workflows/publish-postgres.yml b/.github/workflows/publish-postgres.yml
index 5001f2ed..7445566b 100644
--- a/.github/workflows/publish-postgres.yml
+++ b/.github/workflows/publish-postgres.yml
@@ -11,9 +11,11 @@ on:
 jobs:
   publish-new-release:
     runs-on: ubuntu-latest
+    defaults:
+      run:
+        working-directory: ./postgres
     steps:
     - uses: actions/checkout@v2
-    - run: cd postgres
     - name: Node 12
       uses: actions/setup-node@v2
       with:

From 55d06c6df77ba320f9c6f9bc1e9557068fb57b0c Mon Sep 17 00:00:00 2001
From: Michael Scranton <mscranton@commercehub.com>
Date: Tue, 25 Oct 2022 14:13:26 -0400
Subject: [PATCH 14/62] trying workflow things

---
 postgres/package.json | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/postgres/package.json b/postgres/package.json
index 14f1b305..832ff36d 100644
--- a/postgres/package.json
+++ b/postgres/package.json
@@ -24,7 +24,7 @@
 		"pg-format": "1.0.4"
 	},
 	"devDependencies": {
-		"@dsco/layer-leo": "2.1.29",
+		"@chub-engineering/layer-leo": "2.1.34",
 		"chai": "^4.2.0",
 		"eslint": "^5.13.0",
 		"mocha": "^5.2.0",
@@ -40,4 +40,4 @@
 			}
 		}
 	}
-}
+}
\ No newline at end of file

From b83c0057529d2db8c6775d640e0fd3523c8891b6 Mon Sep 17 00:00:00 2001
From: Michael Scranton <mscranton@commercehub.com>
Date: Tue, 25 Oct 2022 14:25:32 -0400
Subject: [PATCH 15/62] trying workflow things

---
 postgres/package-lock.json | 241 -------------------------------------
 postgres/package.json      |   3 +-
 2 files changed, 1 insertion(+), 243 deletions(-)

diff --git a/postgres/package-lock.json b/postgres/package-lock.json
index c557dff0..21f20995 100644
--- a/postgres/package-lock.json
+++ b/postgres/package-lock.json
@@ -29,81 +29,6 @@
 				"js-tokens": "^4.0.0"
 			}
 		},
-		"@dsco/layer": {
-			"version": "1.0.10",
-			"resolved": "https://registry.npmjs.org/@dsco/layer/-/layer-1.0.10.tgz",
-			"integrity": "sha512-WKeCqbggSSc9uMi+LH/LRk9ibv6FNf0npF8Q7BNKwdSD/f8xbi3FvnNJKuWUvJvK1j/iYJb36ajC+EUWU27gdQ==",
-			"dev": true
-		},
-		"@dsco/layer-leo": {
-			"version": "2.1.29",
-			"resolved": "https://registry.npmjs.org/@dsco/layer-leo/-/layer-leo-2.1.29.tgz",
-			"integrity": "sha512-eFGVSCsrlUVWA8Mh9x43KjxAC/HZOj9+5lXMgS/NGQBfn1sbO2OsUDg2IHYNMHBba5Lql5RwUYqqL6faMjqYJQ==",
-			"dev": true,
-			"requires": {
-				"@dsco/layer": "1.0.10",
-				"leo-auth": "3.0.1",
-				"leo-aws": "2.0.2-beta-x",
-				"leo-cache": "1.0.1",
-				"leo-config": "1.1.0",
-				"leo-connector-common": "4.0.8-rc",
-				"leo-connector-entity-table": "3.0.7-rc",
-				"leo-logger": "1.0.1",
-				"leo-sdk": "5.0.16",
-				"leo-streams": "2.0.0",
-				"moment": "2.24.0",
-				"moment-business-days": "1.1.3",
-				"moment-timezone": "0.5.26"
-			},
-			"dependencies": {
-				"leo-aws": {
-					"version": "2.0.2-beta-x",
-					"resolved": "https://registry.npmjs.org/leo-aws/-/leo-aws-2.0.2-beta-x.tgz",
-					"integrity": "sha512-vHMAuuRMr8pPEqFLAQDgr6SmdJG8ncE7YTMqzyH7AsAUkQluj1fjjZ+W7a0CsmLJz5UARhSVyFc3zCiqGcrgiw==",
-					"dev": true,
-					"requires": {
-						"async": "^2.6.1",
-						"backoff": "^2.5.0",
-						"leo-config": "1.1.0",
-						"leo-logger": "1.0.1",
-						"leo-streams": "2.0.0",
-						"lodash": "^4.17.14",
-						"lodash.merge": "^4.6.2"
-					}
-				},
-				"leo-connector-common": {
-					"version": "4.0.8-rc",
-					"resolved": "https://registry.npmjs.org/leo-connector-common/-/leo-connector-common-4.0.8-rc.tgz",
-					"integrity": "sha512-f9uJVkgDtI+EqJCF/c9BqP/236NlflId0uK3Wq0LwNNISESakDXDyxemb9oTgvaMfXWW7VOHFOSylthdFSLgmg==",
-					"dev": true,
-					"requires": {
-						"async": "2.6.3",
-						"leo-aws": "2.0.1",
-						"leo-logger": "1.0.1",
-						"leo-streams": "2.0.0",
-						"moment": "2.24.0",
-						"sqlstring": "2.3.1"
-					},
-					"dependencies": {
-						"leo-aws": {
-							"version": "2.0.1",
-							"resolved": "https://registry.npmjs.org/leo-aws/-/leo-aws-2.0.1.tgz",
-							"integrity": "sha512-40vq8BQklV96sZvN7fDRRmpKmY7p1CyZLIh1G4IkkokScw2QrqkhxOSs5H+A6xb7HBnt+IUvpgOXhwYe0+uOcQ==",
-							"dev": true,
-							"requires": {
-								"async": "^2.6.1",
-								"backoff": "^2.5.0",
-								"leo-config": "1.1.0",
-								"leo-logger": "1.0.1",
-								"leo-streams": "2.0.0",
-								"lodash": "^4.17.14",
-								"lodash.merge": "^4.6.2"
-							}
-						}
-					}
-				}
-			}
-		},
 		"@sinonjs/commons": {
 			"version": "1.7.1",
 			"resolved": "https://registry.npmjs.org/@sinonjs/commons/-/commons-1.7.1.tgz",
@@ -140,12 +65,6 @@
 			"integrity": "sha512-+iTbntw2IZPb/anVDbypzfQa+ay64MW0Zo8aJ8gZPWMMK6/OubMVb6lUPMagqjOPnmtauXnFCACVl3O7ogjeqQ==",
 			"dev": true
 		},
-		"abbrev": {
-			"version": "1.1.1",
-			"resolved": "https://registry.npmjs.org/abbrev/-/abbrev-1.1.1.tgz",
-			"integrity": "sha512-nne9/IiQ/hzIhY6pdDnbBtz7DjPTKrY00P/zvPSm5pOFkl6xuGrGnXn/VtTNNfNtAfZ9/1RtehkszU9qcTii0Q==",
-			"dev": true
-		},
 		"acorn": {
 			"version": "6.4.1",
 			"resolved": "https://registry.npmjs.org/acorn/-/acorn-6.4.1.tgz",
@@ -365,16 +284,6 @@
 			"integrity": "sha1-2Klr13/Wjfd5OnMDajug1UBdR3s=",
 			"dev": true
 		},
-		"config-chain": {
-			"version": "1.1.13",
-			"resolved": "https://registry.npmjs.org/config-chain/-/config-chain-1.1.13.tgz",
-			"integrity": "sha512-qj+f8APARXHrM0hraqXYb2/bOVSV4PvJQlNZ/DVj0QrmNM2q2euizkeuVckQ57J+W0mRH6Hvi+k50M4Jul2VRQ==",
-			"dev": true,
-			"requires": {
-				"ini": "^1.3.4",
-				"proto-list": "~1.2.1"
-			}
-		},
 		"core-util-is": {
 			"version": "1.0.3",
 			"resolved": "https://registry.npmjs.org/core-util-is/-/core-util-is-1.0.3.tgz",
@@ -471,32 +380,6 @@
 				"stream-shift": "^1.0.0"
 			}
 		},
-		"editorconfig": {
-			"version": "0.15.3",
-			"resolved": "https://registry.npmjs.org/editorconfig/-/editorconfig-0.15.3.tgz",
-			"integrity": "sha512-M9wIMFx96vq0R4F+gRpY3o2exzb8hEj/n9S8unZtHSvYjibBp/iMufSzvmOcV/laG0ZtuTVGtiJggPOSW2r93g==",
-			"dev": true,
-			"requires": {
-				"commander": "^2.19.0",
-				"lru-cache": "^4.1.5",
-				"semver": "^5.6.0",
-				"sigmund": "^1.0.1"
-			},
-			"dependencies": {
-				"commander": {
-					"version": "2.20.3",
-					"resolved": "https://registry.npmjs.org/commander/-/commander-2.20.3.tgz",
-					"integrity": "sha512-GpVkmM8vF2vQUkj2LvZmD35JxeJOLCwJ9cUkugyk2nuhbv3+mJvpLYYt+0+USMxE+oj+ey/lJEnhZw75x/OMcQ==",
-					"dev": true
-				},
-				"semver": {
-					"version": "5.7.1",
-					"resolved": "https://registry.npmjs.org/semver/-/semver-5.7.1.tgz",
-					"integrity": "sha512-sauaDf/PZdVgrLTNYHRtpXa1iRiKcaebiKQ1BJdpQlWH2lCvexQdX55snPFyK7QzpudqbCI0qXFfOasHdyNDGQ==",
-					"dev": true
-				}
-			}
-		},
 		"emoji-regex": {
 			"version": "7.0.3",
 			"resolved": "https://registry.npmjs.org/emoji-regex/-/emoji-regex-7.0.3.tgz",
@@ -961,19 +844,6 @@
 			"integrity": "sha1-6PvzdNxVb/iUehDcsFctYz8s+hA=",
 			"dev": true
 		},
-		"js-beautify": {
-			"version": "1.10.2",
-			"resolved": "https://registry.npmjs.org/js-beautify/-/js-beautify-1.10.2.tgz",
-			"integrity": "sha512-ZtBYyNUYJIsBWERnQP0rPN9KjkrDfJcMjuVGcvXOUJrD1zmOGwhRwQ4msG+HJ+Ni/FA7+sRQEMYVzdTQDvnzvQ==",
-			"dev": true,
-			"requires": {
-				"config-chain": "^1.1.12",
-				"editorconfig": "^0.15.3",
-				"glob": "^7.1.3",
-				"mkdirp": "~0.5.1",
-				"nopt": "~4.0.1"
-			}
-		},
 		"js-tokens": {
 			"version": "4.0.0",
 			"resolved": "https://registry.npmjs.org/js-tokens/-/js-tokens-4.0.0.tgz",
@@ -1008,17 +878,6 @@
 			"integrity": "sha512-ApcjaOdVTJ7y4r08xI5wIqpvwS48Q0PBG4DJROcEkH1f8MdAiNFyFxz3xoL0LWAVwjrwPYZdVHHxhRHcx/uGLA==",
 			"dev": true
 		},
-		"leo-auth": {
-			"version": "3.0.1",
-			"resolved": "https://registry.npmjs.org/leo-auth/-/leo-auth-3.0.1.tgz",
-			"integrity": "sha512-l+vzMxH6PIzjRdJCRVvHX3P4JGAybe71LFz2fg5dQfOcvbqhbTnPMNbfIo4qUcfdHPeFG/ZrZ6wxQJVMzVopEA==",
-			"dev": true,
-			"requires": {
-				"leo-aws": "2.0.1",
-				"leo-config": "1.1.0",
-				"netmask": "^1.0.6"
-			}
-		},
 		"leo-aws": {
 			"version": "2.0.1",
 			"resolved": "https://registry.npmjs.org/leo-aws/-/leo-aws-2.0.1.tgz",
@@ -1033,12 +892,6 @@
 				"lodash.merge": "^4.6.2"
 			}
 		},
-		"leo-cache": {
-			"version": "1.0.1",
-			"resolved": "https://registry.npmjs.org/leo-cache/-/leo-cache-1.0.1.tgz",
-			"integrity": "sha512-61QSKr+GrDIbcmDDtngm64a+ra0Xdqe/4wYKW5aP2DcmtVumaNV9rnfI5vUINeVnkwBcXMXNQBht2jw0yBOy7w==",
-			"dev": true
-		},
 		"leo-config": {
 			"version": "1.1.0",
 			"resolved": "https://registry.npmjs.org/leo-config/-/leo-config-1.1.0.tgz",
@@ -1061,19 +914,6 @@
 				"sqlstring": "2.3.1"
 			}
 		},
-		"leo-connector-entity-table": {
-			"version": "3.0.7-rc",
-			"resolved": "https://registry.npmjs.org/leo-connector-entity-table/-/leo-connector-entity-table-3.0.7-rc.tgz",
-			"integrity": "sha512-5Ws+LLynNTOePkB5bcwaqzRVVyBYgQo2eDx7+ZobP3CR2dq4SYGD6YtmVLAHxnNZLdvtuLaxfxsa3iWiTRbFbQ==",
-			"dev": true,
-			"requires": {
-				"async": "2.6.3",
-				"backoff": "2.5.0",
-				"js-beautify": "1.10.2",
-				"leo-aws": "2.0.1",
-				"leo-logger": "1.0.1"
-			}
-		},
 		"leo-logger": {
 			"version": "1.0.1",
 			"resolved": "https://registry.npmjs.org/leo-logger/-/leo-logger-1.0.1.tgz",
@@ -1204,16 +1044,6 @@
 			"integrity": "sha512-gKO5uExCXvSm6zbF562EvM+rd1kQDnB9AZBbiQVzf1ZmdDpxUSvpnAaVOP83N/31mRK8Ml8/VE8DMvsAZQ+7wg==",
 			"dev": true
 		},
-		"lru-cache": {
-			"version": "4.1.5",
-			"resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-4.1.5.tgz",
-			"integrity": "sha512-sWZlbEP2OsHNkXrMl5GYk/jKk70MBng6UU4YI/qGDYbgf6YbP4EvmqISbXCoJiRKs+1bSpFHVgQxvJ17F2li5g==",
-			"dev": true,
-			"requires": {
-				"pseudomap": "^1.0.2",
-				"yallist": "^2.1.2"
-			}
-		},
 		"map-stream": {
 			"version": "0.0.7",
 			"resolved": "https://registry.npmjs.org/map-stream/-/map-stream-0.0.7.tgz",
@@ -1328,21 +1158,6 @@
 			"resolved": "https://registry.npmjs.org/moment/-/moment-2.24.0.tgz",
 			"integrity": "sha512-bV7f+6l2QigeBBZSM/6yTNq4P2fNpSWj/0e7jQcy87A8e7o2nAfP/34/2ky5Vw4B9S446EtIhodAzkFCcR4dQg=="
 		},
-		"moment-business-days": {
-			"version": "1.1.3",
-			"resolved": "https://registry.npmjs.org/moment-business-days/-/moment-business-days-1.1.3.tgz",
-			"integrity": "sha512-VfpcLQxVWJ9ymq4ljaqPdzoco01kOJblX9sbfOCYguBvqCvVPYDJQTdez49a2gtSoSK8AjL8JfvLvmoQlrBMiQ==",
-			"dev": true
-		},
-		"moment-timezone": {
-			"version": "0.5.26",
-			"resolved": "https://registry.npmjs.org/moment-timezone/-/moment-timezone-0.5.26.tgz",
-			"integrity": "sha512-sFP4cgEKTCymBBKgoxZjYzlSovC20Y6J7y3nanDc5RoBIXKlZhoYwBoZGe3flwU6A372AcRwScH8KiwV6zjy1g==",
-			"dev": true,
-			"requires": {
-				"moment": ">= 2.9.0"
-			}
-		},
 		"ms": {
 			"version": "2.1.2",
 			"resolved": "https://registry.npmjs.org/ms/-/ms-2.1.2.tgz",
@@ -1361,12 +1176,6 @@
 			"integrity": "sha1-Sr6/7tdUHywnrPspvbvRXI1bpPc=",
 			"dev": true
 		},
-		"netmask": {
-			"version": "1.0.6",
-			"resolved": "https://registry.npmjs.org/netmask/-/netmask-1.0.6.tgz",
-			"integrity": "sha512-3DWDqAtIiPSkBXZyYEjwebfK56nrlQfRGt642fu8RPaL+ePu750+HCMHxjJCG3iEHq/0aeMvX6KIzlv7nuhfrA==",
-			"dev": true
-		},
 		"nice-try": {
 			"version": "1.0.5",
 			"resolved": "https://registry.npmjs.org/nice-try/-/nice-try-1.0.5.tgz",
@@ -1397,16 +1206,6 @@
 				}
 			}
 		},
-		"nopt": {
-			"version": "4.0.3",
-			"resolved": "https://registry.npmjs.org/nopt/-/nopt-4.0.3.tgz",
-			"integrity": "sha512-CvaGwVMztSMJLOeXPrez7fyfObdZqNUK1cPAEzLHrTybIua9pMdmmPR5YwtfNftIOMv3DPUhFaxsZMNTQO20Kg==",
-			"dev": true,
-			"requires": {
-				"abbrev": "1",
-				"osenv": "^0.1.4"
-			}
-		},
 		"object-extended": {
 			"version": "0.0.7",
 			"resolved": "https://registry.npmjs.org/object-extended/-/object-extended-0.0.7.tgz",
@@ -1448,28 +1247,12 @@
 				"word-wrap": "~1.2.3"
 			}
 		},
-		"os-homedir": {
-			"version": "1.0.2",
-			"resolved": "https://registry.npmjs.org/os-homedir/-/os-homedir-1.0.2.tgz",
-			"integrity": "sha512-B5JU3cabzk8c67mRRd3ECmROafjYMXbuzlwtqdM8IbS8ktlTix8aFGb2bAGKrSRIlnfKwovGUUr72JUPyOb6kQ==",
-			"dev": true
-		},
 		"os-tmpdir": {
 			"version": "1.0.2",
 			"resolved": "https://registry.npmjs.org/os-tmpdir/-/os-tmpdir-1.0.2.tgz",
 			"integrity": "sha1-u+Z0BseaqFxc/sdm/lc0VV36EnQ=",
 			"dev": true
 		},
-		"osenv": {
-			"version": "0.1.5",
-			"resolved": "https://registry.npmjs.org/osenv/-/osenv-0.1.5.tgz",
-			"integrity": "sha512-0CWcCECdMVc2Rw3U5w9ZjqX6ga6ubk1xDVKxtBQPK7wis/0F2r9T6k4ydGYhecl7YUBxBVxhL5oisPsNxAPe2g==",
-			"dev": true,
-			"requires": {
-				"os-homedir": "^1.0.0",
-				"os-tmpdir": "^1.0.0"
-			}
-		},
 		"packet-reader": {
 			"version": "1.0.0",
 			"resolved": "https://registry.npmjs.org/packet-reader/-/packet-reader-1.0.0.tgz",
@@ -1637,18 +1420,6 @@
 			"integrity": "sha512-7PiHtLll5LdnKIMw100I+8xJXR5gW2QwWYkT6iJva0bXitZKa/XMrSbdmg3r2Xnaidz9Qumd0VPaMrZlF9V9sA==",
 			"dev": true
 		},
-		"proto-list": {
-			"version": "1.2.4",
-			"resolved": "https://registry.npmjs.org/proto-list/-/proto-list-1.2.4.tgz",
-			"integrity": "sha512-vtK/94akxsTMhe0/cbfpR+syPuszcuwhqVjJq26CuNDgFGj682oRBXOP5MJpv2r7JtE8MsiepGIqvvOTBwn2vA==",
-			"dev": true
-		},
-		"pseudomap": {
-			"version": "1.0.2",
-			"resolved": "https://registry.npmjs.org/pseudomap/-/pseudomap-1.0.2.tgz",
-			"integrity": "sha512-b/YwNhb8lk1Zz2+bXXpS/LK9OisiZZ1SNsSLxN1x2OXVEhW2Ckr/7mWE5vrC1ZTiJlD9g19jWszTmJsB+oEpFQ==",
-			"dev": true
-		},
 		"pump": {
 			"version": "3.0.0",
 			"resolved": "https://registry.npmjs.org/pump/-/pump-3.0.0.tgz",
@@ -1778,12 +1549,6 @@
 			"integrity": "sha1-2kL0l0DAtC2yypcoVxyxkMmO/qM=",
 			"dev": true
 		},
-		"sigmund": {
-			"version": "1.0.1",
-			"resolved": "https://registry.npmjs.org/sigmund/-/sigmund-1.0.1.tgz",
-			"integrity": "sha512-fCvEXfh6NWpm+YSuY2bpXb/VIihqWA6hLsgboC+0nl71Q7N7o2eaCW8mJa/NLvQhs6jpd3VZV4UiUQlV6+lc8g==",
-			"dev": true
-		},
 		"signal-exit": {
 			"version": "3.0.2",
 			"resolved": "https://registry.npmjs.org/signal-exit/-/signal-exit-3.0.2.tgz",
@@ -2052,12 +1817,6 @@
 			"version": "4.0.2",
 			"resolved": "https://registry.npmjs.org/xtend/-/xtend-4.0.2.tgz",
 			"integrity": "sha512-LKYU1iAXJXUgAXn9URjiu+MWhyUXHsvfp7mcuYm9dSUKK0/CjtrUwFAxD82/mCWbtLsGjFIad0wIsod4zrTAEQ=="
-		},
-		"yallist": {
-			"version": "2.1.2",
-			"resolved": "https://registry.npmjs.org/yallist/-/yallist-2.1.2.tgz",
-			"integrity": "sha512-ncTzHV7NvsQZkYe1DW7cbDLm0YpzHmZF5r/iyP3ZnQtMiJ+pjzisCiMNI+Sj+xQF5pXhSHxSB3uDbsBTzY/c2A==",
-			"dev": true
 		}
 	}
 }
diff --git a/postgres/package.json b/postgres/package.json
index 832ff36d..9a8e88a1 100644
--- a/postgres/package.json
+++ b/postgres/package.json
@@ -24,7 +24,6 @@
 		"pg-format": "1.0.4"
 	},
 	"devDependencies": {
-		"@chub-engineering/layer-leo": "2.1.34",
 		"chai": "^4.2.0",
 		"eslint": "^5.13.0",
 		"mocha": "^5.2.0",
@@ -40,4 +39,4 @@
 			}
 		}
 	}
-}
\ No newline at end of file
+}

From abf2cb836369a75496295ad6882e71033dfac566 Mon Sep 17 00:00:00 2001
From: Michael Scranton <mscranton@commercehub.com>
Date: Tue, 25 Oct 2022 14:32:26 -0400
Subject: [PATCH 16/62] trying workflow things

---
 .github/workflows/publish-postgres.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/publish-postgres.yml b/.github/workflows/publish-postgres.yml
index 7445566b..5b43fb0d 100644
--- a/.github/workflows/publish-postgres.yml
+++ b/.github/workflows/publish-postgres.yml
@@ -21,7 +21,7 @@ jobs:
       with:
         node-version: '12.x'
         # cache: 'npm'
-        # registry-url: 'https://npm.pkg.github.com'
+        registry-url: 'https://npm.pkg.github.com'
     - run: |
         pwd
         ls

From 1ce11aa744530fdcddcb130d5e95be025cc92854 Mon Sep 17 00:00:00 2001
From: Michael Scranton <mscranton@commercehub.com>
Date: Tue, 25 Oct 2022 14:45:58 -0400
Subject: [PATCH 17/62] trying workflow things

---
 .github/workflows/publish-postgres.yml | 8 +++-----
 1 file changed, 3 insertions(+), 5 deletions(-)

diff --git a/.github/workflows/publish-postgres.yml b/.github/workflows/publish-postgres.yml
index 5b43fb0d..97471026 100644
--- a/.github/workflows/publish-postgres.yml
+++ b/.github/workflows/publish-postgres.yml
@@ -22,15 +22,13 @@ jobs:
         node-version: '12.x'
         # cache: 'npm'
         registry-url: 'https://npm.pkg.github.com'
-    - run: |
-        pwd
-        ls
-        npm ci
+    - run: npm ci
     - run: npm run build --if-present
     # - run: npm run test #Nothing is implemented here currently so this is omitted
     # - run: git config user.email "githubactions@commercehub.com" #skipping this for now since we don't have a servie account for LeoPlatform yet
     - run: git config user.name "GitHub-Actions"
     - run: npm version ${{ github.event.release.tag_name }}
-    - run: npm publish
+    - run: npm config get registry
+        #npm publish
       env:
         NODE_AUTH_TOKEN: ${{ secrets.GITHUB_TOKEN }}

From 75693760c0e7674aa2b18dc849826442936bf1c0 Mon Sep 17 00:00:00 2001
From: Michael Scranton <mscranton@commercehub.com>
Date: Tue, 25 Oct 2022 14:49:34 -0400
Subject: [PATCH 18/62] trying workflow things

---
 .github/workflows/publish-postgres.yml | 8 ++++++--
 1 file changed, 6 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/publish-postgres.yml b/.github/workflows/publish-postgres.yml
index 97471026..80569d3a 100644
--- a/.github/workflows/publish-postgres.yml
+++ b/.github/workflows/publish-postgres.yml
@@ -28,7 +28,11 @@ jobs:
     # - run: git config user.email "githubactions@commercehub.com" #skipping this for now since we don't have a servie account for LeoPlatform yet
     - run: git config user.name "GitHub-Actions"
     - run: npm version ${{ github.event.release.tag_name }}
-    - run: npm config get registry
-        #npm publish
+    - run: |
+        pwd
+        npm config get registry
+        cd ..
+        pwd
+        npm config get registry
       env:
         NODE_AUTH_TOKEN: ${{ secrets.GITHUB_TOKEN }}

From b3d83f71dc98b10645411ea701249d85845b4d4e Mon Sep 17 00:00:00 2001
From: Michael Scranton <mscranton@commercehub.com>
Date: Tue, 25 Oct 2022 15:00:52 -0400
Subject: [PATCH 19/62] trying workflow things

---
 .github/workflows/publish-postgres.yml | 4 ----
 postgres/package.json                  | 3 +++
 2 files changed, 3 insertions(+), 4 deletions(-)

diff --git a/.github/workflows/publish-postgres.yml b/.github/workflows/publish-postgres.yml
index 80569d3a..ba0d9f92 100644
--- a/.github/workflows/publish-postgres.yml
+++ b/.github/workflows/publish-postgres.yml
@@ -29,10 +29,6 @@ jobs:
     - run: git config user.name "GitHub-Actions"
     - run: npm version ${{ github.event.release.tag_name }}
     - run: |
-        pwd
-        npm config get registry
-        cd ..
-        pwd
         npm config get registry
       env:
         NODE_AUTH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
diff --git a/postgres/package.json b/postgres/package.json
index 9a8e88a1..9b19b625 100644
--- a/postgres/package.json
+++ b/postgres/package.json
@@ -1,6 +1,9 @@
 {
 	"name": "connector-postgres",
 	"description": "A Postgres database connector for use with Leo Platform",
+	"publishConfig": {
+		"registry": "https://npm.pkg.github.com"
+	},
 	"repository": {
 		"type": "git",
 		"url": "git://github.com/LeoPlatform/connectors.git"

From 98897fa9e3aea8f4e277c9fc519956f721934efc Mon Sep 17 00:00:00 2001
From: Michael Scranton <mscranton@commercehub.com>
Date: Tue, 25 Oct 2022 15:09:44 -0400
Subject: [PATCH 20/62] trying workflow things

---
 .github/workflows/publish-postgres.yml | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/.github/workflows/publish-postgres.yml b/.github/workflows/publish-postgres.yml
index ba0d9f92..7e9a9d80 100644
--- a/.github/workflows/publish-postgres.yml
+++ b/.github/workflows/publish-postgres.yml
@@ -28,6 +28,9 @@ jobs:
     # - run: git config user.email "githubactions@commercehub.com" #skipping this for now since we don't have a servie account for LeoPlatform yet
     - run: git config user.name "GitHub-Actions"
     - run: npm version ${{ github.event.release.tag_name }}
+    - run: echo "//npm.pkg.github.com/:_authToken=$NODE_AUTH_TOKEN" > .npmrc
+      env:
+        NODE_AUTH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
     - run: |
         npm config get registry
       env:

From 229365b19f4ecdaaf23114e0aaf8f5fda8269889 Mon Sep 17 00:00:00 2001
From: Michael Scranton <mscranton@commercehub.com>
Date: Tue, 25 Oct 2022 15:42:01 -0400
Subject: [PATCH 21/62] trying workflow things

---
 .github/workflows/publish-postgres.yml |   7 +-
 postgres/package-lock.json             | 423 +++++++++++++------------
 postgres/package.json                  |   7 +-
 3 files changed, 228 insertions(+), 209 deletions(-)

diff --git a/.github/workflows/publish-postgres.yml b/.github/workflows/publish-postgres.yml
index 7e9a9d80..04b19654 100644
--- a/.github/workflows/publish-postgres.yml
+++ b/.github/workflows/publish-postgres.yml
@@ -22,16 +22,13 @@ jobs:
         node-version: '12.x'
         # cache: 'npm'
         registry-url: 'https://npm.pkg.github.com'
+        scope: '@leoplatform'
     - run: npm ci
     - run: npm run build --if-present
     # - run: npm run test #Nothing is implemented here currently so this is omitted
     # - run: git config user.email "githubactions@commercehub.com" #skipping this for now since we don't have a servie account for LeoPlatform yet
     - run: git config user.name "GitHub-Actions"
     - run: npm version ${{ github.event.release.tag_name }}
-    - run: echo "//npm.pkg.github.com/:_authToken=$NODE_AUTH_TOKEN" > .npmrc
-      env:
-        NODE_AUTH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-    - run: |
-        npm config get registry
+    - run: npm publish --access=public 
       env:
         NODE_AUTH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
diff --git a/postgres/package-lock.json b/postgres/package-lock.json
index 21f20995..3538a0a0 100644
--- a/postgres/package-lock.json
+++ b/postgres/package-lock.json
@@ -1,38 +1,38 @@
 {
-	"name": "connector-postgres",
+	"name": "@leoplatform/connector-postgres",
 	"requires": true,
 	"lockfileVersion": 1,
 	"dependencies": {
 		"@babel/code-frame": {
-			"version": "7.8.3",
-			"resolved": "https://registry.npmjs.org/@babel/code-frame/-/code-frame-7.8.3.tgz",
-			"integrity": "sha512-a9gxpmdXtZEInkCSHUJDLHZVBgb1QS0jhss4cPP93EW7s+uC5bikET2twEF3KV+7rDblJcmNvTR7VJejqd2C2g==",
+			"version": "7.18.6",
+			"resolved": "https://registry.npmjs.org/@babel/code-frame/-/code-frame-7.18.6.tgz",
+			"integrity": "sha512-TDCmlK5eOvH+eH7cdAFlNXeVJqWIQ7gW9tY1GJIpUtFb6CmjVyq2VM3u71bOyR8CRihcCgMUYoDNyLXao3+70Q==",
 			"dev": true,
 			"requires": {
-				"@babel/highlight": "^7.8.3"
+				"@babel/highlight": "^7.18.6"
 			}
 		},
 		"@babel/helper-validator-identifier": {
-			"version": "7.9.0",
-			"resolved": "https://registry.npmjs.org/@babel/helper-validator-identifier/-/helper-validator-identifier-7.9.0.tgz",
-			"integrity": "sha512-6G8bQKjOh+of4PV/ThDm/rRqlU7+IGoJuofpagU5GlEl29Vv0RGqqt86ZGRV8ZuSOY3o+8yXl5y782SMcG7SHw==",
+			"version": "7.19.1",
+			"resolved": "https://registry.npmjs.org/@babel/helper-validator-identifier/-/helper-validator-identifier-7.19.1.tgz",
+			"integrity": "sha512-awrNfaMtnHUr653GgGEs++LlAvW6w+DcPrOliSMXWCKo597CwL5Acf/wWdNkf/tfEQE3mjkeD1YOVZOUV/od1w==",
 			"dev": true
 		},
 		"@babel/highlight": {
-			"version": "7.9.0",
-			"resolved": "https://registry.npmjs.org/@babel/highlight/-/highlight-7.9.0.tgz",
-			"integrity": "sha512-lJZPilxX7Op3Nv/2cvFdnlepPXDxi29wxteT57Q965oc5R9v86ztx0jfxVrTcBk8C2kcPkkDa2Z4T3ZsPPVWsQ==",
+			"version": "7.18.6",
+			"resolved": "https://registry.npmjs.org/@babel/highlight/-/highlight-7.18.6.tgz",
+			"integrity": "sha512-u7stbOuYjaPezCuLj29hNW1v64M2Md2qupEKP1fHc7WdOA3DgLh37suiSrZYY7haUB7iBeQZ9P1uiRF359do3g==",
 			"dev": true,
 			"requires": {
-				"@babel/helper-validator-identifier": "^7.9.0",
+				"@babel/helper-validator-identifier": "^7.18.6",
 				"chalk": "^2.0.0",
 				"js-tokens": "^4.0.0"
 			}
 		},
 		"@sinonjs/commons": {
-			"version": "1.7.1",
-			"resolved": "https://registry.npmjs.org/@sinonjs/commons/-/commons-1.7.1.tgz",
-			"integrity": "sha512-Debi3Baff1Qu1Unc3mjJ96MgpbwTn43S1+9yJ0llWygPwDNu2aaWBD6yc9y/Z8XDRNhx7U+u2UDg2OGQXkclUQ==",
+			"version": "1.8.3",
+			"resolved": "https://registry.npmjs.org/@sinonjs/commons/-/commons-1.8.3.tgz",
+			"integrity": "sha512-xkNcLAn/wZaX14RPlwizcKicDk9G3F8m2nU3L7Ukm5zBgTwiT0wsoFAHx9Jq56fJA1z/7uKGtCRu16sOUCLIHQ==",
 			"dev": true,
 			"requires": {
 				"type-detect": "4.0.8"
@@ -60,27 +60,27 @@
 			}
 		},
 		"@sinonjs/text-encoding": {
-			"version": "0.7.1",
-			"resolved": "https://registry.npmjs.org/@sinonjs/text-encoding/-/text-encoding-0.7.1.tgz",
-			"integrity": "sha512-+iTbntw2IZPb/anVDbypzfQa+ay64MW0Zo8aJ8gZPWMMK6/OubMVb6lUPMagqjOPnmtauXnFCACVl3O7ogjeqQ==",
+			"version": "0.7.2",
+			"resolved": "https://registry.npmjs.org/@sinonjs/text-encoding/-/text-encoding-0.7.2.tgz",
+			"integrity": "sha512-sXXKG+uL9IrKqViTtao2Ws6dy0znu9sOaP1di/jKGW1M6VssO8vlpXCQcpZ+jisQ1tTFAC5Jo/EOzFbggBagFQ==",
 			"dev": true
 		},
 		"acorn": {
-			"version": "6.4.1",
-			"resolved": "https://registry.npmjs.org/acorn/-/acorn-6.4.1.tgz",
-			"integrity": "sha512-ZVA9k326Nwrj3Cj9jlh3wGFutC2ZornPNARZwsNYqQYgN0EsV2d53w5RN/co65Ohn4sUAUtb1rSUAOD6XN9idA==",
+			"version": "6.4.2",
+			"resolved": "https://registry.npmjs.org/acorn/-/acorn-6.4.2.tgz",
+			"integrity": "sha512-XtGIhXwF8YM8bJhGxG5kXgjkEuNGLTkoYqVE+KMR+aspr4KGYmKYg7yUe3KghyQ9yheNwLnjmzh/7+gfDBmHCQ==",
 			"dev": true
 		},
 		"acorn-jsx": {
-			"version": "5.2.0",
-			"resolved": "https://registry.npmjs.org/acorn-jsx/-/acorn-jsx-5.2.0.tgz",
-			"integrity": "sha512-HiUX/+K2YpkpJ+SzBffkM/AQ2YE03S0U1kjTLVpoJdhZMOWy8qvXVN9JdLqv2QsaQ6MPYQIuNmwD8zOiYUofLQ==",
+			"version": "5.3.2",
+			"resolved": "https://registry.npmjs.org/acorn-jsx/-/acorn-jsx-5.3.2.tgz",
+			"integrity": "sha512-rq9s+JNhf0IChjtDXxllJ7g41oZk5SlXtp0LHwyA5cejwn7vKmKp4pPri6YEePv2PU65sAsegbXtIinmDFDXgQ==",
 			"dev": true
 		},
 		"ajv": {
-			"version": "6.12.0",
-			"resolved": "https://registry.npmjs.org/ajv/-/ajv-6.12.0.tgz",
-			"integrity": "sha512-D6gFiFA0RRLyUbvijN74DWAjXSFxWKaWP7mldxkVhyhAV3+SWA9HEJPHQ2c9soIeTFJqcSdFDGFgdqs1iUU2Hw==",
+			"version": "6.12.6",
+			"resolved": "https://registry.npmjs.org/ajv/-/ajv-6.12.6.tgz",
+			"integrity": "sha512-j3fVLgvTo527anyYyJOGTYJbG+vnnQYvE0m5mmkc1TK+nxAppkCLMIL0aZ4dblVCNoGShhm+kzE4ZUykBoMg4g==",
 			"dev": true,
 			"requires": {
 				"fast-deep-equal": "^3.1.1",
@@ -96,9 +96,9 @@
 			"dev": true
 		},
 		"ansi-regex": {
-			"version": "3.0.0",
-			"resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-3.0.0.tgz",
-			"integrity": "sha1-7QMXwyIGT3lGbAKWa922Bas32Zg=",
+			"version": "3.0.1",
+			"resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-3.0.1.tgz",
+			"integrity": "sha512-+O9Jct8wf++lXxxFc4hc8LsjaSq0HFzzL7cVsw8pRDIPdjKD2mT4ytDZlLuSBZ4cLKZFXIrMGO7DbQCtMJJMKw==",
 			"dev": true
 		},
 		"ansi-styles": {
@@ -122,7 +122,7 @@
 		"arguments-extended": {
 			"version": "0.0.3",
 			"resolved": "https://registry.npmjs.org/arguments-extended/-/arguments-extended-0.0.3.tgz",
-			"integrity": "sha1-YQfkkX0OtvCk3WYyD8Fa/HLvSUY=",
+			"integrity": "sha512-MNYdPKgCiywbgHAmNsYr1tSNLtfbSdwE1akZV+33hU9A8RG0lO5HAK9oMnw7y7bjYUhc04dJpcIBMUaPPYYtXg==",
 			"requires": {
 				"extended": "~0.0.3",
 				"is-extended": "~0.0.8"
@@ -131,7 +131,7 @@
 		"array-extended": {
 			"version": "0.0.11",
 			"resolved": "https://registry.npmjs.org/array-extended/-/array-extended-0.0.11.tgz",
-			"integrity": "sha1-1xRK50jek8pybxIQCdv/FibRZL0=",
+			"integrity": "sha512-Fe4Ti2YgM1onQgrcCD8dUhFuZgHQxzqylSl1C5IDJVVVqY5D07h8RghIXL9sZ6COZ0e+oTL5IusTv5eXABJ9Kw==",
 			"requires": {
 				"arguments-extended": "~0.0.3",
 				"extended": "~0.0.3",
@@ -141,7 +141,7 @@
 		"array-from": {
 			"version": "2.1.1",
 			"resolved": "https://registry.npmjs.org/array-from/-/array-from-2.1.1.tgz",
-			"integrity": "sha1-z+nYwmYoudxa7MYqn12PHzUsEZU=",
+			"integrity": "sha512-GQTc6Uupx1FCavi5mPzBvVT7nEOeWMmUA9P95wpfpW1XwMSKs+KaymD5C2Up7KAUKg/mYwbsUYzdZWcoajlNZg==",
 			"dev": true
 		},
 		"assertion-error": {
@@ -167,15 +167,15 @@
 		"backoff": {
 			"version": "2.5.0",
 			"resolved": "https://registry.npmjs.org/backoff/-/backoff-2.5.0.tgz",
-			"integrity": "sha1-9hbtqdPktmuMp/ynn2lXIsX44m8=",
+			"integrity": "sha512-wC5ihrnUXmR2douXmXLCe5O3zg3GKIyvRi/hi58a/XyRxVI+3/yM0PYueQOZXPXQ9pxBislYkw+sF9b7C/RuMA==",
 			"requires": {
 				"precond": "0.2"
 			}
 		},
 		"balanced-match": {
-			"version": "1.0.0",
-			"resolved": "https://registry.npmjs.org/balanced-match/-/balanced-match-1.0.0.tgz",
-			"integrity": "sha1-ibTRmasr7kneFk6gK4nORi1xt2c=",
+			"version": "1.0.2",
+			"resolved": "https://registry.npmjs.org/balanced-match/-/balanced-match-1.0.2.tgz",
+			"integrity": "sha512-3oSeUO0TMV67hN1AmbXsK4yaqU7tjiHlbxRDZOpH0KW9+CeX4bRAaX0Anxt0tx2MrpRpWwQaPwIlISEJhYU5Pw==",
 			"dev": true
 		},
 		"brace-expansion": {
@@ -206,16 +206,17 @@
 			"dev": true
 		},
 		"chai": {
-			"version": "4.2.0",
-			"resolved": "https://registry.npmjs.org/chai/-/chai-4.2.0.tgz",
-			"integrity": "sha512-XQU3bhBukrOsQCuwZndwGcCVQHyZi53fQ6Ys1Fym7E4olpIqqZZhhoFJoaKVvV17lWQoXYwgWN2nF5crA8J2jw==",
+			"version": "4.3.6",
+			"resolved": "https://registry.npmjs.org/chai/-/chai-4.3.6.tgz",
+			"integrity": "sha512-bbcp3YfHCUzMOvKqsztczerVgBKSsEijCySNlHHbX3VG1nskvqjz5Rfso1gGwD6w6oOV3eI60pKuMOV5MV7p3Q==",
 			"dev": true,
 			"requires": {
 				"assertion-error": "^1.1.0",
 				"check-error": "^1.0.2",
 				"deep-eql": "^3.0.1",
 				"get-func-name": "^2.0.0",
-				"pathval": "^1.1.0",
+				"loupe": "^2.3.1",
+				"pathval": "^1.1.1",
 				"type-detect": "^4.0.5"
 			}
 		},
@@ -239,22 +240,22 @@
 		"check-error": {
 			"version": "1.0.2",
 			"resolved": "https://registry.npmjs.org/check-error/-/check-error-1.0.2.tgz",
-			"integrity": "sha1-V00xLt2Iu13YkS6Sht1sCu1KrII=",
+			"integrity": "sha512-BrgHpW9NURQgzoNyjfq0Wu6VFO6D7IZEmJNdtgNqpzGG8RuNFHt2jQxWlAs4HMe119chBnv+34syEZtc6IhLtA==",
 			"dev": true
 		},
 		"cli-cursor": {
 			"version": "2.1.0",
 			"resolved": "https://registry.npmjs.org/cli-cursor/-/cli-cursor-2.1.0.tgz",
-			"integrity": "sha1-s12sN2R5+sw+lHR9QdDQ9SOP/LU=",
+			"integrity": "sha512-8lgKz8LmCRYZZQDpRyT2m5rKJ08TnU4tR9FFFW2rxpxR1FzWi4PQ/NfyODchAatHaUgnSPVcx/R5w6NuTBzFiw==",
 			"dev": true,
 			"requires": {
 				"restore-cursor": "^2.0.0"
 			}
 		},
 		"cli-width": {
-			"version": "2.2.0",
-			"resolved": "https://registry.npmjs.org/cli-width/-/cli-width-2.2.0.tgz",
-			"integrity": "sha1-/xnt6Kml5XkyQUewwR8PvLq+1jk=",
+			"version": "2.2.1",
+			"resolved": "https://registry.npmjs.org/cli-width/-/cli-width-2.2.1.tgz",
+			"integrity": "sha512-GRMWDxpOB6Dgk2E5Uo+3eEBvtOOlimMmpbFiKuLFnQzYDavtLFY3K5ona41jgN/WdRZtG7utuVSVTL4HbZHGkw==",
 			"dev": true
 		},
 		"color-convert": {
@@ -269,7 +270,7 @@
 		"color-name": {
 			"version": "1.1.3",
 			"resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.3.tgz",
-			"integrity": "sha1-p9BVi9icQveV3UIyj3QIMcpTvCU=",
+			"integrity": "sha512-72fSenhMw2HZMTVHeCA9KCmpEIbzWiQsjN+BHcBbS9vr1mtt+vJjPdksIBNUmKAW8TFUDPJK5SUU3QhE9NEXDw==",
 			"dev": true
 		},
 		"commander": {
@@ -281,7 +282,7 @@
 		"concat-map": {
 			"version": "0.0.1",
 			"resolved": "https://registry.npmjs.org/concat-map/-/concat-map-0.0.1.tgz",
-			"integrity": "sha1-2Klr13/Wjfd5OnMDajug1UBdR3s=",
+			"integrity": "sha512-/Srv4dswyQNBfohGpz9o6Yb3Gz3SrUDqBH5rTuhGR7ahtlbYKnVxw2bCFMRljaA7EXHaXZ8wsHdodFvbkhKmqg==",
 			"dev": true
 		},
 		"core-util-is": {
@@ -313,7 +314,7 @@
 		"date-extended": {
 			"version": "0.0.6",
 			"resolved": "https://registry.npmjs.org/date-extended/-/date-extended-0.0.6.tgz",
-			"integrity": "sha1-I4AtV90b94GIE/4MMuhRqG2iZ8k=",
+			"integrity": "sha512-v9a2QLTVn1GQGXf02TQaSvNfeXA/V1FL2Tr0OQYqjI5+L9T5jEtCpLYG01sxFk+m1OtwMxydkKa8NKcflANAoQ==",
 			"requires": {
 				"array-extended": "~0.0.3",
 				"extended": "~0.0.3",
@@ -321,18 +322,18 @@
 			}
 		},
 		"debug": {
-			"version": "4.1.1",
-			"resolved": "https://registry.npmjs.org/debug/-/debug-4.1.1.tgz",
-			"integrity": "sha512-pYAIzeRo8J6KPEaJ0VWOh5Pzkbw/RetuzehGM7QRRX5he4fPHx2rdKMB256ehJCkX+XRQm16eZLqLNS8RSZXZw==",
+			"version": "4.3.4",
+			"resolved": "https://registry.npmjs.org/debug/-/debug-4.3.4.tgz",
+			"integrity": "sha512-PRWFHuSU3eDtQJPvnNY7Jcket1j0t5OuOsFzPPzsekD52Zl8qUfFIPEiswXqIvHWGVHOgX+7G/vCNNhehwxfkQ==",
 			"dev": true,
 			"requires": {
-				"ms": "^2.1.1"
+				"ms": "2.1.2"
 			}
 		},
 		"declare.js": {
 			"version": "0.0.8",
 			"resolved": "https://registry.npmjs.org/declare.js/-/declare.js-0.0.8.tgz",
-			"integrity": "sha1-BHit/5VkwAT1Hfc9i8E0AZ0o3N4="
+			"integrity": "sha512-O659hy1gcHef7JnwtqdQlrj2c5DAEgtxm8pgFXofW7eUE1L4FjsSLlziovWcrOJAOFlEPaOJshY+0hBWCG/AnA=="
 		},
 		"deep-eql": {
 			"version": "3.0.1",
@@ -344,9 +345,9 @@
 			}
 		},
 		"deep-is": {
-			"version": "0.1.3",
-			"resolved": "https://registry.npmjs.org/deep-is/-/deep-is-0.1.3.tgz",
-			"integrity": "sha1-s2nW+128E+7PUk+RsHD+7cNXzzQ=",
+			"version": "0.1.4",
+			"resolved": "https://registry.npmjs.org/deep-is/-/deep-is-0.1.4.tgz",
+			"integrity": "sha512-oIPzksmTg4/MriiaYGO+okXDT7ztn/w3Eptv/+gSIdMdKsJo0u4CfYNFJPy+4SKMuCqGw2wxnA+URMg3t8a/bQ==",
 			"dev": true
 		},
 		"diff": {
@@ -397,7 +398,7 @@
 		"escape-string-regexp": {
 			"version": "1.0.5",
 			"resolved": "https://registry.npmjs.org/escape-string-regexp/-/escape-string-regexp-1.0.5.tgz",
-			"integrity": "sha1-G2HAViGQqN/2rjuyzwIAyhMLhtQ=",
+			"integrity": "sha512-vbRorB5FUQWvla16U8R/qgaFIya2qGzwDrNmCZuYKrbdSUMG6I1ZCGQRefkRVhuOkIGVne7BQ35DSfo1qvJqFg==",
 			"dev": true
 		},
 		"eslint": {
@@ -472,9 +473,9 @@
 			}
 		},
 		"eslint-visitor-keys": {
-			"version": "1.1.0",
-			"resolved": "https://registry.npmjs.org/eslint-visitor-keys/-/eslint-visitor-keys-1.1.0.tgz",
-			"integrity": "sha512-8y9YjtM1JBJU/A9Kc+SbaOV4y29sSWckBwMHa+FGtVj5gN/sbnKDf6xJUl+8g7FAij9LVaP8C24DUiH/f/2Z9A==",
+			"version": "1.3.0",
+			"resolved": "https://registry.npmjs.org/eslint-visitor-keys/-/eslint-visitor-keys-1.3.0.tgz",
+			"integrity": "sha512-6J72N8UNa462wa/KFODt/PJ3IU60SDpC3QXC1Hjc1BXXpfL2C9R5+AU7jhe0F6GREqVMh4Juu+NY7xn+6dipUQ==",
 			"dev": true
 		},
 		"espree": {
@@ -495,29 +496,37 @@
 			"dev": true
 		},
 		"esquery": {
-			"version": "1.2.0",
-			"resolved": "https://registry.npmjs.org/esquery/-/esquery-1.2.0.tgz",
-			"integrity": "sha512-weltsSqdeWIX9G2qQZz7KlTRJdkkOCTPgLYJUz1Hacf48R4YOwGPHO3+ORfWedqJKbq5WQmsgK90n+pFLIKt/Q==",
+			"version": "1.4.0",
+			"resolved": "https://registry.npmjs.org/esquery/-/esquery-1.4.0.tgz",
+			"integrity": "sha512-cCDispWt5vHHtwMY2YrAQ4ibFkAL8RbH5YGBnZBc90MolvvfkkQcJro/aZiAQUlQ3qgrYS6D6v8Gc5G5CQsc9w==",
 			"dev": true,
 			"requires": {
-				"estraverse": "^5.0.0"
+				"estraverse": "^5.1.0"
 			},
 			"dependencies": {
 				"estraverse": {
-					"version": "5.0.0",
-					"resolved": "https://registry.npmjs.org/estraverse/-/estraverse-5.0.0.tgz",
-					"integrity": "sha512-j3acdrMzqrxmJTNj5dbr1YbjacrYgAxVMeF0gK16E3j494mOe7xygM/ZLIguEQ0ETwAg2hlJCtHRGav+y0Ny5A==",
+					"version": "5.3.0",
+					"resolved": "https://registry.npmjs.org/estraverse/-/estraverse-5.3.0.tgz",
+					"integrity": "sha512-MMdARuVEQziNTeJD8DgMqmhwR11BRQ/cBP+pLtYdSTnf3MIO8fFeiINEbX36ZdNlfU/7A9f3gUw49B3oQsvwBA==",
 					"dev": true
 				}
 			}
 		},
 		"esrecurse": {
-			"version": "4.2.1",
-			"resolved": "https://registry.npmjs.org/esrecurse/-/esrecurse-4.2.1.tgz",
-			"integrity": "sha512-64RBB++fIOAXPw3P9cy89qfMlvZEXZkqqJkjqqXIvzP5ezRZjW+lPWjw35UX/3EhUPFYbg5ER4JYgDw4007/DQ==",
+			"version": "4.3.0",
+			"resolved": "https://registry.npmjs.org/esrecurse/-/esrecurse-4.3.0.tgz",
+			"integrity": "sha512-KmfKL3b6G+RXvP8N1vr3Tq1kL/oCFgn2NYXEtqP8/L3pKapUA4G8cFVaoF3SU323CD4XypR/ffioHmkti6/Tag==",
 			"dev": true,
 			"requires": {
-				"estraverse": "^4.1.0"
+				"estraverse": "^5.2.0"
+			},
+			"dependencies": {
+				"estraverse": {
+					"version": "5.3.0",
+					"resolved": "https://registry.npmjs.org/estraverse/-/estraverse-5.3.0.tgz",
+					"integrity": "sha512-MMdARuVEQziNTeJD8DgMqmhwR11BRQ/cBP+pLtYdSTnf3MIO8fFeiINEbX36ZdNlfU/7A9f3gUw49B3oQsvwBA==",
+					"dev": true
+				}
 			}
 		},
 		"estraverse": {
@@ -554,7 +563,7 @@
 		"extended": {
 			"version": "0.0.6",
 			"resolved": "https://registry.npmjs.org/extended/-/extended-0.0.6.tgz",
-			"integrity": "sha1-f7i/e52uOXWG5IVwrP1kLHjlBmk=",
+			"integrity": "sha512-rvAV3BDGsV1SYGzUOu7aO0k82quhfl0QAyZudYhAcTeIr1rPbBnyOhOlkCLwLpDfP7HyKAWAPNSjRb9p7lE3rg==",
 			"requires": {
 				"extender": "~0.0.5"
 			}
@@ -562,7 +571,7 @@
 		"extender": {
 			"version": "0.0.10",
 			"resolved": "https://registry.npmjs.org/extender/-/extender-0.0.10.tgz",
-			"integrity": "sha1-WJwHSCvmGhRgttgfnCSqZ+jzJM0=",
+			"integrity": "sha512-iPLUHZJaNW6RuOShQX33ZpewEUIlijFBcsXnKWyiYERKWPsFxfKgx8J0xRz29hKQWPFFPACgBW6cHM7Ke1pfaA==",
 			"requires": {
 				"declare.js": "~0.0.4"
 			}
@@ -581,7 +590,7 @@
 		"fast-csv": {
 			"version": "2.4.1",
 			"resolved": "https://registry.npmjs.org/fast-csv/-/fast-csv-2.4.1.tgz",
-			"integrity": "sha1-vX3SaDkfcpNntZRFuN0K0CaIGyY=",
+			"integrity": "sha512-DcNumesovr8oTHMdfVWN9hJeiMALO04c1iLExpxV2+yoTW9Jyv239di81HKL8IQwAqC+T1TYoQsd2XStFb+DhA==",
 			"requires": {
 				"extended": "0.0.6",
 				"is-extended": "0.0.10",
@@ -590,9 +599,9 @@
 			}
 		},
 		"fast-deep-equal": {
-			"version": "3.1.1",
-			"resolved": "https://registry.npmjs.org/fast-deep-equal/-/fast-deep-equal-3.1.1.tgz",
-			"integrity": "sha512-8UEa58QDLauDNfpbrX55Q9jrGHThw2ZMdOky5Gl1CDtVeJDPVrG4Jxx1N8jw2gkWaff5UUuX1KJd+9zGe2B+ZA==",
+			"version": "3.1.3",
+			"resolved": "https://registry.npmjs.org/fast-deep-equal/-/fast-deep-equal-3.1.3.tgz",
+			"integrity": "sha512-f3qQ9oQy9j2AhBe/H9VC91wLmKBCCU/gDOnKNAYG5hswO7BLKj09Hc5HYNz9cGI++xlpDCIgDaitVs03ATR84Q==",
 			"dev": true
 		},
 		"fast-json-stable-stringify": {
@@ -604,13 +613,13 @@
 		"fast-levenshtein": {
 			"version": "2.0.6",
 			"resolved": "https://registry.npmjs.org/fast-levenshtein/-/fast-levenshtein-2.0.6.tgz",
-			"integrity": "sha1-PYpcZog6FqMMqGQ+hR8Zuqd5eRc=",
+			"integrity": "sha512-DCXu6Ifhqcks7TZKY3Hxp3y6qphY5SJZmrWMDrKcERSOXWQdMhU9Ig/PYrzyw/ul9jOIyh0N4M0tbC5hodg8dw==",
 			"dev": true
 		},
 		"figures": {
 			"version": "2.0.0",
 			"resolved": "https://registry.npmjs.org/figures/-/figures-2.0.0.tgz",
-			"integrity": "sha1-OrGi0qYsi/tDGgyUy3l6L84nyWI=",
+			"integrity": "sha512-Oa2M9atig69ZkfwiApY8F2Yy+tzMbazyvqv21R0NsSC8floSOC09BbT1ITWAdoMGQvJ/aZnR1KMwdx9tvHnTNA==",
 			"dev": true,
 			"requires": {
 				"escape-string-regexp": "^1.0.5"
@@ -637,9 +646,9 @@
 			}
 		},
 		"flatted": {
-			"version": "2.0.1",
-			"resolved": "https://registry.npmjs.org/flatted/-/flatted-2.0.1.tgz",
-			"integrity": "sha512-a1hQMktqW9Nmqr5aktAux3JMNqaucxGcjtjWnZLHX7yyPCmlSV3M54nGYbqT8K+0GhF3NBgmJCc3ma+WOgX8Jg==",
+			"version": "2.0.2",
+			"resolved": "https://registry.npmjs.org/flatted/-/flatted-2.0.2.tgz",
+			"integrity": "sha512-r5wGx7YeOwNWNlCA0wQ86zKyDLMQr+/RB8xy74M4hTphfmjlijTSSXGuH8rnvKZnfT9i+75zmd8jcKdMR4O6jA==",
 			"dev": true
 		},
 		"flush-write-stream": {
@@ -666,36 +675,36 @@
 		"from": {
 			"version": "0.1.7",
 			"resolved": "https://registry.npmjs.org/from/-/from-0.1.7.tgz",
-			"integrity": "sha1-g8YK/Fi5xWmXAH7Rp2izqzA6RP4="
+			"integrity": "sha512-twe20eF1OxVxp/ML/kq2p1uc6KvFK/+vs8WjEbeKmV2He22MKm7YF2ANIt+EOqhJ5L3K/SuuPhk0hWQDjOM23g=="
 		},
 		"fs.realpath": {
 			"version": "1.0.0",
 			"resolved": "https://registry.npmjs.org/fs.realpath/-/fs.realpath-1.0.0.tgz",
-			"integrity": "sha1-FQStJSMVjKpA20onh8sBQRmU6k8=",
+			"integrity": "sha512-OO0pH2lK6a0hZnAdau5ItzHPI6pUlvI7jMVnxUQRtw4owF2wk8lOSabtGDCTP4Ggrg2MbGnWO9X8K1t4+fGMDw==",
 			"dev": true
 		},
 		"functional-red-black-tree": {
 			"version": "1.0.1",
 			"resolved": "https://registry.npmjs.org/functional-red-black-tree/-/functional-red-black-tree-1.0.1.tgz",
-			"integrity": "sha1-GwqzvVU7Kg1jmdKcDj6gslIHgyc=",
+			"integrity": "sha512-dsKNQNdj6xA3T+QlADDA7mOSlX0qiMINjn0cgr+eGHGsbSHzTabcIogz2+p/iqP1Xs6EP/sS2SbqH+brGTbq0g==",
 			"dev": true
 		},
 		"get-func-name": {
 			"version": "2.0.0",
 			"resolved": "https://registry.npmjs.org/get-func-name/-/get-func-name-2.0.0.tgz",
-			"integrity": "sha1-6td0q+5y4gQJQzoGY2YCPdaIekE=",
+			"integrity": "sha512-Hm0ixYtaSZ/V7C8FJrtZIuBBI+iSgL+1Aq82zSu8VQNB4S3Gk8e7Qs3VwBDJAhmRZcFqkl3tQu36g/Foh5I5ig==",
 			"dev": true
 		},
 		"glob": {
-			"version": "7.1.6",
-			"resolved": "https://registry.npmjs.org/glob/-/glob-7.1.6.tgz",
-			"integrity": "sha512-LwaxwyZ72Lk7vZINtNNrywX0ZuLyStrdDtabefZKAY5ZGJhVtgdznluResxNmPitE0SAO+O26sWTHeKSI2wMBA==",
+			"version": "7.2.3",
+			"resolved": "https://registry.npmjs.org/glob/-/glob-7.2.3.tgz",
+			"integrity": "sha512-nFR0zLpU2YCaRxwoCJvL6UvCH2JFyFVIvwTLsIf21AuHlMskA1hhTdk+LlYJtOlYt9v6dvszD2BGRqBL+iQK9Q==",
 			"dev": true,
 			"requires": {
 				"fs.realpath": "^1.0.0",
 				"inflight": "^1.0.4",
 				"inherits": "2",
-				"minimatch": "^3.0.4",
+				"minimatch": "^3.1.1",
 				"once": "^1.3.0",
 				"path-is-absolute": "^1.0.0"
 			}
@@ -715,13 +724,13 @@
 		"has-flag": {
 			"version": "3.0.0",
 			"resolved": "https://registry.npmjs.org/has-flag/-/has-flag-3.0.0.tgz",
-			"integrity": "sha1-tdRU3CGZriJWmfNGfloH87lVuv0=",
+			"integrity": "sha512-sKJf1+ceQBr4SMkvQnBDNDtf4TXpVhVGateu0t918bl30FnbE2m4vNLX+VWe/dpjlb+HugGYzW7uQXH98HPEYw==",
 			"dev": true
 		},
 		"he": {
 			"version": "1.1.1",
 			"resolved": "https://registry.npmjs.org/he/-/he-1.1.1.tgz",
-			"integrity": "sha1-k0EP0hsAlzUVH4howvJx80J+I/0=",
+			"integrity": "sha512-z/GDPjlRMNOa2XJiB4em8wJpuuBfrFOlYKTZxtpkdr1uPdibHI8rYA3MY0KDObpVyaes0e/aunid/t88ZI2EKA==",
 			"dev": true
 		},
 		"iconv-lite": {
@@ -740,9 +749,9 @@
 			"dev": true
 		},
 		"import-fresh": {
-			"version": "3.2.1",
-			"resolved": "https://registry.npmjs.org/import-fresh/-/import-fresh-3.2.1.tgz",
-			"integrity": "sha512-6e1q1cnWP2RXD9/keSkxHScg508CdXqXWgWBaETNhyuBFz+kUZlKboh+ISK+bU++DmbHimVBrOz/zzPe0sZ3sQ==",
+			"version": "3.3.0",
+			"resolved": "https://registry.npmjs.org/import-fresh/-/import-fresh-3.3.0.tgz",
+			"integrity": "sha512-veYYhQa+D1QBKznvhUHxb8faxlrwUnxseDAbAp457E0wLNio2bOSKnjYDhMj+YiAq61xrMGhQk9iXVk5FzgQMw==",
 			"dev": true,
 			"requires": {
 				"parent-module": "^1.0.0",
@@ -752,13 +761,13 @@
 		"imurmurhash": {
 			"version": "0.1.4",
 			"resolved": "https://registry.npmjs.org/imurmurhash/-/imurmurhash-0.1.4.tgz",
-			"integrity": "sha1-khi5srkoojixPcT7a21XbyMUU+o=",
+			"integrity": "sha512-JmXMZ6wuvDmLiHEml9ykzqO6lwFbof0GG4IkcGaENdCRDDmMVnny7s5HsIgHCbaq0w2MyPhDqkhTUgS2LU2PHA==",
 			"dev": true
 		},
 		"inflight": {
 			"version": "1.0.6",
 			"resolved": "https://registry.npmjs.org/inflight/-/inflight-1.0.6.tgz",
-			"integrity": "sha1-Sb1jMdfQLQwJvJEKEHW6gWW1bfk=",
+			"integrity": "sha512-k92I/b08q4wvFscXCLvqfsHCrjrF7yiXsQuIVvVE7N82W3+aqpzuUdBbfhWcy/FZR3/4IgflMgKLOsvPDrGCJA==",
 			"dev": true,
 			"requires": {
 				"once": "^1.3.0",
@@ -797,9 +806,9 @@
 			},
 			"dependencies": {
 				"ansi-regex": {
-					"version": "4.1.0",
-					"resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-4.1.0.tgz",
-					"integrity": "sha512-1apePfXM1UOSqw0o9IiFAovVz9M5S1Dg+4TrDwfMewQ6p/rmMueb7tWZjQ1rx4Loy1ArBggoqGpfqqdI4rondg==",
+					"version": "4.1.1",
+					"resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-4.1.1.tgz",
+					"integrity": "sha512-ILlv4k/3f6vfQ4OoP2AGvirOktlQ98ZEL1k9FaQjxa3L1abBgbuTDAdPOpvbGncC0BTVQrl+OM8xZGK6tWXt7g==",
 					"dev": true
 				},
 				"strip-ansi": {
@@ -816,7 +825,7 @@
 		"is-extended": {
 			"version": "0.0.10",
 			"resolved": "https://registry.npmjs.org/is-extended/-/is-extended-0.0.10.tgz",
-			"integrity": "sha1-JE4UDfdbscmjEG9BL/GC+1NKbWI=",
+			"integrity": "sha512-qp+HR+L9QXbgFurvqiVgD+JiGyUboRgICNzCXmbiLtZBFVSNFbxRsI4q7Be9mCWTO5PoO1IxoWp5sl+j5b83FA==",
 			"requires": {
 				"extended": "~0.0.3"
 			}
@@ -824,24 +833,18 @@
 		"is-fullwidth-code-point": {
 			"version": "2.0.0",
 			"resolved": "https://registry.npmjs.org/is-fullwidth-code-point/-/is-fullwidth-code-point-2.0.0.tgz",
-			"integrity": "sha1-o7MKXE8ZkYMWeqq5O+764937ZU8=",
-			"dev": true
-		},
-		"is-promise": {
-			"version": "2.1.0",
-			"resolved": "https://registry.npmjs.org/is-promise/-/is-promise-2.1.0.tgz",
-			"integrity": "sha1-eaKp7OfwlugPNtKy87wWwf9L8/o=",
+			"integrity": "sha512-VHskAKYM8RfSFXwee5t5cbN5PZeq1Wrh6qd5bkyiXIf6UQcN6w/A0eXM9r6t8d+GYOh+o6ZhiEnb88LN/Y8m2w==",
 			"dev": true
 		},
 		"isarray": {
 			"version": "1.0.0",
 			"resolved": "https://registry.npmjs.org/isarray/-/isarray-1.0.0.tgz",
-			"integrity": "sha1-u5NdSFgsuhaMBoNJV6VKPgcSTxE="
+			"integrity": "sha512-VLghIWNM6ELQzo7zwmcg0NmTVyWKYjvIeM83yjp0wRDTmUnrM678fQbcKBo6n2CJEF0szoG//ytg+TKla89ALQ=="
 		},
 		"isexe": {
 			"version": "2.0.0",
 			"resolved": "https://registry.npmjs.org/isexe/-/isexe-2.0.0.tgz",
-			"integrity": "sha1-6PvzdNxVb/iUehDcsFctYz8s+hA=",
+			"integrity": "sha512-RHxMLp9lnKHGHRng9QFhRCMbYAcVpn69smSGcq3f36xjgVVWThj4qqLbTLlq7Ssj8B+fIQ1EuCEGI2lKsyQeIw==",
 			"dev": true
 		},
 		"js-tokens": {
@@ -851,9 +854,9 @@
 			"dev": true
 		},
 		"js-yaml": {
-			"version": "3.13.1",
-			"resolved": "https://registry.npmjs.org/js-yaml/-/js-yaml-3.13.1.tgz",
-			"integrity": "sha512-YfbcO7jXDdyj0DGxYVSlSeQNHbD7XPWvrVWeVUujrQEoZzWJIRrCPoyk6kL6IAjAG2IolMK4T0hNUe0HOUs5Jw==",
+			"version": "3.14.1",
+			"resolved": "https://registry.npmjs.org/js-yaml/-/js-yaml-3.14.1.tgz",
+			"integrity": "sha512-okMH7OXXJ7YrN9Ok3/SXrnu4iX9yOk+25nqX4imS2npuvTYDmo/QEZoqwZkYaIDk3jVvBOTOIEgEhaLOynBS9g==",
 			"dev": true,
 			"requires": {
 				"argparse": "^1.0.7",
@@ -869,13 +872,13 @@
 		"json-stable-stringify-without-jsonify": {
 			"version": "1.0.1",
 			"resolved": "https://registry.npmjs.org/json-stable-stringify-without-jsonify/-/json-stable-stringify-without-jsonify-1.0.1.tgz",
-			"integrity": "sha1-nbe1lJatPzz+8wp1FC0tkwrXJlE=",
+			"integrity": "sha512-Bdboy+l7tA3OGW6FjyFHWkP5LuByj1Tk33Ljyq0axyzdk9//JSi2u3fP1QSmd1KNwq6VOKYGlAu87CisVir6Pw==",
 			"dev": true
 		},
 		"just-extend": {
-			"version": "4.1.0",
-			"resolved": "https://registry.npmjs.org/just-extend/-/just-extend-4.1.0.tgz",
-			"integrity": "sha512-ApcjaOdVTJ7y4r08xI5wIqpvwS48Q0PBG4DJROcEkH1f8MdAiNFyFxz3xoL0LWAVwjrwPYZdVHHxhRHcx/uGLA==",
+			"version": "4.2.1",
+			"resolved": "https://registry.npmjs.org/just-extend/-/just-extend-4.2.1.tgz",
+			"integrity": "sha512-g3UB796vUFIY90VIv/WX3L2c8CS2MdWUww3CNrYmqza1Fg0DURc2K/O4YrnklBdQarSJ/y8JnJYDGc+1iumQjg==",
 			"dev": true
 		},
 		"leo-aws": {
@@ -1021,7 +1024,7 @@
 		"levn": {
 			"version": "0.3.0",
 			"resolved": "https://registry.npmjs.org/levn/-/levn-0.3.0.tgz",
-			"integrity": "sha1-OwmSTt+fCDwEkP3UwLxEIeBHZO4=",
+			"integrity": "sha512-0OO4y2iOHix2W6ujICbKIaEQXvFQHue65vUG3pb5EUomzPI90z9hsA1VsO/dbIIpC53J8gxM9Q4Oho0jrCM/yA==",
 			"dev": true,
 			"requires": {
 				"prelude-ls": "~1.1.2",
@@ -1029,9 +1032,9 @@
 			}
 		},
 		"lodash": {
-			"version": "4.17.15",
-			"resolved": "https://registry.npmjs.org/lodash/-/lodash-4.17.15.tgz",
-			"integrity": "sha512-8xOcRHvCjnocdS5cpwXQXVzmmh5e5+saE2QGoeQmbKmRS6J3VQppPOIt0MnmE+4xlZoumy0GPG0D0MVIQbNA1A=="
+			"version": "4.17.21",
+			"resolved": "https://registry.npmjs.org/lodash/-/lodash-4.17.21.tgz",
+			"integrity": "sha512-v2kDEe57lecTulaDIuNTPy3Ry4gLGJ6Z1O3vE1krgXZNrsQ+LFTGHVxVjcXPs17LhbZVGedAJv8XZ1tvj5FvSg=="
 		},
 		"lodash.merge": {
 			"version": "4.6.2",
@@ -1044,10 +1047,19 @@
 			"integrity": "sha512-gKO5uExCXvSm6zbF562EvM+rd1kQDnB9AZBbiQVzf1ZmdDpxUSvpnAaVOP83N/31mRK8Ml8/VE8DMvsAZQ+7wg==",
 			"dev": true
 		},
+		"loupe": {
+			"version": "2.3.4",
+			"resolved": "https://registry.npmjs.org/loupe/-/loupe-2.3.4.tgz",
+			"integrity": "sha512-OvKfgCC2Ndby6aSTREl5aCCPTNIzlDfQZvZxNUrBrihDhL3xcrYegTblhmEiCrg2kKQz4XsFIaemE5BF4ybSaQ==",
+			"dev": true,
+			"requires": {
+				"get-func-name": "^2.0.0"
+			}
+		},
 		"map-stream": {
 			"version": "0.0.7",
 			"resolved": "https://registry.npmjs.org/map-stream/-/map-stream-0.0.7.tgz",
-			"integrity": "sha1-ih8HiW2CsQkmvTdEokIACfiJdKg="
+			"integrity": "sha512-C0X0KQmGm3N2ftbTGBhSyuydQ+vV1LC3f3zPvT3RXHXNZrvfPZcoXp/N5DOa8vedX/rTMm2CjTtivFg2STJMRQ=="
 		},
 		"mimic-fn": {
 			"version": "1.2.0",
@@ -1056,27 +1068,27 @@
 			"dev": true
 		},
 		"minimatch": {
-			"version": "3.0.4",
-			"resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.0.4.tgz",
-			"integrity": "sha512-yJHVQEhyqPLUTgt9B83PXu6W3rx4MvvHvSUvToogpwoGDOUQ+yDrR0HRot+yOCdCO7u4hX3pWft6kWBBcqh0UA==",
+			"version": "3.1.2",
+			"resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.1.2.tgz",
+			"integrity": "sha512-J7p63hRiAjw1NDEww1W7i37+ByIrOWO5XQQAzZ3VOcL0PNybwpfmV/N05zFAzwQ9USyEcX6t3UO+K5aqBQOIHw==",
 			"dev": true,
 			"requires": {
 				"brace-expansion": "^1.1.7"
 			}
 		},
 		"minimist": {
-			"version": "1.2.5",
-			"resolved": "https://registry.npmjs.org/minimist/-/minimist-1.2.5.tgz",
-			"integrity": "sha512-FM9nNUYrRBAELZQT3xeZQ7fmMOBg6nWNmJKTcgsJeaLstP/UODVpGsr5OhXhhXg6f+qtJ8uiZ+PUxkDWcgIXLw==",
+			"version": "1.2.7",
+			"resolved": "https://registry.npmjs.org/minimist/-/minimist-1.2.7.tgz",
+			"integrity": "sha512-bzfL1YUZsP41gmu/qjrEk0Q6i2ix/cVeAhbCbqH9u3zYutS1cLg00qhrD0M2MVdCcx4Sc0UpP2eBWo9rotpq6g==",
 			"dev": true
 		},
 		"mkdirp": {
-			"version": "0.5.3",
-			"resolved": "https://registry.npmjs.org/mkdirp/-/mkdirp-0.5.3.tgz",
-			"integrity": "sha512-P+2gwrFqx8lhew375MQHHeTlY8AuOJSrGf0R5ddkEndUkmwpgUob/vQuBD1V22/Cw1/lJr4x+EjllSezBThzBg==",
+			"version": "0.5.6",
+			"resolved": "https://registry.npmjs.org/mkdirp/-/mkdirp-0.5.6.tgz",
+			"integrity": "sha512-FP+p8RB8OWpF3YZBCrP5gtADmtXApB5AMLn+vdyA+PyxCjrCs00mjyUozssO33cwDeT3wNGdLxJ5M//YqtHAJw==",
 			"dev": true,
 			"requires": {
-				"minimist": "^1.2.5"
+				"minimist": "^1.2.6"
 			}
 		},
 		"mocha": {
@@ -1121,16 +1133,25 @@
 						"path-is-absolute": "^1.0.0"
 					}
 				},
+				"minimatch": {
+					"version": "3.0.4",
+					"resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.0.4.tgz",
+					"integrity": "sha512-yJHVQEhyqPLUTgt9B83PXu6W3rx4MvvHvSUvToogpwoGDOUQ+yDrR0HRot+yOCdCO7u4hX3pWft6kWBBcqh0UA==",
+					"dev": true,
+					"requires": {
+						"brace-expansion": "^1.1.7"
+					}
+				},
 				"minimist": {
 					"version": "0.0.8",
 					"resolved": "https://registry.npmjs.org/minimist/-/minimist-0.0.8.tgz",
-					"integrity": "sha1-hX/Kv8M5fSYluCKCYuhqp6ARsF0=",
+					"integrity": "sha512-miQKw5Hv4NS1Psg2517mV4e4dYNaO3++hjAvLOAzKqZ61rH8NS1SK+vbfBWZ5PY/Me/bEWhUwqMghEW5Fb9T7Q==",
 					"dev": true
 				},
 				"mkdirp": {
 					"version": "0.5.1",
 					"resolved": "https://registry.npmjs.org/mkdirp/-/mkdirp-0.5.1.tgz",
-					"integrity": "sha1-MAV0OOrGz3+MR2fzhkjWaX11yQM=",
+					"integrity": "sha512-SknJC52obPfGQPnjIkXbmA6+5H15E+fR+E4iR2oQ3zzCLbd7/ONua69R/Gw7AgkTLsRG+r5fzksYwWe1AgTyWA==",
 					"dev": true,
 					"requires": {
 						"minimist": "0.0.8"
@@ -1139,7 +1160,7 @@
 				"ms": {
 					"version": "2.0.0",
 					"resolved": "https://registry.npmjs.org/ms/-/ms-2.0.0.tgz",
-					"integrity": "sha1-VgiurfwAvmwpAd9fmGF4jeDVl8g=",
+					"integrity": "sha512-Tpp60P6IUJDTuOq/5Z8cdskzJujfwqfOTkrwIwj7IRISpnkJnT6SyJ4PCPnGMoFjC9ddhal5KVIYtAt97ix05A==",
 					"dev": true
 				},
 				"supports-color": {
@@ -1167,13 +1188,13 @@
 		"mute-stream": {
 			"version": "0.0.7",
 			"resolved": "https://registry.npmjs.org/mute-stream/-/mute-stream-0.0.7.tgz",
-			"integrity": "sha1-MHXOk7whuPq0PhvE2n6BFe0ee6s=",
+			"integrity": "sha512-r65nCZhrbXXb6dXOACihYApHw2Q6pV0M3V0PSxd74N0+D8nzAdEAITq2oAjA1jVnKI+tGvEBUpqiMh0+rW6zDQ==",
 			"dev": true
 		},
 		"natural-compare": {
 			"version": "1.4.0",
 			"resolved": "https://registry.npmjs.org/natural-compare/-/natural-compare-1.4.0.tgz",
-			"integrity": "sha1-Sr6/7tdUHywnrPspvbvRXI1bpPc=",
+			"integrity": "sha512-OWND8ei3VtNC9h7V60qff3SVobHr996CTwgxubgyQYEpg290h9J0buyECNNJexkFm5sOajh5G116RYA1c8ZMSw==",
 			"dev": true
 		},
 		"nice-try": {
@@ -1209,7 +1230,7 @@
 		"object-extended": {
 			"version": "0.0.7",
 			"resolved": "https://registry.npmjs.org/object-extended/-/object-extended-0.0.7.tgz",
-			"integrity": "sha1-hP0j9WsVWCrrPoiwXLVdJDLWijM=",
+			"integrity": "sha512-2LJYIacEXoZ1glGkAZuvA/4pfJM4Y1ShReAo9jWpBSuz89TiUCdiPqhGJJ6m97F3WjhCSRwrbgaxYEAm9dRYBw==",
 			"requires": {
 				"array-extended": "~0.0.4",
 				"extended": "~0.0.3",
@@ -1219,7 +1240,7 @@
 		"once": {
 			"version": "1.4.0",
 			"resolved": "https://registry.npmjs.org/once/-/once-1.4.0.tgz",
-			"integrity": "sha1-WDsap3WWHUsROsF9nFC6753Xa9E=",
+			"integrity": "sha512-lNaJgI+2Q5URQBkccEKHTQOPaXdUxnZZElQTZY0MFUAuaEqe1E+Nyvgdz/aIyNi6Z9MzO5dv1H8n58/GELp3+w==",
 			"requires": {
 				"wrappy": "1"
 			}
@@ -1227,7 +1248,7 @@
 		"onetime": {
 			"version": "2.0.1",
 			"resolved": "https://registry.npmjs.org/onetime/-/onetime-2.0.1.tgz",
-			"integrity": "sha1-BnQoIw/WdEOyeUsiu6UotoZ5YtQ=",
+			"integrity": "sha512-oyyPpiMaKARvvcgip+JV+7zci5L8D1W9RZIz2l1o08AM3pfspitVWnPt3mzHcBPp12oYMTy0pqrFs/C+m3EwsQ==",
 			"dev": true,
 			"requires": {
 				"mimic-fn": "^1.0.0"
@@ -1250,7 +1271,7 @@
 		"os-tmpdir": {
 			"version": "1.0.2",
 			"resolved": "https://registry.npmjs.org/os-tmpdir/-/os-tmpdir-1.0.2.tgz",
-			"integrity": "sha1-u+Z0BseaqFxc/sdm/lc0VV36EnQ=",
+			"integrity": "sha512-D2FR03Vir7FIu45XBY20mTb+/ZSWB00sjU9jdQXt83gDrI4Ztz5Fs7/yy74g2N5SVQY4xY1qDr4rNddwYRVX0g==",
 			"dev": true
 		},
 		"packet-reader": {
@@ -1270,19 +1291,19 @@
 		"path-is-absolute": {
 			"version": "1.0.1",
 			"resolved": "https://registry.npmjs.org/path-is-absolute/-/path-is-absolute-1.0.1.tgz",
-			"integrity": "sha1-F0uSaHNVNP+8es5r9TpanhtcX18=",
+			"integrity": "sha512-AVbw3UJ2e9bq64vSaS9Am0fje1Pa8pbGqTTsmXfaIiMpnr5DlDhfJOuLj9Sf95ZPVDAUerDfEk88MPmPe7UCQg==",
 			"dev": true
 		},
 		"path-is-inside": {
 			"version": "1.0.2",
 			"resolved": "https://registry.npmjs.org/path-is-inside/-/path-is-inside-1.0.2.tgz",
-			"integrity": "sha1-NlQX3t5EQw0cEa9hAn+s8HS9/FM=",
+			"integrity": "sha512-DUWJr3+ULp4zXmol/SZkFf3JGsS9/SIv+Y3Rt93/UjPpDpklB5f1er4O3POIbUuUJ3FXgqte2Q7SrU6zAqwk8w==",
 			"dev": true
 		},
 		"path-key": {
 			"version": "2.0.1",
 			"resolved": "https://registry.npmjs.org/path-key/-/path-key-2.0.1.tgz",
-			"integrity": "sha1-QRyttXTFoUDTpLGRDUDYDMn0C0A=",
+			"integrity": "sha512-fEHGKCSmUSDPv4uoj8AlD+joPlq3peND+HRYyxFz4KPw4z926S/b8rIuFs2FYJg3BwsxJf6A9/3eIdLaYC+9Dw==",
 			"dev": true
 		},
 		"path-to-regexp": {
@@ -1297,21 +1318,21 @@
 				"isarray": {
 					"version": "0.0.1",
 					"resolved": "https://registry.npmjs.org/isarray/-/isarray-0.0.1.tgz",
-					"integrity": "sha1-ihis/Kmo9Bd+Cav8YDiTmwXR7t8=",
+					"integrity": "sha512-D2S+3GLxWH+uhrNEcoh/fnmYeP8E8/zHl644d/jdA0g2uyXvy3sb0qxotE+ne0LtccHknQzWwZEzhak7oJ0COQ==",
 					"dev": true
 				}
 			}
 		},
 		"pathval": {
-			"version": "1.1.0",
-			"resolved": "https://registry.npmjs.org/pathval/-/pathval-1.1.0.tgz",
-			"integrity": "sha1-uULm1L3mUwBe9rcTYd74cn0GReA=",
+			"version": "1.1.1",
+			"resolved": "https://registry.npmjs.org/pathval/-/pathval-1.1.1.tgz",
+			"integrity": "sha512-Dp6zGqpTdETdR63lehJYPeIOqpiNBNtc7BpWSLrOje7UaIsE5aY92r/AunQA7rsXvet3lrJ3JnZX29UPTKXyKQ==",
 			"dev": true
 		},
 		"pause-stream": {
 			"version": "0.0.11",
 			"resolved": "https://registry.npmjs.org/pause-stream/-/pause-stream-0.0.11.tgz",
-			"integrity": "sha1-/lo0sMvOErWqaitAPuLnO2AvFEU=",
+			"integrity": "sha512-e3FBlXLmN/D1S+zHzanP4E/4Z60oFAa3O051qt1pxa7DEJWKAyil6upYVXCWadEnuoqa4Pkc9oUx9zsxYeRv8A==",
 			"requires": {
 				"through": "~2.3"
 			}
@@ -1333,7 +1354,7 @@
 		"pg-connection-string": {
 			"version": "0.1.3",
 			"resolved": "https://registry.npmjs.org/pg-connection-string/-/pg-connection-string-0.1.3.tgz",
-			"integrity": "sha1-2hhHsglA5C7hSSvq9l1J2RskXfc="
+			"integrity": "sha512-i0NV/CrSkFTaiOQs9AGy3tq0dkSjtTd4d7DfsjeDVZAA4aIHInwfFEmriNYGGJUfZ5x6IAC/QddoUpUJjQAi0w=="
 		},
 		"pg-copy-streams": {
 			"version": "2.2.2",
@@ -1343,7 +1364,7 @@
 		"pg-format": {
 			"version": "1.0.4",
 			"resolved": "https://registry.npmjs.org/pg-format/-/pg-format-1.0.4.tgz",
-			"integrity": "sha1-J3NCNsKtP05QZJFaWTNOIAQKgo4="
+			"integrity": "sha512-YyKEF78pEA6wwTAqOUaHIN/rWpfzzIuMh9KdAhc3rSLQ/7zkRFcCgYBAEGatDstLyZw4g0s9SNICmaTGnBVeyw=="
 		},
 		"pg-int8": {
 			"version": "1.0.1",
@@ -1368,11 +1389,18 @@
 			}
 		},
 		"pgpass": {
-			"version": "1.0.2",
-			"resolved": "https://registry.npmjs.org/pgpass/-/pgpass-1.0.2.tgz",
-			"integrity": "sha1-Knu0G2BltnkH6R2hsHwYR8h3swY=",
+			"version": "1.0.5",
+			"resolved": "https://registry.npmjs.org/pgpass/-/pgpass-1.0.5.tgz",
+			"integrity": "sha512-FdW9r/jQZhSeohs1Z3sI1yxFQNFvMcnmfuj4WBMUTxOrAyLMaTcE1aAMBiTlbMNaXvBCQuVi0R7hd8udDSP7ug==",
 			"requires": {
-				"split": "^1.0.0"
+				"split2": "^4.1.0"
+			},
+			"dependencies": {
+				"split2": {
+					"version": "4.1.0",
+					"resolved": "https://registry.npmjs.org/split2/-/split2-4.1.0.tgz",
+					"integrity": "sha512-VBiJxFkxiXRlUIeyMQi8s4hgvKCSjtknJv/LVYbrgALPwf5zSKmEwV9Lst25AkvMDnvxODugjdl6KZgwKM1WYQ=="
+				}
 			}
 		},
 		"postgres-array": {
@@ -1383,12 +1411,12 @@
 		"postgres-bytea": {
 			"version": "1.0.0",
 			"resolved": "https://registry.npmjs.org/postgres-bytea/-/postgres-bytea-1.0.0.tgz",
-			"integrity": "sha1-AntTPAqokOJtFy1Hz5zOzFIazTU="
+			"integrity": "sha512-xy3pmLuQqRBZBXDULy7KbaitYqLcmxigw14Q5sj8QBVLqEwXfeybIKVWiqAXTlcvdvb0+xkOtDbfQMOf4lST1w=="
 		},
 		"postgres-date": {
-			"version": "1.0.4",
-			"resolved": "https://registry.npmjs.org/postgres-date/-/postgres-date-1.0.4.tgz",
-			"integrity": "sha512-bESRvKVuTrjoBluEcpv2346+6kgB7UlnqWZsnbnCccTNq/pqfj1j6oBaN5+b/NrDXepYUT/HKadqv3iS9lJuVA=="
+			"version": "1.0.7",
+			"resolved": "https://registry.npmjs.org/postgres-date/-/postgres-date-1.0.7.tgz",
+			"integrity": "sha512-suDmjLVQg78nMK2UZ454hAG+OAW+HQPZ6n++TNDUX+L0+uUlLywnoxJKDou51Zm+zTCjrCl0Nq6J9C5hP9vK/Q=="
 		},
 		"postgres-interval": {
 			"version": "1.2.0",
@@ -1401,12 +1429,12 @@
 		"precond": {
 			"version": "0.2.3",
 			"resolved": "https://registry.npmjs.org/precond/-/precond-0.2.3.tgz",
-			"integrity": "sha1-qpWRvKokkj8eD0hJ0kD0fvwQdaw="
+			"integrity": "sha512-QCYG84SgGyGzqJ/vlMsxeXd/pgL/I94ixdNFyh1PusWmTCyVfPJjZ1K1jvHtsbfnXQs2TSkEP2fR7QiMZAnKFQ=="
 		},
 		"prelude-ls": {
 			"version": "1.1.2",
 			"resolved": "https://registry.npmjs.org/prelude-ls/-/prelude-ls-1.1.2.tgz",
-			"integrity": "sha1-IZMqVJ9eUv/ZqCf1cOBL5iqX2lQ=",
+			"integrity": "sha512-ESF23V4SKG6lVSGZgYNpbsiaAkdab6ZgOxe52p7+Kid3W3u3bxR4Vfd/o21dmN7jSt0IwgZ4v5MUd26FEtXE9w==",
 			"dev": true
 		},
 		"process-nextick-args": {
@@ -1485,7 +1513,7 @@
 		"restore-cursor": {
 			"version": "2.0.0",
 			"resolved": "https://registry.npmjs.org/restore-cursor/-/restore-cursor-2.0.0.tgz",
-			"integrity": "sha1-n37ih/gv0ybU/RYpI9YhKe7g368=",
+			"integrity": "sha512-6IzJLuGi4+R14vwagDHX+JrXmPVtPpn4mffDJ1UdR7/Edm87fl6yi8mMBIVvFtJaNTUvjughmW4hwLhRG7gC1Q==",
 			"dev": true,
 			"requires": {
 				"onetime": "^2.0.0",
@@ -1502,18 +1530,15 @@
 			}
 		},
 		"run-async": {
-			"version": "2.4.0",
-			"resolved": "https://registry.npmjs.org/run-async/-/run-async-2.4.0.tgz",
-			"integrity": "sha512-xJTbh/d7Lm7SBhc1tNvTpeCHaEzoyxPrqNlvSdMfBTYwaY++UJFyXUOxAtsRUXjlqOfj8luNaR9vjCh4KeV+pg==",
-			"dev": true,
-			"requires": {
-				"is-promise": "^2.1.0"
-			}
+			"version": "2.4.1",
+			"resolved": "https://registry.npmjs.org/run-async/-/run-async-2.4.1.tgz",
+			"integrity": "sha512-tvVnVv01b8c1RrA6Ep7JkStj85Guv/YrMcwqYQnwjsAS2cTmmPGBBjAjpCW7RrSodNSoE2/qg9O4bceNvUuDgQ==",
+			"dev": true
 		},
 		"rxjs": {
-			"version": "6.5.4",
-			"resolved": "https://registry.npmjs.org/rxjs/-/rxjs-6.5.4.tgz",
-			"integrity": "sha512-naMQXcgEo3csAEGvw/NydRA0fuS2nDZJiw1YUWFKU7aPPAPGZEsD4Iimit96qwCieH6y614MCLYwdkrWx7z/7Q==",
+			"version": "6.6.7",
+			"resolved": "https://registry.npmjs.org/rxjs/-/rxjs-6.6.7.tgz",
+			"integrity": "sha512-hTdwr+7yYNIT5n4AMYp85KA6yw2Va0FLa3Rguvbpa4W3I5xynaBZo41cM3XM+4Q6fRMj3sBYIR1VAmZMXYJvRQ==",
 			"dev": true,
 			"requires": {
 				"tslib": "^1.9.0"
@@ -1532,12 +1557,12 @@
 		"semver": {
 			"version": "4.3.2",
 			"resolved": "https://registry.npmjs.org/semver/-/semver-4.3.2.tgz",
-			"integrity": "sha1-x6BxWKgL7dBSNVt3DYLWZA+AO+c="
+			"integrity": "sha512-VyFUffiBx8hABJ9HYSTXLRwyZtdDHMzMtFmID1aiNAD2BZppBmJm0Hqw3p2jkgxP9BNt1pQ9RnC49P0EcXf6cA=="
 		},
 		"shebang-command": {
 			"version": "1.2.0",
 			"resolved": "https://registry.npmjs.org/shebang-command/-/shebang-command-1.2.0.tgz",
-			"integrity": "sha1-RKrGW2lbAzmJaMOfNj/uXer98eo=",
+			"integrity": "sha512-EV3L1+UQWGor21OmnvojK36mhg+TyIKDh3iFBKBohr5xeXIhNBcx8oWdgkTEEQ+BEFFYdLRuqMfd5L84N1V5Vg==",
 			"dev": true,
 			"requires": {
 				"shebang-regex": "^1.0.0"
@@ -1546,13 +1571,13 @@
 		"shebang-regex": {
 			"version": "1.0.0",
 			"resolved": "https://registry.npmjs.org/shebang-regex/-/shebang-regex-1.0.0.tgz",
-			"integrity": "sha1-2kL0l0DAtC2yypcoVxyxkMmO/qM=",
+			"integrity": "sha512-wpoSFAxys6b2a2wHZ1XpDSgD7N9iVjg29Ph9uV/uaP9Ex/KXlkTZTeddxDPSYQpgvzKLGJke2UU0AzoGCjNIvQ==",
 			"dev": true
 		},
 		"signal-exit": {
-			"version": "3.0.2",
-			"resolved": "https://registry.npmjs.org/signal-exit/-/signal-exit-3.0.2.tgz",
-			"integrity": "sha1-tf3AjxKH6hF4Yo5BXiUTK3NkbG0=",
+			"version": "3.0.7",
+			"resolved": "https://registry.npmjs.org/signal-exit/-/signal-exit-3.0.7.tgz",
+			"integrity": "sha512-wnD2ZE+l+SPC/uoS0vXeE9L1+0wuaMqKlfz9AMUo38JsyLSBWSFcHR1Rri62LZc12vLr1gb3jl7iwQhgwpAbGQ==",
 			"dev": true
 		},
 		"sinon": {
@@ -1600,18 +1625,18 @@
 		"sprintf-js": {
 			"version": "1.0.3",
 			"resolved": "https://registry.npmjs.org/sprintf-js/-/sprintf-js-1.0.3.tgz",
-			"integrity": "sha1-BOaSb2YolTVPPdAVIDYzuFcpfiw=",
+			"integrity": "sha512-D9cPgkvLlV3t3IzL0D0YLvGA9Ahk4PcvVwUbN0dSGr1aP0Nrt4AEnTUbuGvquEC0mA64Gqt1fzirlRs5ibXx8g==",
 			"dev": true
 		},
 		"sqlstring": {
 			"version": "2.3.1",
 			"resolved": "https://registry.npmjs.org/sqlstring/-/sqlstring-2.3.1.tgz",
-			"integrity": "sha1-R1OT/56RR5rqYtyvDKPRSYOn+0A="
+			"integrity": "sha512-ooAzh/7dxIG5+uDik1z/Rd1vli0+38izZhGzSa34FwR7IbelPWCCKSNIl8jlL/F7ERvy8CB2jNeM1E9i9mXMAQ=="
 		},
 		"stream-combiner": {
 			"version": "0.2.2",
 			"resolved": "https://registry.npmjs.org/stream-combiner/-/stream-combiner-0.2.2.tgz",
-			"integrity": "sha1-rsjLrBd7Vrb0+kec7YwZEs7lKFg=",
+			"integrity": "sha512-6yHMqgLYDzQDcAkL+tjJDC5nSNuNIx0vZtRZeiPh7Saef7VHX9H5Ijn9l2VIol2zaNYlYEX6KyuT/237A58qEQ==",
 			"requires": {
 				"duplexer": "~0.1.1",
 				"through": "~2.3.4"
@@ -1625,7 +1650,7 @@
 		"string-extended": {
 			"version": "0.0.8",
 			"resolved": "https://registry.npmjs.org/string-extended/-/string-extended-0.0.8.tgz",
-			"integrity": "sha1-dBlX3/SHsCcqee7FpE8jnubxfM0=",
+			"integrity": "sha512-CK46p3AxBvBhJbBi6WrF9bCcaWH20E4NwlLSzpooG2nXWvcP2gy2YR8VN6fSwZyrbcvL4S4zoNKbR0QG52X4rw==",
 			"requires": {
 				"array-extended": "~0.0.5",
 				"date-extended": "~0.0.3",
@@ -1654,7 +1679,7 @@
 		"strip-ansi": {
 			"version": "4.0.0",
 			"resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-4.0.0.tgz",
-			"integrity": "sha1-qEeQIusaw2iocTibY1JixQXuNo8=",
+			"integrity": "sha512-4XaJ2zQdCzROZDivEVIDPkcQn8LMFSa8kj8Gxb/Lnwzv9A8VctNZ+lfivC/sV3ivW8ElJTERXZoPBRrZKkNKow==",
 			"dev": true,
 			"requires": {
 				"ansi-regex": "^3.0.0"
@@ -1663,7 +1688,7 @@
 		"strip-json-comments": {
 			"version": "2.0.1",
 			"resolved": "https://registry.npmjs.org/strip-json-comments/-/strip-json-comments-2.0.1.tgz",
-			"integrity": "sha1-PFMZQukIwml8DsNEhYwobHygpgo=",
+			"integrity": "sha512-4gB8na07fecVVkOI6Rs4e7T6NOTki5EmL7TUduTs6bu3EdnSycntVJ4re8kgZA+wx9IueI2Y11bfbgwtzuE0KQ==",
 			"dev": true
 		},
 		"supports-color": {
@@ -1688,9 +1713,9 @@
 			},
 			"dependencies": {
 				"ansi-regex": {
-					"version": "4.1.0",
-					"resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-4.1.0.tgz",
-					"integrity": "sha512-1apePfXM1UOSqw0o9IiFAovVz9M5S1Dg+4TrDwfMewQ6p/rmMueb7tWZjQ1rx4Loy1ArBggoqGpfqqdI4rondg==",
+					"version": "4.1.1",
+					"resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-4.1.1.tgz",
+					"integrity": "sha512-ILlv4k/3f6vfQ4OoP2AGvirOktlQ98ZEL1k9FaQjxa3L1abBgbuTDAdPOpvbGncC0BTVQrl+OM8xZGK6tWXt7g==",
 					"dev": true
 				},
 				"string-width": {
@@ -1718,13 +1743,13 @@
 		"text-table": {
 			"version": "0.2.0",
 			"resolved": "https://registry.npmjs.org/text-table/-/text-table-0.2.0.tgz",
-			"integrity": "sha1-f17oI66AUgfACvLfSoTsP8+lcLQ=",
+			"integrity": "sha512-N+8UisAXDGk8PFXP4HAzVR9nbfmVJ3zYLAWiTIoqC5v5isinhr+r5uaO8+7r3BMfuNIufIsA7RdpVgacC2cSpw==",
 			"dev": true
 		},
 		"through": {
 			"version": "2.3.8",
 			"resolved": "https://registry.npmjs.org/through/-/through-2.3.8.tgz",
-			"integrity": "sha1-DdTJ/6q8NXlgsbckEV1+Doai4fU="
+			"integrity": "sha512-w89qg7PI8wAdvX60bMDP+bFoD5Dvhm9oLheFp5O4a2QF0cSBGsBX4qZmadPMvVqlLJBBci+WqGGOAPvcDeNSVg=="
 		},
 		"through2": {
 			"version": "2.0.5",
@@ -1745,15 +1770,15 @@
 			}
 		},
 		"tslib": {
-			"version": "1.11.1",
-			"resolved": "https://registry.npmjs.org/tslib/-/tslib-1.11.1.tgz",
-			"integrity": "sha512-aZW88SY8kQbU7gpV19lN24LtXh/yD4ZZg6qieAJDDg+YBsJcSmLGK9QpnUjAKVG/xefmvJGd1WUmfpT/g6AJGA==",
+			"version": "1.14.1",
+			"resolved": "https://registry.npmjs.org/tslib/-/tslib-1.14.1.tgz",
+			"integrity": "sha512-Xni35NKzjgMrwevysHTCArtLDpPvye8zV/0E4EyYn43P7/7qvQwPh9BGkHewbMulVntbigmcT7rdX3BNo9wRJg==",
 			"dev": true
 		},
 		"type-check": {
 			"version": "0.3.2",
 			"resolved": "https://registry.npmjs.org/type-check/-/type-check-0.3.2.tgz",
-			"integrity": "sha1-WITKtRLPHTVeP7eE8wgEsrUg23I=",
+			"integrity": "sha512-ZCmOJdvOWDBYJlzAoFkC+Q0+bUyEOS1ltgp1MGU03fqHG+dbi9tBFU2Rd9QKiDZFAYrhPh2JUf7rZRIuHRKtOg==",
 			"dev": true,
 			"requires": {
 				"prelude-ls": "~1.1.2"
@@ -1766,9 +1791,9 @@
 			"dev": true
 		},
 		"uri-js": {
-			"version": "4.2.2",
-			"resolved": "https://registry.npmjs.org/uri-js/-/uri-js-4.2.2.tgz",
-			"integrity": "sha512-KY9Frmirql91X2Qgjry0Wd4Y+YTdrdZheS8TFwvkbLWf/G5KNJDCh6pKL5OZctEW4+0Baa5idK2ZQuELRwPznQ==",
+			"version": "4.4.1",
+			"resolved": "https://registry.npmjs.org/uri-js/-/uri-js-4.4.1.tgz",
+			"integrity": "sha512-7rKUyy33Q1yc98pQ1DAmLtwX109F7TIfWlW1Ydo8Wl1ii1SeHieeh0HHfPeL2fMXK6z0s8ecKs9frCuLJvndBg==",
 			"dev": true,
 			"requires": {
 				"punycode": "^2.1.0"
@@ -1777,7 +1802,7 @@
 		"util-deprecate": {
 			"version": "1.0.2",
 			"resolved": "https://registry.npmjs.org/util-deprecate/-/util-deprecate-1.0.2.tgz",
-			"integrity": "sha1-RQ1Nyfpw3nMnYvvS1KKJgUGaDM8="
+			"integrity": "sha512-EPD5q1uXyFxJpCrLnCc1nHnq3gOa6DZBocAIiI2TaSCA7VCJ1UJDMagCzIkXNsUYfD1daK//LTEQ8xiIbrHtcw=="
 		},
 		"uuid": {
 			"version": "3.3.2",
@@ -1802,7 +1827,7 @@
 		"wrappy": {
 			"version": "1.0.2",
 			"resolved": "https://registry.npmjs.org/wrappy/-/wrappy-1.0.2.tgz",
-			"integrity": "sha1-tSQ9jz7BqjXxNkYFvA0QNuMKtp8="
+			"integrity": "sha512-l4Sp/DRseor9wL6EvV2+TuQn63dMkPjZ/sp9XkghTEbV9KlPS1xUsZ3u7/IQO4wxtcFB4bgpQPRcR3QCvezPcQ=="
 		},
 		"write": {
 			"version": "1.0.3",
diff --git a/postgres/package.json b/postgres/package.json
index 9b19b625..d141ab4a 100644
--- a/postgres/package.json
+++ b/postgres/package.json
@@ -1,9 +1,6 @@
 {
-	"name": "connector-postgres",
+	"name": "@leoplatform/connector-postgres",
 	"description": "A Postgres database connector for use with Leo Platform",
-	"publishConfig": {
-		"registry": "https://npm.pkg.github.com"
-	},
 	"repository": {
 		"type": "git",
 		"url": "git://github.com/LeoPlatform/connectors.git"
@@ -42,4 +39,4 @@
 			}
 		}
 	}
-}
+}
\ No newline at end of file

From 61f4104fb769b88594b0f4533b1fe2db1343300f Mon Sep 17 00:00:00 2001
From: Michael Scranton <mscranton@commercehub.com>
Date: Tue, 25 Oct 2022 19:24:10 -0400
Subject: [PATCH 22/62] fix for supported config check

---
 .github/workflows/publish-postgres.yml | 1 -
 postgres/lib/dwconnect.js              | 7 +++++--
 2 files changed, 5 insertions(+), 3 deletions(-)

diff --git a/.github/workflows/publish-postgres.yml b/.github/workflows/publish-postgres.yml
index 04b19654..ddcfe22f 100644
--- a/.github/workflows/publish-postgres.yml
+++ b/.github/workflows/publish-postgres.yml
@@ -20,7 +20,6 @@ jobs:
       uses: actions/setup-node@v2
       with:
         node-version: '12.x'
-        # cache: 'npm'
         registry-url: 'https://npm.pkg.github.com'
         scope: '@leoplatform'
     - run: npm ci
diff --git a/postgres/lib/dwconnect.js b/postgres/lib/dwconnect.js
index 5afb313d..815dcb63 100644
--- a/postgres/lib/dwconnect.js
+++ b/postgres/lib/dwconnect.js
@@ -29,8 +29,11 @@ module.exports = function (config, columnConfig) {
 
 
 	// Control flow for both of these configurations set to true has not been added. An error will be thrown until that is supported.
-	if (config.hashedSurrogateKeys && config.bypassSlowlyChangingDimensions) {
-		logger.error(`Unsupported configuration, bypassSlowlyChangingDimensions:${bypassSlowlyChangingDimensions} hashedSurrogateKeys:${hashedSurrogateKeys}.`);
+	if (
+		(config.hashedSurrogateKeys && !config.bypassSlowlyChangingDimensions) // hashed surrogate keys and slowly changing dimensions not supported
+		|| (config.hashedSurrogateKeys && config.bypassSlowlyChangingDimensions === undefined) // covering the case where slowly changing dimensions has been omitted
+	) {
+		logger.error(`Unsupported configuration, bypassSlowlyChangingDimensions:${config.bypassSlowlyChangingDimensions} hashedSurrogateKeys:${config.hashedSurrogateKeys}.`);
 		process.exit();
 	};
 

From da87a09eafa6fd4ff59562a58ea014130575f967 Mon Sep 17 00:00:00 2001
From: Michael Scranton <mscranton@commercehub.com>
Date: Tue, 25 Oct 2022 19:34:34 -0400
Subject: [PATCH 23/62] seems a comment broke a SQL statement

---
 postgres/lib/dwconnect.js | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/postgres/lib/dwconnect.js b/postgres/lib/dwconnect.js
index 815dcb63..c6147d90 100644
--- a/postgres/lib/dwconnect.js
+++ b/postgres/lib/dwconnect.js
@@ -428,13 +428,13 @@ module.exports = function (config, columnConfig) {
 							tasks.push(done => {
 								connection.query(`INSERT INTO ${qualifiedTable} (${fields.join(',')})
 									SELECT farmFingerPrint64(${nk.map(id => `staging.${id}`).join(`|| '-' ||`)}),
-										   ${allColumns.map(column => `coalesce(staging.${column}, prev.${column})`)}, // Don't think we need the coalesce anymore
+										   ${allColumns.map(column => `coalesce(staging.${column}, prev.${column})`)},
 										   ${dwClient.auditdate} as ${columnConfig._auditdate}, 
 										   case when changes.isNew then '1900-01-01 00:00:00' else ${config.version === 'redshift' ? 'sysdate' : 'now()'} END as ${columnConfig._startdate}, 
 										   '9999-01-01 00:00:00' as ${columnConfig._enddate}, true as ${columnConfig._current}
 									FROM ${qualifiedStagingTable}_changes changes  
 									JOIN ${qualifiedStagingTable} staging on ${nk.map(id => `staging.${id} = changes.${id}`).join(' and ')}
-									LEFT JOIN ${qualifiedTable} as prev on ${nk.map(id => `prev.${id} = changes.${id}`).join(' and ')} and prev.${columnConfig._current} // validate nothing will set _current to false, if so get rid of this join
+									LEFT JOIN ${qualifiedTable} as prev on ${nk.map(id => `prev.${id} = changes.${id}`).join(' and ')} and prev.${columnConfig._current} 
 									WHERE isNew = true		
 									`, done);
 							});

From 1c1a973c43a6924525d112799f23945de998252c Mon Sep 17 00:00:00 2001
From: Michael Scranton <mscranton@commercehub.com>
Date: Tue, 25 Oct 2022 20:14:54 -0400
Subject: [PATCH 24/62] more comments in SQL, probably should have remembered
 the character for SQL comments

---
 postgres/lib/dwconnect.js | 9 ++++++++-
 1 file changed, 8 insertions(+), 1 deletion(-)

diff --git a/postgres/lib/dwconnect.js b/postgres/lib/dwconnect.js
index c6147d90..f3eedc97 100644
--- a/postgres/lib/dwconnect.js
+++ b/postgres/lib/dwconnect.js
@@ -426,6 +426,11 @@ module.exports = function (config, columnConfig) {
 							});
 						} else if (config.hashedSurrogateKeys) {
 							tasks.push(done => {
+								/*
+								* Likely don't need the coalesce anymore
+								* validate nothing will set _current to false, if so get rid of this join
+								* general need to review SCD fields to make sure we can ignore them
+								*/
 								connection.query(`INSERT INTO ${qualifiedTable} (${fields.join(',')})
 									SELECT farmFingerPrint64(${nk.map(id => `staging.${id}`).join(`|| '-' ||`)}),
 										   ${allColumns.map(column => `coalesce(staging.${column}, prev.${column})`)},
@@ -460,11 +465,13 @@ module.exports = function (config, columnConfig) {
 						} else if (config.hashedSurrogateKeys) {
 							tasks.push(done => {
 								columns.push(`"${columnConfig._auditdate}" = ${dwClient.auditdate}`);
+								/*
+								*  might be able to remove the _startdate dependency since we aren't using slowly changing dimensions
+								*/
 								connection.query(`update ${qualifiedTable} as prev
 											set  ${columns.join(', ')}
 											FROM ${qualifiedStagingTable}_changes changes
 											JOIN ${qualifiedStagingTable} staging on ${nk.map(id => `staging.${id} = changes.${id}`).join(' and ')}
-																												// might be able to remove the _startdate dependency since we aren't using slowly changing dimensions
 											WHERE ${nk.map(id => `prev.${id} = changes.${id}`).join(' and ')} and prev.${columnConfig._startdate} != ${config.version === 'redshift' ? 'sysdate' : 'now()'} and changes.isNew = false
 											`, done);
 							});

From f3bdc92b46310532a8776b2ddbd4d804a0e33d4e Mon Sep 17 00:00:00 2001
From: Michael Scranton <mscranton@commercehub.com>
Date: Thu, 27 Oct 2022 16:59:44 -0400
Subject: [PATCH 25/62] Reverting to NPM for package registry

---
 postgres/package.json | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/postgres/package.json b/postgres/package.json
index d141ab4a..83e1a3ba 100644
--- a/postgres/package.json
+++ b/postgres/package.json
@@ -1,5 +1,6 @@
 {
-	"name": "@leoplatform/connector-postgres",
+	"name": "leo-connector-postgres",
+	"version": "4.0.6",
 	"description": "A Postgres database connector for use with Leo Platform",
 	"repository": {
 		"type": "git",
@@ -24,6 +25,7 @@
 		"pg-format": "1.0.4"
 	},
 	"devDependencies": {
+		"@dsco/layer-leo": "2.1.29",
 		"chai": "^4.2.0",
 		"eslint": "^5.13.0",
 		"mocha": "^5.2.0",

From 8cef900f6da72cadff21209ef63b3026077afdd7 Mon Sep 17 00:00:00 2001
From: Michael Scranton <mscranton@commercehub.com>
Date: Fri, 28 Oct 2022 09:50:15 -0400
Subject: [PATCH 26/62] add beta to version

---
 postgres/package.json | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/postgres/package.json b/postgres/package.json
index 83e1a3ba..e535436c 100644
--- a/postgres/package.json
+++ b/postgres/package.json
@@ -1,6 +1,6 @@
 {
 	"name": "leo-connector-postgres",
-	"version": "4.0.6",
+	"version": "4.0.6-beta",
 	"description": "A Postgres database connector for use with Leo Platform",
 	"repository": {
 		"type": "git",

From 9de767f489744a1f607923c0745eb194e909c01f Mon Sep 17 00:00:00 2001
From: Grant Robinson <grant.robinson@dsco.io>
Date: Fri, 28 Oct 2022 13:16:53 -0600
Subject: [PATCH 27/62] * bumped version

---
 postgres/package.json | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/postgres/package.json b/postgres/package.json
index e535436c..a3e39887 100644
--- a/postgres/package.json
+++ b/postgres/package.json
@@ -1,6 +1,6 @@
 {
 	"name": "leo-connector-postgres",
-	"version": "4.0.6-beta",
+	"version": "4.0.7-beta",
 	"description": "A Postgres database connector for use with Leo Platform",
 	"repository": {
 		"type": "git",

From 5e05797a9ba47abbeab79e135dbcd3576c6ec7ff Mon Sep 17 00:00:00 2001
From: Michael Scranton <mscranton@commercehub.com>
Date: Mon, 7 Nov 2022 11:38:42 -0500
Subject: [PATCH 28/62] ETL rework for high volume scaling

---
 postgres/lib/dwconnect.js  | 218 ++++++++++++++++++++++----------
 postgres/package-lock.json | 246 ++++++++++++++++++++++++++++++++++++-
 postgres/package.json      |   4 +-
 3 files changed, 397 insertions(+), 71 deletions(-)

diff --git a/postgres/lib/dwconnect.js b/postgres/lib/dwconnect.js
index f3eedc97..f44e6b76 100644
--- a/postgres/lib/dwconnect.js
+++ b/postgres/lib/dwconnect.js
@@ -198,28 +198,31 @@ module.exports = function (config, columnConfig) {
 
 					let tasks = [];
 					let totalRecords = 0;
+
 					// The following code relies on the fact that now()/sysdate will return the same time during all transaction events
 					tasks.push(done => connection.query(`Begin Transaction`, done));
-					tasks.push(done => {
-						connection.query(`select 1 as total from ${qualifiedTable} limit 1`, (err, results) => {
-							if (err) {
-								return done(err);
-							}
-							totalRecords = results.length;
-							done();
+
+					if (!config.hashedSurrogateKeys) {
+						tasks.push(done => {
+							connection.query(`select 1 as total from ${qualifiedTable} limit 1`, (err, results) => {
+								if (err) {
+									return done(err);
+								}
+								totalRecords = results.length;
+								done();
+							});
 						});
-					});
-					tasks.push(done => {
-						connection.query(`Update ${qualifiedTable} prev
+						tasks.push(done => {
+							connection.query(`Update ${qualifiedTable} prev
 								SET  ${columns.map(column => `${column} = coalesce(staging.${column}, prev.${column})`)}, ${columnConfig._deleted} = coalesce(prev.${columnConfig._deleted}, false), ${columnConfig._auditdate} = ${dwClient.auditdate}
 								FROM ${qualifiedStagingTable} staging
 								where ${ids.map(id => `prev.${id} = staging.${id}`).join(' and ')}
 							`, done);
-					});
+						});
 
-					// Now insert any we were missing
-					tasks.push(done => {
-						connection.query(`INSERT INTO ${qualifiedTable} (${columns.join(',')},${columnConfig._deleted},${columnConfig._auditdate})
+						// Now insert any we were missing
+						tasks.push(done => {
+							connection.query(`INSERT INTO ${qualifiedTable} (${columns.join(',')},${columnConfig._deleted},${columnConfig._auditdate})
 								SELECT ${columns.map(column => `staging.${column}`)},
 										false AS ${columnConfig._deleted},
 										${dwClient.auditdate} as ${columnConfig._auditdate}
@@ -230,7 +233,55 @@ module.exports = function (config, columnConfig) {
 									WHERE ${ids.map(id => `prev.${id} = staging.${id}`).join(' and ')}
 								)	
 							`, done);
-					});
+						});
+					} else {
+						// Retreive copy of existing data
+						tempTables.push(`${qualifiedStagingTable}_previous`);
+						tasks.push(done => client.query(`drop table if exists ${qualifiedStagingTable}_previous;`, done));
+						tasks.push(done => {
+							connection.query(`CREATE TABLE ${qualifiedStagingTable}_previous
+											  DISTSTYLE ALL 
+											  AS SELECT *
+												 FROM   ${qualifiedTable}
+												 LIMIT  0;`, done);
+						});
+						tasks.push(done => {
+							connection.query(`INSERT INTO ${qualifiedStagingTable}_previous
+											  SELECT *
+											  FROM   ${qualifiedTable} AS base
+											  WHERE  EXISTS ( SELECT *
+															  FROM   ${qualifiedStagingTable} AS staging
+															  WHERE  ${ids.map(id => `base.${id} = staging.${id}`).join(` AND `)});`, done);
+						});
+
+						// Merge exiting data into staged copy
+						tasks.push(done => {
+							connection.query(`UPDATE ${qualifiedStagingTable} AS staging
+											  SET    ${columns.map(column => `${column} = COALESCE(staging.${column}, prev.${column})`).join(`,`)}
+											  FROM   ${qualifiedStagingTable}_previous AS prev
+											  WHERE  ${ids.map(id => `prev.${id} = staging.${id}`).join(` AND `)}`, done);
+						});
+
+						// Set audit date for staged data
+						tasks.push(done => {
+							connection.query(`UPDATE ${qualifiedStagingTable}
+					  						  SET ${columnConfig._auditdate} = ${dwClient.auditdate};`, done);
+						});
+
+						// Delete and reinsert data - avoids costly updates on large tables
+						tasks.push(done => {
+							connection.query(`DELETE FROM ${qualifiedTable}
+											  USING ${qualifiedStagingTable}
+											  WHERE ${ids.map(id => `${qualifiedTable}.${id} = ${qualifiedTable}.${id}`).join(` AND `)}`, done);
+						});
+						tasks.push(done => {
+							connection.query(`INSERT INTO ${qualifiedTable} (
+											  ${columns.map(column => `${column}`).join(`,\n`)}
+											  )
+											  SELECT ${columns.map(column => `${column}`).join(`,\n`)}
+											  FROM   ${qualifiedStagingTable};`, done);
+						});
+					};
 
 					async.series(tasks, err => {
 						if (!err) {
@@ -287,7 +338,7 @@ module.exports = function (config, columnConfig) {
 								 from ${qualifiedTable}
 								 limit 0;`, done));
 		};
-		tasks.push(done => client.query(`alter table ${qualifiedStagingTable} drop column ${sk}`, done));
+		// tasks.push(done => client.query(`alter table ${qualifiedStagingTable} drop column ${sk}`, done));
 
 		if (config.version !== 'redshift') {
 			tasks.push(done => {
@@ -376,29 +427,30 @@ module.exports = function (config, columnConfig) {
 						}
 					};
 
-					tempTables.push(`${qualifiedStagingTable}_changes`);
-					connection.query(`create table ${qualifiedStagingTable}_changes as 
+					let fields = [sk].concat(allColumns).concat([columnConfig._auditdate, columnConfig._startdate, columnConfig._enddate, columnConfig._current]);
+					let totalRecords = 0;
+
+					if (!config.hashedSurrogateKeys) {
+						tempTables.push(`${qualifiedStagingTable}_changes`);
+						connection.query(`create table ${qualifiedStagingTable}_changes as 
 									  	select ${nk.map(id => `s.${id}`).join(', ')}, 
 											   d.${nk[0]} is null as isNew
 										${!config.bypassSlowlyChangingDimensions ? `,${scdSQL.join(',\n')}` : ``}
 										FROM ${qualifiedStagingTable} s
 										LEFT JOIN ${qualifiedTable} d on ${nk.map(id => `d.${id} = s.${id}`).join(' and ')} and d.${columnConfig._current}`, (err) => {
-						if (err) {
-							logger.error(err);
-							process.exit();
-						}
+							if (err) {
+								logger.error(err);
+								process.exit();
+							}
 
-						let tasks = [];
-						let rowId = null;
-						let totalRecords = 0;
-						tasks.push(done => connection.query(`analyze ${qualifiedStagingTable}_changes`, done));
+							let tasks = [];
+							let rowId = null;
 
-						// The following code relies on the fact that now()/sysdate will return the same time during all transaction events
-						tasks.push(done => connection.query(`Begin Transaction`, done));
+							tasks.push(done => connection.query(`analyze ${qualifiedStagingTable}_changes`, done));
 
-						let fields = [sk].concat(allColumns).concat([columnConfig._auditdate, columnConfig._startdate, columnConfig._enddate, columnConfig._current]);
+							// The following code relies on the fact that now()/sysdate will return the same time during all transaction events
+							tasks.push(done => connection.query(`Begin Transaction`, done));
 
-						if (!config.hashedSurrogateKeys) {
 							tasks.push(done => {
 								connection.query(`select max(${sk}) as maxid from ${qualifiedTable}`, (err, results) => {
 									if (err) {
@@ -424,30 +476,9 @@ module.exports = function (config, columnConfig) {
 									WHERE (changes.runSCD2 =1 OR changes.runSCD6=1)		
 									`, done);
 							});
-						} else if (config.hashedSurrogateKeys) {
-							tasks.push(done => {
-								/*
-								* Likely don't need the coalesce anymore
-								* validate nothing will set _current to false, if so get rid of this join
-								* general need to review SCD fields to make sure we can ignore them
-								*/
-								connection.query(`INSERT INTO ${qualifiedTable} (${fields.join(',')})
-									SELECT farmFingerPrint64(${nk.map(id => `staging.${id}`).join(`|| '-' ||`)}),
-										   ${allColumns.map(column => `coalesce(staging.${column}, prev.${column})`)},
-										   ${dwClient.auditdate} as ${columnConfig._auditdate}, 
-										   case when changes.isNew then '1900-01-01 00:00:00' else ${config.version === 'redshift' ? 'sysdate' : 'now()'} END as ${columnConfig._startdate}, 
-										   '9999-01-01 00:00:00' as ${columnConfig._enddate}, true as ${columnConfig._current}
-									FROM ${qualifiedStagingTable}_changes changes  
-									JOIN ${qualifiedStagingTable} staging on ${nk.map(id => `staging.${id} = changes.${id}`).join(' and ')}
-									LEFT JOIN ${qualifiedTable} as prev on ${nk.map(id => `prev.${id} = changes.${id}`).join(' and ')} and prev.${columnConfig._current} 
-									WHERE isNew = true		
-									`, done);
-							});
-						}; // Final else is needed to support hashed surrogate keys and slowly changing dimensions
 
-						// This needs to be done last
-						let columns = scd1.map(column => `"${column}" = coalesce(staging."${column}", prev."${column}")`).concat(scd6.map(column => `"current_${column}" = coalesce(staging."${column}", prev."${column}")`));
-						if (!config.bypassSlowlyChangingDimensions) {
+							// This needs to be done last
+							let columns = scd1.map(column => `"${column}" = coalesce(staging."${column}", prev."${column}")`).concat(scd6.map(column => `"current_${column}" = coalesce(staging."${column}", prev."${column}")`));
 							tasks.push(done => {
 								// RUN SCD1 / SCD6 columns  (where we update the old records)
 								columns.push(`"${columnConfig._enddate}" = case when changes.runSCD2 =1 then ${config.version === 'redshift' ? 'sysdate' : 'now()'} else prev."${columnConfig._enddate}" END`);
@@ -462,20 +493,73 @@ module.exports = function (config, columnConfig) {
 												and (changes.runSCD1=1 OR  changes.runSCD6=1 OR changes.runSCD2=1)
 											`, done);
 							});
-						} else if (config.hashedSurrogateKeys) {
-							tasks.push(done => {
-								columns.push(`"${columnConfig._auditdate}" = ${dwClient.auditdate}`);
-								/*
-								*  might be able to remove the _startdate dependency since we aren't using slowly changing dimensions
-								*/
-								connection.query(`update ${qualifiedTable} as prev
-											set  ${columns.join(', ')}
-											FROM ${qualifiedStagingTable}_changes changes
-											JOIN ${qualifiedStagingTable} staging on ${nk.map(id => `staging.${id} = changes.${id}`).join(' and ')}
-											WHERE ${nk.map(id => `prev.${id} = changes.${id}`).join(' and ')} and prev.${columnConfig._startdate} != ${config.version === 'redshift' ? 'sysdate' : 'now()'} and changes.isNew = false
-											`, done);
+
+							async.series(tasks, err => {
+								if (!err) {
+									connection.query(`commit`, e => {
+										connection.release();
+										callback(e || err, {
+											count: totalRecords,
+										});
+									});
+								} else {
+									connection.query(`rollback`, (e, d) => {
+										connection.release();
+										callback(e, d);
+									});
+								}
 							});
-						};
+						});
+					} else {
+						//Retreive copy of existing data
+						tempTables.push(`${qualifiedStagingTable}_previous`);
+						tasks.push(done => client.query(`drop table if exists ${qualifiedStagingTable}_previous;`, done));
+						tasks.push(done => {
+							connection.query(`create table ${qualifiedStagingTable}_previous
+											  diststyle all 
+											  as select *
+												 from ${qualifiedTable}
+												 limit 0;`, done);
+						});
+						tasks.push(done => {
+							connection.query(`INSERT INTO ${qualifiedStagingTable}_previous
+											  SELECT *
+											  FROM   ${qualifiedTable} AS base
+											  WHERE  EXISTS ( SELECT *
+															  FROM   ${qualifiedStagingTable} AS staging
+															  WHERE  ${nk.map(id => `base.${id} = staging.${id}`).join(` AND `)});`
+								, done);
+						});
+
+						// Merge exiting data into staged copy
+						tasks.push(done => {
+							connection.query(`UPDATE ${qualifiedStagingTable} AS staging
+										  	  SET    ${fields.map(column => `${column} = COALESCE(staging.${column}, prev.${column})`).join(`,`)}
+										  	  FROM   ${qualifiedStagingTable}_previous AS prev
+										  	  WHERE  ${nk.map(id => `prev.${id} = staging.${id}`).join(` AND `)}`
+								, done);
+						});
+
+						// Set auditdate and surrogate key for stage data (not sure if the sk is actually necessary)
+						tasks.push(done => {
+							connection.query(`UPDATE ${qualifiedStagingTable}
+				  							  SET ${columnConfig._auditdate} = ${dwClient.auditdate},
+											  ${sk} = farmFingerPrint64(${nk.map(id => `${id}`).join(`|| '-' ||`)});`, done);
+						});
+
+						// Delete and reinsert data - avoids costly updates on large tables
+						tasks.push(done => {
+							connection.query(`DELETE FROM ${qualifiedTable}
+										  	  USING ${qualifiedStagingTable}
+										  	  WHERE ${nk.map(id => `${qualifiedTable}.${id} = ${qualifiedTable}.${id}`).join(` AND `)}`
+								, done);
+						});
+						tasks.push(done => {
+							connection.query(`INSERT INTO ${qualifiedTable}
+										  	  SELECT *
+										  	  FROM   ${qualifiedStagingTable};`
+								, done);
+						});
 
 						async.series(tasks, err => {
 							if (!err) {
@@ -492,7 +576,7 @@ module.exports = function (config, columnConfig) {
 								});
 							}
 						});
-					});
+					};
 				});
 			}).catch(callback);
 		}).catch(callback);
diff --git a/postgres/package-lock.json b/postgres/package-lock.json
index 3538a0a0..e01fa818 100644
--- a/postgres/package-lock.json
+++ b/postgres/package-lock.json
@@ -1,7 +1,8 @@
 {
-	"name": "@leoplatform/connector-postgres",
-	"requires": true,
+	"name": "leo-connector-postgres",
+	"version": "4.0.8-beta",
 	"lockfileVersion": 1,
+	"requires": true,
 	"dependencies": {
 		"@babel/code-frame": {
 			"version": "7.18.6",
@@ -29,6 +30,81 @@
 				"js-tokens": "^4.0.0"
 			}
 		},
+		"@dsco/layer": {
+			"version": "1.0.10",
+			"resolved": "https://registry.npmjs.org/@dsco/layer/-/layer-1.0.10.tgz",
+			"integrity": "sha512-WKeCqbggSSc9uMi+LH/LRk9ibv6FNf0npF8Q7BNKwdSD/f8xbi3FvnNJKuWUvJvK1j/iYJb36ajC+EUWU27gdQ==",
+			"dev": true
+		},
+		"@dsco/layer-leo": {
+			"version": "2.1.29",
+			"resolved": "https://registry.npmjs.org/@dsco/layer-leo/-/layer-leo-2.1.29.tgz",
+			"integrity": "sha512-eFGVSCsrlUVWA8Mh9x43KjxAC/HZOj9+5lXMgS/NGQBfn1sbO2OsUDg2IHYNMHBba5Lql5RwUYqqL6faMjqYJQ==",
+			"dev": true,
+			"requires": {
+				"@dsco/layer": "1.0.10",
+				"leo-auth": "3.0.1",
+				"leo-aws": "2.0.2-beta-x",
+				"leo-cache": "1.0.1",
+				"leo-config": "1.1.0",
+				"leo-connector-common": "4.0.8-rc",
+				"leo-connector-entity-table": "3.0.7-rc",
+				"leo-logger": "1.0.1",
+				"leo-sdk": "5.0.16",
+				"leo-streams": "2.0.0",
+				"moment": "2.24.0",
+				"moment-business-days": "1.1.3",
+				"moment-timezone": "0.5.26"
+			},
+			"dependencies": {
+				"leo-aws": {
+					"version": "2.0.2-beta-x",
+					"resolved": "https://registry.npmjs.org/leo-aws/-/leo-aws-2.0.2-beta-x.tgz",
+					"integrity": "sha512-vHMAuuRMr8pPEqFLAQDgr6SmdJG8ncE7YTMqzyH7AsAUkQluj1fjjZ+W7a0CsmLJz5UARhSVyFc3zCiqGcrgiw==",
+					"dev": true,
+					"requires": {
+						"async": "^2.6.1",
+						"backoff": "^2.5.0",
+						"leo-config": "1.1.0",
+						"leo-logger": "1.0.1",
+						"leo-streams": "2.0.0",
+						"lodash": "^4.17.14",
+						"lodash.merge": "^4.6.2"
+					}
+				},
+				"leo-connector-common": {
+					"version": "4.0.8-rc",
+					"resolved": "https://registry.npmjs.org/leo-connector-common/-/leo-connector-common-4.0.8-rc.tgz",
+					"integrity": "sha512-f9uJVkgDtI+EqJCF/c9BqP/236NlflId0uK3Wq0LwNNISESakDXDyxemb9oTgvaMfXWW7VOHFOSylthdFSLgmg==",
+					"dev": true,
+					"requires": {
+						"async": "2.6.3",
+						"leo-aws": "2.0.1",
+						"leo-logger": "1.0.1",
+						"leo-streams": "2.0.0",
+						"moment": "2.24.0",
+						"sqlstring": "2.3.1"
+					},
+					"dependencies": {
+						"leo-aws": {
+							"version": "2.0.1",
+							"resolved": "https://registry.npmjs.org/leo-aws/-/leo-aws-2.0.1.tgz",
+							"integrity": "sha512-40vq8BQklV96sZvN7fDRRmpKmY7p1CyZLIh1G4IkkokScw2QrqkhxOSs5H+A6xb7HBnt+IUvpgOXhwYe0+uOcQ==",
+							"dev": true,
+							"requires": {
+								"async": "^2.6.1",
+								"backoff": "^2.5.0",
+								"leo-config": "1.1.0",
+								"leo-logger": "1.0.1",
+								"leo-streams": "2.0.0",
+								"lodash": "^4.17.14",
+								"lodash.merge": "^4.6.2"
+							}
+						}
+					}
+				}
+			}
+		},
 		"@sinonjs/commons": {
 			"version": "1.8.3",
 			"resolved": "https://registry.npmjs.org/@sinonjs/commons/-/commons-1.8.3.tgz",
@@ -65,6 +141,12 @@
 			"integrity": "sha512-sXXKG+uL9IrKqViTtao2Ws6dy0znu9sOaP1di/jKGW1M6VssO8vlpXCQcpZ+jisQ1tTFAC5Jo/EOzFbggBagFQ==",
 			"dev": true
 		},
+		"abbrev": {
+			"version": "1.1.1",
+			"resolved": "https://registry.npmjs.org/abbrev/-/abbrev-1.1.1.tgz",
+			"integrity": "sha512-nne9/IiQ/hzIhY6pdDnbBtz7DjPTKrY00P/zvPSm5pOFkl6xuGrGnXn/VtTNNfNtAfZ9/1RtehkszU9qcTii0Q==",
+			"dev": true
+		},
 		"acorn": {
 			"version": "6.4.2",
 			"resolved": "https://registry.npmjs.org/acorn/-/acorn-6.4.2.tgz",
@@ -285,6 +367,16 @@
 			"integrity": "sha512-/Srv4dswyQNBfohGpz9o6Yb3Gz3SrUDqBH5rTuhGR7ahtlbYKnVxw2bCFMRljaA7EXHaXZ8wsHdodFvbkhKmqg==",
 			"dev": true
 		},
+		"config-chain": {
+			"version": "1.1.13",
+			"resolved": "https://registry.npmjs.org/config-chain/-/config-chain-1.1.13.tgz",
+			"integrity": "sha512-qj+f8APARXHrM0hraqXYb2/bOVSV4PvJQlNZ/DVj0QrmNM2q2euizkeuVckQ57J+W0mRH6Hvi+k50M4Jul2VRQ==",
+			"dev": true,
+			"requires": {
+				"ini": "^1.3.4",
+				"proto-list": "~1.2.1"
+			}
+		},
 		"core-util-is": {
 			"version": "1.0.3",
 			"resolved": "https://registry.npmjs.org/core-util-is/-/core-util-is-1.0.3.tgz",
@@ -381,6 +473,32 @@
 				"stream-shift": "^1.0.0"
 			}
 		},
+		"editorconfig": {
+			"version": "0.15.3",
+			"resolved": "https://registry.npmjs.org/editorconfig/-/editorconfig-0.15.3.tgz",
+			"integrity": "sha512-M9wIMFx96vq0R4F+gRpY3o2exzb8hEj/n9S8unZtHSvYjibBp/iMufSzvmOcV/laG0ZtuTVGtiJggPOSW2r93g==",
+			"dev": true,
+			"requires": {
+				"commander": "^2.19.0",
+				"lru-cache": "^4.1.5",
+				"semver": "^5.6.0",
+				"sigmund": "^1.0.1"
+			},
+			"dependencies": {
+				"commander": {
+					"version": "2.20.3",
+					"resolved": "https://registry.npmjs.org/commander/-/commander-2.20.3.tgz",
+					"integrity": "sha512-GpVkmM8vF2vQUkj2LvZmD35JxeJOLCwJ9cUkugyk2nuhbv3+mJvpLYYt+0+USMxE+oj+ey/lJEnhZw75x/OMcQ==",
+					"dev": true
+				},
+				"semver": {
+					"version": "5.7.1",
+					"resolved": "https://registry.npmjs.org/semver/-/semver-5.7.1.tgz",
+					"integrity": "sha512-sauaDf/PZdVgrLTNYHRtpXa1iRiKcaebiKQ1BJdpQlWH2lCvexQdX55snPFyK7QzpudqbCI0qXFfOasHdyNDGQ==",
+					"dev": true
+				}
+			}
+		},
 		"emoji-regex": {
 			"version": "7.0.3",
 			"resolved": "https://registry.npmjs.org/emoji-regex/-/emoji-regex-7.0.3.tgz",
@@ -847,6 +965,19 @@
 			"integrity": "sha512-RHxMLp9lnKHGHRng9QFhRCMbYAcVpn69smSGcq3f36xjgVVWThj4qqLbTLlq7Ssj8B+fIQ1EuCEGI2lKsyQeIw==",
 			"dev": true
 		},
+		"js-beautify": {
+			"version": "1.10.2",
+			"resolved": "https://registry.npmjs.org/js-beautify/-/js-beautify-1.10.2.tgz",
+			"integrity": "sha512-ZtBYyNUYJIsBWERnQP0rPN9KjkrDfJcMjuVGcvXOUJrD1zmOGwhRwQ4msG+HJ+Ni/FA7+sRQEMYVzdTQDvnzvQ==",
+			"dev": true,
+			"requires": {
+				"config-chain": "^1.1.12",
+				"editorconfig": "^0.15.3",
+				"glob": "^7.1.3",
+				"mkdirp": "~0.5.1",
+				"nopt": "~4.0.1"
+			}
+		},
 		"js-tokens": {
 			"version": "4.0.0",
 			"resolved": "https://registry.npmjs.org/js-tokens/-/js-tokens-4.0.0.tgz",
@@ -881,6 +1012,17 @@
 			"integrity": "sha512-g3UB796vUFIY90VIv/WX3L2c8CS2MdWUww3CNrYmqza1Fg0DURc2K/O4YrnklBdQarSJ/y8JnJYDGc+1iumQjg==",
 			"dev": true
 		},
+		"leo-auth": {
+			"version": "3.0.1",
+			"resolved": "https://registry.npmjs.org/leo-auth/-/leo-auth-3.0.1.tgz",
+			"integrity": "sha512-l+vzMxH6PIzjRdJCRVvHX3P4JGAybe71LFz2fg5dQfOcvbqhbTnPMNbfIo4qUcfdHPeFG/ZrZ6wxQJVMzVopEA==",
+			"dev": true,
+			"requires": {
+				"leo-aws": "2.0.1",
+				"leo-config": "1.1.0",
+				"netmask": "^1.0.6"
+			}
+		},
 		"leo-aws": {
 			"version": "2.0.1",
 			"resolved": "https://registry.npmjs.org/leo-aws/-/leo-aws-2.0.1.tgz",
@@ -895,6 +1037,12 @@
 				"lodash.merge": "^4.6.2"
 			}
 		},
+		"leo-cache": {
+			"version": "1.0.1",
+			"resolved": "https://registry.npmjs.org/leo-cache/-/leo-cache-1.0.1.tgz",
+			"integrity": "sha512-61QSKr+GrDIbcmDDtngm64a+ra0Xdqe/4wYKW5aP2DcmtVumaNV9rnfI5vUINeVnkwBcXMXNQBht2jw0yBOy7w==",
+			"dev": true
+		},
 		"leo-config": {
 			"version": "1.1.0",
 			"resolved": "https://registry.npmjs.org/leo-config/-/leo-config-1.1.0.tgz",
@@ -917,6 +1065,19 @@
 				"sqlstring": "2.3.1"
 			}
 		},
+		"leo-connector-entity-table": {
+			"version": "3.0.7-rc",
+			"resolved": "https://registry.npmjs.org/leo-connector-entity-table/-/leo-connector-entity-table-3.0.7-rc.tgz",
+			"integrity": "sha512-5Ws+LLynNTOePkB5bcwaqzRVVyBYgQo2eDx7+ZobP3CR2dq4SYGD6YtmVLAHxnNZLdvtuLaxfxsa3iWiTRbFbQ==",
+			"dev": true,
+			"requires": {
+				"async": "2.6.3",
+				"backoff": "2.5.0",
+				"js-beautify": "1.10.2",
+				"leo-aws": "2.0.1",
+				"leo-logger": "1.0.1"
+			}
+		},
 		"leo-logger": {
 			"version": "1.0.1",
 			"resolved": "https://registry.npmjs.org/leo-logger/-/leo-logger-1.0.1.tgz",
@@ -1056,6 +1217,16 @@
 				"get-func-name": "^2.0.0"
 			}
 		},
+		"lru-cache": {
+			"version": "4.1.5",
+			"resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-4.1.5.tgz",
+			"integrity": "sha512-sWZlbEP2OsHNkXrMl5GYk/jKk70MBng6UU4YI/qGDYbgf6YbP4EvmqISbXCoJiRKs+1bSpFHVgQxvJ17F2li5g==",
+			"dev": true,
+			"requires": {
+				"pseudomap": "^1.0.2",
+				"yallist": "^2.1.2"
+			}
+		},
 		"map-stream": {
 			"version": "0.0.7",
 			"resolved": "https://registry.npmjs.org/map-stream/-/map-stream-0.0.7.tgz",
@@ -1179,6 +1350,21 @@
 			"resolved": "https://registry.npmjs.org/moment/-/moment-2.24.0.tgz",
 			"integrity": "sha512-bV7f+6l2QigeBBZSM/6yTNq4P2fNpSWj/0e7jQcy87A8e7o2nAfP/34/2ky5Vw4B9S446EtIhodAzkFCcR4dQg=="
 		},
+		"moment-business-days": {
+			"version": "1.1.3",
+			"resolved": "https://registry.npmjs.org/moment-business-days/-/moment-business-days-1.1.3.tgz",
+			"integrity": "sha512-VfpcLQxVWJ9ymq4ljaqPdzoco01kOJblX9sbfOCYguBvqCvVPYDJQTdez49a2gtSoSK8AjL8JfvLvmoQlrBMiQ==",
+			"dev": true
+		},
+		"moment-timezone": {
+			"version": "0.5.26",
+			"resolved": "https://registry.npmjs.org/moment-timezone/-/moment-timezone-0.5.26.tgz",
+			"integrity": "sha512-sFP4cgEKTCymBBKgoxZjYzlSovC20Y6J7y3nanDc5RoBIXKlZhoYwBoZGe3flwU6A372AcRwScH8KiwV6zjy1g==",
+			"dev": true,
+			"requires": {
+				"moment": ">= 2.9.0"
+			}
+		},
 		"ms": {
 			"version": "2.1.2",
 			"resolved": "https://registry.npmjs.org/ms/-/ms-2.1.2.tgz",
@@ -1197,6 +1383,12 @@
 			"integrity": "sha512-OWND8ei3VtNC9h7V60qff3SVobHr996CTwgxubgyQYEpg290h9J0buyECNNJexkFm5sOajh5G116RYA1c8ZMSw==",
 			"dev": true
 		},
+		"netmask": {
+			"version": "1.0.6",
+			"resolved": "https://registry.npmjs.org/netmask/-/netmask-1.0.6.tgz",
+			"integrity": "sha512-3DWDqAtIiPSkBXZyYEjwebfK56nrlQfRGt642fu8RPaL+ePu750+HCMHxjJCG3iEHq/0aeMvX6KIzlv7nuhfrA==",
+			"dev": true
+		},
 		"nice-try": {
 			"version": "1.0.5",
 			"resolved": "https://registry.npmjs.org/nice-try/-/nice-try-1.0.5.tgz",
@@ -1227,6 +1419,16 @@
 				}
 			}
 		},
+		"nopt": {
+			"version": "4.0.3",
+			"resolved": "https://registry.npmjs.org/nopt/-/nopt-4.0.3.tgz",
+			"integrity": "sha512-CvaGwVMztSMJLOeXPrez7fyfObdZqNUK1cPAEzLHrTybIua9pMdmmPR5YwtfNftIOMv3DPUhFaxsZMNTQO20Kg==",
+			"dev": true,
+			"requires": {
+				"abbrev": "1",
+				"osenv": "^0.1.4"
+			}
+		},
 		"object-extended": {
 			"version": "0.0.7",
 			"resolved": "https://registry.npmjs.org/object-extended/-/object-extended-0.0.7.tgz",
@@ -1268,12 +1470,28 @@
 				"word-wrap": "~1.2.3"
 			}
 		},
+		"os-homedir": {
+			"version": "1.0.2",
+			"resolved": "https://registry.npmjs.org/os-homedir/-/os-homedir-1.0.2.tgz",
+			"integrity": "sha512-B5JU3cabzk8c67mRRd3ECmROafjYMXbuzlwtqdM8IbS8ktlTix8aFGb2bAGKrSRIlnfKwovGUUr72JUPyOb6kQ==",
+			"dev": true
+		},
 		"os-tmpdir": {
 			"version": "1.0.2",
 			"resolved": "https://registry.npmjs.org/os-tmpdir/-/os-tmpdir-1.0.2.tgz",
 			"integrity": "sha512-D2FR03Vir7FIu45XBY20mTb+/ZSWB00sjU9jdQXt83gDrI4Ztz5Fs7/yy74g2N5SVQY4xY1qDr4rNddwYRVX0g==",
 			"dev": true
 		},
+		"osenv": {
+			"version": "0.1.5",
+			"resolved": "https://registry.npmjs.org/osenv/-/osenv-0.1.5.tgz",
+			"integrity": "sha512-0CWcCECdMVc2Rw3U5w9ZjqX6ga6ubk1xDVKxtBQPK7wis/0F2r9T6k4ydGYhecl7YUBxBVxhL5oisPsNxAPe2g==",
+			"dev": true,
+			"requires": {
+				"os-homedir": "^1.0.0",
+				"os-tmpdir": "^1.0.0"
+			}
+		},
 		"packet-reader": {
 			"version": "1.0.0",
 			"resolved": "https://registry.npmjs.org/packet-reader/-/packet-reader-1.0.0.tgz",
@@ -1448,6 +1666,18 @@
 			"integrity": "sha512-7PiHtLll5LdnKIMw100I+8xJXR5gW2QwWYkT6iJva0bXitZKa/XMrSbdmg3r2Xnaidz9Qumd0VPaMrZlF9V9sA==",
 			"dev": true
 		},
+		"proto-list": {
+			"version": "1.2.4",
+			"resolved": "https://registry.npmjs.org/proto-list/-/proto-list-1.2.4.tgz",
+			"integrity": "sha512-vtK/94akxsTMhe0/cbfpR+syPuszcuwhqVjJq26CuNDgFGj682oRBXOP5MJpv2r7JtE8MsiepGIqvvOTBwn2vA==",
+			"dev": true
+		},
+		"pseudomap": {
+			"version": "1.0.2",
+			"resolved": "https://registry.npmjs.org/pseudomap/-/pseudomap-1.0.2.tgz",
+			"integrity": "sha512-b/YwNhb8lk1Zz2+bXXpS/LK9OisiZZ1SNsSLxN1x2OXVEhW2Ckr/7mWE5vrC1ZTiJlD9g19jWszTmJsB+oEpFQ==",
+			"dev": true
+		},
 		"pump": {
 			"version": "3.0.0",
 			"resolved": "https://registry.npmjs.org/pump/-/pump-3.0.0.tgz",
@@ -1574,6 +1804,12 @@
 			"integrity": "sha512-wpoSFAxys6b2a2wHZ1XpDSgD7N9iVjg29Ph9uV/uaP9Ex/KXlkTZTeddxDPSYQpgvzKLGJke2UU0AzoGCjNIvQ==",
 			"dev": true
 		},
+		"sigmund": {
+			"version": "1.0.1",
+			"resolved": "https://registry.npmjs.org/sigmund/-/sigmund-1.0.1.tgz",
+			"integrity": "sha512-fCvEXfh6NWpm+YSuY2bpXb/VIihqWA6hLsgboC+0nl71Q7N7o2eaCW8mJa/NLvQhs6jpd3VZV4UiUQlV6+lc8g==",
+			"dev": true
+		},
 		"signal-exit": {
 			"version": "3.0.7",
 			"resolved": "https://registry.npmjs.org/signal-exit/-/signal-exit-3.0.7.tgz",
@@ -1842,6 +2078,12 @@
 			"version": "4.0.2",
 			"resolved": "https://registry.npmjs.org/xtend/-/xtend-4.0.2.tgz",
 			"integrity": "sha512-LKYU1iAXJXUgAXn9URjiu+MWhyUXHsvfp7mcuYm9dSUKK0/CjtrUwFAxD82/mCWbtLsGjFIad0wIsod4zrTAEQ=="
+		},
+		"yallist": {
+			"version": "2.1.2",
+			"resolved": "https://registry.npmjs.org/yallist/-/yallist-2.1.2.tgz",
+			"integrity": "sha512-ncTzHV7NvsQZkYe1DW7cbDLm0YpzHmZF5r/iyP3ZnQtMiJ+pjzisCiMNI+Sj+xQF5pXhSHxSB3uDbsBTzY/c2A==",
+			"dev": true
 		}
 	}
 }
diff --git a/postgres/package.json b/postgres/package.json
index e535436c..bd31563e 100644
--- a/postgres/package.json
+++ b/postgres/package.json
@@ -1,6 +1,6 @@
 {
 	"name": "leo-connector-postgres",
-	"version": "4.0.6-beta",
+	"version": "4.0.8-beta",
 	"description": "A Postgres database connector for use with Leo Platform",
 	"repository": {
 		"type": "git",
@@ -41,4 +41,4 @@
 			}
 		}
 	}
-}
\ No newline at end of file
+}

From 27039507b9cd67a3c7b79307c5631d1593d2c213 Mon Sep 17 00:00:00 2001
From: Michael Scranton <mscranton@commercehub.com>
Date: Mon, 7 Nov 2022 11:41:41 -0500
Subject: [PATCH 29/62] Fix version after conflict

---
 postgres/package.json | 6 +-----
 1 file changed, 1 insertion(+), 5 deletions(-)

diff --git a/postgres/package.json b/postgres/package.json
index dd6344fd..a8c7b539 100644
--- a/postgres/package.json
+++ b/postgres/package.json
@@ -1,10 +1,6 @@
 {
 	"name": "leo-connector-postgres",
-<<<<<<< HEAD
 	"version": "4.0.8-beta",
-=======
-	"version": "4.0.7-beta",
->>>>>>> 9de767f489744a1f607923c0745eb194e909c01f
 	"description": "A Postgres database connector for use with Leo Platform",
 	"repository": {
 		"type": "git",
@@ -45,4 +41,4 @@
 			}
 		}
 	}
-}
+}
\ No newline at end of file

From 554a6beea485b0c41274544b8170b9d7a5b876a5 Mon Sep 17 00:00:00 2001
From: Michael Scranton <mscranton@commercehub.com>
Date: Tue, 8 Nov 2022 12:33:32 -0500
Subject: [PATCH 30/62] moving audit date to proper location

---
 postgres/lib/dwconnect.js | 27 ++++++++++++++-------------
 postgres/package.json     |  2 +-
 2 files changed, 15 insertions(+), 14 deletions(-)

diff --git a/postgres/lib/dwconnect.js b/postgres/lib/dwconnect.js
index f44e6b76..46e12046 100644
--- a/postgres/lib/dwconnect.js
+++ b/postgres/lib/dwconnect.js
@@ -254,6 +254,12 @@ module.exports = function (config, columnConfig) {
 															  WHERE  ${ids.map(id => `base.${id} = staging.${id}`).join(` AND `)});`, done);
 						});
 
+						// Set audit date for staged data
+						tasks.push(done => {
+							connection.query(`UPDATE ${qualifiedStagingTable}
+					  						  SET ${columnConfig._auditdate} = ${dwClient.auditdate};`, done);
+						});
+
 						// Merge exiting data into staged copy
 						tasks.push(done => {
 							connection.query(`UPDATE ${qualifiedStagingTable} AS staging
@@ -262,12 +268,6 @@ module.exports = function (config, columnConfig) {
 											  WHERE  ${ids.map(id => `prev.${id} = staging.${id}`).join(` AND `)}`, done);
 						});
 
-						// Set audit date for staged data
-						tasks.push(done => {
-							connection.query(`UPDATE ${qualifiedStagingTable}
-					  						  SET ${columnConfig._auditdate} = ${dwClient.auditdate};`, done);
-						});
-
 						// Delete and reinsert data - avoids costly updates on large tables
 						tasks.push(done => {
 							connection.query(`DELETE FROM ${qualifiedTable}
@@ -531,6 +531,13 @@ module.exports = function (config, columnConfig) {
 								, done);
 						});
 
+						// Set auditdate and surrogate key for stage data (not sure if the sk is actually necessary)
+						tasks.push(done => {
+							connection.query(`UPDATE ${qualifiedStagingTable}
+				  							  SET ${columnConfig._auditdate} = ${dwClient.auditdate},
+											  ${sk} = farmFingerPrint64(${nk.map(id => `${id}`).join(`|| '-' ||`)});`, done);
+						});
+
 						// Merge exiting data into staged copy
 						tasks.push(done => {
 							connection.query(`UPDATE ${qualifiedStagingTable} AS staging
@@ -540,13 +547,6 @@ module.exports = function (config, columnConfig) {
 								, done);
 						});
 
-						// Set auditdate and surrogate key for stage data (not sure if the sk is actually necessary)
-						tasks.push(done => {
-							connection.query(`UPDATE ${qualifiedStagingTable}
-				  							  SET ${columnConfig._auditdate} = ${dwClient.auditdate},
-											  ${sk} = farmFingerPrint64(${nk.map(id => `${id}`).join(`|| '-' ||`)});`, done);
-						});
-
 						// Delete and reinsert data - avoids costly updates on large tables
 						tasks.push(done => {
 							connection.query(`DELETE FROM ${qualifiedTable}
@@ -650,6 +650,7 @@ module.exports = function (config, columnConfig) {
 			let tasks = [];
 			let sets = [];
 
+			const qualifiedTable = `public.${table}`;
 			const linkAuditdate = client.escapeValueNoToLower(new Date().toISOString().replace(/\.\d*Z/, 'Z'));
 
 			// Only run analyze on the table if this is the first load
diff --git a/postgres/package.json b/postgres/package.json
index a8c7b539..bd31563e 100644
--- a/postgres/package.json
+++ b/postgres/package.json
@@ -41,4 +41,4 @@
 			}
 		}
 	}
-}
\ No newline at end of file
+}

From 76d07282c33aae048378519a7a33f7069a085dfc Mon Sep 17 00:00:00 2001
From: Michael Scranton <mscranton@commercehub.com>
Date: Thu, 15 Dec 2022 13:08:47 -0500
Subject: [PATCH 31/62] Various fixes and cleanup

---
 postgres/lib/dwconnect.js | 515 ++++++++++++++++++++++----------------
 1 file changed, 293 insertions(+), 222 deletions(-)

diff --git a/postgres/lib/dwconnect.js b/postgres/lib/dwconnect.js
index 46e12046..8de5433f 100644
--- a/postgres/lib/dwconnect.js
+++ b/postgres/lib/dwconnect.js
@@ -3,6 +3,9 @@ const postgres = require('./connect.js');
 const async = require('async');
 const ls = require('leo-sdk').streams;
 const logger = require('leo-logger');
+const { basename } = require('path');
+const { doesNotThrow } = require('assert');
+const { table } = require('console');
 
 module.exports = function (config, columnConfig) {
 	let client = postgres(config);
@@ -131,61 +134,61 @@ module.exports = function (config, columnConfig) {
 
 		let deleteHandler = deletesSetup(qualifiedTable, schema[qualifiedTable], columnConfig._deleted, true);
 
+		let sortKey;
+
 		tempTables.push(qualifiedStagingTable);
 		tasks.push(done => client.query(`drop table if exists ${qualifiedStagingTable}`, done));
 		if (config.version !== 'redshift') {
 			tasks.push(done => client.query(`create table ${qualifiedStagingTable} (like ${qualifiedTable})`, done));
 			tasks.push(done => client.query(`create index ${stagingTable}_id on ${qualifiedStagingTable} (${ids.join(', ')})`, done));
 		} else {
-			tasks.push(done =>
-				client.query(`create table ${qualifiedStagingTable} 
-							  diststyle all 
-							  as select *
-								 from ${qualifiedTable}
-								 limit 0;`, done));
-		};
-
-		if (config.version !== 'redshift') {
+			// get sortkey for joins
 			tasks.push(done => {
-				ls.pipe(stream, ls.through((obj, done, push) => {
-					if (obj.__leo_delete__) {
-						if (obj.__leo_delete__ === 'id') {
-							push(obj);
-						}
-						deleteHandler.add(obj, done);
-					} else {
-						done(null, obj);
-					}
-				}), client.streamToTable(qualifiedStagingTable), (err) => {
+				client.query(`SELECT sortkey
+					 		  FROM   public.mv_dist_sort_key
+					 		  WHERE  table_name = '${table}';`, (err, results) => {
 					if (err) {
 						return done(err);
 					} else {
-						deleteHandler.flush(done);
-					}
+						if (results[0].sortKey !== null) {
+							sortKey = results[0].sortkey;
+						};
+						done();
+					};
 				});
 			});
-		} else {
-			tasks.push(done => {
-				ls.pipe(stream, ls.through((obj, done, push) => {
-					if (obj.__leo_delete__) {
-						if (obj.__leo_delete__ === 'id') {
-							push(obj);
-						}
-						deleteHandler.add(obj, done);
-					} else {
-						done(null, obj);
-					}
-				}), client.streamToTableFromS3(qualifiedStagingTable, config), (err) => {
-					if (err) {
-						return done(err);
-					} else {
-						deleteHandler.flush(done);
+			tasks.push(done =>
+				client.query(`CREATE TABLE ${qualifiedStagingTable} 
+							  DISTSTYLE ALL 
+							  SORTKEY (${sortKey !== undefined ? sortKey : ids[0]})
+							  AS SELECT *
+								 FROM   ${qualifiedTable}
+								 LIMIT  0;`, done));
+		};
+
+
+		tasks.push(done => {
+			ls.pipe(stream, ls.through((obj, done, push) => {
+				if (obj.__leo_delete__) {
+					if (obj.__leo_delete__ === 'id') {
+						push(obj);
 					}
-				});
+					deleteHandler.add(obj, done);
+				} else {
+					done(null, obj);
+				}
+			}), config.version !== 'redshift' ? client.streamToTable(qualifiedStagingTable) : client.streamToTableFromS3(qualifiedStagingTable, config), (err) => {
+				if (err) {
+					return done(err);
+				} else {
+					deleteHandler.flush(done);
+				}
 			});
-		};
+		});
 
-		tasks.push(done => client.query(`analyze ${qualifiedStagingTable}`, done));
+		if (config.version !== 'redshift') {
+			tasks.push(done => client.query(`analyze ${qualifiedStagingTable}`, done));
+		};
 
 		client.describeTable(table).then(result => {
 			let columns = result.filter(field => !field.column_name.match(/^_/)).map(field => `"${field.column_name}"`);
@@ -214,72 +217,95 @@ module.exports = function (config, columnConfig) {
 						});
 						tasks.push(done => {
 							connection.query(`Update ${qualifiedTable} prev
-								SET  ${columns.map(column => `${column} = coalesce(staging.${column}, prev.${column})`)}, ${columnConfig._deleted} = coalesce(prev.${columnConfig._deleted}, false), ${columnConfig._auditdate} = ${dwClient.auditdate}
-								FROM ${qualifiedStagingTable} staging
-								where ${ids.map(id => `prev.${id} = staging.${id}`).join(' and ')}
-							`, done);
+											  SET  ${columns.map(column => `${column} = coalesce(staging.${column}, prev.${column})`)}, ${columnConfig._deleted} = coalesce(prev.${columnConfig._deleted}, false), ${columnConfig._auditdate} = ${dwClient.auditdate}
+											  FROM ${qualifiedStagingTable} staging
+											  where ${ids.map(id => `prev.${id} = staging.${id}`).join(' and ')}`
+								, done);
 						});
 
 						// Now insert any we were missing
 						tasks.push(done => {
 							connection.query(`INSERT INTO ${qualifiedTable} (${columns.join(',')},${columnConfig._deleted},${columnConfig._auditdate})
-								SELECT ${columns.map(column => `staging.${column}`)},
-										false AS ${columnConfig._deleted},
-										${dwClient.auditdate} as ${columnConfig._auditdate}
-								FROM ${qualifiedStagingTable} staging
-								WHERE NOT EXISTS (
-									SELECT *
-									FROM   ${qualifiedTable} as prev
-									WHERE ${ids.map(id => `prev.${id} = staging.${id}`).join(' and ')}
-								)	
-							`, done);
+											  SELECT ${columns.map(column => `staging.${column}`)}, false AS ${columnConfig._deleted}, ${dwClient.auditdate} as ${columnConfig._auditdate}
+											  FROM ${qualifiedStagingTable} staging
+											  WHERE NOT EXISTS ( SELECT *
+																 FROM   ${qualifiedTable} as prev
+																 WHERE  ${ids.map(id => `prev.${id} = staging.${id}`).join(' and ')})`, done);
 						});
 					} else {
-						// Retreive copy of existing data
-						tempTables.push(`${qualifiedStagingTable}_previous`);
-						tasks.push(done => client.query(`drop table if exists ${qualifiedStagingTable}_previous;`, done));
+						columns = columns.concat([columnConfig._auditdate, columnConfig._deleted]);
+						let naturalKeyLowerBound;
+						let naturalKeyFilter;
+
+						// Get lower bound for natural key to avoid unnecessary scanning
 						tasks.push(done => {
-							connection.query(`CREATE TABLE ${qualifiedStagingTable}_previous
+							connection.query(`SELECT MIN(${(sortKey !== undefined) ? sortKey : ids[0]}) AS minid,
+													 CAST(COUNT(*) AS INT) AS cnt
+											  FROM   ${qualifiedStagingTable};`, (err, results) => {
+								if (err) {
+									return done(err);
+								} else if (results[0].cnt === 0) {
+									totalRecords = results[0].cnt;
+									return done();
+								} else {
+									totalRecords = results[0].cnt;
+									naturalKeyLowerBound = results[0].minid;
+									if (naturalKeyLowerBound !== null) {
+										naturalKeyFilter = (typeof naturalKeyLowerBound === 'string') ? `'${results[0].minid}'` : `${results[0].minid}`
+									};
+									done();
+								};
+							});
+						});
+
+						const qualifiedStagingTablePrevious = `${qualifiedStagingTable}_previous`;
+						tempTables.push(`${qualifiedStagingTablePrevious}`);
+						tasks.push(done => connection.query(`DROP TABLE IF EXISTS ${qualifiedStagingTablePrevious};`, done));
+						tasks.push(done => {
+							connection.query(`CREATE TABLE ${qualifiedStagingTablePrevious}
 											  DISTSTYLE ALL 
 											  AS SELECT *
 												 FROM   ${qualifiedTable}
 												 LIMIT  0;`, done);
 						});
-						tasks.push(done => {
-							connection.query(`INSERT INTO ${qualifiedStagingTable}_previous
-											  SELECT *
-											  FROM   ${qualifiedTable} AS base
-											  WHERE  EXISTS ( SELECT *
-															  FROM   ${qualifiedStagingTable} AS staging
-															  WHERE  ${ids.map(id => `base.${id} = staging.${id}`).join(` AND `)});`, done);
-						});
 
 						// Set audit date for staged data
 						tasks.push(done => {
 							connection.query(`UPDATE ${qualifiedStagingTable}
-					  						  SET ${columnConfig._auditdate} = ${dwClient.auditdate};`, done);
+											  SET ${columnConfig._auditdate} = ${dwClient.auditdate};`, done);
+						});
+
+						// Retreive copy of existing data
+						tasks.push(done => {
+							connection.query(`INSERT INTO ${qualifiedStagingTablePrevious} (${columns.map(column => `${column}`).join(`, `)})
+											  SELECT ${columns.map(column => `${column}`).join(`, `)}
+											  FROM   ${qualifiedTable} AS base
+											  WHERE  EXISTS ( SELECT *
+															  FROM   ${qualifiedStagingTable} AS staging
+															  WHERE  ${ids.map(id => `base.${id} = staging.${id}`).join(` AND `)}
+															  		 ${(naturalKeyFilter !== undefined) ? `AND staging.${(sortKey !== undefined) ? sortKey : ids[0]} >= ${naturalKeyFilter}` : ``})
+													 ${(naturalKeyFilter !== undefined) ? `AND base.${(sortKey !== undefined) ? sortKey : nk[0]} >= ${naturalKeyFilter}` : ``};`, done);
 						});
 
 						// Merge exiting data into staged copy
 						tasks.push(done => {
 							connection.query(`UPDATE ${qualifiedStagingTable} AS staging
 											  SET    ${columns.map(column => `${column} = COALESCE(staging.${column}, prev.${column})`).join(`,`)}
-											  FROM   ${qualifiedStagingTable}_previous AS prev
+											  FROM   ${qualifiedStagingTablePrevious} AS prev
 											  WHERE  ${ids.map(id => `prev.${id} = staging.${id}`).join(` AND `)}`, done);
 						});
 
 						// Delete and reinsert data - avoids costly updates on large tables
 						tasks.push(done => {
 							connection.query(`DELETE FROM ${qualifiedTable}
-											  USING ${qualifiedStagingTable}
-											  WHERE ${ids.map(id => `${qualifiedTable}.${id} = ${qualifiedTable}.${id}`).join(` AND `)}`, done);
+											  USING  ${qualifiedStagingTable}
+											  WHERE  ${ids.map(id => `${qualifiedTable}.${id} = ${qualifiedStagingTable}.${id}`).join(` AND `)}
+											  		 ${(naturalKeyFilter !== undefined) ? `AND ${qualifiedTable}.${sortKey !== undefined ? sortKey : ids[0]} >= ${naturalKeyFilter}` : ``}; `, done);
 						});
 						tasks.push(done => {
-							connection.query(`INSERT INTO ${qualifiedTable} (
-											  ${columns.map(column => `${column}`).join(`,\n`)}
-											  )
-											  SELECT ${columns.map(column => `${column}`).join(`,\n`)}
-											  FROM   ${qualifiedStagingTable};`, done);
+							connection.query(`INSERT INTO ${qualifiedTable} (${columns.map(column => `${column}`).join(`, `)})
+											  SELECT ${columns.map(column => `${column}`).join(`, `)}
+											  FROM   ${qualifiedStagingTable}; `, done);
 						});
 					};
 
@@ -321,66 +347,68 @@ module.exports = function (config, columnConfig) {
 		let tasks = [];
 		let deleteHandler = deletesSetup(qualifiedTable, schema[qualifiedTable], columnConfig._enddate, dwClient.auditdate, `${columnConfig._current} = true`);
 
-		tempTables.push(qualifiedStagingTable);
+		let sortKey;
 
 		// Prepare staging tables
+		tempTables.push(qualifiedStagingTable);
 		tasks.push(done => client.query(`drop table if exists ${qualifiedStagingTable}`, done));
 		tasks.push(done => client.query(`drop table if exists ${qualifiedStagingTable}_changes`, done));
 		if (config.version !== 'redshift') {
 			tasks.push(done => client.query(`create table ${qualifiedStagingTable} (like ${qualifiedTable})`, done));
 			tasks.push(done => client.query(`create index ${stagingTbl}_id on ${qualifiedStagingTable} (${nk.join(', ')})`, done));
 		} else {
-			tasks.push(done =>
-				// Create staging table with DISTSTYLE ALL to prevent cross talk
-				client.query(`create table ${qualifiedStagingTable} 
-							  diststyle all 
-							  as select *
-								 from ${qualifiedTable}
-								 limit 0;`, done));
-		};
-		// tasks.push(done => client.query(`alter table ${qualifiedStagingTable} drop column ${sk}`, done));
-
-		if (config.version !== 'redshift') {
+			// get sortkey for joins
 			tasks.push(done => {
-				ls.pipe(stream, ls.through((obj, done, push) => {
-					if (obj.__leo_delete__) {
-						if (obj.__leo_delete__ === 'id') {
-							push(obj);
-						}
-						deleteHandler.add(obj, done);
-					} else {
-						done(null, obj);
-					}
-				}), client.streamToTable(qualifiedStagingTable), (err) => {
+				client.query(`SELECT sortkey
+					  		  FROM   public.mv_dist_sort_key
+					  		  WHERE  table_name = '${table}';`, (err, results) => {
 					if (err) {
 						return done(err);
 					} else {
-						deleteHandler.flush(done);
-					}
+						if (results[0].sortkey !== null) {
+							sortKey = results[0].sortkey;
+						};
+						done();
+					};
 				});
 			});
-		} else {
-			tasks.push(done => {
-				ls.pipe(stream, ls.through((obj, done, push) => {
-					if (obj.__leo_delete__) {
-						if (obj.__leo_delete__ === 'id') {
-							push(obj);
-						}
-						deleteHandler.add(obj, done);
-					} else {
-						done(null, obj);
-					}
-				}), client.streamToTableFromS3(qualifiedStagingTable, config), (err) => {
-					if (err) {
-						return done(err);
-					} else {
-						deleteHandler.flush(done);
+			tasks.push(done =>
+				// Create staging table with DISTSTYLE ALL to prevent cross talk
+				client.query(`CREATE TABLE ${qualifiedStagingTable} 
+							  DISTSTYLE ALL
+							  SORTKEY(${sortKey !== undefined ? sortKey : nk[0]})
+							  AS SELECT *
+							  FROM ${qualifiedTable}
+								LIMIT 0;`, done));
+		};
+
+		// Retain surrogate key column when using hashed keys on redshift // column must be dropped for postgres
+		if (!config.hashedSurrogateKeys && config.version !== 'redshift') {
+			tasks.push(done => client.query(`alter table ${qualifiedStagingTable} drop column ${sk}`, done));
+		};
+
+		tasks.push(done => {
+			ls.pipe(stream, ls.through((obj, done, push) => {
+				if (obj.__leo_delete__) {
+					if (obj.__leo_delete__ === 'id') {
+						push(obj);
 					}
-				});
+					deleteHandler.add(obj, done);
+				} else {
+					done(null, obj);
+				}
+			}), config.version !== 'redshift' ? client.streamToTable(qualifiedStagingTable) : client.streamToTableFromS3(qualifiedStagingTable, config), (err) => {
+				if (err) {
+					return done(err);
+				} else {
+					deleteHandler.flush(done);
+				}
 			});
-		};
+		});
 
-		tasks.push(done => client.query(`analyze ${qualifiedStagingTable}`, done));
+		if (config.version !== 'redshift') {
+			tasks.push(done => client.query(`analyze ${qualifiedStagingTable}`, done));
+		};
 
 		client.describeTable(table).then(result => {
 			client.connect().then(connection => {
@@ -428,29 +456,25 @@ module.exports = function (config, columnConfig) {
 					};
 
 					let fields = [sk].concat(allColumns).concat([columnConfig._auditdate, columnConfig._startdate, columnConfig._enddate, columnConfig._current]);
+					let tasks = [];
 					let totalRecords = 0;
 
-					if (!config.hashedSurrogateKeys) {
+					if (!config.hashedSurrogateKeys && !config.bypassSlowlyChangingDimensions) {
 						tempTables.push(`${qualifiedStagingTable}_changes`);
-						connection.query(`create table ${qualifiedStagingTable}_changes as 
-									  	select ${nk.map(id => `s.${id}`).join(', ')}, 
-											   d.${nk[0]} is null as isNew
-										${!config.bypassSlowlyChangingDimensions ? `,${scdSQL.join(',\n')}` : ``}
-										FROM ${qualifiedStagingTable} s
-										LEFT JOIN ${qualifiedTable} d on ${nk.map(id => `d.${id} = s.${id}`).join(' and ')} and d.${columnConfig._current}`, (err) => {
+						connection.query(`create table ${qualifiedStagingTable}_changes as
+										  select ${nk.map(id => `s.${id}`).join(', ')},
+												 d.${nk[0]} is null as isNew
+										  ${!config.bypassSlowlyChangingDimensions ? `,${scdSQL.join(',\n')}` : ``}
+										  FROM ${qualifiedStagingTable} s
+										  LEFT JOIN ${qualifiedTable} d on ${nk.map(id => `d.${id} = s.${id}`).join(' and ')} and d.${columnConfig._current}`, (err) => {
 							if (err) {
 								logger.error(err);
 								process.exit();
 							}
 
-							let tasks = [];
 							let rowId = null;
 
 							tasks.push(done => connection.query(`analyze ${qualifiedStagingTable}_changes`, done));
-
-							// The following code relies on the fact that now()/sysdate will return the same time during all transaction events
-							tasks.push(done => connection.query(`Begin Transaction`, done));
-
 							tasks.push(done => {
 								connection.query(`select max(${sk}) as maxid from ${qualifiedTable}`, (err, results) => {
 									if (err) {
@@ -462,121 +486,131 @@ module.exports = function (config, columnConfig) {
 								});
 							});
 
+							// The following code relies on the fact that now()/sysdate will return the same time during all transaction events
+							tasks.push(done => connection.query(`Begin Transaction`, done));
+
 							tasks.push(done => {
 								connection.query(`INSERT INTO ${qualifiedTable} (${fields.join(',')})
-									SELECT row_number() over () + ${rowId}, 
-										   ${allColumns.map(column => `coalesce(staging.${column}, prev.${column})`)}, 
-										   ${dwClient.auditdate} as ${columnConfig._auditdate}, 
-										   case when changes.isNew then '1900-01-01 00:00:00' else ${config.version === 'redshift' ? 'sysdate' : 'now()'} END as ${columnConfig._startdate}, 
-										   '9999-01-01 00:00:00' as ${columnConfig._enddate}, 
-										   true as ${columnConfig._current}
-									FROM ${qualifiedStagingTable}_changes changes  
-									JOIN ${qualifiedStagingTable} staging on ${nk.map(id => `staging.${id} = changes.${id}`).join(' and ')}
-									LEFT JOIN ${qualifiedTable} as prev on ${nk.map(id => `prev.${id} = changes.${id}`).join(' and ')} and prev.${columnConfig._current}
-									WHERE (changes.runSCD2 =1 OR changes.runSCD6=1)		
+												  SELECT row_number() over() + ${rowId}, ${allColumns.map(column => `coalesce(staging.${column}, prev.${column})`)}, ${dwClient.auditdate} as ${columnConfig._auditdate}, 
+														 case when changes.isNew then '1900-01-01 00:00:00' else ${config.version === 'redshift' ? 'sysdate' : 'now()'} END as ${columnConfig._startdate},
+														 '9999-01-01 00:00:00' as ${columnConfig._enddate},
+														 true as ${columnConfig._current}
+												  FROM ${qualifiedStagingTable}_changes changes  
+												  JOIN ${qualifiedStagingTable} staging on ${nk.map(id => `staging.${id} = changes.${id}`).join(' and ')}
+												  LEFT JOIN ${qualifiedTable} as prev on ${nk.map(id => `prev.${id} = changes.${id}`).join(' and ')} and prev.${columnConfig._current}
+												  WHERE (changes.runSCD2 =1 OR changes.runSCD6=1)		
 									`, done);
 							});
 
 							// This needs to be done last
-							let columns = scd1.map(column => `"${column}" = coalesce(staging."${column}", prev."${column}")`).concat(scd6.map(column => `"current_${column}" = coalesce(staging."${column}", prev."${column}")`));
 							tasks.push(done => {
 								// RUN SCD1 / SCD6 columns  (where we update the old records)
+								let columns = scd1.map(column => `"${column}" = coalesce(staging."${column}", prev."${column}")`).concat(scd6.map(column => `"current_${column}" = coalesce(staging."${column}", prev."${column}")`));
 								columns.push(`"${columnConfig._enddate}" = case when changes.runSCD2 =1 then ${config.version === 'redshift' ? 'sysdate' : 'now()'} else prev."${columnConfig._enddate}" END`);
 								columns.push(`"${columnConfig._current}" = case when changes.runSCD2 =1 then false else prev."${columnConfig._current}" END`);
 								columns.push(`"${columnConfig._auditdate}" = ${dwClient.auditdate}`);
 								connection.query(`update ${qualifiedTable} as prev
-											set  ${columns.join(', ')}
-											FROM ${qualifiedStagingTable}_changes changes
-											JOIN ${qualifiedStagingTable} staging on ${nk.map(id => `staging.${id} = changes.${id}`).join(' and ')}
-											where ${nk.map(id => `prev.${id} = changes.${id}`).join(' and ')} and prev.${columnConfig._startdate} != ${config.version === 'redshift' ? 'sysdate' : 'now()'} and changes.isNew = false 
-											/*Need to make sure we are only updating the ones not just inserted through SCD2 otherwise we run into issues with multiple rows having .${columnConfig._current}*/
-												and (changes.runSCD1=1 OR  changes.runSCD6=1 OR changes.runSCD2=1)
+												  set  ${columns.join(', ')}
+												  FROM ${qualifiedStagingTable}_changes changes
+												  JOIN ${qualifiedStagingTable} staging on ${nk.map(id => `staging.${id} = changes.${id}`).join(' and ')}
+												  where ${nk.map(id => `prev.${id} = changes.${id}`).join(' and ')} and prev.${columnConfig._startdate} != ${config.version === 'redshift' ? 'sysdate' : 'now()'} and changes.isNew = false /*Need to make sure we are only updating the ones not just inserted through SCD2 otherwise we run into issues with multiple rows having .${columnConfig._current}*/
+												  and (changes.runSCD1=1 OR  changes.runSCD6=1 OR changes.runSCD2=1)
 											`, done);
 							});
-
-							async.series(tasks, err => {
-								if (!err) {
-									connection.query(`commit`, e => {
-										connection.release();
-										callback(e || err, {
-											count: totalRecords,
-										});
-									});
-								} else {
-									connection.query(`rollback`, (e, d) => {
-										connection.release();
-										callback(e, d);
-									});
-								}
-							});
 						});
 					} else {
-						//Retreive copy of existing data
-						tempTables.push(`${qualifiedStagingTable}_previous`);
-						tasks.push(done => client.query(`drop table if exists ${qualifiedStagingTable}_previous;`, done));
+						let naturalKeyLowerBound;
+						let naturalKeyFilter;
+
+						// Get lower bound for natural key to avoid unnecessary scanning
 						tasks.push(done => {
-							connection.query(`create table ${qualifiedStagingTable}_previous
-											  diststyle all 
-											  as select *
-												 from ${qualifiedTable}
-												 limit 0;`, done);
+							connection.query(`SELECT MIN(${(sortKey !== undefined) ? sortKey : nk[0]}) AS minid,
+													 CAST(COUNT(*) AS INT) AS cnt
+											  FROM   ${qualifiedStagingTable};`, (err, results) => {
+								if (err) {
+									return done(err);
+								} else if (results[0].cnt === 0) {
+									totalRecords = results[0].cnt;
+									return done();
+								} else {
+									totalRecords = results[0].cnt;
+									naturalKeyLowerBound = results[0].minid;
+									if (naturalKeyLowerBound !== null) {
+										naturalKeyFilter = (typeof naturalKeyLowerBound === 'string') ? `'${results[0].minid}'` : `${results[0].minid}`;
+									};
+									done();
+								};
+							});
 						});
+
+						const qualifiedStagingTablePrevious = `${qualifiedStagingTable}_previous`;
+						tempTables.push(`${qualifiedStagingTablePrevious}`);
+						tasks.push(done => connection.query(`DROP TABLE IF EXISTS ${qualifiedStagingTablePrevious};`, done));
 						tasks.push(done => {
-							connection.query(`INSERT INTO ${qualifiedStagingTable}_previous
-											  SELECT *
-											  FROM   ${qualifiedTable} AS base
-											  WHERE  EXISTS ( SELECT *
-															  FROM   ${qualifiedStagingTable} AS staging
-															  WHERE  ${nk.map(id => `base.${id} = staging.${id}`).join(` AND `)});`
-								, done);
+							connection.query(`CREATE TABLE ${qualifiedStagingTablePrevious}
+											  DISTSTYLE ALL
+											  AS SELECT *
+											  FROM ${qualifiedTable}
+											  LIMIT 0;`, done);
 						});
 
 						// Set auditdate and surrogate key for stage data (not sure if the sk is actually necessary)
 						tasks.push(done => {
 							connection.query(`UPDATE ${qualifiedStagingTable}
-				  							  SET ${columnConfig._auditdate} = ${dwClient.auditdate},
-											  ${sk} = farmFingerPrint64(${nk.map(id => `${id}`).join(`|| '-' ||`)});`, done);
+											  SET    ${columnConfig._auditdate} = ${dwClient.auditdate},
+											  		 ${sk} = farmFingerPrint64(${nk.map(id => `${id}`).join(`|| '-' ||`)}); `, done);
+						});
+
+						tasks.push(done => connection.query(`BEGIN TRANSACTION;`, done));
+
+						// Retreive copy of existing data
+						tasks.push(done => {
+							connection.query(`INSERT INTO ${qualifiedStagingTablePrevious}(${fields.map(column => `${column}`).join(`, `)})
+											  SELECT ${fields.map(column => `${column}`).join(`, `)}
+											  FROM   ${qualifiedTable} AS base
+											  WHERE  EXISTS(SELECT *
+											  FROM   ${qualifiedStagingTable} AS staging
+															 WHERE  base.${sk} = staging.${sk}
+																	${(naturalKeyFilter !== undefined) ? `AND staging.${(sortKey !== undefined) ? sortKey : nk[0]} >= ${naturalKeyFilter}` : ``})
+													 ${(naturalKeyFilter !== undefined) ? `AND base.${(sortKey !== undefined) ? sortKey : nk[0]} >= ${naturalKeyFilter}` : ``}; `, done);
 						});
 
 						// Merge exiting data into staged copy
 						tasks.push(done => {
 							connection.query(`UPDATE ${qualifiedStagingTable} AS staging
-										  	  SET    ${fields.map(column => `${column} = COALESCE(staging.${column}, prev.${column})`).join(`,`)}
-										  	  FROM   ${qualifiedStagingTable}_previous AS prev
-										  	  WHERE  ${nk.map(id => `prev.${id} = staging.${id}`).join(` AND `)}`
-								, done);
+											  SET    ${fields.map(column => `${column} = COALESCE(staging.${column}, prev.${column})`).join(`, `)}
+											  FROM   ${qualifiedStagingTablePrevious} AS prev
+											  WHERE  staging.${sk} = prev.${sk}`, done);
 						});
 
 						// Delete and reinsert data - avoids costly updates on large tables
 						tasks.push(done => {
 							connection.query(`DELETE FROM ${qualifiedTable}
-										  	  USING ${qualifiedStagingTable}
-										  	  WHERE ${nk.map(id => `${qualifiedTable}.${id} = ${qualifiedTable}.${id}`).join(` AND `)}`
-								, done);
+											  USING  ${qualifiedStagingTable}
+											  WHERE  ${qualifiedTable}.${sk} = ${qualifiedStagingTable}.${sk}
+											  		 ${(naturalKeyFilter !== undefined) ? `AND ${qualifiedTable}.${sortKey !== undefined ? sortKey : nk[0]} >= ${naturalKeyFilter}` : ``}; `, done);
 						});
 						tasks.push(done => {
-							connection.query(`INSERT INTO ${qualifiedTable}
-										  	  SELECT *
-										  	  FROM   ${qualifiedStagingTable};`
-								, done);
-						});
-
-						async.series(tasks, err => {
-							if (!err) {
-								connection.query(`commit`, e => {
-									connection.release();
-									callback(e || err, {
-										count: totalRecords,
-									});
-								});
-							} else {
-								connection.query(`rollback`, (e, d) => {
-									connection.release();
-									callback(e, d);
-								});
-							}
+							connection.query(`INSERT INTO ${qualifiedTable} (${fields.map(column => `${column}`).join(`, `)})
+											  SELECT ${fields.map(column => `${column}`).join(`, `)}
+											  FROM   ${qualifiedStagingTable}; `, done);
 						});
 					};
+					async.series(tasks, err => {
+						if (!err) {
+							connection.query(`commit`, e => {
+								connection.release();
+								callback(e || err, {
+									count: totalRecords,
+								});
+							});
+						} else {
+							connection.query(`rollback`, (e, d) => {
+								connection.release();
+								callback(e, d);
+							});
+						}
+					});
 				});
 			}).catch(callback);
 		}).catch(callback);
@@ -631,7 +665,7 @@ module.exports = function (config, columnConfig) {
 							let rowId = results[0].maxid || 10000;
 							let _auditdate = dwClient.auditdate;
 							let unionQuery = unions[table].join('\nUNION\n');
-							transaction.query(`insert into ${table} (${sk}, ${nk}, ${columnConfig._auditdate}, ${columnConfig._startdate}, ${columnConfig._enddate}, ${columnConfig._current}) select row_number() over () + ${rowId}, sub.id, max(${_auditdate})::timestamp, '1900-01-01 00:00:00', '9999-01-01 00:00:00', true from (${unionQuery}) as sub where sub.id is not null group by sub.id`, (err) => {
+							transaction.query(`insert into ${table} (${sk}, ${nk}, ${columnConfig._auditdate}, ${columnConfig._startdate}, ${columnConfig._enddate}, ${columnConfig._current}) select row_number() over() + ${rowId}, sub.id, max(${_auditdate})::timestamp, '1900-01-01 00:00:00', '9999-01-01 00:00:00', true from(${unionQuery}) as sub where sub.id is not null group by sub.id`, (err) => {
 								done(err);
 							});
 						});
@@ -654,9 +688,46 @@ module.exports = function (config, columnConfig) {
 			const linkAuditdate = client.escapeValueNoToLower(new Date().toISOString().replace(/\.\d*Z/, 'Z'));
 
 			// Only run analyze on the table if this is the first load
-			if (tableStatus === 'First Load') {
+			if (tableStatus === 'First Load' && config.version !== 'redshift') {
 				tasks.push(done => client.query(`ANALYZE ${table}`, done));
 			}
+
+			const qualifiedStagingTable = `${columnConfig.stageSchema}.${columnConfig.stageTablePrefix}_${table} `;
+			let naturalKeyLowerBound;
+			let naturalKeyFilter;
+			let sortKey;
+			// Get lower bound for natural key to avoid unnecessary scanning
+			if (config.version === 'redshift') {
+				tasks.push(done => {
+					client.query(`SELECT sortkey
+								  FROM   public.mv_dist_sort_key
+								  WHERE  table_name = '${table}'; `, (err, results) => {
+						if (err) {
+							return done(err);
+						} else {
+							if (results[0].sortKey !== null) {
+								sortKey = results[0].sortkey;
+							};
+							done();
+						};
+					});
+				});
+				tasks.push(done => {
+					client.query(`SELECT MIN(${(sortKey !== undefined) ? sortKey : nk[0]}) AS minid
+							  	  FROM   ${qualifiedStagingTable}; `, (err, results) => {
+						if (err) {
+							return done(err);
+						} else {
+							naturalKeyLowerBound = results[0].minid;
+							if (naturalKeyLowerBound !== null) {
+								naturalKeyFilter = (typeof naturalKeyLowerBound === 'string') ? `'${results[0].minid}'` : `${results[0].minid} `
+							};
+							done();
+						};
+					});
+				});
+			};
+
 			tasks.push(done => {
 				let joinTables = links.map(link => {
 					if (columnConfig.useSurrogateDateKeys && (link.table === 'd_datetime' || link.table === 'datetime' || link.table === 'dim_datetime')) {
@@ -672,27 +743,27 @@ module.exports = function (config, columnConfig) {
 							return ``;
 						} else {
 							sets.push(`${link.destination} = coalesce(${link.join_id}_join_table.${link.sk}, 1)`);
-							var joinOn = `${link.join_id}_join_table.${link.on} = t.${link.source}`;
+							var joinOn = `${link.join_id}_join_table.${link.on} = t.${link.source} `;
 
 							if (Array.isArray(link.source)) {
-								joinOn = link.source.map((v, i) => `${link.join_id}_join_table.${link.on[i]} = t.${v}`).join(' AND ');
+								joinOn = link.source.map((v, i) => `${link.join_id}_join_table.${link.on[i]} = t.${v} `).join(' AND ');
 							}
 							return `LEFT JOIN ${link.table} ${link.join_id}_join_table
 									ON ${joinOn} 
 									AND t.${link.link_date} >= ${link.join_id}_join_table.${columnConfig._startdate}
-									AND (t.${link.link_date} <= ${link.join_id}_join_table.${columnConfig._enddate} or ${link.join_id}_join_table.${columnConfig._current})`;
+									AND(t.${link.link_date} <= ${link.join_id}_join_table.${columnConfig._enddate} or ${link.join_id}_join_table.${columnConfig._current})`;
 						};
 					}
 				});
 
 				if (sets.length) {
 					client.query(`UPDATE ${table} dm
-                        SET  ${sets.join(', ')}, ${columnConfig._auditdate} = ${linkAuditdate}
-                        FROM ${table} t
-						${config.hashedSurrogateKeys ? '' : joinTables.join('\n')}
-                        WHERE ${nk.map(id => `dm.${id} = t.${id}`).join(' AND ')}
-							AND dm.${columnConfig._auditdate} = ${dwClient.auditdate} AND t.${columnConfig._auditdate} = ${dwClient.auditdate}
-                    `, done);
+                       			  SET  ${sets.join(', ')}, ${columnConfig._auditdate} = ${linkAuditdate}
+                        		  FROM ${table} t
+								  ${config.hashedSurrogateKeys ? '' : joinTables.join('\n')}
+                        		  WHERE ${nk.map(id => `dm.${id} = t.${id}`).join(' AND ')}
+								  AND dm.${columnConfig._auditdate} = ${dwClient.auditdate} AND t.${columnConfig._auditdate} = ${dwClient.auditdate}
+								  ${(naturalKeyFilter !== undefined) ? `AND t.${(sortKey !== undefined) ? sortKey : nk[0]} >= ${naturalKeyFilter}` : ``} `, done);
 				} else {
 					done();
 				}
@@ -851,7 +922,7 @@ module.exports = function (config, columnConfig) {
 		});
 
 		let sql = `create table ${table} (
-				${fields.join(',\n')}
+		${fields.join(',\n')}
 			)`;
 
 		/*
@@ -953,7 +1024,7 @@ module.exports = function (config, columnConfig) {
 		});
 		let sqls = [`alter table  ${table} 
 				add column ${fields.join(',\n add column ')}
-			`];
+	`];
 
 		// redshift doesn't support multi 'add column' in one query
 		if (config.version === 'redshift') {
@@ -1013,7 +1084,7 @@ module.exports = function (config, columnConfig) {
 			client.query(`select to_char(min(${columnConfig._auditdate}), 'YYYY-MM-DD HH24:MI:SS') as oldest, count(*) as count
         FROM ${client.escapeId(table)}
         ${where}
-        `, (err, result) => {
+	`, (err, result) => {
 				if (err) {
 					callback(err);
 				}
@@ -1141,4 +1212,4 @@ module.exports = function (config, columnConfig) {
 	};
 
 	return client;
-};
+};
\ No newline at end of file

From cec0732575d8a5d9e31390eab26437c95f78a80a Mon Sep 17 00:00:00 2001
From: Michael Scranton <mscranton@commercehub.com>
Date: Thu, 15 Dec 2022 13:39:27 -0500
Subject: [PATCH 32/62] Bump version

---
 postgres/package.json | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/postgres/package.json b/postgres/package.json
index bd31563e..e3fa4149 100644
--- a/postgres/package.json
+++ b/postgres/package.json
@@ -1,6 +1,6 @@
 {
 	"name": "leo-connector-postgres",
-	"version": "4.0.8-beta",
+	"version": "4.0.9-beta",
 	"description": "A Postgres database connector for use with Leo Platform",
 	"repository": {
 		"type": "git",
@@ -41,4 +41,4 @@
 			}
 		}
 	}
-}
+}
\ No newline at end of file

From 1ba3a4083e9cc772021a6ea6475f933619162dd2 Mon Sep 17 00:00:00 2001
From: Michael Scranton <mscranton@commercehub.com>
Date: Thu, 19 Jan 2023 13:35:24 -0500
Subject: [PATCH 33/62] etl fixes and version bump

---
 postgres/lib/dwconnect.js | 61 ++++++++++++++++++++++-----------------
 postgres/package.json     |  2 +-
 2 files changed, 35 insertions(+), 28 deletions(-)

diff --git a/postgres/lib/dwconnect.js b/postgres/lib/dwconnect.js
index 8de5433f..993db0d9 100644
--- a/postgres/lib/dwconnect.js
+++ b/postgres/lib/dwconnect.js
@@ -3,9 +3,6 @@ const postgres = require('./connect.js');
 const async = require('async');
 const ls = require('leo-sdk').streams;
 const logger = require('leo-logger');
-const { basename } = require('path');
-const { doesNotThrow } = require('assert');
-const { table } = require('console');
 
 module.exports = function (config, columnConfig) {
 	let client = postgres(config);
@@ -244,9 +241,6 @@ module.exports = function (config, columnConfig) {
 											  FROM   ${qualifiedStagingTable};`, (err, results) => {
 								if (err) {
 									return done(err);
-								} else if (results[0].cnt === 0) {
-									totalRecords = results[0].cnt;
-									return done();
 								} else {
 									totalRecords = results[0].cnt;
 									naturalKeyLowerBound = results[0].minid;
@@ -488,7 +482,6 @@ module.exports = function (config, columnConfig) {
 
 							// The following code relies on the fact that now()/sysdate will return the same time during all transaction events
 							tasks.push(done => connection.query(`Begin Transaction`, done));
-
 							tasks.push(done => {
 								connection.query(`INSERT INTO ${qualifiedTable} (${fields.join(',')})
 												  SELECT row_number() over() + ${rowId}, ${allColumns.map(column => `coalesce(staging.${column}, prev.${column})`)}, ${dwClient.auditdate} as ${columnConfig._auditdate}, 
@@ -517,6 +510,22 @@ module.exports = function (config, columnConfig) {
 												  and (changes.runSCD1=1 OR  changes.runSCD6=1 OR changes.runSCD2=1)
 											`, done);
 							});
+
+							async.series(tasks, err => {
+								if (!err) {
+									connection.query(`commit`, e => {
+										connection.release();
+										callback(e || err, {
+											count: totalRecords,
+										});
+									});
+								} else {
+									connection.query(`rollback`, (e, d) => {
+										connection.release();
+										callback(e, d);
+									});
+								}
+							});
 						});
 					} else {
 						let naturalKeyLowerBound;
@@ -529,9 +538,6 @@ module.exports = function (config, columnConfig) {
 											  FROM   ${qualifiedStagingTable};`, (err, results) => {
 								if (err) {
 									return done(err);
-								} else if (results[0].cnt === 0) {
-									totalRecords = results[0].cnt;
-									return done();
 								} else {
 									totalRecords = results[0].cnt;
 									naturalKeyLowerBound = results[0].minid;
@@ -595,22 +601,23 @@ module.exports = function (config, columnConfig) {
 											  SELECT ${fields.map(column => `${column}`).join(`, `)}
 											  FROM   ${qualifiedStagingTable}; `, done);
 						});
-					};
-					async.series(tasks, err => {
-						if (!err) {
-							connection.query(`commit`, e => {
-								connection.release();
-								callback(e || err, {
-									count: totalRecords,
+
+						async.series(tasks, err => {
+							if (!err) {
+								connection.query(`commit`, e => {
+									connection.release();
+									callback(e || err, {
+										count: totalRecords,
+									});
 								});
-							});
-						} else {
-							connection.query(`rollback`, (e, d) => {
-								connection.release();
-								callback(e, d);
-							});
-						}
-					});
+							} else {
+								connection.query(`rollback`, (e, d) => {
+									connection.release();
+									callback(e, d);
+								});
+							}
+						});
+					};
 				});
 			}).catch(callback);
 		}).catch(callback);
@@ -746,12 +753,12 @@ module.exports = function (config, columnConfig) {
 							var joinOn = `${link.join_id}_join_table.${link.on} = t.${link.source} `;
 
 							if (Array.isArray(link.source)) {
-								joinOn = link.source.map((v, i) => `${link.join_id}_join_table.${link.on[i]} = t.${v} `).join(' AND ');
+								joinOn = link.source.map((v, i) => `${link.join_id}_join_table.${link.on[i]} = t.${v}`).join(' AND ');
 							}
 							return `LEFT JOIN ${link.table} ${link.join_id}_join_table
 									ON ${joinOn} 
 									AND t.${link.link_date} >= ${link.join_id}_join_table.${columnConfig._startdate}
-									AND(t.${link.link_date} <= ${link.join_id}_join_table.${columnConfig._enddate} or ${link.join_id}_join_table.${columnConfig._current})`;
+									AND (t.${link.link_date} <= ${link.join_id}_join_table.${columnConfig._enddate} or ${link.join_id}_join_table.${columnConfig._current})`;
 						};
 					}
 				});
diff --git a/postgres/package.json b/postgres/package.json
index e3fa4149..69f33a04 100644
--- a/postgres/package.json
+++ b/postgres/package.json
@@ -1,6 +1,6 @@
 {
 	"name": "leo-connector-postgres",
-	"version": "4.0.9-beta",
+	"version": "4.0.10-beta",
 	"description": "A Postgres database connector for use with Leo Platform",
 	"repository": {
 		"type": "git",

From a71ce57cda2a163e2c809f68b28f3df8921ff940 Mon Sep 17 00:00:00 2001
From: Michael Scranton <mscranton@commercehub.com>
Date: Thu, 19 Jan 2023 14:00:42 -0500
Subject: [PATCH 34/62] reverting uneeded changes

---
 postgres/lib/dwconnect.js | 7 +++----
 1 file changed, 3 insertions(+), 4 deletions(-)

diff --git a/postgres/lib/dwconnect.js b/postgres/lib/dwconnect.js
index 993db0d9..46c246ac 100644
--- a/postgres/lib/dwconnect.js
+++ b/postgres/lib/dwconnect.js
@@ -278,7 +278,7 @@ module.exports = function (config, columnConfig) {
 															  FROM   ${qualifiedStagingTable} AS staging
 															  WHERE  ${ids.map(id => `base.${id} = staging.${id}`).join(` AND `)}
 															  		 ${(naturalKeyFilter !== undefined) ? `AND staging.${(sortKey !== undefined) ? sortKey : ids[0]} >= ${naturalKeyFilter}` : ``})
-													 ${(naturalKeyFilter !== undefined) ? `AND base.${(sortKey !== undefined) ? sortKey : nk[0]} >= ${naturalKeyFilter}` : ``};`, done);
+													 ${(naturalKeyFilter !== undefined) ? `AND base.${(sortKey !== undefined) ? sortKey : ids[0]} >= ${naturalKeyFilter}` : ``};`, done);
 						});
 
 						// Merge exiting data into staged copy
@@ -456,9 +456,8 @@ module.exports = function (config, columnConfig) {
 					if (!config.hashedSurrogateKeys && !config.bypassSlowlyChangingDimensions) {
 						tempTables.push(`${qualifiedStagingTable}_changes`);
 						connection.query(`create table ${qualifiedStagingTable}_changes as
-										  select ${nk.map(id => `s.${id}`).join(', ')},
-												 d.${nk[0]} is null as isNew
-										  ${!config.bypassSlowlyChangingDimensions ? `,${scdSQL.join(',\n')}` : ``}
+										  select ${nk.map(id => `s.${id}`).join(', ')}, d.${nk[0]} is null as isNew,
+												 ${scdSQL.join(',\n')}
 										  FROM ${qualifiedStagingTable} s
 										  LEFT JOIN ${qualifiedTable} d on ${nk.map(id => `d.${id} = s.${id}`).join(' and ')} and d.${columnConfig._current}`, (err) => {
 							if (err) {

From d733d75d4bcd14ff1b1e8d45b070b4d9eee86675 Mon Sep 17 00:00:00 2001
From: Michael Scranton <mscranton@commercehub.com>
Date: Tue, 31 Jan 2023 10:50:40 -0500
Subject: [PATCH 35/62] sortkey type handling and d_time fix

---
 postgres/lib/dwconnect.js | 55 +++++++++++++++++++++++++++++----------
 postgres/package.json     |  4 +--
 2 files changed, 43 insertions(+), 16 deletions(-)

diff --git a/postgres/lib/dwconnect.js b/postgres/lib/dwconnect.js
index 46c246ac..1925ef01 100644
--- a/postgres/lib/dwconnect.js
+++ b/postgres/lib/dwconnect.js
@@ -132,6 +132,7 @@ module.exports = function (config, columnConfig) {
 		let deleteHandler = deletesSetup(qualifiedTable, schema[qualifiedTable], columnConfig._deleted, true);
 
 		let sortKey;
+		let sortKeyType;
 
 		tempTables.push(qualifiedStagingTable);
 		tasks.push(done => client.query(`drop table if exists ${qualifiedStagingTable}`, done));
@@ -141,14 +142,16 @@ module.exports = function (config, columnConfig) {
 		} else {
 			// get sortkey for joins
 			tasks.push(done => {
-				client.query(`SELECT sortkey
-					 		  FROM   public.mv_dist_sort_key
+				client.query(`SELECT sortkey,
+								     sortkeytype
+					 		  FROM   public.v_dist_sort_key
 					 		  WHERE  table_name = '${table}';`, (err, results) => {
 					if (err) {
 						return done(err);
 					} else {
 						if (results[0].sortKey !== null) {
 							sortKey = results[0].sortkey;
+							sortKeyType = results[0].sortkeytype;
 						};
 						done();
 					};
@@ -245,7 +248,13 @@ module.exports = function (config, columnConfig) {
 									totalRecords = results[0].cnt;
 									naturalKeyLowerBound = results[0].minid;
 									if (naturalKeyLowerBound !== null) {
-										naturalKeyFilter = (typeof naturalKeyLowerBound === 'string') ? `'${results[0].minid}'` : `${results[0].minid}`
+										if (sortKeyType === 'int4' || sortKeyType === 'int8') {
+											naturalKeyFilter = `${results[0].minid}`;
+										} else if (sortKeyType === 'varchar') {
+											naturalKeyFilter = `'${results[0].minid}'`;
+										} else if (sortKeyType === 'timestamp' && Date.parse(results[0].minid.split('  ')[1])) {
+											naturalKeyFilter = `'${results[0].minid.split('  ')[1]}'`;
+										};
 									};
 									done();
 								};
@@ -342,6 +351,7 @@ module.exports = function (config, columnConfig) {
 		let deleteHandler = deletesSetup(qualifiedTable, schema[qualifiedTable], columnConfig._enddate, dwClient.auditdate, `${columnConfig._current} = true`);
 
 		let sortKey;
+		let sortKeyType;
 
 		// Prepare staging tables
 		tempTables.push(qualifiedStagingTable);
@@ -353,14 +363,16 @@ module.exports = function (config, columnConfig) {
 		} else {
 			// get sortkey for joins
 			tasks.push(done => {
-				client.query(`SELECT sortkey
-					  		  FROM   public.mv_dist_sort_key
+				client.query(`SELECT sortkey,
+									 sortkeytype
+					  		  FROM   public.v_dist_sort_key
 					  		  WHERE  table_name = '${table}';`, (err, results) => {
 					if (err) {
 						return done(err);
 					} else {
 						if (results[0].sortkey !== null) {
 							sortKey = results[0].sortkey;
+							sortKeyType = results[0].sortkeytype;
 						};
 						done();
 					};
@@ -541,7 +553,13 @@ module.exports = function (config, columnConfig) {
 									totalRecords = results[0].cnt;
 									naturalKeyLowerBound = results[0].minid;
 									if (naturalKeyLowerBound !== null) {
-										naturalKeyFilter = (typeof naturalKeyLowerBound === 'string') ? `'${results[0].minid}'` : `${results[0].minid}`;
+										if (sortKeyType === 'int4' || sortKeyType === 'int8') {
+											naturalKeyFilter = `${results[0].minid}`;
+										} else if (sortKeyType === 'varchar') {
+											naturalKeyFilter = `'${results[0].minid}'`;
+										} else if (sortKeyType === 'timestamp' && Date.parse(results[0].minid.split('  ')[1])) {
+											naturalKeyFilter = `'${results[0].minid.split('  ')[1]}'`;
+										};
 									};
 									done();
 								};
@@ -698,21 +716,24 @@ module.exports = function (config, columnConfig) {
 				tasks.push(done => client.query(`ANALYZE ${table}`, done));
 			}
 
-			const qualifiedStagingTable = `${columnConfig.stageSchema}.${columnConfig.stageTablePrefix}_${table} `;
+			const qualifiedStagingTable = `${columnConfig.stageSchema}.${columnConfig.stageTablePrefix}_${table}`;
 			let naturalKeyLowerBound;
 			let naturalKeyFilter;
 			let sortKey;
+			let sortKeyType;
 			// Get lower bound for natural key to avoid unnecessary scanning
 			if (config.version === 'redshift') {
 				tasks.push(done => {
-					client.query(`SELECT sortkey
-								  FROM   public.mv_dist_sort_key
-								  WHERE  table_name = '${table}'; `, (err, results) => {
+					client.query(`SELECT sortkey,
+										 sortkeytype
+								  FROM   public.v_dist_sort_key
+								  WHERE  table_name = '${table}';`, (err, results) => {
 						if (err) {
 							return done(err);
 						} else {
 							if (results[0].sortKey !== null) {
 								sortKey = results[0].sortkey;
+								sortKeyType = results[0].sortkeytype;
 							};
 							done();
 						};
@@ -726,7 +747,13 @@ module.exports = function (config, columnConfig) {
 						} else {
 							naturalKeyLowerBound = results[0].minid;
 							if (naturalKeyLowerBound !== null) {
-								naturalKeyFilter = (typeof naturalKeyLowerBound === 'string') ? `'${results[0].minid}'` : `${results[0].minid} `
+								if (sortKeyType === 'int4' || sortKeyType === 'int8') {
+									naturalKeyFilter = `${results[0].minid}`;
+								} else if (sortKeyType === 'varchar') {
+									naturalKeyFilter = `'${results[0].minid}'`;
+								} else if (sortKeyType === 'timestamp' && Date.parse(results[0].minid.split('  ')[1])) {
+									naturalKeyFilter = `'${results[0].minid.split('  ')[1]}'`;
+								};
 							};
 							done();
 						};
@@ -738,11 +765,11 @@ module.exports = function (config, columnConfig) {
 				let joinTables = links.map(link => {
 					if (columnConfig.useSurrogateDateKeys && (link.table === 'd_datetime' || link.table === 'datetime' || link.table === 'dim_datetime')) {
 						sets.push(`${link.destination}_date = coalesce(t.${link.source}::date - '1400-01-01'::date + 10000, 1)`);
-						sets.push(`${link.destination}_time = coalesce(EXTRACT(EPOCH from t.${link.source}::time) + 10000, 1)`);
+						sets.push(`${link.destination}_time = coalesce(EXTRACT(EPOCH FROM ${(config.version !== 'redshift') ? `` : `'1970-01-01'::date +`} t.${link.source}::time) + 10000, 1)`);
 					} else if (columnConfig.useSurrogateDateKeys && (link.table === 'd_date' || link.table === 'date' || link.table === 'dim_date')) {
 						sets.push(`${link.destination}_date = coalesce(t.${link.source}::date - '1400-01-01'::date + 10000, 1)`);
 					} else if (columnConfig.useSurrogateDateKeys && (link.table === 'd_time' || link.table === 'time' || link.table === 'dim_time')) {
-						sets.push(`${link.destination}_time = coalesce(EXTRACT(EPOCH from t.${link.source}::time) + 10000, 1)`);
+						sets.push(`${link.destination}_time = coalesce(EXTRACT(EPOCH FROM ${(config.version !== 'redshift') ? `` : `'1970-01-01'::date +`} t.${link.source}::time) + 10000, 1)`);
 					} else {
 						if (config.hashedSurrogateKeys) {
 							sets.push(`${link.destination} = coalesce(farmFingerPrint64(t.${link.source}), 1)`);
@@ -769,7 +796,7 @@ module.exports = function (config, columnConfig) {
 								  ${config.hashedSurrogateKeys ? '' : joinTables.join('\n')}
                         		  WHERE ${nk.map(id => `dm.${id} = t.${id}`).join(' AND ')}
 								  AND dm.${columnConfig._auditdate} = ${dwClient.auditdate} AND t.${columnConfig._auditdate} = ${dwClient.auditdate}
-								  ${(naturalKeyFilter !== undefined) ? `AND t.${(sortKey !== undefined) ? sortKey : nk[0]} >= ${naturalKeyFilter}` : ``} `, done);
+								  ${(naturalKeyFilter !== undefined) ? `AND t.${(sortKey !== undefined) ? sortKey : nk[0]} >= ${naturalKeyFilter}` : ``}`, done);
 				} else {
 					done();
 				}
diff --git a/postgres/package.json b/postgres/package.json
index 69f33a04..f34e77cf 100644
--- a/postgres/package.json
+++ b/postgres/package.json
@@ -1,6 +1,6 @@
 {
 	"name": "leo-connector-postgres",
-	"version": "4.0.10-beta",
+	"version": "4.0.11-beta",
 	"description": "A Postgres database connector for use with Leo Platform",
 	"repository": {
 		"type": "git",
@@ -41,4 +41,4 @@
 			}
 		}
 	}
-}
\ No newline at end of file
+}

From d41d75b9b015554ae40ba7cd109f9fe10b6b7c0c Mon Sep 17 00:00:00 2001
From: John Cronin <jcronin@ip-192-168-1-163.ec2.internal>
Date: Mon, 6 Feb 2023 11:39:39 -0500
Subject: [PATCH 36/62] change TIMEFORMAT to auto on copy command

---
 postgres/lib/connect.js | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/postgres/lib/connect.js b/postgres/lib/connect.js
index b32c2181..1b1d9a45 100644
--- a/postgres/lib/connect.js
+++ b/postgres/lib/connect.js
@@ -705,7 +705,7 @@ function create (pool, parentCache) {
 									let role = config.loaderRole;
 									myClient.query(
 										`copy ${table} (${f}) from '${file}' ${manifest} ${role ? `credentials 'aws_iam_role=${role}'` : ''
-										} NULL AS '\\\\N' format csv DELIMITER '|' ACCEPTINVCHARS TRUNCATECOLUMNS ACCEPTANYDATE TIMEFORMAT 'YYYY-MM-DD HH:MI:SS' COMPUPDATE OFF`,
+										} NULL AS '\\\\N' format csv DELIMITER '|' ACCEPTINVCHARS TRUNCATECOLUMNS ACCEPTANYDATE TIMEFORMAT 'auto' COMPUPDATE OFF`,
 										copyErr => {
 											if (keepS3Files) {
 												innerDone(copyErr);

From c3adc4b2f95e196b4712a875e0080c12479343e7 Mon Sep 17 00:00:00 2001
From: John Cronin <jcronin@ip-192-168-1-163.ec2.internal>
Date: Tue, 7 Feb 2023 11:08:16 -0500
Subject: [PATCH 37/62] bump leo-connector-postgres version to 4.0.12-beta

---
 postgres/package.json | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/postgres/package.json b/postgres/package.json
index f34e77cf..d2de037b 100644
--- a/postgres/package.json
+++ b/postgres/package.json
@@ -1,6 +1,6 @@
 {
 	"name": "leo-connector-postgres",
-	"version": "4.0.11-beta",
+	"version": "4.0.12-beta",
 	"description": "A Postgres database connector for use with Leo Platform",
 	"repository": {
 		"type": "git",

From c5e8fed6ae5ccd87ab88fafa4b52c6e36865947b Mon Sep 17 00:00:00 2001
From: Michael Scranton <mscranton@commercehub.com>
Date: Fri, 10 Feb 2023 11:52:22 -0500
Subject: [PATCH 38/62] fix for SCD columns

---
 postgres/lib/dwconnect.js | 8 ++++++++
 postgres/package.json     | 2 +-
 2 files changed, 9 insertions(+), 1 deletion(-)

diff --git a/postgres/lib/dwconnect.js b/postgres/lib/dwconnect.js
index 1925ef01..048baf0e 100644
--- a/postgres/lib/dwconnect.js
+++ b/postgres/lib/dwconnect.js
@@ -606,6 +606,14 @@ module.exports = function (config, columnConfig) {
 											  WHERE  staging.${sk} = prev.${sk}`, done);
 						});
 
+						// Set default SCD values
+						tasks.push(done => {
+							connection.query(`UPDATE ${qualifiedStagingTable}
+											  SET    ${columnConfig._startdate} = COALESCE(${columnConfig._startdate},'1900-01-01 00:00:00'),
+													 ${columnConfig._enddate} = COALESCE(${columnConfig._enddate},'9999-01-01 00:00:00'),
+													 ${columnConfig._current} = COALESCE(${columnConfig._current},true);`, done);
+						});
+
 						// Delete and reinsert data - avoids costly updates on large tables
 						tasks.push(done => {
 							connection.query(`DELETE FROM ${qualifiedTable}
diff --git a/postgres/package.json b/postgres/package.json
index d2de037b..fd9499e0 100644
--- a/postgres/package.json
+++ b/postgres/package.json
@@ -1,6 +1,6 @@
 {
 	"name": "leo-connector-postgres",
-	"version": "4.0.12-beta",
+	"version": "4.0.13-beta",
 	"description": "A Postgres database connector for use with Leo Platform",
 	"repository": {
 		"type": "git",

From b5e6cddfb8b09b304085c946b0510cd3b92316d3 Mon Sep 17 00:00:00 2001
From: Michael Scranton <mscranton@commercehub.com>
Date: Fri, 10 Feb 2023 11:53:35 -0500
Subject: [PATCH 39/62] missed the package-lock

---
 postgres/package-lock.json | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/postgres/package-lock.json b/postgres/package-lock.json
index e01fa818..3055b685 100644
--- a/postgres/package-lock.json
+++ b/postgres/package-lock.json
@@ -1,6 +1,6 @@
 {
 	"name": "leo-connector-postgres",
-	"version": "4.0.8-beta",
+	"version": "4.0.13-beta",
 	"lockfileVersion": 1,
 	"requires": true,
 	"dependencies": {

From e0a64f80501ea9310db60891b91cc7e42aa28321 Mon Sep 17 00:00:00 2001
From: Michael Scranton <mscranton@commercehub.com>
Date: Tue, 28 Feb 2023 13:31:53 -0500
Subject: [PATCH 40/62] Fix for _deleted on fact tables

---
 postgres/lib/dwconnect.js | 22 ++++++++++------------
 1 file changed, 10 insertions(+), 12 deletions(-)

diff --git a/postgres/lib/dwconnect.js b/postgres/lib/dwconnect.js
index 048baf0e..ff718823 100644
--- a/postgres/lib/dwconnect.js
+++ b/postgres/lib/dwconnect.js
@@ -233,7 +233,6 @@ module.exports = function (config, columnConfig) {
 																 WHERE  ${ids.map(id => `prev.${id} = staging.${id}`).join(' and ')})`, done);
 						});
 					} else {
-						columns = columns.concat([columnConfig._auditdate, columnConfig._deleted]);
 						let naturalKeyLowerBound;
 						let naturalKeyFilter;
 
@@ -272,16 +271,11 @@ module.exports = function (config, columnConfig) {
 												 LIMIT  0;`, done);
 						});
 
-						// Set audit date for staged data
-						tasks.push(done => {
-							connection.query(`UPDATE ${qualifiedStagingTable}
-											  SET ${columnConfig._auditdate} = ${dwClient.auditdate};`, done);
-						});
-
 						// Retreive copy of existing data
 						tasks.push(done => {
-							connection.query(`INSERT INTO ${qualifiedStagingTablePrevious} (${columns.map(column => `${column}`).join(`, `)})
-											  SELECT ${columns.map(column => `${column}`).join(`, `)}
+							connection.query(`INSERT INTO ${qualifiedStagingTablePrevious} (${columns.map(column => `${column}`).join(`, `)}, ${columnConfig._deleted})
+											  SELECT ${columns.map(column => `${column}`).join(`, `)},
+													 ${columnConfig._deleted}
 											  FROM   ${qualifiedTable} AS base
 											  WHERE  EXISTS ( SELECT *
 															  FROM   ${qualifiedStagingTable} AS staging
@@ -293,7 +287,9 @@ module.exports = function (config, columnConfig) {
 						// Merge exiting data into staged copy
 						tasks.push(done => {
 							connection.query(`UPDATE ${qualifiedStagingTable} AS staging
-											  SET    ${columns.map(column => `${column} = COALESCE(staging.${column}, prev.${column})`).join(`,`)}
+											  SET    ${columns.map(column => `${column} = COALESCE(staging.${column}, prev.${column})`).join(`,`)},
+											  		 ${columnConfig._auditdate} = ${dwClient.auditdate},
+													 ${columnConfig._deleted} = COALESCE(prev.${columnConfig._deleted}, false)
 											  FROM   ${qualifiedStagingTablePrevious} AS prev
 											  WHERE  ${ids.map(id => `prev.${id} = staging.${id}`).join(` AND `)}`, done);
 						});
@@ -306,8 +302,10 @@ module.exports = function (config, columnConfig) {
 											  		 ${(naturalKeyFilter !== undefined) ? `AND ${qualifiedTable}.${sortKey !== undefined ? sortKey : ids[0]} >= ${naturalKeyFilter}` : ``}; `, done);
 						});
 						tasks.push(done => {
-							connection.query(`INSERT INTO ${qualifiedTable} (${columns.map(column => `${column}`).join(`, `)})
-											  SELECT ${columns.map(column => `${column}`).join(`, `)}
+							connection.query(`INSERT INTO ${qualifiedTable} (${columns.map(column => `${column}`).join(`, `)}, ${columnConfig._auditdate}, ${columnConfig._deleted})
+											  SELECT ${columns.map(column => `${column}`).join(`, `)},
+											  		 ${columnConfig._auditdate},
+													 ${columnConfig._deleted}
 											  FROM   ${qualifiedStagingTable}; `, done);
 						});
 					};

From d44380288565ffb663a33dee16d16f61e1741f7f Mon Sep 17 00:00:00 2001
From: Michael Scranton <mscranton@commercehub.com>
Date: Wed, 1 Mar 2023 10:02:37 -0500
Subject: [PATCH 41/62] fact fix and newline fix

---
 postgres/lib/connect.js   | 8 ++++++--
 postgres/lib/dwconnect.js | 8 +++++++-
 2 files changed, 13 insertions(+), 3 deletions(-)

diff --git a/postgres/lib/connect.js b/postgres/lib/connect.js
index 1b1d9a45..7b22fc0b 100644
--- a/postgres/lib/connect.js
+++ b/postgres/lib/connect.js
@@ -641,8 +641,12 @@ function create (pool, parentCache) {
 			function nonNull(v) {
 				if (v === '' || v === null || v === undefined) {
 					return '\\N';
-				} else if (typeof v === 'string' && v.search(/\r/) !== -1) {
-					return v.replace(/\r\n?/g, '\n');
+				} else if (typeof v === 'string' && (v.search(/\r/) !== -1 || v.search(/\n/) !== -1)) {
+					if (config.version !== 'redshift') {
+						return v.replace(/\r\n?/g, '\n');
+					} else {
+						return v.replace(/\r\n?/g, '\n').slice(0, -1).replace(/\n/g, `\\n`) + v.slice(-1);
+					}
 				} else {
 					return v;
 				}
diff --git a/postgres/lib/dwconnect.js b/postgres/lib/dwconnect.js
index ff718823..5ee10bdd 100644
--- a/postgres/lib/dwconnect.js
+++ b/postgres/lib/dwconnect.js
@@ -288,12 +288,18 @@ module.exports = function (config, columnConfig) {
 						tasks.push(done => {
 							connection.query(`UPDATE ${qualifiedStagingTable} AS staging
 											  SET    ${columns.map(column => `${column} = COALESCE(staging.${column}, prev.${column})`).join(`,`)},
-											  		 ${columnConfig._auditdate} = ${dwClient.auditdate},
 													 ${columnConfig._deleted} = COALESCE(prev.${columnConfig._deleted}, false)
 											  FROM   ${qualifiedStagingTablePrevious} AS prev
 											  WHERE  ${ids.map(id => `prev.${id} = staging.${id}`).join(` AND `)}`, done);
 						});
 
+						// Set auditdate and _deleted for stage data
+						tasks.push(done => {
+							connection.query(`UPDATE ${qualifiedStagingTable}
+												SET    ${columnConfig._auditdate} = ${dwClient.auditdate},
+													   ${columnConfig._deleted} = COALESCE(${columnConfig._deleted}, false); `, done);
+						});
+
 						// Delete and reinsert data - avoids costly updates on large tables
 						tasks.push(done => {
 							connection.query(`DELETE FROM ${qualifiedTable}

From 6bf2587746c96a1e2313c72a96413c01f4e7d98d Mon Sep 17 00:00:00 2001
From: Michael Scranton <mscranton@commercehub.com>
Date: Wed, 1 Mar 2023 11:19:26 -0500
Subject: [PATCH 42/62] package version bump

---
 postgres/package.json | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/postgres/package.json b/postgres/package.json
index fd9499e0..890e5c49 100644
--- a/postgres/package.json
+++ b/postgres/package.json
@@ -1,6 +1,6 @@
 {
 	"name": "leo-connector-postgres",
-	"version": "4.0.13-beta",
+	"version": "4.0.14-beta",
 	"description": "A Postgres database connector for use with Leo Platform",
 	"repository": {
 		"type": "git",

From 2ea9d6978e02f8c64e78049bfd2085085ba844d4 Mon Sep 17 00:00:00 2001
From: Clint Zirker <clint.zirker@dsco.io>
Date: Fri, 3 Mar 2023 13:32:53 -0700
Subject: [PATCH 43/62] Fixing bug with related to posting errors to a
 different queue when loading into the DW and then having a processing failure

---
 common/datawarehouse/load.js | 13 +++++++------
 common/package.json          |  2 +-
 2 files changed, 8 insertions(+), 7 deletions(-)

diff --git a/common/datawarehouse/load.js b/common/datawarehouse/load.js
index 15692a30..04b06c64 100644
--- a/common/datawarehouse/load.js
+++ b/common/datawarehouse/load.js
@@ -9,7 +9,7 @@ const async = require('async');
 const validate = require('./../utils/validation');
 let errorStream;
 
-module.exports = function (ID, source, client, tableConfig, stream, callback) {
+module.exports = function(ID, source, client, tableConfig, stream, callback) {
 
 	// adding backwards compatibility for ID and source
 	if (!callback) {
@@ -39,7 +39,7 @@ module.exports = function (ID, source, client, tableConfig, stream, callback) {
 		});
 	});
 
-	let checkforDelete = ls.through(function (obj, done) {
+	let checkforDelete = ls.through(function(obj, done) {
 		if (obj.payload.type === 'delete') {
 			let data = obj.payload.data || {};
 			let ids = data.in || [];
@@ -68,7 +68,7 @@ module.exports = function (ID, source, client, tableConfig, stream, callback) {
 		}
 	});
 
-	let validateData = ls.through(function (obj, done) {
+	let validateData = ls.through(function(obj, done) {
 		let eventObj = obj.payload.data;
 		let invalid = false;
 
@@ -336,7 +336,7 @@ module.exports = function (ID, source, client, tableConfig, stream, callback) {
  * @param eventObj {Object}
  * @param error {string}
  */
-function handleFailedValidation (ID, source, eventObj, error) {
+function handleFailedValidation(ID, source, eventObj, error) {
 	if (!errorStream || !errorStream.Writable) {
 		errorStream = streams.passthrough({
 			objectMode: true
@@ -345,8 +345,9 @@ function handleFailedValidation (ID, source, eventObj, error) {
 		streams.pipe(
 			errorStream,
 
-			ls.process(ID, obj => {
-				return obj;
+			ls.process(ID, function(obj) {
+				this.push(obj, { partial: true });
+				done();
 			}),
 
 			leo.load(ID, `${source}_error`),
diff --git a/common/package.json b/common/package.json
index db34c29b..452ab9f2 100644
--- a/common/package.json
+++ b/common/package.json
@@ -1,6 +1,6 @@
 {
 	"name": "leo-connector-common",
-	"version": "4.0.6",
+	"version": "4.0.10-beta",
 	"description": "Common package for all Leo Platform database connectors",
 	"main": "index.js",
 	"directories": {},

From 8a747053cf95d4051edff64b90702cebcb7973f5 Mon Sep 17 00:00:00 2001
From: Clint Zirker <clint.zirker@dsco.io>
Date: Fri, 3 Mar 2023 14:50:15 -0700
Subject: [PATCH 44/62] fix bad checkpointing

---
 common/datawarehouse/load.js | 9 ++++-----
 common/package.json          | 2 +-
 2 files changed, 5 insertions(+), 6 deletions(-)

diff --git a/common/datawarehouse/load.js b/common/datawarehouse/load.js
index 04b06c64..340150ad 100644
--- a/common/datawarehouse/load.js
+++ b/common/datawarehouse/load.js
@@ -346,11 +346,10 @@ function handleFailedValidation(ID, source, eventObj, error) {
 			errorStream,
 
 			ls.process(ID, function(obj) {
-				this.push(obj, { partial: true });
-				done();
-			}),
-
-			leo.load(ID, `${source}_error`),
+				return obj;
+			}, `${source}_error`),
+			ls.toLeo(ID),
+			ls.devnull(),
 
 			(err) => {
 				err && logger.error('GOT ERROR', err);
diff --git a/common/package.json b/common/package.json
index 452ab9f2..a0230ca7 100644
--- a/common/package.json
+++ b/common/package.json
@@ -1,6 +1,6 @@
 {
 	"name": "leo-connector-common",
-	"version": "4.0.10-beta",
+	"version": "4.0.11-beta",
 	"description": "Common package for all Leo Platform database connectors",
 	"main": "index.js",
 	"directories": {},

From c3f9811e8083c54d22f2032d91f58099e5043ace Mon Sep 17 00:00:00 2001
From: Michael Scranton <mscranton@commercehub.com>
Date: Mon, 6 Mar 2023 11:41:24 -0500
Subject: [PATCH 45/62] Newline fix and version bump

---
 postgres/lib/connect.js    | 2 +-
 postgres/package-lock.json | 2 +-
 postgres/package.json      | 2 +-
 3 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/postgres/lib/connect.js b/postgres/lib/connect.js
index 7b22fc0b..206c908d 100644
--- a/postgres/lib/connect.js
+++ b/postgres/lib/connect.js
@@ -645,7 +645,7 @@ function create (pool, parentCache) {
 					if (config.version !== 'redshift') {
 						return v.replace(/\r\n?/g, '\n');
 					} else {
-						return v.replace(/\r\n?/g, '\n').slice(0, -1).replace(/\n/g, `\\n`) + v.slice(-1);
+						return v.replace(/\r\n?/g, '\n').replace(/\n/g, `\\n`);
 					}
 				} else {
 					return v;
diff --git a/postgres/package-lock.json b/postgres/package-lock.json
index 3055b685..1b9c08e6 100644
--- a/postgres/package-lock.json
+++ b/postgres/package-lock.json
@@ -1,6 +1,6 @@
 {
 	"name": "leo-connector-postgres",
-	"version": "4.0.13-beta",
+	"version": "4.0.15-beta",
 	"lockfileVersion": 1,
 	"requires": true,
 	"dependencies": {
diff --git a/postgres/package.json b/postgres/package.json
index 890e5c49..98a779ba 100644
--- a/postgres/package.json
+++ b/postgres/package.json
@@ -1,6 +1,6 @@
 {
 	"name": "leo-connector-postgres",
-	"version": "4.0.14-beta",
+	"version": "4.0.15-beta",
 	"description": "A Postgres database connector for use with Leo Platform",
 	"repository": {
 		"type": "git",

From 4cb98bf8326545ad83835cb9e0ee87bddbb32e29 Mon Sep 17 00:00:00 2001
From: Michael Scranton <mscranton@commercehub.com>
Date: Mon, 20 Mar 2023 11:20:36 -0400
Subject: [PATCH 46/62] Add bug fix from development

---
 postgres/lib/dwconnect.js | 2 +-
 postgres/package.json     | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/postgres/lib/dwconnect.js b/postgres/lib/dwconnect.js
index 5ee10bdd..743dcb2a 100644
--- a/postgres/lib/dwconnect.js
+++ b/postgres/lib/dwconnect.js
@@ -54,7 +54,7 @@ module.exports = function (config, columnConfig) {
 		function tryFlushDelete(done, force = false) {
 			if (force || toDeleteCount >= 1000) {
 				let deleteTasks = Object.keys(toDelete).map(col => {
-					return deleteDone => client.query(`update ${qualifiedTable} set ${field} = ${value} where ${col} in (${toDelete[col].join(',')}) ${where}`, deleteDone);
+					return deleteDone => client.query(`update ${qualifiedTable} set ${field} = ${value}, ${columnConfig._auditdate} = ${dwClient.auditdate} where ${col} in (${toDelete[col].join(',')}) ${where}`, deleteDone);
 				});
 				async.parallelLimit(deleteTasks, 1, (err) => {
 					if (!err) {
diff --git a/postgres/package.json b/postgres/package.json
index 98a779ba..bff10e75 100644
--- a/postgres/package.json
+++ b/postgres/package.json
@@ -1,6 +1,6 @@
 {
 	"name": "leo-connector-postgres",
-	"version": "4.0.15-beta",
+	"version": "4.0.16-beta",
 	"description": "A Postgres database connector for use with Leo Platform",
 	"repository": {
 		"type": "git",

From 87bd935bc80f3924815e657669d4d76c82689cf8 Mon Sep 17 00:00:00 2001
From: Michael Scranton <mscranton@commercehub.com>
Date: Mon, 20 Mar 2023 11:55:55 -0400
Subject: [PATCH 47/62] version bump

---
 postgres/package-lock.json | 2 +-
 postgres/package.json      | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/postgres/package-lock.json b/postgres/package-lock.json
index 1b9c08e6..6768801e 100644
--- a/postgres/package-lock.json
+++ b/postgres/package-lock.json
@@ -1,6 +1,6 @@
 {
 	"name": "leo-connector-postgres",
-	"version": "4.0.15-beta",
+	"version": "4.0.17-beta",
 	"lockfileVersion": 1,
 	"requires": true,
 	"dependencies": {
diff --git a/postgres/package.json b/postgres/package.json
index bff10e75..86eef07f 100644
--- a/postgres/package.json
+++ b/postgres/package.json
@@ -1,6 +1,6 @@
 {
 	"name": "leo-connector-postgres",
-	"version": "4.0.16-beta",
+	"version": "4.0.17-beta",
 	"description": "A Postgres database connector for use with Leo Platform",
 	"repository": {
 		"type": "git",

From 29091c00550968717b55af78c2507895e7433b37 Mon Sep 17 00:00:00 2001
From: Michael Scranton <mscranton@commercehub.com>
Date: Mon, 20 Mar 2023 12:27:52 -0400
Subject: [PATCH 48/62] Version bump again for leo-connector-common

---
 postgres/package-lock.json | 8 ++++----
 postgres/package.json      | 4 ++--
 2 files changed, 6 insertions(+), 6 deletions(-)

diff --git a/postgres/package-lock.json b/postgres/package-lock.json
index 6768801e..fe493d05 100644
--- a/postgres/package-lock.json
+++ b/postgres/package-lock.json
@@ -1,6 +1,6 @@
 {
 	"name": "leo-connector-postgres",
-	"version": "4.0.17-beta",
+	"version": "4.0.18-beta",
 	"lockfileVersion": 1,
 	"requires": true,
 	"dependencies": {
@@ -1052,9 +1052,9 @@
 			}
 		},
 		"leo-connector-common": {
-			"version": "4.0.6",
-			"resolved": "https://registry.npmjs.org/leo-connector-common/-/leo-connector-common-4.0.6.tgz",
-			"integrity": "sha512-PGvbJ08g6EQe7fAQ8AJyrdV6/WY0CAlBm2Ka/IYYE4mDFMpnLzTZdlf/q+QQSdRvXXPfZ3rL2sG6uHRygULjdA==",
+			"version": "4.0.11-beta",
+			"resolved": "https://registry.npmjs.org/leo-connector-common/-/leo-connector-common-4.0.11-beta.tgz",
+			"integrity": "sha512-LcBdN+4FcRXvJO0gY6vzSfNMPlNVXsr4yFLt0/Av180JK664L1eIvzaWyR8WsgiOiOx9+cgmlw0+yFNTK6s7Gg==",
 			"requires": {
 				"async": "2.6.3",
 				"leo-aws": "2.0.1",
diff --git a/postgres/package.json b/postgres/package.json
index 86eef07f..b7232fb0 100644
--- a/postgres/package.json
+++ b/postgres/package.json
@@ -1,6 +1,6 @@
 {
 	"name": "leo-connector-postgres",
-	"version": "4.0.17-beta",
+	"version": "4.0.18-beta",
 	"description": "A Postgres database connector for use with Leo Platform",
 	"repository": {
 		"type": "git",
@@ -19,7 +19,7 @@
 	"homepage": "https://github.com/LeoPlatform/connectors#readme",
 	"dependencies": {
 		"fast-csv": "2.4.1",
-		"leo-connector-common": "4.0.6",
+		"leo-connector-common": "4.0.11-beta",
 		"pg": "7.8.2",
 		"pg-copy-streams": "2.2.2",
 		"pg-format": "1.0.4"

From 96ed636c9cbaf3a95e203b8d242cb742f5f96952 Mon Sep 17 00:00:00 2001
From: Michael Scranton <mscranton@commercehub.com>
Date: Thu, 6 Apr 2023 12:28:10 -0400
Subject: [PATCH 49/62] update schema management for hash keys

---
 postgres/lib/dwconnect.js  | 8 ++++----
 postgres/package-lock.json | 2 +-
 postgres/package.json      | 2 +-
 3 files changed, 6 insertions(+), 6 deletions(-)

diff --git a/postgres/lib/dwconnect.js b/postgres/lib/dwconnect.js
index 743dcb2a..80d83b28 100644
--- a/postgres/lib/dwconnect.js
+++ b/postgres/lib/dwconnect.js
@@ -846,7 +846,7 @@ module.exports = function (config, columnConfig) {
 								} else if (field.dimension && !(columnConfig.dimColumnTransform(f, field) in fieldLookup)) {
 									let missingDim = columnConfig.dimColumnTransform(f, field);
 									missingFields[missingDim] = {
-										type: 'integer',
+										type: (config.hashedSurrogateKeys) ? 'bigint' : 'integer',
 									};
 								}
 							});
@@ -904,7 +904,7 @@ module.exports = function (config, columnConfig) {
 			if (field === 'sk') {
 				field = {
 					sk: true,
-					type: 'integer primary key',
+					type: `${(config.hashedSurrogateKeys) ? 'bigint' : 'integer'} primary key`,
 				};
 			} else if (typeof field === 'string') {
 				field = {
@@ -953,7 +953,7 @@ module.exports = function (config, columnConfig) {
 					});
 				}
 			} else if (field.dimension) {
-				fields.push(`${columnConfig.dimColumnTransform(key, field)} integer`);
+				fields.push(`${columnConfig.dimColumnTransform(key, field)} ${(config.hashedSurrogateKeys) ? 'bigint' : 'integer'}`);
 				defaults.push({
 					column: columnConfig.dimColumnTransform(key, field),
 					value: 1,
@@ -1033,7 +1033,7 @@ module.exports = function (config, columnConfig) {
 			let field = definition[key];
 			if (field === 'sk') {
 				field = {
-					type: 'integer primary key',
+					type: `${(config.hashedSurrogateKeys) ? 'bigint' : 'integer'} primary key`,
 				};
 			} else if (typeof field === 'string') {
 				field = {
diff --git a/postgres/package-lock.json b/postgres/package-lock.json
index fe493d05..bf874b5c 100644
--- a/postgres/package-lock.json
+++ b/postgres/package-lock.json
@@ -1,6 +1,6 @@
 {
 	"name": "leo-connector-postgres",
-	"version": "4.0.18-beta",
+	"version": "4.0.19-beta",
 	"lockfileVersion": 1,
 	"requires": true,
 	"dependencies": {
diff --git a/postgres/package.json b/postgres/package.json
index b7232fb0..c3231be6 100644
--- a/postgres/package.json
+++ b/postgres/package.json
@@ -1,6 +1,6 @@
 {
 	"name": "leo-connector-postgres",
-	"version": "4.0.18-beta",
+	"version": "4.0.19-beta",
 	"description": "A Postgres database connector for use with Leo Platform",
 	"repository": {
 		"type": "git",

From d4ce694beb4f142fb0516d574e185d9e49bf473b Mon Sep 17 00:00:00 2001
From: Clint Zirker <clint.zirker@dsco.io>
Date: Wed, 9 Aug 2023 14:37:21 -0600
Subject: [PATCH 50/62] Fixing postgress loader to pass along errors that
 occurred before the rollback

---
 postgres/lib/dwconnect.js | 70 +++++++++++++++++++++------------------
 1 file changed, 38 insertions(+), 32 deletions(-)

diff --git a/postgres/lib/dwconnect.js b/postgres/lib/dwconnect.js
index 80d83b28..34578739 100644
--- a/postgres/lib/dwconnect.js
+++ b/postgres/lib/dwconnect.js
@@ -4,7 +4,7 @@ const async = require('async');
 const ls = require('leo-sdk').streams;
 const logger = require('leo-logger');
 
-module.exports = function (config, columnConfig) {
+module.exports = function(config, columnConfig) {
 	let client = postgres(config);
 	let dwClient = client;
 	let tempTables = [];
@@ -69,7 +69,7 @@ module.exports = function (config, columnConfig) {
 		}
 
 		return {
-			add: function (obj, done) {
+			add: function(obj, done) {
 				let field = obj.__leo_delete__;
 				let id = obj.__leo_delete_id__;
 				if (id !== undefined && colLookup[field]) {
@@ -96,26 +96,32 @@ module.exports = function (config, columnConfig) {
 		if (tempTables.length) {
 			let tasks = [];
 
-			tempTables.forEach(table => {
-				tasks.push(done => client.query(`drop table ${table}`, done));
-			});
+			try {
+				tempTables.forEach(table => {
+					tasks.push(done => client.query(`drop table if exists ${table}`, done));
+				});
 
-			return new Promise(resolve => {
-				async.series(tasks, err => {
-					if (err) {
-						throw err;
-					} else {
-						tempTables = [];
-						return resolve('Cleaned up temp tables');
-					}
+				await new Promise((resolve, reject) => {
+					async.series(tasks, err => {
+						if (err) {
+							reject(err);
+						} else {
+							tempTables = [];
+							resolve('Cleaned up temp tables');
+						}
+					});
 				});
-			});
+			} catch (dropError) {
+				// Temp tables are cleaned up before use in the next invocation
+				// so just log it and move on
+				logger.info('Error dropping temp tables', dropError);
+			}
 		}
 
 		return true;
 	};
 
-	client.importFact = function (stream, table, ids, callback) {
+	client.importFact = function(stream, table, ids, callback) {
 		const stagingTable = `${columnConfig.stageTablePrefix}_${table}`;
 		const qualifiedStagingTable = `${columnConfig.stageSchema}.${stagingTable}`;
 		const qualifiedTable = `public.${table}`;
@@ -327,7 +333,7 @@ module.exports = function (config, columnConfig) {
 						} else {
 							connection.query(`rollback`, (e, d) => {
 								connection.release();
-								callback(e, d);
+								callback(e || err, d);
 							});
 						}
 					});
@@ -336,7 +342,7 @@ module.exports = function (config, columnConfig) {
 		}).catch(callback);
 	};
 
-	client.importDimension = function (stream, table, sk, nk, scds, callback, tableDef = {}) {
+	client.importDimension = function(stream, table, sk, nk, scds, callback, tableDef = {}) {
 		const stagingTbl = `${columnConfig.stageTablePrefix}_${table}`;
 		const qualifiedStagingTable = `${columnConfig.stageSchema}.${stagingTbl}`;
 		const qualifiedTable = `public.${table}`;
@@ -537,7 +543,7 @@ module.exports = function (config, columnConfig) {
 								} else {
 									connection.query(`rollback`, (e, d) => {
 										connection.release();
-										callback(e, d);
+										callback(e || err, d);
 									});
 								}
 							});
@@ -642,7 +648,7 @@ module.exports = function (config, columnConfig) {
 							} else {
 								connection.query(`rollback`, (e, d) => {
 									connection.release();
-									callback(e, d);
+									callback(e || err, d);
 								});
 							}
 						});
@@ -652,7 +658,7 @@ module.exports = function (config, columnConfig) {
 		}).catch(callback);
 	};
 
-	client.insertMissingDimensions = function (usedTables, tableConfig, tableSks, tableNks, callback) {
+	client.insertMissingDimensions = function(usedTables, tableConfig, tableSks, tableNks, callback) {
 		if (config.hashedSurrogateKeys) {
 			callback(null);
 		} else {
@@ -715,7 +721,7 @@ module.exports = function (config, columnConfig) {
 		};
 	};
 
-	client.linkDimensions = function (table, links, nk, callback, tableStatus) {
+	client.linkDimensions = function(table, links, nk, callback, tableStatus) {
 		client.describeTable(table).then(() => {
 			let tasks = [];
 			let sets = [];
@@ -821,7 +827,7 @@ module.exports = function (config, columnConfig) {
 		});
 	};
 
-	client.changeTableStructure = async function (structures) {
+	client.changeTableStructure = async function(structures) {
 		let tasks = [];
 		let tableResults = {};
 
@@ -888,7 +894,7 @@ module.exports = function (config, columnConfig) {
 		});
 	};
 
-	client.createTable = async function (table, definition) {
+	client.createTable = async function(table, definition) {
 		let fields = [];
 		let defaults = [];
 		let dbType = (config.type || '').toLowerCase();
@@ -1026,7 +1032,7 @@ module.exports = function (config, columnConfig) {
 			});
 		});
 	};
-	client.updateTable = async function (table, definition) {
+	client.updateTable = async function(table, definition) {
 		let fields = [];
 		let queries = [];
 		Object.keys(definition).forEach(key => {
@@ -1081,7 +1087,7 @@ module.exports = function (config, columnConfig) {
 		});
 
 		return new Promise(resolve => {
-			async.eachSeries(sqls, function (sql, done) {
+			async.eachSeries(sqls, function(sql, done) {
 				client.query(sql, err => done(err));
 			}, err => {
 				if (err) {
@@ -1093,7 +1099,7 @@ module.exports = function (config, columnConfig) {
 		});
 	};
 
-	client.findAuditDate = function (table, callback) {
+	client.findAuditDate = function(table, callback) {
 		client.query(`select to_char(max(${columnConfig._auditdate}), 'YYYY-MM-DD HH24:MI:SS') as max FROM ${client.escapeId(table)}`, (err, auditdate) => {
 			if (err) {
 				callback(err);
@@ -1110,7 +1116,7 @@ module.exports = function (config, columnConfig) {
 		});
 	};
 
-	client.exportChanges = function (table, fields, remoteAuditdate, opts, callback) {
+	client.exportChanges = function(table, fields, remoteAuditdate, opts, callback) {
 		let auditdateCompare = remoteAuditdate.auditdate != null ? `${columnConfig._auditdate} >= ${client.escapeValue(remoteAuditdate.auditdate)}` : `${columnConfig._auditdate} is null`;
 		client.query(`select count(*) as count FROM ${client.escapeId(table)} WHERE ${auditdateCompare}`, (err, result) => {
 			if (err) {
@@ -1178,7 +1184,7 @@ module.exports = function (config, columnConfig) {
 		});
 	};
 
-	client.importChanges = function (file, table, fields, opts, callback) {
+	client.importChanges = function(file, table, fields, opts, callback) {
 		if (typeof opts === 'function') {
 			callback = opts;
 			opts = {};
@@ -1239,16 +1245,16 @@ module.exports = function (config, columnConfig) {
 				}
 			});
 		}
-		tasks.push(function (done) {
+		tasks.push(function(done) {
 			client.query(`insert into ${tableName} select * from ${qualifiedStagingTable}`, done);
 		});
-		tasks.push(function (done) {
+		tasks.push(function(done) {
 			client.query(`select count(*) from ${qualifiedStagingTable}`, (err, result) => {
 				loadCount = result && parseInt(result[0].count);
 				done(err);
 			});
 		});
-		tasks.push(function (done) {
+		tasks.push(function(done) {
 			client.query(`drop table if exists ${qualifiedStagingTable}`, done);
 		});
 		async.series(tasks, (err) => {
@@ -1257,4 +1263,4 @@ module.exports = function (config, columnConfig) {
 	};
 
 	return client;
-};
\ No newline at end of file
+};

From 26051fe96c5b1e963109da291fc400b963b91e8d Mon Sep 17 00:00:00 2001
From: Clint Zirker <clint.zirker@dsco.io>
Date: Wed, 9 Aug 2023 14:41:28 -0600
Subject: [PATCH 51/62] increment postgres package version

---
 postgres/package.json | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/postgres/package.json b/postgres/package.json
index c3231be6..0e965990 100644
--- a/postgres/package.json
+++ b/postgres/package.json
@@ -1,6 +1,6 @@
 {
 	"name": "leo-connector-postgres",
-	"version": "4.0.19-beta",
+	"version": "4.0.20-beta",
 	"description": "A Postgres database connector for use with Leo Platform",
 	"repository": {
 		"type": "git",

From caf9e7d24bbd89424e68978dd29f14c469e89a86 Mon Sep 17 00:00:00 2001
From: Clint Zirker <clint.zirker@dsco.io>
Date: Wed, 18 Oct 2023 10:03:19 -0600
Subject: [PATCH 52/62] RSTREAM-209 Fixing issue with sortKey when creating the
 stating table.  It now checks for null and undefined

---
 postgres/lib/dwconnect.js | 34 ++++++++++++++++++----------------
 1 file changed, 18 insertions(+), 16 deletions(-)

diff --git a/postgres/lib/dwconnect.js b/postgres/lib/dwconnect.js
index 34578739..35fec117 100644
--- a/postgres/lib/dwconnect.js
+++ b/postgres/lib/dwconnect.js
@@ -40,6 +40,8 @@ module.exports = function(config, columnConfig) {
 	client.getDimensionColumn = columnConfig.dimColumnTransform;
 	client.columnConfig = columnConfig;
 
+	let deleteFlushCount = config.deleteFlushCount || 1000;
+
 	function deletesSetup(qualifiedTable, schema, field, value, where = '') {
 		let colLookup = {};
 		schema.map(col => {
@@ -52,7 +54,7 @@ module.exports = function(config, columnConfig) {
 		}
 
 		function tryFlushDelete(done, force = false) {
-			if (force || toDeleteCount >= 1000) {
+			if (force || toDeleteCount >= deleteFlushCount) {
 				let deleteTasks = Object.keys(toDelete).map(col => {
 					return deleteDone => client.query(`update ${qualifiedTable} set ${field} = ${value}, ${columnConfig._auditdate} = ${dwClient.auditdate} where ${col} in (${toDelete[col].join(',')}) ${where}`, deleteDone);
 				});
@@ -155,7 +157,7 @@ module.exports = function(config, columnConfig) {
 					if (err) {
 						return done(err);
 					} else {
-						if (results[0].sortKey !== null) {
+						if (results[0].sortKey != null) {
 							sortKey = results[0].sortkey;
 							sortKeyType = results[0].sortkeytype;
 						};
@@ -166,7 +168,7 @@ module.exports = function(config, columnConfig) {
 			tasks.push(done =>
 				client.query(`CREATE TABLE ${qualifiedStagingTable} 
 							  DISTSTYLE ALL 
-							  SORTKEY (${sortKey !== undefined ? sortKey : ids[0]})
+							  SORTKEY (${sortKey != null ? sortKey : ids[0]})
 							  AS SELECT *
 								 FROM   ${qualifiedTable}
 								 LIMIT  0;`, done));
@@ -244,7 +246,7 @@ module.exports = function(config, columnConfig) {
 
 						// Get lower bound for natural key to avoid unnecessary scanning
 						tasks.push(done => {
-							connection.query(`SELECT MIN(${(sortKey !== undefined) ? sortKey : ids[0]}) AS minid,
+							connection.query(`SELECT MIN(${(sortKey != null) ? sortKey : ids[0]}) AS minid,
 													 CAST(COUNT(*) AS INT) AS cnt
 											  FROM   ${qualifiedStagingTable};`, (err, results) => {
 								if (err) {
@@ -286,8 +288,8 @@ module.exports = function(config, columnConfig) {
 											  WHERE  EXISTS ( SELECT *
 															  FROM   ${qualifiedStagingTable} AS staging
 															  WHERE  ${ids.map(id => `base.${id} = staging.${id}`).join(` AND `)}
-															  		 ${(naturalKeyFilter !== undefined) ? `AND staging.${(sortKey !== undefined) ? sortKey : ids[0]} >= ${naturalKeyFilter}` : ``})
-													 ${(naturalKeyFilter !== undefined) ? `AND base.${(sortKey !== undefined) ? sortKey : ids[0]} >= ${naturalKeyFilter}` : ``};`, done);
+															  		 ${(naturalKeyFilter !== undefined) ? `AND staging.${(sortKey != null) ? sortKey : ids[0]} >= ${naturalKeyFilter}` : ``})
+													 ${(naturalKeyFilter !== undefined) ? `AND base.${(sortKey != null) ? sortKey : ids[0]} >= ${naturalKeyFilter}` : ``};`, done);
 						});
 
 						// Merge exiting data into staged copy
@@ -311,7 +313,7 @@ module.exports = function(config, columnConfig) {
 							connection.query(`DELETE FROM ${qualifiedTable}
 											  USING  ${qualifiedStagingTable}
 											  WHERE  ${ids.map(id => `${qualifiedTable}.${id} = ${qualifiedStagingTable}.${id}`).join(` AND `)}
-											  		 ${(naturalKeyFilter !== undefined) ? `AND ${qualifiedTable}.${sortKey !== undefined ? sortKey : ids[0]} >= ${naturalKeyFilter}` : ``}; `, done);
+											  		 ${(naturalKeyFilter !== undefined) ? `AND ${qualifiedTable}.${sortKey != null ? sortKey : ids[0]} >= ${naturalKeyFilter}` : ``}; `, done);
 						});
 						tasks.push(done => {
 							connection.query(`INSERT INTO ${qualifiedTable} (${columns.map(column => `${column}`).join(`, `)}, ${columnConfig._auditdate}, ${columnConfig._deleted})
@@ -380,7 +382,7 @@ module.exports = function(config, columnConfig) {
 					if (err) {
 						return done(err);
 					} else {
-						if (results[0].sortkey !== null) {
+						if (results[0].sortkey != null) {
 							sortKey = results[0].sortkey;
 							sortKeyType = results[0].sortkeytype;
 						};
@@ -392,7 +394,7 @@ module.exports = function(config, columnConfig) {
 				// Create staging table with DISTSTYLE ALL to prevent cross talk
 				client.query(`CREATE TABLE ${qualifiedStagingTable} 
 							  DISTSTYLE ALL
-							  SORTKEY(${sortKey !== undefined ? sortKey : nk[0]})
+							  SORTKEY(${sortKey != null ? sortKey : nk[0]})
 							  AS SELECT *
 							  FROM ${qualifiedTable}
 								LIMIT 0;`, done));
@@ -554,7 +556,7 @@ module.exports = function(config, columnConfig) {
 
 						// Get lower bound for natural key to avoid unnecessary scanning
 						tasks.push(done => {
-							connection.query(`SELECT MIN(${(sortKey !== undefined) ? sortKey : nk[0]}) AS minid,
+							connection.query(`SELECT MIN(${(sortKey != null) ? sortKey : nk[0]}) AS minid,
 													 CAST(COUNT(*) AS INT) AS cnt
 											  FROM   ${qualifiedStagingTable};`, (err, results) => {
 								if (err) {
@@ -604,8 +606,8 @@ module.exports = function(config, columnConfig) {
 											  WHERE  EXISTS(SELECT *
 											  FROM   ${qualifiedStagingTable} AS staging
 															 WHERE  base.${sk} = staging.${sk}
-																	${(naturalKeyFilter !== undefined) ? `AND staging.${(sortKey !== undefined) ? sortKey : nk[0]} >= ${naturalKeyFilter}` : ``})
-													 ${(naturalKeyFilter !== undefined) ? `AND base.${(sortKey !== undefined) ? sortKey : nk[0]} >= ${naturalKeyFilter}` : ``}; `, done);
+																	${(naturalKeyFilter !== undefined) ? `AND staging.${(sortKey != null) ? sortKey : nk[0]} >= ${naturalKeyFilter}` : ``})
+													 ${(naturalKeyFilter !== undefined) ? `AND base.${(sortKey != null) ? sortKey : nk[0]} >= ${naturalKeyFilter}` : ``}; `, done);
 						});
 
 						// Merge exiting data into staged copy
@@ -629,7 +631,7 @@ module.exports = function(config, columnConfig) {
 							connection.query(`DELETE FROM ${qualifiedTable}
 											  USING  ${qualifiedStagingTable}
 											  WHERE  ${qualifiedTable}.${sk} = ${qualifiedStagingTable}.${sk}
-											  		 ${(naturalKeyFilter !== undefined) ? `AND ${qualifiedTable}.${sortKey !== undefined ? sortKey : nk[0]} >= ${naturalKeyFilter}` : ``}; `, done);
+											  		 ${(naturalKeyFilter !== undefined) ? `AND ${qualifiedTable}.${sortKey != null ? sortKey : nk[0]} >= ${naturalKeyFilter}` : ``}; `, done);
 						});
 						tasks.push(done => {
 							connection.query(`INSERT INTO ${qualifiedTable} (${fields.map(column => `${column}`).join(`, `)})
@@ -749,7 +751,7 @@ module.exports = function(config, columnConfig) {
 						if (err) {
 							return done(err);
 						} else {
-							if (results[0].sortKey !== null) {
+							if (results[0].sortKey != null) {
 								sortKey = results[0].sortkey;
 								sortKeyType = results[0].sortkeytype;
 							};
@@ -758,7 +760,7 @@ module.exports = function(config, columnConfig) {
 					});
 				});
 				tasks.push(done => {
-					client.query(`SELECT MIN(${(sortKey !== undefined) ? sortKey : nk[0]}) AS minid
+					client.query(`SELECT MIN(${(sortKey != null) ? sortKey : nk[0]}) AS minid
 							  	  FROM   ${qualifiedStagingTable}; `, (err, results) => {
 						if (err) {
 							return done(err);
@@ -814,7 +816,7 @@ module.exports = function(config, columnConfig) {
 								  ${config.hashedSurrogateKeys ? '' : joinTables.join('\n')}
                         		  WHERE ${nk.map(id => `dm.${id} = t.${id}`).join(' AND ')}
 								  AND dm.${columnConfig._auditdate} = ${dwClient.auditdate} AND t.${columnConfig._auditdate} = ${dwClient.auditdate}
-								  ${(naturalKeyFilter !== undefined) ? `AND t.${(sortKey !== undefined) ? sortKey : nk[0]} >= ${naturalKeyFilter}` : ``}`, done);
+								  ${(naturalKeyFilter !== undefined) ? `AND t.${(sortKey != null) ? sortKey : nk[0]} >= ${naturalKeyFilter}` : ``}`, done);
 				} else {
 					done();
 				}

From 11f7f62fdf423a306ab14000f47525059ee1213d Mon Sep 17 00:00:00 2001
From: Clint Zirker <clint.zirker@dsco.io>
Date: Wed, 29 Nov 2023 16:04:15 -0700
Subject: [PATCH 53/62] updated version to 4.0.21-beta

---
 postgres/package.json | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/postgres/package.json b/postgres/package.json
index 0e965990..32875fc7 100644
--- a/postgres/package.json
+++ b/postgres/package.json
@@ -1,6 +1,6 @@
 {
 	"name": "leo-connector-postgres",
-	"version": "4.0.20-beta",
+	"version": "4.0.21-beta",
 	"description": "A Postgres database connector for use with Leo Platform",
 	"repository": {
 		"type": "git",

From 79e81c497a98a7a1b2c2c7bc6b8453ac41d7db03 Mon Sep 17 00:00:00 2001
From: Grant Robinson <grant.robinson@rithum.com>
Date: Thu, 30 Jan 2025 17:57:08 -0700
Subject: [PATCH 54/62] ES-2352 - bumped version # again

---
 entity-table/package-lock.json | 2 +-
 entity-table/package.json      | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/entity-table/package-lock.json b/entity-table/package-lock.json
index bca9bea4..64e19be7 100644
--- a/entity-table/package-lock.json
+++ b/entity-table/package-lock.json
@@ -1,6 +1,6 @@
 {
 	"name": "leo-connector-entity-table",
-	"version": "3.0.21-rc",
+	"version": "3.0.22-rc",
 	"lockfileVersion": 1,
 	"requires": true,
 	"dependencies": {
diff --git a/entity-table/package.json b/entity-table/package.json
index 4f7a2bdf..5848098b 100644
--- a/entity-table/package.json
+++ b/entity-table/package.json
@@ -1,6 +1,6 @@
 {
 	"name": "leo-connector-entity-table",
-	"version": "3.0.21-rc",
+	"version": "3.0.22-rc",
 	"description": "A Leo entity table connector",
 	"repository": {
 		"type": "git",

From e5d9c963539bd73284de36089af251d03c5eec4b Mon Sep 17 00:00:00 2001
From: Grant Robinson <grant.robinson@rithum.com>
Date: Fri, 31 Jan 2025 11:40:15 -0700
Subject: [PATCH 55/62] ES-2516 - reset deleted flag on update or insert

---
 postgres/lib/dwconnect.js | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/postgres/lib/dwconnect.js b/postgres/lib/dwconnect.js
index dd3105c7..d31c9707 100644
--- a/postgres/lib/dwconnect.js
+++ b/postgres/lib/dwconnect.js
@@ -172,7 +172,7 @@ module.exports = function (config, columnConfig) {
 					});
 					tasks.push(done => {
 						connection.query(`Update ${qualifiedTable} prev
-								SET  ${columns.map(column => `${column} = coalesce(staging.${column}, prev.${column})`)}, ${columnConfig._deleted} = coalesce(prev.${columnConfig._deleted}, false), ${columnConfig._auditdate} = ${dwClient.auditdate}
+								SET  ${columns.map(column => `${column} = coalesce(staging.${column}, prev.${column})`)}, ${columnConfig._deleted} = false, ${columnConfig._auditdate} = ${dwClient.auditdate}
 								FROM ${qualifiedStagingTable} staging
 								where ${ids.map(id => `prev.${id} = staging.${id}`).join(' and ')}
 							`, done);
@@ -181,7 +181,7 @@ module.exports = function (config, columnConfig) {
 					// Now insert any we were missing
 					tasks.push(done => {
 						connection.query(`INSERT INTO ${qualifiedTable} (${columns.join(',')},${columnConfig._deleted},${columnConfig._auditdate})
-								SELECT ${columns.map(column => `coalesce(staging.${column}, prev.${column})`)}, coalesce(prev.${columnConfig._deleted}, false), ${dwClient.auditdate} as ${columnConfig._auditdate}
+								SELECT ${columns.map(column => `coalesce(staging.${column}, prev.${column})`)}, false, ${dwClient.auditdate} as ${columnConfig._auditdate}
 								FROM ${qualifiedStagingTable} staging
 								LEFT JOIN ${qualifiedTable} as prev on ${ids.map(id => `prev.${id} = staging.${id}`).join(' and ')}
 								WHERE prev.${ids[0]} is null	

From 833d65d3e048689264cf3beeab9ef693214b717c Mon Sep 17 00:00:00 2001
From: Grant Robinson <grant.robinson@rithum.com>
Date: Fri, 31 Jan 2025 11:46:57 -0700
Subject: [PATCH 56/62] ES-2516 - updating version tags

---
 common/package-lock.json   | 2 +-
 common/package.json        | 2 +-
 postgres/package-lock.json | 2 +-
 postgres/package.json      | 4 ++--
 4 files changed, 5 insertions(+), 5 deletions(-)

diff --git a/common/package-lock.json b/common/package-lock.json
index caff21bf..402c88ae 100644
--- a/common/package-lock.json
+++ b/common/package-lock.json
@@ -1,6 +1,6 @@
 {
 	"name": "leo-connector-common",
-	"version": "4.0.5",
+	"version": "4.0.12-rc",
 	"lockfileVersion": 1,
 	"requires": true,
 	"dependencies": {
diff --git a/common/package.json b/common/package.json
index a0230ca7..c0ba21f5 100644
--- a/common/package.json
+++ b/common/package.json
@@ -1,6 +1,6 @@
 {
 	"name": "leo-connector-common",
-	"version": "4.0.11-beta",
+	"version": "4.0.12-rc",
 	"description": "Common package for all Leo Platform database connectors",
 	"main": "index.js",
 	"directories": {},
diff --git a/postgres/package-lock.json b/postgres/package-lock.json
index bf874b5c..c1638448 100644
--- a/postgres/package-lock.json
+++ b/postgres/package-lock.json
@@ -1,6 +1,6 @@
 {
 	"name": "leo-connector-postgres",
-	"version": "4.0.19-beta",
+	"version": "4.0.22-rc",
 	"lockfileVersion": 1,
 	"requires": true,
 	"dependencies": {
diff --git a/postgres/package.json b/postgres/package.json
index 32875fc7..061d87fe 100644
--- a/postgres/package.json
+++ b/postgres/package.json
@@ -1,6 +1,6 @@
 {
 	"name": "leo-connector-postgres",
-	"version": "4.0.21-beta",
+	"version": "4.0.22-rc",
 	"description": "A Postgres database connector for use with Leo Platform",
 	"repository": {
 		"type": "git",
@@ -19,7 +19,7 @@
 	"homepage": "https://github.com/LeoPlatform/connectors#readme",
 	"dependencies": {
 		"fast-csv": "2.4.1",
-		"leo-connector-common": "4.0.11-beta",
+		"leo-connector-common": "4.0.12-rc",
 		"pg": "7.8.2",
 		"pg-copy-streams": "2.2.2",
 		"pg-format": "1.0.4"

From 6f3535ab2db1ac281863492d4288f9761af6c190 Mon Sep 17 00:00:00 2001
From: Grant Robinson <grant.robinson@rithum.com>
Date: Fri, 31 Jan 2025 13:21:09 -0700
Subject: [PATCH 57/62] ES-2516 - fix for new natural key loading

---
 postgres/lib/dwconnect.js | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/postgres/lib/dwconnect.js b/postgres/lib/dwconnect.js
index ae79b33d..44c484d1 100644
--- a/postgres/lib/dwconnect.js
+++ b/postgres/lib/dwconnect.js
@@ -292,7 +292,7 @@ module.exports = function(config, columnConfig) {
 													 ${(naturalKeyFilter !== undefined) ? `AND base.${(sortKey != null) ? sortKey : ids[0]} >= ${naturalKeyFilter}` : ``};`, done);
 						});
 
-						// Merge exiting data into staged copy
+						// Merge existing data into staged copy
 						tasks.push(done => {
 							connection.query(`UPDATE ${qualifiedStagingTable} AS staging
 											  SET    ${columns.map(column => `${column} = COALESCE(staging.${column}, prev.${column})`).join(`,`)},
@@ -319,7 +319,7 @@ module.exports = function(config, columnConfig) {
 							connection.query(`INSERT INTO ${qualifiedTable} (${columns.map(column => `${column}`).join(`, `)}, ${columnConfig._auditdate}, ${columnConfig._deleted})
 											  SELECT ${columns.map(column => `${column}`).join(`, `)},
 											  		 ${columnConfig._auditdate},
-													 false
+													 false as ${columnConfig._deleted}
 											  FROM   ${qualifiedStagingTable}; `, done);
 						});
 					}

From 516e252090dcb34e8aca2e48c23f4c2b73d08a00 Mon Sep 17 00:00:00 2001
From: Grant Robinson <grant.robinson@rithum.com>
Date: Fri, 31 Jan 2025 13:41:21 -0700
Subject: [PATCH 58/62] ES-2516 - ran NPM install

---
 postgres/package-lock.json | 225 ++++++++++++++++++++++++++++++++++---
 1 file changed, 207 insertions(+), 18 deletions(-)

diff --git a/postgres/package-lock.json b/postgres/package-lock.json
index 231b01fe..9f99cd3c 100644
--- a/postgres/package-lock.json
+++ b/postgres/package-lock.json
@@ -457,6 +457,12 @@
 				"esutils": "^2.0.2"
 			}
 		},
+		"duplexer": {
+			"version": "0.1.2",
+			"resolved": "https://registry.npmjs.org/duplexer/-/duplexer-0.1.2.tgz",
+			"integrity": "sha512-jtD6YG370ZCIi/9GTaJKQxWTZD045+4R4hTk/x1UyoqadyJ9x9CgSi1RlVDQF8U2sxLLSnFkCaMihqljHIWgMg==",
+			"dev": true
+		},
 		"duplexify": {
 			"version": "3.7.1",
 			"resolved": "https://registry.npmjs.org/duplexify/-/duplexify-3.7.1.tgz",
@@ -654,6 +660,27 @@
 			"integrity": "sha512-kVscqXk4OCp68SZ0dkgEKVi6/8ij300KBWTJq32P/dYeWTSwK41WyTxalN1eRmA5Z9UU/LX9D7FWSmV9SAYx6g==",
 			"dev": true
 		},
+		"event-stream": {
+			"version": "4.0.1",
+			"resolved": "https://registry.npmjs.org/event-stream/-/event-stream-4.0.1.tgz",
+			"integrity": "sha512-qACXdu/9VHPBzcyhdOWR5/IahhGMf0roTeZJfzz077GwylcDd90yOHLouhmv7GJ5XzPi6ekaQWd8AvPP2nOvpA==",
+			"dev": true,
+			"requires": {
+				"duplexer": "^0.1.1",
+				"from": "^0.1.7",
+				"map-stream": "0.0.7",
+				"pause-stream": "^0.0.11",
+				"split": "^1.0.1",
+				"stream-combiner": "^0.2.2",
+				"through": "^2.3.8"
+			}
+		},
+		"extend": {
+			"version": "3.0.2",
+			"resolved": "https://registry.npmjs.org/extend/-/extend-3.0.2.tgz",
+			"integrity": "sha512-fjquC59cD7CyW6urNXK0FBufkZcoiGG80wTuPujX590cB5Ttln20E2UB4S/WARVqhXffZl2LNgS+gQdPIIim/g==",
+			"dev": true
+		},
 		"extended": {
 			"version": "0.0.6",
 			"resolved": "https://registry.npmjs.org/extended/-/extended-0.0.6.tgz",
@@ -749,6 +776,7 @@
 			"version": "2.0.0",
 			"resolved": "https://registry.npmjs.org/flush-write-stream/-/flush-write-stream-2.0.0.tgz",
 			"integrity": "sha512-uXClqPxT4xW0lcdSBheb2ObVU+kuqUk3Jk64EwieirEXZx9XUrVwp/JuBfKAWaM4T5Td/VL7QLDWPXp/MvGm/g==",
+			"dev": true,
 			"requires": {
 				"inherits": "^2.0.3",
 				"readable-stream": "^3.1.1"
@@ -758,6 +786,7 @@
 					"version": "3.6.0",
 					"resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-3.6.0.tgz",
 					"integrity": "sha512-BViHy7LKeTz4oNnkcLJ+lVSL6vpiFeX6/d3oSH8zCW7UxP2onchk+vTGB143xuFjHS3deTgkKoXXymXqymiIdA==",
+					"dev": true,
 					"requires": {
 						"inherits": "^2.0.3",
 						"string_decoder": "^1.1.1",
@@ -769,7 +798,8 @@
 		"from": {
 			"version": "0.1.7",
 			"resolved": "https://registry.npmjs.org/from/-/from-0.1.7.tgz",
-			"integrity": "sha512-twe20eF1OxVxp/ML/kq2p1uc6KvFK/+vs8WjEbeKmV2He22MKm7YF2ANIt+EOqhJ5L3K/SuuPhk0hWQDjOM23g=="
+			"integrity": "sha512-twe20eF1OxVxp/ML/kq2p1uc6KvFK/+vs8WjEbeKmV2He22MKm7YF2ANIt+EOqhJ5L3K/SuuPhk0hWQDjOM23g==",
+			"dev": true
 		},
 		"fs.realpath": {
 			"version": "1.0.0",
@@ -873,6 +903,12 @@
 			"resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.4.tgz",
 			"integrity": "sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ=="
 		},
+		"ini": {
+			"version": "1.3.8",
+			"resolved": "https://registry.npmjs.org/ini/-/ini-1.3.8.tgz",
+			"integrity": "sha512-JV/yugV2uzW5iMRSiZAyDtQd+nxtUnjeLt0acNdw98kKLrvuRVyB80tsREOE7yvGVgalhZ6RNXCmEHkUKBKxew==",
+			"dev": true
+		},
 		"inquirer": {
 			"version": "6.5.2",
 			"resolved": "https://registry.npmjs.org/inquirer/-/inquirer-6.5.2.tgz",
@@ -992,6 +1028,23 @@
 				"leo-aws": "2.0.1",
 				"leo-config": "1.1.0",
 				"netmask": "^1.0.6"
+			},
+			"dependencies": {
+				"leo-aws": {
+					"version": "2.0.1",
+					"resolved": "https://registry.npmjs.org/leo-aws/-/leo-aws-2.0.1.tgz",
+					"integrity": "sha512-40vq8BQklV96sZvN7fDRRmpKmY7p1CyZLIh1G4IkkokScw2QrqkhxOSs5H+A6xb7HBnt+IUvpgOXhwYe0+uOcQ==",
+					"dev": true,
+					"requires": {
+						"async": "^2.6.1",
+						"backoff": "^2.5.0",
+						"leo-config": "1.1.0",
+						"leo-logger": "1.0.1",
+						"leo-streams": "2.0.0",
+						"lodash": "^4.17.14",
+						"lodash.merge": "^4.6.2"
+					}
+				}
 			}
 		},
 		"leo-aws": {
@@ -1023,9 +1076,9 @@
 			}
 		},
 		"leo-connector-common": {
-			"version": "4.0.11-beta",
-			"resolved": "https://registry.npmjs.org/leo-connector-common/-/leo-connector-common-4.0.11-beta.tgz",
-			"integrity": "sha512-LcBdN+4FcRXvJO0gY6vzSfNMPlNVXsr4yFLt0/Av180JK664L1eIvzaWyR8WsgiOiOx9+cgmlw0+yFNTK6s7Gg==",
+			"version": "4.0.12-rc",
+			"resolved": "https://registry.npmjs.org/leo-connector-common/-/leo-connector-common-4.0.12-rc.tgz",
+			"integrity": "sha512-JO19U6uEaUUTViNUCSF63lcyOiVQPZxgWL+gRTzbbWyAXXeJLf222RlnSOaiXCTukTLJjd/SqIXl/0fzjA6lTA==",
 			"requires": {
 				"async": "2.6.3",
 				"leo-aws": "^2.0.3",
@@ -1046,6 +1099,23 @@
 				"js-beautify": "1.10.2",
 				"leo-aws": "2.0.1",
 				"leo-logger": "1.0.1"
+			},
+			"dependencies": {
+				"leo-aws": {
+					"version": "2.0.1",
+					"resolved": "https://registry.npmjs.org/leo-aws/-/leo-aws-2.0.1.tgz",
+					"integrity": "sha512-40vq8BQklV96sZvN7fDRRmpKmY7p1CyZLIh1G4IkkokScw2QrqkhxOSs5H+A6xb7HBnt+IUvpgOXhwYe0+uOcQ==",
+					"dev": true,
+					"requires": {
+						"async": "^2.6.1",
+						"backoff": "^2.5.0",
+						"leo-config": "1.1.0",
+						"leo-logger": "1.0.1",
+						"leo-streams": "2.0.0",
+						"lodash": "^4.17.14",
+						"lodash.merge": "^4.6.2"
+					}
+				}
 			}
 		},
 		"leo-logger": {
@@ -1056,6 +1126,113 @@
 				"lodash": "^4.17.10"
 			}
 		},
+		"leo-sdk": {
+			"version": "5.0.16",
+			"resolved": "https://registry.npmjs.org/leo-sdk/-/leo-sdk-5.0.16.tgz",
+			"integrity": "sha512-i7eREO8v18fdBl7WJlIW4RVYstcQCPbiKG5qsuqRxDEvvAy/UmB4A73qVSbcvxSQaXiPq2QbiqaFPjHKJwrw6Q==",
+			"dev": true,
+			"requires": {
+				"async": "2.6.2",
+				"backoff": "2.5.0",
+				"event-stream": "4.0.1",
+				"extend": "3.0.2",
+				"fast-csv": "2.5.0",
+				"flush-write-stream": "2.0.0",
+				"ini": "1.3.5",
+				"leo-aws": "2.0.1",
+				"leo-config": "1.1.0",
+				"leo-logger": "1.0.1",
+				"leo-streams": "2.0.0",
+				"lodash": "4.17.19",
+				"moment": "2.24.0",
+				"pump": "3.0.0",
+				"pumpify": "1.5.1",
+				"readable-stream": "3.4.0",
+				"split2": "3.1.1",
+				"through2": "3.0.1",
+				"uuid": "3.3.2"
+			},
+			"dependencies": {
+				"async": {
+					"version": "2.6.2",
+					"resolved": "https://registry.npmjs.org/async/-/async-2.6.2.tgz",
+					"integrity": "sha512-H1qVYh1MYhEEFLsP97cVKqCGo7KfCyTt6uEWqsTBr9SO84oK9Uwbyd/yCW+6rKJLHksBNUVWZDAjfS+Ccx0Bbg==",
+					"dev": true,
+					"requires": {
+						"lodash": "^4.17.11"
+					}
+				},
+				"fast-csv": {
+					"version": "2.5.0",
+					"resolved": "https://registry.npmjs.org/fast-csv/-/fast-csv-2.5.0.tgz",
+					"integrity": "sha512-M/9ezLU9/uDwvDZTt9sNFJa0iLDUsbhYJwPtnE0D9MjeuB6DY9wRCyUPZta9iI6cSz5wBWGaUPL61QH8h92cNA==",
+					"dev": true,
+					"requires": {
+						"extended": "0.0.6",
+						"is-extended": "0.0.10",
+						"object-extended": "0.0.7",
+						"safer-buffer": "^2.1.2",
+						"string-extended": "0.0.8"
+					}
+				},
+				"ini": {
+					"version": "1.3.5",
+					"resolved": "https://registry.npmjs.org/ini/-/ini-1.3.5.tgz",
+					"integrity": "sha512-RZY5huIKCMRWDUqZlEi72f/lmXKMvuszcMBduliQ3nnWbx9X/ZBQO7DijMEYS9EhHBb2qacRUMtC7svLwe0lcw==",
+					"dev": true
+				},
+				"leo-aws": {
+					"version": "2.0.1",
+					"resolved": "https://registry.npmjs.org/leo-aws/-/leo-aws-2.0.1.tgz",
+					"integrity": "sha512-40vq8BQklV96sZvN7fDRRmpKmY7p1CyZLIh1G4IkkokScw2QrqkhxOSs5H+A6xb7HBnt+IUvpgOXhwYe0+uOcQ==",
+					"dev": true,
+					"requires": {
+						"async": "^2.6.1",
+						"backoff": "^2.5.0",
+						"leo-config": "1.1.0",
+						"leo-logger": "1.0.1",
+						"leo-streams": "2.0.0",
+						"lodash": "^4.17.14",
+						"lodash.merge": "^4.6.2"
+					}
+				},
+				"lodash": {
+					"version": "4.17.19",
+					"resolved": "https://registry.npmjs.org/lodash/-/lodash-4.17.19.tgz",
+					"integrity": "sha512-JNvd8XER9GQX0v2qJgsaN/mzFCNA5BRe/j8JN9d+tWyGLSodKQHKFicdwNYzWwI3wjRnaKPsGj1XkBjx/F96DQ==",
+					"dev": true
+				},
+				"readable-stream": {
+					"version": "3.4.0",
+					"resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-3.4.0.tgz",
+					"integrity": "sha512-jItXPLmrSR8jmTRmRWJXCnGJsfy85mB3Wd/uINMXA65yrnFo0cPClFIUWzo2najVNSl+mx7/4W8ttlLWJe99pQ==",
+					"dev": true,
+					"requires": {
+						"inherits": "^2.0.3",
+						"string_decoder": "^1.1.1",
+						"util-deprecate": "^1.0.1"
+					}
+				},
+				"split2": {
+					"version": "3.1.1",
+					"resolved": "https://registry.npmjs.org/split2/-/split2-3.1.1.tgz",
+					"integrity": "sha512-emNzr1s7ruq4N+1993yht631/JH+jaj0NYBosuKmLcq+JkGQ9MmTw1RB1fGaTCzUuseRIClrlSLHRNYGwWQ58Q==",
+					"dev": true,
+					"requires": {
+						"readable-stream": "^3.0.0"
+					}
+				},
+				"through2": {
+					"version": "3.0.1",
+					"resolved": "https://registry.npmjs.org/through2/-/through2-3.0.1.tgz",
+					"integrity": "sha512-M96dvTalPT3YbYLaKaCuwu+j06D/8Jfib0o/PxbVt6Amhv3dUAtW6rTV1jPgJSBG83I/e04Y6xkVdVhSRhi0ww==",
+					"dev": true,
+					"requires": {
+						"readable-stream": "2 || 3"
+					}
+				}
+			}
+		},
 		"leo-streams": {
 			"version": "2.0.0",
 			"resolved": "https://registry.npmjs.org/leo-streams/-/leo-streams-2.0.0.tgz",
@@ -1121,7 +1298,8 @@
 		"map-stream": {
 			"version": "0.0.7",
 			"resolved": "https://registry.npmjs.org/map-stream/-/map-stream-0.0.7.tgz",
-			"integrity": "sha512-C0X0KQmGm3N2ftbTGBhSyuydQ+vV1LC3f3zPvT3RXHXNZrvfPZcoXp/N5DOa8vedX/rTMm2CjTtivFg2STJMRQ=="
+			"integrity": "sha512-C0X0KQmGm3N2ftbTGBhSyuydQ+vV1LC3f3zPvT3RXHXNZrvfPZcoXp/N5DOa8vedX/rTMm2CjTtivFg2STJMRQ==",
+			"dev": true
 		},
 		"mimic-fn": {
 			"version": "1.2.0",
@@ -1442,6 +1620,7 @@
 			"version": "0.0.11",
 			"resolved": "https://registry.npmjs.org/pause-stream/-/pause-stream-0.0.11.tgz",
 			"integrity": "sha512-e3FBlXLmN/D1S+zHzanP4E/4Z60oFAa3O051qt1pxa7DEJWKAyil6upYVXCWadEnuoqa4Pkc9oUx9zsxYeRv8A==",
+			"dev": true,
 			"requires": {
 				"through": "~2.3"
 			}
@@ -1459,6 +1638,13 @@
 				"pg-protocol": "^1.6.0",
 				"pg-types": "^2.1.0",
 				"pgpass": "1.x"
+			},
+			"dependencies": {
+				"pg-connection-string": {
+					"version": "2.7.0",
+					"resolved": "https://registry.npmjs.org/pg-connection-string/-/pg-connection-string-2.7.0.tgz",
+					"integrity": "sha512-PI2W9mv53rXJQEOb8xNR8lH7Hr+EKa6oJa38zsK0S/ky2er16ios1wLKhZyxzD7jUReiWokc9WK5nxSnC7W1TA=="
+				}
 			}
 		},
 		"pg-cloudflare": {
@@ -1467,11 +1653,6 @@
 			"integrity": "sha512-xWPagP/4B6BgFO+EKz3JONXv3YDgvkbVrGw2mTo3D6tVDQRh1e7cqVGvyR3BE+eQgAvx1XhW/iEASj4/jCWl3Q==",
 			"optional": true
 		},
-		"pg-connection-string": {
-			"version": "0.1.3",
-			"resolved": "https://registry.npmjs.org/pg-connection-string/-/pg-connection-string-0.1.3.tgz",
-			"integrity": "sha512-i0NV/CrSkFTaiOQs9AGy3tq0dkSjtTd4d7DfsjeDVZAA4aIHInwfFEmriNYGGJUfZ5x6IAC/QddoUpUJjQAi0w=="
-		},
 		"pg-copy-streams": {
 			"version": "2.2.2",
 			"resolved": "https://registry.npmjs.org/pg-copy-streams/-/pg-copy-streams-2.2.2.tgz",
@@ -1685,12 +1866,8 @@
 		"safer-buffer": {
 			"version": "2.1.2",
 			"resolved": "https://registry.npmjs.org/safer-buffer/-/safer-buffer-2.1.2.tgz",
-			"integrity": "sha512-YZo3K82SD7Riyi0E1EQPojLz7kpepnSQI9IyPbHHg1XXXevb5dJI7tpyN2ADxGcQbHG7vcyRHk0cbwqcQriUtg=="
-		},
-		"semver": {
-			"version": "4.3.2",
-			"resolved": "https://registry.npmjs.org/semver/-/semver-4.3.2.tgz",
-			"integrity": "sha512-VyFUffiBx8hABJ9HYSTXLRwyZtdDHMzMtFmID1aiNAD2BZppBmJm0Hqw3p2jkgxP9BNt1pQ9RnC49P0EcXf6cA=="
+			"integrity": "sha512-YZo3K82SD7Riyi0E1EQPojLz7kpepnSQI9IyPbHHg1XXXevb5dJI7tpyN2ADxGcQbHG7vcyRHk0cbwqcQriUtg==",
+			"dev": true
 		},
 		"shebang-command": {
 			"version": "1.2.0",
@@ -1745,6 +1922,15 @@
 				"is-fullwidth-code-point": "^2.0.0"
 			}
 		},
+		"split": {
+			"version": "1.0.1",
+			"resolved": "https://registry.npmjs.org/split/-/split-1.0.1.tgz",
+			"integrity": "sha512-mTyOoPbrivtXnwnIxZRFYRrPNtEFKlpB2fvjSnCQUiAA6qAZzqwna5envK4uk6OIeP17CsdF3rSBGYVBsU0Tkg==",
+			"dev": true,
+			"requires": {
+				"through": "2"
+			}
+		},
 		"split2": {
 			"version": "2.2.0",
 			"resolved": "https://registry.npmjs.org/split2/-/split2-2.2.0.tgz",
@@ -1768,6 +1954,7 @@
 			"version": "0.2.2",
 			"resolved": "https://registry.npmjs.org/stream-combiner/-/stream-combiner-0.2.2.tgz",
 			"integrity": "sha512-6yHMqgLYDzQDcAkL+tjJDC5nSNuNIx0vZtRZeiPh7Saef7VHX9H5Ijn9l2VIol2zaNYlYEX6KyuT/237A58qEQ==",
+			"dev": true,
 			"requires": {
 				"duplexer": "~0.1.1",
 				"through": "~2.3.4"
@@ -1880,7 +2067,8 @@
 		"through": {
 			"version": "2.3.8",
 			"resolved": "https://registry.npmjs.org/through/-/through-2.3.8.tgz",
-			"integrity": "sha512-w89qg7PI8wAdvX60bMDP+bFoD5Dvhm9oLheFp5O4a2QF0cSBGsBX4qZmadPMvVqlLJBBci+WqGGOAPvcDeNSVg=="
+			"integrity": "sha512-w89qg7PI8wAdvX60bMDP+bFoD5Dvhm9oLheFp5O4a2QF0cSBGsBX4qZmadPMvVqlLJBBci+WqGGOAPvcDeNSVg==",
+			"dev": true
 		},
 		"through2": {
 			"version": "2.0.5",
@@ -1938,7 +2126,8 @@
 		"uuid": {
 			"version": "3.3.2",
 			"resolved": "https://registry.npmjs.org/uuid/-/uuid-3.3.2.tgz",
-			"integrity": "sha512-yXJmeNaw3DnnKAOKJE51sL/ZaYfWJRl1pK9dr19YFCu0ObS231AB1/LbqTKRAQ5kw8A90rA6fr4riOUpTZvQZA=="
+			"integrity": "sha512-yXJmeNaw3DnnKAOKJE51sL/ZaYfWJRl1pK9dr19YFCu0ObS231AB1/LbqTKRAQ5kw8A90rA6fr4riOUpTZvQZA==",
+			"dev": true
 		},
 		"which": {
 			"version": "1.3.1",

From f40c904439d8ab7258fb1d5bee773f5f5a281530 Mon Sep 17 00:00:00 2001
From: Grant Robinson <grant.robinson@rithum.com>
Date: Fri, 31 Jan 2025 16:35:54 -0700
Subject: [PATCH 59/62] ES-2516 - ensure we are setting _deleted to false when
 it is updated

---
 postgres/lib/dwconnect.js  | 4 +++-
 postgres/package-lock.json | 2 +-
 postgres/package.json      | 2 +-
 3 files changed, 5 insertions(+), 3 deletions(-)

diff --git a/postgres/lib/dwconnect.js b/postgres/lib/dwconnect.js
index 44c484d1..b0f65df1 100644
--- a/postgres/lib/dwconnect.js
+++ b/postgres/lib/dwconnect.js
@@ -27,6 +27,8 @@ module.exports = function(config, columnConfig) {
 		useSurrogateDateKeys: true,
 	}, columnConfig || {});
 
+	console.log(`has delete_fix in-place`);
+
 
 	// Control flow for both of these configurations set to true has not been added. An error will be thrown until that is supported.
 	if (
@@ -305,7 +307,7 @@ module.exports = function(config, columnConfig) {
 						tasks.push(done => {
 							connection.query(`UPDATE ${qualifiedStagingTable}
 												SET    ${columnConfig._auditdate} = ${dwClient.auditdate},
-													   ${columnConfig._deleted} = COALESCE(${columnConfig._deleted}, false); `, done);
+													   ${columnConfig._deleted} = false; `, done);
 						});
 
 						// Delete and reinsert data - avoids costly updates on large tables
diff --git a/postgres/package-lock.json b/postgres/package-lock.json
index 9f99cd3c..2e685d18 100644
--- a/postgres/package-lock.json
+++ b/postgres/package-lock.json
@@ -1,6 +1,6 @@
 {
 	"name": "leo-connector-postgres",
-	"version": "4.0.22-rc",
+	"version": "4.0.23-rc",
 	"lockfileVersion": 1,
 	"requires": true,
 	"dependencies": {
diff --git a/postgres/package.json b/postgres/package.json
index 39dd86e7..321a4576 100644
--- a/postgres/package.json
+++ b/postgres/package.json
@@ -1,6 +1,6 @@
 {
 	"name": "leo-connector-postgres",
-	"version": "4.0.22-rc",
+	"version": "4.0.23-rc",
 	"description": "A Postgres database connector for use with Leo Platform",
 	"repository": {
 		"type": "git",

From 90830af4caaf49b7da294e5411596064c84cb395 Mon Sep 17 00:00:00 2001
From: Grant Robinson <grant.robinson@rithum.com>
Date: Wed, 5 Feb 2025 18:25:12 -0600
Subject: [PATCH 60/62] ES-2516 - don't write deletes to the CSV file/staging
 table

---
 common/datawarehouse/combine.js |  5 ++++-
 postgres/lib/dwconnect.js       | 20 +++++++++++++++++---
 2 files changed, 21 insertions(+), 4 deletions(-)

diff --git a/common/datawarehouse/combine.js b/common/datawarehouse/combine.js
index 804ef6a9..40bb5ab8 100644
--- a/common/datawarehouse/combine.js
+++ b/common/datawarehouse/combine.js
@@ -118,7 +118,10 @@ function combine(file) {
 				process.exit();
 			}
 			if (lastObj && id === lastId) {
-				if (data.__leo_delete__) {
+				if (data.__leo_delete__ || lastObj.__leo_delete__) {
+					// if our current row is a delete, then we throw away the previous updates
+					// if our previous row was a delete, but the current one is not, we throw away the delete, because it was updated
+					// after the delete, so it should NOT be deleted.
 					lastObj = data;
 				} else {
 					lastObj = merge(lastObj, data);
diff --git a/postgres/lib/dwconnect.js b/postgres/lib/dwconnect.js
index b0f65df1..e652af11 100644
--- a/postgres/lib/dwconnect.js
+++ b/postgres/lib/dwconnect.js
@@ -179,10 +179,24 @@ module.exports = function(config, columnConfig) {
 
 		tasks.push(done => {
 			ls.pipe(stream, ls.through((obj, done, push) => {
+				if (push) {
+					// do nothing, just getting rid of un-used error
+				}
 				if (obj.__leo_delete__) {
-					if (obj.__leo_delete__ === 'id') {
-						push(obj);
-					}
+					/**
+					 * The code that merges/combines all the updates already favors a DELETE over any other type of operation,
+					 * and it will throw away any other updates to the thing that is about to be deleted.
+					 * 
+					 * If we push this delete onto the next pipeline step, it will get loaded into the staging table.
+					 * This will prevent us from being able to tell what has NOT been deleted and what has been, as we are now
+					 * mixing the deletes with the inserts/updates. Because of this, I am commenting this out so that we can un-delete
+					 * facts in tables that are essentially lookup tables for transient relationships, like a connection between accounts or other
+					 * types of transient relationships. Adding it to the delete handler ensures that it will be deleted, and we
+					 * flush the deletes BEFORE we do any of the other operations on the staging tables.
+					 */
+					// if (obj.__leo_delete__ === 'id') {
+					// 	push(obj);
+					// }
 					deleteHandler.add(obj, done);
 				} else {
 					done(null, obj);

From 6662c11d83c0ade3413c5d34e9a5b3f799f14425 Mon Sep 17 00:00:00 2001
From: Grant Robinson <grant.robinson@rithum.com>
Date: Wed, 5 Feb 2025 18:27:29 -0600
Subject: [PATCH 61/62] ES-2516 - updated version #'s

---
 common/package-lock.json   | 2 +-
 common/package.json        | 2 +-
 postgres/package-lock.json | 2 +-
 postgres/package.json      | 2 +-
 4 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/common/package-lock.json b/common/package-lock.json
index cda79c5c..e82f601a 100644
--- a/common/package-lock.json
+++ b/common/package-lock.json
@@ -1,6 +1,6 @@
 {
 	"name": "leo-connector-common",
-	"version": "4.0.12-rc",
+	"version": "4.0.13-rc",
 	"lockfileVersion": 1,
 	"requires": true,
 	"dependencies": {
diff --git a/common/package.json b/common/package.json
index edee2290..5596763d 100644
--- a/common/package.json
+++ b/common/package.json
@@ -1,6 +1,6 @@
 {
 	"name": "leo-connector-common",
-	"version": "4.0.12-rc",
+	"version": "4.0.13-rc",
 	"description": "Common package for all Leo Platform database connectors",
 	"main": "index.js",
 	"directories": {},
diff --git a/postgres/package-lock.json b/postgres/package-lock.json
index 2e685d18..f084116a 100644
--- a/postgres/package-lock.json
+++ b/postgres/package-lock.json
@@ -1,6 +1,6 @@
 {
 	"name": "leo-connector-postgres",
-	"version": "4.0.23-rc",
+	"version": "4.0.24-rc",
 	"lockfileVersion": 1,
 	"requires": true,
 	"dependencies": {
diff --git a/postgres/package.json b/postgres/package.json
index 321a4576..1676e589 100644
--- a/postgres/package.json
+++ b/postgres/package.json
@@ -1,6 +1,6 @@
 {
 	"name": "leo-connector-postgres",
-	"version": "4.0.23-rc",
+	"version": "4.0.24-rc",
 	"description": "A Postgres database connector for use with Leo Platform",
 	"repository": {
 		"type": "git",

From 9c44258b8f8aaf899cc411c541cf9aa80f38ed7e Mon Sep 17 00:00:00 2001
From: Grant Robinson <grant.robinson@rithum.com>
Date: Wed, 5 Feb 2025 18:28:58 -0600
Subject: [PATCH 62/62] ES-2516 - use new version of the common connector

---
 postgres/package-lock.json | 6 +++---
 postgres/package.json      | 2 +-
 2 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/postgres/package-lock.json b/postgres/package-lock.json
index f084116a..6e3787b2 100644
--- a/postgres/package-lock.json
+++ b/postgres/package-lock.json
@@ -1076,9 +1076,9 @@
 			}
 		},
 		"leo-connector-common": {
-			"version": "4.0.12-rc",
-			"resolved": "https://registry.npmjs.org/leo-connector-common/-/leo-connector-common-4.0.12-rc.tgz",
-			"integrity": "sha512-JO19U6uEaUUTViNUCSF63lcyOiVQPZxgWL+gRTzbbWyAXXeJLf222RlnSOaiXCTukTLJjd/SqIXl/0fzjA6lTA==",
+			"version": "4.0.13-rc",
+			"resolved": "https://registry.npmjs.org/leo-connector-common/-/leo-connector-common-4.0.13-rc.tgz",
+			"integrity": "sha512-RXBVMTBOY7oc9EdFCHYxg4GePeKTvhwsgbsohNJ3hpw9ognHkdX3rqsrnhcjXeJEt/BS9/f3jZt4Ra4dk+SdXQ==",
 			"requires": {
 				"async": "2.6.3",
 				"leo-aws": "^2.0.3",
diff --git a/postgres/package.json b/postgres/package.json
index 1676e589..cc385a2a 100644
--- a/postgres/package.json
+++ b/postgres/package.json
@@ -19,7 +19,7 @@
 	"homepage": "https://github.com/LeoPlatform/connectors#readme",
 	"dependencies": {
 		"fast-csv": "2.4.1",
-		"leo-connector-common": "4.0.12-rc",
+		"leo-connector-common": "4.0.13-rc",
 		"pg": "^8.11.3",
 		"pg-copy-streams": "2.2.2",
 		"pg-format": "1.0.4",