diff --git a/.github/workflows/publish-postgres.yml b/.github/workflows/publish-postgres.yml new file mode 100644 index 00000000..ddcfe22f --- /dev/null +++ b/.github/workflows/publish-postgres.yml @@ -0,0 +1,33 @@ +# This workflow will do a clean install of node dependencies, cache/restore them, build the source code and run tests across different versions of node +# For more information see: https://help.github.com/actions/language-and-framework-guides/using-nodejs-with-github-actions + +name: Publish-Postgres + +on: + release: + types: [published] + # branches: [ $default-branch ] + +jobs: + publish-new-release: + runs-on: ubuntu-latest + defaults: + run: + working-directory: ./postgres + steps: + - uses: actions/checkout@v2 + - name: Node 12 + uses: actions/setup-node@v2 + with: + node-version: '12.x' + registry-url: 'https://npm.pkg.github.com' + scope: '@leoplatform' + - run: npm ci + - run: npm run build --if-present + # - run: npm run test #Nothing is implemented here currently so this is omitted + # - run: git config user.email "githubactions@commercehub.com" #skipping this for now since we don't have a servie account for LeoPlatform yet + - run: git config user.name "GitHub-Actions" + - run: npm version ${{ github.event.release.tag_name }} + - run: npm publish --access=public + env: + NODE_AUTH_TOKEN: ${{ secrets.GITHUB_TOKEN }} diff --git a/common/datawarehouse/combine.js b/common/datawarehouse/combine.js index 804ef6a9..40bb5ab8 100644 --- a/common/datawarehouse/combine.js +++ b/common/datawarehouse/combine.js @@ -118,7 +118,10 @@ function combine(file) { process.exit(); } if (lastObj && id === lastId) { - if (data.__leo_delete__) { + if (data.__leo_delete__ || lastObj.__leo_delete__) { + // if our current row is a delete, then we throw away the previous updates + // if our previous row was a delete, but the current one is not, we throw away the delete, because it was updated + // after the delete, so it should NOT be deleted. lastObj = data; } else { lastObj = merge(lastObj, data); diff --git a/common/datawarehouse/load.js b/common/datawarehouse/load.js index 15692a30..340150ad 100644 --- a/common/datawarehouse/load.js +++ b/common/datawarehouse/load.js @@ -9,7 +9,7 @@ const async = require('async'); const validate = require('./../utils/validation'); let errorStream; -module.exports = function (ID, source, client, tableConfig, stream, callback) { +module.exports = function(ID, source, client, tableConfig, stream, callback) { // adding backwards compatibility for ID and source if (!callback) { @@ -39,7 +39,7 @@ module.exports = function (ID, source, client, tableConfig, stream, callback) { }); }); - let checkforDelete = ls.through(function (obj, done) { + let checkforDelete = ls.through(function(obj, done) { if (obj.payload.type === 'delete') { let data = obj.payload.data || {}; let ids = data.in || []; @@ -68,7 +68,7 @@ module.exports = function (ID, source, client, tableConfig, stream, callback) { } }); - let validateData = ls.through(function (obj, done) { + let validateData = ls.through(function(obj, done) { let eventObj = obj.payload.data; let invalid = false; @@ -336,7 +336,7 @@ module.exports = function (ID, source, client, tableConfig, stream, callback) { * @param eventObj {Object} * @param error {string} */ -function handleFailedValidation (ID, source, eventObj, error) { +function handleFailedValidation(ID, source, eventObj, error) { if (!errorStream || !errorStream.Writable) { errorStream = streams.passthrough({ objectMode: true @@ -345,11 +345,11 @@ function handleFailedValidation (ID, source, eventObj, error) { streams.pipe( errorStream, - ls.process(ID, obj => { + ls.process(ID, function(obj) { return obj; - }), - - leo.load(ID, `${source}_error`), + }, `${source}_error`), + ls.toLeo(ID), + ls.devnull(), (err) => { err && logger.error('GOT ERROR', err); diff --git a/common/package-lock.json b/common/package-lock.json index 6b644f77..e82f601a 100644 --- a/common/package-lock.json +++ b/common/package-lock.json @@ -1,6 +1,6 @@ { "name": "leo-connector-common", - "version": "4.0.10-rc", + "version": "4.0.13-rc", "lockfileVersion": 1, "requires": true, "dependencies": { diff --git a/common/package.json b/common/package.json index 38733304..5596763d 100644 --- a/common/package.json +++ b/common/package.json @@ -1,6 +1,6 @@ { "name": "leo-connector-common", - "version": "4.0.11-rc", + "version": "4.0.13-rc", "description": "Common package for all Leo Platform database connectors", "main": "index.js", "directories": {}, diff --git a/entity-table/package-lock.json b/entity-table/package-lock.json index bca9bea4..64e19be7 100644 --- a/entity-table/package-lock.json +++ b/entity-table/package-lock.json @@ -1,6 +1,6 @@ { "name": "leo-connector-entity-table", - "version": "3.0.21-rc", + "version": "3.0.22-rc", "lockfileVersion": 1, "requires": true, "dependencies": { diff --git a/entity-table/package.json b/entity-table/package.json index 4f7a2bdf..5848098b 100644 --- a/entity-table/package.json +++ b/entity-table/package.json @@ -1,6 +1,6 @@ { "name": "leo-connector-entity-table", - "version": "3.0.21-rc", + "version": "3.0.22-rc", "description": "A Leo entity table connector", "repository": { "type": "git", diff --git a/postgres/lib/connect.js b/postgres/lib/connect.js index 9815bda4..206c908d 100644 --- a/postgres/lib/connect.js +++ b/postgres/lib/connect.js @@ -11,14 +11,16 @@ var copyFrom = require('pg-copy-streams').from; var copyTo = require('pg-copy-streams').to; let csv = require('fast-csv'); +// need to confirm there is no issue adding these dependencies +const leo = require('leo-sdk'); +const ls = leo.streams; + require('pg').types.setTypeParser(1114, (val) => { val += 'Z'; logger.debug(val); return moment(val).unix() + ' ' + moment(val).utc().format(); }); -const ls = require('leo-sdk').streams; - let queryCount = 0; module.exports = function (config) { const pool = new Pool(Object.assign({ @@ -571,8 +573,176 @@ function create (pool, parentCache) { records: opts.records, }); }, - streamToTableFromS3: () => { - // opts = Object.assign({}, opts || {}); + streamToTableFromS3: (table, config) => { + const ts = table.split('.'); + let schema = 'public'; + let shortTable = table; + if (ts.length > 1) { + schema = ts[0]; + shortTable = ts[1]; + } + + let columns = []; + let stream; + let myClient = null; + let pending = null; + let ended = false; + let csvopts = { delimiter: '|' }; + let keepS3Files = config.keepS3Files != null ? config.keepS3Files : false; + + // Get prefix for S3 file path + let s3prefix = + config.s3prefix || + process.env.AWS_LAMBDA_FUNCTION_NAME || + 'dw_redshift_ingest'; + s3prefix = s3prefix.replace(/^\/*(.*?)\/*$/, '$1'); // Remove leading and trailing '/' + + // clean audit date to use in S3 file path + let cleanAuditDate = client.auditdate + .replace(/'/g, '') + .replace(/:/g, '-'); + let s3FileName = `files/${s3prefix}/${cleanAuditDate}/${table}.csv`; + + client.connect().then( + c => { + client + .describeTable(shortTable, schema) + .then(result => { + if (ended) { + c.release(true); + return; + } + columns = result.map(f => f.column_name); + myClient = c; + + stream = ls.toS3(leo.configuration.resources.LeoS3, s3FileName); + // copyFrom uses `end` but s3 `finish` so pipe finish to end + stream.on('finish', () => stream.emit('end')); + + stream.on('error', function(err) { + logger.error(`COPY error: ${err.where}`, err); + process.exit(); + }); + if (pending) { + pending(); + } + }) + .catch(err => { + logger.error(err); + }); + }, + err => { + logger.error(err); + } + ); + + let count = 0; + + function nonNull(v) { + if (v === '' || v === null || v === undefined) { + return '\\N'; + } else if (typeof v === 'string' && (v.search(/\r/) !== -1 || v.search(/\n/) !== -1)) { + if (config.version !== 'redshift') { + return v.replace(/\r\n?/g, '\n'); + } else { + return v.replace(/\r\n?/g, '\n').replace(/\n/g, `\\n`); + } + } else { + return v; + } + } + + + return ls.pipeline( + csv.createWriteStream({ + ...csvopts, + headers: false, + transform: (row, done) => { + if (!myClient) { + pending = () => { + done( + null, + columns.map(f => nonNull(row[f])) + ); + }; + } else { + done( + null, + columns.map(f => nonNull(row[f])) + ); + } + }, + }), + ls.write( + (r, done) => { + count++; + if (count % 10000 === 0) { + logger.info(table + ': ' + count); + } + if (!stream.write(r)) { + stream.once('drain', done); + } else { + done(null); + } + }, + done => { + ended = true; + logger.debug(table + ': stream done'); + if (stream) { + stream.on('end', err => { + logger.debug(table + ': stream ended', err || ''); + + // wrap done callback to release the connection + function innerDone(err) { + myClient.release(true); + logger.debug(table + ': stream client released', err || ''); + done(err); + } + + if (err) { + innerDone(err); + } else { + // Once the S3 file is complete run copy to load the staging table + let f = columns.map(f => `"${f}"`); + let file = `s3://${leo.configuration.s3}/${s3FileName}`; + let manifest = ''; + let role = config.loaderRole; + myClient.query( + `copy ${table} (${f}) from '${file}' ${manifest} ${role ? `credentials 'aws_iam_role=${role}'` : '' + } NULL AS '\\\\N' format csv DELIMITER '|' ACCEPTINVCHARS TRUNCATECOLUMNS ACCEPTANYDATE TIMEFORMAT 'auto' COMPUPDATE OFF`, + copyErr => { + if (keepS3Files) { + innerDone(copyErr); + } else { + // Delete the S3 files when done + ls.s3.deleteObject( + { + Bucket: leo.configuration.s3, + Key: s3FileName, + }, + deleteError => { + if (deleteError) { + logger.info( + 'file failed to delete:', + s3FileName, + deleteError + ); + } + innerDone(copyErr); + } + ); + } + } + ); + } + }); + stream.end(); + } else { + done(); + } + } + ) + ); }, }; diff --git a/postgres/lib/dwconnect.js b/postgres/lib/dwconnect.js index dd3105c7..e652af11 100644 --- a/postgres/lib/dwconnect.js +++ b/postgres/lib/dwconnect.js @@ -4,7 +4,7 @@ const async = require('async'); const ls = require('leo-sdk').streams; const logger = require('leo-logger'); -module.exports = function (config, columnConfig) { +module.exports = function(config, columnConfig) { let client = postgres(config); let dwClient = client; let tempTables = []; @@ -27,10 +27,24 @@ module.exports = function (config, columnConfig) { useSurrogateDateKeys: true, }, columnConfig || {}); + console.log(`has delete_fix in-place`); + + + // Control flow for both of these configurations set to true has not been added. An error will be thrown until that is supported. + if ( + (config.hashedSurrogateKeys && !config.bypassSlowlyChangingDimensions) // hashed surrogate keys and slowly changing dimensions not supported + || (config.hashedSurrogateKeys && config.bypassSlowlyChangingDimensions === undefined) // covering the case where slowly changing dimensions has been omitted + ) { + logger.error(`Unsupported configuration, bypassSlowlyChangingDimensions:${config.bypassSlowlyChangingDimensions} hashedSurrogateKeys:${config.hashedSurrogateKeys}.`); + process.exit(); + }; + client.getDimensionColumn = columnConfig.dimColumnTransform; client.columnConfig = columnConfig; - function deletesSetup (qualifiedTable, schema, field, value, where = '') { + let deleteFlushCount = config.deleteFlushCount || 1000; + + function deletesSetup(qualifiedTable, schema, field, value, where = '') { let colLookup = {}; schema.map(col => { colLookup[col.column_name] = true; @@ -41,8 +55,8 @@ module.exports = function (config, columnConfig) { where = `and ${where}`; } - function tryFlushDelete (done, force = false) { - if (force || toDeleteCount >= 1000) { + function tryFlushDelete(done, force = false) { + if (force || toDeleteCount >= deleteFlushCount) { let deleteTasks = Object.keys(toDelete).map(col => { return deleteDone => client.query(`update ${qualifiedTable} set ${field} = ${value}, ${columnConfig._auditdate} = ${dwClient.auditdate} where ${col} in (${toDelete[col].join(',')}) ${where}`, deleteDone); }); @@ -59,7 +73,7 @@ module.exports = function (config, columnConfig) { } return { - add: function (obj, done) { + add: function(obj, done) { let field = obj.__leo_delete__; let id = obj.__leo_delete_id__; if (id !== undefined && colLookup[field]) { @@ -86,26 +100,32 @@ module.exports = function (config, columnConfig) { if (tempTables.length) { let tasks = []; - tempTables.forEach(table => { - tasks.push(done => client.query(`drop table ${table}`, done)); - }); + try { + tempTables.forEach(table => { + tasks.push(done => client.query(`drop table if exists ${table}`, done)); + }); - return new Promise(resolve => { - async.series(tasks, err => { - if (err) { - throw err; - } else { - tempTables = []; - return resolve('Cleaned up temp tables'); - } + await new Promise((resolve, reject) => { + async.series(tasks, err => { + if (err) { + reject(err); + } else { + tempTables = []; + resolve('Cleaned up temp tables'); + } + }); }); - }); + } catch (dropError) { + // Temp tables are cleaned up before use in the next invocation + // so just log it and move on + logger.info('Error dropping temp tables', dropError); + } } return true; }; - client.importFact = function (stream, table, ids, callback) { + client.importFact = function(stream, table, ids, callback) { const stagingTable = `${columnConfig.stageTablePrefix}_${table}`; const qualifiedStagingTable = `${columnConfig.stageSchema}.${stagingTable}`; const qualifiedTable = `public.${table}`; @@ -121,23 +141,67 @@ module.exports = function (config, columnConfig) { let deleteHandler = deletesSetup(qualifiedTable, schema[qualifiedTable], columnConfig._deleted, true); + let sortKey; + let sortKeyType; + tempTables.push(qualifiedStagingTable); tasks.push(done => client.query(`drop table if exists ${qualifiedStagingTable}`, done)); - tasks.push(done => client.query(`drop table if exists ${qualifiedStagingTable}_changes`, done)); - tasks.push(done => client.query(`create table ${qualifiedStagingTable} (like ${qualifiedTable})`, done)); - tasks.push(done => client.query(`create index ${stagingTable}_id on ${qualifiedStagingTable} (${ids.join(', ')})`, done)); - // tasks.push(done => ls.pipe(stream, client.streamToTable(qualifiedStagingTable), done)); + if (config.version !== 'redshift') { + tasks.push(done => client.query(`create table ${qualifiedStagingTable} (like ${qualifiedTable})`, done)); + tasks.push(done => client.query(`create index ${stagingTable}_id on ${qualifiedStagingTable} (${ids.join(', ')})`, done)); + } else { + // get sortkey for joins + tasks.push(done => { + client.query(`SELECT sortkey, + sortkeytype + FROM public.v_dist_sort_key + WHERE table_name = '${table}';`, (err, results) => { + if (err) { + return done(err); + } else { + if (results[0].sortKey != null) { + sortKey = results[0].sortkey; + sortKeyType = results[0].sortkeytype; + }; + done(); + }; + }); + }); + tasks.push(done => + client.query(`CREATE TABLE ${qualifiedStagingTable} + DISTSTYLE ALL + SORTKEY (${sortKey != null ? sortKey : ids[0]}) + AS SELECT * + FROM ${qualifiedTable} + LIMIT 0;`, done)); + }; + + tasks.push(done => { ls.pipe(stream, ls.through((obj, done, push) => { + if (push) { + // do nothing, just getting rid of un-used error + } if (obj.__leo_delete__) { - if (obj.__leo_delete__ === 'id') { - push(obj); - } + /** + * The code that merges/combines all the updates already favors a DELETE over any other type of operation, + * and it will throw away any other updates to the thing that is about to be deleted. + * + * If we push this delete onto the next pipeline step, it will get loaded into the staging table. + * This will prevent us from being able to tell what has NOT been deleted and what has been, as we are now + * mixing the deletes with the inserts/updates. Because of this, I am commenting this out so that we can un-delete + * facts in tables that are essentially lookup tables for transient relationships, like a connection between accounts or other + * types of transient relationships. Adding it to the delete handler ensures that it will be deleted, and we + * flush the deletes BEFORE we do any of the other operations on the staging tables. + */ + // if (obj.__leo_delete__ === 'id') { + // push(obj); + // } deleteHandler.add(obj, done); } else { done(null, obj); } - }), client.streamToTable(qualifiedStagingTable), (err) => { + }), config.version !== 'redshift' ? client.streamToTable(qualifiedStagingTable) : client.streamToTableFromS3(qualifiedStagingTable, config), (err) => { if (err) { return done(err); } else { @@ -146,7 +210,9 @@ module.exports = function (config, columnConfig) { }); }); - tasks.push(done => client.query(`analyze ${qualifiedStagingTable}`, done)); + if (config.version !== 'redshift') { + tasks.push(done => client.query(`analyze ${qualifiedStagingTable}`, done)); + }; client.describeTable(table).then(result => { let columns = result.filter(field => !field.column_name.match(/^_/)).map(field => `"${field.column_name}"`); @@ -159,35 +225,120 @@ module.exports = function (config, columnConfig) { let tasks = []; let totalRecords = 0; - // The following code relies on the fact that now() will return the same time during all transaction events + + // The following code relies on the fact that now()/sysdate will return the same time during all transaction events tasks.push(done => connection.query(`Begin Transaction`, done)); - tasks.push(done => { - connection.query(`select 1 as total from ${qualifiedTable} limit 1`, (err, results) => { - if (err) { - return done(err); - } - totalRecords = results.length; - done(); + + if (!config.hashedSurrogateKeys) { + tasks.push(done => { + connection.query(`select 1 as total from ${qualifiedTable} limit 1`, (err, results) => { + if (err) { + return done(err); + } + totalRecords = results.length; + done(); + }); + }); + tasks.push(done => { + connection.query(`Update ${qualifiedTable} prev + SET ${columns.map(column => `${column} = coalesce(staging.${column}, prev.${column})`)}, ${columnConfig._deleted} = false, ${columnConfig._auditdate} = ${dwClient.auditdate} + FROM ${qualifiedStagingTable} staging + where ${ids.map(id => `prev.${id} = staging.${id}`).join(' and ')}` + , done); }); - }); - tasks.push(done => { - connection.query(`Update ${qualifiedTable} prev - SET ${columns.map(column => `${column} = coalesce(staging.${column}, prev.${column})`)}, ${columnConfig._deleted} = coalesce(prev.${columnConfig._deleted}, false), ${columnConfig._auditdate} = ${dwClient.auditdate} - FROM ${qualifiedStagingTable} staging - where ${ids.map(id => `prev.${id} = staging.${id}`).join(' and ')} - `, done); - }); - // Now insert any we were missing - tasks.push(done => { - connection.query(`INSERT INTO ${qualifiedTable} (${columns.join(',')},${columnConfig._deleted},${columnConfig._auditdate}) - SELECT ${columns.map(column => `coalesce(staging.${column}, prev.${column})`)}, coalesce(prev.${columnConfig._deleted}, false), ${dwClient.auditdate} as ${columnConfig._auditdate} - FROM ${qualifiedStagingTable} staging - LEFT JOIN ${qualifiedTable} as prev on ${ids.map(id => `prev.${id} = staging.${id}`).join(' and ')} - WHERE prev.${ids[0]} is null - `, done); - }); - // tasks.push(done => connection.query(`drop table ${stagingTbl}`, done)); + // Now insert any we were missing + tasks.push(done => { + connection.query(`INSERT INTO ${qualifiedTable} (${columns.join(',')},${columnConfig._deleted},${columnConfig._auditdate}) + SELECT ${columns.map(column => `staging.${column}`)}, false AS ${columnConfig._deleted}, ${dwClient.auditdate} as ${columnConfig._auditdate} + FROM ${qualifiedStagingTable} staging + WHERE NOT EXISTS ( SELECT * + FROM ${qualifiedTable} as prev + WHERE ${ids.map(id => `prev.${id} = staging.${id}`).join(' and ')})`, done); + }); + } else { + let naturalKeyLowerBound; + let naturalKeyFilter; + + // Get lower bound for natural key to avoid unnecessary scanning + tasks.push(done => { + connection.query(`SELECT MIN(${(sortKey != null) ? sortKey : ids[0]}) AS minid, + CAST(COUNT(*) AS INT) AS cnt + FROM ${qualifiedStagingTable};`, (err, results) => { + if (err) { + return done(err); + } else { + totalRecords = results[0].cnt; + naturalKeyLowerBound = results[0].minid; + if (naturalKeyLowerBound !== null) { + if (sortKeyType === 'int4' || sortKeyType === 'int8') { + naturalKeyFilter = `${results[0].minid}`; + } else if (sortKeyType === 'varchar') { + naturalKeyFilter = `'${results[0].minid}'`; + } else if (sortKeyType === 'timestamp' && Date.parse(results[0].minid.split(' ')[1])) { + naturalKeyFilter = `'${results[0].minid.split(' ')[1]}'`; + }; + }; + done(); + }; + }); + }); + + const qualifiedStagingTablePrevious = `${qualifiedStagingTable}_previous`; + tempTables.push(`${qualifiedStagingTablePrevious}`); + tasks.push(done => connection.query(`DROP TABLE IF EXISTS ${qualifiedStagingTablePrevious};`, done)); + tasks.push(done => { + connection.query(`CREATE TABLE ${qualifiedStagingTablePrevious} + DISTSTYLE ALL + AS SELECT * + FROM ${qualifiedTable} + LIMIT 0;`, done); + }); + + // Retreive copy of existing data + tasks.push(done => { + connection.query(`INSERT INTO ${qualifiedStagingTablePrevious} (${columns.map(column => `${column}`).join(`, `)}, ${columnConfig._deleted}) + SELECT ${columns.map(column => `${column}`).join(`, `)}, + ${columnConfig._deleted} + FROM ${qualifiedTable} AS base + WHERE EXISTS ( SELECT * + FROM ${qualifiedStagingTable} AS staging + WHERE ${ids.map(id => `base.${id} = staging.${id}`).join(` AND `)} + ${(naturalKeyFilter !== undefined) ? `AND staging.${(sortKey != null) ? sortKey : ids[0]} >= ${naturalKeyFilter}` : ``}) + ${(naturalKeyFilter !== undefined) ? `AND base.${(sortKey != null) ? sortKey : ids[0]} >= ${naturalKeyFilter}` : ``};`, done); + }); + + // Merge existing data into staged copy + tasks.push(done => { + connection.query(`UPDATE ${qualifiedStagingTable} AS staging + SET ${columns.map(column => `${column} = COALESCE(staging.${column}, prev.${column})`).join(`,`)}, + ${columnConfig._deleted} = false + FROM ${qualifiedStagingTablePrevious} AS prev + WHERE ${ids.map(id => `prev.${id} = staging.${id}`).join(` AND `)}`, done); + }); + + // Set auditdate and _deleted for stage data + tasks.push(done => { + connection.query(`UPDATE ${qualifiedStagingTable} + SET ${columnConfig._auditdate} = ${dwClient.auditdate}, + ${columnConfig._deleted} = false; `, done); + }); + + // Delete and reinsert data - avoids costly updates on large tables + tasks.push(done => { + connection.query(`DELETE FROM ${qualifiedTable} + USING ${qualifiedStagingTable} + WHERE ${ids.map(id => `${qualifiedTable}.${id} = ${qualifiedStagingTable}.${id}`).join(` AND `)} + ${(naturalKeyFilter !== undefined) ? `AND ${qualifiedTable}.${sortKey != null ? sortKey : ids[0]} >= ${naturalKeyFilter}` : ``}; `, done); + }); + tasks.push(done => { + connection.query(`INSERT INTO ${qualifiedTable} (${columns.map(column => `${column}`).join(`, `)}, ${columnConfig._auditdate}, ${columnConfig._deleted}) + SELECT ${columns.map(column => `${column}`).join(`, `)}, + ${columnConfig._auditdate}, + false as ${columnConfig._deleted} + FROM ${qualifiedStagingTable}; `, done); + }); + } async.series(tasks, err => { if (!err) { @@ -200,7 +351,7 @@ module.exports = function (config, columnConfig) { } else { connection.query(`rollback`, (e, d) => { connection.release(); - callback(e, d); + callback(e || err, d); }); } }); @@ -209,7 +360,7 @@ module.exports = function (config, columnConfig) { }).catch(callback); }; - client.importDimension = function (stream, table, sk, nk, scds, callback, tableDef = {}) { + client.importDimension = function(stream, table, sk, nk, scds, callback, tableDef = {}) { const stagingTbl = `${columnConfig.stageTablePrefix}_${table}`; const qualifiedStagingTable = `${columnConfig.stageSchema}.${stagingTbl}`; const qualifiedTable = `public.${table}`; @@ -227,13 +378,49 @@ module.exports = function (config, columnConfig) { let tasks = []; let deleteHandler = deletesSetup(qualifiedTable, schema[qualifiedTable], columnConfig._enddate, dwClient.auditdate, `${columnConfig._current} = true`); + let sortKey; + let sortKeyType; + + // Prepare staging tables tempTables.push(qualifiedStagingTable); tasks.push(done => client.query(`drop table if exists ${qualifiedStagingTable}`, done)); tasks.push(done => client.query(`drop table if exists ${qualifiedStagingTable}_changes`, done)); - tasks.push(done => client.query(`create table ${qualifiedStagingTable} (like ${qualifiedTable})`, done)); - tasks.push(done => client.query(`create index ${stagingTbl}_id on ${qualifiedStagingTable} (${nk.join(', ')})`, done)); - tasks.push(done => client.query(`alter table ${qualifiedStagingTable} drop column ${sk}`, done)); - // tasks.push(done => ls.pipe(stream, client.streamToTable(qualifiedStagingTable), done)); + if (config.version !== 'redshift') { + tasks.push(done => client.query(`create table ${qualifiedStagingTable} (like ${qualifiedTable})`, done)); + tasks.push(done => client.query(`create index ${stagingTbl}_id on ${qualifiedStagingTable} (${nk.join(', ')})`, done)); + } else { + // get sortkey for joins + tasks.push(done => { + client.query(`SELECT sortkey, + sortkeytype + FROM public.v_dist_sort_key + WHERE table_name = '${table}';`, (err, results) => { + if (err) { + return done(err); + } else { + if (results[0].sortkey != null) { + sortKey = results[0].sortkey; + sortKeyType = results[0].sortkeytype; + }; + done(); + }; + }); + }); + tasks.push(done => + // Create staging table with DISTSTYLE ALL to prevent cross talk + client.query(`CREATE TABLE ${qualifiedStagingTable} + DISTSTYLE ALL + SORTKEY(${sortKey != null ? sortKey : nk[0]}) + AS SELECT * + FROM ${qualifiedTable} + LIMIT 0;`, done)); + }; + + // Retain surrogate key column when using hashed keys on redshift // column must be dropped for postgres + if (!config.hashedSurrogateKeys && config.version !== 'redshift') { + tasks.push(done => client.query(`alter table ${qualifiedStagingTable} drop column ${sk}`, done)); + }; + tasks.push(done => { ls.pipe(stream, ls.through((obj, done, push) => { if (obj.__leo_delete__) { @@ -244,7 +431,7 @@ module.exports = function (config, columnConfig) { } else { done(null, obj); } - }), client.streamToTable(qualifiedStagingTable), (err) => { + }), config.version !== 'redshift' ? client.streamToTable(qualifiedStagingTable) : client.streamToTableFromS3(qualifiedStagingTable, config), (err) => { if (err) { return done(err); } else { @@ -253,7 +440,9 @@ module.exports = function (config, columnConfig) { }); }); - tasks.push(done => client.query(`analyze ${qualifiedStagingTable}`, done)); + if (config.version !== 'redshift') { + tasks.push(done => client.query(`analyze ${qualifiedStagingTable}`, done)); + }; client.describeTable(table).then(result => { client.connect().then(connection => { @@ -262,7 +451,6 @@ module.exports = function (config, columnConfig) { return callback(err); } - // let scd0 = scds[0] || []; // Not Used let scd2 = scds[2] || []; let scd3 = scds[3] || []; let scd6 = Object.keys(scds[6] || {}); @@ -278,88 +466,195 @@ module.exports = function (config, columnConfig) { let scdSQL = []; - // if (!scd2.length && !scd3.length && !scd6.length) { - // scdSQL.push(`1 as runSCD1`); - // } else - if (scd1.length) { - scdSQL.push(`CASE WHEN md5(${scd1.map(f => 'md5(coalesce(s.' + f + '::text,\'\'))').join(' || ')}) = md5(${scd1.map(f => 'md5(coalesce(d.' + f + '::text,\'\'))').join(' || ')}) THEN 0 WHEN d.${nk[0]} is null then 0 ELSE 1 END as runSCD1`); - } else { - scdSQL.push(`0 as runSCD1`); - } - if (scd2.length) { - scdSQL.push(`CASE WHEN d.${nk[0]} is null then 1 WHEN md5(${scd2.map(f => 'md5(coalesce(s.' + f + '::text,\'\'))').join(' || ')}) = md5(${scd2.map(f => 'md5(coalesce(d.' + f + '::text,\'\'))').join(' || ')}) THEN 0 ELSE 1 END as runSCD2`); - } else { - scdSQL.push(`CASE WHEN d.${nk[0]} is null then 1 ELSE 0 END as runSCD2`); - } - if (scd3.length) { - scdSQL.push(`CASE WHEN md5(${scd3.map(f => 'md5(coalesce(s.' + f + '::text,\'\'))').join(' || ')}) = md5(${scd3.map(f => 'md5(coalesce(d.' + f + '::text,\'\'))').join(' || ')}) THEN 0 WHEN d.${nk[0]} is null then 0 ELSE 1 END as runSCD3`); - } else { - scdSQL.push(`0 as runSCD3`); - } - if (scd6.length) { - scdSQL.push(`CASE WHEN md5(${scd6.map(f => 'md5(coalesce(s.' + f + '::text,\'\'))').join(' || ')}) = md5(${scd6.map(f => 'md5(coalesce(d.' + f + '::text,\'\'))').join(' || ')}) THEN 0 WHEN d.${nk[0]} is null then 0 ELSE 1 END as runSCD6`); + if (!config.bypassSlowlyChangingDimensions) { + if (scd1.length) { + scdSQL.push(`CASE WHEN md5(${scd1.map(f => 'md5(coalesce(s.' + f + '::text,\'\'))').join(' || ')}) = md5(${scd1.map(f => 'md5(coalesce(d.' + f + '::text,\'\'))').join(' || ')}) THEN 0 WHEN d.${nk[0]} is null then 0 ELSE 1 END as runSCD1`); + } else { + scdSQL.push(`0 as runSCD1`); + } + if (scd2.length) { + scdSQL.push(`CASE WHEN d.${nk[0]} is null then 1 WHEN md5(${scd2.map(f => 'md5(coalesce(s.' + f + '::text,\'\'))').join(' || ')}) = md5(${scd2.map(f => 'md5(coalesce(d.' + f + '::text,\'\'))').join(' || ')}) THEN 0 ELSE 1 END as runSCD2`); + } else { + scdSQL.push(`CASE WHEN d.${nk[0]} is null then 1 ELSE 0 END as runSCD2`); + } + if (scd3.length) { + scdSQL.push(`CASE WHEN md5(${scd3.map(f => 'md5(coalesce(s.' + f + '::text,\'\'))').join(' || ')}) = md5(${scd3.map(f => 'md5(coalesce(d.' + f + '::text,\'\'))').join(' || ')}) THEN 0 WHEN d.${nk[0]} is null then 0 ELSE 1 END as runSCD3`); + } else { + scdSQL.push(`0 as runSCD3`); + } + if (scd6.length) { + scdSQL.push(`CASE WHEN md5(${scd6.map(f => 'md5(coalesce(s.' + f + '::text,\'\'))').join(' || ')}) = md5(${scd6.map(f => 'md5(coalesce(d.' + f + '::text,\'\'))').join(' || ')}) THEN 0 WHEN d.${nk[0]} is null then 0 ELSE 1 END as runSCD6`); + } else { + scdSQL.push(`0 as runSCD6`); + } + }; + + let fields = [sk].concat(allColumns).concat([columnConfig._auditdate, columnConfig._startdate, columnConfig._enddate, columnConfig._current]); + let tasks = []; + let totalRecords = 0; + + if (!config.hashedSurrogateKeys && !config.bypassSlowlyChangingDimensions) { + tempTables.push(`${qualifiedStagingTable}_changes`); + connection.query(`create table ${qualifiedStagingTable}_changes as + select ${nk.map(id => `s.${id}`).join(', ')}, d.${nk[0]} is null as isNew, + ${scdSQL.join(',\n')} + FROM ${qualifiedStagingTable} s + LEFT JOIN ${qualifiedTable} d on ${nk.map(id => `d.${id} = s.${id}`).join(' and ')} and d.${columnConfig._current}`, (err) => { + if (err) { + logger.error(err); + process.exit(); + } + + let rowId = null; + + tasks.push(done => connection.query(`analyze ${qualifiedStagingTable}_changes`, done)); + tasks.push(done => { + connection.query(`select max(${sk}) as maxid from ${qualifiedTable}`, (err, results) => { + if (err) { + return done(err); + } + rowId = results[0].maxid || 10000; + totalRecords = (rowId - 10000); + done(); + }); + }); + + // The following code relies on the fact that now()/sysdate will return the same time during all transaction events + tasks.push(done => connection.query(`Begin Transaction`, done)); + tasks.push(done => { + connection.query(`INSERT INTO ${qualifiedTable} (${fields.join(',')}) + SELECT row_number() over() + ${rowId}, ${allColumns.map(column => `coalesce(staging.${column}, prev.${column})`)}, ${dwClient.auditdate} as ${columnConfig._auditdate}, + case when changes.isNew then '1900-01-01 00:00:00' else ${config.version === 'redshift' ? 'sysdate' : 'now()'} END as ${columnConfig._startdate}, + '9999-01-01 00:00:00' as ${columnConfig._enddate}, + true as ${columnConfig._current} + FROM ${qualifiedStagingTable}_changes changes + JOIN ${qualifiedStagingTable} staging on ${nk.map(id => `staging.${id} = changes.${id}`).join(' and ')} + LEFT JOIN ${qualifiedTable} as prev on ${nk.map(id => `prev.${id} = changes.${id}`).join(' and ')} and prev.${columnConfig._current} + WHERE (changes.runSCD2 =1 OR changes.runSCD6=1) + `, done); + }); + + // This needs to be done last + tasks.push(done => { + // RUN SCD1 / SCD6 columns (where we update the old records) + let columns = scd1.map(column => `"${column}" = coalesce(staging."${column}", prev."${column}")`).concat(scd6.map(column => `"current_${column}" = coalesce(staging."${column}", prev."${column}")`)); + columns.push(`"${columnConfig._enddate}" = case when changes.runSCD2 =1 then ${config.version === 'redshift' ? 'sysdate' : 'now()'} else prev."${columnConfig._enddate}" END`); + columns.push(`"${columnConfig._current}" = case when changes.runSCD2 =1 then false else prev."${columnConfig._current}" END`); + columns.push(`"${columnConfig._auditdate}" = ${dwClient.auditdate}`); + connection.query(`update ${qualifiedTable} as prev + set ${columns.join(', ')} + FROM ${qualifiedStagingTable}_changes changes + JOIN ${qualifiedStagingTable} staging on ${nk.map(id => `staging.${id} = changes.${id}`).join(' and ')} + where ${nk.map(id => `prev.${id} = changes.${id}`).join(' and ')} and prev.${columnConfig._startdate} != ${config.version === 'redshift' ? 'sysdate' : 'now()'} and changes.isNew = false /*Need to make sure we are only updating the ones not just inserted through SCD2 otherwise we run into issues with multiple rows having .${columnConfig._current}*/ + and (changes.runSCD1=1 OR changes.runSCD6=1 OR changes.runSCD2=1) + `, done); + }); + + async.series(tasks, err => { + if (!err) { + connection.query(`commit`, e => { + connection.release(); + callback(e || err, { + count: totalRecords, + }); + }); + } else { + connection.query(`rollback`, (e, d) => { + connection.release(); + callback(e || err, d); + }); + } + }); + }); } else { - scdSQL.push(`0 as runSCD6`); - } + let naturalKeyLowerBound; + let naturalKeyFilter; - // let's figure out which SCDs needs to happen - tempTables.push(`${qualifiedStagingTable}_changes`); - connection.query(`create table ${qualifiedStagingTable}_changes as - select ${nk.map(id => `s.${id}`).join(', ')}, d.${nk[0]} is null as isNew, - ${scdSQL.join(',\n')} - FROM ${qualifiedStagingTable} s - LEFT JOIN ${qualifiedTable} d on ${nk.map(id => `d.${id} = s.${id}`).join(' and ')} and d.${columnConfig._current}`, (err) => { - if (err) { - logger.error(err); - process.exit(); - } - let tasks = []; - let rowId = null; - let totalRecords = 0; - tasks.push(done => connection.query(`analyze ${qualifiedStagingTable}_changes`, done)); + // Get lower bound for natural key to avoid unnecessary scanning tasks.push(done => { - connection.query(`select max(${sk}) as maxid from ${qualifiedTable}`, (err, results) => { + connection.query(`SELECT MIN(${(sortKey != null) ? sortKey : nk[0]}) AS minid, + CAST(COUNT(*) AS INT) AS cnt + FROM ${qualifiedStagingTable};`, (err, results) => { if (err) { return done(err); - } - rowId = results[0].maxid || 10000; - totalRecords = (rowId - 10000); - done(); + } else { + totalRecords = results[0].cnt; + naturalKeyLowerBound = results[0].minid; + if (naturalKeyLowerBound !== null) { + if (sortKeyType === 'int4' || sortKeyType === 'int8') { + naturalKeyFilter = `${results[0].minid}`; + } else if (sortKeyType === 'varchar') { + naturalKeyFilter = `'${results[0].minid}'`; + } else if (sortKeyType === 'timestamp' && Date.parse(results[0].minid.split(' ')[1])) { + naturalKeyFilter = `'${results[0].minid.split(' ')[1]}'`; + }; + }; + done(); + }; }); }); - // The following code relies on the fact that now() will return the same time during all transaction events - tasks.push(done => connection.query(`Begin Transaction`, done)); + const qualifiedStagingTablePrevious = `${qualifiedStagingTable}_previous`; + tempTables.push(`${qualifiedStagingTablePrevious}`); + tasks.push(done => connection.query(`DROP TABLE IF EXISTS ${qualifiedStagingTablePrevious};`, done)); + tasks.push(done => { + connection.query(`CREATE TABLE ${qualifiedStagingTablePrevious} + DISTSTYLE ALL + AS SELECT * + FROM ${qualifiedTable} + LIMIT 0;`, done); + }); + + // Set auditdate and surrogate key for stage data (not sure if the sk is actually necessary) + tasks.push(done => { + connection.query(`UPDATE ${qualifiedStagingTable} + SET ${columnConfig._auditdate} = ${dwClient.auditdate}, + ${sk} = farmFingerPrint64(${nk.map(id => `${id}`).join(`|| '-' ||`)}); `, done); + }); + + tasks.push(done => connection.query(`BEGIN TRANSACTION;`, done)); + + // Retreive copy of existing data + tasks.push(done => { + connection.query(`INSERT INTO ${qualifiedStagingTablePrevious}(${fields.map(column => `${column}`).join(`, `)}) + SELECT ${fields.map(column => `${column}`).join(`, `)} + FROM ${qualifiedTable} AS base + WHERE EXISTS(SELECT * + FROM ${qualifiedStagingTable} AS staging + WHERE base.${sk} = staging.${sk} + ${(naturalKeyFilter !== undefined) ? `AND staging.${(sortKey != null) ? sortKey : nk[0]} >= ${naturalKeyFilter}` : ``}) + ${(naturalKeyFilter !== undefined) ? `AND base.${(sortKey != null) ? sortKey : nk[0]} >= ${naturalKeyFilter}` : ``}; `, done); + }); + + // Merge exiting data into staged copy + tasks.push(done => { + connection.query(`UPDATE ${qualifiedStagingTable} AS staging + SET ${fields.map(column => `${column} = COALESCE(staging.${column}, prev.${column})`).join(`, `)} + FROM ${qualifiedStagingTablePrevious} AS prev + WHERE staging.${sk} = prev.${sk}`, done); + }); + // Set default SCD values tasks.push(done => { - let fields = [sk].concat(allColumns).concat([columnConfig._auditdate, columnConfig._startdate, columnConfig._enddate, columnConfig._current]); - connection.query(`INSERT INTO ${qualifiedTable} (${fields.join(',')}) - SELECT row_number() over () + ${rowId}, ${allColumns.map(column => `coalesce(staging.${column}, prev.${column})`)}, ${dwClient.auditdate} as ${columnConfig._auditdate}, case when changes.isNew then '1900-01-01 00:00:00' else now() END as ${columnConfig._startdate}, '9999-01-01 00:00:00' as ${columnConfig._enddate}, true as ${columnConfig._current} - FROM ${qualifiedStagingTable}_changes changes - JOIN ${qualifiedStagingTable} staging on ${nk.map(id => `staging.${id} = changes.${id}`).join(' and ')} - LEFT JOIN ${qualifiedTable} as prev on ${nk.map(id => `prev.${id} = changes.${id}`).join(' and ')} and prev.${columnConfig._current} - WHERE (changes.runSCD2 =1 OR changes.runSCD6=1) - `, done); + connection.query(`UPDATE ${qualifiedStagingTable} + SET ${columnConfig._startdate} = COALESCE(${columnConfig._startdate},'1900-01-01 00:00:00'), + ${columnConfig._enddate} = COALESCE(${columnConfig._enddate},'9999-01-01 00:00:00'), + ${columnConfig._current} = COALESCE(${columnConfig._current},true);`, done); }); - // This needs to be done last + // Delete and reinsert data - avoids costly updates on large tables tasks.push(done => { - // RUN SCD1 / SCD6 columns (where we update the old records) - let columns = scd1.map(column => `"${column}" = coalesce(staging."${column}", prev."${column}")`).concat(scd6.map(column => `"current_${column}" = coalesce(staging."${column}", prev."${column}")`)); - columns.push(`"${columnConfig._enddate}" = case when changes.runSCD2 =1 then now() else prev."${columnConfig._enddate}" END`); - columns.push(`"${columnConfig._current}" = case when changes.runSCD2 =1 then false else prev."${columnConfig._current}" END`); - columns.push(`"${columnConfig._auditdate}" = ${dwClient.auditdate}`); - connection.query(`update ${qualifiedTable} as prev - set ${columns.join(', ')} - FROM ${qualifiedStagingTable}_changes changes - JOIN ${qualifiedStagingTable} staging on ${nk.map(id => `staging.${id} = changes.${id}`).join(' and ')} - where ${nk.map(id => `prev.${id} = changes.${id}`).join(' and ')} and prev.${columnConfig._startdate} != now() and changes.isNew = false /*Need to make sure we are only updating the ones not just inserted through SCD2 otherwise we run into issues with multiple rows having .${columnConfig._current}*/ - and (changes.runSCD1=1 OR changes.runSCD6=1 OR changes.runSCD2=1) - `, done); + connection.query(`DELETE FROM ${qualifiedTable} + USING ${qualifiedStagingTable} + WHERE ${qualifiedTable}.${sk} = ${qualifiedStagingTable}.${sk} + ${(naturalKeyFilter !== undefined) ? `AND ${qualifiedTable}.${sortKey != null ? sortKey : nk[0]} >= ${naturalKeyFilter}` : ``}; `, done); + }); + tasks.push(done => { + connection.query(`INSERT INTO ${qualifiedTable} (${fields.map(column => `${column}`).join(`, `)}) + SELECT ${fields.map(column => `${column}`).join(`, `)} + FROM ${qualifiedStagingTable}; `, done); }); - // tasks.push(done => connection.query(`drop table ${qualifiedStagingTable}_changes`, done)); - // tasks.push(done => connection.query(`drop table ${qualifiedStagingTable}`, done)); async.series(tasks, err => { if (!err) { connection.query(`commit`, e => { @@ -371,116 +666,173 @@ module.exports = function (config, columnConfig) { } else { connection.query(`rollback`, (e, d) => { connection.release(); - callback(e, d); + callback(e || err, d); }); } }); - }); + }; }); }).catch(callback); }).catch(callback); }; - client.insertMissingDimensions = function (usedTables, tableConfig, tableSks, tableNks, callback) { - let unions = {}; - let isDate = { - d_date: true, - d_datetime: true, - d_time: true, - date: true, - datetime: true, - dim_date: true, - dim_datetime: true, - dim_time: true, - time: true, - }; - Object.keys(usedTables).map(table => { - Object.keys(tableConfig[table].structure).map(column => { - let field = tableConfig[table].structure[column]; - if (field.dimension && !isDate[field.dimension]) { - if (!(unions[field.dimension])) { - unions[field.dimension] = []; - } - if (typeof tableNks[field.dimension] === 'undefined') { - throw new Error(`${field.dimension} not found in tableNks`); + client.insertMissingDimensions = function(usedTables, tableConfig, tableSks, tableNks, callback) { + if (config.hashedSurrogateKeys) { + callback(null); + } else { + let unions = {}; + let isDate = { + d_date: true, + d_datetime: true, + d_time: true, + date: true, + datetime: true, + dim_date: true, + dim_datetime: true, + dim_time: true, + time: true, + }; + Object.keys(usedTables).map(table => { + Object.keys(tableConfig[table].structure).map(column => { + let field = tableConfig[table].structure[column]; + if (field.dimension && !isDate[field.dimension]) { + if (!(unions[field.dimension])) { + unions[field.dimension] = []; + } + if (typeof tableNks[field.dimension] === 'undefined') { + throw new Error(`${field.dimension} not found in tableNks`); + } + let dimTableNk = tableNks[field.dimension][0]; + unions[field.dimension].push(`select ${table}.${column} as id from ${table} left join ${field.dimension} on ${field.dimension}.${dimTableNk} = ${table}.${column} where ${field.dimension}.${dimTableNk} is null and ${table}.${columnConfig._auditdate} = ${dwClient.auditdate}`); } - let dimTableNk = tableNks[field.dimension][0]; - unions[field.dimension].push(`select ${table}.${column} as id from ${table} left join ${field.dimension} on ${field.dimension}.${dimTableNk} = ${table}.${column} where ${field.dimension}.${dimTableNk} is null and ${table}.${columnConfig._auditdate} = ${dwClient.auditdate}`); - } + }); }); - }); - let missingDimTasks = Object.keys(unions).map(table => { - let sk = tableSks[table]; - let nk = tableNks[table][0]; - return (callback) => { - let done = (err, data) => { - trx && trx.release(); - callback(err, data); - }; - let trx; - client.connect().then(transaction => { - trx = transaction; - transaction.query(`select max(${sk}) as maxid from ${table}`, (err, results) => { - if (err) { - return done(err); - } - let rowId = results[0].maxid || 10000; - let _auditdate = dwClient.auditdate; // results[0]._auditdate ? `'${results[0]._auditdate.replace(/^\d* */,"")}'` : "now()"; - let unionQuery = unions[table].join('\nUNION\n'); - transaction.query(`insert into ${table} (${sk}, ${nk}, ${columnConfig._auditdate}, ${columnConfig._startdate}, ${columnConfig._enddate}, ${columnConfig._current}) select row_number() over () + ${rowId}, sub.id, max(${_auditdate})::timestamp, '1900-01-01 00:00:00', '9999-01-01 00:00:00', true from (${unionQuery}) as sub where sub.id is not null group by sub.id`, (err) => { - done(err); + let missingDimTasks = Object.keys(unions).map(table => { + let sk = tableSks[table]; + let nk = tableNks[table][0]; + return (callback) => { + let done = (err, data) => { + trx && trx.release(); + callback(err, data); + }; + let trx; + client.connect().then(transaction => { + trx = transaction; + transaction.query(`select max(${sk}) as maxid from ${table}`, (err, results) => { + if (err) { + return done(err); + } + let rowId = results[0].maxid || 10000; + let _auditdate = dwClient.auditdate; + let unionQuery = unions[table].join('\nUNION\n'); + transaction.query(`insert into ${table} (${sk}, ${nk}, ${columnConfig._auditdate}, ${columnConfig._startdate}, ${columnConfig._enddate}, ${columnConfig._current}) select row_number() over() + ${rowId}, sub.id, max(${_auditdate})::timestamp, '1900-01-01 00:00:00', '9999-01-01 00:00:00', true from(${unionQuery}) as sub where sub.id is not null group by sub.id`, (err) => { + done(err); + }); }); - }); - }).catch(done); - }; - }); - async.parallelLimit(missingDimTasks, 10, (missingDimError) => { - logger.info(`Missing Dimensions ${!missingDimError && 'Inserted'} ----------------------------`, missingDimError || ''); - callback(missingDimError); - }); + }).catch(done); + }; + }); + async.parallelLimit(missingDimTasks, 10, (missingDimError) => { + logger.info(`Missing Dimensions ${!missingDimError && 'Inserted'} ----------------------------`, missingDimError || ''); + callback(missingDimError); + }); + }; }; - client.linkDimensions = function (table, links, nk, callback, tableStatus) { + client.linkDimensions = function(table, links, nk, callback, tableStatus) { client.describeTable(table).then(() => { let tasks = []; let sets = []; + const qualifiedTable = `public.${table}`; const linkAuditdate = client.escapeValueNoToLower(new Date().toISOString().replace(/\.\d*Z/, 'Z')); // Only run analyze on the table if this is the first load - if (tableStatus === 'First Load') { - tasks.push(done => client.query(`analyze ${table}`, done)); + if (tableStatus === 'First Load' && config.version !== 'redshift') { + tasks.push(done => client.query(`ANALYZE ${table}`, done)); } + + const qualifiedStagingTable = `${columnConfig.stageSchema}.${columnConfig.stageTablePrefix}_${table}`; + let naturalKeyLowerBound; + let naturalKeyFilter; + let sortKey; + let sortKeyType; + // Get lower bound for natural key to avoid unnecessary scanning + if (config.version === 'redshift') { + tasks.push(done => { + client.query(`SELECT sortkey, + sortkeytype + FROM public.v_dist_sort_key + WHERE table_name = '${table}';`, (err, results) => { + if (err) { + return done(err); + } else { + if (results[0].sortKey != null) { + sortKey = results[0].sortkey; + sortKeyType = results[0].sortkeytype; + }; + done(); + }; + }); + }); + tasks.push(done => { + client.query(`SELECT MIN(${(sortKey != null) ? sortKey : nk[0]}) AS minid + FROM ${qualifiedStagingTable}; `, (err, results) => { + if (err) { + return done(err); + } else { + naturalKeyLowerBound = results[0].minid; + if (naturalKeyLowerBound !== null) { + if (sortKeyType === 'int4' || sortKeyType === 'int8') { + naturalKeyFilter = `${results[0].minid}`; + } else if (sortKeyType === 'varchar') { + naturalKeyFilter = `'${results[0].minid}'`; + } else if (sortKeyType === 'timestamp' && Date.parse(results[0].minid.split(' ')[1])) { + naturalKeyFilter = `'${results[0].minid.split(' ')[1]}'`; + }; + }; + done(); + }; + }); + }); + }; + tasks.push(done => { let joinTables = links.map(link => { if (columnConfig.useSurrogateDateKeys && (link.table === 'd_datetime' || link.table === 'datetime' || link.table === 'dim_datetime')) { sets.push(`${link.destination}_date = coalesce(t.${link.source}::date - '1400-01-01'::date + 10000, 1)`); - sets.push(`${link.destination}_time = coalesce(EXTRACT(EPOCH from t.${link.source}::time) + 10000, 1)`); + sets.push(`${link.destination}_time = coalesce(EXTRACT(EPOCH FROM ${(config.version !== 'redshift') ? `` : `'1970-01-01'::date +`} t.${link.source}::time) + 10000, 1)`); } else if (columnConfig.useSurrogateDateKeys && (link.table === 'd_date' || link.table === 'date' || link.table === 'dim_date')) { sets.push(`${link.destination}_date = coalesce(t.${link.source}::date - '1400-01-01'::date + 10000, 1)`); } else if (columnConfig.useSurrogateDateKeys && (link.table === 'd_time' || link.table === 'time' || link.table === 'dim_time')) { - sets.push(`${link.destination}_time = coalesce(EXTRACT(EPOCH from t.${link.source}::time) + 10000, 1)`); + sets.push(`${link.destination}_time = coalesce(EXTRACT(EPOCH FROM ${(config.version !== 'redshift') ? `` : `'1970-01-01'::date +`} t.${link.source}::time) + 10000, 1)`); } else { - sets.push(`${link.destination} = coalesce(${link.join_id}_join_table.${link.sk}, 1)`); - var joinOn = `${link.join_id}_join_table.${link.on} = t.${link.source}`; - if (Array.isArray(link.source)) { - joinOn = link.source.map((v, i) => `${link.join_id}_join_table.${link.on[i]} = t.${v}`).join(' AND '); - } - return `LEFT JOIN ${link.table} ${link.join_id}_join_table - on ${joinOn} - and t.${link.link_date} >= ${link.join_id}_join_table.${columnConfig._startdate} - and (t.${link.link_date} <= ${link.join_id}_join_table.${columnConfig._enddate} or ${link.join_id}_join_table.${columnConfig._current})`; + if (config.hashedSurrogateKeys) { + sets.push(`${link.destination} = coalesce(farmFingerPrint64(t.${link.source}), 1)`); + return ``; + } else { + sets.push(`${link.destination} = coalesce(${link.join_id}_join_table.${link.sk}, 1)`); + var joinOn = `${link.join_id}_join_table.${link.on} = t.${link.source} `; + + if (Array.isArray(link.source)) { + joinOn = link.source.map((v, i) => `${link.join_id}_join_table.${link.on[i]} = t.${v}`).join(' AND '); + } + return `LEFT JOIN ${link.table} ${link.join_id}_join_table + ON ${joinOn} + AND t.${link.link_date} >= ${link.join_id}_join_table.${columnConfig._startdate} + AND (t.${link.link_date} <= ${link.join_id}_join_table.${columnConfig._enddate} or ${link.join_id}_join_table.${columnConfig._current})`; + }; } }); if (sets.length) { - client.query(`Update ${table} dm - SET ${sets.join(', ')}, ${columnConfig._auditdate} = ${linkAuditdate} - FROM ${table} t - ${joinTables.join('\n')} - where ${nk.map(id => `dm.${id} = t.${id}`).join(' and ')} - AND dm.${columnConfig._auditdate} = ${dwClient.auditdate} AND t.${columnConfig._auditdate} = ${dwClient.auditdate} - `, done); + client.query(`UPDATE ${table} dm + SET ${sets.join(', ')}, ${columnConfig._auditdate} = ${linkAuditdate} + FROM ${table} t + ${config.hashedSurrogateKeys ? '' : joinTables.join('\n')} + WHERE ${nk.map(id => `dm.${id} = t.${id}`).join(' AND ')} + AND dm.${columnConfig._auditdate} = ${dwClient.auditdate} AND t.${columnConfig._auditdate} = ${dwClient.auditdate} + ${(naturalKeyFilter !== undefined) ? `AND t.${(sortKey != null) ? sortKey : nk[0]} >= ${naturalKeyFilter}` : ``}`, done); } else { done(); } @@ -493,7 +845,7 @@ module.exports = function (config, columnConfig) { }); }; - client.changeTableStructure = async function (structures) { + client.changeTableStructure = async function(structures) { let tasks = []; let tableResults = {}; @@ -518,7 +870,7 @@ module.exports = function (config, columnConfig) { } else if (field.dimension && !(columnConfig.dimColumnTransform(f, field) in fieldLookup)) { let missingDim = columnConfig.dimColumnTransform(f, field); missingFields[missingDim] = { - type: 'integer', + type: (config.hashedSurrogateKeys) ? 'bigint' : 'integer', }; } }); @@ -560,7 +912,7 @@ module.exports = function (config, columnConfig) { }); }; - client.createTable = async function (table, definition) { + client.createTable = async function(table, definition) { let fields = []; let defaults = []; let dbType = (config.type || '').toLowerCase(); @@ -576,7 +928,7 @@ module.exports = function (config, columnConfig) { if (field === 'sk') { field = { sk: true, - type: 'integer primary key', + type: `${(config.hashedSurrogateKeys) ? 'bigint' : 'integer'} primary key`, }; } else if (typeof field === 'string') { field = { @@ -625,7 +977,7 @@ module.exports = function (config, columnConfig) { }); } } else if (field.dimension) { - fields.push(`${columnConfig.dimColumnTransform(key, field)} integer`); + fields.push(`${columnConfig.dimColumnTransform(key, field)} ${(config.hashedSurrogateKeys) ? 'bigint' : 'integer'}`); defaults.push({ column: columnConfig.dimColumnTransform(key, field), value: 1, @@ -639,7 +991,7 @@ module.exports = function (config, columnConfig) { }); let sql = `create table ${table} ( - ${fields.join(',\n')} + ${fields.join(',\n')} )`; /* @@ -663,7 +1015,7 @@ module.exports = function (config, columnConfig) { // Add empty row to new dim defaults = defaults.concat([{ column: columnConfig._auditdate, - value: 'now()', + value: config.version === 'redshift' ? 'sysdate' : 'now()', }, { column: columnConfig._startdate, value: client.escapeValueNoToLower('1900-01-01 00:00:00'), @@ -698,14 +1050,14 @@ module.exports = function (config, columnConfig) { }); }); }; - client.updateTable = async function (table, definition) { + client.updateTable = async function(table, definition) { let fields = []; let queries = []; Object.keys(definition).forEach(key => { let field = definition[key]; if (field === 'sk') { field = { - type: 'integer primary key', + type: `${(config.hashedSurrogateKeys) ? 'bigint' : 'integer'} primary key`, }; } else if (typeof field === 'string') { field = { @@ -741,7 +1093,7 @@ module.exports = function (config, columnConfig) { }); let sqls = [`alter table ${table} add column ${fields.join(',\n add column ')} - `]; + `]; // redshift doesn't support multi 'add column' in one query if (config.version === 'redshift') { @@ -753,7 +1105,7 @@ module.exports = function (config, columnConfig) { }); return new Promise(resolve => { - async.eachSeries(sqls, function (sql, done) { + async.eachSeries(sqls, function(sql, done) { client.query(sql, err => done(err)); }, err => { if (err) { @@ -765,7 +1117,7 @@ module.exports = function (config, columnConfig) { }); }; - client.findAuditDate = function (table, callback) { + client.findAuditDate = function(table, callback) { client.query(`select to_char(max(${columnConfig._auditdate}), 'YYYY-MM-DD HH24:MI:SS') as max FROM ${client.escapeId(table)}`, (err, auditdate) => { if (err) { callback(err); @@ -782,7 +1134,7 @@ module.exports = function (config, columnConfig) { }); }; - client.exportChanges = function (table, fields, remoteAuditdate, opts, callback) { + client.exportChanges = function(table, fields, remoteAuditdate, opts, callback) { let auditdateCompare = remoteAuditdate.auditdate != null ? `${columnConfig._auditdate} >= ${client.escapeValue(remoteAuditdate.auditdate)}` : `${columnConfig._auditdate} is null`; client.query(`select count(*) as count FROM ${client.escapeId(table)} WHERE ${auditdateCompare}`, (err, result) => { if (err) { @@ -801,7 +1153,7 @@ module.exports = function (config, columnConfig) { client.query(`select to_char(min(${columnConfig._auditdate}), 'YYYY-MM-DD HH24:MI:SS') as oldest, count(*) as count FROM ${client.escapeId(table)} ${where} - `, (err, result) => { + `, (err, result) => { if (err) { callback(err); } @@ -850,7 +1202,7 @@ module.exports = function (config, columnConfig) { }); }; - client.importChanges = function (file, table, fields, opts, callback) { + client.importChanges = function(file, table, fields, opts, callback) { if (typeof opts === 'function') { callback = opts; opts = {}; @@ -911,16 +1263,16 @@ module.exports = function (config, columnConfig) { } }); } - tasks.push(function (done) { + tasks.push(function(done) { client.query(`insert into ${tableName} select * from ${qualifiedStagingTable}`, done); }); - tasks.push(function (done) { + tasks.push(function(done) { client.query(`select count(*) from ${qualifiedStagingTable}`, (err, result) => { loadCount = result && parseInt(result[0].count); done(err); }); }); - tasks.push(function (done) { + tasks.push(function(done) { client.query(`drop table if exists ${qualifiedStagingTable}`, done); }); async.series(tasks, (err) => { diff --git a/postgres/package-lock.json b/postgres/package-lock.json index 07f305be..6e3787b2 100644 --- a/postgres/package-lock.json +++ b/postgres/package-lock.json @@ -1,39 +1,114 @@ { "name": "leo-connector-postgres", - "version": "4.0.7", + "version": "4.0.24-rc", "lockfileVersion": 1, "requires": true, "dependencies": { "@babel/code-frame": { - "version": "7.8.3", - "resolved": "https://registry.npmjs.org/@babel/code-frame/-/code-frame-7.8.3.tgz", - "integrity": "sha512-a9gxpmdXtZEInkCSHUJDLHZVBgb1QS0jhss4cPP93EW7s+uC5bikET2twEF3KV+7rDblJcmNvTR7VJejqd2C2g==", + "version": "7.18.6", + "resolved": "https://registry.npmjs.org/@babel/code-frame/-/code-frame-7.18.6.tgz", + "integrity": "sha512-TDCmlK5eOvH+eH7cdAFlNXeVJqWIQ7gW9tY1GJIpUtFb6CmjVyq2VM3u71bOyR8CRihcCgMUYoDNyLXao3+70Q==", "dev": true, "requires": { - "@babel/highlight": "^7.8.3" + "@babel/highlight": "^7.18.6" } }, "@babel/helper-validator-identifier": { - "version": "7.9.0", - "resolved": "https://registry.npmjs.org/@babel/helper-validator-identifier/-/helper-validator-identifier-7.9.0.tgz", - "integrity": "sha512-6G8bQKjOh+of4PV/ThDm/rRqlU7+IGoJuofpagU5GlEl29Vv0RGqqt86ZGRV8ZuSOY3o+8yXl5y782SMcG7SHw==", + "version": "7.19.1", + "resolved": "https://registry.npmjs.org/@babel/helper-validator-identifier/-/helper-validator-identifier-7.19.1.tgz", + "integrity": "sha512-awrNfaMtnHUr653GgGEs++LlAvW6w+DcPrOliSMXWCKo597CwL5Acf/wWdNkf/tfEQE3mjkeD1YOVZOUV/od1w==", "dev": true }, "@babel/highlight": { - "version": "7.9.0", - "resolved": "https://registry.npmjs.org/@babel/highlight/-/highlight-7.9.0.tgz", - "integrity": "sha512-lJZPilxX7Op3Nv/2cvFdnlepPXDxi29wxteT57Q965oc5R9v86ztx0jfxVrTcBk8C2kcPkkDa2Z4T3ZsPPVWsQ==", + "version": "7.18.6", + "resolved": "https://registry.npmjs.org/@babel/highlight/-/highlight-7.18.6.tgz", + "integrity": "sha512-u7stbOuYjaPezCuLj29hNW1v64M2Md2qupEKP1fHc7WdOA3DgLh37suiSrZYY7haUB7iBeQZ9P1uiRF359do3g==", "dev": true, "requires": { - "@babel/helper-validator-identifier": "^7.9.0", + "@babel/helper-validator-identifier": "^7.18.6", "chalk": "^2.0.0", "js-tokens": "^4.0.0" } }, + "@dsco/layer": { + "version": "1.0.10", + "resolved": "https://registry.npmjs.org/@dsco/layer/-/layer-1.0.10.tgz", + "integrity": "sha512-WKeCqbggSSc9uMi+LH/LRk9ibv6FNf0npF8Q7BNKwdSD/f8xbi3FvnNJKuWUvJvK1j/iYJb36ajC+EUWU27gdQ==", + "dev": true + }, + "@dsco/layer-leo": { + "version": "2.1.29", + "resolved": "https://registry.npmjs.org/@dsco/layer-leo/-/layer-leo-2.1.29.tgz", + "integrity": "sha512-eFGVSCsrlUVWA8Mh9x43KjxAC/HZOj9+5lXMgS/NGQBfn1sbO2OsUDg2IHYNMHBba5Lql5RwUYqqL6faMjqYJQ==", + "dev": true, + "requires": { + "@dsco/layer": "1.0.10", + "leo-auth": "3.0.1", + "leo-aws": "2.0.2-beta-x", + "leo-cache": "1.0.1", + "leo-config": "1.1.0", + "leo-connector-common": "4.0.8-rc", + "leo-connector-entity-table": "3.0.7-rc", + "leo-logger": "1.0.1", + "leo-sdk": "5.0.16", + "leo-streams": "2.0.0", + "moment": "2.24.0", + "moment-business-days": "1.1.3", + "moment-timezone": "0.5.26" + }, + "dependencies": { + "leo-aws": { + "version": "2.0.2-beta-x", + "resolved": "https://registry.npmjs.org/leo-aws/-/leo-aws-2.0.2-beta-x.tgz", + "integrity": "sha512-vHMAuuRMr8pPEqFLAQDgr6SmdJG8ncE7YTMqzyH7AsAUkQluj1fjjZ+W7a0CsmLJz5UARhSVyFc3zCiqGcrgiw==", + "dev": true, + "requires": { + "async": "^2.6.1", + "backoff": "^2.5.0", + "leo-config": "1.1.0", + "leo-logger": "1.0.1", + "leo-streams": "2.0.0", + "lodash": "^4.17.14", + "lodash.merge": "^4.6.2" + } + }, + "leo-connector-common": { + "version": "4.0.8-rc", + "resolved": "https://registry.npmjs.org/leo-connector-common/-/leo-connector-common-4.0.8-rc.tgz", + "integrity": "sha512-f9uJVkgDtI+EqJCF/c9BqP/236NlflId0uK3Wq0LwNNISESakDXDyxemb9oTgvaMfXWW7VOHFOSylthdFSLgmg==", + "dev": true, + "requires": { + "async": "2.6.3", + "leo-aws": "2.0.1", + "leo-logger": "1.0.1", + "leo-streams": "2.0.0", + "moment": "2.24.0", + "sqlstring": "2.3.1" + }, + "dependencies": { + "leo-aws": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/leo-aws/-/leo-aws-2.0.1.tgz", + "integrity": "sha512-40vq8BQklV96sZvN7fDRRmpKmY7p1CyZLIh1G4IkkokScw2QrqkhxOSs5H+A6xb7HBnt+IUvpgOXhwYe0+uOcQ==", + "dev": true, + "requires": { + "async": "^2.6.1", + "backoff": "^2.5.0", + "leo-config": "1.1.0", + "leo-logger": "1.0.1", + "leo-streams": "2.0.0", + "lodash": "^4.17.14", + "lodash.merge": "^4.6.2" + } + } + } + } + } + }, "@sinonjs/commons": { - "version": "1.7.1", - "resolved": "https://registry.npmjs.org/@sinonjs/commons/-/commons-1.7.1.tgz", - "integrity": "sha512-Debi3Baff1Qu1Unc3mjJ96MgpbwTn43S1+9yJ0llWygPwDNu2aaWBD6yc9y/Z8XDRNhx7U+u2UDg2OGQXkclUQ==", + "version": "1.8.3", + "resolved": "https://registry.npmjs.org/@sinonjs/commons/-/commons-1.8.3.tgz", + "integrity": "sha512-xkNcLAn/wZaX14RPlwizcKicDk9G3F8m2nU3L7Ukm5zBgTwiT0wsoFAHx9Jq56fJA1z/7uKGtCRu16sOUCLIHQ==", "dev": true, "requires": { "type-detect": "4.0.8" @@ -61,27 +136,33 @@ } }, "@sinonjs/text-encoding": { - "version": "0.7.1", - "resolved": "https://registry.npmjs.org/@sinonjs/text-encoding/-/text-encoding-0.7.1.tgz", - "integrity": "sha512-+iTbntw2IZPb/anVDbypzfQa+ay64MW0Zo8aJ8gZPWMMK6/OubMVb6lUPMagqjOPnmtauXnFCACVl3O7ogjeqQ==", + "version": "0.7.2", + "resolved": "https://registry.npmjs.org/@sinonjs/text-encoding/-/text-encoding-0.7.2.tgz", + "integrity": "sha512-sXXKG+uL9IrKqViTtao2Ws6dy0znu9sOaP1di/jKGW1M6VssO8vlpXCQcpZ+jisQ1tTFAC5Jo/EOzFbggBagFQ==", + "dev": true + }, + "abbrev": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/abbrev/-/abbrev-1.1.1.tgz", + "integrity": "sha512-nne9/IiQ/hzIhY6pdDnbBtz7DjPTKrY00P/zvPSm5pOFkl6xuGrGnXn/VtTNNfNtAfZ9/1RtehkszU9qcTii0Q==", "dev": true }, "acorn": { - "version": "6.4.1", - "resolved": "https://registry.npmjs.org/acorn/-/acorn-6.4.1.tgz", - "integrity": "sha512-ZVA9k326Nwrj3Cj9jlh3wGFutC2ZornPNARZwsNYqQYgN0EsV2d53w5RN/co65Ohn4sUAUtb1rSUAOD6XN9idA==", + "version": "6.4.2", + "resolved": "https://registry.npmjs.org/acorn/-/acorn-6.4.2.tgz", + "integrity": "sha512-XtGIhXwF8YM8bJhGxG5kXgjkEuNGLTkoYqVE+KMR+aspr4KGYmKYg7yUe3KghyQ9yheNwLnjmzh/7+gfDBmHCQ==", "dev": true }, "acorn-jsx": { - "version": "5.2.0", - "resolved": "https://registry.npmjs.org/acorn-jsx/-/acorn-jsx-5.2.0.tgz", - "integrity": "sha512-HiUX/+K2YpkpJ+SzBffkM/AQ2YE03S0U1kjTLVpoJdhZMOWy8qvXVN9JdLqv2QsaQ6MPYQIuNmwD8zOiYUofLQ==", + "version": "5.3.2", + "resolved": "https://registry.npmjs.org/acorn-jsx/-/acorn-jsx-5.3.2.tgz", + "integrity": "sha512-rq9s+JNhf0IChjtDXxllJ7g41oZk5SlXtp0LHwyA5cejwn7vKmKp4pPri6YEePv2PU65sAsegbXtIinmDFDXgQ==", "dev": true }, "ajv": { - "version": "6.12.0", - "resolved": "https://registry.npmjs.org/ajv/-/ajv-6.12.0.tgz", - "integrity": "sha512-D6gFiFA0RRLyUbvijN74DWAjXSFxWKaWP7mldxkVhyhAV3+SWA9HEJPHQ2c9soIeTFJqcSdFDGFgdqs1iUU2Hw==", + "version": "6.12.6", + "resolved": "https://registry.npmjs.org/ajv/-/ajv-6.12.6.tgz", + "integrity": "sha512-j3fVLgvTo527anyYyJOGTYJbG+vnnQYvE0m5mmkc1TK+nxAppkCLMIL0aZ4dblVCNoGShhm+kzE4ZUykBoMg4g==", "dev": true, "requires": { "fast-deep-equal": "^3.1.1", @@ -97,9 +178,9 @@ "dev": true }, "ansi-regex": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-3.0.0.tgz", - "integrity": "sha1-7QMXwyIGT3lGbAKWa922Bas32Zg=", + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-3.0.1.tgz", + "integrity": "sha512-+O9Jct8wf++lXxxFc4hc8LsjaSq0HFzzL7cVsw8pRDIPdjKD2mT4ytDZlLuSBZ4cLKZFXIrMGO7DbQCtMJJMKw==", "dev": true }, "ansi-styles": { @@ -123,7 +204,7 @@ "arguments-extended": { "version": "0.0.3", "resolved": "https://registry.npmjs.org/arguments-extended/-/arguments-extended-0.0.3.tgz", - "integrity": "sha1-YQfkkX0OtvCk3WYyD8Fa/HLvSUY=", + "integrity": "sha512-MNYdPKgCiywbgHAmNsYr1tSNLtfbSdwE1akZV+33hU9A8RG0lO5HAK9oMnw7y7bjYUhc04dJpcIBMUaPPYYtXg==", "requires": { "extended": "~0.0.3", "is-extended": "~0.0.8" @@ -132,7 +213,7 @@ "array-extended": { "version": "0.0.11", "resolved": "https://registry.npmjs.org/array-extended/-/array-extended-0.0.11.tgz", - "integrity": "sha1-1xRK50jek8pybxIQCdv/FibRZL0=", + "integrity": "sha512-Fe4Ti2YgM1onQgrcCD8dUhFuZgHQxzqylSl1C5IDJVVVqY5D07h8RghIXL9sZ6COZ0e+oTL5IusTv5eXABJ9Kw==", "requires": { "arguments-extended": "~0.0.3", "extended": "~0.0.3", @@ -142,7 +223,7 @@ "array-from": { "version": "2.1.1", "resolved": "https://registry.npmjs.org/array-from/-/array-from-2.1.1.tgz", - "integrity": "sha1-z+nYwmYoudxa7MYqn12PHzUsEZU=", + "integrity": "sha512-GQTc6Uupx1FCavi5mPzBvVT7nEOeWMmUA9P95wpfpW1XwMSKs+KaymD5C2Up7KAUKg/mYwbsUYzdZWcoajlNZg==", "dev": true }, "assertion-error": { @@ -174,9 +255,9 @@ } }, "balanced-match": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/balanced-match/-/balanced-match-1.0.0.tgz", - "integrity": "sha1-ibTRmasr7kneFk6gK4nORi1xt2c=", + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/balanced-match/-/balanced-match-1.0.2.tgz", + "integrity": "sha512-3oSeUO0TMV67hN1AmbXsK4yaqU7tjiHlbxRDZOpH0KW9+CeX4bRAaX0Anxt0tx2MrpRpWwQaPwIlISEJhYU5Pw==", "dev": true }, "brace-expansion": { @@ -207,16 +288,17 @@ "dev": true }, "chai": { - "version": "4.2.0", - "resolved": "https://registry.npmjs.org/chai/-/chai-4.2.0.tgz", - "integrity": "sha512-XQU3bhBukrOsQCuwZndwGcCVQHyZi53fQ6Ys1Fym7E4olpIqqZZhhoFJoaKVvV17lWQoXYwgWN2nF5crA8J2jw==", + "version": "4.3.6", + "resolved": "https://registry.npmjs.org/chai/-/chai-4.3.6.tgz", + "integrity": "sha512-bbcp3YfHCUzMOvKqsztczerVgBKSsEijCySNlHHbX3VG1nskvqjz5Rfso1gGwD6w6oOV3eI60pKuMOV5MV7p3Q==", "dev": true, "requires": { "assertion-error": "^1.1.0", "check-error": "^1.0.2", "deep-eql": "^3.0.1", "get-func-name": "^2.0.0", - "pathval": "^1.1.0", + "loupe": "^2.3.1", + "pathval": "^1.1.1", "type-detect": "^4.0.5" } }, @@ -240,22 +322,22 @@ "check-error": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/check-error/-/check-error-1.0.2.tgz", - "integrity": "sha1-V00xLt2Iu13YkS6Sht1sCu1KrII=", + "integrity": "sha512-BrgHpW9NURQgzoNyjfq0Wu6VFO6D7IZEmJNdtgNqpzGG8RuNFHt2jQxWlAs4HMe119chBnv+34syEZtc6IhLtA==", "dev": true }, "cli-cursor": { "version": "2.1.0", "resolved": "https://registry.npmjs.org/cli-cursor/-/cli-cursor-2.1.0.tgz", - "integrity": "sha1-s12sN2R5+sw+lHR9QdDQ9SOP/LU=", + "integrity": "sha512-8lgKz8LmCRYZZQDpRyT2m5rKJ08TnU4tR9FFFW2rxpxR1FzWi4PQ/NfyODchAatHaUgnSPVcx/R5w6NuTBzFiw==", "dev": true, "requires": { "restore-cursor": "^2.0.0" } }, "cli-width": { - "version": "2.2.0", - "resolved": "https://registry.npmjs.org/cli-width/-/cli-width-2.2.0.tgz", - "integrity": "sha1-/xnt6Kml5XkyQUewwR8PvLq+1jk=", + "version": "2.2.1", + "resolved": "https://registry.npmjs.org/cli-width/-/cli-width-2.2.1.tgz", + "integrity": "sha512-GRMWDxpOB6Dgk2E5Uo+3eEBvtOOlimMmpbFiKuLFnQzYDavtLFY3K5ona41jgN/WdRZtG7utuVSVTL4HbZHGkw==", "dev": true }, "color-convert": { @@ -270,7 +352,7 @@ "color-name": { "version": "1.1.3", "resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.3.tgz", - "integrity": "sha1-p9BVi9icQveV3UIyj3QIMcpTvCU=", + "integrity": "sha512-72fSenhMw2HZMTVHeCA9KCmpEIbzWiQsjN+BHcBbS9vr1mtt+vJjPdksIBNUmKAW8TFUDPJK5SUU3QhE9NEXDw==", "dev": true }, "commander": { @@ -282,9 +364,19 @@ "concat-map": { "version": "0.0.1", "resolved": "https://registry.npmjs.org/concat-map/-/concat-map-0.0.1.tgz", - "integrity": "sha1-2Klr13/Wjfd5OnMDajug1UBdR3s=", + "integrity": "sha512-/Srv4dswyQNBfohGpz9o6Yb3Gz3SrUDqBH5rTuhGR7ahtlbYKnVxw2bCFMRljaA7EXHaXZ8wsHdodFvbkhKmqg==", "dev": true }, + "config-chain": { + "version": "1.1.13", + "resolved": "https://registry.npmjs.org/config-chain/-/config-chain-1.1.13.tgz", + "integrity": "sha512-qj+f8APARXHrM0hraqXYb2/bOVSV4PvJQlNZ/DVj0QrmNM2q2euizkeuVckQ57J+W0mRH6Hvi+k50M4Jul2VRQ==", + "dev": true, + "requires": { + "ini": "^1.3.4", + "proto-list": "~1.2.1" + } + }, "core-util-is": { "version": "1.0.3", "resolved": "https://registry.npmjs.org/core-util-is/-/core-util-is-1.0.3.tgz", @@ -314,7 +406,7 @@ "date-extended": { "version": "0.0.6", "resolved": "https://registry.npmjs.org/date-extended/-/date-extended-0.0.6.tgz", - "integrity": "sha1-I4AtV90b94GIE/4MMuhRqG2iZ8k=", + "integrity": "sha512-v9a2QLTVn1GQGXf02TQaSvNfeXA/V1FL2Tr0OQYqjI5+L9T5jEtCpLYG01sxFk+m1OtwMxydkKa8NKcflANAoQ==", "requires": { "array-extended": "~0.0.3", "extended": "~0.0.3", @@ -322,18 +414,18 @@ } }, "debug": { - "version": "4.1.1", - "resolved": "https://registry.npmjs.org/debug/-/debug-4.1.1.tgz", - "integrity": "sha512-pYAIzeRo8J6KPEaJ0VWOh5Pzkbw/RetuzehGM7QRRX5he4fPHx2rdKMB256ehJCkX+XRQm16eZLqLNS8RSZXZw==", + "version": "4.3.4", + "resolved": "https://registry.npmjs.org/debug/-/debug-4.3.4.tgz", + "integrity": "sha512-PRWFHuSU3eDtQJPvnNY7Jcket1j0t5OuOsFzPPzsekD52Zl8qUfFIPEiswXqIvHWGVHOgX+7G/vCNNhehwxfkQ==", "dev": true, "requires": { - "ms": "^2.1.1" + "ms": "2.1.2" } }, "declare.js": { "version": "0.0.8", "resolved": "https://registry.npmjs.org/declare.js/-/declare.js-0.0.8.tgz", - "integrity": "sha1-BHit/5VkwAT1Hfc9i8E0AZ0o3N4=" + "integrity": "sha512-O659hy1gcHef7JnwtqdQlrj2c5DAEgtxm8pgFXofW7eUE1L4FjsSLlziovWcrOJAOFlEPaOJshY+0hBWCG/AnA==" }, "deep-eql": { "version": "3.0.1", @@ -345,9 +437,9 @@ } }, "deep-is": { - "version": "0.1.3", - "resolved": "https://registry.npmjs.org/deep-is/-/deep-is-0.1.3.tgz", - "integrity": "sha1-s2nW+128E+7PUk+RsHD+7cNXzzQ=", + "version": "0.1.4", + "resolved": "https://registry.npmjs.org/deep-is/-/deep-is-0.1.4.tgz", + "integrity": "sha512-oIPzksmTg4/MriiaYGO+okXDT7ztn/w3Eptv/+gSIdMdKsJo0u4CfYNFJPy+4SKMuCqGw2wxnA+URMg3t8a/bQ==", "dev": true }, "diff": { @@ -365,6 +457,12 @@ "esutils": "^2.0.2" } }, + "duplexer": { + "version": "0.1.2", + "resolved": "https://registry.npmjs.org/duplexer/-/duplexer-0.1.2.tgz", + "integrity": "sha512-jtD6YG370ZCIi/9GTaJKQxWTZD045+4R4hTk/x1UyoqadyJ9x9CgSi1RlVDQF8U2sxLLSnFkCaMihqljHIWgMg==", + "dev": true + }, "duplexify": { "version": "3.7.1", "resolved": "https://registry.npmjs.org/duplexify/-/duplexify-3.7.1.tgz", @@ -376,6 +474,32 @@ "stream-shift": "^1.0.0" } }, + "editorconfig": { + "version": "0.15.3", + "resolved": "https://registry.npmjs.org/editorconfig/-/editorconfig-0.15.3.tgz", + "integrity": "sha512-M9wIMFx96vq0R4F+gRpY3o2exzb8hEj/n9S8unZtHSvYjibBp/iMufSzvmOcV/laG0ZtuTVGtiJggPOSW2r93g==", + "dev": true, + "requires": { + "commander": "^2.19.0", + "lru-cache": "^4.1.5", + "semver": "^5.6.0", + "sigmund": "^1.0.1" + }, + "dependencies": { + "commander": { + "version": "2.20.3", + "resolved": "https://registry.npmjs.org/commander/-/commander-2.20.3.tgz", + "integrity": "sha512-GpVkmM8vF2vQUkj2LvZmD35JxeJOLCwJ9cUkugyk2nuhbv3+mJvpLYYt+0+USMxE+oj+ey/lJEnhZw75x/OMcQ==", + "dev": true + }, + "semver": { + "version": "5.7.1", + "resolved": "https://registry.npmjs.org/semver/-/semver-5.7.1.tgz", + "integrity": "sha512-sauaDf/PZdVgrLTNYHRtpXa1iRiKcaebiKQ1BJdpQlWH2lCvexQdX55snPFyK7QzpudqbCI0qXFfOasHdyNDGQ==", + "dev": true + } + } + }, "emoji-regex": { "version": "7.0.3", "resolved": "https://registry.npmjs.org/emoji-regex/-/emoji-regex-7.0.3.tgz", @@ -393,7 +517,7 @@ "escape-string-regexp": { "version": "1.0.5", "resolved": "https://registry.npmjs.org/escape-string-regexp/-/escape-string-regexp-1.0.5.tgz", - "integrity": "sha1-G2HAViGQqN/2rjuyzwIAyhMLhtQ=", + "integrity": "sha512-vbRorB5FUQWvla16U8R/qgaFIya2qGzwDrNmCZuYKrbdSUMG6I1ZCGQRefkRVhuOkIGVne7BQ35DSfo1qvJqFg==", "dev": true }, "eslint": { @@ -468,9 +592,9 @@ } }, "eslint-visitor-keys": { - "version": "1.1.0", - "resolved": "https://registry.npmjs.org/eslint-visitor-keys/-/eslint-visitor-keys-1.1.0.tgz", - "integrity": "sha512-8y9YjtM1JBJU/A9Kc+SbaOV4y29sSWckBwMHa+FGtVj5gN/sbnKDf6xJUl+8g7FAij9LVaP8C24DUiH/f/2Z9A==", + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/eslint-visitor-keys/-/eslint-visitor-keys-1.3.0.tgz", + "integrity": "sha512-6J72N8UNa462wa/KFODt/PJ3IU60SDpC3QXC1Hjc1BXXpfL2C9R5+AU7jhe0F6GREqVMh4Juu+NY7xn+6dipUQ==", "dev": true }, "espree": { @@ -491,29 +615,37 @@ "dev": true }, "esquery": { - "version": "1.2.0", - "resolved": "https://registry.npmjs.org/esquery/-/esquery-1.2.0.tgz", - "integrity": "sha512-weltsSqdeWIX9G2qQZz7KlTRJdkkOCTPgLYJUz1Hacf48R4YOwGPHO3+ORfWedqJKbq5WQmsgK90n+pFLIKt/Q==", + "version": "1.4.0", + "resolved": "https://registry.npmjs.org/esquery/-/esquery-1.4.0.tgz", + "integrity": "sha512-cCDispWt5vHHtwMY2YrAQ4ibFkAL8RbH5YGBnZBc90MolvvfkkQcJro/aZiAQUlQ3qgrYS6D6v8Gc5G5CQsc9w==", "dev": true, "requires": { - "estraverse": "^5.0.0" + "estraverse": "^5.1.0" }, "dependencies": { "estraverse": { - "version": "5.0.0", - "resolved": "https://registry.npmjs.org/estraverse/-/estraverse-5.0.0.tgz", - "integrity": "sha512-j3acdrMzqrxmJTNj5dbr1YbjacrYgAxVMeF0gK16E3j494mOe7xygM/ZLIguEQ0ETwAg2hlJCtHRGav+y0Ny5A==", + "version": "5.3.0", + "resolved": "https://registry.npmjs.org/estraverse/-/estraverse-5.3.0.tgz", + "integrity": "sha512-MMdARuVEQziNTeJD8DgMqmhwR11BRQ/cBP+pLtYdSTnf3MIO8fFeiINEbX36ZdNlfU/7A9f3gUw49B3oQsvwBA==", "dev": true } } }, "esrecurse": { - "version": "4.2.1", - "resolved": "https://registry.npmjs.org/esrecurse/-/esrecurse-4.2.1.tgz", - "integrity": "sha512-64RBB++fIOAXPw3P9cy89qfMlvZEXZkqqJkjqqXIvzP5ezRZjW+lPWjw35UX/3EhUPFYbg5ER4JYgDw4007/DQ==", + "version": "4.3.0", + "resolved": "https://registry.npmjs.org/esrecurse/-/esrecurse-4.3.0.tgz", + "integrity": "sha512-KmfKL3b6G+RXvP8N1vr3Tq1kL/oCFgn2NYXEtqP8/L3pKapUA4G8cFVaoF3SU323CD4XypR/ffioHmkti6/Tag==", "dev": true, "requires": { - "estraverse": "^4.1.0" + "estraverse": "^5.2.0" + }, + "dependencies": { + "estraverse": { + "version": "5.3.0", + "resolved": "https://registry.npmjs.org/estraverse/-/estraverse-5.3.0.tgz", + "integrity": "sha512-MMdARuVEQziNTeJD8DgMqmhwR11BRQ/cBP+pLtYdSTnf3MIO8fFeiINEbX36ZdNlfU/7A9f3gUw49B3oQsvwBA==", + "dev": true + } } }, "estraverse": { @@ -528,10 +660,31 @@ "integrity": "sha512-kVscqXk4OCp68SZ0dkgEKVi6/8ij300KBWTJq32P/dYeWTSwK41WyTxalN1eRmA5Z9UU/LX9D7FWSmV9SAYx6g==", "dev": true }, + "event-stream": { + "version": "4.0.1", + "resolved": "https://registry.npmjs.org/event-stream/-/event-stream-4.0.1.tgz", + "integrity": "sha512-qACXdu/9VHPBzcyhdOWR5/IahhGMf0roTeZJfzz077GwylcDd90yOHLouhmv7GJ5XzPi6ekaQWd8AvPP2nOvpA==", + "dev": true, + "requires": { + "duplexer": "^0.1.1", + "from": "^0.1.7", + "map-stream": "0.0.7", + "pause-stream": "^0.0.11", + "split": "^1.0.1", + "stream-combiner": "^0.2.2", + "through": "^2.3.8" + } + }, + "extend": { + "version": "3.0.2", + "resolved": "https://registry.npmjs.org/extend/-/extend-3.0.2.tgz", + "integrity": "sha512-fjquC59cD7CyW6urNXK0FBufkZcoiGG80wTuPujX590cB5Ttln20E2UB4S/WARVqhXffZl2LNgS+gQdPIIim/g==", + "dev": true + }, "extended": { "version": "0.0.6", "resolved": "https://registry.npmjs.org/extended/-/extended-0.0.6.tgz", - "integrity": "sha1-f7i/e52uOXWG5IVwrP1kLHjlBmk=", + "integrity": "sha512-rvAV3BDGsV1SYGzUOu7aO0k82quhfl0QAyZudYhAcTeIr1rPbBnyOhOlkCLwLpDfP7HyKAWAPNSjRb9p7lE3rg==", "requires": { "extender": "~0.0.5" } @@ -539,7 +692,7 @@ "extender": { "version": "0.0.10", "resolved": "https://registry.npmjs.org/extender/-/extender-0.0.10.tgz", - "integrity": "sha1-WJwHSCvmGhRgttgfnCSqZ+jzJM0=", + "integrity": "sha512-iPLUHZJaNW6RuOShQX33ZpewEUIlijFBcsXnKWyiYERKWPsFxfKgx8J0xRz29hKQWPFFPACgBW6cHM7Ke1pfaA==", "requires": { "declare.js": "~0.0.4" } @@ -558,7 +711,7 @@ "fast-csv": { "version": "2.4.1", "resolved": "https://registry.npmjs.org/fast-csv/-/fast-csv-2.4.1.tgz", - "integrity": "sha1-vX3SaDkfcpNntZRFuN0K0CaIGyY=", + "integrity": "sha512-DcNumesovr8oTHMdfVWN9hJeiMALO04c1iLExpxV2+yoTW9Jyv239di81HKL8IQwAqC+T1TYoQsd2XStFb+DhA==", "requires": { "extended": "0.0.6", "is-extended": "0.0.10", @@ -567,9 +720,9 @@ } }, "fast-deep-equal": { - "version": "3.1.1", - "resolved": "https://registry.npmjs.org/fast-deep-equal/-/fast-deep-equal-3.1.1.tgz", - "integrity": "sha512-8UEa58QDLauDNfpbrX55Q9jrGHThw2ZMdOky5Gl1CDtVeJDPVrG4Jxx1N8jw2gkWaff5UUuX1KJd+9zGe2B+ZA==", + "version": "3.1.3", + "resolved": "https://registry.npmjs.org/fast-deep-equal/-/fast-deep-equal-3.1.3.tgz", + "integrity": "sha512-f3qQ9oQy9j2AhBe/H9VC91wLmKBCCU/gDOnKNAYG5hswO7BLKj09Hc5HYNz9cGI++xlpDCIgDaitVs03ATR84Q==", "dev": true }, "fast-json-stable-stringify": { @@ -581,13 +734,13 @@ "fast-levenshtein": { "version": "2.0.6", "resolved": "https://registry.npmjs.org/fast-levenshtein/-/fast-levenshtein-2.0.6.tgz", - "integrity": "sha1-PYpcZog6FqMMqGQ+hR8Zuqd5eRc=", + "integrity": "sha512-DCXu6Ifhqcks7TZKY3Hxp3y6qphY5SJZmrWMDrKcERSOXWQdMhU9Ig/PYrzyw/ul9jOIyh0N4M0tbC5hodg8dw==", "dev": true }, "figures": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/figures/-/figures-2.0.0.tgz", - "integrity": "sha1-OrGi0qYsi/tDGgyUy3l6L84nyWI=", + "integrity": "sha512-Oa2M9atig69ZkfwiApY8F2Yy+tzMbazyvqv21R0NsSC8floSOC09BbT1ITWAdoMGQvJ/aZnR1KMwdx9tvHnTNA==", "dev": true, "requires": { "escape-string-regexp": "^1.0.5" @@ -614,39 +767,68 @@ } }, "flatted": { - "version": "2.0.1", - "resolved": "https://registry.npmjs.org/flatted/-/flatted-2.0.1.tgz", - "integrity": "sha512-a1hQMktqW9Nmqr5aktAux3JMNqaucxGcjtjWnZLHX7yyPCmlSV3M54nGYbqT8K+0GhF3NBgmJCc3ma+WOgX8Jg==", + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/flatted/-/flatted-2.0.2.tgz", + "integrity": "sha512-r5wGx7YeOwNWNlCA0wQ86zKyDLMQr+/RB8xy74M4hTphfmjlijTSSXGuH8rnvKZnfT9i+75zmd8jcKdMR4O6jA==", + "dev": true + }, + "flush-write-stream": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/flush-write-stream/-/flush-write-stream-2.0.0.tgz", + "integrity": "sha512-uXClqPxT4xW0lcdSBheb2ObVU+kuqUk3Jk64EwieirEXZx9XUrVwp/JuBfKAWaM4T5Td/VL7QLDWPXp/MvGm/g==", + "dev": true, + "requires": { + "inherits": "^2.0.3", + "readable-stream": "^3.1.1" + }, + "dependencies": { + "readable-stream": { + "version": "3.6.0", + "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-3.6.0.tgz", + "integrity": "sha512-BViHy7LKeTz4oNnkcLJ+lVSL6vpiFeX6/d3oSH8zCW7UxP2onchk+vTGB143xuFjHS3deTgkKoXXymXqymiIdA==", + "dev": true, + "requires": { + "inherits": "^2.0.3", + "string_decoder": "^1.1.1", + "util-deprecate": "^1.0.1" + } + } + } + }, + "from": { + "version": "0.1.7", + "resolved": "https://registry.npmjs.org/from/-/from-0.1.7.tgz", + "integrity": "sha512-twe20eF1OxVxp/ML/kq2p1uc6KvFK/+vs8WjEbeKmV2He22MKm7YF2ANIt+EOqhJ5L3K/SuuPhk0hWQDjOM23g==", "dev": true }, "fs.realpath": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/fs.realpath/-/fs.realpath-1.0.0.tgz", - "integrity": "sha1-FQStJSMVjKpA20onh8sBQRmU6k8=", + "integrity": "sha512-OO0pH2lK6a0hZnAdau5ItzHPI6pUlvI7jMVnxUQRtw4owF2wk8lOSabtGDCTP4Ggrg2MbGnWO9X8K1t4+fGMDw==", "dev": true }, "functional-red-black-tree": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/functional-red-black-tree/-/functional-red-black-tree-1.0.1.tgz", - "integrity": "sha1-GwqzvVU7Kg1jmdKcDj6gslIHgyc=", + "integrity": "sha512-dsKNQNdj6xA3T+QlADDA7mOSlX0qiMINjn0cgr+eGHGsbSHzTabcIogz2+p/iqP1Xs6EP/sS2SbqH+brGTbq0g==", "dev": true }, "get-func-name": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/get-func-name/-/get-func-name-2.0.0.tgz", - "integrity": "sha1-6td0q+5y4gQJQzoGY2YCPdaIekE=", + "integrity": "sha512-Hm0ixYtaSZ/V7C8FJrtZIuBBI+iSgL+1Aq82zSu8VQNB4S3Gk8e7Qs3VwBDJAhmRZcFqkl3tQu36g/Foh5I5ig==", "dev": true }, "glob": { - "version": "7.1.6", - "resolved": "https://registry.npmjs.org/glob/-/glob-7.1.6.tgz", - "integrity": "sha512-LwaxwyZ72Lk7vZINtNNrywX0ZuLyStrdDtabefZKAY5ZGJhVtgdznluResxNmPitE0SAO+O26sWTHeKSI2wMBA==", + "version": "7.2.3", + "resolved": "https://registry.npmjs.org/glob/-/glob-7.2.3.tgz", + "integrity": "sha512-nFR0zLpU2YCaRxwoCJvL6UvCH2JFyFVIvwTLsIf21AuHlMskA1hhTdk+LlYJtOlYt9v6dvszD2BGRqBL+iQK9Q==", "dev": true, "requires": { "fs.realpath": "^1.0.0", "inflight": "^1.0.4", "inherits": "2", - "minimatch": "^3.0.4", + "minimatch": "^3.1.1", "once": "^1.3.0", "path-is-absolute": "^1.0.0" } @@ -666,13 +848,13 @@ "has-flag": { "version": "3.0.0", "resolved": "https://registry.npmjs.org/has-flag/-/has-flag-3.0.0.tgz", - "integrity": "sha1-tdRU3CGZriJWmfNGfloH87lVuv0=", + "integrity": "sha512-sKJf1+ceQBr4SMkvQnBDNDtf4TXpVhVGateu0t918bl30FnbE2m4vNLX+VWe/dpjlb+HugGYzW7uQXH98HPEYw==", "dev": true }, "he": { "version": "1.1.1", "resolved": "https://registry.npmjs.org/he/-/he-1.1.1.tgz", - "integrity": "sha1-k0EP0hsAlzUVH4howvJx80J+I/0=", + "integrity": "sha512-z/GDPjlRMNOa2XJiB4em8wJpuuBfrFOlYKTZxtpkdr1uPdibHI8rYA3MY0KDObpVyaes0e/aunid/t88ZI2EKA==", "dev": true }, "iconv-lite": { @@ -691,9 +873,9 @@ "dev": true }, "import-fresh": { - "version": "3.2.1", - "resolved": "https://registry.npmjs.org/import-fresh/-/import-fresh-3.2.1.tgz", - "integrity": "sha512-6e1q1cnWP2RXD9/keSkxHScg508CdXqXWgWBaETNhyuBFz+kUZlKboh+ISK+bU++DmbHimVBrOz/zzPe0sZ3sQ==", + "version": "3.3.0", + "resolved": "https://registry.npmjs.org/import-fresh/-/import-fresh-3.3.0.tgz", + "integrity": "sha512-veYYhQa+D1QBKznvhUHxb8faxlrwUnxseDAbAp457E0wLNio2bOSKnjYDhMj+YiAq61xrMGhQk9iXVk5FzgQMw==", "dev": true, "requires": { "parent-module": "^1.0.0", @@ -703,13 +885,13 @@ "imurmurhash": { "version": "0.1.4", "resolved": "https://registry.npmjs.org/imurmurhash/-/imurmurhash-0.1.4.tgz", - "integrity": "sha1-khi5srkoojixPcT7a21XbyMUU+o=", + "integrity": "sha512-JmXMZ6wuvDmLiHEml9ykzqO6lwFbof0GG4IkcGaENdCRDDmMVnny7s5HsIgHCbaq0w2MyPhDqkhTUgS2LU2PHA==", "dev": true }, "inflight": { "version": "1.0.6", "resolved": "https://registry.npmjs.org/inflight/-/inflight-1.0.6.tgz", - "integrity": "sha1-Sb1jMdfQLQwJvJEKEHW6gWW1bfk=", + "integrity": "sha512-k92I/b08q4wvFscXCLvqfsHCrjrF7yiXsQuIVvVE7N82W3+aqpzuUdBbfhWcy/FZR3/4IgflMgKLOsvPDrGCJA==", "dev": true, "requires": { "once": "^1.3.0", @@ -721,6 +903,12 @@ "resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.4.tgz", "integrity": "sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ==" }, + "ini": { + "version": "1.3.8", + "resolved": "https://registry.npmjs.org/ini/-/ini-1.3.8.tgz", + "integrity": "sha512-JV/yugV2uzW5iMRSiZAyDtQd+nxtUnjeLt0acNdw98kKLrvuRVyB80tsREOE7yvGVgalhZ6RNXCmEHkUKBKxew==", + "dev": true + }, "inquirer": { "version": "6.5.2", "resolved": "https://registry.npmjs.org/inquirer/-/inquirer-6.5.2.tgz", @@ -743,9 +931,9 @@ }, "dependencies": { "ansi-regex": { - "version": "4.1.0", - "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-4.1.0.tgz", - "integrity": "sha512-1apePfXM1UOSqw0o9IiFAovVz9M5S1Dg+4TrDwfMewQ6p/rmMueb7tWZjQ1rx4Loy1ArBggoqGpfqqdI4rondg==", + "version": "4.1.1", + "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-4.1.1.tgz", + "integrity": "sha512-ILlv4k/3f6vfQ4OoP2AGvirOktlQ98ZEL1k9FaQjxa3L1abBgbuTDAdPOpvbGncC0BTVQrl+OM8xZGK6tWXt7g==", "dev": true }, "strip-ansi": { @@ -762,7 +950,7 @@ "is-extended": { "version": "0.0.10", "resolved": "https://registry.npmjs.org/is-extended/-/is-extended-0.0.10.tgz", - "integrity": "sha1-JE4UDfdbscmjEG9BL/GC+1NKbWI=", + "integrity": "sha512-qp+HR+L9QXbgFurvqiVgD+JiGyUboRgICNzCXmbiLtZBFVSNFbxRsI4q7Be9mCWTO5PoO1IxoWp5sl+j5b83FA==", "requires": { "extended": "~0.0.3" } @@ -770,13 +958,7 @@ "is-fullwidth-code-point": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/is-fullwidth-code-point/-/is-fullwidth-code-point-2.0.0.tgz", - "integrity": "sha1-o7MKXE8ZkYMWeqq5O+764937ZU8=", - "dev": true - }, - "is-promise": { - "version": "2.1.0", - "resolved": "https://registry.npmjs.org/is-promise/-/is-promise-2.1.0.tgz", - "integrity": "sha1-eaKp7OfwlugPNtKy87wWwf9L8/o=", + "integrity": "sha512-VHskAKYM8RfSFXwee5t5cbN5PZeq1Wrh6qd5bkyiXIf6UQcN6w/A0eXM9r6t8d+GYOh+o6ZhiEnb88LN/Y8m2w==", "dev": true }, "isarray": { @@ -787,9 +969,22 @@ "isexe": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/isexe/-/isexe-2.0.0.tgz", - "integrity": "sha1-6PvzdNxVb/iUehDcsFctYz8s+hA=", + "integrity": "sha512-RHxMLp9lnKHGHRng9QFhRCMbYAcVpn69smSGcq3f36xjgVVWThj4qqLbTLlq7Ssj8B+fIQ1EuCEGI2lKsyQeIw==", "dev": true }, + "js-beautify": { + "version": "1.10.2", + "resolved": "https://registry.npmjs.org/js-beautify/-/js-beautify-1.10.2.tgz", + "integrity": "sha512-ZtBYyNUYJIsBWERnQP0rPN9KjkrDfJcMjuVGcvXOUJrD1zmOGwhRwQ4msG+HJ+Ni/FA7+sRQEMYVzdTQDvnzvQ==", + "dev": true, + "requires": { + "config-chain": "^1.1.12", + "editorconfig": "^0.15.3", + "glob": "^7.1.3", + "mkdirp": "~0.5.1", + "nopt": "~4.0.1" + } + }, "js-tokens": { "version": "4.0.0", "resolved": "https://registry.npmjs.org/js-tokens/-/js-tokens-4.0.0.tgz", @@ -797,9 +992,9 @@ "dev": true }, "js-yaml": { - "version": "3.13.1", - "resolved": "https://registry.npmjs.org/js-yaml/-/js-yaml-3.13.1.tgz", - "integrity": "sha512-YfbcO7jXDdyj0DGxYVSlSeQNHbD7XPWvrVWeVUujrQEoZzWJIRrCPoyk6kL6IAjAG2IolMK4T0hNUe0HOUs5Jw==", + "version": "3.14.1", + "resolved": "https://registry.npmjs.org/js-yaml/-/js-yaml-3.14.1.tgz", + "integrity": "sha512-okMH7OXXJ7YrN9Ok3/SXrnu4iX9yOk+25nqX4imS2npuvTYDmo/QEZoqwZkYaIDk3jVvBOTOIEgEhaLOynBS9g==", "dev": true, "requires": { "argparse": "^1.0.7", @@ -815,15 +1010,43 @@ "json-stable-stringify-without-jsonify": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/json-stable-stringify-without-jsonify/-/json-stable-stringify-without-jsonify-1.0.1.tgz", - "integrity": "sha1-nbe1lJatPzz+8wp1FC0tkwrXJlE=", + "integrity": "sha512-Bdboy+l7tA3OGW6FjyFHWkP5LuByj1Tk33Ljyq0axyzdk9//JSi2u3fP1QSmd1KNwq6VOKYGlAu87CisVir6Pw==", "dev": true }, "just-extend": { - "version": "4.1.0", - "resolved": "https://registry.npmjs.org/just-extend/-/just-extend-4.1.0.tgz", - "integrity": "sha512-ApcjaOdVTJ7y4r08xI5wIqpvwS48Q0PBG4DJROcEkH1f8MdAiNFyFxz3xoL0LWAVwjrwPYZdVHHxhRHcx/uGLA==", + "version": "4.2.1", + "resolved": "https://registry.npmjs.org/just-extend/-/just-extend-4.2.1.tgz", + "integrity": "sha512-g3UB796vUFIY90VIv/WX3L2c8CS2MdWUww3CNrYmqza1Fg0DURc2K/O4YrnklBdQarSJ/y8JnJYDGc+1iumQjg==", "dev": true }, + "leo-auth": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/leo-auth/-/leo-auth-3.0.1.tgz", + "integrity": "sha512-l+vzMxH6PIzjRdJCRVvHX3P4JGAybe71LFz2fg5dQfOcvbqhbTnPMNbfIo4qUcfdHPeFG/ZrZ6wxQJVMzVopEA==", + "dev": true, + "requires": { + "leo-aws": "2.0.1", + "leo-config": "1.1.0", + "netmask": "^1.0.6" + }, + "dependencies": { + "leo-aws": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/leo-aws/-/leo-aws-2.0.1.tgz", + "integrity": "sha512-40vq8BQklV96sZvN7fDRRmpKmY7p1CyZLIh1G4IkkokScw2QrqkhxOSs5H+A6xb7HBnt+IUvpgOXhwYe0+uOcQ==", + "dev": true, + "requires": { + "async": "^2.6.1", + "backoff": "^2.5.0", + "leo-config": "1.1.0", + "leo-logger": "1.0.1", + "leo-streams": "2.0.0", + "lodash": "^4.17.14", + "lodash.merge": "^4.6.2" + } + } + } + }, "leo-aws": { "version": "2.0.3", "resolved": "https://registry.npmjs.org/leo-aws/-/leo-aws-2.0.3.tgz", @@ -838,6 +1061,12 @@ "lodash.merge": "^4.6.2" } }, + "leo-cache": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/leo-cache/-/leo-cache-1.0.1.tgz", + "integrity": "sha512-61QSKr+GrDIbcmDDtngm64a+ra0Xdqe/4wYKW5aP2DcmtVumaNV9rnfI5vUINeVnkwBcXMXNQBht2jw0yBOy7w==", + "dev": true + }, "leo-config": { "version": "1.1.0", "resolved": "https://registry.npmjs.org/leo-config/-/leo-config-1.1.0.tgz", @@ -847,9 +1076,9 @@ } }, "leo-connector-common": { - "version": "4.0.11-rc", - "resolved": "https://registry.npmjs.org/leo-connector-common/-/leo-connector-common-4.0.11-rc.tgz", - "integrity": "sha512-aDiG/OTYSZGOA/MTO1B+DXD7u8akTD8AOys2wJEf+ysJz1ivm4Yt5Jzr/0+hRGCvMOiSA6gxDRj7/F1bcx02rQ==", + "version": "4.0.13-rc", + "resolved": "https://registry.npmjs.org/leo-connector-common/-/leo-connector-common-4.0.13-rc.tgz", + "integrity": "sha512-RXBVMTBOY7oc9EdFCHYxg4GePeKTvhwsgbsohNJ3hpw9ognHkdX3rqsrnhcjXeJEt/BS9/f3jZt4Ra4dk+SdXQ==", "requires": { "async": "2.6.3", "leo-aws": "^2.0.3", @@ -859,6 +1088,36 @@ "sqlstring": "2.3.1" } }, + "leo-connector-entity-table": { + "version": "3.0.7-rc", + "resolved": "https://registry.npmjs.org/leo-connector-entity-table/-/leo-connector-entity-table-3.0.7-rc.tgz", + "integrity": "sha512-5Ws+LLynNTOePkB5bcwaqzRVVyBYgQo2eDx7+ZobP3CR2dq4SYGD6YtmVLAHxnNZLdvtuLaxfxsa3iWiTRbFbQ==", + "dev": true, + "requires": { + "async": "2.6.3", + "backoff": "2.5.0", + "js-beautify": "1.10.2", + "leo-aws": "2.0.1", + "leo-logger": "1.0.1" + }, + "dependencies": { + "leo-aws": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/leo-aws/-/leo-aws-2.0.1.tgz", + "integrity": "sha512-40vq8BQklV96sZvN7fDRRmpKmY7p1CyZLIh1G4IkkokScw2QrqkhxOSs5H+A6xb7HBnt+IUvpgOXhwYe0+uOcQ==", + "dev": true, + "requires": { + "async": "^2.6.1", + "backoff": "^2.5.0", + "leo-config": "1.1.0", + "leo-logger": "1.0.1", + "leo-streams": "2.0.0", + "lodash": "^4.17.14", + "lodash.merge": "^4.6.2" + } + } + } + }, "leo-logger": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/leo-logger/-/leo-logger-1.0.1.tgz", @@ -867,6 +1126,113 @@ "lodash": "^4.17.10" } }, + "leo-sdk": { + "version": "5.0.16", + "resolved": "https://registry.npmjs.org/leo-sdk/-/leo-sdk-5.0.16.tgz", + "integrity": "sha512-i7eREO8v18fdBl7WJlIW4RVYstcQCPbiKG5qsuqRxDEvvAy/UmB4A73qVSbcvxSQaXiPq2QbiqaFPjHKJwrw6Q==", + "dev": true, + "requires": { + "async": "2.6.2", + "backoff": "2.5.0", + "event-stream": "4.0.1", + "extend": "3.0.2", + "fast-csv": "2.5.0", + "flush-write-stream": "2.0.0", + "ini": "1.3.5", + "leo-aws": "2.0.1", + "leo-config": "1.1.0", + "leo-logger": "1.0.1", + "leo-streams": "2.0.0", + "lodash": "4.17.19", + "moment": "2.24.0", + "pump": "3.0.0", + "pumpify": "1.5.1", + "readable-stream": "3.4.0", + "split2": "3.1.1", + "through2": "3.0.1", + "uuid": "3.3.2" + }, + "dependencies": { + "async": { + "version": "2.6.2", + "resolved": "https://registry.npmjs.org/async/-/async-2.6.2.tgz", + "integrity": "sha512-H1qVYh1MYhEEFLsP97cVKqCGo7KfCyTt6uEWqsTBr9SO84oK9Uwbyd/yCW+6rKJLHksBNUVWZDAjfS+Ccx0Bbg==", + "dev": true, + "requires": { + "lodash": "^4.17.11" + } + }, + "fast-csv": { + "version": "2.5.0", + "resolved": "https://registry.npmjs.org/fast-csv/-/fast-csv-2.5.0.tgz", + "integrity": "sha512-M/9ezLU9/uDwvDZTt9sNFJa0iLDUsbhYJwPtnE0D9MjeuB6DY9wRCyUPZta9iI6cSz5wBWGaUPL61QH8h92cNA==", + "dev": true, + "requires": { + "extended": "0.0.6", + "is-extended": "0.0.10", + "object-extended": "0.0.7", + "safer-buffer": "^2.1.2", + "string-extended": "0.0.8" + } + }, + "ini": { + "version": "1.3.5", + "resolved": "https://registry.npmjs.org/ini/-/ini-1.3.5.tgz", + "integrity": "sha512-RZY5huIKCMRWDUqZlEi72f/lmXKMvuszcMBduliQ3nnWbx9X/ZBQO7DijMEYS9EhHBb2qacRUMtC7svLwe0lcw==", + "dev": true + }, + "leo-aws": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/leo-aws/-/leo-aws-2.0.1.tgz", + "integrity": "sha512-40vq8BQklV96sZvN7fDRRmpKmY7p1CyZLIh1G4IkkokScw2QrqkhxOSs5H+A6xb7HBnt+IUvpgOXhwYe0+uOcQ==", + "dev": true, + "requires": { + "async": "^2.6.1", + "backoff": "^2.5.0", + "leo-config": "1.1.0", + "leo-logger": "1.0.1", + "leo-streams": "2.0.0", + "lodash": "^4.17.14", + "lodash.merge": "^4.6.2" + } + }, + "lodash": { + "version": "4.17.19", + "resolved": "https://registry.npmjs.org/lodash/-/lodash-4.17.19.tgz", + "integrity": "sha512-JNvd8XER9GQX0v2qJgsaN/mzFCNA5BRe/j8JN9d+tWyGLSodKQHKFicdwNYzWwI3wjRnaKPsGj1XkBjx/F96DQ==", + "dev": true + }, + "readable-stream": { + "version": "3.4.0", + "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-3.4.0.tgz", + "integrity": "sha512-jItXPLmrSR8jmTRmRWJXCnGJsfy85mB3Wd/uINMXA65yrnFo0cPClFIUWzo2najVNSl+mx7/4W8ttlLWJe99pQ==", + "dev": true, + "requires": { + "inherits": "^2.0.3", + "string_decoder": "^1.1.1", + "util-deprecate": "^1.0.1" + } + }, + "split2": { + "version": "3.1.1", + "resolved": "https://registry.npmjs.org/split2/-/split2-3.1.1.tgz", + "integrity": "sha512-emNzr1s7ruq4N+1993yht631/JH+jaj0NYBosuKmLcq+JkGQ9MmTw1RB1fGaTCzUuseRIClrlSLHRNYGwWQ58Q==", + "dev": true, + "requires": { + "readable-stream": "^3.0.0" + } + }, + "through2": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/through2/-/through2-3.0.1.tgz", + "integrity": "sha512-M96dvTalPT3YbYLaKaCuwu+j06D/8Jfib0o/PxbVt6Amhv3dUAtW6rTV1jPgJSBG83I/e04Y6xkVdVhSRhi0ww==", + "dev": true, + "requires": { + "readable-stream": "2 || 3" + } + } + } + }, "leo-streams": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/leo-streams/-/leo-streams-2.0.0.tgz", @@ -887,7 +1253,7 @@ "levn": { "version": "0.3.0", "resolved": "https://registry.npmjs.org/levn/-/levn-0.3.0.tgz", - "integrity": "sha1-OwmSTt+fCDwEkP3UwLxEIeBHZO4=", + "integrity": "sha512-0OO4y2iOHix2W6ujICbKIaEQXvFQHue65vUG3pb5EUomzPI90z9hsA1VsO/dbIIpC53J8gxM9Q4Oho0jrCM/yA==", "dev": true, "requires": { "prelude-ls": "~1.1.2", @@ -895,9 +1261,9 @@ } }, "lodash": { - "version": "4.17.15", - "resolved": "https://registry.npmjs.org/lodash/-/lodash-4.17.15.tgz", - "integrity": "sha512-8xOcRHvCjnocdS5cpwXQXVzmmh5e5+saE2QGoeQmbKmRS6J3VQppPOIt0MnmE+4xlZoumy0GPG0D0MVIQbNA1A==" + "version": "4.17.21", + "resolved": "https://registry.npmjs.org/lodash/-/lodash-4.17.21.tgz", + "integrity": "sha512-v2kDEe57lecTulaDIuNTPy3Ry4gLGJ6Z1O3vE1krgXZNrsQ+LFTGHVxVjcXPs17LhbZVGedAJv8XZ1tvj5FvSg==" }, "lodash.merge": { "version": "4.6.2", @@ -910,6 +1276,31 @@ "integrity": "sha512-gKO5uExCXvSm6zbF562EvM+rd1kQDnB9AZBbiQVzf1ZmdDpxUSvpnAaVOP83N/31mRK8Ml8/VE8DMvsAZQ+7wg==", "dev": true }, + "loupe": { + "version": "2.3.4", + "resolved": "https://registry.npmjs.org/loupe/-/loupe-2.3.4.tgz", + "integrity": "sha512-OvKfgCC2Ndby6aSTREl5aCCPTNIzlDfQZvZxNUrBrihDhL3xcrYegTblhmEiCrg2kKQz4XsFIaemE5BF4ybSaQ==", + "dev": true, + "requires": { + "get-func-name": "^2.0.0" + } + }, + "lru-cache": { + "version": "4.1.5", + "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-4.1.5.tgz", + "integrity": "sha512-sWZlbEP2OsHNkXrMl5GYk/jKk70MBng6UU4YI/qGDYbgf6YbP4EvmqISbXCoJiRKs+1bSpFHVgQxvJ17F2li5g==", + "dev": true, + "requires": { + "pseudomap": "^1.0.2", + "yallist": "^2.1.2" + } + }, + "map-stream": { + "version": "0.0.7", + "resolved": "https://registry.npmjs.org/map-stream/-/map-stream-0.0.7.tgz", + "integrity": "sha512-C0X0KQmGm3N2ftbTGBhSyuydQ+vV1LC3f3zPvT3RXHXNZrvfPZcoXp/N5DOa8vedX/rTMm2CjTtivFg2STJMRQ==", + "dev": true + }, "mimic-fn": { "version": "1.2.0", "resolved": "https://registry.npmjs.org/mimic-fn/-/mimic-fn-1.2.0.tgz", @@ -917,27 +1308,27 @@ "dev": true }, "minimatch": { - "version": "3.0.4", - "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.0.4.tgz", - "integrity": "sha512-yJHVQEhyqPLUTgt9B83PXu6W3rx4MvvHvSUvToogpwoGDOUQ+yDrR0HRot+yOCdCO7u4hX3pWft6kWBBcqh0UA==", + "version": "3.1.2", + "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.1.2.tgz", + "integrity": "sha512-J7p63hRiAjw1NDEww1W7i37+ByIrOWO5XQQAzZ3VOcL0PNybwpfmV/N05zFAzwQ9USyEcX6t3UO+K5aqBQOIHw==", "dev": true, "requires": { "brace-expansion": "^1.1.7" } }, "minimist": { - "version": "1.2.5", - "resolved": "https://registry.npmjs.org/minimist/-/minimist-1.2.5.tgz", - "integrity": "sha512-FM9nNUYrRBAELZQT3xeZQ7fmMOBg6nWNmJKTcgsJeaLstP/UODVpGsr5OhXhhXg6f+qtJ8uiZ+PUxkDWcgIXLw==", + "version": "1.2.7", + "resolved": "https://registry.npmjs.org/minimist/-/minimist-1.2.7.tgz", + "integrity": "sha512-bzfL1YUZsP41gmu/qjrEk0Q6i2ix/cVeAhbCbqH9u3zYutS1cLg00qhrD0M2MVdCcx4Sc0UpP2eBWo9rotpq6g==", "dev": true }, "mkdirp": { - "version": "0.5.3", - "resolved": "https://registry.npmjs.org/mkdirp/-/mkdirp-0.5.3.tgz", - "integrity": "sha512-P+2gwrFqx8lhew375MQHHeTlY8AuOJSrGf0R5ddkEndUkmwpgUob/vQuBD1V22/Cw1/lJr4x+EjllSezBThzBg==", + "version": "0.5.6", + "resolved": "https://registry.npmjs.org/mkdirp/-/mkdirp-0.5.6.tgz", + "integrity": "sha512-FP+p8RB8OWpF3YZBCrP5gtADmtXApB5AMLn+vdyA+PyxCjrCs00mjyUozssO33cwDeT3wNGdLxJ5M//YqtHAJw==", "dev": true, "requires": { - "minimist": "^1.2.5" + "minimist": "^1.2.6" } }, "mocha": { @@ -982,16 +1373,25 @@ "path-is-absolute": "^1.0.0" } }, + "minimatch": { + "version": "3.0.4", + "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.0.4.tgz", + "integrity": "sha512-yJHVQEhyqPLUTgt9B83PXu6W3rx4MvvHvSUvToogpwoGDOUQ+yDrR0HRot+yOCdCO7u4hX3pWft6kWBBcqh0UA==", + "dev": true, + "requires": { + "brace-expansion": "^1.1.7" + } + }, "minimist": { "version": "0.0.8", "resolved": "https://registry.npmjs.org/minimist/-/minimist-0.0.8.tgz", - "integrity": "sha1-hX/Kv8M5fSYluCKCYuhqp6ARsF0=", + "integrity": "sha512-miQKw5Hv4NS1Psg2517mV4e4dYNaO3++hjAvLOAzKqZ61rH8NS1SK+vbfBWZ5PY/Me/bEWhUwqMghEW5Fb9T7Q==", "dev": true }, "mkdirp": { "version": "0.5.1", "resolved": "https://registry.npmjs.org/mkdirp/-/mkdirp-0.5.1.tgz", - "integrity": "sha1-MAV0OOrGz3+MR2fzhkjWaX11yQM=", + "integrity": "sha512-SknJC52obPfGQPnjIkXbmA6+5H15E+fR+E4iR2oQ3zzCLbd7/ONua69R/Gw7AgkTLsRG+r5fzksYwWe1AgTyWA==", "dev": true, "requires": { "minimist": "0.0.8" @@ -1000,7 +1400,7 @@ "ms": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/ms/-/ms-2.0.0.tgz", - "integrity": "sha1-VgiurfwAvmwpAd9fmGF4jeDVl8g=", + "integrity": "sha512-Tpp60P6IUJDTuOq/5Z8cdskzJujfwqfOTkrwIwj7IRISpnkJnT6SyJ4PCPnGMoFjC9ddhal5KVIYtAt97ix05A==", "dev": true }, "supports-color": { @@ -1019,6 +1419,21 @@ "resolved": "https://registry.npmjs.org/moment/-/moment-2.24.0.tgz", "integrity": "sha512-bV7f+6l2QigeBBZSM/6yTNq4P2fNpSWj/0e7jQcy87A8e7o2nAfP/34/2ky5Vw4B9S446EtIhodAzkFCcR4dQg==" }, + "moment-business-days": { + "version": "1.1.3", + "resolved": "https://registry.npmjs.org/moment-business-days/-/moment-business-days-1.1.3.tgz", + "integrity": "sha512-VfpcLQxVWJ9ymq4ljaqPdzoco01kOJblX9sbfOCYguBvqCvVPYDJQTdez49a2gtSoSK8AjL8JfvLvmoQlrBMiQ==", + "dev": true + }, + "moment-timezone": { + "version": "0.5.26", + "resolved": "https://registry.npmjs.org/moment-timezone/-/moment-timezone-0.5.26.tgz", + "integrity": "sha512-sFP4cgEKTCymBBKgoxZjYzlSovC20Y6J7y3nanDc5RoBIXKlZhoYwBoZGe3flwU6A372AcRwScH8KiwV6zjy1g==", + "dev": true, + "requires": { + "moment": ">= 2.9.0" + } + }, "ms": { "version": "2.1.2", "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.2.tgz", @@ -1028,13 +1443,19 @@ "mute-stream": { "version": "0.0.7", "resolved": "https://registry.npmjs.org/mute-stream/-/mute-stream-0.0.7.tgz", - "integrity": "sha1-MHXOk7whuPq0PhvE2n6BFe0ee6s=", + "integrity": "sha512-r65nCZhrbXXb6dXOACihYApHw2Q6pV0M3V0PSxd74N0+D8nzAdEAITq2oAjA1jVnKI+tGvEBUpqiMh0+rW6zDQ==", "dev": true }, "natural-compare": { "version": "1.4.0", "resolved": "https://registry.npmjs.org/natural-compare/-/natural-compare-1.4.0.tgz", - "integrity": "sha1-Sr6/7tdUHywnrPspvbvRXI1bpPc=", + "integrity": "sha512-OWND8ei3VtNC9h7V60qff3SVobHr996CTwgxubgyQYEpg290h9J0buyECNNJexkFm5sOajh5G116RYA1c8ZMSw==", + "dev": true + }, + "netmask": { + "version": "1.0.6", + "resolved": "https://registry.npmjs.org/netmask/-/netmask-1.0.6.tgz", + "integrity": "sha512-3DWDqAtIiPSkBXZyYEjwebfK56nrlQfRGt642fu8RPaL+ePu750+HCMHxjJCG3iEHq/0aeMvX6KIzlv7nuhfrA==", "dev": true }, "nice-try": { @@ -1067,10 +1488,20 @@ } } }, + "nopt": { + "version": "4.0.3", + "resolved": "https://registry.npmjs.org/nopt/-/nopt-4.0.3.tgz", + "integrity": "sha512-CvaGwVMztSMJLOeXPrez7fyfObdZqNUK1cPAEzLHrTybIua9pMdmmPR5YwtfNftIOMv3DPUhFaxsZMNTQO20Kg==", + "dev": true, + "requires": { + "abbrev": "1", + "osenv": "^0.1.4" + } + }, "object-extended": { "version": "0.0.7", "resolved": "https://registry.npmjs.org/object-extended/-/object-extended-0.0.7.tgz", - "integrity": "sha1-hP0j9WsVWCrrPoiwXLVdJDLWijM=", + "integrity": "sha512-2LJYIacEXoZ1glGkAZuvA/4pfJM4Y1ShReAo9jWpBSuz89TiUCdiPqhGJJ6m97F3WjhCSRwrbgaxYEAm9dRYBw==", "requires": { "array-extended": "~0.0.4", "extended": "~0.0.3", @@ -1080,7 +1511,7 @@ "once": { "version": "1.4.0", "resolved": "https://registry.npmjs.org/once/-/once-1.4.0.tgz", - "integrity": "sha1-WDsap3WWHUsROsF9nFC6753Xa9E=", + "integrity": "sha512-lNaJgI+2Q5URQBkccEKHTQOPaXdUxnZZElQTZY0MFUAuaEqe1E+Nyvgdz/aIyNi6Z9MzO5dv1H8n58/GELp3+w==", "requires": { "wrappy": "1" } @@ -1088,7 +1519,7 @@ "onetime": { "version": "2.0.1", "resolved": "https://registry.npmjs.org/onetime/-/onetime-2.0.1.tgz", - "integrity": "sha1-BnQoIw/WdEOyeUsiu6UotoZ5YtQ=", + "integrity": "sha512-oyyPpiMaKARvvcgip+JV+7zci5L8D1W9RZIz2l1o08AM3pfspitVWnPt3mzHcBPp12oYMTy0pqrFs/C+m3EwsQ==", "dev": true, "requires": { "mimic-fn": "^1.0.0" @@ -1108,12 +1539,28 @@ "word-wrap": "~1.2.3" } }, + "os-homedir": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/os-homedir/-/os-homedir-1.0.2.tgz", + "integrity": "sha512-B5JU3cabzk8c67mRRd3ECmROafjYMXbuzlwtqdM8IbS8ktlTix8aFGb2bAGKrSRIlnfKwovGUUr72JUPyOb6kQ==", + "dev": true + }, "os-tmpdir": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/os-tmpdir/-/os-tmpdir-1.0.2.tgz", - "integrity": "sha1-u+Z0BseaqFxc/sdm/lc0VV36EnQ=", + "integrity": "sha512-D2FR03Vir7FIu45XBY20mTb+/ZSWB00sjU9jdQXt83gDrI4Ztz5Fs7/yy74g2N5SVQY4xY1qDr4rNddwYRVX0g==", "dev": true }, + "osenv": { + "version": "0.1.5", + "resolved": "https://registry.npmjs.org/osenv/-/osenv-0.1.5.tgz", + "integrity": "sha512-0CWcCECdMVc2Rw3U5w9ZjqX6ga6ubk1xDVKxtBQPK7wis/0F2r9T6k4ydGYhecl7YUBxBVxhL5oisPsNxAPe2g==", + "dev": true, + "requires": { + "os-homedir": "^1.0.0", + "os-tmpdir": "^1.0.0" + } + }, "packet-reader": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/packet-reader/-/packet-reader-1.0.0.tgz", @@ -1131,19 +1578,19 @@ "path-is-absolute": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/path-is-absolute/-/path-is-absolute-1.0.1.tgz", - "integrity": "sha1-F0uSaHNVNP+8es5r9TpanhtcX18=", + "integrity": "sha512-AVbw3UJ2e9bq64vSaS9Am0fje1Pa8pbGqTTsmXfaIiMpnr5DlDhfJOuLj9Sf95ZPVDAUerDfEk88MPmPe7UCQg==", "dev": true }, "path-is-inside": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/path-is-inside/-/path-is-inside-1.0.2.tgz", - "integrity": "sha1-NlQX3t5EQw0cEa9hAn+s8HS9/FM=", + "integrity": "sha512-DUWJr3+ULp4zXmol/SZkFf3JGsS9/SIv+Y3Rt93/UjPpDpklB5f1er4O3POIbUuUJ3FXgqte2Q7SrU6zAqwk8w==", "dev": true }, "path-key": { "version": "2.0.1", "resolved": "https://registry.npmjs.org/path-key/-/path-key-2.0.1.tgz", - "integrity": "sha1-QRyttXTFoUDTpLGRDUDYDMn0C0A=", + "integrity": "sha512-fEHGKCSmUSDPv4uoj8AlD+joPlq3peND+HRYyxFz4KPw4z926S/b8rIuFs2FYJg3BwsxJf6A9/3eIdLaYC+9Dw==", "dev": true }, "path-to-regexp": { @@ -1158,17 +1605,26 @@ "isarray": { "version": "0.0.1", "resolved": "https://registry.npmjs.org/isarray/-/isarray-0.0.1.tgz", - "integrity": "sha1-ihis/Kmo9Bd+Cav8YDiTmwXR7t8=", + "integrity": "sha512-D2S+3GLxWH+uhrNEcoh/fnmYeP8E8/zHl644d/jdA0g2uyXvy3sb0qxotE+ne0LtccHknQzWwZEzhak7oJ0COQ==", "dev": true } } }, "pathval": { - "version": "1.1.0", - "resolved": "https://registry.npmjs.org/pathval/-/pathval-1.1.0.tgz", - "integrity": "sha1-uULm1L3mUwBe9rcTYd74cn0GReA=", + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/pathval/-/pathval-1.1.1.tgz", + "integrity": "sha512-Dp6zGqpTdETdR63lehJYPeIOqpiNBNtc7BpWSLrOje7UaIsE5aY92r/AunQA7rsXvet3lrJ3JnZX29UPTKXyKQ==", "dev": true }, + "pause-stream": { + "version": "0.0.11", + "resolved": "https://registry.npmjs.org/pause-stream/-/pause-stream-0.0.11.tgz", + "integrity": "sha512-e3FBlXLmN/D1S+zHzanP4E/4Z60oFAa3O051qt1pxa7DEJWKAyil6upYVXCWadEnuoqa4Pkc9oUx9zsxYeRv8A==", + "dev": true, + "requires": { + "through": "~2.3" + } + }, "pg": { "version": "8.11.3", "resolved": "https://registry.npmjs.org/pg/-/pg-8.11.3.tgz", @@ -1182,6 +1638,13 @@ "pg-protocol": "^1.6.0", "pg-types": "^2.1.0", "pgpass": "1.x" + }, + "dependencies": { + "pg-connection-string": { + "version": "2.7.0", + "resolved": "https://registry.npmjs.org/pg-connection-string/-/pg-connection-string-2.7.0.tgz", + "integrity": "sha512-PI2W9mv53rXJQEOb8xNR8lH7Hr+EKa6oJa38zsK0S/ky2er16ios1wLKhZyxzD7jUReiWokc9WK5nxSnC7W1TA==" + } } }, "pg-cloudflare": { @@ -1190,11 +1653,6 @@ "integrity": "sha512-xWPagP/4B6BgFO+EKz3JONXv3YDgvkbVrGw2mTo3D6tVDQRh1e7cqVGvyR3BE+eQgAvx1XhW/iEASj4/jCWl3Q==", "optional": true }, - "pg-connection-string": { - "version": "2.6.2", - "resolved": "https://registry.npmjs.org/pg-connection-string/-/pg-connection-string-2.6.2.tgz", - "integrity": "sha512-ch6OwaeaPYcova4kKZ15sbJ2hKb/VP48ZD2gE7i1J+L4MspCtBMAx8nMgz7bksc7IojCIIWuEhHibSMFH8m8oA==" - }, "pg-copy-streams": { "version": "2.2.2", "resolved": "https://registry.npmjs.org/pg-copy-streams/-/pg-copy-streams-2.2.2.tgz", @@ -1203,7 +1661,7 @@ "pg-format": { "version": "1.0.4", "resolved": "https://registry.npmjs.org/pg-format/-/pg-format-1.0.4.tgz", - "integrity": "sha1-J3NCNsKtP05QZJFaWTNOIAQKgo4=" + "integrity": "sha512-YyKEF78pEA6wwTAqOUaHIN/rWpfzzIuMh9KdAhc3rSLQ/7zkRFcCgYBAEGatDstLyZw4g0s9SNICmaTGnBVeyw==" }, "pg-int8": { "version": "1.0.1", @@ -1241,9 +1699,9 @@ }, "dependencies": { "split2": { - "version": "4.2.0", - "resolved": "https://registry.npmjs.org/split2/-/split2-4.2.0.tgz", - "integrity": "sha512-UcjcJOWknrNkF6PLX83qcHM6KHgVKNkV62Y8a5uYDVv9ydGQVwAHMKqHdJje1VTWpljG0WYpCDhrCdAOYH4TWg==" + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/split2/-/split2-4.1.0.tgz", + "integrity": "sha512-VBiJxFkxiXRlUIeyMQi8s4hgvKCSjtknJv/LVYbrgALPwf5zSKmEwV9Lst25AkvMDnvxODugjdl6KZgwKM1WYQ==" } } }, @@ -1278,7 +1736,7 @@ "prelude-ls": { "version": "1.1.2", "resolved": "https://registry.npmjs.org/prelude-ls/-/prelude-ls-1.1.2.tgz", - "integrity": "sha1-IZMqVJ9eUv/ZqCf1cOBL5iqX2lQ=", + "integrity": "sha512-ESF23V4SKG6lVSGZgYNpbsiaAkdab6ZgOxe52p7+Kid3W3u3bxR4Vfd/o21dmN7jSt0IwgZ4v5MUd26FEtXE9w==", "dev": true }, "process-nextick-args": { @@ -1292,6 +1750,18 @@ "integrity": "sha512-7PiHtLll5LdnKIMw100I+8xJXR5gW2QwWYkT6iJva0bXitZKa/XMrSbdmg3r2Xnaidz9Qumd0VPaMrZlF9V9sA==", "dev": true }, + "proto-list": { + "version": "1.2.4", + "resolved": "https://registry.npmjs.org/proto-list/-/proto-list-1.2.4.tgz", + "integrity": "sha512-vtK/94akxsTMhe0/cbfpR+syPuszcuwhqVjJq26CuNDgFGj682oRBXOP5MJpv2r7JtE8MsiepGIqvvOTBwn2vA==", + "dev": true + }, + "pseudomap": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/pseudomap/-/pseudomap-1.0.2.tgz", + "integrity": "sha512-b/YwNhb8lk1Zz2+bXXpS/LK9OisiZZ1SNsSLxN1x2OXVEhW2Ckr/7mWE5vrC1ZTiJlD9g19jWszTmJsB+oEpFQ==", + "dev": true + }, "pump": { "version": "3.0.0", "resolved": "https://registry.npmjs.org/pump/-/pump-3.0.0.tgz", @@ -1357,7 +1827,7 @@ "restore-cursor": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/restore-cursor/-/restore-cursor-2.0.0.tgz", - "integrity": "sha1-n37ih/gv0ybU/RYpI9YhKe7g368=", + "integrity": "sha512-6IzJLuGi4+R14vwagDHX+JrXmPVtPpn4mffDJ1UdR7/Edm87fl6yi8mMBIVvFtJaNTUvjughmW4hwLhRG7gC1Q==", "dev": true, "requires": { "onetime": "^2.0.0", @@ -1374,18 +1844,15 @@ } }, "run-async": { - "version": "2.4.0", - "resolved": "https://registry.npmjs.org/run-async/-/run-async-2.4.0.tgz", - "integrity": "sha512-xJTbh/d7Lm7SBhc1tNvTpeCHaEzoyxPrqNlvSdMfBTYwaY++UJFyXUOxAtsRUXjlqOfj8luNaR9vjCh4KeV+pg==", - "dev": true, - "requires": { - "is-promise": "^2.1.0" - } + "version": "2.4.1", + "resolved": "https://registry.npmjs.org/run-async/-/run-async-2.4.1.tgz", + "integrity": "sha512-tvVnVv01b8c1RrA6Ep7JkStj85Guv/YrMcwqYQnwjsAS2cTmmPGBBjAjpCW7RrSodNSoE2/qg9O4bceNvUuDgQ==", + "dev": true }, "rxjs": { - "version": "6.5.4", - "resolved": "https://registry.npmjs.org/rxjs/-/rxjs-6.5.4.tgz", - "integrity": "sha512-naMQXcgEo3csAEGvw/NydRA0fuS2nDZJiw1YUWFKU7aPPAPGZEsD4Iimit96qwCieH6y614MCLYwdkrWx7z/7Q==", + "version": "6.6.7", + "resolved": "https://registry.npmjs.org/rxjs/-/rxjs-6.6.7.tgz", + "integrity": "sha512-hTdwr+7yYNIT5n4AMYp85KA6yw2Va0FLa3Rguvbpa4W3I5xynaBZo41cM3XM+4Q6fRMj3sBYIR1VAmZMXYJvRQ==", "dev": true, "requires": { "tslib": "^1.9.0" @@ -1405,7 +1872,7 @@ "shebang-command": { "version": "1.2.0", "resolved": "https://registry.npmjs.org/shebang-command/-/shebang-command-1.2.0.tgz", - "integrity": "sha1-RKrGW2lbAzmJaMOfNj/uXer98eo=", + "integrity": "sha512-EV3L1+UQWGor21OmnvojK36mhg+TyIKDh3iFBKBohr5xeXIhNBcx8oWdgkTEEQ+BEFFYdLRuqMfd5L84N1V5Vg==", "dev": true, "requires": { "shebang-regex": "^1.0.0" @@ -1414,13 +1881,19 @@ "shebang-regex": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/shebang-regex/-/shebang-regex-1.0.0.tgz", - "integrity": "sha1-2kL0l0DAtC2yypcoVxyxkMmO/qM=", + "integrity": "sha512-wpoSFAxys6b2a2wHZ1XpDSgD7N9iVjg29Ph9uV/uaP9Ex/KXlkTZTeddxDPSYQpgvzKLGJke2UU0AzoGCjNIvQ==", + "dev": true + }, + "sigmund": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/sigmund/-/sigmund-1.0.1.tgz", + "integrity": "sha512-fCvEXfh6NWpm+YSuY2bpXb/VIihqWA6hLsgboC+0nl71Q7N7o2eaCW8mJa/NLvQhs6jpd3VZV4UiUQlV6+lc8g==", "dev": true }, "signal-exit": { - "version": "3.0.2", - "resolved": "https://registry.npmjs.org/signal-exit/-/signal-exit-3.0.2.tgz", - "integrity": "sha1-tf3AjxKH6hF4Yo5BXiUTK3NkbG0=", + "version": "3.0.7", + "resolved": "https://registry.npmjs.org/signal-exit/-/signal-exit-3.0.7.tgz", + "integrity": "sha512-wnD2ZE+l+SPC/uoS0vXeE9L1+0wuaMqKlfz9AMUo38JsyLSBWSFcHR1Rri62LZc12vLr1gb3jl7iwQhgwpAbGQ==", "dev": true }, "sinon": { @@ -1449,6 +1922,15 @@ "is-fullwidth-code-point": "^2.0.0" } }, + "split": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/split/-/split-1.0.1.tgz", + "integrity": "sha512-mTyOoPbrivtXnwnIxZRFYRrPNtEFKlpB2fvjSnCQUiAA6qAZzqwna5envK4uk6OIeP17CsdF3rSBGYVBsU0Tkg==", + "dev": true, + "requires": { + "through": "2" + } + }, "split2": { "version": "2.2.0", "resolved": "https://registry.npmjs.org/split2/-/split2-2.2.0.tgz", @@ -1460,7 +1942,7 @@ "sprintf-js": { "version": "1.0.3", "resolved": "https://registry.npmjs.org/sprintf-js/-/sprintf-js-1.0.3.tgz", - "integrity": "sha1-BOaSb2YolTVPPdAVIDYzuFcpfiw=", + "integrity": "sha512-D9cPgkvLlV3t3IzL0D0YLvGA9Ahk4PcvVwUbN0dSGr1aP0Nrt4AEnTUbuGvquEC0mA64Gqt1fzirlRs5ibXx8g==", "dev": true }, "sqlstring": { @@ -1468,6 +1950,16 @@ "resolved": "https://registry.npmjs.org/sqlstring/-/sqlstring-2.3.1.tgz", "integrity": "sha512-ooAzh/7dxIG5+uDik1z/Rd1vli0+38izZhGzSa34FwR7IbelPWCCKSNIl8jlL/F7ERvy8CB2jNeM1E9i9mXMAQ==" }, + "stream-combiner": { + "version": "0.2.2", + "resolved": "https://registry.npmjs.org/stream-combiner/-/stream-combiner-0.2.2.tgz", + "integrity": "sha512-6yHMqgLYDzQDcAkL+tjJDC5nSNuNIx0vZtRZeiPh7Saef7VHX9H5Ijn9l2VIol2zaNYlYEX6KyuT/237A58qEQ==", + "dev": true, + "requires": { + "duplexer": "~0.1.1", + "through": "~2.3.4" + } + }, "stream-shift": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/stream-shift/-/stream-shift-1.0.1.tgz", @@ -1476,7 +1968,7 @@ "string-extended": { "version": "0.0.8", "resolved": "https://registry.npmjs.org/string-extended/-/string-extended-0.0.8.tgz", - "integrity": "sha1-dBlX3/SHsCcqee7FpE8jnubxfM0=", + "integrity": "sha512-CK46p3AxBvBhJbBi6WrF9bCcaWH20E4NwlLSzpooG2nXWvcP2gy2YR8VN6fSwZyrbcvL4S4zoNKbR0QG52X4rw==", "requires": { "array-extended": "~0.0.5", "date-extended": "~0.0.3", @@ -1505,7 +1997,7 @@ "strip-ansi": { "version": "4.0.0", "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-4.0.0.tgz", - "integrity": "sha1-qEeQIusaw2iocTibY1JixQXuNo8=", + "integrity": "sha512-4XaJ2zQdCzROZDivEVIDPkcQn8LMFSa8kj8Gxb/Lnwzv9A8VctNZ+lfivC/sV3ivW8ElJTERXZoPBRrZKkNKow==", "dev": true, "requires": { "ansi-regex": "^3.0.0" @@ -1514,7 +2006,7 @@ "strip-json-comments": { "version": "2.0.1", "resolved": "https://registry.npmjs.org/strip-json-comments/-/strip-json-comments-2.0.1.tgz", - "integrity": "sha1-PFMZQukIwml8DsNEhYwobHygpgo=", + "integrity": "sha512-4gB8na07fecVVkOI6Rs4e7T6NOTki5EmL7TUduTs6bu3EdnSycntVJ4re8kgZA+wx9IueI2Y11bfbgwtzuE0KQ==", "dev": true }, "supports-color": { @@ -1539,9 +2031,9 @@ }, "dependencies": { "ansi-regex": { - "version": "4.1.0", - "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-4.1.0.tgz", - "integrity": "sha512-1apePfXM1UOSqw0o9IiFAovVz9M5S1Dg+4TrDwfMewQ6p/rmMueb7tWZjQ1rx4Loy1ArBggoqGpfqqdI4rondg==", + "version": "4.1.1", + "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-4.1.1.tgz", + "integrity": "sha512-ILlv4k/3f6vfQ4OoP2AGvirOktlQ98ZEL1k9FaQjxa3L1abBgbuTDAdPOpvbGncC0BTVQrl+OM8xZGK6tWXt7g==", "dev": true }, "string-width": { @@ -1569,13 +2061,13 @@ "text-table": { "version": "0.2.0", "resolved": "https://registry.npmjs.org/text-table/-/text-table-0.2.0.tgz", - "integrity": "sha1-f17oI66AUgfACvLfSoTsP8+lcLQ=", + "integrity": "sha512-N+8UisAXDGk8PFXP4HAzVR9nbfmVJ3zYLAWiTIoqC5v5isinhr+r5uaO8+7r3BMfuNIufIsA7RdpVgacC2cSpw==", "dev": true }, "through": { "version": "2.3.8", "resolved": "https://registry.npmjs.org/through/-/through-2.3.8.tgz", - "integrity": "sha1-DdTJ/6q8NXlgsbckEV1+Doai4fU=", + "integrity": "sha512-w89qg7PI8wAdvX60bMDP+bFoD5Dvhm9oLheFp5O4a2QF0cSBGsBX4qZmadPMvVqlLJBBci+WqGGOAPvcDeNSVg==", "dev": true }, "through2": { @@ -1597,15 +2089,15 @@ } }, "tslib": { - "version": "1.11.1", - "resolved": "https://registry.npmjs.org/tslib/-/tslib-1.11.1.tgz", - "integrity": "sha512-aZW88SY8kQbU7gpV19lN24LtXh/yD4ZZg6qieAJDDg+YBsJcSmLGK9QpnUjAKVG/xefmvJGd1WUmfpT/g6AJGA==", + "version": "1.14.1", + "resolved": "https://registry.npmjs.org/tslib/-/tslib-1.14.1.tgz", + "integrity": "sha512-Xni35NKzjgMrwevysHTCArtLDpPvye8zV/0E4EyYn43P7/7qvQwPh9BGkHewbMulVntbigmcT7rdX3BNo9wRJg==", "dev": true }, "type-check": { "version": "0.3.2", "resolved": "https://registry.npmjs.org/type-check/-/type-check-0.3.2.tgz", - "integrity": "sha1-WITKtRLPHTVeP7eE8wgEsrUg23I=", + "integrity": "sha512-ZCmOJdvOWDBYJlzAoFkC+Q0+bUyEOS1ltgp1MGU03fqHG+dbi9tBFU2Rd9QKiDZFAYrhPh2JUf7rZRIuHRKtOg==", "dev": true, "requires": { "prelude-ls": "~1.1.2" @@ -1618,9 +2110,9 @@ "dev": true }, "uri-js": { - "version": "4.2.2", - "resolved": "https://registry.npmjs.org/uri-js/-/uri-js-4.2.2.tgz", - "integrity": "sha512-KY9Frmirql91X2Qgjry0Wd4Y+YTdrdZheS8TFwvkbLWf/G5KNJDCh6pKL5OZctEW4+0Baa5idK2ZQuELRwPznQ==", + "version": "4.4.1", + "resolved": "https://registry.npmjs.org/uri-js/-/uri-js-4.4.1.tgz", + "integrity": "sha512-7rKUyy33Q1yc98pQ1DAmLtwX109F7TIfWlW1Ydo8Wl1ii1SeHieeh0HHfPeL2fMXK6z0s8ecKs9frCuLJvndBg==", "dev": true, "requires": { "punycode": "^2.1.0" @@ -1631,6 +2123,12 @@ "resolved": "https://registry.npmjs.org/util-deprecate/-/util-deprecate-1.0.2.tgz", "integrity": "sha512-EPD5q1uXyFxJpCrLnCc1nHnq3gOa6DZBocAIiI2TaSCA7VCJ1UJDMagCzIkXNsUYfD1daK//LTEQ8xiIbrHtcw==" }, + "uuid": { + "version": "3.3.2", + "resolved": "https://registry.npmjs.org/uuid/-/uuid-3.3.2.tgz", + "integrity": "sha512-yXJmeNaw3DnnKAOKJE51sL/ZaYfWJRl1pK9dr19YFCu0ObS231AB1/LbqTKRAQ5kw8A90rA6fr4riOUpTZvQZA==", + "dev": true + }, "which": { "version": "1.3.1", "resolved": "https://registry.npmjs.org/which/-/which-1.3.1.tgz", @@ -1649,7 +2147,7 @@ "wrappy": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/wrappy/-/wrappy-1.0.2.tgz", - "integrity": "sha1-tSQ9jz7BqjXxNkYFvA0QNuMKtp8=" + "integrity": "sha512-l4Sp/DRseor9wL6EvV2+TuQn63dMkPjZ/sp9XkghTEbV9KlPS1xUsZ3u7/IQO4wxtcFB4bgpQPRcR3QCvezPcQ==" }, "write": { "version": "1.0.3", @@ -1664,6 +2162,12 @@ "version": "4.0.2", "resolved": "https://registry.npmjs.org/xtend/-/xtend-4.0.2.tgz", "integrity": "sha512-LKYU1iAXJXUgAXn9URjiu+MWhyUXHsvfp7mcuYm9dSUKK0/CjtrUwFAxD82/mCWbtLsGjFIad0wIsod4zrTAEQ==" + }, + "yallist": { + "version": "2.1.2", + "resolved": "https://registry.npmjs.org/yallist/-/yallist-2.1.2.tgz", + "integrity": "sha512-ncTzHV7NvsQZkYe1DW7cbDLm0YpzHmZF5r/iyP3ZnQtMiJ+pjzisCiMNI+Sj+xQF5pXhSHxSB3uDbsBTzY/c2A==", + "dev": true } } } diff --git a/postgres/package.json b/postgres/package.json index 076870f3..cc385a2a 100644 --- a/postgres/package.json +++ b/postgres/package.json @@ -1,6 +1,6 @@ { "name": "leo-connector-postgres", - "version": "4.0.7", + "version": "4.0.24-rc", "description": "A Postgres database connector for use with Leo Platform", "repository": { "type": "git", @@ -19,13 +19,14 @@ "homepage": "https://github.com/LeoPlatform/connectors#readme", "dependencies": { "fast-csv": "2.4.1", - "leo-connector-common": "^4.0.11-rc", + "leo-connector-common": "4.0.13-rc", "pg": "^8.11.3", "pg-copy-streams": "2.2.2", "pg-format": "1.0.4", "pg-pool": "^3.6.1" }, "devDependencies": { + "@dsco/layer-leo": "2.1.29", "chai": "^4.2.0", "eslint": "^5.13.0", "mocha": "^5.2.0",