diff --git a/adapter/integration-test/src/adapter-stateless.test.js b/adapter/integration-test/src/adapter-stateless.test.js index c4fdc9b13..426d2d8fa 100644 --- a/adapter/integration-test/src/adapter-stateless.test.js +++ b/adapter/integration-test/src/adapter-stateless.test.js @@ -1,229 +1,310 @@ -const { execPath } = require('process') -const request = require('supertest') +const { execPath } = require('process'); +const request = require('supertest'); -const { - ADAPTER_URL, - MOCK_SERVER_URL -} = require('./env') -const { waitForServicesToBeReady } = require('./waitForServices') +const { ADAPTER_URL, MOCK_SERVER_URL } = require('./env'); +const { waitForServicesToBeReady } = require('./waitForServices'); -const TIMEOUT = 10000 +const TIMEOUT = 10000; describe('Stateless data import', () => { beforeAll(async () => { - await waitForServicesToBeReady() - }, 60000) - - test('Should respond with semantic version [GET /version]', async () => { - const response = await request(ADAPTER_URL).get('/version') - expect(response.status).toEqual(200) - expect(response.type).toEqual('text/plain') - - const semanticVersionRegEx = '^(0|[1-9]\\d*)\\.(0|[1-9]\\d*)\\.(0|[1-9]\\d*)' - expect(response.text).toMatch(new RegExp(semanticVersionRegEx)) - }, TIMEOUT) - - test('Should respond with all available formats [GET /formats]', async () => { - const response = await request(ADAPTER_URL).get('/formats') - expect(response.status).toEqual(200) - expect(response.type).toEqual('application/json') - expect(response.body.length).toBeGreaterThanOrEqual(2) - - response.body.forEach(e => { - expect(e.type).toBeDefined() - expect(e.parameters).toBeDefined() - }) - }, TIMEOUT) - - test('Should respond with all available protocols [GET /protocols]', async () => { - const response = await request(ADAPTER_URL).get('/protocols') - expect(response.status).toEqual(200) - expect(response.type).toEqual('application/json') - expect(response.body.length).toBeGreaterThanOrEqual(1) - - response.body.forEach(e => { - expect(e.type).toBeDefined() - expect(e.parameters).toBeDefined() - }) - }, TIMEOUT) - - test('Should import json data', async () => { - const reqBody = { - protocol: { - type: 'HTTP', - parameters: { - location: MOCK_SERVER_URL + '/json', - encoding: 'UTF-8' - } - }, - format: { - type: 'JSON' - } - } - - const response = await request(ADAPTER_URL) - .post('/preview') - .send(reqBody) - expect(response.status).toEqual(200) - const importedData = response.body.data - expect(importedData).toEqual({ whateverwillbe: 'willbe', quesera: 'sera' }) - }, TIMEOUT) - - test('Should import raw xml data', async () => { - const reqBody = { - protocol: { - type: 'HTTP', - parameters: { - location: MOCK_SERVER_URL + '/xml', - encoding: 'UTF-8' - } - } - } - - const response = await request(ADAPTER_URL) - .post('/preview/raw') - .send(reqBody) - - expect(response.status).toEqual(200) - expect(response.body.data).toEqual( - '' + - 'RickMorty' - ) - }, TIMEOUT) - - test('Should import and format xml data', async () => { - const reqBody = { - protocol: { - type: 'HTTP', - parameters: { - location: MOCK_SERVER_URL + '/xml', - encoding: 'UTF-8' - } - }, - format: { - type: 'XML' - } - } - - const response = await request(ADAPTER_URL) - .post('/preview') - .send(reqBody) - expect(response.status).toEqual(200) - const importedData = response.body.data - expect(importedData).toEqual({ from: 'Rick', to: 'Morty' }) - }, TIMEOUT) - - test('Should import and format csv data', async () => { - const reqBody = { - protocol: { - type: 'HTTP', - parameters: { - location: MOCK_SERVER_URL + '/csv', - encoding: 'UTF-8' - } - }, - format: { - type: 'CSV', - parameters: { - columnSeparator: ',', - lineSeparator: '\n', - skipFirstDataRow: false, - firstRowAsHeader: true - } - } - } - const response = await request(ADAPTER_URL) - .post('/preview') - .send(reqBody) - expect(response.status).toEqual(200) - const importedData = response.body.data - const expected = [ - { - col1: 'val11', - col2: 'val12', - col3: 'val13' - }, { - col1: 'val21', - col2: 'val22', - col3: 'val23' - }] - - expect(importedData).toEqual(expected) - }, TIMEOUT) - - test('Should return 400 BAD_REQUEST for unsupported protocol [POST /preview]', async () => { - const reqBody = { - protocol: { - type: 'UNSUPPORTED', - parameters: { - location: MOCK_SERVER_URL + '/json', - encoding: 'UTF-8' - } - }, - format: { - type: 'JSON' - } - } - const response = await request(ADAPTER_URL) - .post('/preview') - .send(reqBody) - expect(response.status).toEqual(400) - }, TIMEOUT) - - test('Should return 400 BAD_REQUEST for unsupported format [POST /preview]', async () => { - const reqBody = { - protocol: { - type: 'HTTP', - parameters: { - location: MOCK_SERVER_URL + '/json', - encoding: 'UTF-8' - } - }, - format: { - type: 'UNSUPPORTED' - } - } - const response = await request(ADAPTER_URL) - .post('/preview') - .send(reqBody) - expect(response.status).toEqual(400) - }, TIMEOUT) - - test('Should return 400 BAD_REQUEST for invalid location [POST /preview]', async () => { - const reqBody = { - protocol: { - type: 'HTTP', - parameters: { - location: 'invalid-location', - encoding: 'UTF-8' - } - }, - format: { - type: 'JSON' - } - } - const response = await request(ADAPTER_URL) - .post('/preview') - .send(reqBody) - expect(response.status).toEqual(400) - }, TIMEOUT) - - test('Should return 500 INTERNAL_SERVER_ERROR for data not found [POST /preview]', async () => { - const reqBody = { - protocol: { - type: 'HTTP', - parameters: { - location: MOCK_SERVER_URL + '/not-found', - encoding: 'UTF-8' - } - }, - format: { - type: 'JSON' - } - } - const response = await request(ADAPTER_URL) - .post('/preview') - .send(reqBody) - expect(response.status).toEqual(500) - }, TIMEOUT) -}) + await waitForServicesToBeReady(); + }, 60000); + + test( + 'Should respond with semantic version [GET /version]', + async () => { + const response = await request(ADAPTER_URL).get('/version'); + expect(response.status).toEqual(200); + expect(response.type).toEqual('text/plain'); + + const semanticVersionRegEx = + '^(0|[1-9]\\d*)\\.(0|[1-9]\\d*)\\.(0|[1-9]\\d*)'; + expect(response.text).toMatch(new RegExp(semanticVersionRegEx)); + }, + TIMEOUT, + ); + + test( + 'Should respond with all available formats [GET /formats]', + async () => { + const response = await request(ADAPTER_URL).get('/formats'); + expect(response.status).toEqual(200); + expect(response.type).toEqual('application/json'); + expect(response.body.length).toBeGreaterThanOrEqual(2); + + response.body.forEach((e) => { + expect(e.type).toBeDefined(); + expect(e.parameters).toBeDefined(); + }); + }, + TIMEOUT, + ); + + test( + 'Should respond with all available protocols [GET /protocols]', + async () => { + const response = await request(ADAPTER_URL).get('/protocols'); + expect(response.status).toEqual(200); + expect(response.type).toEqual('application/json'); + expect(response.body.length).toBeGreaterThanOrEqual(1); + + response.body.forEach((e) => { + expect(e.type).toBeDefined(); + expect(e.parameters).toBeDefined(); + }); + }, + TIMEOUT, + ); + + test( + 'Should import json data', + async () => { + const reqBody = { + protocol: { + type: 'HTTP', + parameters: { + location: MOCK_SERVER_URL + '/json', + encoding: 'UTF-8', + }, + }, + format: { + type: 'JSON', + }, + }; + + const response = await request(ADAPTER_URL) + .post('/preview') + .send(reqBody); + expect(response.status).toEqual(200); + const importedData = response.body.data; + expect(importedData).toEqual({ + whateverwillbe: 'willbe', + quesera: 'sera', + }); + }, + TIMEOUT, + ); + + test( + 'Should import raw xml data', + async () => { + const reqBody = { + protocol: { + type: 'HTTP', + parameters: { + location: MOCK_SERVER_URL + '/xml', + encoding: 'UTF-8', + }, + }, + }; + + const response = await request(ADAPTER_URL) + .post('/preview/raw') + .send(reqBody); + + expect(response.status).toEqual(200); + expect(response.body.data).toEqual( + '' + + 'RickMorty', + ); + }, + TIMEOUT, + ); + + test( + 'Should import and format xml data', + async () => { + const reqBody = { + protocol: { + type: 'HTTP', + parameters: { + location: MOCK_SERVER_URL + '/xml', + encoding: 'UTF-8', + }, + }, + format: { + type: 'XML', + }, + }; + + const response = await request(ADAPTER_URL) + .post('/preview') + .send(reqBody); + expect(response.status).toEqual(200); + const importedData = response.body.data; + expect(JSON.parse(importedData)).toEqual({ + root: { from: 'Rick', to: 'Morty' }, + }); + }, + TIMEOUT, + ); + + test( + 'Should import and format more xml data', + async () => { + const reqBody = { + protocol: { + type: 'HTTP', + parameters: { + location: MOCK_SERVER_URL + '/xmlbigger', + encoding: 'UTF-8', + }, + }, + format: { + type: 'XML', + }, + }; + + const response = await request(ADAPTER_URL) + .post('/preview') + .send(reqBody); + expect(response.status).toEqual(200); + const importedData = response.body.data; + expect(JSON.parse(importedData)).toEqual({ + root: { + from: 'Rick', + to: 'Morty', + test: { hello: 'hello', servus: 'servus' }, + }, + }); + }, + TIMEOUT, + ); + + test( + 'Should import and format csv data', + async () => { + const reqBody = { + protocol: { + type: 'HTTP', + parameters: { + location: MOCK_SERVER_URL + '/csv', + encoding: 'UTF-8', + }, + }, + format: { + type: 'CSV', + parameters: { + columnSeparator: ',', + lineSeparator: '\n', + skipFirstDataRow: false, + firstRowAsHeader: true, + }, + }, + }; + const response = await request(ADAPTER_URL) + .post('/preview') + .send(reqBody); + expect(response.status).toEqual(200); + const importedData = response.body.data; + const expected = [ + { + col1: 'val11', + col2: 'val12', + col3: 'val13', + }, + { + col1: 'val21', + col2: 'val22', + col3: 'val23', + }, + ]; + + expect(JSON.parse(importedData)).toEqual(expected); + }, + TIMEOUT, + ); + + test( + 'Should return 400 BAD_REQUEST for unsupported protocol [POST /preview]', + async () => { + const reqBody = { + protocol: { + type: 'UNSUPPORTED', + parameters: { + location: MOCK_SERVER_URL + '/json', + encoding: 'UTF-8', + }, + }, + format: { + type: 'JSON', + }, + }; + const response = await request(ADAPTER_URL) + .post('/preview') + .send(reqBody); + expect(response.status).toEqual(400); + }, + TIMEOUT, + ); + + test( + 'Should return 400 BAD_REQUEST for unsupported format [POST /preview]', + async () => { + const reqBody = { + protocol: { + type: 'HTTP', + parameters: { + location: MOCK_SERVER_URL + '/json', + encoding: 'UTF-8', + }, + }, + format: { + type: 'UNSUPPORTED', + }, + }; + const response = await request(ADAPTER_URL) + .post('/preview') + .send(reqBody); + expect(response.status).toEqual(400); + }, + TIMEOUT, + ); + + test( + 'Should return 400 BAD_REQUEST for invalid location [POST /preview]', + async () => { + const reqBody = { + protocol: { + type: 'HTTP', + parameters: { + location: 'invalid-location', + encoding: 'UTF-8', + }, + }, + format: { + type: 'JSON', + }, + }; + const response = await request(ADAPTER_URL) + .post('/preview') + .send(reqBody); + expect(response.status).toEqual(400); + }, + TIMEOUT, + ); + + test( + 'Should return 500 INTERNAL_SERVER_ERROR for data not found [POST /preview]', + async () => { + const reqBody = { + protocol: { + type: 'HTTP', + parameters: { + location: MOCK_SERVER_URL + '/not-found', + encoding: 'UTF-8', + }, + }, + format: { + type: 'JSON', + }, + }; + const response = await request(ADAPTER_URL) + .post('/preview') + .send(reqBody); + expect(response.status).toEqual(500); + }, + TIMEOUT, + ); +}); diff --git a/adapter/integration-test/src/mock.server.js b/adapter/integration-test/src/mock.server.js index 8207cae1a..92acf11ab 100644 --- a/adapter/integration-test/src/mock.server.js +++ b/adapter/integration-test/src/mock.server.js @@ -1,64 +1,74 @@ -const Koa = require('koa') -const Router = require('koa-router') -const router = new Router() +const Koa = require('koa'); +const Router = require('koa-router'); +const router = new Router(); -const app = new Koa() +const app = new Koa(); -const { MOCK_SERVER_PORT } = require('./env') +const { MOCK_SERVER_PORT } = require('./env'); -router.get('/', async ctx => { - ctx.type = 'text/plain' - ctx.body = 'ok' -}) +router.get('/', async (ctx) => { + ctx.type = 'text/plain'; + ctx.body = 'ok'; +}); -router.get('/not-found', async ctx => { - ctx.type = 'text/plain' - ctx.status = 404 - ctx.body = '404 NOT FOUND Error' -}) +router.get('/not-found', async (ctx) => { + ctx.type = 'text/plain'; + ctx.status = 404; + ctx.body = '404 NOT FOUND Error'; +}); -router.get('/json', async ctx => { - console.log('GET /json') - ctx.body = { whateverwillbe: 'willbe', quesera: 'sera' } -}) +router.get('/json', async (ctx) => { + console.log('GET /json'); + ctx.body = { whateverwillbe: 'willbe', quesera: 'sera' }; +}); -router.get('/json/:id', async ctx => { - console.log('Get /json/' + ctx.params.id) - ctx.body = { id: ctx.params.id } -}) +router.get('/json/:id', async (ctx) => { + console.log('Get /json/' + ctx.params.id); + ctx.body = { id: ctx.params.id }; +}); -router.get('/xml', async ctx => { - console.log('GET /xml') +router.get('/xml', async (ctx) => { + console.log('GET /xml'); - ctx.type = 'text/xml' + ctx.type = 'text/xml'; ctx.body = '' + - 'RickMorty' -}) + 'RickMorty'; +}); -router.get('/csv', async ctx => { - console.log('GET /CSV') +router.get('/xmlbigger', async (ctx) => { + console.log('GET /xml'); - ctx.type = 'text/csv' + ctx.type = 'text/xml'; ctx.body = - 'col1,col2,col3\n' + - 'val11,val12,val13\n' + - 'val21,val22,val23' -}) + '' + + 'RickMorty' + + 'helloservus' + + ''; +}); + +router.get('/csv', async (ctx) => { + console.log('GET /CSV'); + + ctx.type = 'text/csv'; + ctx.body = 'col1,col2,col3\n' + 'val11,val12,val13\n' + 'val21,val22,val23'; +}); -app.use(router.routes()) +app.use(router.routes()); -const server = app.listen(MOCK_SERVER_PORT, () => console.log('Starting mock server on port ' + MOCK_SERVER_PORT)) +const server = app.listen(MOCK_SERVER_PORT, () => + console.log('Starting mock server on port ' + MOCK_SERVER_PORT), +); process.on('SIGTERM', async () => { - console.info('Mock-Server: SIGTERM signal received.') + console.info('Mock-Server: SIGTERM signal received.'); try { - await server.close() + await server.close(); } catch (e) { - console.error('Could not shutdown server') - console.error(e) - process.exit(-1) + console.error('Could not shutdown server'); + console.error(e); + process.exit(-1); } -}) +}); -module.exports = server +module.exports = server; diff --git a/adapter/package-lock.json b/adapter/package-lock.json index 2aa6c2fd6..3c5cbd6fd 100644 --- a/adapter/package-lock.json +++ b/adapter/package-lock.json @@ -17,6 +17,7 @@ "cors": "^2.8.5", "csvtojson": "^2.0.10", "express": "^4.17.1", + "fast-xml-parser": "^4.0.7", "jackson-js": "^1.1.0", "knex": "^1.0.4", "prisma": "^3.10.0", @@ -31,6 +32,7 @@ "@types/node": "^17.0.22", "@types/supertest": "^2.0.11", "@types/uuid": "^8.3.4", + "@types/xml2js": "^0.4.11", "@typescript-eslint/eslint-plugin": "^4.30.0", "@typescript-eslint/parser": "^4.30.0", "eslint": "^7.31.0", @@ -1696,6 +1698,15 @@ "integrity": "sha512-c/I8ZRb51j+pYGAu5CrFMRxqZ2ke4y2grEBO5AUjgSkSk+qT2Ea+OdWElz/OiMf5MNpn2b17kuVBwZLQJXzihw==", "dev": true }, + "node_modules/@types/xml2js": { + "version": "0.4.11", + "resolved": "https://registry.npmjs.org/@types/xml2js/-/xml2js-0.4.11.tgz", + "integrity": "sha512-JdigeAKmCyoJUiQljjr7tQG3if9NkqGUgwEUqBvV0N7LM4HyQk7UXCnusRa1lnvXAEYJ8mw8GtZWioagNztOwA==", + "dev": true, + "dependencies": { + "@types/node": "*" + } + }, "node_modules/@types/yargs": { "version": "16.0.4", "resolved": "https://registry.npmjs.org/@types/yargs/-/yargs-16.0.4.tgz", @@ -3794,6 +3805,21 @@ "integrity": "sha1-PYpcZog6FqMMqGQ+hR8Zuqd5eRc=", "dev": true }, + "node_modules/fast-xml-parser": { + "version": "4.0.7", + "resolved": "https://registry.npmjs.org/fast-xml-parser/-/fast-xml-parser-4.0.7.tgz", + "integrity": "sha512-dMtibyus3kC7nbxj1CpVtysLzO13UOAZEFAb5vpQg3T4O6qvetmSePpXKFx5KPNCHKoGwjtgjfF5DOyn7s1ylQ==", + "dependencies": { + "strnum": "^1.0.5" + }, + "bin": { + "fxparser": "src/cli/cli.js" + }, + "funding": { + "type": "paypal", + "url": "https://paypal.me/naturalintelligence" + } + }, "node_modules/fastq": { "version": "1.13.0", "resolved": "https://registry.npmjs.org/fastq/-/fastq-1.13.0.tgz", @@ -8533,6 +8559,11 @@ "url": "https://github.com/sponsors/sindresorhus" } }, + "node_modules/strnum": { + "version": "1.0.5", + "resolved": "https://registry.npmjs.org/strnum/-/strnum-1.0.5.tgz", + "integrity": "sha512-J8bbNyKKXl5qYcR36TIO8W3mVGVHrmmxsd5PAItGkmyzwJvybiw2IVq5nqd0i4LSNSkB/sx9VHllbfFdr9k1JA==" + }, "node_modules/supports-color": { "version": "5.5.0", "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-5.5.0.tgz", @@ -10604,6 +10635,15 @@ "integrity": "sha512-c/I8ZRb51j+pYGAu5CrFMRxqZ2ke4y2grEBO5AUjgSkSk+qT2Ea+OdWElz/OiMf5MNpn2b17kuVBwZLQJXzihw==", "dev": true }, + "@types/xml2js": { + "version": "0.4.11", + "resolved": "https://registry.npmjs.org/@types/xml2js/-/xml2js-0.4.11.tgz", + "integrity": "sha512-JdigeAKmCyoJUiQljjr7tQG3if9NkqGUgwEUqBvV0N7LM4HyQk7UXCnusRa1lnvXAEYJ8mw8GtZWioagNztOwA==", + "dev": true, + "requires": { + "@types/node": "*" + } + }, "@types/yargs": { "version": "16.0.4", "resolved": "https://registry.npmjs.org/@types/yargs/-/yargs-16.0.4.tgz", @@ -12198,6 +12238,14 @@ "integrity": "sha1-PYpcZog6FqMMqGQ+hR8Zuqd5eRc=", "dev": true }, + "fast-xml-parser": { + "version": "4.0.7", + "resolved": "https://registry.npmjs.org/fast-xml-parser/-/fast-xml-parser-4.0.7.tgz", + "integrity": "sha512-dMtibyus3kC7nbxj1CpVtysLzO13UOAZEFAb5vpQg3T4O6qvetmSePpXKFx5KPNCHKoGwjtgjfF5DOyn7s1ylQ==", + "requires": { + "strnum": "^1.0.5" + } + }, "fastq": { "version": "1.13.0", "resolved": "https://registry.npmjs.org/fastq/-/fastq-1.13.0.tgz", @@ -15667,6 +15715,11 @@ "integrity": "sha512-6fPc+R4ihwqP6N/aIv2f1gMH8lOVtWQHoqC4yK6oSDVVocumAsfCqjkXnqiYMhmMwS/mEHLp7Vehlt3ql6lEig==", "dev": true }, + "strnum": { + "version": "1.0.5", + "resolved": "https://registry.npmjs.org/strnum/-/strnum-1.0.5.tgz", + "integrity": "sha512-J8bbNyKKXl5qYcR36TIO8W3mVGVHrmmxsd5PAItGkmyzwJvybiw2IVq5nqd0i4LSNSkB/sx9VHllbfFdr9k1JA==" + }, "supports-color": { "version": "5.5.0", "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-5.5.0.tgz", diff --git a/adapter/package.json b/adapter/package.json index 15cf6dc06..20f995afc 100644 --- a/adapter/package.json +++ b/adapter/package.json @@ -23,11 +23,10 @@ "cors": "^2.8.5", "csvtojson": "^2.0.10", "express": "^4.17.1", - "jackson-js": "^1.1.0", + "fast-xml-parser": "^4.0.7", "knex": "^1.0.4", "prisma": "^3.10.0", - "uuid": "^8.3.2", - "xml2js": "^0.4.23" + "uuid": "^8.3.2" }, "devDependencies": { "@jvalue/eslint-config-jvalue": "^1.1.0", diff --git a/adapter/src/adapter/api/rest/adapterEndpoint.ts b/adapter/src/adapter/api/rest/adapterEndpoint.ts index 0b6f2965a..a6e55bad1 100644 --- a/adapter/src/adapter/api/rest/adapterEndpoint.ts +++ b/adapter/src/adapter/api/rest/adapterEndpoint.ts @@ -1,20 +1,25 @@ import express from 'express'; -import { AdapterConfig, AdapterConfigValidator } from '../../model/AdapterConfig'; - -import { asyncHandler } from './utils'; -import {ProtocolConfig, ProtocolConfigValidator} from "../../model/ProtocolConfig"; +import { Importer } from '../../importer/Importer'; +import { Interpreter } from '../../interpreter/Interpreter'; +import { + AdapterConfig, + AdapterConfigValidator, +} from '../../model/AdapterConfig'; import { Format } from '../../model/enum/Format'; -import { AdapterService, adapterService } from '../../services/adapterService'; -import { FormatConfig } from '../../model/FormatConfig'; import { Protocol } from '../../model/enum/Protocol'; import { ImporterParameterError } from '../../model/exceptions/ImporterParameterError'; +import { FormatConfig } from '../../model/FormatConfig'; +import { + ProtocolConfig, + ProtocolConfigValidator, +} from '../../model/ProtocolConfig'; +import { AdapterService } from '../../services/adapterService'; +import { asyncHandler } from './utils'; -const APP_VERSION = "0.0.1" +const APP_VERSION = '0.0.1'; export class AdapterEndpoint { - constructor() {} - registerRoutes = (app: express.Application): void => { app.post('/preview', asyncHandler(this.handleExecuteDataImport)); app.post('/preview/raw', asyncHandler(this.handleExecuteRawPreview)); @@ -23,70 +28,63 @@ export class AdapterEndpoint { app.get('/version', asyncHandler(this.handleGetApplicationVersion)); }; - // Adapter Endpoint - /** - * @RestController - @AllArgsConstructor - public class AdapterEndpoint { - private final Adapter adapter; - - @PostMapping(Mappings.IMPORT_PATH) - public DataImportResponse executeDataImport(@Valid @RequestBody AdapterConfig config) - throws ImporterParameterException, InterpreterParameterException, IOException { - return adapter.executeJob(config); - } - - @PostMapping(Mappings.RAW_IMPORT_PATH) - public DataImportResponse executeRawPreview(@Valid @RequestBody ProtocolConfig config) - throws ImporterParameterException { - return adapter.executeRawImport(config); - } - } - */ - handleExecuteDataImport = async ( req: express.Request, res: express.Response, ): Promise => { const validator = new AdapterConfigValidator(); - const adapterconfigforValidator = req.body; - if (!validator.validate(adapterconfigforValidator)) { + if (!validator.validate(req.body)) { res.status(400).json({ errors: validator.getErrors() }); return; } // Check protocol type - const protocolType = AdapterEndpoint.getProtocol(req.body.protocol.type) - if(protocolType === "unsupported"){ - res.status(400).send("Protocol " + req.body.protocol.type + " not supported") + let protocolType: Importer; + try { + protocolType = AdapterEndpoint.getProtocol(req.body.protocol.type); + } catch (e) { + res.status(400).send('Protocol not supported'); return; } - const protocolConfigObj: ProtocolConfig = {protocol: new Protocol(protocolType), parameters: req.body.protocol.parameters} + + const protocolConfigObj: ProtocolConfig = { + protocol: new Protocol(protocolType), + parameters: req.body.protocol.parameters, + }; // Check format type - const formatType = AdapterEndpoint.getFormat(req.body.format.type) - if(formatType === "unsupported"){ - res.status(400).send("Format " + req.body.format.type + " not supported") + let formatType: Interpreter; + try { + formatType = AdapterEndpoint.getFormat(req.body.format.type); + } catch (e) { + res.status(400).send('Format not supported'); return; } - // Check location (???) + const format = new Format(formatType); + const formatConfigObj: FormatConfig = { + format: format, + parameters: req.body.format.parameters, + }; - const format = new Format(formatType) - const formatConfigObj: FormatConfig = {format: format, parameters: req.body.format.parameters} - const adapterConfig:AdapterConfig = {protocolConfig: protocolConfigObj, formatConfig: formatConfigObj} - console.log(adapterConfig) + const adapterConfig: AdapterConfig = { + protocolConfig: protocolConfigObj, + formatConfig: formatConfigObj, + }; + console.log(adapterConfig); - let returnDataImportResponse = null - try{ - returnDataImportResponse = await AdapterService.getInstance().executeJob(adapterConfig); - }catch(e){ - if(e instanceof ImporterParameterError){ - res.status(400).send(e.message) + try { + const returnDataImportResponse = + await AdapterService.getInstance().executeJob(adapterConfig); + res.status(200).send(returnDataImportResponse); + } catch (e) { + if (e instanceof ImporterParameterError) { + res.status(400).send(e.message); return; } + if (e instanceof Error) { + res.status(500).send(e.message); + } } - - res.status(200).send(returnDataImportResponse); }; handleExecuteRawPreview = async ( @@ -94,85 +92,84 @@ export class AdapterEndpoint { res: express.Response, ): Promise => { const validator = new ProtocolConfigValidator(); - const protcolConfigForValidator = req.body.protocol; - if (!validator.validate(protcolConfigForValidator)) { + if (!validator.validate(req.body)) { res.status(400).json({ errors: validator.getErrors() }); return; } - const protocolConfigObj: ProtocolConfig = {protocol: new Protocol(Protocol.HTTP), parameters: req.body.protocol.parameters} - const returnDataImportResponse = await AdapterService.getInstance().executeRawJob(protocolConfigObj); - res.status(200).send(returnDataImportResponse); + const protocolConfigObj: ProtocolConfig = { + protocol: new Protocol(Protocol.HTTP), + parameters: req.body.protocol.parameters, + }; + try { + const returnDataImportResponse = + await AdapterService.getInstance().executeRawJob(protocolConfigObj); + res.status(200).send(returnDataImportResponse); + } catch (e) { + if (e instanceof ImporterParameterError) { + res.status(400).send(e.message); + return; + } + if (e instanceof Error) { + res.status(500).send(e.message); + } + } }; /* - returns Collection of Importerj + Returns Collection of Interpreter } */ - handleGetFormat = async ( - req: express.Request, - res: express.Response, - ): Promise => { - try { - const interpreters = AdapterService.getInstance().getAllFormats(); - res.setHeader("Content-Type", "application/json") - res.status(200).json(interpreters); - } catch (e) { - //res.status(500).send('Error finding formats'); - throw e - } + handleGetFormat = (req: express.Request, res: express.Response): void => { + const interpreters = AdapterService.getInstance().getAllFormats(); + res.setHeader('Content-Type', 'application/json'); + res.status(200).json(interpreters); }; /* - returns Collection of Importer + Returns Collection of Importer */ - handleGetProtocols = async ( - req: express.Request, - res: express.Response, - ): Promise => { + handleGetProtocols = (req: express.Request, res: express.Response): void => { try { - const protocols = AdapterService.getInstance().getAllProtocols(); - res.status(200).json(protocols); - } catch (e) { - res.status(500).send('Error finding protocols'); - } + const protocols = AdapterService.getInstance().getAllProtocols(); + res.status(200).json(protocols); + } catch (e) { + res.status(500).send('Error finding protocols'); + } }; - handleGetApplicationVersion = async ( + handleGetApplicationVersion = ( req: express.Request, res: express.Response, - ): Promise => { - res.setHeader("Content-Type", "text/plain"); + ): void => { + res.setHeader('Content-Type', 'text/plain'); res.status(200).send(APP_VERSION); }; - static getFormat(type: any): any { - switch(type) { - case "JSON": { - return Format.JSON; + static getFormat(type: string): Interpreter { + switch (type) { + case 'JSON': { + return Format.JSON; } - case "CSV": { - return Format.CSV; + case 'CSV': { + return Format.CSV; } - case "XML": { + case 'XML': { return Format.XML; - } + } default: { - return "unsupported"; + throw new Error('Format not found'); } - } + } } - static getProtocol(type: any): any { - switch(type) { - case "HTTP": { - return Protocol.HTTP; + static getProtocol(type: string): Importer { + switch (type) { + case 'HTTP': { + return Protocol.HTTP; } default: { - return "unsupported" + throw new Error('Protocol not found'); } - } + } } - -}; - - +} diff --git a/adapter/src/adapter/api/rest/utils.ts b/adapter/src/adapter/api/rest/utils.ts index 8e793a472..6061d3835 100644 --- a/adapter/src/adapter/api/rest/utils.ts +++ b/adapter/src/adapter/api/rest/utils.ts @@ -16,4 +16,4 @@ export function asyncHandler( const handlerResult = handler(req, res, next); Promise.resolve(handlerResult).catch(next); }; -} \ No newline at end of file +} diff --git a/adapter/src/adapter/importer/HttpImporter.ts b/adapter/src/adapter/importer/HttpImporter.ts index 883cc5bda..201b61037 100644 --- a/adapter/src/adapter/importer/HttpImporter.ts +++ b/adapter/src/adapter/importer/HttpImporter.ts @@ -1,88 +1,82 @@ -import { ImporterParameterError } from "../model/exceptions/ImporterParameterError"; -import { Importer } from "./Importer"; -import { ImporterParameterDescription } from "./ImporterParameterDescription"; -const axios = require('axios'); +import axios, { AxiosError } from 'axios'; -export class HttpImporter extends Importer { +import { ImporterParameterError } from '../model/exceptions/ImporterParameterError'; - //TODO RuntimeParameters type is probably wrong - type: string = "HTTP" - description: string = "Plain HTTP" - parameters:ImporterParameterDescription[] = [new ImporterParameterDescription({name:"location", description:"String of the URI for the HTTP call", type:"string"}), - new ImporterParameterDescription({name:"encoding", description:"Encoding of the source. Available encodings: ISO-8859-1, US-ASCII, UTF-8", type:"string"}), - new ImporterParameterDescription({name:"defaultParameters", description:"Default values for open parameters in the URI", required:false, type:"RuntimeParameters"})] +import { Importer } from './Importer'; +import { ImporterParameterDescription } from './ImporterParameterDescription'; - // Override annotation is not necessary, but will be used for a better understanding of the code - override getType(): string { - return this.type - } - - override getDescription(): string { - return this.description - } - - override getAvailableParameters(): ImporterParameterDescription[] { - return this.parameters; - } +export class HttpImporter extends Importer { + type = 'HTTP'; + description = 'Plain HTTP'; + parameters: ImporterParameterDescription[] = [ + new ImporterParameterDescription({ + name: 'location', + description: 'String of the URI for the HTTP call', + type: 'string', + }), + new ImporterParameterDescription({ + name: 'encoding', + description: + 'Encoding of the source. Available encodings: ISO-8859-1, US-ASCII, UTF-8', + type: 'string', + }), + new ImporterParameterDescription({ + name: 'defaultParameters', + description: 'Default values for open parameters in the URI', + required: false, + type: 'RuntimeParameters', + }), + ]; - override validateParameters(inputParameters: Record): void { - super.validateParameters(inputParameters); - const encoding: string = inputParameters.encoding as string; + override getType(): string { + return this.type; + } - // TODO CHECK IF ENCODING ARE WRITTEN CORRECT - if (encoding !== "ISO-8859-1" && encoding !== "US-ASCII" && encoding !== "UTF-8") { - throw new Error(this.getType() + " interpreter requires parameter encoding to have value " + - "ISO-8859-1" + ", " + - "US-ASCII" + ", " + - "UTF-8" - + ". Your given value " + encoding + " is invalid!"); - } - } - /** - protected void validateParameters(Map inputParameters) throws ImporterParameterException { - super.validateParameters(inputParameters); + override getDescription(): string { + return this.description; + } - String encoding = (String) inputParameters.get("encoding"); - if (!encoding.equals(StandardCharsets.ISO_8859_1.name()) && !encoding.equals(StandardCharsets.US_ASCII.name()) && !encoding.equals(StandardCharsets.UTF_8.name())) { - throw new IllegalArgumentException(getType() + " interpreter requires parameter encoding to have value " + - StandardCharsets.ISO_8859_1 + ", " + - StandardCharsets.US_ASCII + ", " + - StandardCharsets.UTF_8 - + ". Your given value " + encoding + " is invalid!"); - } - } - */ - override async doFetch(parameters: Record): Promise { - let uri = parameters.location - console.log(parameters) - let encoding = parameters.encoding - // TODO see if encoding from response is good - return axios({ - method: 'get', - url: uri, - responseEncoding: encoding - }).then(function (response: any) { - console.log(response.data) - return response.data - }).catch(function (error: any) { - console.error(error) - throw new ImporterParameterError("Could not Fetch from URI:" + uri) - }); - } + override getAvailableParameters(): ImporterParameterDescription[] { + return this.parameters; + } - /* + override validateParameters(inputParameters: Record): void { + super.validateParameters(inputParameters); + const encoding: string = inputParameters.encoding as string; - @Override - protected String doFetch(Map parameters) throws ImporterParameterException { - String location = parameters.get("location").toString(); - try { - URI uri = URI.create(location); - byte[] rawResponse = restTemplate.getForEntity(uri, byte[].class).getBody(); - return new String(rawResponse, Charset.forName((String) parameters.get("encoding"))); - } catch (IllegalArgumentException e) { - throw new ImporterParameterException(e.getMessage()); + if ( + encoding !== 'ISO-8859-1' && + encoding !== 'US-ASCII' && + encoding !== 'UTF-8' + ) { + throw new Error( + this.getType() + + ' interpreter requires parameter encoding to have value ' + + 'ISO-8859-1' + + ', ' + + 'US-ASCII' + + ', ' + + 'UTF-8' + + '. Your given value ' + + encoding + + ' is invalid!', + ); } } - */ -} \ No newline at end of file + override async doFetch(parameters: Record): Promise { + const uri = parameters.location as string; + const encoding = parameters.encoding as string; + + const result = await axios + .get(uri, { responseEncoding: encoding }) + .catch((error: AxiosError) => { + if (error.response) { + console.log(error.response); + throw new Error('Could not Fetch from URI:' + uri); + } + throw new ImporterParameterError('Could not Fetch from URI:' + uri); + }); + return result.data as string; + } +} diff --git a/adapter/src/adapter/importer/Importer.ts b/adapter/src/adapter/importer/Importer.ts index 0f60690fb..10916cf80 100644 --- a/adapter/src/adapter/importer/Importer.ts +++ b/adapter/src/adapter/importer/Importer.ts @@ -1,71 +1,82 @@ -import { ImporterParameterDescription } from "./ImporterParameterDescription"; -import { ImporterParameterError } from "../model/exceptions/ImporterParameterError"; -import { generateKeySync } from "crypto"; +import { ImporterParameterError } from '../model/exceptions/ImporterParameterError'; + +import { ImporterParameterDescription } from './ImporterParameterDescription'; export abstract class Importer { type: string | undefined; description: string | undefined; getRequiredParameters(): Array { - return this.getAvailableParameters().filter((item: any) => item.required) as Array + return this.getAvailableParameters().filter( + (item: ImporterParameterDescription) => item.required, + ); } - //@JsonProperty("parameters") - abstract getAvailableParameters() :Array; + abstract getAvailableParameters(): Array; - async fetch(parameters:Record ): Promise { //throws ImporterParameterException - this.validateParameters(parameters); - const x = await this.doFetch(parameters); - return x; - //return JSON.stringify(x); + async fetch(parameters: Record): Promise { + this.validateParameters(parameters); + const x = await this.doFetch(parameters); + return x; } abstract getType(): string; abstract getDescription(): string; - abstract doFetch(parameters: Record): Promise; //throws ImporterParameterException - - validateParameters(inputParameters: Record) { //throws ImporterParameterException; + abstract doFetch(parameters: Record): Promise; - let illegalArguments: boolean = false; - let illegalArgumentsMessage: string = ""; + validateParameters(inputParameters: Record): void { + let illegalArguments = false; + let illegalArgumentsMessage = ''; - const possibleParameters: Array = this.getAvailableParameters(); + const possibleParameters: Array = + this.getAvailableParameters(); - let unnecessaryArguments = []; - const names = possibleParameters.map(a => a.name); + const unnecessaryArguments = []; + const names = possibleParameters.map((a) => a.name); const keys = Object.keys(inputParameters); for (const entry of keys) { - if(!names.includes(entry)) { + if (!names.includes(entry)) { unnecessaryArguments.push(entry); } } - if(unnecessaryArguments.length > 0){ + if (unnecessaryArguments.length > 0) { illegalArguments = true; - for(const argument of unnecessaryArguments){ - illegalArgumentsMessage += argument + " is not needed by importer \n" + for (const argument of unnecessaryArguments) { + illegalArgumentsMessage += argument + ' is not needed by importer \n'; } } - const requiredParameters = this.getRequiredParameters() - for (const requiredParameter of requiredParameters){ - // TODO is that OK? - const checkType = (inputParameters[requiredParameter.name] as any).constructor.name - if (inputParameters[requiredParameter.name] == null){ + + const requiredParameters = this.getRequiredParameters(); + for (const requiredParameter of requiredParameters) { + const param = inputParameters[ + requiredParameter.name + ] as ImporterParameterDescription; + + if (param === undefined) { illegalArguments = true; - illegalArgumentsMessage = illegalArgumentsMessage + this.type + "importer requires parameter " + requiredParameter.name + "\n"; + illegalArgumentsMessage += this.type; + illegalArgumentsMessage += 'importer requires parameter '; + illegalArgumentsMessage += requiredParameter.name; + illegalArgumentsMessage += '\n'; + break; } - - else if(checkType.toLowerCase() != requiredParameter.type){ + const checkType = param.constructor.name; + if (checkType.toLowerCase() !== requiredParameter.type) { illegalArguments = true; - illegalArgumentsMessage = illegalArgumentsMessage + this.type + " importer requires parameter " - + requiredParameter.name + " to be type " + (requiredParameter.type as string) + "\n"; + illegalArgumentsMessage += this.type; + illegalArgumentsMessage += ' importer requires parameter '; + illegalArgumentsMessage += requiredParameter.name; + illegalArgumentsMessage += ' to be type '; + illegalArgumentsMessage += requiredParameter.type; + illegalArgumentsMessage += '\n'; + break; } } - if(illegalArguments){ + if (illegalArguments) { throw new ImporterParameterError(illegalArgumentsMessage); } - } } diff --git a/adapter/src/adapter/importer/ImporterParameterDescription.ts b/adapter/src/adapter/importer/ImporterParameterDescription.ts index 00696e168..319dc2073 100644 --- a/adapter/src/adapter/importer/ImporterParameterDescription.ts +++ b/adapter/src/adapter/importer/ImporterParameterDescription.ts @@ -1,15 +1,23 @@ export class ImporterParameterDescription { + name: string; + description: string; + required: boolean; + type: unknown; + + constructor({ + name, + description, + required = true, + type, + }: { name: string; description: string; - required: boolean; - - //TODO @Georg: need to check how to store the class in typescript, can use instance.constructor.name to get the name probably -> is stored as string + required?: boolean; type: unknown; - - constructor ({name,description,required=true,type}: {name:string, description:string, required?:boolean, type:unknown}) { - this.name = name; - this.description = description; - this.required = required; - this.type = type; - } -} \ No newline at end of file + }) { + this.name = name; + this.description = description; + this.required = required; + this.type = type; + } +} diff --git a/adapter/src/adapter/interpreter/CsvInterpreter.ts b/adapter/src/adapter/interpreter/CsvInterpreter.ts index d3c97cabf..e5effca0c 100644 --- a/adapter/src/adapter/interpreter/CsvInterpreter.ts +++ b/adapter/src/adapter/interpreter/CsvInterpreter.ts @@ -1,78 +1,103 @@ -import {Interpreter} from "./Interpreter"; -import { InterpreterParameterDescription } from "./InterpreterParameterDescription"; -const csv=require('csvtojson') +import csv from 'csvtojson'; +import { Interpreter } from './Interpreter'; +import { InterpreterParameterDescription } from './InterpreterParameterDescription'; export class CsvInterpreter extends Interpreter { + type = 'CSV'; - type: string = "CSV" - description: string = "Interpret data as CSV data" - parameters: InterpreterParameterDescription[] = [new InterpreterParameterDescription("columnSeparator", "Column delimiter character, only one character supported", "string"), - new InterpreterParameterDescription("lineSeparator", "Line delimiter character, only \\r, \\r\\n, and \\n supported", "string",), - new InterpreterParameterDescription("skipFirstDataRow", "Skip first data row (after header)", "boolean"), - new InterpreterParameterDescription("firstRowAsHeader", "Interpret first row as header for columns", "boolean")] - - - + description = 'Interpret data as CSV data'; + parameters: InterpreterParameterDescription[] = [ + new InterpreterParameterDescription( + 'columnSeparator', + 'Column delimiter character, only one character supported', + 'string', + ), + new InterpreterParameterDescription( + 'lineSeparator', + 'Line delimiter character, only \\r, \\r\\n, and \\n supported', + 'string', + ), + new InterpreterParameterDescription( + 'skipFirstDataRow', + 'Skip first data row (after header)', + 'boolean', + ), + new InterpreterParameterDescription( + 'firstRowAsHeader', + 'Interpret first row as header for columns', + 'boolean', + ), + ]; override getType(): string { - return this.type + return this.type; } override getDescription(): string { - return this.description + return this.description; } override getAvailableParameters(): InterpreterParameterDescription[] { return this.parameters; } - override doInterpret(data: string, parameters: Record): Promise { - data = 'col1,col2,col3\n' + - 'val11,val12,val13\n' + - 'val21,val22,val23'; + override async doInterpret( + data: string, + parameters: Record, + ): Promise { + const columnSeparator = (parameters.columnSeparator as string).charAt(0); + const lineSeparator: string = parameters.lineSeparator as string; + // Be Careful: Need to Invert the boolean here + // True = With header, False = WithoutHeader + const firstRowAsHeader = !parameters.firstRowAsHeader; + const skipFirstDataRow: boolean = parameters.skipFirstDataRow as boolean; - let columnSeparator = (parameters.columnSeparator as string).charAt(0) - let lineSeparator: string = parameters.lineSeparator as string; - let firstRowAsHeader: boolean = parameters.firstRowAsHeader as boolean; // True = With header, False = WithoutHeader - let skipFirstDataRow: boolean = parameters.skipFirstDataRow as boolean; - - let json: any[] = []; - return csv({ - noheader: !firstRowAsHeader, // Be Careful: Need to Invert the boolean here - output: "json", + const json: string[] = []; + await csv({ + noheader: firstRowAsHeader, + output: 'json', delimiter: columnSeparator, - eol: lineSeparator + eol: lineSeparator, }) - .fromString(data) - .subscribe((csvRow: any, index:any)=>{ - // Todo need to test if this works - if(skipFirstDataRow && ((index == 0 && !firstRowAsHeader) || (index == 1 && firstRowAsHeader))) { - // Skip First Row - } - else { - json.push(csvRow); - } - }).on('done', (error:any) => { - return new Promise(function(resolve, reject){ - resolve(JSON.stringify(json)); + .fromString(data) + .subscribe((csvRow: string, index: number) => { + if (skipFirstDataRow && index === 0) { + // Skip First Row + } else { + json.push(csvRow); + } }); - }) - - + return new Promise(function (resolve) { + resolve(JSON.stringify(json)); + }); } override validateParameters(inputParameters: Record): void { - super.validateParameters(inputParameters); - const lineSeparator: string = inputParameters.lineSeparator as string; + super.validateParameters(inputParameters); + const lineSeparator: string = inputParameters.lineSeparator as string; - if (lineSeparator !== "\n" && lineSeparator !== "\r" && lineSeparator !== "\r\n") { - throw new Error(this.getType() + " interpreter requires parameter lineSeparator to have" + - " value \\n, \\r, or \\r\\n. Your given value " + lineSeparator + " is invalid!"); - } + if ( + lineSeparator !== '\n' && + lineSeparator !== '\r' && + lineSeparator !== '\r\n' + ) { + throw new Error( + this.getType() + + ' interpreter requires parameter lineSeparator to have' + + ' value \\n, \\r, or \\r\\n. Your given value ' + + lineSeparator + + ' is invalid!', + ); + } - var columnSeparator: string = inputParameters.columnSeparator as string; - if (columnSeparator.length !== 1) { - throw new Error(this.getType() + " interpreter requires parameter columnSeparator to have" + - " length 1. Your given value " + columnSeparator + " is invalid!"); + const columnSeparator: string = inputParameters.columnSeparator as string; + if (columnSeparator.length !== 1) { + throw new Error( + this.getType() + + ' interpreter requires parameter columnSeparator to have' + + ' length 1. Your given value ' + + columnSeparator + + ' is invalid!', + ); } } } diff --git a/adapter/src/adapter/interpreter/Interpreter.ts b/adapter/src/adapter/interpreter/Interpreter.ts index 9f35c5d8c..cd9365338 100644 --- a/adapter/src/adapter/interpreter/Interpreter.ts +++ b/adapter/src/adapter/interpreter/Interpreter.ts @@ -1,40 +1,81 @@ -import { stringifiers } from "@jvalue/node-dry-basics"; -import { InterpreterParameterError } from "../model/exceptions/InterpreterParameterError"; -import { InterpreterParameterDescription } from "./InterpreterParameterDescription"; +import { InterpreterParameterError } from '../model/exceptions/InterpreterParameterError'; + +import { InterpreterParameterDescription } from './InterpreterParameterDescription'; export abstract class Interpreter { type: string | undefined; description: string | undefined; - - async interpret(data: string, parameters: Record): Promise { + async interpret( + data: string, + parameters: Record, + ): Promise { this.validateParameters(parameters); return await this.doInterpret(data, parameters); } abstract getType(): string; abstract getDescription(): string; - abstract doInterpret(data: string, parameters: Record): Promise + abstract doInterpret( + data: string, + parameters?: Record, + ): Promise; abstract getAvailableParameters(): Array; - validateParameters(inputParameters: Record) { - let illegalArguments: boolean = false; - let illegalArgumentsMessage: string = ""; - - for (const requiredParameter of this.getAvailableParameters()){ - const param = (inputParameters[requiredParameter.name] as InterpreterParameterDescription) - if (param == null){ + validateParameters(inputParameters: Record): void { + let illegalArguments = false; + let illegalArgumentsMessage = ''; + + const possibleParameters: Array = + this.getAvailableParameters(); + + if (possibleParameters.length === 0) { + return; + } + + const unnecessaryArguments = []; + const names = possibleParameters.map((a) => a.name); + const keys = Object.keys(inputParameters); + + for (const entry of keys) { + if (!names.includes(entry)) { + unnecessaryArguments.push(entry); + } + } + + if (unnecessaryArguments.length > 0) { + illegalArguments = true; + for (const argument of unnecessaryArguments) { + illegalArgumentsMessage += argument + ' is not needed by importer \n'; + } + } + const requiredParameters = this.getAvailableParameters(); + for (const requiredParameter of requiredParameters) { + const param = inputParameters[ + requiredParameter.name + ] as InterpreterParameterDescription; + + if (param === undefined) { illegalArguments = true; - illegalArgumentsMessage = illegalArgumentsMessage + this.type + "interpreter requires parameter " + requiredParameter.name + "\n"; + illegalArgumentsMessage += this.type; + illegalArgumentsMessage += 'interpreter requires parameter '; + illegalArgumentsMessage += requiredParameter.name; + illegalArgumentsMessage += '\n'; + break; } - // TODO is that OK? - /*else if(((inputParameters.requiredParameter as InterpreterParameterDescription).name) as any)).constructor.name != requiredParameter.type){ + const checkType = param.constructor.name; + if (checkType.toLowerCase() !== requiredParameter.type) { illegalArguments = true; - illegalArgumentsMessage = illegalArgumentsMessage + this.type + " interpreter requires parameter " - + requiredParameter.name + " to be type " + (requiredParameter.type as string) + "\n"; - }*/ + illegalArgumentsMessage += this.type; + illegalArgumentsMessage += ' interpreter requires parameter '; + illegalArgumentsMessage += requiredParameter.name; + illegalArgumentsMessage += ' to be type '; + illegalArgumentsMessage += requiredParameter.type; + illegalArgumentsMessage += '\n'; + break; + } } - if(illegalArguments){ + if (illegalArguments) { throw new InterpreterParameterError(illegalArgumentsMessage); } } diff --git a/adapter/src/adapter/interpreter/InterpreterParameterDescription.ts b/adapter/src/adapter/interpreter/InterpreterParameterDescription.ts index 59252aabc..fac5eaff0 100644 --- a/adapter/src/adapter/interpreter/InterpreterParameterDescription.ts +++ b/adapter/src/adapter/interpreter/InterpreterParameterDescription.ts @@ -1,15 +1,12 @@ -export class InterpreterParameterDescription{ +export class InterpreterParameterDescription { + name: string; + description: string; + required: boolean | undefined; + type: unknown; - name: string; - description: string; - // TODO default value? is not used in Interpreters - required: boolean | undefined; - //TODO @Georg: need to check how to store the class in typescript, can use instance.constructor.name to get the name probably -> is stored as string -> Same as in Importer - type: unknown; - - constructor (name: string, description: string, type: unknown) { - this.name = name; - this.description = description; - this.type = type; - } -} \ No newline at end of file + constructor(name: string, description: string, type: unknown) { + this.name = name; + this.description = description; + this.type = type; + } +} diff --git a/adapter/src/adapter/interpreter/JsonInterpreter.ts b/adapter/src/adapter/interpreter/JsonInterpreter.ts index 60972e6b7..e2025704b 100644 --- a/adapter/src/adapter/interpreter/JsonInterpreter.ts +++ b/adapter/src/adapter/interpreter/JsonInterpreter.ts @@ -1,27 +1,30 @@ -import {Interpreter} from "./Interpreter"; -import { InterpreterParameterDescription } from "./InterpreterParameterDescription"; +import { Interpreter } from './Interpreter'; +import { InterpreterParameterDescription } from './InterpreterParameterDescription'; export class JsonInterpreter extends Interpreter { - - type: string = "JSON" - description: string = "Interpret data as JSON data" - parameters: InterpreterParameterDescription[] = [] + type = 'JSON'; + description = 'Interpret data as JSON data'; + parameters: InterpreterParameterDescription[] = []; override getType(): string { - return this.type + return this.type; } - + override getDescription(): string { - return this.description + return this.description; } override getAvailableParameters(): InterpreterParameterDescription[] { return this.parameters; } - override doInterpret(data: string, parameters: Record): Promise { - return new Promise(function(resolve, reject){ - resolve(data); - }); + override doInterpret( + data: string, + // eslint-disable-next-line @typescript-eslint/no-unused-vars + _parameters: Record, + ): Promise { + return new Promise(function (resolve) { + resolve(data); + }); } -} \ No newline at end of file +} diff --git a/adapter/src/adapter/interpreter/XmlInterpreter.ts b/adapter/src/adapter/interpreter/XmlInterpreter.ts index 3ec566f1b..77b224e89 100644 --- a/adapter/src/adapter/interpreter/XmlInterpreter.ts +++ b/adapter/src/adapter/interpreter/XmlInterpreter.ts @@ -1,38 +1,38 @@ -import {Interpreter} from "./Interpreter"; -import { InterpreterParameterDescription } from "./InterpreterParameterDescription"; -const xml2js = require('xml2js'); +import { XMLParser } from 'fast-xml-parser'; -export class XmlInterpreter extends Interpreter{ +import { Interpreter } from './Interpreter'; +import { InterpreterParameterDescription } from './InterpreterParameterDescription'; - type: string = "XML" - description: string = "Interpret data as XML data" - parameters: InterpreterParameterDescription[] = [] +export class XmlInterpreter extends Interpreter { + type = 'XML'; + description = 'Interpret data as XML data'; + parameters: InterpreterParameterDescription[] = []; override getType(): string { - return this.type + return this.type; } override getDescription(): string { - return this.description + return this.description; } override getAvailableParameters(): InterpreterParameterDescription[] { return this.parameters; } - // TODO @Georg check if this package can be used.. - override doInterpret(data: string, parameters: Record): Promise { - const parser = new xml2js.Parser({explicitArray: false}); - - return parser.parseStringPromise(data).then(function (result:any) { - // `result` is a JavaScript object - // convert it to a JSON string - return result.root - //const json = JSON.stringify(result.root); - //return json; - }) - .catch(function (err:any) { - throw err - }); + override doInterpret( + data: string, + // eslint-disable-next-line @typescript-eslint/no-unused-vars + parameters: Record, + ): Promise { + const options = { + ignoreDeclaration: true, + }; + + const parser = new XMLParser(options); + const result = parser.parse(data) as Record; + return new Promise(function (resolve) { + resolve(JSON.stringify(result)); + }); } } diff --git a/adapter/src/adapter/model/AdapterConfig.ts b/adapter/src/adapter/model/AdapterConfig.ts index 9b60da09e..a3f9d5d1e 100644 --- a/adapter/src/adapter/model/AdapterConfig.ts +++ b/adapter/src/adapter/model/AdapterConfig.ts @@ -1,7 +1,8 @@ -import { FormatConfig } from "./FormatConfig"; -import { ProtocolConfig } from "./ProtocolConfig"; import { validators } from '@jvalue/node-dry-basics'; -import { JsonAlias, JsonClassType, JsonProperty } from "jackson-js"; + +import { AdapterConfigDTO } from './EndpointDTOs'; +import { FormatConfig } from './FormatConfig'; +import { ProtocolConfig } from './ProtocolConfig'; export interface AdapterConfig { protocolConfig: ProtocolConfig; @@ -11,7 +12,7 @@ export interface AdapterConfig { export class AdapterConfigValidator { private errors: string[] = []; - validate(request: unknown): request is AdapterConfig { + validate(request: AdapterConfigDTO): request is AdapterConfigDTO { this.errors = []; if (!validators.isObject(request)) { this.errors.push("'AdapterConfig' must be an object"); diff --git a/adapter/src/adapter/model/EndpointDTOs.ts b/adapter/src/adapter/model/EndpointDTOs.ts new file mode 100644 index 000000000..f3a2fb23b --- /dev/null +++ b/adapter/src/adapter/model/EndpointDTOs.ts @@ -0,0 +1,12 @@ +export interface AdapterConfigDTO { + protocol: ProtocolConfigDTO; + format: FormatConfigDTO; +} +export interface ProtocolConfigDTO { + type: string; + parameters: Record; +} +export interface FormatConfigDTO { + type: string; + parameters: Record; +} diff --git a/adapter/src/adapter/model/FormatConfig.ts b/adapter/src/adapter/model/FormatConfig.ts index ba600fefd..b7fbc5a12 100644 --- a/adapter/src/adapter/model/FormatConfig.ts +++ b/adapter/src/adapter/model/FormatConfig.ts @@ -1,6 +1,6 @@ -import { Format } from "./enum/Format"; +import { Format } from './enum/Format'; export interface FormatConfig { - format:Format; - parameters: Record; - } + format: Format; + parameters: Record; +} diff --git a/adapter/src/adapter/model/ProtocolConfig.ts b/adapter/src/adapter/model/ProtocolConfig.ts index 7bc9ebf3d..382ad949f 100644 --- a/adapter/src/adapter/model/ProtocolConfig.ts +++ b/adapter/src/adapter/model/ProtocolConfig.ts @@ -1,30 +1,31 @@ -import { Protocol } from "./enum/Protocol"; import { validators } from '@jvalue/node-dry-basics'; -import { stringify } from "querystring"; -import { JsonAlias, JsonClassType, JsonProperty } from "jackson-js"; + +import { AdapterConfigDTO } from './EndpointDTOs'; +import { Protocol } from './enum/Protocol'; export interface ProtocolConfig { - protocol:Protocol; - parameters: Record; + protocol: Protocol; + parameters: Record; } export class ProtocolConfigValidator { private errors: string[] = []; - validate(request: unknown): request is ProtocolConfig { + validate(request: AdapterConfigDTO): request is AdapterConfigDTO { this.errors = []; + if (!validators.isObject(request)) { this.errors.push("'ProtocolConfig' must be an object"); return false; } - if (!validators.hasProperty(request, 'type')) { + if (!validators.hasProperty(request.protocol, 'type')) { this.errors.push("'type' property is missing"); - } else if (!validators.isString(request.type)) { + } else if (!validators.isString(request.protocol.type)) { this.errors.push("'type' must be a string"); } - if (!validators.hasProperty(request, 'parameters')) { + if (!validators.hasProperty(request.protocol, 'parameters')) { this.errors.push("'parameters' property is missing"); - } else if (!validators.isObject(request.parameters)) { + } else if (!validators.isObject(request.protocol.parameters)) { this.errors.push("'parameters' must be an object or array"); } return this.errors.length === 0; diff --git a/adapter/src/adapter/model/enum/Format.ts b/adapter/src/adapter/model/enum/Format.ts index 6262f2da9..eb41db3dc 100644 --- a/adapter/src/adapter/model/enum/Format.ts +++ b/adapter/src/adapter/model/enum/Format.ts @@ -1,19 +1,19 @@ -import { CsvInterpreter } from "../../interpreter/CsvInterpreter"; -import { Interpreter } from "../../interpreter/Interpreter"; -import { JsonInterpreter } from "../../interpreter/JsonInterpreter"; -import { XmlInterpreter } from "../../interpreter/XmlInterpreter"; +import { CsvInterpreter } from '../../interpreter/CsvInterpreter'; +import { Interpreter } from '../../interpreter/Interpreter'; +import { JsonInterpreter } from '../../interpreter/JsonInterpreter'; +import { XmlInterpreter } from '../../interpreter/XmlInterpreter'; export class Format { - static readonly JSON = new JsonInterpreter(); + static readonly JSON = new JsonInterpreter(); static readonly XML = new XmlInterpreter(); - static readonly CSV = new CsvInterpreter(); + static readonly CSV = new CsvInterpreter(); interpreter: Interpreter; constructor(interpreter: Interpreter) { this.interpreter = interpreter; } - getInterpreter() { + getInterpreter(): Interpreter { return this.interpreter; } -} \ No newline at end of file +} diff --git a/adapter/src/adapter/model/enum/Protocol.ts b/adapter/src/adapter/model/enum/Protocol.ts index c75813883..a20a1aafd 100644 --- a/adapter/src/adapter/model/enum/Protocol.ts +++ b/adapter/src/adapter/model/enum/Protocol.ts @@ -1,15 +1,15 @@ -import { HttpImporter } from "../../importer/HttpImporter"; -import { Importer } from "../../importer/Importer"; +import { HttpImporter } from '../../importer/HttpImporter'; +import { Importer } from '../../importer/Importer'; export class Protocol { - static readonly HTTP = new HttpImporter(); - + static readonly HTTP = new HttpImporter(); + importer: Importer; constructor(importer: Importer) { this.importer = importer; } - getImporter() { - return this.importer + getImporter(): Importer { + return this.importer; } -} \ No newline at end of file +} diff --git a/adapter/src/adapter/model/exceptions/AdapterError.ts b/adapter/src/adapter/model/exceptions/AdapterError.ts index 3a235cda0..026d48bd4 100644 --- a/adapter/src/adapter/model/exceptions/AdapterError.ts +++ b/adapter/src/adapter/model/exceptions/AdapterError.ts @@ -1,5 +1,5 @@ -export class AdapterError extends Error{ - constructor(msg: string){ - super(msg); - } -} \ No newline at end of file +export class AdapterError extends Error { + constructor(msg: string) { + super(msg); + } +} diff --git a/adapter/src/adapter/model/exceptions/ImporterParameterError.ts b/adapter/src/adapter/model/exceptions/ImporterParameterError.ts index f7ffcd91d..3278dd1ac 100644 --- a/adapter/src/adapter/model/exceptions/ImporterParameterError.ts +++ b/adapter/src/adapter/model/exceptions/ImporterParameterError.ts @@ -1,8 +1,7 @@ -import { AdapterError } from "./AdapterError"; +import { AdapterError } from './AdapterError'; -export class ImporterParameterError extends AdapterError{ - - constructor(msg: string){ - super(msg); - } -} \ No newline at end of file +export class ImporterParameterError extends AdapterError { + constructor(msg: string) { + super(msg); + } +} diff --git a/adapter/src/adapter/model/exceptions/InterpreterParameterError.ts b/adapter/src/adapter/model/exceptions/InterpreterParameterError.ts index 238ce7666..0fef375a5 100644 --- a/adapter/src/adapter/model/exceptions/InterpreterParameterError.ts +++ b/adapter/src/adapter/model/exceptions/InterpreterParameterError.ts @@ -1,8 +1,7 @@ -import { AdapterError } from "./AdapterError"; +import { AdapterError } from './AdapterError'; -export class InterpreterParameterError extends AdapterError{ - - constructor(msg: string){ - super(msg); - } -} \ No newline at end of file +export class InterpreterParameterError extends AdapterError { + constructor(msg: string) { + super(msg); + } +} diff --git a/adapter/src/adapter/services/adapterService.ts b/adapter/src/adapter/services/adapterService.ts index 03ab543e3..bcded2897 100644 --- a/adapter/src/adapter/services/adapterService.ts +++ b/adapter/src/adapter/services/adapterService.ts @@ -1,63 +1,77 @@ -import { JsonRawValue } from "jackson-js"; -import { Importer } from "../importer/Importer"; -import { Interpreter } from "../interpreter/Interpreter"; -import { AdapterConfig } from "../model/AdapterConfig"; -import { DataImportResponse } from "../model/DataImportResponse"; -import { Format } from "../model/enum/Format"; -import { Protocol } from "../model/enum/Protocol"; -import { FormatConfig } from "../model/FormatConfig"; - -import { ProtocolConfig } from "../model/ProtocolConfig"; +import { Importer } from '../importer/Importer'; +import { Interpreter } from '../interpreter/Interpreter'; +import { AdapterConfig } from '../model/AdapterConfig'; +import { DataImportResponse } from '../model/DataImportResponse'; +import { Format } from '../model/enum/Format'; +import { Protocol } from '../model/enum/Protocol'; +import { FormatConfig } from '../model/FormatConfig'; +import { ProtocolConfig } from '../model/ProtocolConfig'; export class AdapterService { - /** - * @description Create an instance of AdapterService - */ - private static instance: AdapterService; + /** + * @description Create an instance of AdapterService + */ + private static instance: AdapterService; - constructor () { + static getInstance(): AdapterService { + if (!AdapterService.instance) { + AdapterService.instance = new AdapterService(); } - public static getInstance(): AdapterService { - if (!AdapterService.instance) { - AdapterService.instance = new AdapterService(); - } + return AdapterService.instance; + } - return AdapterService.instance; - } + getAllFormats(): Array { + return [Format.CSV, Format.JSON, Format.XML]; + } + getAllProtocols(): Array { + return [Protocol.HTTP]; + } - // To Implement - public getAllFormats(): Array { - return [Format.CSV, Format.JSON, Format.XML] - } + /** + * Executes an adapter configuration + * + * @param _adapterConfig the adapter configuration + * @return the imported and interpreted data + * @throws ImporterParameterError on errors in the interpreter config (e.g. missing parameters, ...) + * @throws InterpreterParameterError on errors in the interpreter config (e.g. missing parameters, ...) + * @throws Error on response errors when importing the data + */ + async executeJob(_adapterConfig: AdapterConfig): Promise { + const rawData = await this.executeProtocol(_adapterConfig.protocolConfig); + const result = await this.executeFormat( + rawData, + _adapterConfig.formatConfig, + ); + const returnValue: DataImportResponse = { data: result }; + return returnValue; + } + /** + * Executes an protocol configuration + * + * @param _protocolConfig the protocol configuration + * @return the imported and interpreted data + * @throws ImporterParameterError on errors in the interpreter config (e.g. missing parameters, ...) + * @throws Error on response errors when importing the data + */ + async executeRawJob( + _protocolConfig: ProtocolConfig, + ): Promise { + const value = await this.executeProtocol(_protocolConfig); + const returnValue: DataImportResponse = { data: value }; + return returnValue; + } - public getAllProtocols(): Array { - return [Protocol.HTTP] - } - - public async executeJob(_adapterConfig: AdapterConfig): Promise { - const rawData = await this.executeProtocol(_adapterConfig.protocolConfig); - const result = await this.executeFormat(rawData, _adapterConfig.formatConfig); - const returnValue: DataImportResponse = {data: result}; - return returnValue; - } + async executeProtocol(config: ProtocolConfig): Promise { + const importer = config.protocol.getImporter(); + return await importer.fetch(config.parameters); + } - public async executeRawJob(_protocolConfig: ProtocolConfig): Promise { - const value = await this.executeProtocol(_protocolConfig) - const returnValue: DataImportResponse = {data: value}; - return returnValue; - } - - public async executeProtocol (config: ProtocolConfig): Promise { - const importer = config.protocol.getImporter(); - return await importer.fetch(config.parameters) - } - - public async executeFormat(rawData: string, config: FormatConfig): Promise { - const interpreter = config.format.getInterpreter(); - return await interpreter.interpret(rawData, config.parameters); - } + async executeFormat(rawData: string, config: FormatConfig): Promise { + const interpreter = config.format.getInterpreter(); + return await interpreter.interpret(rawData, config.parameters); + } } -export const adapterService = AdapterService.getInstance(); \ No newline at end of file +export const adapterService = AdapterService.getInstance(); diff --git a/adapter/src/datasource/api/rest/dataImportEndpoint.ts b/adapter/src/datasource/api/rest/dataImportEndpoint.ts index 8bbd09551..65e55eb30 100644 --- a/adapter/src/datasource/api/rest/dataImportEndpoint.ts +++ b/adapter/src/datasource/api/rest/dataImportEndpoint.ts @@ -1,32 +1,53 @@ -import express, {Express, json} from "express"; -import {asyncHandler} from "../../../adapter/api/rest/utils"; -import {DataImportRepository} from "../../repository/dataImportRepository"; -import {KnexHelper} from "../../repository/knexHelper"; +import express, { Express, json } from 'express'; -const dataImportRepository: DataImportRepository = new DataImportRepository(); +import { asyncHandler } from '../../../adapter/api/rest/utils'; +import { DataImportRepository } from '../../repository/dataImportRepository'; +import { KnexHelper } from '../../repository/knexHelper'; +const dataImportRepository: DataImportRepository = new DataImportRepository(); export class DataImportEndpoint { - registerRoutes = (app: express.Application): void => { - app.get('/datasources/:datasourceId/imports', asyncHandler(this.getMetaDataImportsForDatasource)); - app.get('/datasources/:datasourceId/imports/latest', asyncHandler(this.getLatestMetaDataImportForDatasource)); - app.get('/datasources/:datasourceId/imports/latest/data', asyncHandler(this.getLatestDataImportForDatasource)); - app.get('/datasources/:datasourceId/imports/:dataImportId', asyncHandler(this.getMetadataForDataImport)); - app.get('/datasources/:datasourceId/imports/:dataImportId/data', asyncHandler(this.getDataFromDataImport)); + app.get( + '/datasources/:datasourceId/imports', + asyncHandler(this.getMetaDataImportsForDatasource), + ); + app.get( + '/datasources/:datasourceId/imports/latest', + asyncHandler(this.getLatestMetaDataImportForDatasource), + ); + app.get( + '/datasources/:datasourceId/imports/latest/data', + asyncHandler(this.getLatestDataImportForDatasource), + ); + app.get( + '/datasources/:datasourceId/imports/:dataImportId', + asyncHandler(this.getMetadataForDataImport), + ); + app.get( + '/datasources/:datasourceId/imports/:dataImportId/data', + asyncHandler(this.getDataFromDataImport), + ); }; -//TODO Transactional bei gets??? + // TODO Transactional bei gets??? getMetaDataImportsForDatasource = async ( req: express.Request, res: express.Response, ): Promise => { - let result = await dataImportRepository.getMetaDataImportByDatasource(req.params.datasourceId) + const result = await dataImportRepository.getMetaDataImportByDatasource( + req.params.datasourceId, + ); let i = 0; result.forEach(function (el: any) { - let dataImportId = el.id; - result[i]["location"] = "/datasources/" + req.params.datasourceId + "/imports/" + dataImportId + "/data"; + const dataImportId = el.id; + result[i].location = + '/datasources/' + + req.params.datasourceId + + '/imports/' + + dataImportId + + '/data'; i++; - }) + }); res.status(200).send(result); }; @@ -35,10 +56,16 @@ export class DataImportEndpoint { req: express.Request, res: express.Response, ): Promise => { - let id = req.params.datasourceId; - let result = await dataImportRepository.getLatestMetaDataImportByDatasourceId(id); - let dataImportId = result[0].id; - result[0]["location"] = "/datasources/" + req.params.datasourceId + "/imports/" + dataImportId + "/data"; + const id = req.params.datasourceId; + const result = + await dataImportRepository.getLatestMetaDataImportByDatasourceId(id); + const dataImportId = result[0].id; + result[0].location = + '/datasources/' + + req.params.datasourceId + + '/imports/' + + dataImportId + + '/data'; res.status(200).send(result[0]); }; @@ -46,9 +73,11 @@ export class DataImportEndpoint { req: express.Request, res: express.Response, ): Promise => { - let id = req.params.datasourceId; - let result = await dataImportRepository.getLatestDataImportByDatasourceId(id); - const stringResult = KnexHelper.stringFromUTF8Array(result[0].data) + const id = req.params.datasourceId; + const result = await dataImportRepository.getLatestDataImportByDatasourceId( + id, + ); + const stringResult = KnexHelper.stringFromUTF8Array(result[0].data); res.status(200).send(stringResult); }; @@ -56,9 +85,12 @@ export class DataImportEndpoint { req: express.Request, res: express.Response, ): Promise => { - let datasourceId = req.params.datasourceId; - let dataImportId = req.params.dataImportId; - let result = await dataImportRepository.getMetadataForDataImport(datasourceId, dataImportId); + const datasourceId = req.params.datasourceId; + const dataImportId = req.params.dataImportId; + const result = await dataImportRepository.getMetadataForDataImport( + datasourceId, + dataImportId, + ); res.status(200).send(result[0]); }; @@ -66,11 +98,13 @@ export class DataImportEndpoint { req: express.Request, res: express.Response, ): Promise => { - let datasourceId = req.params.datasourceId; - let dataImportId = req.params.dataImportId; - let result = await dataImportRepository.getDataFromDataImport(datasourceId, dataImportId); - const stringFromUTF8Array = KnexHelper.stringFromUTF8Array(result[0].data) + const datasourceId = req.params.datasourceId; + const dataImportId = req.params.dataImportId; + const result = await dataImportRepository.getDataFromDataImport( + datasourceId, + dataImportId, + ); + const stringFromUTF8Array = KnexHelper.stringFromUTF8Array(result[0].data); res.status(200).send(stringFromUTF8Array); }; - } diff --git a/adapter/src/datasource/api/rest/dataSourceEndpoint.ts b/adapter/src/datasource/api/rest/dataSourceEndpoint.ts index 568fa3fe5..57f2d52fd 100644 --- a/adapter/src/datasource/api/rest/dataSourceEndpoint.ts +++ b/adapter/src/datasource/api/rest/dataSourceEndpoint.ts @@ -1,40 +1,44 @@ -import express from "express"; -import {asyncHandler} from "../../../adapter/api/rest/utils"; -import {AdapterConfig} from "../../../adapter/model/AdapterConfig"; -import {AdapterService} from "../../../adapter/services/adapterService"; -import {ProtocolConfig} from "../../../adapter/model/ProtocolConfig"; -import {Protocol} from "../../../adapter/model/enum/Protocol"; -import {Format} from "../../../adapter/model/enum/Format"; -import {FormatConfig} from "../../../adapter/model/FormatConfig"; -import {AdapterEndpoint} from "../../../adapter/api/rest/adapterEndpoint"; -import {DatasourceRepository} from "../../repository/datasourceRepository"; -import {KnexHelper} from "../../repository/knexHelper"; -import {OutboxRepository} from "../../repository/outboxRepository"; -import {DatasourceModelForAmqp} from "../../model/datasourceModelForAmqp"; -import {ADAPTER_AMQP_DATASOURCE_UPDATED_TOPIC} from "../../../env"; - +import express from 'express'; + +import { AdapterEndpoint } from '../../../adapter/api/rest/adapterEndpoint'; +import { asyncHandler } from '../../../adapter/api/rest/utils'; +import { AdapterConfig } from '../../../adapter/model/AdapterConfig'; +import { Format } from '../../../adapter/model/enum/Format'; +import { Protocol } from '../../../adapter/model/enum/Protocol'; +import { FormatConfig } from '../../../adapter/model/FormatConfig'; +import { ProtocolConfig } from '../../../adapter/model/ProtocolConfig'; +import { AdapterService } from '../../../adapter/services/adapterService'; +import { ADAPTER_AMQP_DATASOURCE_UPDATED_TOPIC } from '../../../env'; +import { DatasourceModelForAmqp } from '../../model/datasourceModelForAmqp'; +import { DatasourceRepository } from '../../repository/datasourceRepository'; +import { KnexHelper } from '../../repository/knexHelper'; +import { OutboxRepository } from '../../repository/outboxRepository'; const datasourceRepository: DatasourceRepository = new DatasourceRepository(); const outboxRepository: OutboxRepository = new OutboxRepository(); - export class DataSourceEndpoint { - registerRoutes = (app: express.Application): void => { app.get('/datasources', asyncHandler(this.getAllDataSources)); app.get('/datasources/:datasourceId', asyncHandler(this.getDataSource)); app.post('/datasources', asyncHandler(this.addDatasource)); app.put('/datasources/:datasourceId', asyncHandler(this.updateDatasource)); app.delete('/datasources/', asyncHandler(this.deleteAllDatasources)); - app.delete('/datasources/:datasourceId', asyncHandler(this.deleteDatasource)); - app.post('/datasources/:datasourceId/trigger', asyncHandler(this.triggerDataImportForDatasource)); + app.delete( + '/datasources/:datasourceId', + asyncHandler(this.deleteDatasource), + ); + app.post( + '/datasources/:datasourceId/trigger', + asyncHandler(this.triggerDataImportForDatasource), + ); }; getAllDataSources = async ( req: express.Request, res: express.Response, ): Promise => { const result = await datasourceRepository.getAllDataSources(); - let datasource = KnexHelper.createDatasourceFromResultArray(result); + const datasource = KnexHelper.createDatasourceFromResultArray(result); res.status(200).send(datasource); }; @@ -42,10 +46,11 @@ export class DataSourceEndpoint { req: express.Request, res: express.Response, ): Promise => { - - console.log(req.params.datasourceId) - const result = await datasourceRepository.getDataSourceById(req.params.datasourceId) - let datasource = KnexHelper.createDatasourceFromResult(result); + console.log(req.params.datasourceId); + const result = await datasourceRepository.getDataSourceById( + req.params.datasourceId, + ); + const datasource = KnexHelper.createDatasourceFromResult(result); res.status(200).send(datasource); }; @@ -53,16 +58,18 @@ export class DataSourceEndpoint { req: express.Request, res: express.Response, ): Promise => { - //routingkey == topic - //TODO typisierung Datasource & Dataimport - let insertStatement = KnexHelper.getInsertStatementForDataSource(req) - let datasource = await datasourceRepository.addDatasource(insertStatement); - let datasouceModelForAmqp:DatasourceModelForAmqp={ - datasource:datasource - } - // let routingKey=ADAPTER_AMQP_DATASOURCE_CREATED_TOPIC; - let routingKey="datasource.config.created"; - await outboxRepository.publishToOutbox(datasouceModelForAmqp,routingKey); + // Routingkey == topic + // TODO typisierung Datasource & Dataimport + const insertStatement = KnexHelper.getInsertStatementForDataSource(req); + const datasource = await datasourceRepository.addDatasource( + insertStatement, + ); + const datasouceModelForAmqp: DatasourceModelForAmqp = { + datasource: datasource, + }; + // Let routingKey=ADAPTER_AMQP_DATASOURCE_CREATED_TOPIC; + const routingKey = 'datasource.config.created'; + await outboxRepository.publishToOutbox(datasouceModelForAmqp, routingKey); res.status(201).send(datasource); }; @@ -70,15 +77,18 @@ export class DataSourceEndpoint { req: express.Request, res: express.Response, ): Promise => { - //TODO check response 204 with no body ?! - let insertStatement = KnexHelper.getInsertStatementForDataSource(req) - let datasource = await datasourceRepository.updateDatasource(insertStatement, req.params.datasourceId); - let datasourceModelForAmqp:DatasourceModelForAmqp ={ - datasource:datasource - } - // let routingKey=ADAPTER_AMQP_DATASOURCE_UPDATED_TOPIC; - let routingKey="datasource.config.updated"; - await outboxRepository.publishToOutbox(datasourceModelForAmqp,routingKey) + // TODO check response 204 with no body ?! + const insertStatement = KnexHelper.getInsertStatementForDataSource(req); + const datasource = await datasourceRepository.updateDatasource( + insertStatement, + req.params.datasourceId, + ); + const datasourceModelForAmqp: DatasourceModelForAmqp = { + datasource: datasource, + }; + // Let routingKey=ADAPTER_AMQP_DATASOURCE_UPDATED_TOPIC; + const routingKey = 'datasource.config.updated'; + await outboxRepository.publishToOutbox(datasourceModelForAmqp, routingKey); res.status(200).send(datasource); }; @@ -102,26 +112,30 @@ export class DataSourceEndpoint { req: express.Request, res: express.Response, ): Promise => { - //TODO add parameters from request to trigger - //TODO Error Handling general here when datasource == null - let id = req.params.datasourceId; - let result = await datasourceRepository.getDataSourceById(id); - let datasource = KnexHelper.createDatasourceFromResult(result); - let protocolConfigObj: ProtocolConfig = { + // TODO add parameters from request to trigger + // TODO Error Handling general here when datasource == null + const id = req.params.datasourceId; + const result = await datasourceRepository.getDataSourceById(id); + const datasource = KnexHelper.createDatasourceFromResult(result); + const protocolConfigObj: ProtocolConfig = { protocol: new Protocol(Protocol.HTTP), - parameters: datasource.protocol.parameters - } - let format = new Format(AdapterEndpoint.getFormat(datasource.format.type)) - let formatConfigObj: FormatConfig = {format: format, parameters: datasource.format.parameters} - let adapterConfig: AdapterConfig = {protocolConfig: protocolConfigObj, formatConfig: formatConfigObj} - let returnDataImportResponse = await AdapterService.getInstance().executeJob(adapterConfig); + parameters: datasource.protocol.parameters, + }; + const format = new Format( + AdapterEndpoint.getFormat(datasource.format.type), + ); + const formatConfigObj: FormatConfig = { + format: format, + parameters: datasource.format.parameters, + }; + const adapterConfig: AdapterConfig = { + protocolConfig: protocolConfigObj, + formatConfig: formatConfigObj, + }; + const returnDataImportResponse = + await AdapterService.getInstance().executeJob(adapterConfig); // //TODO save response in dataimport table - //TODO check correct response + // TODO check correct response res.status(200).send(returnDataImportResponse); }; - - } - - - diff --git a/adapter/src/datasource/model/datasourceModelForAmqp.ts b/adapter/src/datasource/model/datasourceModelForAmqp.ts index 4a5d05ebe..39a2b066b 100644 --- a/adapter/src/datasource/model/datasourceModelForAmqp.ts +++ b/adapter/src/datasource/model/datasourceModelForAmqp.ts @@ -1,3 +1,3 @@ export interface DatasourceModelForAmqp { - datasource: any + datasource: any; } diff --git a/adapter/src/datasource/model/outboxEvent.ts b/adapter/src/datasource/model/outboxEvent.ts index 4f94a2d49..5997885ce 100644 --- a/adapter/src/datasource/model/outboxEvent.ts +++ b/adapter/src/datasource/model/outboxEvent.ts @@ -1,5 +1,5 @@ export interface OutboxEvent { - id:any, - routing_key:string, - payload:any + id: any; + routing_key: string; + payload: any; } diff --git a/adapter/src/datasource/repository/dataImportRepository.ts b/adapter/src/datasource/repository/dataImportRepository.ts index 8631766f6..016c79f97 100644 --- a/adapter/src/datasource/repository/dataImportRepository.ts +++ b/adapter/src/datasource/repository/dataImportRepository.ts @@ -6,16 +6,15 @@ const knex = require('knex')({ user: 'adapterservice', password: 'admin', database: 'adapterservice', - asyncStackTraces: true - } + asyncStackTraces: true, + }, }); export class DataImportRepository { - async getMetaDataImportByDatasource(datasourceId: string) { - return await knex + return await knex .select('id', 'timestamp', 'health', 'error_messages') .from('public.data_import') - .where('datasource_id', datasourceId) + .where('datasource_id', datasourceId); } async getLatestMetaDataImportByDatasourceId(id: string) { @@ -27,11 +26,11 @@ export class DataImportRepository { } async getLatestDataImportByDatasourceId(id: string) { - return await knex + return await knex .select('data') .from('public.data_import') .where('datasource_id', id) - .orderBy('timestamp', 'desc') + .orderBy('timestamp', 'desc'); } async getMetadataForDataImport(datasourceId: string, dataImportId: string) { @@ -39,14 +38,14 @@ export class DataImportRepository { .select('id', 'timestamp', 'health', 'error_messages') .from('public.data_import') .where('datasource_id', datasourceId) - .andWhere('id', dataImportId) + .andWhere('id', dataImportId); } async getDataFromDataImport(datasourceId: string, dataImportId: string) { - return await knex + return await knex .select('data') .from('public.data_import') .where('datasource_id', datasourceId) - .andWhere('id', dataImportId) + .andWhere('id', dataImportId); } } diff --git a/adapter/src/datasource/repository/datasourceRepository.ts b/adapter/src/datasource/repository/datasourceRepository.ts index be3d28b7c..46d32aea4 100644 --- a/adapter/src/datasource/repository/datasourceRepository.ts +++ b/adapter/src/datasource/repository/datasourceRepository.ts @@ -1,5 +1,6 @@ -import {KnexHelper} from "./knexHelper"; -import {DatasourceInsertStatement} from "../model/DatasourceInsertStatement"; +import { DatasourceInsertStatement } from '../model/DatasourceInsertStatement'; + +import { KnexHelper } from './knexHelper'; const knex = require('knex')({ client: 'pg', @@ -9,24 +10,17 @@ const knex = require('knex')({ user: 'adapterservice', password: 'admin', database: 'adapterservice', - asyncStackTraces: true - } + asyncStackTraces: true, + }, }); export class DatasourceRepository { - - async getAllDataSources() { - return await knex - .select() - .from('public.datasource') + return await knex.select().from('public.datasource'); } async getDataSourceById(id: any) { - return await knex - .select() - .from('public.datasource') - .where('id', id); + return await knex.select().from('public.datasource').where('id', id); } async addDatasource(insertStatement: DatasourceInsertStatement) { @@ -34,22 +28,25 @@ export class DatasourceRepository { .insert(insertStatement) .returning('id') .then(function (id: any) { - console.log(id) - console.log("neuer code geht") + console.log(id); + console.log('neuer code geht'); return knex .select() .from('public.datasource') .where('id', id[0].id) .then(function (result: any) { return KnexHelper.createDatasourceFromResult(result); - }) + }); }) .catch(function (err: any) { - console.log(err) - }) + console.log(err); + }); } - async updateDatasource(insertStatement: DatasourceInsertStatement, datasourceId: string) { + async updateDatasource( + insertStatement: DatasourceInsertStatement, + datasourceId: string, + ) { return await knex('public.datasource') .where('id', datasourceId) .update(insertStatement) @@ -59,13 +56,13 @@ export class DatasourceRepository { .from('public.datasource') .where('id', datasourceId) .then(function (result: any) { - console.log(result) + console.log(result); return KnexHelper.createDatasourceFromResult(result); - }) + }); }) .catch(function (err: any) { - console.log(err) - }) + console.log(err); + }); } async deleteDatasourceById(datasourceId: string) { @@ -79,8 +76,6 @@ export class DatasourceRepository { return await knex .delete() .from('public.datasource') - .where('id', '!=', "-1") + .where('id', '!=', '-1'); } - - } diff --git a/adapter/src/datasource/repository/knexHelper.ts b/adapter/src/datasource/repository/knexHelper.ts index 39d39055e..21da416b6 100644 --- a/adapter/src/datasource/repository/knexHelper.ts +++ b/adapter/src/datasource/repository/knexHelper.ts @@ -1,80 +1,75 @@ -import {DatasourceInsertStatement} from "../model/DatasourceInsertStatement"; - +import { DatasourceInsertStatement } from '../model/DatasourceInsertStatement'; export class KnexHelper { static createDatasourceFromResultArray(result: any) { - var test = []; - for (var i in result) { - var el = result[i]; - let protocolParameters = JSON.parse(el.protocol_parameters); - let formatParameters = JSON.parse(el.format_parameters); - let x = { - "protocol": { - "type": el.protocol_type, - "parameters": - protocolParameters - + const test = []; + for (const i in result) { + const el = result[i]; + const protocolParameters = JSON.parse(el.protocol_parameters); + const formatParameters = JSON.parse(el.format_parameters); + const x = { + protocol: { + type: el.protocol_type, + parameters: protocolParameters, }, - "format": { - "type": el.format_type, - "parameters": formatParameters + format: { + type: el.format_type, + parameters: formatParameters, }, - "metadata": { - "author": el.author, - "license": el.license, - "displayName": el.display_name, - "description": el.description, - "creationTimestamp": el.creation_timestamp + metadata: { + author: el.author, + license: el.license, + displayName: el.display_name, + description: el.description, + creationTimestamp: el.creation_timestamp, }, - "trigger": { - "periodic": el.periodic, - "firstExecution": el.first_execution, - "interval": el.interval + trigger: { + periodic: el.periodic, + firstExecution: el.first_execution, + interval: el.interval, }, - "schema": el.schema, - "id": el.id - } + schema: el.schema, + id: el.id, + }; console.log(x); - test.push(x) + test.push(x); } - console.log("durch") - console.log(test) - + console.log('durch'); + console.log(test); return test; } static createDatasourceFromResult(result: any) { - let protocolParameters = JSON.parse(result[0].protocol_parameters); - let formatParameters = JSON.parse(result[0].format_parameters); - let x = { - "protocol": { - "type": result[0].protocol_type, - "parameters": protocolParameters + const protocolParameters = JSON.parse(result[0].protocol_parameters); + const formatParameters = JSON.parse(result[0].format_parameters); + const x = { + protocol: { + type: result[0].protocol_type, + parameters: protocolParameters, }, - "format": { - "type": result[0].format_type, - "parameters": formatParameters + format: { + type: result[0].format_type, + parameters: formatParameters, }, - "metadata": { - "author": result[0].author, - "license": result[0].license, - "displayName": result[0].display_name, - "description": result[0].description, - "creationTimestamp": result[0].creation_timestamp + metadata: { + author: result[0].author, + license: result[0].license, + displayName: result[0].display_name, + description: result[0].description, + creationTimestamp: result[0].creation_timestamp, }, - "trigger": { - "periodic": result[0].periodic, - "firstExecution": result[0].first_execution, - "interval": result[0].interval + trigger: { + periodic: result[0].periodic, + firstExecution: result[0].first_execution, + interval: result[0].interval, }, - "schema": result[0].schema, - "id": result[0].id - } + schema: result[0].schema, + id: result[0].id, + }; console.log(x); - return x; } @@ -91,30 +86,32 @@ export class KnexHelper { protocol_type: req.body.protocol.type, first_execution: req.body.trigger.firstExecution, interval: req.body.trigger.interval, - periodic: req.body.trigger.periodic + periodic: req.body.trigger.periodic, }; } - //from: https://weblog.rogueamoeba.com/2017/02/27/javascript-correctly-converting-a-byte-array-to-a-utf-8-string/ + // From: https://weblog.rogueamoeba.com/2017/02/27/javascript-correctly-converting-a-byte-array-to-a-utf-8-string/ static stringFromUTF8Array(data: any) { const extraByteMap = [1, 1, 1, 1, 2, 2, 3, 0]; - var count = data.length; - var str = ""; + const count = data.length; + let str = ''; - for (var index = 0; index < count;) { - var ch = data[index++]; + for (let index = 0; index < count; ) { + let ch = data[index++]; if (ch & 0x80) { - var extra = extraByteMap[(ch >> 3) & 0x07]; - if (!(ch & 0x40) || !extra || ((index + extra) > count)) + let extra = extraByteMap[(ch >> 3) & 0x07]; + if (!(ch & 0x40) || !extra || index + extra > count) { return null; + } - ch = ch & (0x3F >> extra); + ch = ch & (0x3f >> extra); for (; extra > 0; extra -= 1) { - var chx = data[index++]; - if ((chx & 0xC0) != 0x80) + const chx = data[index++]; + if ((chx & 0xc0) != 0x80) { return null; + } - ch = (ch << 6) | (chx & 0x3F); + ch = (ch << 6) | (chx & 0x3f); } } diff --git a/adapter/src/datasource/repository/outboxRepository.ts b/adapter/src/datasource/repository/outboxRepository.ts index 6dbe0a7e1..f9cb8b52b 100644 --- a/adapter/src/datasource/repository/outboxRepository.ts +++ b/adapter/src/datasource/repository/outboxRepository.ts @@ -1,7 +1,8 @@ -import {OutboxEvent} from "../model/outboxEvent"; -import {v4 as uuidv4} from "uuid"; -import {KnexHelper} from "./knexHelper"; +import { v4 as uuidv4 } from 'uuid'; +import { OutboxEvent } from '../model/outboxEvent'; + +import { KnexHelper } from './knexHelper'; const knex = require('knex')({ client: 'pg', @@ -11,28 +12,27 @@ const knex = require('knex')({ user: 'adapterservice', password: 'admin', database: 'adapterservice', - asyncStackTraces: true - } + asyncStackTraces: true, + }, }); export class OutboxRepository { - async publishToOutbox(payload: any, routingKey: string) { - let id =uuidv4(); - let outboxEvent: OutboxEvent = { - id:id, + const id = uuidv4(); + const outboxEvent: OutboxEvent = { + id: id, payload: payload, - routing_key: routingKey - } + routing_key: routingKey, + }; return await knex('public.outbox') .insert(outboxEvent) .returning('id') .then(function (id: any) { - console.log(id) - console.log("neuer code geht") + console.log(id); + console.log('neuer code geht'); }) .catch(function (err: any) { - console.log(err) - }) + console.log(err); + }); } } diff --git a/adapter/src/dummy.spec.ts b/adapter/src/dummy.spec.ts deleted file mode 100644 index 1d0194a8e..000000000 --- a/adapter/src/dummy.spec.ts +++ /dev/null @@ -1,9 +0,0 @@ -/* eslint-env jest */ -describe('dummy', () => { - test('dummy test', () => { - console.log( - 'Dummy test - please remove after first real test implemented!', - ); - expect(true).toBeTruthy(); - }); -}); diff --git a/adapter/src/index.ts b/adapter/src/index.ts index c19fe7696..127670ee0 100644 --- a/adapter/src/index.ts +++ b/adapter/src/index.ts @@ -3,16 +3,14 @@ import { Server } from 'http'; import bodyParser from 'body-parser'; import cors from 'cors'; import express from 'express'; + import { AdapterEndpoint } from './adapter/api/rest/adapterEndpoint'; +import { DataImportEndpoint } from './datasource/api/rest/dataImportEndpoint'; import { DataSourceEndpoint } from './datasource/api/rest/dataSourceEndpoint'; -import { AdapterService } from './adapter/services/adapterService'; -import {DataImportEndpoint} from "./datasource/api/rest/dataImportEndpoint"; export const port = 8080; -const API_VERSION = '0.0.1'; export let server: Server | undefined; - // Await will be needed in the future, so for now ignore this linter issue and remove the disable later // eslint-disable-next-line @typescript-eslint/require-await async function main(): Promise { diff --git a/adapter/src/tests/adapter/CsvInterpreter.spec.ts b/adapter/src/tests/adapter/CsvInterpreter.spec.ts new file mode 100644 index 000000000..0b18d3c88 --- /dev/null +++ b/adapter/src/tests/adapter/CsvInterpreter.spec.ts @@ -0,0 +1,265 @@ +import { Interpreter } from '../../adapter/interpreter/Interpreter'; +import { InterpreterParameterDescription } from '../../adapter/interpreter/InterpreterParameterDescription'; +import { Format } from '../../adapter/model/enum/Format'; +import { InterpreterParameterError } from '../../adapter/model/exceptions/InterpreterParameterError'; + +describe('doInterpret CSV Format returns valid JSON', () => { + test('convert standard CSV to JSON with lineSeperator \n and Column Seperator ,', async () => { + const csvFormat = Format.CSV; + const data = + 'id,first_name,last_name,email,gender,ip_address\n' + + '1,Ewell,Mathwin,emathwin0@hibu.com,Male,226.172.125.251\n' + + '2,Fayth,Blampy,fblampy1@hubpages.com,Female,212.76.208.25\n' + + '3,Kelli,Cornock,kcornock2@boston.com,Female,171.5.66.30\n'; + const parameters = { + columnSeparator: ',', + lineSeparator: '\n', + skipFirstDataRow: false, + firstRowAsHeader: true, + }; + const expected = [ + { + id: '1', + first_name: 'Ewell', + last_name: 'Mathwin', + email: 'emathwin0@hibu.com', + gender: 'Male', + ip_address: '226.172.125.251', + }, + { + id: '2', + first_name: 'Fayth', + last_name: 'Blampy', + email: 'fblampy1@hubpages.com', + gender: 'Female', + ip_address: '212.76.208.25', + }, + { + id: '3', + first_name: 'Kelli', + last_name: 'Cornock', + email: 'kcornock2@boston.com', + gender: 'Female', + ip_address: '171.5.66.30', + }, + ]; + const res = await csvFormat.doInterpret(data, parameters); + expect(JSON.parse(res)).toEqual(expected); + }); + + test('convert standard CSV to JSON with lineSeperator \r and ColumnSeperator ;', async () => { + const csvFormat = Format.CSV; + const data = + 'id;first_name;last_name;email;gender;ip_address\r' + + '1;Ewell;Mathwin;emathwin0@hibu.com;Male;226.172.125.251\r' + + '2;Fayth;Blampy;fblampy1@hubpages.com;Female;212.76.208.25\r' + + '3;Kelli;Cornock;kcornock2@boston.com;Female;171.5.66.30\r'; + const parameters = { + columnSeparator: ';', + lineSeparator: '\r', + skipFirstDataRow: false, + firstRowAsHeader: true, + }; + const expected = [ + { + id: '1', + first_name: 'Ewell', + last_name: 'Mathwin', + email: 'emathwin0@hibu.com', + gender: 'Male', + ip_address: '226.172.125.251', + }, + { + id: '2', + first_name: 'Fayth', + last_name: 'Blampy', + email: 'fblampy1@hubpages.com', + gender: 'Female', + ip_address: '212.76.208.25', + }, + { + id: '3', + first_name: 'Kelli', + last_name: 'Cornock', + email: 'kcornock2@boston.com', + gender: 'Female', + ip_address: '171.5.66.30', + }, + ]; + const res = await csvFormat.doInterpret(data, parameters); + expect(JSON.parse(res)).toEqual(expected); + }); +}); + +describe('doInterpret CSV Format with SkipFirstRow = TRUE returns valid JSON with missing first Row', () => { + test('convert standard CSV to JSON with skipFirstRow', async () => { + const csvFormat = Format.CSV; + const data = + 'id,first_name,last_name,email,gender,ip_address\n' + + '1,Ewell,Mathwin,emathwin0@hibu.com,Male,226.172.125.251\n' + + '2,Fayth,Blampy,fblampy1@hubpages.com,Female,212.76.208.25\n' + + '3,Kelli,Cornock,kcornock2@boston.com,Female,171.5.66.30\n'; + const parameters = { + columnSeparator: ',', + lineSeparator: '\n', + skipFirstDataRow: true, + firstRowAsHeader: true, + }; + const expected = [ + { + id: '2', + first_name: 'Fayth', + last_name: 'Blampy', + email: 'fblampy1@hubpages.com', + gender: 'Female', + ip_address: '212.76.208.25', + }, + { + id: '3', + first_name: 'Kelli', + last_name: 'Cornock', + email: 'kcornock2@boston.com', + gender: 'Female', + ip_address: '171.5.66.30', + }, + ]; + const res = await csvFormat.doInterpret(data, parameters); + expect(JSON.parse(res)).toEqual(expected); + }); +}); + +describe('doInterpret CSV Format with FirstRowAsHeader = FALSE returns valid JSON', () => { + test('convert standard CSV to JSON with skipFirstRow', async () => { + const csvFormat = Format.CSV; + const data = + '1,Ewell,Mathwin,emathwin0@hibu.com,Male,226.172.125.251\n' + + '2,Fayth,Blampy,fblampy1@hubpages.com,Female,212.76.208.25\n' + + '3,Kelli,Cornock,kcornock2@boston.com\n'; + const parameters = { + columnSeparator: ',', + lineSeparator: '\n', + skipFirstDataRow: false, + firstRowAsHeader: false, + }; + const expected = [ + { + field1: '1', + field2: 'Ewell', + field3: 'Mathwin', + field4: 'emathwin0@hibu.com', + field5: 'Male', + field6: '226.172.125.251', + }, + { + field1: '2', + field2: 'Fayth', + field3: 'Blampy', + field4: 'fblampy1@hubpages.com', + field5: 'Female', + field6: '212.76.208.25', + }, + { + field1: '3', + field2: 'Kelli', + field3: 'Cornock', + field4: 'kcornock2@boston.com', + }, + ]; + const res = await csvFormat.doInterpret(data, parameters); + expect(JSON.parse(res)).toEqual(expected); + }); +}); + +describe('getAvaialbleParameters Tests', () => { + test('getAvaiableParameters are of type string', () => { + const interpreter: Interpreter = Format.CSV; + const availableParameters: InterpreterParameterDescription[] = + interpreter.getAvailableParameters(); + expect(availableParameters[0].type).toBe('string'); + expect(availableParameters[1].type).toBe('string'); + expect(availableParameters[2].type).toBe('boolean'); + expect(availableParameters[3].type).toBe('boolean'); + }); + + test('getAvailableParameters is of Type Array', () => { + const interpreter: Interpreter = Format.CSV; + const availableParameters: InterpreterParameterDescription[] = + interpreter.getAvailableParameters(); + expect(availableParameters).toBeInstanceOf(Array); + }); +}); + +describe('validateParameters from CSV Interpreter', () => { + test('validateParameters with valid Parameters works', () => { + const interpreter: Interpreter = Format.CSV; + const parameters = { + columnSeparator: ',', + lineSeparator: '\n', + skipFirstDataRow: true, + firstRowAsHeader: false, + }; + interpreter.validateParameters(parameters); + }); + + test('validateParameters with Parameter misses columnSeperator throws Exception', () => { + const interpreter: Interpreter = Format.CSV; + const parameters = { + lineSeparator: '\n', + skipFirstDataRow: true, + firstRowAsHeader: false, + }; + expect(() => { + interpreter.validateParameters(parameters); + }).toThrow(InterpreterParameterError); + }); + + test('validateParameters with Parameter misses lineSeperator throws Exception', () => { + const interpreter: Interpreter = Format.CSV; + const parameters = { + columnSeparator: ',', + skipFirstDataRow: true, + firstRowAsHeader: false, + }; + expect(() => { + interpreter.validateParameters(parameters); + }).toThrow(InterpreterParameterError); + }); + + test('validateParameters with Parameter misses skipFirstDataRow throws Exception', () => { + const interpreter: Interpreter = Format.CSV; + const parameters = { + columnSeparator: ',', + lineSeparator: '\n', + firstRowAsHeader: false, + }; + expect(() => { + interpreter.validateParameters(parameters); + }).toThrow(InterpreterParameterError); + }); + + test('validateParameters with Parameter misses firstRowAsHeader throws Exception', () => { + const interpreter: Interpreter = Format.CSV; + const parameters = { + columnSeparator: ',', + lineSeparator: '\n', + skipFirstDataRow: true, + }; + expect(() => { + interpreter.validateParameters(parameters); + }).toThrow(InterpreterParameterError); + }); + + test('validateParameters with too many Parameters throws Error', () => { + const interpreter: Interpreter = Format.CSV; + const parameters = { + columnSeparator: ',', + lineSeparator: '\n', + skipFirstDataRow: true, + firstRowAsHeader: false, + extraParameter: false, + }; + expect(() => { + interpreter.validateParameters(parameters); + }).toThrow(InterpreterParameterError); + }); +}); diff --git a/adapter/src/tests/adapter/HttpImporter.spec.ts b/adapter/src/tests/adapter/HttpImporter.spec.ts new file mode 100644 index 000000000..9495f8d2e --- /dev/null +++ b/adapter/src/tests/adapter/HttpImporter.spec.ts @@ -0,0 +1,111 @@ +import { Importer } from '../../adapter/importer/Importer'; +import { ImporterParameterDescription } from '../../adapter/importer/ImporterParameterDescription'; +import { Protocol } from '../../adapter/model/enum/Protocol'; +import { ImporterParameterError } from '../../adapter/model/exceptions/ImporterParameterError'; + +/* eslint-env jest */ +describe('getAvaialbleParameters Tests', () => { + test('getAvaiableParameters are of type string', () => { + const importer: Importer = Protocol.HTTP; + const availableParameters: ImporterParameterDescription[] = + importer.getAvailableParameters(); + expect(availableParameters[0].type).toBe('string'); + expect(availableParameters[1].type).toBe('string'); + }); + + test('getAvailableParameters is of Type Array', () => { + const importer: Importer = Protocol.HTTP; + const availableParameters = importer.getAvailableParameters(); + expect(availableParameters).toBeInstanceOf(Array); + }); +}); + +describe('validateParameters from Abstract Parent Class Tests', () => { + test('validateParameters with valid Parameters works', () => { + const importer: Importer = Protocol.HTTP; + const parameters = { + location: + 'https://www.pegelonline.wsv.de/webservices/rest-api/v2/stations.json', + encoding: 'UTF-8', + }; + importer.validateParameters(parameters); + }); + + test('validateParameters finds unavailable Parameter', () => { + const importer: Importer = Protocol.HTTP; + const wrongParameters = { + location: + 'https://www.pegelonline.wsv.de/webservices/rest-api/v2/stations.json', + encoding: 'UTF-8', + parameter3: 'komischer parameter', + }; + expect(() => { + importer.validateParameters(wrongParameters); + }).toThrow(ImporterParameterError); + }); + + test('validateParameters is missing encoding Parameter', () => { + const importer: Importer = Protocol.HTTP; + const wrongParameters = { + location: + 'https://www.pegelonline.wsv.de/webservices/rest-api/v2/stations.json', + }; + expect(() => { + importer.validateParameters(wrongParameters); + }).toThrow(ImporterParameterError); + }); + + test('validateParameters is missing location Parameter', () => { + const importer: Importer = Protocol.HTTP; + const wrongParameters = { + encoding: 'UTF-8', + }; + expect(() => { + importer.validateParameters(wrongParameters); + }).toThrow(ImporterParameterError); + }); +}); + +describe('validateParameters HTTP Importer Tests', () => { + test('validateParameters HTTP Importer with UTF-8 works', () => { + const importer: Importer = Protocol.HTTP; + const parameters = { + location: + 'https://www.pegelonline.wsv.de/webservices/rest-api/v2/stations.json', + encoding: 'UTF-8', + }; + importer.validateParameters(parameters); + }); + + test('validateParameters HTTP Importer with US-ASCII works', () => { + const importer: Importer = Protocol.HTTP; + const parameters = { + location: + 'https://www.pegelonline.wsv.de/webservices/rest-api/v2/stations.json', + encoding: 'US-ASCII', + }; + importer.validateParameters(parameters); + }); + + test('validateParameters HTTP Importer with ISO-8859-1 works', () => { + const importer: Importer = Protocol.HTTP; + const parameters = { + location: + 'https://www.pegelonline.wsv.de/webservices/rest-api/v2/stations.json', + encoding: 'ISO-8859-1', + }; + importer.validateParameters(parameters); + }); + + test('validateParameters HTTP Importer with a wrong Encoding does not work', () => { + const importer: Importer = Protocol.HTTP; + const parameters = { + location: + 'https://www.pegelonline.wsv.de/webservices/rest-api/v2/stations.json', + encoding: 'encodingwrong', + }; + expect(() => { + importer.validateParameters(parameters); + }).toThrow(Error); + }); +}); diff --git a/adapter/src/tests/adapter/JsonInterpreter.spec.ts b/adapter/src/tests/adapter/JsonInterpreter.spec.ts new file mode 100644 index 000000000..25df70a8a --- /dev/null +++ b/adapter/src/tests/adapter/JsonInterpreter.spec.ts @@ -0,0 +1,13 @@ +import { Format } from '../../adapter/model/enum/Format'; + +describe('doInterpret Json Format returns valid JSON', () => { + test('doInterpret JSON data test', () => { + const jsonFormat = Format.JSON; + const data = + '"{"uuid":"47174d8f-1b8e-4599-8a59-b580dd55bc87","number":"48900237","shortname":"EITZE","longname":"EITZE","km":9.56,"agency":"VERDEN",' + + '"longitude":9.276769435375872,"latitude":52.90406544743417,"water":{"shortname":"ALLER","longname":"ALLER"}},"'; + return jsonFormat.doInterpret(data, {}).then((res) => { + expect(res).toBe(data); + }); + }); +}); diff --git a/adapter/src/tests/adapter/XmlInterpreter.spec.ts b/adapter/src/tests/adapter/XmlInterpreter.spec.ts new file mode 100644 index 000000000..e5c748e70 --- /dev/null +++ b/adapter/src/tests/adapter/XmlInterpreter.spec.ts @@ -0,0 +1,54 @@ +import { Format } from '../../adapter/model/enum/Format'; + +describe('doInterpret XML Format Returns valid JSON', () => { + test('convert standard XML to JSON test', () => { + const xmlFormat = Format.XML; + const data = + 'ToveJaniReminderDon\'t forget me this weekend!'; + return xmlFormat.doInterpret(data, {}).then((res) => { + expect(JSON.parse(res)).toEqual({ + root: { + to: 'Tove', + from: 'Jani', + heading: 'Reminder', + body: "Don't forget me this weekend!", + }, + }); + }); + }); + + test('convert nested XML to JSON test', () => { + const xmlFormat = Format.XML; + const data = + 'ToveJaniReminderSubheadingReminderDon\'t forget me this weekend!'; + return xmlFormat.doInterpret(data, {}).then((res) => { + expect(JSON.parse(res)).toEqual({ + root: { + to: 'Tove', + from: 'Jani', + heading: { subheading: 'ReminderSubheading', Reminder: 'Reminder' }, + body: "Don't forget me this weekend!", + }, + }); + }); + }); + + test('convert nested XML to JSON test', () => { + const xmlFormat = Format.XML; + const data = + 'ToveJaniReminderSubheadingReminder1Reminder2Reminder3Don\'t forget me this weekend!'; + return xmlFormat.doInterpret(data, {}).then((res) => { + expect(JSON.parse(res)).toEqual({ + root: { + to: 'Tove', + from: 'Jani', + heading: { + subheading: 'ReminderSubheading', + Reminder: ['Reminder1', 'Reminder2', 'Reminder3'], + }, + body: "Don't forget me this weekend!", + }, + }); + }); + }); +}); diff --git a/adapter/src/tests/adapter/adapterEndpoint.spec.ts b/adapter/src/tests/adapter/adapterEndpoint.spec.ts new file mode 100644 index 000000000..a58e4d948 --- /dev/null +++ b/adapter/src/tests/adapter/adapterEndpoint.spec.ts @@ -0,0 +1,36 @@ +import { AdapterEndpoint } from '../../adapter/api/rest/adapterEndpoint'; +import { Format } from '../../adapter/model/enum/Format'; +import { Protocol } from '../../adapter/model/enum/Protocol'; + +/* eslint-env jest */ +describe('getFormat should return correct Result', () => { + test('getFormat test throws exception for not existing format', () => { + expect(() => { + AdapterEndpoint.getFormat('not here'); + }).toThrow(Error); + }); + test('getFormat test for CSV', () => { + const result = AdapterEndpoint.getFormat('CSV'); + expect(result).toBe(Format.CSV); + }); + test('getFormat test for XML', () => { + const result = AdapterEndpoint.getFormat('XML'); + expect(result).toBe(Format.XML); + }); + test('getFormat test for JSON', () => { + const result = AdapterEndpoint.getFormat('JSON'); + expect(result).toBe(Format.JSON); + }); +}); + +describe('getProtocol should return correct Result', () => { + test('getProtocol test for HTTP', () => { + const result = AdapterEndpoint.getProtocol('HTTP'); + expect(result).toBe(Protocol.HTTP); + }); + test('getProtocol test throws exception for not existing Protocol', () => { + expect(() => { + AdapterEndpoint.getProtocol('not here'); + }).toThrow(Error); + }); +});