From bed7472b790d67eb864278832d92f855bfe512e4 Mon Sep 17 00:00:00 2001 From: = <=> Date: Wed, 20 Apr 2022 13:43:38 +0200 Subject: [PATCH 01/40] undefined + typo + constructor raus with @PVahldiek --- adapter/src/adapter/api/rest/adapterEndpoint.ts | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/adapter/src/adapter/api/rest/adapterEndpoint.ts b/adapter/src/adapter/api/rest/adapterEndpoint.ts index 0b6f2965a..fe54d767b 100644 --- a/adapter/src/adapter/api/rest/adapterEndpoint.ts +++ b/adapter/src/adapter/api/rest/adapterEndpoint.ts @@ -13,7 +13,6 @@ import { ImporterParameterError } from '../../model/exceptions/ImporterParameter const APP_VERSION = "0.0.1" export class AdapterEndpoint { - constructor() {} registerRoutes = (app: express.Application): void => { app.post('/preview', asyncHandler(this.handleExecuteDataImport)); @@ -56,7 +55,7 @@ export class AdapterEndpoint { } // Check protocol type const protocolType = AdapterEndpoint.getProtocol(req.body.protocol.type) - if(protocolType === "unsupported"){ + if(protocolType === undefined){ res.status(400).send("Protocol " + req.body.protocol.type + " not supported") return; } @@ -64,7 +63,7 @@ export class AdapterEndpoint { // Check format type const formatType = AdapterEndpoint.getFormat(req.body.format.type) - if(formatType === "unsupported"){ + if(formatType === undefined){ res.status(400).send("Format " + req.body.format.type + " not supported") return; } @@ -105,7 +104,7 @@ export class AdapterEndpoint { }; /* - returns Collection of Importerj + returns Collection of Importer } */ handleGetFormat = async ( @@ -157,7 +156,7 @@ export class AdapterEndpoint { return Format.XML; } default: { - return "unsupported"; + return undefined; } } } @@ -168,7 +167,7 @@ export class AdapterEndpoint { return Protocol.HTTP; } default: { - return "unsupported" + return undefined; } } } From cc8af1043d673dd9892afc140f9eaf0c5d361150 Mon Sep 17 00:00:00 2001 From: = <=> Date: Wed, 20 Apr 2022 14:59:56 +0200 Subject: [PATCH 02/40] linter first fixes with @PVahldiek --- .../src/adapter/api/rest/adapterEndpoint.ts | 171 +++++++++--------- adapter/src/adapter/model/AdapterConfig.ts | 9 +- adapter/src/adapter/model/EndpointDTOs.ts | 24 +++ adapter/src/adapter/model/enum/Format.ts | 16 +- adapter/src/adapter/model/enum/Protocol.ts | 14 +- adapter/src/index.ts | 5 +- 6 files changed, 135 insertions(+), 104 deletions(-) create mode 100644 adapter/src/adapter/model/EndpointDTOs.ts diff --git a/adapter/src/adapter/api/rest/adapterEndpoint.ts b/adapter/src/adapter/api/rest/adapterEndpoint.ts index fe54d767b..48776e6a3 100644 --- a/adapter/src/adapter/api/rest/adapterEndpoint.ts +++ b/adapter/src/adapter/api/rest/adapterEndpoint.ts @@ -1,19 +1,25 @@ import express from 'express'; -import { AdapterConfig, AdapterConfigValidator } from '../../model/AdapterConfig'; - -import { asyncHandler } from './utils'; -import {ProtocolConfig, ProtocolConfigValidator} from "../../model/ProtocolConfig"; +import { Importer } from '../../importer/Importer'; +import { Interpreter } from '../../interpreter/Interpreter'; +import { + AdapterConfig, + AdapterConfigValidator, +} from '../../model/AdapterConfig'; import { Format } from '../../model/enum/Format'; -import { AdapterService, adapterService } from '../../services/adapterService'; -import { FormatConfig } from '../../model/FormatConfig'; import { Protocol } from '../../model/enum/Protocol'; import { ImporterParameterError } from '../../model/exceptions/ImporterParameterError'; +import { FormatConfig } from '../../model/FormatConfig'; +import { + ProtocolConfig, + ProtocolConfigValidator, +} from '../../model/ProtocolConfig'; +import { AdapterService } from '../../services/adapterService'; +import { asyncHandler } from './utils'; -const APP_VERSION = "0.0.1" +const APP_VERSION = '0.0.1'; export class AdapterEndpoint { - registerRoutes = (app: express.Application): void => { app.post('/preview', asyncHandler(this.handleExecuteDataImport)); app.post('/preview/raw', asyncHandler(this.handleExecuteRawPreview)); @@ -22,65 +28,64 @@ export class AdapterEndpoint { app.get('/version', asyncHandler(this.handleGetApplicationVersion)); }; - // Adapter Endpoint - /** - * @RestController - @AllArgsConstructor - public class AdapterEndpoint { - private final Adapter adapter; - - @PostMapping(Mappings.IMPORT_PATH) - public DataImportResponse executeDataImport(@Valid @RequestBody AdapterConfig config) - throws ImporterParameterException, InterpreterParameterException, IOException { - return adapter.executeJob(config); - } - - @PostMapping(Mappings.RAW_IMPORT_PATH) - public DataImportResponse executeRawPreview(@Valid @RequestBody ProtocolConfig config) - throws ImporterParameterException { - return adapter.executeRawImport(config); - } - } - */ - handleExecuteDataImport = async ( req: express.Request, res: express.Response, ): Promise => { const validator = new AdapterConfigValidator(); - const adapterconfigforValidator = req.body; + const adapterconfigforValidator: unknown = req.body; if (!validator.validate(adapterconfigforValidator)) { res.status(400).json({ errors: validator.getErrors() }); return; } // Check protocol type - const protocolType = AdapterEndpoint.getProtocol(req.body.protocol.type) - if(protocolType === undefined){ - res.status(400).send("Protocol " + req.body.protocol.type + " not supported") + let protocolType: Importer; + try { + protocolType = AdapterEndpoint.getProtocol( + adapterconfigforValidator.protocol.type, + ); + } catch (e) { + res.status(400).send('Protocol not supported'); return; } - const protocolConfigObj: ProtocolConfig = {protocol: new Protocol(protocolType), parameters: req.body.protocol.parameters} + + const protocolConfigObj: ProtocolConfig = { + protocol: new Protocol(protocolType), + parameters: adapterconfigforValidator.protocol.parameters, + }; // Check format type - const formatType = AdapterEndpoint.getFormat(req.body.format.type) - if(formatType === undefined){ - res.status(400).send("Format " + req.body.format.type + " not supported") + let formatType: Interpreter; + try { + formatType = AdapterEndpoint.getFormat( + adapterconfigforValidator.format.type, + ); + } catch (e) { + res.status(400).send('Format not supported'); return; } // Check location (???) - - const format = new Format(formatType) - const formatConfigObj: FormatConfig = {format: format, parameters: req.body.format.parameters} - const adapterConfig:AdapterConfig = {protocolConfig: protocolConfigObj, formatConfig: formatConfigObj} - console.log(adapterConfig) - - let returnDataImportResponse = null - try{ - returnDataImportResponse = await AdapterService.getInstance().executeJob(adapterConfig); - }catch(e){ - if(e instanceof ImporterParameterError){ - res.status(400).send(e.message) + const format = new Format(formatType); + const formatConfigObj: FormatConfig = { + format: format, + parameters: adapterconfigforValidator.format.parameters, + }; + + const adapterConfig: AdapterConfig = { + protocolConfig: protocolConfigObj, + formatConfig: formatConfigObj, + }; + console.log(adapterConfig); + + let returnDataImportResponse = null; + try { + returnDataImportResponse = await AdapterService.getInstance().executeJob( + adapterConfig, + ); + } catch (e) { + if (e instanceof ImporterParameterError) { + res.status(400).send(e.message); return; } } @@ -94,17 +99,22 @@ export class AdapterEndpoint { ): Promise => { const validator = new ProtocolConfigValidator(); const protcolConfigForValidator = req.body.protocol; + if (!validator.validate(protcolConfigForValidator)) { res.status(400).json({ errors: validator.getErrors() }); return; } - const protocolConfigObj: ProtocolConfig = {protocol: new Protocol(Protocol.HTTP), parameters: req.body.protocol.parameters} - const returnDataImportResponse = await AdapterService.getInstance().executeRawJob(protocolConfigObj); + const protocolConfigObj: ProtocolConfig = { + protocol: new Protocol(Protocol.HTTP), + parameters: req.body.protocol.parameters, + }; + const returnDataImportResponse = + await AdapterService.getInstance().executeRawJob(protocolConfigObj); res.status(200).send(returnDataImportResponse); }; /* - returns Collection of Importer + Returns Collection of Importer } */ handleGetFormat = async ( @@ -113,65 +123,62 @@ export class AdapterEndpoint { ): Promise => { try { const interpreters = AdapterService.getInstance().getAllFormats(); - res.setHeader("Content-Type", "application/json") + res.setHeader('Content-Type', 'application/json'); res.status(200).json(interpreters); } catch (e) { - //res.status(500).send('Error finding formats'); - throw e + // Res.status(500).send('Error finding formats'); + throw e; } }; /* - returns Collection of Importer + Returns Collection of Importer */ handleGetProtocols = async ( req: express.Request, res: express.Response, ): Promise => { try { - const protocols = AdapterService.getInstance().getAllProtocols(); - res.status(200).json(protocols); - } catch (e) { - res.status(500).send('Error finding protocols'); - } + const protocols = AdapterService.getInstance().getAllProtocols(); + res.status(200).json(protocols); + } catch (e) { + res.status(500).send('Error finding protocols'); + } }; handleGetApplicationVersion = async ( req: express.Request, res: express.Response, ): Promise => { - res.setHeader("Content-Type", "text/plain"); + res.setHeader('Content-Type', 'text/plain'); res.status(200).send(APP_VERSION); }; - static getFormat(type: any): any { - switch(type) { - case "JSON": { - return Format.JSON; + static getFormat(type: string): Interpreter { + switch (type) { + case 'JSON': { + return Format.JSON; } - case "CSV": { - return Format.CSV; + case 'CSV': { + return Format.CSV; } - case "XML": { + case 'XML': { return Format.XML; - } + } default: { - return undefined; + throw new Error('asdasd'); } - } + } } - static getProtocol(type: any): any { - switch(type) { - case "HTTP": { - return Protocol.HTTP; + static getProtocol(type: string): Importer { + switch (type) { + case 'HTTP': { + return Protocol.HTTP; } default: { - return undefined; + throw new Error('asdasd'); } - } + } } - -}; - - +} diff --git a/adapter/src/adapter/model/AdapterConfig.ts b/adapter/src/adapter/model/AdapterConfig.ts index 9b60da09e..c4f21c638 100644 --- a/adapter/src/adapter/model/AdapterConfig.ts +++ b/adapter/src/adapter/model/AdapterConfig.ts @@ -1,7 +1,8 @@ -import { FormatConfig } from "./FormatConfig"; -import { ProtocolConfig } from "./ProtocolConfig"; import { validators } from '@jvalue/node-dry-basics'; -import { JsonAlias, JsonClassType, JsonProperty } from "jackson-js"; + +import { AdapterConfigDTO } from './EndpointDTOs'; +import { FormatConfig } from './FormatConfig'; +import { ProtocolConfig } from './ProtocolConfig'; export interface AdapterConfig { protocolConfig: ProtocolConfig; @@ -11,7 +12,7 @@ export interface AdapterConfig { export class AdapterConfigValidator { private errors: string[] = []; - validate(request: unknown): request is AdapterConfig { + validate(request: unknown): request is AdapterConfigDTO { this.errors = []; if (!validators.isObject(request)) { this.errors.push("'AdapterConfig' must be an object"); diff --git a/adapter/src/adapter/model/EndpointDTOs.ts b/adapter/src/adapter/model/EndpointDTOs.ts new file mode 100644 index 000000000..12d9210b5 --- /dev/null +++ b/adapter/src/adapter/model/EndpointDTOs.ts @@ -0,0 +1,24 @@ +export interface AdapterConfigDTO { + protocol: ProtocolConfigDTO; + format: FormatConfigDTO; +} + +export interface ProtocolConfigDTO { + type: string; + parameters: Record; +} + +export interface FormatConfigDTO { + type: string; + parameters: Record; +} + +/* Export interface FormatParameterDTO { + location: string; + encoding: string; +} + +export interface ProtocolParameterDTO { + type: string; + parameters: Record; +}*/ diff --git a/adapter/src/adapter/model/enum/Format.ts b/adapter/src/adapter/model/enum/Format.ts index 6262f2da9..eb41db3dc 100644 --- a/adapter/src/adapter/model/enum/Format.ts +++ b/adapter/src/adapter/model/enum/Format.ts @@ -1,19 +1,19 @@ -import { CsvInterpreter } from "../../interpreter/CsvInterpreter"; -import { Interpreter } from "../../interpreter/Interpreter"; -import { JsonInterpreter } from "../../interpreter/JsonInterpreter"; -import { XmlInterpreter } from "../../interpreter/XmlInterpreter"; +import { CsvInterpreter } from '../../interpreter/CsvInterpreter'; +import { Interpreter } from '../../interpreter/Interpreter'; +import { JsonInterpreter } from '../../interpreter/JsonInterpreter'; +import { XmlInterpreter } from '../../interpreter/XmlInterpreter'; export class Format { - static readonly JSON = new JsonInterpreter(); + static readonly JSON = new JsonInterpreter(); static readonly XML = new XmlInterpreter(); - static readonly CSV = new CsvInterpreter(); + static readonly CSV = new CsvInterpreter(); interpreter: Interpreter; constructor(interpreter: Interpreter) { this.interpreter = interpreter; } - getInterpreter() { + getInterpreter(): Interpreter { return this.interpreter; } -} \ No newline at end of file +} diff --git a/adapter/src/adapter/model/enum/Protocol.ts b/adapter/src/adapter/model/enum/Protocol.ts index c75813883..a20a1aafd 100644 --- a/adapter/src/adapter/model/enum/Protocol.ts +++ b/adapter/src/adapter/model/enum/Protocol.ts @@ -1,15 +1,15 @@ -import { HttpImporter } from "../../importer/HttpImporter"; -import { Importer } from "../../importer/Importer"; +import { HttpImporter } from '../../importer/HttpImporter'; +import { Importer } from '../../importer/Importer'; export class Protocol { - static readonly HTTP = new HttpImporter(); - + static readonly HTTP = new HttpImporter(); + importer: Importer; constructor(importer: Importer) { this.importer = importer; } - getImporter() { - return this.importer + getImporter(): Importer { + return this.importer; } -} \ No newline at end of file +} diff --git a/adapter/src/index.ts b/adapter/src/index.ts index c19fe7696..a0cad1bed 100644 --- a/adapter/src/index.ts +++ b/adapter/src/index.ts @@ -3,16 +3,15 @@ import { Server } from 'http'; import bodyParser from 'body-parser'; import cors from 'cors'; import express from 'express'; + import { AdapterEndpoint } from './adapter/api/rest/adapterEndpoint'; +import { DataImportEndpoint } from './datasource/api/rest/dataImportEndpoint'; import { DataSourceEndpoint } from './datasource/api/rest/dataSourceEndpoint'; -import { AdapterService } from './adapter/services/adapterService'; -import {DataImportEndpoint} from "./datasource/api/rest/dataImportEndpoint"; export const port = 8080; const API_VERSION = '0.0.1'; export let server: Server | undefined; - // Await will be needed in the future, so for now ignore this linter issue and remove the disable later // eslint-disable-next-line @typescript-eslint/require-await async function main(): Promise { From fa2de593daeb752eb8126332a303af5151c5d751 Mon Sep 17 00:00:00 2001 From: = <=> Date: Wed, 20 Apr 2022 15:08:03 +0200 Subject: [PATCH 03/40] preview raw linter with @PVahldiek --- adapter/src/adapter/api/rest/adapterEndpoint.ts | 8 +++----- adapter/src/adapter/model/ProtocolConfig.ts | 16 ++++++++++------ 2 files changed, 13 insertions(+), 11 deletions(-) diff --git a/adapter/src/adapter/api/rest/adapterEndpoint.ts b/adapter/src/adapter/api/rest/adapterEndpoint.ts index 48776e6a3..97641e95f 100644 --- a/adapter/src/adapter/api/rest/adapterEndpoint.ts +++ b/adapter/src/adapter/api/rest/adapterEndpoint.ts @@ -71,7 +71,7 @@ export class AdapterEndpoint { format: format, parameters: adapterconfigforValidator.format.parameters, }; - + const adapterConfig: AdapterConfig = { protocolConfig: protocolConfigObj, formatConfig: formatConfigObj, @@ -98,15 +98,13 @@ export class AdapterEndpoint { res: express.Response, ): Promise => { const validator = new ProtocolConfigValidator(); - const protcolConfigForValidator = req.body.protocol; - - if (!validator.validate(protcolConfigForValidator)) { + if (!validator.validate(req.body)) { res.status(400).json({ errors: validator.getErrors() }); return; } const protocolConfigObj: ProtocolConfig = { protocol: new Protocol(Protocol.HTTP), - parameters: req.body.protocol.parameters, + parameters: req.body.parameters, }; const returnDataImportResponse = await AdapterService.getInstance().executeRawJob(protocolConfigObj); diff --git a/adapter/src/adapter/model/ProtocolConfig.ts b/adapter/src/adapter/model/ProtocolConfig.ts index 7bc9ebf3d..74a2f6632 100644 --- a/adapter/src/adapter/model/ProtocolConfig.ts +++ b/adapter/src/adapter/model/ProtocolConfig.ts @@ -1,18 +1,22 @@ -import { Protocol } from "./enum/Protocol"; +import { stringify } from 'querystring'; + import { validators } from '@jvalue/node-dry-basics'; -import { stringify } from "querystring"; -import { JsonAlias, JsonClassType, JsonProperty } from "jackson-js"; +import { JsonAlias, JsonClassType, JsonProperty } from 'jackson-js'; + +import { ProtocolConfigDTO } from './EndpointDTOs'; +import { Protocol } from './enum/Protocol'; export interface ProtocolConfig { - protocol:Protocol; - parameters: Record; + protocol: Protocol; + parameters: Record; } export class ProtocolConfigValidator { private errors: string[] = []; - validate(request: unknown): request is ProtocolConfig { + validate(request: ProtocolConfigDTO): request is ProtocolConfigDTO { this.errors = []; + if (!validators.isObject(request)) { this.errors.push("'ProtocolConfig' must be an object"); return false; From 337f013bbadec9f2f471ad9079192e22a61323f1 Mon Sep 17 00:00:00 2001 From: = <=> Date: Wed, 20 Apr 2022 15:29:14 +0200 Subject: [PATCH 04/40] linter with @PVahldiek --- .../src/adapter/api/rest/adapterEndpoint.ts | 28 +--- adapter/src/adapter/api/rest/utils.ts | 2 +- adapter/src/adapter/importer/HttpImporter.ts | 155 +++++++++--------- adapter/src/adapter/importer/Importer.ts | 79 +++++---- .../importer/ImporterParameterDescription.ts | 32 ++-- .../src/adapter/interpreter/CsvInterpreter.ts | 134 +++++++++------ .../src/adapter/interpreter/Interpreter.ts | 46 +++--- .../InterpreterParameterDescription.ts | 27 ++- .../adapter/interpreter/JsonInterpreter.ts | 30 ++-- .../src/adapter/interpreter/XmlInterpreter.ts | 49 +++--- adapter/src/adapter/model/AdapterConfig.ts | 2 +- adapter/src/adapter/model/EndpointDTOs.ts | 10 -- adapter/src/adapter/model/FormatConfig.ts | 8 +- adapter/src/adapter/model/ProtocolConfig.ts | 13 +- .../adapter/model/exceptions/AdapterError.ts | 10 +- .../exceptions/ImporterParameterError.ts | 13 +- .../exceptions/InterpreterParameterError.ts | 13 +- .../src/adapter/services/adapterService.ts | 100 +++++------ 18 files changed, 398 insertions(+), 353 deletions(-) diff --git a/adapter/src/adapter/api/rest/adapterEndpoint.ts b/adapter/src/adapter/api/rest/adapterEndpoint.ts index 97641e95f..5fd798b1e 100644 --- a/adapter/src/adapter/api/rest/adapterEndpoint.ts +++ b/adapter/src/adapter/api/rest/adapterEndpoint.ts @@ -33,17 +33,14 @@ export class AdapterEndpoint { res: express.Response, ): Promise => { const validator = new AdapterConfigValidator(); - const adapterconfigforValidator: unknown = req.body; - if (!validator.validate(adapterconfigforValidator)) { + if (!validator.validate(req.body)) { res.status(400).json({ errors: validator.getErrors() }); return; } // Check protocol type let protocolType: Importer; try { - protocolType = AdapterEndpoint.getProtocol( - adapterconfigforValidator.protocol.type, - ); + protocolType = AdapterEndpoint.getProtocol(req.body.protocol.type); } catch (e) { res.status(400).send('Protocol not supported'); return; @@ -51,15 +48,13 @@ export class AdapterEndpoint { const protocolConfigObj: ProtocolConfig = { protocol: new Protocol(protocolType), - parameters: adapterconfigforValidator.protocol.parameters, + parameters: req.body.protocol.parameters, }; // Check format type let formatType: Interpreter; try { - formatType = AdapterEndpoint.getFormat( - adapterconfigforValidator.format.type, - ); + formatType = AdapterEndpoint.getFormat(req.body.format.type); } catch (e) { res.status(400).send('Format not supported'); return; @@ -69,7 +64,7 @@ export class AdapterEndpoint { const format = new Format(formatType); const formatConfigObj: FormatConfig = { format: format, - parameters: adapterconfigforValidator.format.parameters, + parameters: req.body.format.parameters, }; const adapterConfig: AdapterConfig = { @@ -104,7 +99,7 @@ export class AdapterEndpoint { } const protocolConfigObj: ProtocolConfig = { protocol: new Protocol(Protocol.HTTP), - parameters: req.body.parameters, + parameters: req.body.protocol.parameters, }; const returnDataImportResponse = await AdapterService.getInstance().executeRawJob(protocolConfigObj); @@ -119,14 +114,9 @@ export class AdapterEndpoint { req: express.Request, res: express.Response, ): Promise => { - try { - const interpreters = AdapterService.getInstance().getAllFormats(); - res.setHeader('Content-Type', 'application/json'); - res.status(200).json(interpreters); - } catch (e) { - // Res.status(500).send('Error finding formats'); - throw e; - } + const interpreters = AdapterService.getInstance().getAllFormats(); + res.setHeader('Content-Type', 'application/json'); + res.status(200).json(interpreters); }; /* diff --git a/adapter/src/adapter/api/rest/utils.ts b/adapter/src/adapter/api/rest/utils.ts index 8e793a472..6061d3835 100644 --- a/adapter/src/adapter/api/rest/utils.ts +++ b/adapter/src/adapter/api/rest/utils.ts @@ -16,4 +16,4 @@ export function asyncHandler( const handlerResult = handler(req, res, next); Promise.resolve(handlerResult).catch(next); }; -} \ No newline at end of file +} diff --git a/adapter/src/adapter/importer/HttpImporter.ts b/adapter/src/adapter/importer/HttpImporter.ts index 883cc5bda..2b508cc99 100644 --- a/adapter/src/adapter/importer/HttpImporter.ts +++ b/adapter/src/adapter/importer/HttpImporter.ts @@ -1,88 +1,89 @@ -import { ImporterParameterError } from "../model/exceptions/ImporterParameterError"; -import { Importer } from "./Importer"; -import { ImporterParameterDescription } from "./ImporterParameterDescription"; -const axios = require('axios'); +import axios from 'axios'; -export class HttpImporter extends Importer { +import { ImporterParameterError } from '../model/exceptions/ImporterParameterError'; - //TODO RuntimeParameters type is probably wrong - type: string = "HTTP" - description: string = "Plain HTTP" - parameters:ImporterParameterDescription[] = [new ImporterParameterDescription({name:"location", description:"String of the URI for the HTTP call", type:"string"}), - new ImporterParameterDescription({name:"encoding", description:"Encoding of the source. Available encodings: ISO-8859-1, US-ASCII, UTF-8", type:"string"}), - new ImporterParameterDescription({name:"defaultParameters", description:"Default values for open parameters in the URI", required:false, type:"RuntimeParameters"})] +import { Importer } from './Importer'; +import { ImporterParameterDescription } from './ImporterParameterDescription'; - // Override annotation is not necessary, but will be used for a better understanding of the code - override getType(): string { - return this.type - } - - override getDescription(): string { - return this.description - } - - override getAvailableParameters(): ImporterParameterDescription[] { - return this.parameters; - } +export class HttpImporter extends Importer { + // TODO RuntimeParameters type is probably wrong + type = 'HTTP'; + description = 'Plain HTTP'; + parameters: ImporterParameterDescription[] = [ + new ImporterParameterDescription({ + name: 'location', + description: 'String of the URI for the HTTP call', + type: 'string', + }), + new ImporterParameterDescription({ + name: 'encoding', + description: + 'Encoding of the source. Available encodings: ISO-8859-1, US-ASCII, UTF-8', + type: 'string', + }), + new ImporterParameterDescription({ + name: 'defaultParameters', + description: 'Default values for open parameters in the URI', + required: false, + type: 'RuntimeParameters', + }), + ]; - override validateParameters(inputParameters: Record): void { - super.validateParameters(inputParameters); - const encoding: string = inputParameters.encoding as string; + // Override annotation is not necessary, but will be used for a better understanding of the code + override getType(): string { + return this.type; + } - // TODO CHECK IF ENCODING ARE WRITTEN CORRECT - if (encoding !== "ISO-8859-1" && encoding !== "US-ASCII" && encoding !== "UTF-8") { - throw new Error(this.getType() + " interpreter requires parameter encoding to have value " + - "ISO-8859-1" + ", " + - "US-ASCII" + ", " + - "UTF-8" - + ". Your given value " + encoding + " is invalid!"); - } - } - /** - protected void validateParameters(Map inputParameters) throws ImporterParameterException { - super.validateParameters(inputParameters); + override getDescription(): string { + return this.description; + } - String encoding = (String) inputParameters.get("encoding"); - if (!encoding.equals(StandardCharsets.ISO_8859_1.name()) && !encoding.equals(StandardCharsets.US_ASCII.name()) && !encoding.equals(StandardCharsets.UTF_8.name())) { - throw new IllegalArgumentException(getType() + " interpreter requires parameter encoding to have value " + - StandardCharsets.ISO_8859_1 + ", " + - StandardCharsets.US_ASCII + ", " + - StandardCharsets.UTF_8 - + ". Your given value " + encoding + " is invalid!"); - } - } - */ - override async doFetch(parameters: Record): Promise { - let uri = parameters.location - console.log(parameters) - let encoding = parameters.encoding - // TODO see if encoding from response is good - return axios({ - method: 'get', - url: uri, - responseEncoding: encoding - }).then(function (response: any) { - console.log(response.data) - return response.data - }).catch(function (error: any) { - console.error(error) - throw new ImporterParameterError("Could not Fetch from URI:" + uri) - }); - } + override getAvailableParameters(): ImporterParameterDescription[] { + return this.parameters; + } - /* + override validateParameters(inputParameters: Record): void { + super.validateParameters(inputParameters); + const encoding: string = inputParameters.encoding as string; - @Override - protected String doFetch(Map parameters) throws ImporterParameterException { - String location = parameters.get("location").toString(); - try { - URI uri = URI.create(location); - byte[] rawResponse = restTemplate.getForEntity(uri, byte[].class).getBody(); - return new String(rawResponse, Charset.forName((String) parameters.get("encoding"))); - } catch (IllegalArgumentException e) { - throw new ImporterParameterException(e.getMessage()); + // TODO CHECK IF ENCODING ARE WRITTEN CORRECT + if ( + encoding !== 'ISO-8859-1' && + encoding !== 'US-ASCII' && + encoding !== 'UTF-8' + ) { + throw new Error( + this.getType() + + ' interpreter requires parameter encoding to have value ' + + 'ISO-8859-1' + + ', ' + + 'US-ASCII' + + ', ' + + 'UTF-8' + + '. Your given value ' + + encoding + + ' is invalid!', + ); } } - */ -} \ No newline at end of file + override async doFetch(parameters: Record): Promise { + const uri = parameters.location; + console.log(parameters); + const encoding = parameters.encoding; + // TODO see if encoding from response is good + return axios({ + method: 'get', + url: uri, + responseEncoding: encoding, + }) + .then(function (response: any) { + console.log(response.data); + return response.data; + }) + .catch(function (error: any) { + console.error(error); + throw new ImporterParameterError('Could not Fetch from URI:' + uri); + }); + } +} diff --git a/adapter/src/adapter/importer/Importer.ts b/adapter/src/adapter/importer/Importer.ts index 0f60690fb..d07a1d246 100644 --- a/adapter/src/adapter/importer/Importer.ts +++ b/adapter/src/adapter/importer/Importer.ts @@ -1,71 +1,82 @@ -import { ImporterParameterDescription } from "./ImporterParameterDescription"; -import { ImporterParameterError } from "../model/exceptions/ImporterParameterError"; -import { generateKeySync } from "crypto"; +import { generateKeySync } from 'crypto'; + +import { ImporterParameterError } from '../model/exceptions/ImporterParameterError'; + +import { ImporterParameterDescription } from './ImporterParameterDescription'; export abstract class Importer { type: string | undefined; description: string | undefined; getRequiredParameters(): Array { - return this.getAvailableParameters().filter((item: any) => item.required) as Array + return this.getAvailableParameters().filter( + (item: ImporterParameterDescription) => item.required, + ); } - //@JsonProperty("parameters") - abstract getAvailableParameters() :Array; + abstract getAvailableParameters(): Array; - async fetch(parameters:Record ): Promise { //throws ImporterParameterException - this.validateParameters(parameters); - const x = await this.doFetch(parameters); - return x; - //return JSON.stringify(x); + async fetch(parameters: Record): Promise { + // Throws ImporterParameterException + this.validateParameters(parameters); + const x = await this.doFetch(parameters); + return x; } abstract getType(): string; abstract getDescription(): string; - abstract doFetch(parameters: Record): Promise; //throws ImporterParameterException + abstract doFetch(parameters: Record): Promise; // Throws ImporterParameterException - validateParameters(inputParameters: Record) { //throws ImporterParameterException; + validateParameters(inputParameters: Record): void { + // Throws ImporterParameterException; - let illegalArguments: boolean = false; - let illegalArgumentsMessage: string = ""; + let illegalArguments = false; + let illegalArgumentsMessage = ''; - const possibleParameters: Array = this.getAvailableParameters(); + const possibleParameters: Array = + this.getAvailableParameters(); - let unnecessaryArguments = []; - const names = possibleParameters.map(a => a.name); + const unnecessaryArguments = []; + const names = possibleParameters.map((a) => a.name); const keys = Object.keys(inputParameters); for (const entry of keys) { - if(!names.includes(entry)) { + if (!names.includes(entry)) { unnecessaryArguments.push(entry); } } - if(unnecessaryArguments.length > 0){ + if (unnecessaryArguments.length > 0) { illegalArguments = true; - for(const argument of unnecessaryArguments){ - illegalArgumentsMessage += argument + " is not needed by importer \n" + for (const argument of unnecessaryArguments) { + illegalArgumentsMessage += argument + ' is not needed by importer \n'; } } - const requiredParameters = this.getRequiredParameters() - for (const requiredParameter of requiredParameters){ + const requiredParameters = this.getRequiredParameters(); + for (const requiredParameter of requiredParameters) { // TODO is that OK? - const checkType = (inputParameters[requiredParameter.name] as any).constructor.name - if (inputParameters[requiredParameter.name] == null){ + const name = requiredParameter.name; + const checkType = (inputParameters[name] as Record) + .constructor.name; + if (inputParameters[requiredParameter.name] == null) { illegalArguments = true; - illegalArgumentsMessage = illegalArgumentsMessage + this.type + "importer requires parameter " + requiredParameter.name + "\n"; - } - - else if(checkType.toLowerCase() != requiredParameter.type){ + illegalArgumentsMessage += this.type; + illegalArgumentsMessage += 'importer requires parameter '; + illegalArgumentsMessage += name; + illegalArgumentsMessage += '\n'; + } else if (checkType.toLowerCase() !== requiredParameter.type) { illegalArguments = true; - illegalArgumentsMessage = illegalArgumentsMessage + this.type + " importer requires parameter " - + requiredParameter.name + " to be type " + (requiredParameter.type as string) + "\n"; + illegalArgumentsMessage += this.type; + illegalArgumentsMessage += ' importer requires parameter '; + illegalArgumentsMessage += name; + illegalArgumentsMessage += ' to be type '; + illegalArgumentsMessage += requiredParameter.type; + illegalArgumentsMessage += '\n'; } } - if(illegalArguments){ + if (illegalArguments) { throw new ImporterParameterError(illegalArgumentsMessage); } - } } diff --git a/adapter/src/adapter/importer/ImporterParameterDescription.ts b/adapter/src/adapter/importer/ImporterParameterDescription.ts index 00696e168..36e5ea449 100644 --- a/adapter/src/adapter/importer/ImporterParameterDescription.ts +++ b/adapter/src/adapter/importer/ImporterParameterDescription.ts @@ -1,15 +1,25 @@ export class ImporterParameterDescription { + name: string; + description: string; + required: boolean; + + // TODO @Georg: need to check how to store the class in typescript, can use instance.constructor.name to get the name probably -> is stored as string + type: unknown; + + constructor({ + name, + description, + required = true, + type, + }: { name: string; description: string; - required: boolean; - - //TODO @Georg: need to check how to store the class in typescript, can use instance.constructor.name to get the name probably -> is stored as string + required?: boolean; type: unknown; - - constructor ({name,description,required=true,type}: {name:string, description:string, required?:boolean, type:unknown}) { - this.name = name; - this.description = description; - this.required = required; - this.type = type; - } -} \ No newline at end of file + }) { + this.name = name; + this.description = description; + this.required = required; + this.type = type; + } +} diff --git a/adapter/src/adapter/interpreter/CsvInterpreter.ts b/adapter/src/adapter/interpreter/CsvInterpreter.ts index d3c97cabf..4422cb265 100644 --- a/adapter/src/adapter/interpreter/CsvInterpreter.ts +++ b/adapter/src/adapter/interpreter/CsvInterpreter.ts @@ -1,78 +1,110 @@ -import {Interpreter} from "./Interpreter"; -import { InterpreterParameterDescription } from "./InterpreterParameterDescription"; -const csv=require('csvtojson') +import csv from 'csvtojson'; +import { Interpreter } from './Interpreter'; +import { InterpreterParameterDescription } from './InterpreterParameterDescription'; export class CsvInterpreter extends Interpreter { + type = 'CSV'; - type: string = "CSV" - description: string = "Interpret data as CSV data" - parameters: InterpreterParameterDescription[] = [new InterpreterParameterDescription("columnSeparator", "Column delimiter character, only one character supported", "string"), - new InterpreterParameterDescription("lineSeparator", "Line delimiter character, only \\r, \\r\\n, and \\n supported", "string",), - new InterpreterParameterDescription("skipFirstDataRow", "Skip first data row (after header)", "boolean"), - new InterpreterParameterDescription("firstRowAsHeader", "Interpret first row as header for columns", "boolean")] - - - + description = 'Interpret data as CSV data'; + parameters: InterpreterParameterDescription[] = [ + new InterpreterParameterDescription( + 'columnSeparator', + 'Column delimiter character, only one character supported', + 'string', + ), + new InterpreterParameterDescription( + 'lineSeparator', + 'Line delimiter character, only \\r, \\r\\n, and \\n supported', + 'string', + ), + new InterpreterParameterDescription( + 'skipFirstDataRow', + 'Skip first data row (after header)', + 'boolean', + ), + new InterpreterParameterDescription( + 'firstRowAsHeader', + 'Interpret first row as header for columns', + 'boolean', + ), + ]; override getType(): string { - return this.type + return this.type; } override getDescription(): string { - return this.description + return this.description; } override getAvailableParameters(): InterpreterParameterDescription[] { return this.parameters; } - override doInterpret(data: string, parameters: Record): Promise { - data = 'col1,col2,col3\n' + - 'val11,val12,val13\n' + - 'val21,val22,val23'; + override doInterpret( + data: string, + parameters: Record, + ): Promise { + data = 'col1,col2,col3\n' + 'val11,val12,val13\n' + 'val21,val22,val23'; + + const columnSeparator = (parameters.columnSeparator as string).charAt(0); + const lineSeparator: string = parameters.lineSeparator as string; + const firstRowAsHeader: boolean = parameters.firstRowAsHeader as boolean; // True = With header, False = WithoutHeader + const skipFirstDataRow: boolean = parameters.skipFirstDataRow as boolean; - let columnSeparator = (parameters.columnSeparator as string).charAt(0) - let lineSeparator: string = parameters.lineSeparator as string; - let firstRowAsHeader: boolean = parameters.firstRowAsHeader as boolean; // True = With header, False = WithoutHeader - let skipFirstDataRow: boolean = parameters.skipFirstDataRow as boolean; - - let json: any[] = []; + const json: any[] = []; return csv({ noheader: !firstRowAsHeader, // Be Careful: Need to Invert the boolean here - output: "json", + output: 'json', delimiter: columnSeparator, - eol: lineSeparator + eol: lineSeparator, }) - .fromString(data) - .subscribe((csvRow: any, index:any)=>{ - // Todo need to test if this works - if(skipFirstDataRow && ((index == 0 && !firstRowAsHeader) || (index == 1 && firstRowAsHeader))) { - // Skip First Row - } - else { - json.push(csvRow); - } - }).on('done', (error:any) => { - return new Promise(function(resolve, reject){ - resolve(JSON.stringify(json)); + .fromString(data) + .subscribe((csvRow: any, index: any) => { + // Todo need to test if this works + if ( + skipFirstDataRow && + ((index == 0 && !firstRowAsHeader) || + (index == 1 && firstRowAsHeader)) + ) { + // Skip First Row + } else { + json.push(csvRow); + } + }) + .on('done', (error: any) => { + return new Promise(function (resolve, reject) { + resolve(JSON.stringify(json)); + }); }); - }) - - } override validateParameters(inputParameters: Record): void { - super.validateParameters(inputParameters); - const lineSeparator: string = inputParameters.lineSeparator as string; + super.validateParameters(inputParameters); + const lineSeparator: string = inputParameters.lineSeparator as string; - if (lineSeparator !== "\n" && lineSeparator !== "\r" && lineSeparator !== "\r\n") { - throw new Error(this.getType() + " interpreter requires parameter lineSeparator to have" + - " value \\n, \\r, or \\r\\n. Your given value " + lineSeparator + " is invalid!"); - } + if ( + lineSeparator !== '\n' && + lineSeparator !== '\r' && + lineSeparator !== '\r\n' + ) { + throw new Error( + this.getType() + + ' interpreter requires parameter lineSeparator to have' + + ' value \\n, \\r, or \\r\\n. Your given value ' + + lineSeparator + + ' is invalid!', + ); + } - var columnSeparator: string = inputParameters.columnSeparator as string; - if (columnSeparator.length !== 1) { - throw new Error(this.getType() + " interpreter requires parameter columnSeparator to have" + - " length 1. Your given value " + columnSeparator + " is invalid!"); + const columnSeparator: string = inputParameters.columnSeparator as string; + if (columnSeparator.length !== 1) { + throw new Error( + this.getType() + + ' interpreter requires parameter columnSeparator to have' + + ' length 1. Your given value ' + + columnSeparator + + ' is invalid!', + ); } } } diff --git a/adapter/src/adapter/interpreter/Interpreter.ts b/adapter/src/adapter/interpreter/Interpreter.ts index 9f35c5d8c..e8e21097d 100644 --- a/adapter/src/adapter/interpreter/Interpreter.ts +++ b/adapter/src/adapter/interpreter/Interpreter.ts @@ -1,40 +1,44 @@ -import { stringifiers } from "@jvalue/node-dry-basics"; -import { InterpreterParameterError } from "../model/exceptions/InterpreterParameterError"; -import { InterpreterParameterDescription } from "./InterpreterParameterDescription"; +import { InterpreterParameterError } from '../model/exceptions/InterpreterParameterError'; + +import { InterpreterParameterDescription } from './InterpreterParameterDescription'; export abstract class Interpreter { type: string | undefined; description: string | undefined; - - async interpret(data: string, parameters: Record): Promise { + async interpret( + data: string, + parameters: Record, + ): Promise { this.validateParameters(parameters); return await this.doInterpret(data, parameters); } abstract getType(): string; abstract getDescription(): string; - abstract doInterpret(data: string, parameters: Record): Promise + abstract doInterpret( + data: string, + parameters: Record, + ): Promise; abstract getAvailableParameters(): Array; - validateParameters(inputParameters: Record) { - let illegalArguments: boolean = false; - let illegalArgumentsMessage: string = ""; - - for (const requiredParameter of this.getAvailableParameters()){ - const param = (inputParameters[requiredParameter.name] as InterpreterParameterDescription) - if (param == null){ + validateParameters(inputParameters: Record): void { + let illegalArguments = false; + let illegalArgumentsMessage = ''; + + for (const requiredParameter of this.getAvailableParameters()) { + const param = inputParameters[ + requiredParameter.name + ] as InterpreterParameterDescription; + if (param == null) { illegalArguments = true; - illegalArgumentsMessage = illegalArgumentsMessage + this.type + "interpreter requires parameter " + requiredParameter.name + "\n"; + illegalArgumentsMessage += this.type; + illegalArgumentsMessage += 'interpreter requires parameter '; + illegalArgumentsMessage += requiredParameter.name; + illegalArgumentsMessage += '\n'; } - // TODO is that OK? - /*else if(((inputParameters.requiredParameter as InterpreterParameterDescription).name) as any)).constructor.name != requiredParameter.type){ - illegalArguments = true; - illegalArgumentsMessage = illegalArgumentsMessage + this.type + " interpreter requires parameter " - + requiredParameter.name + " to be type " + (requiredParameter.type as string) + "\n"; - }*/ } - if(illegalArguments){ + if (illegalArguments) { throw new InterpreterParameterError(illegalArgumentsMessage); } } diff --git a/adapter/src/adapter/interpreter/InterpreterParameterDescription.ts b/adapter/src/adapter/interpreter/InterpreterParameterDescription.ts index 59252aabc..badda2955 100644 --- a/adapter/src/adapter/interpreter/InterpreterParameterDescription.ts +++ b/adapter/src/adapter/interpreter/InterpreterParameterDescription.ts @@ -1,15 +1,14 @@ -export class InterpreterParameterDescription{ +export class InterpreterParameterDescription { + name: string; + description: string; + // TODO default value? is not used in Interpreters + required: boolean | undefined; + // TODO @Georg: need to check how to store the class in typescript, can use instance.constructor.name to get the name probably -> is stored as string -> Same as in Importer + type: unknown; - name: string; - description: string; - // TODO default value? is not used in Interpreters - required: boolean | undefined; - //TODO @Georg: need to check how to store the class in typescript, can use instance.constructor.name to get the name probably -> is stored as string -> Same as in Importer - type: unknown; - - constructor (name: string, description: string, type: unknown) { - this.name = name; - this.description = description; - this.type = type; - } -} \ No newline at end of file + constructor(name: string, description: string, type: unknown) { + this.name = name; + this.description = description; + this.type = type; + } +} diff --git a/adapter/src/adapter/interpreter/JsonInterpreter.ts b/adapter/src/adapter/interpreter/JsonInterpreter.ts index 60972e6b7..833238d4f 100644 --- a/adapter/src/adapter/interpreter/JsonInterpreter.ts +++ b/adapter/src/adapter/interpreter/JsonInterpreter.ts @@ -1,27 +1,29 @@ -import {Interpreter} from "./Interpreter"; -import { InterpreterParameterDescription } from "./InterpreterParameterDescription"; +import { Interpreter } from './Interpreter'; +import { InterpreterParameterDescription } from './InterpreterParameterDescription'; export class JsonInterpreter extends Interpreter { - - type: string = "JSON" - description: string = "Interpret data as JSON data" - parameters: InterpreterParameterDescription[] = [] + type = 'JSON'; + description = 'Interpret data as JSON data'; + parameters: InterpreterParameterDescription[] = []; override getType(): string { - return this.type + return this.type; } - + override getDescription(): string { - return this.description + return this.description; } override getAvailableParameters(): InterpreterParameterDescription[] { return this.parameters; } - override doInterpret(data: string, parameters: Record): Promise { - return new Promise(function(resolve, reject){ - resolve(data); - }); + override doInterpret( + data: string, + _parameters: Record, + ): Promise { + return new Promise(function (resolve) { + resolve(data); + }); } -} \ No newline at end of file +} diff --git a/adapter/src/adapter/interpreter/XmlInterpreter.ts b/adapter/src/adapter/interpreter/XmlInterpreter.ts index 3ec566f1b..af46dae29 100644 --- a/adapter/src/adapter/interpreter/XmlInterpreter.ts +++ b/adapter/src/adapter/interpreter/XmlInterpreter.ts @@ -1,19 +1,19 @@ -import {Interpreter} from "./Interpreter"; -import { InterpreterParameterDescription } from "./InterpreterParameterDescription"; -const xml2js = require('xml2js'); +import xml2js from 'xml2js'; -export class XmlInterpreter extends Interpreter{ +import { Interpreter } from './Interpreter'; +import { InterpreterParameterDescription } from './InterpreterParameterDescription'; - type: string = "XML" - description: string = "Interpret data as XML data" - parameters: InterpreterParameterDescription[] = [] +export class XmlInterpreter extends Interpreter { + type = 'XML'; + description = 'Interpret data as XML data'; + parameters: InterpreterParameterDescription[] = []; override getType(): string { - return this.type + return this.type; } override getDescription(): string { - return this.description + return this.description; } override getAvailableParameters(): InterpreterParameterDescription[] { @@ -21,18 +21,23 @@ export class XmlInterpreter extends Interpreter{ } // TODO @Georg check if this package can be used.. - override doInterpret(data: string, parameters: Record): Promise { - const parser = new xml2js.Parser({explicitArray: false}); - - return parser.parseStringPromise(data).then(function (result:any) { - // `result` is a JavaScript object - // convert it to a JSON string - return result.root - //const json = JSON.stringify(result.root); - //return json; - }) - .catch(function (err:any) { - throw err - }); + override doInterpret( + data: string, + parameters: Record, + ): Promise { + const parser = new xml2js.Parser({ explicitArray: false }); + + return parser + .parseStringPromise(data) + .then(function (result: any) { + // `result` is a JavaScript object + // Convert it to a JSON string + return result.root; + // Const json = JSON.stringify(result.root); + // Return json; + }) + .catch(function (err: any) { + throw err; + }); } } diff --git a/adapter/src/adapter/model/AdapterConfig.ts b/adapter/src/adapter/model/AdapterConfig.ts index c4f21c638..a3f9d5d1e 100644 --- a/adapter/src/adapter/model/AdapterConfig.ts +++ b/adapter/src/adapter/model/AdapterConfig.ts @@ -12,7 +12,7 @@ export interface AdapterConfig { export class AdapterConfigValidator { private errors: string[] = []; - validate(request: unknown): request is AdapterConfigDTO { + validate(request: AdapterConfigDTO): request is AdapterConfigDTO { this.errors = []; if (!validators.isObject(request)) { this.errors.push("'AdapterConfig' must be an object"); diff --git a/adapter/src/adapter/model/EndpointDTOs.ts b/adapter/src/adapter/model/EndpointDTOs.ts index 12d9210b5..a646c180d 100644 --- a/adapter/src/adapter/model/EndpointDTOs.ts +++ b/adapter/src/adapter/model/EndpointDTOs.ts @@ -12,13 +12,3 @@ export interface FormatConfigDTO { type: string; parameters: Record; } - -/* Export interface FormatParameterDTO { - location: string; - encoding: string; -} - -export interface ProtocolParameterDTO { - type: string; - parameters: Record; -}*/ diff --git a/adapter/src/adapter/model/FormatConfig.ts b/adapter/src/adapter/model/FormatConfig.ts index ba600fefd..b7fbc5a12 100644 --- a/adapter/src/adapter/model/FormatConfig.ts +++ b/adapter/src/adapter/model/FormatConfig.ts @@ -1,6 +1,6 @@ -import { Format } from "./enum/Format"; +import { Format } from './enum/Format'; export interface FormatConfig { - format:Format; - parameters: Record; - } + format: Format; + parameters: Record; +} diff --git a/adapter/src/adapter/model/ProtocolConfig.ts b/adapter/src/adapter/model/ProtocolConfig.ts index 74a2f6632..e7f9ad477 100644 --- a/adapter/src/adapter/model/ProtocolConfig.ts +++ b/adapter/src/adapter/model/ProtocolConfig.ts @@ -3,7 +3,8 @@ import { stringify } from 'querystring'; import { validators } from '@jvalue/node-dry-basics'; import { JsonAlias, JsonClassType, JsonProperty } from 'jackson-js'; -import { ProtocolConfigDTO } from './EndpointDTOs'; +import { AdapterConfig } from './AdapterConfig'; +import { AdapterConfigDTO, ProtocolConfigDTO } from './EndpointDTOs'; import { Protocol } from './enum/Protocol'; export interface ProtocolConfig { @@ -14,21 +15,21 @@ export interface ProtocolConfig { export class ProtocolConfigValidator { private errors: string[] = []; - validate(request: ProtocolConfigDTO): request is ProtocolConfigDTO { + validate(request: AdapterConfigDTO): request is AdapterConfigDTO { this.errors = []; if (!validators.isObject(request)) { this.errors.push("'ProtocolConfig' must be an object"); return false; } - if (!validators.hasProperty(request, 'type')) { + if (!validators.hasProperty(request.protocol, 'type')) { this.errors.push("'type' property is missing"); - } else if (!validators.isString(request.type)) { + } else if (!validators.isString(request.protocol.type)) { this.errors.push("'type' must be a string"); } - if (!validators.hasProperty(request, 'parameters')) { + if (!validators.hasProperty(request.protocol, 'parameters')) { this.errors.push("'parameters' property is missing"); - } else if (!validators.isObject(request.parameters)) { + } else if (!validators.isObject(request.protocol.parameters)) { this.errors.push("'parameters' must be an object or array"); } return this.errors.length === 0; diff --git a/adapter/src/adapter/model/exceptions/AdapterError.ts b/adapter/src/adapter/model/exceptions/AdapterError.ts index 3a235cda0..026d48bd4 100644 --- a/adapter/src/adapter/model/exceptions/AdapterError.ts +++ b/adapter/src/adapter/model/exceptions/AdapterError.ts @@ -1,5 +1,5 @@ -export class AdapterError extends Error{ - constructor(msg: string){ - super(msg); - } -} \ No newline at end of file +export class AdapterError extends Error { + constructor(msg: string) { + super(msg); + } +} diff --git a/adapter/src/adapter/model/exceptions/ImporterParameterError.ts b/adapter/src/adapter/model/exceptions/ImporterParameterError.ts index f7ffcd91d..3278dd1ac 100644 --- a/adapter/src/adapter/model/exceptions/ImporterParameterError.ts +++ b/adapter/src/adapter/model/exceptions/ImporterParameterError.ts @@ -1,8 +1,7 @@ -import { AdapterError } from "./AdapterError"; +import { AdapterError } from './AdapterError'; -export class ImporterParameterError extends AdapterError{ - - constructor(msg: string){ - super(msg); - } -} \ No newline at end of file +export class ImporterParameterError extends AdapterError { + constructor(msg: string) { + super(msg); + } +} diff --git a/adapter/src/adapter/model/exceptions/InterpreterParameterError.ts b/adapter/src/adapter/model/exceptions/InterpreterParameterError.ts index 238ce7666..0fef375a5 100644 --- a/adapter/src/adapter/model/exceptions/InterpreterParameterError.ts +++ b/adapter/src/adapter/model/exceptions/InterpreterParameterError.ts @@ -1,8 +1,7 @@ -import { AdapterError } from "./AdapterError"; +import { AdapterError } from './AdapterError'; -export class InterpreterParameterError extends AdapterError{ - - constructor(msg: string){ - super(msg); - } -} \ No newline at end of file +export class InterpreterParameterError extends AdapterError { + constructor(msg: string) { + super(msg); + } +} diff --git a/adapter/src/adapter/services/adapterService.ts b/adapter/src/adapter/services/adapterService.ts index 03ab543e3..91b91a4cf 100644 --- a/adapter/src/adapter/services/adapterService.ts +++ b/adapter/src/adapter/services/adapterService.ts @@ -1,63 +1,65 @@ -import { JsonRawValue } from "jackson-js"; -import { Importer } from "../importer/Importer"; -import { Interpreter } from "../interpreter/Interpreter"; -import { AdapterConfig } from "../model/AdapterConfig"; -import { DataImportResponse } from "../model/DataImportResponse"; -import { Format } from "../model/enum/Format"; -import { Protocol } from "../model/enum/Protocol"; -import { FormatConfig } from "../model/FormatConfig"; +import { JsonRawValue } from 'jackson-js'; -import { ProtocolConfig } from "../model/ProtocolConfig"; +import { Importer } from '../importer/Importer'; +import { Interpreter } from '../interpreter/Interpreter'; +import { AdapterConfig } from '../model/AdapterConfig'; +import { DataImportResponse } from '../model/DataImportResponse'; +import { Format } from '../model/enum/Format'; +import { Protocol } from '../model/enum/Protocol'; +import { FormatConfig } from '../model/FormatConfig'; +import { ProtocolConfig } from '../model/ProtocolConfig'; export class AdapterService { - /** - * @description Create an instance of AdapterService - */ - private static instance: AdapterService; + /** + * @description Create an instance of AdapterService + */ + private static instance: AdapterService; - constructor () { - } - - public static getInstance(): AdapterService { - if (!AdapterService.instance) { - AdapterService.instance = new AdapterService(); - } + constructor() {} - return AdapterService.instance; + static getInstance(): AdapterService { + if (!AdapterService.instance) { + AdapterService.instance = new AdapterService(); } + return AdapterService.instance; + } - // To Implement - public getAllFormats(): Array { - return [Format.CSV, Format.JSON, Format.XML] - } + // To Implement + getAllFormats(): Array { + return [Format.CSV, Format.JSON, Format.XML]; + } + getAllProtocols(): Array { + return [Protocol.HTTP]; + } - public getAllProtocols(): Array { - return [Protocol.HTTP] - } + async executeJob(_adapterConfig: AdapterConfig): Promise { + const rawData = await this.executeProtocol(_adapterConfig.protocolConfig); + const result = await this.executeFormat( + rawData, + _adapterConfig.formatConfig, + ); + const returnValue: DataImportResponse = { data: result }; + return returnValue; + } - public async executeJob(_adapterConfig: AdapterConfig): Promise { - const rawData = await this.executeProtocol(_adapterConfig.protocolConfig); - const result = await this.executeFormat(rawData, _adapterConfig.formatConfig); - const returnValue: DataImportResponse = {data: result}; - return returnValue; - } + async executeRawJob( + _protocolConfig: ProtocolConfig, + ): Promise { + const value = await this.executeProtocol(_protocolConfig); + const returnValue: DataImportResponse = { data: value }; + return returnValue; + } - public async executeRawJob(_protocolConfig: ProtocolConfig): Promise { - const value = await this.executeProtocol(_protocolConfig) - const returnValue: DataImportResponse = {data: value}; - return returnValue; - } + async executeProtocol(config: ProtocolConfig): Promise { + const importer = config.protocol.getImporter(); + return await importer.fetch(config.parameters); + } - public async executeProtocol (config: ProtocolConfig): Promise { - const importer = config.protocol.getImporter(); - return await importer.fetch(config.parameters) - } - - public async executeFormat(rawData: string, config: FormatConfig): Promise { - const interpreter = config.format.getInterpreter(); - return await interpreter.interpret(rawData, config.parameters); - } + async executeFormat(rawData: string, config: FormatConfig): Promise { + const interpreter = config.format.getInterpreter(); + return await interpreter.interpret(rawData, config.parameters); + } } -export const adapterService = AdapterService.getInstance(); \ No newline at end of file +export const adapterService = AdapterService.getInstance(); From 2dc514cb73eb86d5f5bcdf60bfcb346e5809a588 Mon Sep 17 00:00:00 2001 From: = <=> Date: Wed, 20 Apr 2022 15:31:19 +0200 Subject: [PATCH 05/40] linter in datasource with @PVahldiek --- .../src/adapter/services/adapterService.ts | 2 - .../datasource/api/rest/dataImportEndpoint.ts | 96 +++++++++---- .../datasource/api/rest/dataSourceEndpoint.ts | 136 ++++++++++-------- .../model/datasourceModelForAmqp.ts | 2 +- adapter/src/datasource/model/outboxEvent.ts | 6 +- .../repository/dataImportRepository.ts | 19 ++- .../repository/datasourceRepository.ts | 47 +++--- .../src/datasource/repository/knexHelper.ts | 133 +++++++++-------- .../datasource/repository/outboxRepository.ts | 30 ++-- 9 files changed, 254 insertions(+), 217 deletions(-) diff --git a/adapter/src/adapter/services/adapterService.ts b/adapter/src/adapter/services/adapterService.ts index 91b91a4cf..ae336c4e0 100644 --- a/adapter/src/adapter/services/adapterService.ts +++ b/adapter/src/adapter/services/adapterService.ts @@ -15,8 +15,6 @@ export class AdapterService { */ private static instance: AdapterService; - constructor() {} - static getInstance(): AdapterService { if (!AdapterService.instance) { AdapterService.instance = new AdapterService(); diff --git a/adapter/src/datasource/api/rest/dataImportEndpoint.ts b/adapter/src/datasource/api/rest/dataImportEndpoint.ts index 8bbd09551..65e55eb30 100644 --- a/adapter/src/datasource/api/rest/dataImportEndpoint.ts +++ b/adapter/src/datasource/api/rest/dataImportEndpoint.ts @@ -1,32 +1,53 @@ -import express, {Express, json} from "express"; -import {asyncHandler} from "../../../adapter/api/rest/utils"; -import {DataImportRepository} from "../../repository/dataImportRepository"; -import {KnexHelper} from "../../repository/knexHelper"; +import express, { Express, json } from 'express'; -const dataImportRepository: DataImportRepository = new DataImportRepository(); +import { asyncHandler } from '../../../adapter/api/rest/utils'; +import { DataImportRepository } from '../../repository/dataImportRepository'; +import { KnexHelper } from '../../repository/knexHelper'; +const dataImportRepository: DataImportRepository = new DataImportRepository(); export class DataImportEndpoint { - registerRoutes = (app: express.Application): void => { - app.get('/datasources/:datasourceId/imports', asyncHandler(this.getMetaDataImportsForDatasource)); - app.get('/datasources/:datasourceId/imports/latest', asyncHandler(this.getLatestMetaDataImportForDatasource)); - app.get('/datasources/:datasourceId/imports/latest/data', asyncHandler(this.getLatestDataImportForDatasource)); - app.get('/datasources/:datasourceId/imports/:dataImportId', asyncHandler(this.getMetadataForDataImport)); - app.get('/datasources/:datasourceId/imports/:dataImportId/data', asyncHandler(this.getDataFromDataImport)); + app.get( + '/datasources/:datasourceId/imports', + asyncHandler(this.getMetaDataImportsForDatasource), + ); + app.get( + '/datasources/:datasourceId/imports/latest', + asyncHandler(this.getLatestMetaDataImportForDatasource), + ); + app.get( + '/datasources/:datasourceId/imports/latest/data', + asyncHandler(this.getLatestDataImportForDatasource), + ); + app.get( + '/datasources/:datasourceId/imports/:dataImportId', + asyncHandler(this.getMetadataForDataImport), + ); + app.get( + '/datasources/:datasourceId/imports/:dataImportId/data', + asyncHandler(this.getDataFromDataImport), + ); }; -//TODO Transactional bei gets??? + // TODO Transactional bei gets??? getMetaDataImportsForDatasource = async ( req: express.Request, res: express.Response, ): Promise => { - let result = await dataImportRepository.getMetaDataImportByDatasource(req.params.datasourceId) + const result = await dataImportRepository.getMetaDataImportByDatasource( + req.params.datasourceId, + ); let i = 0; result.forEach(function (el: any) { - let dataImportId = el.id; - result[i]["location"] = "/datasources/" + req.params.datasourceId + "/imports/" + dataImportId + "/data"; + const dataImportId = el.id; + result[i].location = + '/datasources/' + + req.params.datasourceId + + '/imports/' + + dataImportId + + '/data'; i++; - }) + }); res.status(200).send(result); }; @@ -35,10 +56,16 @@ export class DataImportEndpoint { req: express.Request, res: express.Response, ): Promise => { - let id = req.params.datasourceId; - let result = await dataImportRepository.getLatestMetaDataImportByDatasourceId(id); - let dataImportId = result[0].id; - result[0]["location"] = "/datasources/" + req.params.datasourceId + "/imports/" + dataImportId + "/data"; + const id = req.params.datasourceId; + const result = + await dataImportRepository.getLatestMetaDataImportByDatasourceId(id); + const dataImportId = result[0].id; + result[0].location = + '/datasources/' + + req.params.datasourceId + + '/imports/' + + dataImportId + + '/data'; res.status(200).send(result[0]); }; @@ -46,9 +73,11 @@ export class DataImportEndpoint { req: express.Request, res: express.Response, ): Promise => { - let id = req.params.datasourceId; - let result = await dataImportRepository.getLatestDataImportByDatasourceId(id); - const stringResult = KnexHelper.stringFromUTF8Array(result[0].data) + const id = req.params.datasourceId; + const result = await dataImportRepository.getLatestDataImportByDatasourceId( + id, + ); + const stringResult = KnexHelper.stringFromUTF8Array(result[0].data); res.status(200).send(stringResult); }; @@ -56,9 +85,12 @@ export class DataImportEndpoint { req: express.Request, res: express.Response, ): Promise => { - let datasourceId = req.params.datasourceId; - let dataImportId = req.params.dataImportId; - let result = await dataImportRepository.getMetadataForDataImport(datasourceId, dataImportId); + const datasourceId = req.params.datasourceId; + const dataImportId = req.params.dataImportId; + const result = await dataImportRepository.getMetadataForDataImport( + datasourceId, + dataImportId, + ); res.status(200).send(result[0]); }; @@ -66,11 +98,13 @@ export class DataImportEndpoint { req: express.Request, res: express.Response, ): Promise => { - let datasourceId = req.params.datasourceId; - let dataImportId = req.params.dataImportId; - let result = await dataImportRepository.getDataFromDataImport(datasourceId, dataImportId); - const stringFromUTF8Array = KnexHelper.stringFromUTF8Array(result[0].data) + const datasourceId = req.params.datasourceId; + const dataImportId = req.params.dataImportId; + const result = await dataImportRepository.getDataFromDataImport( + datasourceId, + dataImportId, + ); + const stringFromUTF8Array = KnexHelper.stringFromUTF8Array(result[0].data); res.status(200).send(stringFromUTF8Array); }; - } diff --git a/adapter/src/datasource/api/rest/dataSourceEndpoint.ts b/adapter/src/datasource/api/rest/dataSourceEndpoint.ts index 568fa3fe5..57f2d52fd 100644 --- a/adapter/src/datasource/api/rest/dataSourceEndpoint.ts +++ b/adapter/src/datasource/api/rest/dataSourceEndpoint.ts @@ -1,40 +1,44 @@ -import express from "express"; -import {asyncHandler} from "../../../adapter/api/rest/utils"; -import {AdapterConfig} from "../../../adapter/model/AdapterConfig"; -import {AdapterService} from "../../../adapter/services/adapterService"; -import {ProtocolConfig} from "../../../adapter/model/ProtocolConfig"; -import {Protocol} from "../../../adapter/model/enum/Protocol"; -import {Format} from "../../../adapter/model/enum/Format"; -import {FormatConfig} from "../../../adapter/model/FormatConfig"; -import {AdapterEndpoint} from "../../../adapter/api/rest/adapterEndpoint"; -import {DatasourceRepository} from "../../repository/datasourceRepository"; -import {KnexHelper} from "../../repository/knexHelper"; -import {OutboxRepository} from "../../repository/outboxRepository"; -import {DatasourceModelForAmqp} from "../../model/datasourceModelForAmqp"; -import {ADAPTER_AMQP_DATASOURCE_UPDATED_TOPIC} from "../../../env"; - +import express from 'express'; + +import { AdapterEndpoint } from '../../../adapter/api/rest/adapterEndpoint'; +import { asyncHandler } from '../../../adapter/api/rest/utils'; +import { AdapterConfig } from '../../../adapter/model/AdapterConfig'; +import { Format } from '../../../adapter/model/enum/Format'; +import { Protocol } from '../../../adapter/model/enum/Protocol'; +import { FormatConfig } from '../../../adapter/model/FormatConfig'; +import { ProtocolConfig } from '../../../adapter/model/ProtocolConfig'; +import { AdapterService } from '../../../adapter/services/adapterService'; +import { ADAPTER_AMQP_DATASOURCE_UPDATED_TOPIC } from '../../../env'; +import { DatasourceModelForAmqp } from '../../model/datasourceModelForAmqp'; +import { DatasourceRepository } from '../../repository/datasourceRepository'; +import { KnexHelper } from '../../repository/knexHelper'; +import { OutboxRepository } from '../../repository/outboxRepository'; const datasourceRepository: DatasourceRepository = new DatasourceRepository(); const outboxRepository: OutboxRepository = new OutboxRepository(); - export class DataSourceEndpoint { - registerRoutes = (app: express.Application): void => { app.get('/datasources', asyncHandler(this.getAllDataSources)); app.get('/datasources/:datasourceId', asyncHandler(this.getDataSource)); app.post('/datasources', asyncHandler(this.addDatasource)); app.put('/datasources/:datasourceId', asyncHandler(this.updateDatasource)); app.delete('/datasources/', asyncHandler(this.deleteAllDatasources)); - app.delete('/datasources/:datasourceId', asyncHandler(this.deleteDatasource)); - app.post('/datasources/:datasourceId/trigger', asyncHandler(this.triggerDataImportForDatasource)); + app.delete( + '/datasources/:datasourceId', + asyncHandler(this.deleteDatasource), + ); + app.post( + '/datasources/:datasourceId/trigger', + asyncHandler(this.triggerDataImportForDatasource), + ); }; getAllDataSources = async ( req: express.Request, res: express.Response, ): Promise => { const result = await datasourceRepository.getAllDataSources(); - let datasource = KnexHelper.createDatasourceFromResultArray(result); + const datasource = KnexHelper.createDatasourceFromResultArray(result); res.status(200).send(datasource); }; @@ -42,10 +46,11 @@ export class DataSourceEndpoint { req: express.Request, res: express.Response, ): Promise => { - - console.log(req.params.datasourceId) - const result = await datasourceRepository.getDataSourceById(req.params.datasourceId) - let datasource = KnexHelper.createDatasourceFromResult(result); + console.log(req.params.datasourceId); + const result = await datasourceRepository.getDataSourceById( + req.params.datasourceId, + ); + const datasource = KnexHelper.createDatasourceFromResult(result); res.status(200).send(datasource); }; @@ -53,16 +58,18 @@ export class DataSourceEndpoint { req: express.Request, res: express.Response, ): Promise => { - //routingkey == topic - //TODO typisierung Datasource & Dataimport - let insertStatement = KnexHelper.getInsertStatementForDataSource(req) - let datasource = await datasourceRepository.addDatasource(insertStatement); - let datasouceModelForAmqp:DatasourceModelForAmqp={ - datasource:datasource - } - // let routingKey=ADAPTER_AMQP_DATASOURCE_CREATED_TOPIC; - let routingKey="datasource.config.created"; - await outboxRepository.publishToOutbox(datasouceModelForAmqp,routingKey); + // Routingkey == topic + // TODO typisierung Datasource & Dataimport + const insertStatement = KnexHelper.getInsertStatementForDataSource(req); + const datasource = await datasourceRepository.addDatasource( + insertStatement, + ); + const datasouceModelForAmqp: DatasourceModelForAmqp = { + datasource: datasource, + }; + // Let routingKey=ADAPTER_AMQP_DATASOURCE_CREATED_TOPIC; + const routingKey = 'datasource.config.created'; + await outboxRepository.publishToOutbox(datasouceModelForAmqp, routingKey); res.status(201).send(datasource); }; @@ -70,15 +77,18 @@ export class DataSourceEndpoint { req: express.Request, res: express.Response, ): Promise => { - //TODO check response 204 with no body ?! - let insertStatement = KnexHelper.getInsertStatementForDataSource(req) - let datasource = await datasourceRepository.updateDatasource(insertStatement, req.params.datasourceId); - let datasourceModelForAmqp:DatasourceModelForAmqp ={ - datasource:datasource - } - // let routingKey=ADAPTER_AMQP_DATASOURCE_UPDATED_TOPIC; - let routingKey="datasource.config.updated"; - await outboxRepository.publishToOutbox(datasourceModelForAmqp,routingKey) + // TODO check response 204 with no body ?! + const insertStatement = KnexHelper.getInsertStatementForDataSource(req); + const datasource = await datasourceRepository.updateDatasource( + insertStatement, + req.params.datasourceId, + ); + const datasourceModelForAmqp: DatasourceModelForAmqp = { + datasource: datasource, + }; + // Let routingKey=ADAPTER_AMQP_DATASOURCE_UPDATED_TOPIC; + const routingKey = 'datasource.config.updated'; + await outboxRepository.publishToOutbox(datasourceModelForAmqp, routingKey); res.status(200).send(datasource); }; @@ -102,26 +112,30 @@ export class DataSourceEndpoint { req: express.Request, res: express.Response, ): Promise => { - //TODO add parameters from request to trigger - //TODO Error Handling general here when datasource == null - let id = req.params.datasourceId; - let result = await datasourceRepository.getDataSourceById(id); - let datasource = KnexHelper.createDatasourceFromResult(result); - let protocolConfigObj: ProtocolConfig = { + // TODO add parameters from request to trigger + // TODO Error Handling general here when datasource == null + const id = req.params.datasourceId; + const result = await datasourceRepository.getDataSourceById(id); + const datasource = KnexHelper.createDatasourceFromResult(result); + const protocolConfigObj: ProtocolConfig = { protocol: new Protocol(Protocol.HTTP), - parameters: datasource.protocol.parameters - } - let format = new Format(AdapterEndpoint.getFormat(datasource.format.type)) - let formatConfigObj: FormatConfig = {format: format, parameters: datasource.format.parameters} - let adapterConfig: AdapterConfig = {protocolConfig: protocolConfigObj, formatConfig: formatConfigObj} - let returnDataImportResponse = await AdapterService.getInstance().executeJob(adapterConfig); + parameters: datasource.protocol.parameters, + }; + const format = new Format( + AdapterEndpoint.getFormat(datasource.format.type), + ); + const formatConfigObj: FormatConfig = { + format: format, + parameters: datasource.format.parameters, + }; + const adapterConfig: AdapterConfig = { + protocolConfig: protocolConfigObj, + formatConfig: formatConfigObj, + }; + const returnDataImportResponse = + await AdapterService.getInstance().executeJob(adapterConfig); // //TODO save response in dataimport table - //TODO check correct response + // TODO check correct response res.status(200).send(returnDataImportResponse); }; - - } - - - diff --git a/adapter/src/datasource/model/datasourceModelForAmqp.ts b/adapter/src/datasource/model/datasourceModelForAmqp.ts index 4a5d05ebe..39a2b066b 100644 --- a/adapter/src/datasource/model/datasourceModelForAmqp.ts +++ b/adapter/src/datasource/model/datasourceModelForAmqp.ts @@ -1,3 +1,3 @@ export interface DatasourceModelForAmqp { - datasource: any + datasource: any; } diff --git a/adapter/src/datasource/model/outboxEvent.ts b/adapter/src/datasource/model/outboxEvent.ts index 4f94a2d49..5997885ce 100644 --- a/adapter/src/datasource/model/outboxEvent.ts +++ b/adapter/src/datasource/model/outboxEvent.ts @@ -1,5 +1,5 @@ export interface OutboxEvent { - id:any, - routing_key:string, - payload:any + id: any; + routing_key: string; + payload: any; } diff --git a/adapter/src/datasource/repository/dataImportRepository.ts b/adapter/src/datasource/repository/dataImportRepository.ts index 8631766f6..016c79f97 100644 --- a/adapter/src/datasource/repository/dataImportRepository.ts +++ b/adapter/src/datasource/repository/dataImportRepository.ts @@ -6,16 +6,15 @@ const knex = require('knex')({ user: 'adapterservice', password: 'admin', database: 'adapterservice', - asyncStackTraces: true - } + asyncStackTraces: true, + }, }); export class DataImportRepository { - async getMetaDataImportByDatasource(datasourceId: string) { - return await knex + return await knex .select('id', 'timestamp', 'health', 'error_messages') .from('public.data_import') - .where('datasource_id', datasourceId) + .where('datasource_id', datasourceId); } async getLatestMetaDataImportByDatasourceId(id: string) { @@ -27,11 +26,11 @@ export class DataImportRepository { } async getLatestDataImportByDatasourceId(id: string) { - return await knex + return await knex .select('data') .from('public.data_import') .where('datasource_id', id) - .orderBy('timestamp', 'desc') + .orderBy('timestamp', 'desc'); } async getMetadataForDataImport(datasourceId: string, dataImportId: string) { @@ -39,14 +38,14 @@ export class DataImportRepository { .select('id', 'timestamp', 'health', 'error_messages') .from('public.data_import') .where('datasource_id', datasourceId) - .andWhere('id', dataImportId) + .andWhere('id', dataImportId); } async getDataFromDataImport(datasourceId: string, dataImportId: string) { - return await knex + return await knex .select('data') .from('public.data_import') .where('datasource_id', datasourceId) - .andWhere('id', dataImportId) + .andWhere('id', dataImportId); } } diff --git a/adapter/src/datasource/repository/datasourceRepository.ts b/adapter/src/datasource/repository/datasourceRepository.ts index be3d28b7c..46d32aea4 100644 --- a/adapter/src/datasource/repository/datasourceRepository.ts +++ b/adapter/src/datasource/repository/datasourceRepository.ts @@ -1,5 +1,6 @@ -import {KnexHelper} from "./knexHelper"; -import {DatasourceInsertStatement} from "../model/DatasourceInsertStatement"; +import { DatasourceInsertStatement } from '../model/DatasourceInsertStatement'; + +import { KnexHelper } from './knexHelper'; const knex = require('knex')({ client: 'pg', @@ -9,24 +10,17 @@ const knex = require('knex')({ user: 'adapterservice', password: 'admin', database: 'adapterservice', - asyncStackTraces: true - } + asyncStackTraces: true, + }, }); export class DatasourceRepository { - - async getAllDataSources() { - return await knex - .select() - .from('public.datasource') + return await knex.select().from('public.datasource'); } async getDataSourceById(id: any) { - return await knex - .select() - .from('public.datasource') - .where('id', id); + return await knex.select().from('public.datasource').where('id', id); } async addDatasource(insertStatement: DatasourceInsertStatement) { @@ -34,22 +28,25 @@ export class DatasourceRepository { .insert(insertStatement) .returning('id') .then(function (id: any) { - console.log(id) - console.log("neuer code geht") + console.log(id); + console.log('neuer code geht'); return knex .select() .from('public.datasource') .where('id', id[0].id) .then(function (result: any) { return KnexHelper.createDatasourceFromResult(result); - }) + }); }) .catch(function (err: any) { - console.log(err) - }) + console.log(err); + }); } - async updateDatasource(insertStatement: DatasourceInsertStatement, datasourceId: string) { + async updateDatasource( + insertStatement: DatasourceInsertStatement, + datasourceId: string, + ) { return await knex('public.datasource') .where('id', datasourceId) .update(insertStatement) @@ -59,13 +56,13 @@ export class DatasourceRepository { .from('public.datasource') .where('id', datasourceId) .then(function (result: any) { - console.log(result) + console.log(result); return KnexHelper.createDatasourceFromResult(result); - }) + }); }) .catch(function (err: any) { - console.log(err) - }) + console.log(err); + }); } async deleteDatasourceById(datasourceId: string) { @@ -79,8 +76,6 @@ export class DatasourceRepository { return await knex .delete() .from('public.datasource') - .where('id', '!=', "-1") + .where('id', '!=', '-1'); } - - } diff --git a/adapter/src/datasource/repository/knexHelper.ts b/adapter/src/datasource/repository/knexHelper.ts index 39d39055e..21da416b6 100644 --- a/adapter/src/datasource/repository/knexHelper.ts +++ b/adapter/src/datasource/repository/knexHelper.ts @@ -1,80 +1,75 @@ -import {DatasourceInsertStatement} from "../model/DatasourceInsertStatement"; - +import { DatasourceInsertStatement } from '../model/DatasourceInsertStatement'; export class KnexHelper { static createDatasourceFromResultArray(result: any) { - var test = []; - for (var i in result) { - var el = result[i]; - let protocolParameters = JSON.parse(el.protocol_parameters); - let formatParameters = JSON.parse(el.format_parameters); - let x = { - "protocol": { - "type": el.protocol_type, - "parameters": - protocolParameters - + const test = []; + for (const i in result) { + const el = result[i]; + const protocolParameters = JSON.parse(el.protocol_parameters); + const formatParameters = JSON.parse(el.format_parameters); + const x = { + protocol: { + type: el.protocol_type, + parameters: protocolParameters, }, - "format": { - "type": el.format_type, - "parameters": formatParameters + format: { + type: el.format_type, + parameters: formatParameters, }, - "metadata": { - "author": el.author, - "license": el.license, - "displayName": el.display_name, - "description": el.description, - "creationTimestamp": el.creation_timestamp + metadata: { + author: el.author, + license: el.license, + displayName: el.display_name, + description: el.description, + creationTimestamp: el.creation_timestamp, }, - "trigger": { - "periodic": el.periodic, - "firstExecution": el.first_execution, - "interval": el.interval + trigger: { + periodic: el.periodic, + firstExecution: el.first_execution, + interval: el.interval, }, - "schema": el.schema, - "id": el.id - } + schema: el.schema, + id: el.id, + }; console.log(x); - test.push(x) + test.push(x); } - console.log("durch") - console.log(test) - + console.log('durch'); + console.log(test); return test; } static createDatasourceFromResult(result: any) { - let protocolParameters = JSON.parse(result[0].protocol_parameters); - let formatParameters = JSON.parse(result[0].format_parameters); - let x = { - "protocol": { - "type": result[0].protocol_type, - "parameters": protocolParameters + const protocolParameters = JSON.parse(result[0].protocol_parameters); + const formatParameters = JSON.parse(result[0].format_parameters); + const x = { + protocol: { + type: result[0].protocol_type, + parameters: protocolParameters, }, - "format": { - "type": result[0].format_type, - "parameters": formatParameters + format: { + type: result[0].format_type, + parameters: formatParameters, }, - "metadata": { - "author": result[0].author, - "license": result[0].license, - "displayName": result[0].display_name, - "description": result[0].description, - "creationTimestamp": result[0].creation_timestamp + metadata: { + author: result[0].author, + license: result[0].license, + displayName: result[0].display_name, + description: result[0].description, + creationTimestamp: result[0].creation_timestamp, }, - "trigger": { - "periodic": result[0].periodic, - "firstExecution": result[0].first_execution, - "interval": result[0].interval + trigger: { + periodic: result[0].periodic, + firstExecution: result[0].first_execution, + interval: result[0].interval, }, - "schema": result[0].schema, - "id": result[0].id - } + schema: result[0].schema, + id: result[0].id, + }; console.log(x); - return x; } @@ -91,30 +86,32 @@ export class KnexHelper { protocol_type: req.body.protocol.type, first_execution: req.body.trigger.firstExecution, interval: req.body.trigger.interval, - periodic: req.body.trigger.periodic + periodic: req.body.trigger.periodic, }; } - //from: https://weblog.rogueamoeba.com/2017/02/27/javascript-correctly-converting-a-byte-array-to-a-utf-8-string/ + // From: https://weblog.rogueamoeba.com/2017/02/27/javascript-correctly-converting-a-byte-array-to-a-utf-8-string/ static stringFromUTF8Array(data: any) { const extraByteMap = [1, 1, 1, 1, 2, 2, 3, 0]; - var count = data.length; - var str = ""; + const count = data.length; + let str = ''; - for (var index = 0; index < count;) { - var ch = data[index++]; + for (let index = 0; index < count; ) { + let ch = data[index++]; if (ch & 0x80) { - var extra = extraByteMap[(ch >> 3) & 0x07]; - if (!(ch & 0x40) || !extra || ((index + extra) > count)) + let extra = extraByteMap[(ch >> 3) & 0x07]; + if (!(ch & 0x40) || !extra || index + extra > count) { return null; + } - ch = ch & (0x3F >> extra); + ch = ch & (0x3f >> extra); for (; extra > 0; extra -= 1) { - var chx = data[index++]; - if ((chx & 0xC0) != 0x80) + const chx = data[index++]; + if ((chx & 0xc0) != 0x80) { return null; + } - ch = (ch << 6) | (chx & 0x3F); + ch = (ch << 6) | (chx & 0x3f); } } diff --git a/adapter/src/datasource/repository/outboxRepository.ts b/adapter/src/datasource/repository/outboxRepository.ts index 6dbe0a7e1..f9cb8b52b 100644 --- a/adapter/src/datasource/repository/outboxRepository.ts +++ b/adapter/src/datasource/repository/outboxRepository.ts @@ -1,7 +1,8 @@ -import {OutboxEvent} from "../model/outboxEvent"; -import {v4 as uuidv4} from "uuid"; -import {KnexHelper} from "./knexHelper"; +import { v4 as uuidv4 } from 'uuid'; +import { OutboxEvent } from '../model/outboxEvent'; + +import { KnexHelper } from './knexHelper'; const knex = require('knex')({ client: 'pg', @@ -11,28 +12,27 @@ const knex = require('knex')({ user: 'adapterservice', password: 'admin', database: 'adapterservice', - asyncStackTraces: true - } + asyncStackTraces: true, + }, }); export class OutboxRepository { - async publishToOutbox(payload: any, routingKey: string) { - let id =uuidv4(); - let outboxEvent: OutboxEvent = { - id:id, + const id = uuidv4(); + const outboxEvent: OutboxEvent = { + id: id, payload: payload, - routing_key: routingKey - } + routing_key: routingKey, + }; return await knex('public.outbox') .insert(outboxEvent) .returning('id') .then(function (id: any) { - console.log(id) - console.log("neuer code geht") + console.log(id); + console.log('neuer code geht'); }) .catch(function (err: any) { - console.log(err) - }) + console.log(err); + }); } } From 7b956e514e3f874fdadd628f49d342fffc1b5270 Mon Sep 17 00:00:00 2001 From: = <=> Date: Wed, 20 Apr 2022 15:34:53 +0200 Subject: [PATCH 06/40] linter with @PVahldiek --- adapter/package-lock.json | 19 +++++++++++++++++++ adapter/package.json | 1 + adapter/src/adapter/importer/Importer.ts | 2 -- .../src/adapter/interpreter/CsvInterpreter.ts | 4 ++-- adapter/src/adapter/model/ProtocolConfig.ts | 6 +----- .../src/adapter/services/adapterService.ts | 2 -- 6 files changed, 23 insertions(+), 11 deletions(-) diff --git a/adapter/package-lock.json b/adapter/package-lock.json index 2aa6c2fd6..f6f14486d 100644 --- a/adapter/package-lock.json +++ b/adapter/package-lock.json @@ -31,6 +31,7 @@ "@types/node": "^17.0.22", "@types/supertest": "^2.0.11", "@types/uuid": "^8.3.4", + "@types/xml2js": "^0.4.11", "@typescript-eslint/eslint-plugin": "^4.30.0", "@typescript-eslint/parser": "^4.30.0", "eslint": "^7.31.0", @@ -1696,6 +1697,15 @@ "integrity": "sha512-c/I8ZRb51j+pYGAu5CrFMRxqZ2ke4y2grEBO5AUjgSkSk+qT2Ea+OdWElz/OiMf5MNpn2b17kuVBwZLQJXzihw==", "dev": true }, + "node_modules/@types/xml2js": { + "version": "0.4.11", + "resolved": "https://registry.npmjs.org/@types/xml2js/-/xml2js-0.4.11.tgz", + "integrity": "sha512-JdigeAKmCyoJUiQljjr7tQG3if9NkqGUgwEUqBvV0N7LM4HyQk7UXCnusRa1lnvXAEYJ8mw8GtZWioagNztOwA==", + "dev": true, + "dependencies": { + "@types/node": "*" + } + }, "node_modules/@types/yargs": { "version": "16.0.4", "resolved": "https://registry.npmjs.org/@types/yargs/-/yargs-16.0.4.tgz", @@ -10604,6 +10614,15 @@ "integrity": "sha512-c/I8ZRb51j+pYGAu5CrFMRxqZ2ke4y2grEBO5AUjgSkSk+qT2Ea+OdWElz/OiMf5MNpn2b17kuVBwZLQJXzihw==", "dev": true }, + "@types/xml2js": { + "version": "0.4.11", + "resolved": "https://registry.npmjs.org/@types/xml2js/-/xml2js-0.4.11.tgz", + "integrity": "sha512-JdigeAKmCyoJUiQljjr7tQG3if9NkqGUgwEUqBvV0N7LM4HyQk7UXCnusRa1lnvXAEYJ8mw8GtZWioagNztOwA==", + "dev": true, + "requires": { + "@types/node": "*" + } + }, "@types/yargs": { "version": "16.0.4", "resolved": "https://registry.npmjs.org/@types/yargs/-/yargs-16.0.4.tgz", diff --git a/adapter/package.json b/adapter/package.json index 15cf6dc06..4c4ef8dbb 100644 --- a/adapter/package.json +++ b/adapter/package.json @@ -37,6 +37,7 @@ "@types/node": "^17.0.22", "@types/supertest": "^2.0.11", "@types/uuid": "^8.3.4", + "@types/xml2js": "^0.4.11", "@typescript-eslint/eslint-plugin": "^4.30.0", "@typescript-eslint/parser": "^4.30.0", "eslint": "^7.31.0", diff --git a/adapter/src/adapter/importer/Importer.ts b/adapter/src/adapter/importer/Importer.ts index d07a1d246..770437760 100644 --- a/adapter/src/adapter/importer/Importer.ts +++ b/adapter/src/adapter/importer/Importer.ts @@ -1,5 +1,3 @@ -import { generateKeySync } from 'crypto'; - import { ImporterParameterError } from '../model/exceptions/ImporterParameterError'; import { ImporterParameterDescription } from './ImporterParameterDescription'; diff --git a/adapter/src/adapter/interpreter/CsvInterpreter.ts b/adapter/src/adapter/interpreter/CsvInterpreter.ts index 4422cb265..a0024deee 100644 --- a/adapter/src/adapter/interpreter/CsvInterpreter.ts +++ b/adapter/src/adapter/interpreter/CsvInterpreter.ts @@ -63,8 +63,8 @@ export class CsvInterpreter extends Interpreter { // Todo need to test if this works if ( skipFirstDataRow && - ((index == 0 && !firstRowAsHeader) || - (index == 1 && firstRowAsHeader)) + ((index === 0 && !firstRowAsHeader) || + (index === 1 && firstRowAsHeader)) ) { // Skip First Row } else { diff --git a/adapter/src/adapter/model/ProtocolConfig.ts b/adapter/src/adapter/model/ProtocolConfig.ts index e7f9ad477..382ad949f 100644 --- a/adapter/src/adapter/model/ProtocolConfig.ts +++ b/adapter/src/adapter/model/ProtocolConfig.ts @@ -1,10 +1,6 @@ -import { stringify } from 'querystring'; - import { validators } from '@jvalue/node-dry-basics'; -import { JsonAlias, JsonClassType, JsonProperty } from 'jackson-js'; -import { AdapterConfig } from './AdapterConfig'; -import { AdapterConfigDTO, ProtocolConfigDTO } from './EndpointDTOs'; +import { AdapterConfigDTO } from './EndpointDTOs'; import { Protocol } from './enum/Protocol'; export interface ProtocolConfig { diff --git a/adapter/src/adapter/services/adapterService.ts b/adapter/src/adapter/services/adapterService.ts index ae336c4e0..d9c442c82 100644 --- a/adapter/src/adapter/services/adapterService.ts +++ b/adapter/src/adapter/services/adapterService.ts @@ -1,5 +1,3 @@ -import { JsonRawValue } from 'jackson-js'; - import { Importer } from '../importer/Importer'; import { Interpreter } from '../interpreter/Interpreter'; import { AdapterConfig } from '../model/AdapterConfig'; From 21241405eded86aa836827fd7b2f0587f5dbd364 Mon Sep 17 00:00:00 2001 From: = <=> Date: Wed, 20 Apr 2022 16:16:54 +0200 Subject: [PATCH 07/40] csv interpreter fixed with @PVahdliek --- adapter/src/adapter/importer/HttpImporter.ts | 7 +++---- adapter/src/adapter/interpreter/CsvInterpreter.ts | 14 +++++++++++--- 2 files changed, 14 insertions(+), 7 deletions(-) diff --git a/adapter/src/adapter/importer/HttpImporter.ts b/adapter/src/adapter/importer/HttpImporter.ts index 2b508cc99..2e0bbe1a7 100644 --- a/adapter/src/adapter/importer/HttpImporter.ts +++ b/adapter/src/adapter/importer/HttpImporter.ts @@ -68,9 +68,8 @@ export class HttpImporter extends Importer { } override async doFetch(parameters: Record): Promise { - const uri = parameters.location; - console.log(parameters); - const encoding = parameters.encoding; + const uri = parameters.location as string; + const encoding = parameters.encoding as string; // TODO see if encoding from response is good return axios({ method: 'get', @@ -79,7 +78,7 @@ export class HttpImporter extends Importer { }) .then(function (response: any) { console.log(response.data); - return response.data; + return response.data as string; }) .catch(function (error: any) { console.error(error); diff --git a/adapter/src/adapter/interpreter/CsvInterpreter.ts b/adapter/src/adapter/interpreter/CsvInterpreter.ts index a0024deee..c3ade380d 100644 --- a/adapter/src/adapter/interpreter/CsvInterpreter.ts +++ b/adapter/src/adapter/interpreter/CsvInterpreter.ts @@ -40,7 +40,7 @@ export class CsvInterpreter extends Interpreter { return this.parameters; } - override doInterpret( + override async doInterpret( data: string, parameters: Record, ): Promise { @@ -51,8 +51,12 @@ export class CsvInterpreter extends Interpreter { const firstRowAsHeader: boolean = parameters.firstRowAsHeader as boolean; // True = With header, False = WithoutHeader const skipFirstDataRow: boolean = parameters.skipFirstDataRow as boolean; + /*return new Promise(function (resolve) { + resolve(data); + });*/ + const json: any[] = []; - return csv({ + const cssv = await csv({ noheader: !firstRowAsHeader, // Be Careful: Need to Invert the boolean here output: 'json', delimiter: columnSeparator, @@ -71,11 +75,15 @@ export class CsvInterpreter extends Interpreter { json.push(csvRow); } }) - .on('done', (error: any) => { + /*.on('done', (error: any) => { return new Promise(function (resolve, reject) { resolve(JSON.stringify(json)); }); + });*/ + return new Promise(function (resolve, reject) { + resolve(JSON.stringify(json)); }); + } override validateParameters(inputParameters: Record): void { From ffaa2fae4acbb246322d7d8566ff22516bbc8877 Mon Sep 17 00:00:00 2001 From: = <=> Date: Wed, 20 Apr 2022 16:33:47 +0200 Subject: [PATCH 08/40] linter with @PVahldiek --- adapter/src/adapter/importer/HttpImporter.ts | 20 ++++++---------- .../src/adapter/interpreter/CsvInterpreter.ts | 24 ++++++------------- .../src/adapter/interpreter/XmlInterpreter.ts | 6 ++--- adapter/src/index.ts | 1 - 4 files changed, 17 insertions(+), 34 deletions(-) diff --git a/adapter/src/adapter/importer/HttpImporter.ts b/adapter/src/adapter/importer/HttpImporter.ts index 2e0bbe1a7..d07a1d6f8 100644 --- a/adapter/src/adapter/importer/HttpImporter.ts +++ b/adapter/src/adapter/importer/HttpImporter.ts @@ -71,18 +71,12 @@ export class HttpImporter extends Importer { const uri = parameters.location as string; const encoding = parameters.encoding as string; // TODO see if encoding from response is good - return axios({ - method: 'get', - url: uri, - responseEncoding: encoding, - }) - .then(function (response: any) { - console.log(response.data); - return response.data as string; - }) - .catch(function (error: any) { - console.error(error); - throw new ImporterParameterError('Could not Fetch from URI:' + uri); - }); + try { + const result = await axios.get(uri, { responseEncoding: encoding }); + return result.data as string; + } catch (e) { + console.error(e); + throw new ImporterParameterError('Could not Fetch from URI:' + uri); + } } } diff --git a/adapter/src/adapter/interpreter/CsvInterpreter.ts b/adapter/src/adapter/interpreter/CsvInterpreter.ts index c3ade380d..7648709d2 100644 --- a/adapter/src/adapter/interpreter/CsvInterpreter.ts +++ b/adapter/src/adapter/interpreter/CsvInterpreter.ts @@ -44,26 +44,22 @@ export class CsvInterpreter extends Interpreter { data: string, parameters: Record, ): Promise { - data = 'col1,col2,col3\n' + 'val11,val12,val13\n' + 'val21,val22,val23'; + // Data = 'col1,col2,col3\n' + 'val11,val12,val13\n' + 'val21,val22,val23'; const columnSeparator = (parameters.columnSeparator as string).charAt(0); const lineSeparator: string = parameters.lineSeparator as string; const firstRowAsHeader: boolean = parameters.firstRowAsHeader as boolean; // True = With header, False = WithoutHeader const skipFirstDataRow: boolean = parameters.skipFirstDataRow as boolean; - /*return new Promise(function (resolve) { - resolve(data); - });*/ - - const json: any[] = []; - const cssv = await csv({ + const json: string[] = []; + await csv({ noheader: !firstRowAsHeader, // Be Careful: Need to Invert the boolean here output: 'json', delimiter: columnSeparator, eol: lineSeparator, }) .fromString(data) - .subscribe((csvRow: any, index: any) => { + .subscribe((csvRow: string, index: number) => { // Todo need to test if this works if ( skipFirstDataRow && @@ -74,16 +70,10 @@ export class CsvInterpreter extends Interpreter { } else { json.push(csvRow); } - }) - /*.on('done', (error: any) => { - return new Promise(function (resolve, reject) { - resolve(JSON.stringify(json)); - }); - });*/ - return new Promise(function (resolve, reject) { - resolve(JSON.stringify(json)); }); - + return new Promise(function (resolve) { + resolve(JSON.stringify(json)); + }); } override validateParameters(inputParameters: Record): void { diff --git a/adapter/src/adapter/interpreter/XmlInterpreter.ts b/adapter/src/adapter/interpreter/XmlInterpreter.ts index af46dae29..53d5b2ea4 100644 --- a/adapter/src/adapter/interpreter/XmlInterpreter.ts +++ b/adapter/src/adapter/interpreter/XmlInterpreter.ts @@ -27,14 +27,14 @@ export class XmlInterpreter extends Interpreter { ): Promise { const parser = new xml2js.Parser({ explicitArray: false }); + /* Const result: ????? = await parser.parseStringPromise(data); + return result.root;*/ + return parser .parseStringPromise(data) .then(function (result: any) { // `result` is a JavaScript object - // Convert it to a JSON string return result.root; - // Const json = JSON.stringify(result.root); - // Return json; }) .catch(function (err: any) { throw err; diff --git a/adapter/src/index.ts b/adapter/src/index.ts index a0cad1bed..127670ee0 100644 --- a/adapter/src/index.ts +++ b/adapter/src/index.ts @@ -9,7 +9,6 @@ import { DataImportEndpoint } from './datasource/api/rest/dataImportEndpoint'; import { DataSourceEndpoint } from './datasource/api/rest/dataSourceEndpoint'; export const port = 8080; -const API_VERSION = '0.0.1'; export let server: Server | undefined; // Await will be needed in the future, so for now ignore this linter issue and remove the disable later From 27c74d50958ea49cb44bf4b1587193e4a95f9b5e Mon Sep 17 00:00:00 2001 From: = <=> Date: Wed, 20 Apr 2022 23:34:32 +0200 Subject: [PATCH 09/40] linter + xmlinterpreter with @PVahldiek --- .../src/adapter/api/rest/adapterEndpoint.ts | 16 +++------ .../src/adapter/interpreter/XmlInterpreter.ts | 35 +++++++++++-------- adapter/src/adapter/model/EndpointDTOs.ts | 2 -- 3 files changed, 26 insertions(+), 27 deletions(-) diff --git a/adapter/src/adapter/api/rest/adapterEndpoint.ts b/adapter/src/adapter/api/rest/adapterEndpoint.ts index 5fd798b1e..48198f24b 100644 --- a/adapter/src/adapter/api/rest/adapterEndpoint.ts +++ b/adapter/src/adapter/api/rest/adapterEndpoint.ts @@ -107,13 +107,10 @@ export class AdapterEndpoint { }; /* - Returns Collection of Importer + Returns Collection of Interpreter } */ - handleGetFormat = async ( - req: express.Request, - res: express.Response, - ): Promise => { + handleGetFormat = (req: express.Request, res: express.Response): void => { const interpreters = AdapterService.getInstance().getAllFormats(); res.setHeader('Content-Type', 'application/json'); res.status(200).json(interpreters); @@ -122,10 +119,7 @@ export class AdapterEndpoint { /* Returns Collection of Importer */ - handleGetProtocols = async ( - req: express.Request, - res: express.Response, - ): Promise => { + handleGetProtocols = (req: express.Request, res: express.Response): void => { try { const protocols = AdapterService.getInstance().getAllProtocols(); res.status(200).json(protocols); @@ -134,10 +128,10 @@ export class AdapterEndpoint { } }; - handleGetApplicationVersion = async ( + handleGetApplicationVersion = ( req: express.Request, res: express.Response, - ): Promise => { + ): void => { res.setHeader('Content-Type', 'text/plain'); res.status(200).send(APP_VERSION); }; diff --git a/adapter/src/adapter/interpreter/XmlInterpreter.ts b/adapter/src/adapter/interpreter/XmlInterpreter.ts index 53d5b2ea4..f5d23c1d3 100644 --- a/adapter/src/adapter/interpreter/XmlInterpreter.ts +++ b/adapter/src/adapter/interpreter/XmlInterpreter.ts @@ -25,19 +25,26 @@ export class XmlInterpreter extends Interpreter { data: string, parameters: Record, ): Promise { - const parser = new xml2js.Parser({ explicitArray: false }); - - /* Const result: ????? = await parser.parseStringPromise(data); - return result.root;*/ - - return parser - .parseStringPromise(data) - .then(function (result: any) { - // `result` is a JavaScript object - return result.root; - }) - .catch(function (err: any) { - throw err; - }); + data = + '' + + 'RickMorty' + + 'RickMorty'; + + const result = this.parseXmlToJson(data); + return new Promise(function (resolve) { + resolve(JSON.stringify(result)); + }); + } + + parseXmlToJson(xml: string): Record { + const json: Record = {}; + for (const res of xml.matchAll( + /(?:<(\w*)(?:\s[^>]*)*>)((?:(?!<\1).)*)(?:<\/\1>)|<(\w*)(?:\s*)*\/>/gm, + )) { + const key: string = res[1] || res[3]; + const value = res[2] && this.parseXmlToJson(res[2]); + json[key] = (value && Object.keys(value).length ? value : res[2]) || null; + } + return json; } } diff --git a/adapter/src/adapter/model/EndpointDTOs.ts b/adapter/src/adapter/model/EndpointDTOs.ts index a646c180d..f3a2fb23b 100644 --- a/adapter/src/adapter/model/EndpointDTOs.ts +++ b/adapter/src/adapter/model/EndpointDTOs.ts @@ -2,12 +2,10 @@ export interface AdapterConfigDTO { protocol: ProtocolConfigDTO; format: FormatConfigDTO; } - export interface ProtocolConfigDTO { type: string; parameters: Record; } - export interface FormatConfigDTO { type: string; parameters: Record; From 5f8f58bf764dfafb347cdc05f44a7aa426e335bb Mon Sep 17 00:00:00 2001 From: = <=> Date: Thu, 21 Apr 2022 00:22:51 +0200 Subject: [PATCH 10/40] xmlInterpreter with new function for converting xml to record --- adapter/src/adapter/interpreter/XmlInterpreter.ts | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/adapter/src/adapter/interpreter/XmlInterpreter.ts b/adapter/src/adapter/interpreter/XmlInterpreter.ts index f5d23c1d3..f487d99c0 100644 --- a/adapter/src/adapter/interpreter/XmlInterpreter.ts +++ b/adapter/src/adapter/interpreter/XmlInterpreter.ts @@ -27,12 +27,19 @@ export class XmlInterpreter extends Interpreter { ): Promise { data = '' + - 'RickMorty' + - 'RickMorty'; + 'RickMorty' + + 'RickMorty'; const result = this.parseXmlToJson(data); + const updatedResult: Record = {}; + const resultKey: Record = result[ + Object.keys(result)[0] + ] as Record; + for (const [key, value] of Object.entries(resultKey)) { + updatedResult[key] = value; + } return new Promise(function (resolve) { - resolve(JSON.stringify(result)); + resolve(updatedResult); }); } From c23f3b0b6fb91e228c467c9cf4b328b60cb62308 Mon Sep 17 00:00:00 2001 From: = <=> Date: Thu, 21 Apr 2022 00:30:24 +0200 Subject: [PATCH 11/40] xml interpreter with @PVahldiek --- adapter/src/adapter/interpreter/XmlInterpreter.ts | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/adapter/src/adapter/interpreter/XmlInterpreter.ts b/adapter/src/adapter/interpreter/XmlInterpreter.ts index f487d99c0..c53dcf1d3 100644 --- a/adapter/src/adapter/interpreter/XmlInterpreter.ts +++ b/adapter/src/adapter/interpreter/XmlInterpreter.ts @@ -39,7 +39,8 @@ export class XmlInterpreter extends Interpreter { updatedResult[key] = value; } return new Promise(function (resolve) { - resolve(updatedResult); + // TODO we can not stringify here -> makes \ in the json result + resolve(JSON.stringify(updatedResult)); }); } From 40874de8061c8b48b31233d4f178337a6970a05d Mon Sep 17 00:00:00 2001 From: Pascal Vahldiek Date: Thu, 21 Apr 2022 01:52:58 +0200 Subject: [PATCH 12/40] re-changed to json.parse for json.stringify decoding with @MarcoDoell --- adapter/integration-test/src/adapter-stateless.test.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/adapter/integration-test/src/adapter-stateless.test.js b/adapter/integration-test/src/adapter-stateless.test.js index c4fdc9b13..f54c369b4 100644 --- a/adapter/integration-test/src/adapter-stateless.test.js +++ b/adapter/integration-test/src/adapter-stateless.test.js @@ -110,7 +110,7 @@ describe('Stateless data import', () => { .send(reqBody) expect(response.status).toEqual(200) const importedData = response.body.data - expect(importedData).toEqual({ from: 'Rick', to: 'Morty' }) + expect(JSON.parse(importedData)).toEqual({ from: 'Rick', to: 'Morty' }) }, TIMEOUT) test('Should import and format csv data', async () => { @@ -148,7 +148,7 @@ describe('Stateless data import', () => { col3: 'val23' }] - expect(importedData).toEqual(expected) + expect(JSON.parse(importedData)).toEqual(expected) }, TIMEOUT) test('Should return 400 BAD_REQUEST for unsupported protocol [POST /preview]', async () => { From df395f61eb89285174ce8af3d99193b58d6d88bc Mon Sep 17 00:00:00 2001 From: = <=> Date: Thu, 21 Apr 2022 12:53:31 +0200 Subject: [PATCH 13/40] additional test in stateless with @PVahldiek --- .../src/adapter-stateless.test.js | 525 ++++++++++-------- adapter/integration-test/src/mock.server.js | 96 ++-- .../src/adapter/interpreter/XmlInterpreter.ts | 6 +- 3 files changed, 356 insertions(+), 271 deletions(-) diff --git a/adapter/integration-test/src/adapter-stateless.test.js b/adapter/integration-test/src/adapter-stateless.test.js index f54c369b4..f64efb43c 100644 --- a/adapter/integration-test/src/adapter-stateless.test.js +++ b/adapter/integration-test/src/adapter-stateless.test.js @@ -1,229 +1,306 @@ -const { execPath } = require('process') -const request = require('supertest') +const { execPath } = require('process'); +const request = require('supertest'); -const { - ADAPTER_URL, - MOCK_SERVER_URL -} = require('./env') -const { waitForServicesToBeReady } = require('./waitForServices') +const { ADAPTER_URL, MOCK_SERVER_URL } = require('./env'); +const { waitForServicesToBeReady } = require('./waitForServices'); -const TIMEOUT = 10000 +const TIMEOUT = 10000; describe('Stateless data import', () => { beforeAll(async () => { - await waitForServicesToBeReady() - }, 60000) - - test('Should respond with semantic version [GET /version]', async () => { - const response = await request(ADAPTER_URL).get('/version') - expect(response.status).toEqual(200) - expect(response.type).toEqual('text/plain') - - const semanticVersionRegEx = '^(0|[1-9]\\d*)\\.(0|[1-9]\\d*)\\.(0|[1-9]\\d*)' - expect(response.text).toMatch(new RegExp(semanticVersionRegEx)) - }, TIMEOUT) - - test('Should respond with all available formats [GET /formats]', async () => { - const response = await request(ADAPTER_URL).get('/formats') - expect(response.status).toEqual(200) - expect(response.type).toEqual('application/json') - expect(response.body.length).toBeGreaterThanOrEqual(2) - - response.body.forEach(e => { - expect(e.type).toBeDefined() - expect(e.parameters).toBeDefined() - }) - }, TIMEOUT) - - test('Should respond with all available protocols [GET /protocols]', async () => { - const response = await request(ADAPTER_URL).get('/protocols') - expect(response.status).toEqual(200) - expect(response.type).toEqual('application/json') - expect(response.body.length).toBeGreaterThanOrEqual(1) - - response.body.forEach(e => { - expect(e.type).toBeDefined() - expect(e.parameters).toBeDefined() - }) - }, TIMEOUT) - - test('Should import json data', async () => { - const reqBody = { - protocol: { - type: 'HTTP', - parameters: { - location: MOCK_SERVER_URL + '/json', - encoding: 'UTF-8' - } - }, - format: { - type: 'JSON' - } - } - - const response = await request(ADAPTER_URL) - .post('/preview') - .send(reqBody) - expect(response.status).toEqual(200) - const importedData = response.body.data - expect(importedData).toEqual({ whateverwillbe: 'willbe', quesera: 'sera' }) - }, TIMEOUT) - - test('Should import raw xml data', async () => { - const reqBody = { - protocol: { - type: 'HTTP', - parameters: { - location: MOCK_SERVER_URL + '/xml', - encoding: 'UTF-8' - } - } - } - - const response = await request(ADAPTER_URL) - .post('/preview/raw') - .send(reqBody) - - expect(response.status).toEqual(200) - expect(response.body.data).toEqual( - '' + - 'RickMorty' - ) - }, TIMEOUT) - - test('Should import and format xml data', async () => { - const reqBody = { - protocol: { - type: 'HTTP', - parameters: { - location: MOCK_SERVER_URL + '/xml', - encoding: 'UTF-8' - } - }, - format: { - type: 'XML' - } - } - - const response = await request(ADAPTER_URL) - .post('/preview') - .send(reqBody) - expect(response.status).toEqual(200) - const importedData = response.body.data - expect(JSON.parse(importedData)).toEqual({ from: 'Rick', to: 'Morty' }) - }, TIMEOUT) - - test('Should import and format csv data', async () => { - const reqBody = { - protocol: { - type: 'HTTP', - parameters: { - location: MOCK_SERVER_URL + '/csv', - encoding: 'UTF-8' - } - }, - format: { - type: 'CSV', - parameters: { - columnSeparator: ',', - lineSeparator: '\n', - skipFirstDataRow: false, - firstRowAsHeader: true - } - } - } - const response = await request(ADAPTER_URL) - .post('/preview') - .send(reqBody) - expect(response.status).toEqual(200) - const importedData = response.body.data - const expected = [ - { - col1: 'val11', - col2: 'val12', - col3: 'val13' - }, { - col1: 'val21', - col2: 'val22', - col3: 'val23' - }] - - expect(JSON.parse(importedData)).toEqual(expected) - }, TIMEOUT) - - test('Should return 400 BAD_REQUEST for unsupported protocol [POST /preview]', async () => { - const reqBody = { - protocol: { - type: 'UNSUPPORTED', - parameters: { - location: MOCK_SERVER_URL + '/json', - encoding: 'UTF-8' - } - }, - format: { - type: 'JSON' - } - } - const response = await request(ADAPTER_URL) - .post('/preview') - .send(reqBody) - expect(response.status).toEqual(400) - }, TIMEOUT) - - test('Should return 400 BAD_REQUEST for unsupported format [POST /preview]', async () => { - const reqBody = { - protocol: { - type: 'HTTP', - parameters: { - location: MOCK_SERVER_URL + '/json', - encoding: 'UTF-8' - } - }, - format: { - type: 'UNSUPPORTED' - } - } - const response = await request(ADAPTER_URL) - .post('/preview') - .send(reqBody) - expect(response.status).toEqual(400) - }, TIMEOUT) - - test('Should return 400 BAD_REQUEST for invalid location [POST /preview]', async () => { - const reqBody = { - protocol: { - type: 'HTTP', - parameters: { - location: 'invalid-location', - encoding: 'UTF-8' - } - }, - format: { - type: 'JSON' - } - } - const response = await request(ADAPTER_URL) - .post('/preview') - .send(reqBody) - expect(response.status).toEqual(400) - }, TIMEOUT) - - test('Should return 500 INTERNAL_SERVER_ERROR for data not found [POST /preview]', async () => { - const reqBody = { - protocol: { - type: 'HTTP', - parameters: { - location: MOCK_SERVER_URL + '/not-found', - encoding: 'UTF-8' - } - }, - format: { - type: 'JSON' - } - } - const response = await request(ADAPTER_URL) - .post('/preview') - .send(reqBody) - expect(response.status).toEqual(500) - }, TIMEOUT) -}) + await waitForServicesToBeReady(); + }, 60000); + + test( + 'Should respond with semantic version [GET /version]', + async () => { + const response = await request(ADAPTER_URL).get('/version'); + expect(response.status).toEqual(200); + expect(response.type).toEqual('text/plain'); + + const semanticVersionRegEx = + '^(0|[1-9]\\d*)\\.(0|[1-9]\\d*)\\.(0|[1-9]\\d*)'; + expect(response.text).toMatch(new RegExp(semanticVersionRegEx)); + }, + TIMEOUT, + ); + + test( + 'Should respond with all available formats [GET /formats]', + async () => { + const response = await request(ADAPTER_URL).get('/formats'); + expect(response.status).toEqual(200); + expect(response.type).toEqual('application/json'); + expect(response.body.length).toBeGreaterThanOrEqual(2); + + response.body.forEach((e) => { + expect(e.type).toBeDefined(); + expect(e.parameters).toBeDefined(); + }); + }, + TIMEOUT, + ); + + test( + 'Should respond with all available protocols [GET /protocols]', + async () => { + const response = await request(ADAPTER_URL).get('/protocols'); + expect(response.status).toEqual(200); + expect(response.type).toEqual('application/json'); + expect(response.body.length).toBeGreaterThanOrEqual(1); + + response.body.forEach((e) => { + expect(e.type).toBeDefined(); + expect(e.parameters).toBeDefined(); + }); + }, + TIMEOUT, + ); + + test( + 'Should import json data', + async () => { + const reqBody = { + protocol: { + type: 'HTTP', + parameters: { + location: MOCK_SERVER_URL + '/json', + encoding: 'UTF-8', + }, + }, + format: { + type: 'JSON', + }, + }; + + const response = await request(ADAPTER_URL) + .post('/preview') + .send(reqBody); + expect(response.status).toEqual(200); + const importedData = response.body.data; + expect(importedData).toEqual({ + whateverwillbe: 'willbe', + quesera: 'sera', + }); + }, + TIMEOUT, + ); + + test( + 'Should import raw xml data', + async () => { + const reqBody = { + protocol: { + type: 'HTTP', + parameters: { + location: MOCK_SERVER_URL + '/xml', + encoding: 'UTF-8', + }, + }, + }; + + const response = await request(ADAPTER_URL) + .post('/preview/raw') + .send(reqBody); + + expect(response.status).toEqual(200); + expect(response.body.data).toEqual( + '' + + 'RickMorty', + ); + }, + TIMEOUT, + ); + + test( + 'Should import and format xml data', + async () => { + const reqBody = { + protocol: { + type: 'HTTP', + parameters: { + location: MOCK_SERVER_URL + '/xml', + encoding: 'UTF-8', + }, + }, + format: { + type: 'XML', + }, + }; + + const response = await request(ADAPTER_URL) + .post('/preview') + .send(reqBody); + expect(response.status).toEqual(200); + const importedData = response.body.data; + expect(JSON.parse(importedData)).toEqual({ from: 'Rick', to: 'Morty' }); + }, + TIMEOUT, + ); + + test( + 'Should import and format more xml data', + async () => { + const reqBody = { + protocol: { + type: 'HTTP', + parameters: { + location: MOCK_SERVER_URL + '/xmlbigger', + encoding: 'UTF-8', + }, + }, + format: { + type: 'XML', + }, + }; + + const response = await request(ADAPTER_URL) + .post('/preview') + .send(reqBody); + expect(response.status).toEqual(200); + const importedData = response.body.data; + expect(JSON.parse(importedData)).toEqual({ + from: 'Rick', + to: 'Morty', + test: { hello: 'hello', servus: 'servus' }, + }); + }, + TIMEOUT, + ); + + test( + 'Should import and format csv data', + async () => { + const reqBody = { + protocol: { + type: 'HTTP', + parameters: { + location: MOCK_SERVER_URL + '/csv', + encoding: 'UTF-8', + }, + }, + format: { + type: 'CSV', + parameters: { + columnSeparator: ',', + lineSeparator: '\n', + skipFirstDataRow: false, + firstRowAsHeader: true, + }, + }, + }; + const response = await request(ADAPTER_URL) + .post('/preview') + .send(reqBody); + expect(response.status).toEqual(200); + const importedData = response.body.data; + const expected = [ + { + col1: 'val11', + col2: 'val12', + col3: 'val13', + }, + { + col1: 'val21', + col2: 'val22', + col3: 'val23', + }, + ]; + + expect(JSON.parse(importedData)).toEqual(expected); + }, + TIMEOUT, + ); + + test( + 'Should return 400 BAD_REQUEST for unsupported protocol [POST /preview]', + async () => { + const reqBody = { + protocol: { + type: 'UNSUPPORTED', + parameters: { + location: MOCK_SERVER_URL + '/json', + encoding: 'UTF-8', + }, + }, + format: { + type: 'JSON', + }, + }; + const response = await request(ADAPTER_URL) + .post('/preview') + .send(reqBody); + expect(response.status).toEqual(400); + }, + TIMEOUT, + ); + + test( + 'Should return 400 BAD_REQUEST for unsupported format [POST /preview]', + async () => { + const reqBody = { + protocol: { + type: 'HTTP', + parameters: { + location: MOCK_SERVER_URL + '/json', + encoding: 'UTF-8', + }, + }, + format: { + type: 'UNSUPPORTED', + }, + }; + const response = await request(ADAPTER_URL) + .post('/preview') + .send(reqBody); + expect(response.status).toEqual(400); + }, + TIMEOUT, + ); + + test( + 'Should return 400 BAD_REQUEST for invalid location [POST /preview]', + async () => { + const reqBody = { + protocol: { + type: 'HTTP', + parameters: { + location: 'invalid-location', + encoding: 'UTF-8', + }, + }, + format: { + type: 'JSON', + }, + }; + const response = await request(ADAPTER_URL) + .post('/preview') + .send(reqBody); + expect(response.status).toEqual(400); + }, + TIMEOUT, + ); + + test( + 'Should return 500 INTERNAL_SERVER_ERROR for data not found [POST /preview]', + async () => { + const reqBody = { + protocol: { + type: 'HTTP', + parameters: { + location: MOCK_SERVER_URL + '/not-found', + encoding: 'UTF-8', + }, + }, + format: { + type: 'JSON', + }, + }; + const response = await request(ADAPTER_URL) + .post('/preview') + .send(reqBody); + expect(response.status).toEqual(500); + }, + TIMEOUT, + ); +}); diff --git a/adapter/integration-test/src/mock.server.js b/adapter/integration-test/src/mock.server.js index 8207cae1a..92acf11ab 100644 --- a/adapter/integration-test/src/mock.server.js +++ b/adapter/integration-test/src/mock.server.js @@ -1,64 +1,74 @@ -const Koa = require('koa') -const Router = require('koa-router') -const router = new Router() +const Koa = require('koa'); +const Router = require('koa-router'); +const router = new Router(); -const app = new Koa() +const app = new Koa(); -const { MOCK_SERVER_PORT } = require('./env') +const { MOCK_SERVER_PORT } = require('./env'); -router.get('/', async ctx => { - ctx.type = 'text/plain' - ctx.body = 'ok' -}) +router.get('/', async (ctx) => { + ctx.type = 'text/plain'; + ctx.body = 'ok'; +}); -router.get('/not-found', async ctx => { - ctx.type = 'text/plain' - ctx.status = 404 - ctx.body = '404 NOT FOUND Error' -}) +router.get('/not-found', async (ctx) => { + ctx.type = 'text/plain'; + ctx.status = 404; + ctx.body = '404 NOT FOUND Error'; +}); -router.get('/json', async ctx => { - console.log('GET /json') - ctx.body = { whateverwillbe: 'willbe', quesera: 'sera' } -}) +router.get('/json', async (ctx) => { + console.log('GET /json'); + ctx.body = { whateverwillbe: 'willbe', quesera: 'sera' }; +}); -router.get('/json/:id', async ctx => { - console.log('Get /json/' + ctx.params.id) - ctx.body = { id: ctx.params.id } -}) +router.get('/json/:id', async (ctx) => { + console.log('Get /json/' + ctx.params.id); + ctx.body = { id: ctx.params.id }; +}); -router.get('/xml', async ctx => { - console.log('GET /xml') +router.get('/xml', async (ctx) => { + console.log('GET /xml'); - ctx.type = 'text/xml' + ctx.type = 'text/xml'; ctx.body = '' + - 'RickMorty' -}) + 'RickMorty'; +}); -router.get('/csv', async ctx => { - console.log('GET /CSV') +router.get('/xmlbigger', async (ctx) => { + console.log('GET /xml'); - ctx.type = 'text/csv' + ctx.type = 'text/xml'; ctx.body = - 'col1,col2,col3\n' + - 'val11,val12,val13\n' + - 'val21,val22,val23' -}) + '' + + 'RickMorty' + + 'helloservus' + + ''; +}); + +router.get('/csv', async (ctx) => { + console.log('GET /CSV'); + + ctx.type = 'text/csv'; + ctx.body = 'col1,col2,col3\n' + 'val11,val12,val13\n' + 'val21,val22,val23'; +}); -app.use(router.routes()) +app.use(router.routes()); -const server = app.listen(MOCK_SERVER_PORT, () => console.log('Starting mock server on port ' + MOCK_SERVER_PORT)) +const server = app.listen(MOCK_SERVER_PORT, () => + console.log('Starting mock server on port ' + MOCK_SERVER_PORT), +); process.on('SIGTERM', async () => { - console.info('Mock-Server: SIGTERM signal received.') + console.info('Mock-Server: SIGTERM signal received.'); try { - await server.close() + await server.close(); } catch (e) { - console.error('Could not shutdown server') - console.error(e) - process.exit(-1) + console.error('Could not shutdown server'); + console.error(e); + process.exit(-1); } -}) +}); -module.exports = server +module.exports = server; diff --git a/adapter/src/adapter/interpreter/XmlInterpreter.ts b/adapter/src/adapter/interpreter/XmlInterpreter.ts index c53dcf1d3..c60068ee8 100644 --- a/adapter/src/adapter/interpreter/XmlInterpreter.ts +++ b/adapter/src/adapter/interpreter/XmlInterpreter.ts @@ -20,15 +20,14 @@ export class XmlInterpreter extends Interpreter { return this.parameters; } - // TODO @Georg check if this package can be used.. override doInterpret( data: string, parameters: Record, ): Promise { - data = + /* Data = '' + 'RickMorty' + - 'RickMorty'; + 'RickMorty';*/ const result = this.parseXmlToJson(data); const updatedResult: Record = {}; @@ -39,7 +38,6 @@ export class XmlInterpreter extends Interpreter { updatedResult[key] = value; } return new Promise(function (resolve) { - // TODO we can not stringify here -> makes \ in the json result resolve(JSON.stringify(updatedResult)); }); } From ecd62bb93ed85497517895dea9e2cef580ef82c4 Mon Sep 17 00:00:00 2001 From: = <=> Date: Thu, 21 Apr 2022 13:22:05 +0200 Subject: [PATCH 14/40] =?UTF-8?q?unterscheidung=20internal=20server=20erro?= =?UTF-8?q?r=20und=20bad=20request=20f=C3=BCr=20httpimporter=20with=20@PVa?= =?UTF-8?q?hldiek?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../src/adapter/api/rest/adapterEndpoint.ts | 28 +++++++++++++------ adapter/src/adapter/importer/HttpImporter.ts | 20 +++++++------ .../InterpreterParameterDescription.ts | 2 -- 3 files changed, 31 insertions(+), 19 deletions(-) diff --git a/adapter/src/adapter/api/rest/adapterEndpoint.ts b/adapter/src/adapter/api/rest/adapterEndpoint.ts index 48198f24b..b2fb81a19 100644 --- a/adapter/src/adapter/api/rest/adapterEndpoint.ts +++ b/adapter/src/adapter/api/rest/adapterEndpoint.ts @@ -73,19 +73,19 @@ export class AdapterEndpoint { }; console.log(adapterConfig); - let returnDataImportResponse = null; try { - returnDataImportResponse = await AdapterService.getInstance().executeJob( - adapterConfig, - ); + const returnDataImportResponse = + await AdapterService.getInstance().executeJob(adapterConfig); + res.status(200).send(returnDataImportResponse); } catch (e) { if (e instanceof ImporterParameterError) { res.status(400).send(e.message); return; } + if (e instanceof Error) { + res.status(500).send(e.message); + } } - - res.status(200).send(returnDataImportResponse); }; handleExecuteRawPreview = async ( @@ -101,9 +101,19 @@ export class AdapterEndpoint { protocol: new Protocol(Protocol.HTTP), parameters: req.body.protocol.parameters, }; - const returnDataImportResponse = - await AdapterService.getInstance().executeRawJob(protocolConfigObj); - res.status(200).send(returnDataImportResponse); + try { + const returnDataImportResponse = + await AdapterService.getInstance().executeRawJob(protocolConfigObj); + res.status(200).send(returnDataImportResponse); + } catch (e) { + if (e instanceof ImporterParameterError) { + res.status(400).send(e.message); + return; + } + if (e instanceof Error) { + res.status(500).send(e.message); + } + } }; /* diff --git a/adapter/src/adapter/importer/HttpImporter.ts b/adapter/src/adapter/importer/HttpImporter.ts index d07a1d6f8..86a395910 100644 --- a/adapter/src/adapter/importer/HttpImporter.ts +++ b/adapter/src/adapter/importer/HttpImporter.ts @@ -1,4 +1,4 @@ -import axios from 'axios'; +import axios, { Axios, AxiosError } from 'axios'; import { ImporterParameterError } from '../model/exceptions/ImporterParameterError'; @@ -71,12 +71,16 @@ export class HttpImporter extends Importer { const uri = parameters.location as string; const encoding = parameters.encoding as string; // TODO see if encoding from response is good - try { - const result = await axios.get(uri, { responseEncoding: encoding }); - return result.data as string; - } catch (e) { - console.error(e); - throw new ImporterParameterError('Could not Fetch from URI:' + uri); - } + + const result = await axios + .get(uri, { responseEncoding: encoding }) + .catch((error: AxiosError) => { + if (error.response) { + console.log(error.response); + throw new Error('Could not Fetch from URI:' + uri); + } + throw new ImporterParameterError('Could not Fetch from URI:' + uri); + }); + return result.data as string; } } diff --git a/adapter/src/adapter/interpreter/InterpreterParameterDescription.ts b/adapter/src/adapter/interpreter/InterpreterParameterDescription.ts index badda2955..fac5eaff0 100644 --- a/adapter/src/adapter/interpreter/InterpreterParameterDescription.ts +++ b/adapter/src/adapter/interpreter/InterpreterParameterDescription.ts @@ -1,9 +1,7 @@ export class InterpreterParameterDescription { name: string; description: string; - // TODO default value? is not used in Interpreters required: boolean | undefined; - // TODO @Georg: need to check how to store the class in typescript, can use instance.constructor.name to get the name probably -> is stored as string -> Same as in Importer type: unknown; constructor(name: string, description: string, type: unknown) { From a4e33c089a694fc2e7d3317e0c12b28462dff9bb Mon Sep 17 00:00:00 2001 From: = <=> Date: Thu, 21 Apr 2022 13:28:09 +0200 Subject: [PATCH 15/40] convertXmlToJson new function with @PVahldiek --- adapter/src/adapter/importer/HttpImporter.ts | 2 +- .../src/adapter/interpreter/XmlInterpreter.ts | 19 +++++++++---------- 2 files changed, 10 insertions(+), 11 deletions(-) diff --git a/adapter/src/adapter/importer/HttpImporter.ts b/adapter/src/adapter/importer/HttpImporter.ts index 86a395910..429c0d6ab 100644 --- a/adapter/src/adapter/importer/HttpImporter.ts +++ b/adapter/src/adapter/importer/HttpImporter.ts @@ -1,4 +1,4 @@ -import axios, { Axios, AxiosError } from 'axios'; +import axios, { AxiosError } from 'axios'; import { ImporterParameterError } from '../model/exceptions/ImporterParameterError'; diff --git a/adapter/src/adapter/interpreter/XmlInterpreter.ts b/adapter/src/adapter/interpreter/XmlInterpreter.ts index c60068ee8..245ce11fa 100644 --- a/adapter/src/adapter/interpreter/XmlInterpreter.ts +++ b/adapter/src/adapter/interpreter/XmlInterpreter.ts @@ -1,5 +1,3 @@ -import xml2js from 'xml2js'; - import { Interpreter } from './Interpreter'; import { InterpreterParameterDescription } from './InterpreterParameterDescription'; @@ -29,7 +27,7 @@ export class XmlInterpreter extends Interpreter { 'RickMorty' + 'RickMorty';*/ - const result = this.parseXmlToJson(data); + const result = this.convertXmlToJson(data); const updatedResult: Record = {}; const resultKey: Record = result[ Object.keys(result)[0] @@ -42,15 +40,16 @@ export class XmlInterpreter extends Interpreter { }); } - parseXmlToJson(xml: string): Record { - const json: Record = {}; - for (const res of xml.matchAll( + convertXmlToJson(inputData: string): Record { + const returnJson: Record = {}; + for (const result of inputData.matchAll( /(?:<(\w*)(?:\s[^>]*)*>)((?:(?!<\1).)*)(?:<\/\1>)|<(\w*)(?:\s*)*\/>/gm, )) { - const key: string = res[1] || res[3]; - const value = res[2] && this.parseXmlToJson(res[2]); - json[key] = (value && Object.keys(value).length ? value : res[2]) || null; + const key: string = result[1] || result[3]; + const value = result[2] && this.convertXmlToJson(result[2]); + returnJson[key] = + (value && Object.keys(value).length ? value : result[2]) || null; } - return json; + return returnJson; } } From 4041b36221ec30f3dda0aa55fc12119161764c5f Mon Sep 17 00:00:00 2001 From: = <=> Date: Thu, 21 Apr 2022 13:44:43 +0200 Subject: [PATCH 16/40] unit tests for adapter Endpoint with @PVahldiek --- .../src/adapter/api/rest/adapterEndpoint.ts | 5 +- .../src/adapter/services/adapterService.ts | 1 - adapter/src/adapterEndpoint.spec.ts | 48 +++++++++++++++++++ adapter/src/dummy.spec.ts | 9 ---- 4 files changed, 50 insertions(+), 13 deletions(-) create mode 100644 adapter/src/adapterEndpoint.spec.ts delete mode 100644 adapter/src/dummy.spec.ts diff --git a/adapter/src/adapter/api/rest/adapterEndpoint.ts b/adapter/src/adapter/api/rest/adapterEndpoint.ts index b2fb81a19..a6e55bad1 100644 --- a/adapter/src/adapter/api/rest/adapterEndpoint.ts +++ b/adapter/src/adapter/api/rest/adapterEndpoint.ts @@ -60,7 +60,6 @@ export class AdapterEndpoint { return; } - // Check location (???) const format = new Format(formatType); const formatConfigObj: FormatConfig = { format: format, @@ -158,7 +157,7 @@ export class AdapterEndpoint { return Format.XML; } default: { - throw new Error('asdasd'); + throw new Error('Format not found'); } } } @@ -169,7 +168,7 @@ export class AdapterEndpoint { return Protocol.HTTP; } default: { - throw new Error('asdasd'); + throw new Error('Protocol not found'); } } } diff --git a/adapter/src/adapter/services/adapterService.ts b/adapter/src/adapter/services/adapterService.ts index d9c442c82..eda25a384 100644 --- a/adapter/src/adapter/services/adapterService.ts +++ b/adapter/src/adapter/services/adapterService.ts @@ -21,7 +21,6 @@ export class AdapterService { return AdapterService.instance; } - // To Implement getAllFormats(): Array { return [Format.CSV, Format.JSON, Format.XML]; } diff --git a/adapter/src/adapterEndpoint.spec.ts b/adapter/src/adapterEndpoint.spec.ts new file mode 100644 index 000000000..9251d469e --- /dev/null +++ b/adapter/src/adapterEndpoint.spec.ts @@ -0,0 +1,48 @@ +import { AdapterEndpoint } from './adapter/api/rest/adapterEndpoint'; +import { Format } from './adapter/model/enum/Format'; +import { Protocol } from './adapter/model/enum/Protocol'; + +/* eslint-env jest */ +describe('getFormatShouldReturnErrorWhenNotFound', () => { + test('getFormat test', () => { + expect(() => { + AdapterEndpoint.getFormat('not here'); + }).toThrow(Error); + }); +}); + +describe('getFormatShouldReturnCSV', () => { + test('getFormat test', () => { + const result = AdapterEndpoint.getFormat('CSV'); + expect(result).toBe(Format.CSV); + }); +}); + +describe('getFormatShouldReturnXML', () => { + test('getFormat test', () => { + const result = AdapterEndpoint.getFormat('XML'); + expect(result).toBe(Format.XML); + }); +}); + +describe('getFormatShouldReturnJSON', () => { + test('getFormat test', () => { + const result = AdapterEndpoint.getFormat('JSON'); + expect(result).toBe(Format.JSON); + }); +}); + +describe('getProtocolShouldReturnHTTP', () => { + test('getProtocol test', () => { + const result = AdapterEndpoint.getProtocol('HTTP'); + expect(result).toBe(Protocol.HTTP); + }); +}); + +describe('getProtocolShouldReturnError', () => { + test('getProtocol test', () => { + expect(() => { + AdapterEndpoint.getProtocol('not here'); + }).toThrow(Error); + }); +}); diff --git a/adapter/src/dummy.spec.ts b/adapter/src/dummy.spec.ts deleted file mode 100644 index 1d0194a8e..000000000 --- a/adapter/src/dummy.spec.ts +++ /dev/null @@ -1,9 +0,0 @@ -/* eslint-env jest */ -describe('dummy', () => { - test('dummy test', () => { - console.log( - 'Dummy test - please remove after first real test implemented!', - ); - expect(true).toBeTruthy(); - }); -}); From 509ba96252b3bff9ba6cfde08ecd6abe96eabc6e Mon Sep 17 00:00:00 2001 From: = <=> Date: Thu, 21 Apr 2022 13:54:23 +0200 Subject: [PATCH 17/40] JEST Unit Tests for JsonInterpreter with @Pvahldiek --- adapter/src/JsonInterpreter.spec.ts | 13 +++++++++++++ adapter/src/XmlInterpreter.spec.ts | 13 +++++++++++++ 2 files changed, 26 insertions(+) create mode 100644 adapter/src/JsonInterpreter.spec.ts create mode 100644 adapter/src/XmlInterpreter.spec.ts diff --git a/adapter/src/JsonInterpreter.spec.ts b/adapter/src/JsonInterpreter.spec.ts new file mode 100644 index 000000000..3b8068dbe --- /dev/null +++ b/adapter/src/JsonInterpreter.spec.ts @@ -0,0 +1,13 @@ +import { Format } from './adapter/model/enum/Format'; + +describe('doInterpretJsonFormatReturnsValidJSON', () => { + test('getFormat test', () => { + const jsonFormat = Format.JSON; + const data = + '"{"uuid":"47174d8f-1b8e-4599-8a59-b580dd55bc87","number":"48900237","shortname":"EITZE","longname":"EITZE","km":9.56,"agency":"VERDEN",' + + '"longitude":9.276769435375872,"latitude":52.90406544743417,"water":{"shortname":"ALLER","longname":"ALLER"}},"'; + return jsonFormat.doInterpret(data, {}).then((res) => { + expect(res).toBe(data); + }); + }); +}); diff --git a/adapter/src/XmlInterpreter.spec.ts b/adapter/src/XmlInterpreter.spec.ts new file mode 100644 index 000000000..3b8068dbe --- /dev/null +++ b/adapter/src/XmlInterpreter.spec.ts @@ -0,0 +1,13 @@ +import { Format } from './adapter/model/enum/Format'; + +describe('doInterpretJsonFormatReturnsValidJSON', () => { + test('getFormat test', () => { + const jsonFormat = Format.JSON; + const data = + '"{"uuid":"47174d8f-1b8e-4599-8a59-b580dd55bc87","number":"48900237","shortname":"EITZE","longname":"EITZE","km":9.56,"agency":"VERDEN",' + + '"longitude":9.276769435375872,"latitude":52.90406544743417,"water":{"shortname":"ALLER","longname":"ALLER"}},"'; + return jsonFormat.doInterpret(data, {}).then((res) => { + expect(res).toBe(data); + }); + }); +}); From d92f434861897c96a02786bf09a3210b2a883d03 Mon Sep 17 00:00:00 2001 From: = <=> Date: Thu, 21 Apr 2022 13:55:18 +0200 Subject: [PATCH 18/40] added csv interpreter jest file with @PVahldiek --- adapter/src/CsvInterpreter.spec.ts | 13 +++++++++++++ 1 file changed, 13 insertions(+) create mode 100644 adapter/src/CsvInterpreter.spec.ts diff --git a/adapter/src/CsvInterpreter.spec.ts b/adapter/src/CsvInterpreter.spec.ts new file mode 100644 index 000000000..3b8068dbe --- /dev/null +++ b/adapter/src/CsvInterpreter.spec.ts @@ -0,0 +1,13 @@ +import { Format } from './adapter/model/enum/Format'; + +describe('doInterpretJsonFormatReturnsValidJSON', () => { + test('getFormat test', () => { + const jsonFormat = Format.JSON; + const data = + '"{"uuid":"47174d8f-1b8e-4599-8a59-b580dd55bc87","number":"48900237","shortname":"EITZE","longname":"EITZE","km":9.56,"agency":"VERDEN",' + + '"longitude":9.276769435375872,"latitude":52.90406544743417,"water":{"shortname":"ALLER","longname":"ALLER"}},"'; + return jsonFormat.doInterpret(data, {}).then((res) => { + expect(res).toBe(data); + }); + }); +}); From 96c269c7ea2c0f80627175cff87a085df55893f5 Mon Sep 17 00:00:00 2001 From: = <=> Date: Thu, 21 Apr 2022 14:02:18 +0200 Subject: [PATCH 19/40] test push --- adapter/src/adapter/interpreter/XmlInterpreter.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/adapter/src/adapter/interpreter/XmlInterpreter.ts b/adapter/src/adapter/interpreter/XmlInterpreter.ts index 245ce11fa..a8d43e02b 100644 --- a/adapter/src/adapter/interpreter/XmlInterpreter.ts +++ b/adapter/src/adapter/interpreter/XmlInterpreter.ts @@ -25,7 +25,7 @@ export class XmlInterpreter extends Interpreter { /* Data = '' + 'RickMorty' + - 'RickMorty';*/ + 'RickMorty';s*/ const result = this.convertXmlToJson(data); const updatedResult: Record = {}; From 421f34161414d3ff00c48f18156f6ae6ebf6d097 Mon Sep 17 00:00:00 2001 From: Pascal Vahldiek Date: Thu, 21 Apr 2022 14:35:26 +0200 Subject: [PATCH 20/40] jest unit test for xmlinterpreter with @MarcoDoell --- adapter/src/XmlInterpreter.spec.ts | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/adapter/src/XmlInterpreter.spec.ts b/adapter/src/XmlInterpreter.spec.ts index 3b8068dbe..8073c7508 100644 --- a/adapter/src/XmlInterpreter.spec.ts +++ b/adapter/src/XmlInterpreter.spec.ts @@ -1,13 +1,17 @@ import { Format } from './adapter/model/enum/Format'; -describe('doInterpretJsonFormatReturnsValidJSON', () => { +describe('doInterpretXMLFormatReturnsValidJSON', () => { test('getFormat test', () => { - const jsonFormat = Format.JSON; + const xmlFormat = Format.XML; const data = - '"{"uuid":"47174d8f-1b8e-4599-8a59-b580dd55bc87","number":"48900237","shortname":"EITZE","longname":"EITZE","km":9.56,"agency":"VERDEN",' + - '"longitude":9.276769435375872,"latitude":52.90406544743417,"water":{"shortname":"ALLER","longname":"ALLER"}},"'; - return jsonFormat.doInterpret(data, {}).then((res) => { - expect(res).toBe(data); + 'ToveJaniReminderDon\'t forget me this weekend!'; + return xmlFormat.doInterpret(data, {}).then((res) => { + expect(JSON.parse(res)).toEqual({ + to: 'Tove', + from: 'Jani', + heading: 'Reminder', + body: "Don't forget me this weekend!", + }); }); }); }); From e4192e2bc399c497fa0e07b68d9f5d5ff89af69f Mon Sep 17 00:00:00 2001 From: Pascal Vahldiek Date: Thu, 21 Apr 2022 15:03:43 +0200 Subject: [PATCH 21/40] jest tests formatted and xmlinterpreter unit tests with @MarcoDoell --- adapter/src/CsvInterpreter.spec.ts | 4 ++-- adapter/src/JsonInterpreter.spec.ts | 2 +- adapter/src/XmlInterpreter.spec.ts | 15 ++++++++++++++- adapter/src/adapterEndpoint.spec.ts | 28 ++++++++-------------------- 4 files changed, 25 insertions(+), 24 deletions(-) diff --git a/adapter/src/CsvInterpreter.spec.ts b/adapter/src/CsvInterpreter.spec.ts index 3b8068dbe..d14be72aa 100644 --- a/adapter/src/CsvInterpreter.spec.ts +++ b/adapter/src/CsvInterpreter.spec.ts @@ -1,7 +1,7 @@ import { Format } from './adapter/model/enum/Format'; -describe('doInterpretJsonFormatReturnsValidJSON', () => { - test('getFormat test', () => { +describe('doInterpretCSVFormatReturnsValidJSON', () => { + test('convert standard CSV to JSON', () => { const jsonFormat = Format.JSON; const data = '"{"uuid":"47174d8f-1b8e-4599-8a59-b580dd55bc87","number":"48900237","shortname":"EITZE","longname":"EITZE","km":9.56,"agency":"VERDEN",' + diff --git a/adapter/src/JsonInterpreter.spec.ts b/adapter/src/JsonInterpreter.spec.ts index 3b8068dbe..f7b5c8424 100644 --- a/adapter/src/JsonInterpreter.spec.ts +++ b/adapter/src/JsonInterpreter.spec.ts @@ -1,7 +1,7 @@ import { Format } from './adapter/model/enum/Format'; describe('doInterpretJsonFormatReturnsValidJSON', () => { - test('getFormat test', () => { + test('doInterpret JSON data test', () => { const jsonFormat = Format.JSON; const data = '"{"uuid":"47174d8f-1b8e-4599-8a59-b580dd55bc87","number":"48900237","shortname":"EITZE","longname":"EITZE","km":9.56,"agency":"VERDEN",' + diff --git a/adapter/src/XmlInterpreter.spec.ts b/adapter/src/XmlInterpreter.spec.ts index 8073c7508..e937c7a87 100644 --- a/adapter/src/XmlInterpreter.spec.ts +++ b/adapter/src/XmlInterpreter.spec.ts @@ -1,7 +1,7 @@ import { Format } from './adapter/model/enum/Format'; describe('doInterpretXMLFormatReturnsValidJSON', () => { - test('getFormat test', () => { + test('convert standard XML to JSON test', () => { const xmlFormat = Format.XML; const data = 'ToveJaniReminderDon\'t forget me this weekend!'; @@ -14,4 +14,17 @@ describe('doInterpretXMLFormatReturnsValidJSON', () => { }); }); }); + test('convert nested XML to JSON test', () => { + const xmlFormat = Format.XML; + const data = + 'ToveJaniReminderSubheadingReminderDon\'t forget me this weekend!'; + return xmlFormat.doInterpret(data, {}).then((res) => { + expect(JSON.parse(res)).toEqual({ + to: 'Tove', + from: 'Jani', + heading: { subheading: 'ReminderSubheading', Reminder: 'Reminder' }, + body: "Don't forget me this weekend!", + }); + }); + }); }); diff --git a/adapter/src/adapterEndpoint.spec.ts b/adapter/src/adapterEndpoint.spec.ts index 9251d469e..96d87a704 100644 --- a/adapter/src/adapterEndpoint.spec.ts +++ b/adapter/src/adapterEndpoint.spec.ts @@ -3,44 +3,32 @@ import { Format } from './adapter/model/enum/Format'; import { Protocol } from './adapter/model/enum/Protocol'; /* eslint-env jest */ -describe('getFormatShouldReturnErrorWhenNotFound', () => { - test('getFormat test', () => { +describe('getFormatShouldReturnCorrectResult', () => { + test('getFormat test throws exception for not existing format', () => { expect(() => { AdapterEndpoint.getFormat('not here'); }).toThrow(Error); }); -}); - -describe('getFormatShouldReturnCSV', () => { - test('getFormat test', () => { + test('getFormat test for CSV', () => { const result = AdapterEndpoint.getFormat('CSV'); expect(result).toBe(Format.CSV); }); -}); - -describe('getFormatShouldReturnXML', () => { - test('getFormat test', () => { + test('getFormat test for XML', () => { const result = AdapterEndpoint.getFormat('XML'); expect(result).toBe(Format.XML); }); -}); - -describe('getFormatShouldReturnJSON', () => { - test('getFormat test', () => { + test('getFormat test for JSON', () => { const result = AdapterEndpoint.getFormat('JSON'); expect(result).toBe(Format.JSON); }); }); -describe('getProtocolShouldReturnHTTP', () => { - test('getProtocol test', () => { +describe('getProtocolShouldReturnCorrectResult', () => { + test('getProtocol test for HTTP', () => { const result = AdapterEndpoint.getProtocol('HTTP'); expect(result).toBe(Protocol.HTTP); }); -}); - -describe('getProtocolShouldReturnError', () => { - test('getProtocol test', () => { + test('getProtocol test throws exception for not existing Protocol', () => { expect(() => { AdapterEndpoint.getProtocol('not here'); }).toThrow(Error); From c06d305c4ca5294b1959c7aaa75fb5afbcc102b4 Mon Sep 17 00:00:00 2001 From: Pascal Vahldiek Date: Thu, 21 Apr 2022 15:33:54 +0200 Subject: [PATCH 22/40] CSV Interpreter Unit test with @MarcoDoell --- adapter/src/CsvInterpreter.spec.ts | 47 +++++++++++++++++++++++++----- 1 file changed, 40 insertions(+), 7 deletions(-) diff --git a/adapter/src/CsvInterpreter.spec.ts b/adapter/src/CsvInterpreter.spec.ts index d14be72aa..773957253 100644 --- a/adapter/src/CsvInterpreter.spec.ts +++ b/adapter/src/CsvInterpreter.spec.ts @@ -1,13 +1,46 @@ import { Format } from './adapter/model/enum/Format'; describe('doInterpretCSVFormatReturnsValidJSON', () => { - test('convert standard CSV to JSON', () => { - const jsonFormat = Format.JSON; + test('convert standard CSV to JSON', async () => { + const csvFormat = Format.CSV; const data = - '"{"uuid":"47174d8f-1b8e-4599-8a59-b580dd55bc87","number":"48900237","shortname":"EITZE","longname":"EITZE","km":9.56,"agency":"VERDEN",' + - '"longitude":9.276769435375872,"latitude":52.90406544743417,"water":{"shortname":"ALLER","longname":"ALLER"}},"'; - return jsonFormat.doInterpret(data, {}).then((res) => { - expect(res).toBe(data); - }); + 'id,first_name,last_name,email,gender,ip_address\n' + + '1,Ewell,Mathwin,emathwin0@hibu.com,Male,226.172.125.251\n' + + '2,Fayth,Blampy,fblampy1@hubpages.com,Female,212.76.208.25\n' + + '3,Kelli,Cornock,kcornock2@boston.com,Female,171.5.66.30\n'; + const parameters = { + columnSeparator: ',', + lineSeparator: '\n', + skipFirstDataRow: false, + firstRowAsHeader: true, + }; + const expected = [ + { + id: '1', + first_name: 'Ewell', + last_name: 'Mathwin', + email: 'emathwin0@hibu.com', + gender: 'Male', + ip_address: '226.172.125.251', + }, + { + id: '2', + first_name: 'Fayth', + last_name: 'Blampy', + email: 'fblampy1@hubpages.com', + gender: 'Female', + ip_address: '212.76.208.25', + }, + { + id: '3', + first_name: 'Kelli', + last_name: 'Cornock', + email: 'kcornock2@boston.com', + gender: 'Female', + ip_address: '171.5.66.30', + }, + ]; + const res = await csvFormat.doInterpret(data, parameters); + expect(JSON.parse(res)).toEqual(expected); }); }); From e750733dc7fd163891bb31588f8a0a1d018e0955 Mon Sep 17 00:00:00 2001 From: = <=> Date: Fri, 22 Apr 2022 11:40:33 +0200 Subject: [PATCH 23/40] linter finished for adapter with @PVahldiek --- adapter/src/adapter/interpreter/CsvInterpreter.ts | 2 -- adapter/src/adapter/interpreter/Interpreter.ts | 2 +- adapter/src/adapter/interpreter/JsonInterpreter.ts | 1 + adapter/src/adapter/interpreter/XmlInterpreter.ts | 8 ++------ 4 files changed, 4 insertions(+), 9 deletions(-) diff --git a/adapter/src/adapter/interpreter/CsvInterpreter.ts b/adapter/src/adapter/interpreter/CsvInterpreter.ts index 7648709d2..80814ead0 100644 --- a/adapter/src/adapter/interpreter/CsvInterpreter.ts +++ b/adapter/src/adapter/interpreter/CsvInterpreter.ts @@ -44,8 +44,6 @@ export class CsvInterpreter extends Interpreter { data: string, parameters: Record, ): Promise { - // Data = 'col1,col2,col3\n' + 'val11,val12,val13\n' + 'val21,val22,val23'; - const columnSeparator = (parameters.columnSeparator as string).charAt(0); const lineSeparator: string = parameters.lineSeparator as string; const firstRowAsHeader: boolean = parameters.firstRowAsHeader as boolean; // True = With header, False = WithoutHeader diff --git a/adapter/src/adapter/interpreter/Interpreter.ts b/adapter/src/adapter/interpreter/Interpreter.ts index e8e21097d..3975c5040 100644 --- a/adapter/src/adapter/interpreter/Interpreter.ts +++ b/adapter/src/adapter/interpreter/Interpreter.ts @@ -18,7 +18,7 @@ export abstract class Interpreter { abstract getDescription(): string; abstract doInterpret( data: string, - parameters: Record, + parameters?: Record, ): Promise; abstract getAvailableParameters(): Array; diff --git a/adapter/src/adapter/interpreter/JsonInterpreter.ts b/adapter/src/adapter/interpreter/JsonInterpreter.ts index 833238d4f..e2025704b 100644 --- a/adapter/src/adapter/interpreter/JsonInterpreter.ts +++ b/adapter/src/adapter/interpreter/JsonInterpreter.ts @@ -20,6 +20,7 @@ export class JsonInterpreter extends Interpreter { override doInterpret( data: string, + // eslint-disable-next-line @typescript-eslint/no-unused-vars _parameters: Record, ): Promise { return new Promise(function (resolve) { diff --git a/adapter/src/adapter/interpreter/XmlInterpreter.ts b/adapter/src/adapter/interpreter/XmlInterpreter.ts index a8d43e02b..ae5dda006 100644 --- a/adapter/src/adapter/interpreter/XmlInterpreter.ts +++ b/adapter/src/adapter/interpreter/XmlInterpreter.ts @@ -20,13 +20,9 @@ export class XmlInterpreter extends Interpreter { override doInterpret( data: string, + // eslint-disable-next-line @typescript-eslint/no-unused-vars parameters: Record, ): Promise { - /* Data = - '' + - 'RickMorty' + - 'RickMorty';s*/ - const result = this.convertXmlToJson(data); const updatedResult: Record = {}; const resultKey: Record = result[ @@ -45,8 +41,8 @@ export class XmlInterpreter extends Interpreter { for (const result of inputData.matchAll( /(?:<(\w*)(?:\s[^>]*)*>)((?:(?!<\1).)*)(?:<\/\1>)|<(\w*)(?:\s*)*\/>/gm, )) { - const key: string = result[1] || result[3]; const value = result[2] && this.convertXmlToJson(result[2]); + const key: string = result[1] || result[3]; returnJson[key] = (value && Object.keys(value).length ? value : result[2]) || null; } From 7fb3b2281743a34470521abfc3cc93c17c2b78de Mon Sep 17 00:00:00 2001 From: = <=> Date: Fri, 22 Apr 2022 11:42:09 +0200 Subject: [PATCH 24/40] folder structure for tests with @PVahldiek --- adapter/src/{ => tests/adapter}/CsvInterpreter.spec.ts | 2 +- adapter/src/{ => tests/adapter}/JsonInterpreter.spec.ts | 2 +- adapter/src/{ => tests/adapter}/XmlInterpreter.spec.ts | 2 +- adapter/src/{ => tests/adapter}/adapterEndpoint.spec.ts | 6 +++--- 4 files changed, 6 insertions(+), 6 deletions(-) rename adapter/src/{ => tests/adapter}/CsvInterpreter.spec.ts (95%) rename adapter/src/{ => tests/adapter}/JsonInterpreter.spec.ts (90%) rename adapter/src/{ => tests/adapter}/XmlInterpreter.spec.ts (95%) rename adapter/src/{ => tests/adapter}/adapterEndpoint.spec.ts (84%) diff --git a/adapter/src/CsvInterpreter.spec.ts b/adapter/src/tests/adapter/CsvInterpreter.spec.ts similarity index 95% rename from adapter/src/CsvInterpreter.spec.ts rename to adapter/src/tests/adapter/CsvInterpreter.spec.ts index 773957253..4e49d41ef 100644 --- a/adapter/src/CsvInterpreter.spec.ts +++ b/adapter/src/tests/adapter/CsvInterpreter.spec.ts @@ -1,4 +1,4 @@ -import { Format } from './adapter/model/enum/Format'; +import { Format } from '../../adapter/model/enum/Format'; describe('doInterpretCSVFormatReturnsValidJSON', () => { test('convert standard CSV to JSON', async () => { diff --git a/adapter/src/JsonInterpreter.spec.ts b/adapter/src/tests/adapter/JsonInterpreter.spec.ts similarity index 90% rename from adapter/src/JsonInterpreter.spec.ts rename to adapter/src/tests/adapter/JsonInterpreter.spec.ts index f7b5c8424..038296815 100644 --- a/adapter/src/JsonInterpreter.spec.ts +++ b/adapter/src/tests/adapter/JsonInterpreter.spec.ts @@ -1,4 +1,4 @@ -import { Format } from './adapter/model/enum/Format'; +import { Format } from '../../adapter/model/enum/Format'; describe('doInterpretJsonFormatReturnsValidJSON', () => { test('doInterpret JSON data test', () => { diff --git a/adapter/src/XmlInterpreter.spec.ts b/adapter/src/tests/adapter/XmlInterpreter.spec.ts similarity index 95% rename from adapter/src/XmlInterpreter.spec.ts rename to adapter/src/tests/adapter/XmlInterpreter.spec.ts index e937c7a87..8b752dd5d 100644 --- a/adapter/src/XmlInterpreter.spec.ts +++ b/adapter/src/tests/adapter/XmlInterpreter.spec.ts @@ -1,4 +1,4 @@ -import { Format } from './adapter/model/enum/Format'; +import { Format } from '../../adapter/model/enum/Format'; describe('doInterpretXMLFormatReturnsValidJSON', () => { test('convert standard XML to JSON test', () => { diff --git a/adapter/src/adapterEndpoint.spec.ts b/adapter/src/tests/adapter/adapterEndpoint.spec.ts similarity index 84% rename from adapter/src/adapterEndpoint.spec.ts rename to adapter/src/tests/adapter/adapterEndpoint.spec.ts index 96d87a704..3ce015c3c 100644 --- a/adapter/src/adapterEndpoint.spec.ts +++ b/adapter/src/tests/adapter/adapterEndpoint.spec.ts @@ -1,6 +1,6 @@ -import { AdapterEndpoint } from './adapter/api/rest/adapterEndpoint'; -import { Format } from './adapter/model/enum/Format'; -import { Protocol } from './adapter/model/enum/Protocol'; +import { AdapterEndpoint } from '../../adapter/api/rest/adapterEndpoint'; +import { Format } from '../../adapter/model/enum/Format'; +import { Protocol } from '../../adapter/model/enum/Protocol'; /* eslint-env jest */ describe('getFormatShouldReturnCorrectResult', () => { From 48b12e650bf4162e2129d4649713003ce3fe4c6c Mon Sep 17 00:00:00 2001 From: = <=> Date: Fri, 22 Apr 2022 11:52:52 +0200 Subject: [PATCH 25/40] firstRowHeader fixed in CSV Format with @PVahldiek --- .../src/adapter/interpreter/CsvInterpreter.ts | 19 +++++---- .../src/tests/adapter/CsvInterpreter.spec.ts | 39 ++++++++++++++++++- .../src/tests/adapter/JsonInterpreter.spec.ts | 2 +- .../src/tests/adapter/XmlInterpreter.spec.ts | 2 +- .../src/tests/adapter/adapterEndpoint.spec.ts | 4 +- 5 files changed, 53 insertions(+), 13 deletions(-) diff --git a/adapter/src/adapter/interpreter/CsvInterpreter.ts b/adapter/src/adapter/interpreter/CsvInterpreter.ts index 80814ead0..172cece31 100644 --- a/adapter/src/adapter/interpreter/CsvInterpreter.ts +++ b/adapter/src/adapter/interpreter/CsvInterpreter.ts @@ -46,24 +46,27 @@ export class CsvInterpreter extends Interpreter { ): Promise { const columnSeparator = (parameters.columnSeparator as string).charAt(0); const lineSeparator: string = parameters.lineSeparator as string; - const firstRowAsHeader: boolean = parameters.firstRowAsHeader as boolean; // True = With header, False = WithoutHeader + // Be Careful: Need to Invert the boolean here + // True = With header, False = WithoutHeader + const firstRowAsHeader = !parameters.firstRowAsHeader; const skipFirstDataRow: boolean = parameters.skipFirstDataRow as boolean; + data = + 'id,first_name,last_name,email,gender,ip_address\n' + + '1,Ewell,Mathwin,emathwin0@hibu.com,Male,226.172.125.251\n' + + '2,Fayth,Blampy,fblampy1@hubpages.com,Female,212.76.208.25\n' + + '3,Kelli,Cornock,kcornock2@boston.com,Female,171.5.66.30\n'; + const json: string[] = []; await csv({ - noheader: !firstRowAsHeader, // Be Careful: Need to Invert the boolean here + noheader: firstRowAsHeader, output: 'json', delimiter: columnSeparator, eol: lineSeparator, }) .fromString(data) .subscribe((csvRow: string, index: number) => { - // Todo need to test if this works - if ( - skipFirstDataRow && - ((index === 0 && !firstRowAsHeader) || - (index === 1 && firstRowAsHeader)) - ) { + if (skipFirstDataRow && index === 0) { // Skip First Row } else { json.push(csvRow); diff --git a/adapter/src/tests/adapter/CsvInterpreter.spec.ts b/adapter/src/tests/adapter/CsvInterpreter.spec.ts index 4e49d41ef..0049ced5d 100644 --- a/adapter/src/tests/adapter/CsvInterpreter.spec.ts +++ b/adapter/src/tests/adapter/CsvInterpreter.spec.ts @@ -1,6 +1,6 @@ import { Format } from '../../adapter/model/enum/Format'; -describe('doInterpretCSVFormatReturnsValidJSON', () => { +describe('doInterpret CSV Format returns valid JSON', () => { test('convert standard CSV to JSON', async () => { const csvFormat = Format.CSV; const data = @@ -44,3 +44,40 @@ describe('doInterpretCSVFormatReturnsValidJSON', () => { expect(JSON.parse(res)).toEqual(expected); }); }); + +describe('doInterpret CSV Format with SkipFirstRow = TRUE returns valid JSON with missing first Row', () => { + test('convert standard CSV to JSON with skipFirstRow', async () => { + const csvFormat = Format.CSV; + const data = + 'id,first_name,last_name,email,gender,ip_address\n' + + '1,Ewell,Mathwin,emathwin0@hibu.com,Male,226.172.125.251\n' + + '2,Fayth,Blampy,fblampy1@hubpages.com,Female,212.76.208.25\n' + + '3,Kelli,Cornock,kcornock2@boston.com,Female,171.5.66.30\n'; + const parameters = { + columnSeparator: ',', + lineSeparator: '\n', + skipFirstDataRow: true, + firstRowAsHeader: true, + }; + const expected = [ + { + id: '2', + first_name: 'Fayth', + last_name: 'Blampy', + email: 'fblampy1@hubpages.com', + gender: 'Female', + ip_address: '212.76.208.25', + }, + { + id: '3', + first_name: 'Kelli', + last_name: 'Cornock', + email: 'kcornock2@boston.com', + gender: 'Female', + ip_address: '171.5.66.30', + }, + ]; + const res = await csvFormat.doInterpret(data, parameters); + expect(JSON.parse(res)).toEqual(expected); + }); +}); diff --git a/adapter/src/tests/adapter/JsonInterpreter.spec.ts b/adapter/src/tests/adapter/JsonInterpreter.spec.ts index 038296815..25df70a8a 100644 --- a/adapter/src/tests/adapter/JsonInterpreter.spec.ts +++ b/adapter/src/tests/adapter/JsonInterpreter.spec.ts @@ -1,6 +1,6 @@ import { Format } from '../../adapter/model/enum/Format'; -describe('doInterpretJsonFormatReturnsValidJSON', () => { +describe('doInterpret Json Format returns valid JSON', () => { test('doInterpret JSON data test', () => { const jsonFormat = Format.JSON; const data = diff --git a/adapter/src/tests/adapter/XmlInterpreter.spec.ts b/adapter/src/tests/adapter/XmlInterpreter.spec.ts index 8b752dd5d..e7f157c63 100644 --- a/adapter/src/tests/adapter/XmlInterpreter.spec.ts +++ b/adapter/src/tests/adapter/XmlInterpreter.spec.ts @@ -1,6 +1,6 @@ import { Format } from '../../adapter/model/enum/Format'; -describe('doInterpretXMLFormatReturnsValidJSON', () => { +describe('doInterpret XML Format Returns valid JSON', () => { test('convert standard XML to JSON test', () => { const xmlFormat = Format.XML; const data = diff --git a/adapter/src/tests/adapter/adapterEndpoint.spec.ts b/adapter/src/tests/adapter/adapterEndpoint.spec.ts index 3ce015c3c..a58e4d948 100644 --- a/adapter/src/tests/adapter/adapterEndpoint.spec.ts +++ b/adapter/src/tests/adapter/adapterEndpoint.spec.ts @@ -3,7 +3,7 @@ import { Format } from '../../adapter/model/enum/Format'; import { Protocol } from '../../adapter/model/enum/Protocol'; /* eslint-env jest */ -describe('getFormatShouldReturnCorrectResult', () => { +describe('getFormat should return correct Result', () => { test('getFormat test throws exception for not existing format', () => { expect(() => { AdapterEndpoint.getFormat('not here'); @@ -23,7 +23,7 @@ describe('getFormatShouldReturnCorrectResult', () => { }); }); -describe('getProtocolShouldReturnCorrectResult', () => { +describe('getProtocol should return correct Result', () => { test('getProtocol test for HTTP', () => { const result = AdapterEndpoint.getProtocol('HTTP'); expect(result).toBe(Protocol.HTTP); From 479e910e4ae8ef4e3abd4d8e61b48f65670ba3e3 Mon Sep 17 00:00:00 2001 From: = <=> Date: Fri, 22 Apr 2022 12:05:23 +0200 Subject: [PATCH 26/40] line seperator for test in csv interpreter with @PVahldiek --- .../src/adapter/interpreter/CsvInterpreter.ts | 6 -- .../src/tests/adapter/CsvInterpreter.spec.ts | 87 ++++++++++++++++++- 2 files changed, 86 insertions(+), 7 deletions(-) diff --git a/adapter/src/adapter/interpreter/CsvInterpreter.ts b/adapter/src/adapter/interpreter/CsvInterpreter.ts index 172cece31..e5effca0c 100644 --- a/adapter/src/adapter/interpreter/CsvInterpreter.ts +++ b/adapter/src/adapter/interpreter/CsvInterpreter.ts @@ -51,12 +51,6 @@ export class CsvInterpreter extends Interpreter { const firstRowAsHeader = !parameters.firstRowAsHeader; const skipFirstDataRow: boolean = parameters.skipFirstDataRow as boolean; - data = - 'id,first_name,last_name,email,gender,ip_address\n' + - '1,Ewell,Mathwin,emathwin0@hibu.com,Male,226.172.125.251\n' + - '2,Fayth,Blampy,fblampy1@hubpages.com,Female,212.76.208.25\n' + - '3,Kelli,Cornock,kcornock2@boston.com,Female,171.5.66.30\n'; - const json: string[] = []; await csv({ noheader: firstRowAsHeader, diff --git a/adapter/src/tests/adapter/CsvInterpreter.spec.ts b/adapter/src/tests/adapter/CsvInterpreter.spec.ts index 0049ced5d..c574e3ca9 100644 --- a/adapter/src/tests/adapter/CsvInterpreter.spec.ts +++ b/adapter/src/tests/adapter/CsvInterpreter.spec.ts @@ -1,7 +1,7 @@ import { Format } from '../../adapter/model/enum/Format'; describe('doInterpret CSV Format returns valid JSON', () => { - test('convert standard CSV to JSON', async () => { + test('convert standard CSV to JSON with lineSeperator \n and Column Seperator ,', async () => { const csvFormat = Format.CSV; const data = 'id,first_name,last_name,email,gender,ip_address\n' + @@ -43,6 +43,49 @@ describe('doInterpret CSV Format returns valid JSON', () => { const res = await csvFormat.doInterpret(data, parameters); expect(JSON.parse(res)).toEqual(expected); }); + + test('convert standard CSV to JSON with lineSeperator \r and ColumnSeperator ;', async () => { + const csvFormat = Format.CSV; + const data = + 'id;first_name;last_name;email;gender;ip_address\r' + + '1;Ewell;Mathwin;emathwin0@hibu.com;Male;226.172.125.251\r' + + '2;Fayth;Blampy;fblampy1@hubpages.com;Female;212.76.208.25\r' + + '3;Kelli;Cornock;kcornock2@boston.com;Female;171.5.66.30\r'; + const parameters = { + columnSeparator: ';', + lineSeparator: '\r', + skipFirstDataRow: false, + firstRowAsHeader: true, + }; + const expected = [ + { + id: '1', + first_name: 'Ewell', + last_name: 'Mathwin', + email: 'emathwin0@hibu.com', + gender: 'Male', + ip_address: '226.172.125.251', + }, + { + id: '2', + first_name: 'Fayth', + last_name: 'Blampy', + email: 'fblampy1@hubpages.com', + gender: 'Female', + ip_address: '212.76.208.25', + }, + { + id: '3', + first_name: 'Kelli', + last_name: 'Cornock', + email: 'kcornock2@boston.com', + gender: 'Female', + ip_address: '171.5.66.30', + }, + ]; + const res = await csvFormat.doInterpret(data, parameters); + expect(JSON.parse(res)).toEqual(expected); + }); }); describe('doInterpret CSV Format with SkipFirstRow = TRUE returns valid JSON with missing first Row', () => { @@ -81,3 +124,45 @@ describe('doInterpret CSV Format with SkipFirstRow = TRUE returns valid JSON wit expect(JSON.parse(res)).toEqual(expected); }); }); + +describe('doInterpret CSV Format with FirstRowAsHeader = FALSE returns valid JSON', () => { + test('convert standard CSV to JSON with skipFirstRow', async () => { + const csvFormat = Format.CSV; + const data = + '1,Ewell,Mathwin,emathwin0@hibu.com,Male,226.172.125.251\n' + + '2,Fayth,Blampy,fblampy1@hubpages.com,Female,212.76.208.25\n' + + '3,Kelli,Cornock,kcornock2@boston.com\n'; + const parameters = { + columnSeparator: ',', + lineSeparator: '\n', + skipFirstDataRow: false, + firstRowAsHeader: false, + }; + const expected = [ + { + field1: '1', + field2: 'Ewell', + field3: 'Mathwin', + field4: 'emathwin0@hibu.com', + field5: 'Male', + field6: '226.172.125.251', + }, + { + field1: '2', + field2: 'Fayth', + field3: 'Blampy', + field4: 'fblampy1@hubpages.com', + field5: 'Female', + field6: '212.76.208.25', + }, + { + field1: '3', + field2: 'Kelli', + field3: 'Cornock', + field4: 'kcornock2@boston.com', + }, + ]; + const res = await csvFormat.doInterpret(data, parameters); + expect(JSON.parse(res)).toEqual(expected); + }); +}); From 1ae519406a3085aecbcd5e20748bc90e6a8fafcb Mon Sep 17 00:00:00 2001 From: = <=> Date: Fri, 22 Apr 2022 12:25:51 +0200 Subject: [PATCH 27/40] further tests for xmlinterpreter with @PVahldiek --- .../src/tests/adapter/XmlInterpreter.spec.ts | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/adapter/src/tests/adapter/XmlInterpreter.spec.ts b/adapter/src/tests/adapter/XmlInterpreter.spec.ts index e7f157c63..fa19b067c 100644 --- a/adapter/src/tests/adapter/XmlInterpreter.spec.ts +++ b/adapter/src/tests/adapter/XmlInterpreter.spec.ts @@ -27,4 +27,21 @@ describe('doInterpret XML Format Returns valid JSON', () => { }); }); }); + + test('convert nested XML to JSON test', () => { + const xmlFormat = Format.XML; + const data = + 'ToveJaniReminderSubheadingReminder1Reminder2Reminder3Don\'t forget me this weekend!'; + return xmlFormat.doInterpret(data, {}).then((res) => { + expect(JSON.parse(res)).toEqual({ + to: 'Tove', + from: 'Jani', + heading: { + subheading: 'ReminderSubheading', + Reminder: ['Reminder1', 'Reminder2', 'Reminder3'], + }, + body: "Don't forget me this weekend!", + }); + }); + }); }); From e9910127a90cc192d2d9546a282706b26719a018 Mon Sep 17 00:00:00 2001 From: = <=> Date: Fri, 22 Apr 2022 12:41:42 +0200 Subject: [PATCH 28/40] jest Unit Tests for HTTP Importer with @PVahldiek --- adapter/src/adapter/importer/HttpImporter.ts | 5 +- adapter/src/adapter/importer/Importer.ts | 5 +- .../src/tests/adapter/HttpImporter.spec.ts | 64 +++++++++++++++++++ 3 files changed, 66 insertions(+), 8 deletions(-) create mode 100644 adapter/src/tests/adapter/HttpImporter.spec.ts diff --git a/adapter/src/adapter/importer/HttpImporter.ts b/adapter/src/adapter/importer/HttpImporter.ts index 429c0d6ab..062126c9a 100644 --- a/adapter/src/adapter/importer/HttpImporter.ts +++ b/adapter/src/adapter/importer/HttpImporter.ts @@ -6,7 +6,6 @@ import { Importer } from './Importer'; import { ImporterParameterDescription } from './ImporterParameterDescription'; export class HttpImporter extends Importer { - // TODO RuntimeParameters type is probably wrong type = 'HTTP'; description = 'Plain HTTP'; parameters: ImporterParameterDescription[] = [ @@ -21,6 +20,7 @@ export class HttpImporter extends Importer { 'Encoding of the source. Available encodings: ISO-8859-1, US-ASCII, UTF-8', type: 'string', }), + // TODO RuntimeParameters type is probably wrong new ImporterParameterDescription({ name: 'defaultParameters', description: 'Default values for open parameters in the URI', @@ -29,7 +29,6 @@ export class HttpImporter extends Importer { }), ]; - // Override annotation is not necessary, but will be used for a better understanding of the code override getType(): string { return this.type; } @@ -46,7 +45,6 @@ export class HttpImporter extends Importer { super.validateParameters(inputParameters); const encoding: string = inputParameters.encoding as string; - // TODO CHECK IF ENCODING ARE WRITTEN CORRECT if ( encoding !== 'ISO-8859-1' && encoding !== 'US-ASCII' && @@ -70,7 +68,6 @@ export class HttpImporter extends Importer { override async doFetch(parameters: Record): Promise { const uri = parameters.location as string; const encoding = parameters.encoding as string; - // TODO see if encoding from response is good const result = await axios .get(uri, { responseEncoding: encoding }) diff --git a/adapter/src/adapter/importer/Importer.ts b/adapter/src/adapter/importer/Importer.ts index 770437760..5d905fecc 100644 --- a/adapter/src/adapter/importer/Importer.ts +++ b/adapter/src/adapter/importer/Importer.ts @@ -15,7 +15,6 @@ export abstract class Importer { abstract getAvailableParameters(): Array; async fetch(parameters: Record): Promise { - // Throws ImporterParameterException this.validateParameters(parameters); const x = await this.doFetch(parameters); return x; @@ -24,11 +23,9 @@ export abstract class Importer { abstract getType(): string; abstract getDescription(): string; - abstract doFetch(parameters: Record): Promise; // Throws ImporterParameterException + abstract doFetch(parameters: Record): Promise; validateParameters(inputParameters: Record): void { - // Throws ImporterParameterException; - let illegalArguments = false; let illegalArgumentsMessage = ''; diff --git a/adapter/src/tests/adapter/HttpImporter.spec.ts b/adapter/src/tests/adapter/HttpImporter.spec.ts new file mode 100644 index 000000000..3e8172557 --- /dev/null +++ b/adapter/src/tests/adapter/HttpImporter.spec.ts @@ -0,0 +1,64 @@ +import { Importer } from '../../adapter/importer/Importer'; +import { ImporterParameterDescription } from '../../adapter/importer/ImporterParameterDescription'; +import { Protocol } from '../../adapter/model/enum/Protocol'; + +/* eslint-env jest */ +describe('getAvaialbleParameters Tests', () => { + test('getFormat test throws exception for not existing format', () => { + const importer: Importer = Protocol.HTTP; + const availableParameters: ImporterParameterDescription[] = + importer.getAvailableParameters(); + expect(availableParameters[0].type).toBe('string'); + expect(availableParameters[1].type).toBe('string'); + }); + + test('getAvailableParameters is of Type ImporterParameterDescription', () => { + const importer: Importer = Protocol.HTTP; + const availableParameters = importer.getAvailableParameters(); + expect(availableParameters).toBeInstanceOf(Array); + }); +}); + +describe('validateParameters HTTP Importer Tests', () => { + test('validateParameters HTTP Importer with UTF-8 works', () => { + const importer: Importer = Protocol.HTTP; + const parameters = { + location: + 'https://www.pegelonline.wsv.de/webservices/rest-api/v2/stations.json', + encoding: 'UTF-8', + }; + importer.validateParameters(parameters); + }); + + test('validateParameters HTTP Importer with US-ASCII works', () => { + const importer: Importer = Protocol.HTTP; + const parameters = { + location: + 'https://www.pegelonline.wsv.de/webservices/rest-api/v2/stations.json', + encoding: 'US-ASCII', + }; + importer.validateParameters(parameters); + }); + + test('validateParameters HTTP Importer with ISO-8859-1 works', () => { + const importer: Importer = Protocol.HTTP; + const parameters = { + location: + 'https://www.pegelonline.wsv.de/webservices/rest-api/v2/stations.json', + encoding: 'ISO-8859-1', + }; + importer.validateParameters(parameters); + }); + + test('validateParameters HTTP Importer with a wrong Encoding does not work', () => { + const importer: Importer = Protocol.HTTP; + const parameters = { + location: + 'https://www.pegelonline.wsv.de/webservices/rest-api/v2/stations.json', + encoding: 'encodingwrong', + }; + expect(() => { + importer.validateParameters(parameters); + }).toThrow(Error); + }); +}); From 2f5e988fb523fbb7ebfdbb2eae91702b1b5c9a7d Mon Sep 17 00:00:00 2001 From: = <=> Date: Fri, 22 Apr 2022 13:02:40 +0200 Subject: [PATCH 29/40] further tests for validateParameters in HTTP Importer with @PVahldiek --- adapter/src/adapter/importer/Importer.ts | 13 +++-- .../src/tests/adapter/HttpImporter.spec.ts | 49 ++++++++++++++++++- 2 files changed, 56 insertions(+), 6 deletions(-) diff --git a/adapter/src/adapter/importer/Importer.ts b/adapter/src/adapter/importer/Importer.ts index 5d905fecc..31dc24475 100644 --- a/adapter/src/adapter/importer/Importer.ts +++ b/adapter/src/adapter/importer/Importer.ts @@ -48,19 +48,21 @@ export abstract class Importer { illegalArgumentsMessage += argument + ' is not needed by importer \n'; } } + const requiredParameters = this.getRequiredParameters(); for (const requiredParameter of requiredParameters) { - // TODO is that OK? const name = requiredParameter.name; - const checkType = (inputParameters[name] as Record) - .constructor.name; - if (inputParameters[requiredParameter.name] == null) { + if (inputParameters[name] === undefined) { illegalArguments = true; illegalArgumentsMessage += this.type; illegalArgumentsMessage += 'importer requires parameter '; illegalArgumentsMessage += name; illegalArgumentsMessage += '\n'; - } else if (checkType.toLowerCase() !== requiredParameter.type) { + break; + } + const checkType = (inputParameters[name] as Record) + .constructor.name; + if (checkType.toLowerCase() !== requiredParameter.type) { illegalArguments = true; illegalArgumentsMessage += this.type; illegalArgumentsMessage += ' importer requires parameter '; @@ -68,6 +70,7 @@ export abstract class Importer { illegalArgumentsMessage += ' to be type '; illegalArgumentsMessage += requiredParameter.type; illegalArgumentsMessage += '\n'; + break; } } if (illegalArguments) { diff --git a/adapter/src/tests/adapter/HttpImporter.spec.ts b/adapter/src/tests/adapter/HttpImporter.spec.ts index 3e8172557..da34ea46a 100644 --- a/adapter/src/tests/adapter/HttpImporter.spec.ts +++ b/adapter/src/tests/adapter/HttpImporter.spec.ts @@ -1,10 +1,11 @@ import { Importer } from '../../adapter/importer/Importer'; import { ImporterParameterDescription } from '../../adapter/importer/ImporterParameterDescription'; import { Protocol } from '../../adapter/model/enum/Protocol'; +import { ImporterParameterError } from '../../adapter/model/exceptions/ImporterParameterError'; /* eslint-env jest */ describe('getAvaialbleParameters Tests', () => { - test('getFormat test throws exception for not existing format', () => { + test('getAvaiableParameters are of type string', () => { const importer: Importer = Protocol.HTTP; const availableParameters: ImporterParameterDescription[] = importer.getAvailableParameters(); @@ -19,6 +20,52 @@ describe('getAvaialbleParameters Tests', () => { }); }); +describe('validateParameters from Abstract Parent Class Tests', () => { + test('validateParameters with valid Parameters works', () => { + const importer: Importer = Protocol.HTTP; + const parameters = { + location: + 'https://www.pegelonline.wsv.de/webservices/rest-api/v2/stations.json', + encoding: 'UTF-8', + }; + importer.validateParameters(parameters); + }); + + test('validateParameters finds unavailable Parameter', () => { + const importer: Importer = Protocol.HTTP; + const wrongParameters = { + location: + 'https://www.pegelonline.wsv.de/webservices/rest-api/v2/stations.json', + encoding: 'UTF-8', + parameter3: 'komischer parameter', + }; + expect(() => { + importer.validateParameters(wrongParameters); + }).toThrow(ImporterParameterError); + }); + + test('validateParameters is missing encoding Parameter', () => { + const importer: Importer = Protocol.HTTP; + const wrongParameters = { + location: + 'https://www.pegelonline.wsv.de/webservices/rest-api/v2/stations.json', + }; + expect(() => { + importer.validateParameters(wrongParameters); + }).toThrow(ImporterParameterError); + }); + + test('validateParameters is missing location Parameter', () => { + const importer: Importer = Protocol.HTTP; + const wrongParameters = { + encoding: 'UTF-8', + }; + expect(() => { + importer.validateParameters(wrongParameters); + }).toThrow(ImporterParameterError); + }); +}); + describe('validateParameters HTTP Importer Tests', () => { test('validateParameters HTTP Importer with UTF-8 works', () => { const importer: Importer = Protocol.HTTP; From 00b5d8a9f65f6e8ac3963609480807b4d6e17176 Mon Sep 17 00:00:00 2001 From: = <=> Date: Fri, 22 Apr 2022 14:33:07 +0200 Subject: [PATCH 30/40] added fast xml parser with @PVahldiek --- adapter/package-lock.json | 34 ++++++++++++++++++++++++++++++++++ adapter/package.json | 1 + 2 files changed, 35 insertions(+) diff --git a/adapter/package-lock.json b/adapter/package-lock.json index f6f14486d..3c5cbd6fd 100644 --- a/adapter/package-lock.json +++ b/adapter/package-lock.json @@ -17,6 +17,7 @@ "cors": "^2.8.5", "csvtojson": "^2.0.10", "express": "^4.17.1", + "fast-xml-parser": "^4.0.7", "jackson-js": "^1.1.0", "knex": "^1.0.4", "prisma": "^3.10.0", @@ -3804,6 +3805,21 @@ "integrity": "sha1-PYpcZog6FqMMqGQ+hR8Zuqd5eRc=", "dev": true }, + "node_modules/fast-xml-parser": { + "version": "4.0.7", + "resolved": "https://registry.npmjs.org/fast-xml-parser/-/fast-xml-parser-4.0.7.tgz", + "integrity": "sha512-dMtibyus3kC7nbxj1CpVtysLzO13UOAZEFAb5vpQg3T4O6qvetmSePpXKFx5KPNCHKoGwjtgjfF5DOyn7s1ylQ==", + "dependencies": { + "strnum": "^1.0.5" + }, + "bin": { + "fxparser": "src/cli/cli.js" + }, + "funding": { + "type": "paypal", + "url": "https://paypal.me/naturalintelligence" + } + }, "node_modules/fastq": { "version": "1.13.0", "resolved": "https://registry.npmjs.org/fastq/-/fastq-1.13.0.tgz", @@ -8543,6 +8559,11 @@ "url": "https://github.com/sponsors/sindresorhus" } }, + "node_modules/strnum": { + "version": "1.0.5", + "resolved": "https://registry.npmjs.org/strnum/-/strnum-1.0.5.tgz", + "integrity": "sha512-J8bbNyKKXl5qYcR36TIO8W3mVGVHrmmxsd5PAItGkmyzwJvybiw2IVq5nqd0i4LSNSkB/sx9VHllbfFdr9k1JA==" + }, "node_modules/supports-color": { "version": "5.5.0", "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-5.5.0.tgz", @@ -12217,6 +12238,14 @@ "integrity": "sha1-PYpcZog6FqMMqGQ+hR8Zuqd5eRc=", "dev": true }, + "fast-xml-parser": { + "version": "4.0.7", + "resolved": "https://registry.npmjs.org/fast-xml-parser/-/fast-xml-parser-4.0.7.tgz", + "integrity": "sha512-dMtibyus3kC7nbxj1CpVtysLzO13UOAZEFAb5vpQg3T4O6qvetmSePpXKFx5KPNCHKoGwjtgjfF5DOyn7s1ylQ==", + "requires": { + "strnum": "^1.0.5" + } + }, "fastq": { "version": "1.13.0", "resolved": "https://registry.npmjs.org/fastq/-/fastq-1.13.0.tgz", @@ -15686,6 +15715,11 @@ "integrity": "sha512-6fPc+R4ihwqP6N/aIv2f1gMH8lOVtWQHoqC4yK6oSDVVocumAsfCqjkXnqiYMhmMwS/mEHLp7Vehlt3ql6lEig==", "dev": true }, + "strnum": { + "version": "1.0.5", + "resolved": "https://registry.npmjs.org/strnum/-/strnum-1.0.5.tgz", + "integrity": "sha512-J8bbNyKKXl5qYcR36TIO8W3mVGVHrmmxsd5PAItGkmyzwJvybiw2IVq5nqd0i4LSNSkB/sx9VHllbfFdr9k1JA==" + }, "supports-color": { "version": "5.5.0", "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-5.5.0.tgz", diff --git a/adapter/package.json b/adapter/package.json index 4c4ef8dbb..0312bbddf 100644 --- a/adapter/package.json +++ b/adapter/package.json @@ -23,6 +23,7 @@ "cors": "^2.8.5", "csvtojson": "^2.0.10", "express": "^4.17.1", + "fast-xml-parser": "^4.0.7", "jackson-js": "^1.1.0", "knex": "^1.0.4", "prisma": "^3.10.0", From e637af4fc71f61036265e34d8a02e0ca1baeb9d1 Mon Sep 17 00:00:00 2001 From: = <=> Date: Fri, 22 Apr 2022 14:49:02 +0200 Subject: [PATCH 31/40] =?UTF-8?q?new=20xmlparser=20library=20+=20manuelle?= =?UTF-8?q?=20l=C3=B6schung=20vom=20root=20element=20with=20@PVahldiek?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../src/adapter/interpreter/XmlInterpreter.ts | 24 ++++++++++++++++--- .../src/tests/adapter/XmlInterpreter.spec.ts | 7 +++--- 2 files changed, 25 insertions(+), 6 deletions(-) diff --git a/adapter/src/adapter/interpreter/XmlInterpreter.ts b/adapter/src/adapter/interpreter/XmlInterpreter.ts index ae5dda006..ba3c51df6 100644 --- a/adapter/src/adapter/interpreter/XmlInterpreter.ts +++ b/adapter/src/adapter/interpreter/XmlInterpreter.ts @@ -1,3 +1,5 @@ +import { XMLParser } from 'fast-xml-parser'; + import { Interpreter } from './Interpreter'; import { InterpreterParameterDescription } from './InterpreterParameterDescription'; @@ -23,7 +25,12 @@ export class XmlInterpreter extends Interpreter { // eslint-disable-next-line @typescript-eslint/no-unused-vars parameters: Record, ): Promise { - const result = this.convertXmlToJson(data); + const options = { + ignoreDeclaration: true, + }; + + const parser = new XMLParser(options); + const result = parser.parse(data) as Record; const updatedResult: Record = {}; const resultKey: Record = result[ Object.keys(result)[0] @@ -34,9 +41,20 @@ export class XmlInterpreter extends Interpreter { return new Promise(function (resolve) { resolve(JSON.stringify(updatedResult)); }); + /* Const result = this.convertXmlToJson(data); + const updatedResult: Record = {}; + const resultKey: Record = result[ + Object.keys(result)[0] + ] as Record; + for (const [key, value] of Object.entries(resultKey)) { + updatedResult[key] = value; + } + return new Promise(function (resolve) { + resolve(JSON.stringify(updatedResult)); + });*/ } - convertXmlToJson(inputData: string): Record { + /* ConvertXmlToJson(inputData: string): Record { const returnJson: Record = {}; for (const result of inputData.matchAll( /(?:<(\w*)(?:\s[^>]*)*>)((?:(?!<\1).)*)(?:<\/\1>)|<(\w*)(?:\s*)*\/>/gm, @@ -47,5 +65,5 @@ export class XmlInterpreter extends Interpreter { (value && Object.keys(value).length ? value : result[2]) || null; } return returnJson; - } + }*/ } diff --git a/adapter/src/tests/adapter/XmlInterpreter.spec.ts b/adapter/src/tests/adapter/XmlInterpreter.spec.ts index fa19b067c..c6a8e3cc4 100644 --- a/adapter/src/tests/adapter/XmlInterpreter.spec.ts +++ b/adapter/src/tests/adapter/XmlInterpreter.spec.ts @@ -4,7 +4,7 @@ describe('doInterpret XML Format Returns valid JSON', () => { test('convert standard XML to JSON test', () => { const xmlFormat = Format.XML; const data = - 'ToveJaniReminderDon\'t forget me this weekend!'; + 'ToveJaniReminderDon\'t forget me this weekend!'; return xmlFormat.doInterpret(data, {}).then((res) => { expect(JSON.parse(res)).toEqual({ to: 'Tove', @@ -14,10 +14,11 @@ describe('doInterpret XML Format Returns valid JSON', () => { }); }); }); + test('convert nested XML to JSON test', () => { const xmlFormat = Format.XML; const data = - 'ToveJaniReminderSubheadingReminderDon\'t forget me this weekend!'; + 'ToveJaniReminderSubheadingReminderDon\'t forget me this weekend!'; return xmlFormat.doInterpret(data, {}).then((res) => { expect(JSON.parse(res)).toEqual({ to: 'Tove', @@ -31,7 +32,7 @@ describe('doInterpret XML Format Returns valid JSON', () => { test('convert nested XML to JSON test', () => { const xmlFormat = Format.XML; const data = - 'ToveJaniReminderSubheadingReminder1Reminder2Reminder3Don\'t forget me this weekend!'; + 'ToveJaniReminderSubheadingReminder1Reminder2Reminder3Don\'t forget me this weekend!'; return xmlFormat.doInterpret(data, {}).then((res) => { expect(JSON.parse(res)).toEqual({ to: 'Tove', From 61293420a96527cb288ebeddeced49b3ea2533eb Mon Sep 17 00:00:00 2001 From: = <=> Date: Fri, 22 Apr 2022 14:51:59 +0200 Subject: [PATCH 32/40] =?UTF-8?q?unn=C3=B6tige=20dependencies=20raus=20wit?= =?UTF-8?q?h=20@PVahldiek?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- adapter/package.json | 3 --- .../src/adapter/interpreter/XmlInterpreter.ts | 24 ------------------- adapter/src/tests/adapter/interpreter.spec.ts | 16 +++++++++++++ 3 files changed, 16 insertions(+), 27 deletions(-) create mode 100644 adapter/src/tests/adapter/interpreter.spec.ts diff --git a/adapter/package.json b/adapter/package.json index 0312bbddf..e74ae992f 100644 --- a/adapter/package.json +++ b/adapter/package.json @@ -24,11 +24,9 @@ "csvtojson": "^2.0.10", "express": "^4.17.1", "fast-xml-parser": "^4.0.7", - "jackson-js": "^1.1.0", "knex": "^1.0.4", "prisma": "^3.10.0", "uuid": "^8.3.2", - "xml2js": "^0.4.23" }, "devDependencies": { "@jvalue/eslint-config-jvalue": "^1.1.0", @@ -38,7 +36,6 @@ "@types/node": "^17.0.22", "@types/supertest": "^2.0.11", "@types/uuid": "^8.3.4", - "@types/xml2js": "^0.4.11", "@typescript-eslint/eslint-plugin": "^4.30.0", "@typescript-eslint/parser": "^4.30.0", "eslint": "^7.31.0", diff --git a/adapter/src/adapter/interpreter/XmlInterpreter.ts b/adapter/src/adapter/interpreter/XmlInterpreter.ts index ba3c51df6..3fe471203 100644 --- a/adapter/src/adapter/interpreter/XmlInterpreter.ts +++ b/adapter/src/adapter/interpreter/XmlInterpreter.ts @@ -41,29 +41,5 @@ export class XmlInterpreter extends Interpreter { return new Promise(function (resolve) { resolve(JSON.stringify(updatedResult)); }); - /* Const result = this.convertXmlToJson(data); - const updatedResult: Record = {}; - const resultKey: Record = result[ - Object.keys(result)[0] - ] as Record; - for (const [key, value] of Object.entries(resultKey)) { - updatedResult[key] = value; - } - return new Promise(function (resolve) { - resolve(JSON.stringify(updatedResult)); - });*/ } - - /* ConvertXmlToJson(inputData: string): Record { - const returnJson: Record = {}; - for (const result of inputData.matchAll( - /(?:<(\w*)(?:\s[^>]*)*>)((?:(?!<\1).)*)(?:<\/\1>)|<(\w*)(?:\s*)*\/>/gm, - )) { - const value = result[2] && this.convertXmlToJson(result[2]); - const key: string = result[1] || result[3]; - returnJson[key] = - (value && Object.keys(value).length ? value : result[2]) || null; - } - return returnJson; - }*/ } diff --git a/adapter/src/tests/adapter/interpreter.spec.ts b/adapter/src/tests/adapter/interpreter.spec.ts new file mode 100644 index 000000000..f0aa1c642 --- /dev/null +++ b/adapter/src/tests/adapter/interpreter.spec.ts @@ -0,0 +1,16 @@ +import { AdapterEndpoint } from '../../adapter/api/rest/adapterEndpoint'; +import { Format } from '../../adapter/model/enum/Format'; +import { Protocol } from '../../adapter/model/enum/Protocol'; + +/* eslint-env jest */ +describe('getFormat should return correct Result', () => { + test('getFormat test throws exception for not existing format', () => { + expect(() => { + AdapterEndpoint.getFormat('not here'); + }).toThrow(Error); + }); + +}); + +d +}); From 9f9a2c95c87409e58c121ff39cf35c5f7029fd4c Mon Sep 17 00:00:00 2001 From: = <=> Date: Fri, 22 Apr 2022 15:01:57 +0200 Subject: [PATCH 33/40] additional test for csv interpreter + validateparameters changed with @PVahldiek --- adapter/package.json | 2 +- .../src/adapter/interpreter/Interpreter.ts | 23 ++++++++- .../src/tests/adapter/CsvInterpreter.spec.ts | 49 +++++++++++++++++++ .../src/tests/adapter/HttpImporter.spec.ts | 2 +- adapter/src/tests/adapter/interpreter.spec.ts | 16 ------ 5 files changed, 73 insertions(+), 19 deletions(-) delete mode 100644 adapter/src/tests/adapter/interpreter.spec.ts diff --git a/adapter/package.json b/adapter/package.json index e74ae992f..20f995afc 100644 --- a/adapter/package.json +++ b/adapter/package.json @@ -26,7 +26,7 @@ "fast-xml-parser": "^4.0.7", "knex": "^1.0.4", "prisma": "^3.10.0", - "uuid": "^8.3.2", + "uuid": "^8.3.2" }, "devDependencies": { "@jvalue/eslint-config-jvalue": "^1.1.0", diff --git a/adapter/src/adapter/interpreter/Interpreter.ts b/adapter/src/adapter/interpreter/Interpreter.ts index 3975c5040..229cb8a19 100644 --- a/adapter/src/adapter/interpreter/Interpreter.ts +++ b/adapter/src/adapter/interpreter/Interpreter.ts @@ -26,11 +26,32 @@ export abstract class Interpreter { let illegalArguments = false; let illegalArgumentsMessage = ''; + const possibleParameters: Array = + this.getAvailableParameters(); + + const unnecessaryArguments = []; + const names = possibleParameters.map((a) => a.name); + const keys = Object.keys(inputParameters); + + for (const entry of keys) { + if (!names.includes(entry)) { + unnecessaryArguments.push(entry); + } + } + + if (unnecessaryArguments.length > 0) { + illegalArguments = true; + for (const argument of unnecessaryArguments) { + illegalArgumentsMessage += argument + ' is not needed by importer \n'; + } + } + for (const requiredParameter of this.getAvailableParameters()) { const param = inputParameters[ requiredParameter.name ] as InterpreterParameterDescription; - if (param == null) { + + if (param === undefined) { illegalArguments = true; illegalArgumentsMessage += this.type; illegalArgumentsMessage += 'interpreter requires parameter '; diff --git a/adapter/src/tests/adapter/CsvInterpreter.spec.ts b/adapter/src/tests/adapter/CsvInterpreter.spec.ts index c574e3ca9..ef638ab19 100644 --- a/adapter/src/tests/adapter/CsvInterpreter.spec.ts +++ b/adapter/src/tests/adapter/CsvInterpreter.spec.ts @@ -1,4 +1,7 @@ +import { Interpreter } from '../../adapter/interpreter/Interpreter'; +import { InterpreterParameterDescription } from '../../adapter/interpreter/InterpreterParameterDescription'; import { Format } from '../../adapter/model/enum/Format'; +import { InterpreterParameterError } from '../../adapter/model/exceptions/InterpreterParameterError'; describe('doInterpret CSV Format returns valid JSON', () => { test('convert standard CSV to JSON with lineSeperator \n and Column Seperator ,', async () => { @@ -166,3 +169,49 @@ describe('doInterpret CSV Format with FirstRowAsHeader = FALSE returns valid JSO expect(JSON.parse(res)).toEqual(expected); }); }); + +describe('getAvaialbleParameters Tests', () => { + test('getAvaiableParameters are of type string', () => { + const interpreter: Interpreter = Format.CSV; + const availableParameters: InterpreterParameterDescription[] = + interpreter.getAvailableParameters(); + expect(availableParameters[0].type).toBe('string'); + expect(availableParameters[1].type).toBe('string'); + expect(availableParameters[2].type).toBe('boolean'); + expect(availableParameters[3].type).toBe('boolean'); + }); + + test('getAvailableParameters is of Type Array', () => { + const interpreter: Interpreter = Format.CSV; + const availableParameters: InterpreterParameterDescription[] = + interpreter.getAvailableParameters(); + expect(availableParameters).toBeInstanceOf(Array); + }); +}); + +describe('validateParameters from CSV Interpreter', () => { + test('validateParameters with valid Parameters works', () => { + const interpreter: Interpreter = Format.CSV; + const parameters = { + columnSeparator: ',', + lineSeparator: '\n', + skipFirstDataRow: true, + firstRowAsHeader: false, + }; + interpreter.validateParameters(parameters); + }); + + test('validateParameters with too many Parameters throws Error', () => { + const interpreter: Interpreter = Format.CSV; + const parameters = { + columnSeparator: ',', + lineSeparator: '\n', + skipFirstDataRow: true, + firstRowAsHeader: false, + extraParameter: false, + }; + expect(() => { + interpreter.validateParameters(parameters); + }).toThrow(InterpreterParameterError); + }); +}); diff --git a/adapter/src/tests/adapter/HttpImporter.spec.ts b/adapter/src/tests/adapter/HttpImporter.spec.ts index da34ea46a..9495f8d2e 100644 --- a/adapter/src/tests/adapter/HttpImporter.spec.ts +++ b/adapter/src/tests/adapter/HttpImporter.spec.ts @@ -13,7 +13,7 @@ describe('getAvaialbleParameters Tests', () => { expect(availableParameters[1].type).toBe('string'); }); - test('getAvailableParameters is of Type ImporterParameterDescription', () => { + test('getAvailableParameters is of Type Array', () => { const importer: Importer = Protocol.HTTP; const availableParameters = importer.getAvailableParameters(); expect(availableParameters).toBeInstanceOf(Array); diff --git a/adapter/src/tests/adapter/interpreter.spec.ts b/adapter/src/tests/adapter/interpreter.spec.ts deleted file mode 100644 index f0aa1c642..000000000 --- a/adapter/src/tests/adapter/interpreter.spec.ts +++ /dev/null @@ -1,16 +0,0 @@ -import { AdapterEndpoint } from '../../adapter/api/rest/adapterEndpoint'; -import { Format } from '../../adapter/model/enum/Format'; -import { Protocol } from '../../adapter/model/enum/Protocol'; - -/* eslint-env jest */ -describe('getFormat should return correct Result', () => { - test('getFormat test throws exception for not existing format', () => { - expect(() => { - AdapterEndpoint.getFormat('not here'); - }).toThrow(Error); - }); - -}); - -d -}); From 90446b6d2ac69ce6c1892e9928081f0029d969e9 Mon Sep 17 00:00:00 2001 From: = <=> Date: Fri, 22 Apr 2022 15:14:45 +0200 Subject: [PATCH 34/40] =?UTF-8?q?validateParameters=20in=20CSV=20Interpret?= =?UTF-8?q?er=20angepasst=20-=20weitere=20Tests=20f=C3=BCr=20Interpreter?= =?UTF-8?q?=20with=20@PVahldiek?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- adapter/src/adapter/importer/Importer.ts | 14 +++--- .../importer/ImporterParameterDescription.ts | 2 - .../src/adapter/interpreter/Interpreter.ts | 16 ++++++- .../src/tests/adapter/CsvInterpreter.spec.ts | 48 +++++++++++++++++++ 4 files changed, 70 insertions(+), 10 deletions(-) diff --git a/adapter/src/adapter/importer/Importer.ts b/adapter/src/adapter/importer/Importer.ts index 31dc24475..10916cf80 100644 --- a/adapter/src/adapter/importer/Importer.ts +++ b/adapter/src/adapter/importer/Importer.ts @@ -51,22 +51,24 @@ export abstract class Importer { const requiredParameters = this.getRequiredParameters(); for (const requiredParameter of requiredParameters) { - const name = requiredParameter.name; - if (inputParameters[name] === undefined) { + const param = inputParameters[ + requiredParameter.name + ] as ImporterParameterDescription; + + if (param === undefined) { illegalArguments = true; illegalArgumentsMessage += this.type; illegalArgumentsMessage += 'importer requires parameter '; - illegalArgumentsMessage += name; + illegalArgumentsMessage += requiredParameter.name; illegalArgumentsMessage += '\n'; break; } - const checkType = (inputParameters[name] as Record) - .constructor.name; + const checkType = param.constructor.name; if (checkType.toLowerCase() !== requiredParameter.type) { illegalArguments = true; illegalArgumentsMessage += this.type; illegalArgumentsMessage += ' importer requires parameter '; - illegalArgumentsMessage += name; + illegalArgumentsMessage += requiredParameter.name; illegalArgumentsMessage += ' to be type '; illegalArgumentsMessage += requiredParameter.type; illegalArgumentsMessage += '\n'; diff --git a/adapter/src/adapter/importer/ImporterParameterDescription.ts b/adapter/src/adapter/importer/ImporterParameterDescription.ts index 36e5ea449..319dc2073 100644 --- a/adapter/src/adapter/importer/ImporterParameterDescription.ts +++ b/adapter/src/adapter/importer/ImporterParameterDescription.ts @@ -2,8 +2,6 @@ export class ImporterParameterDescription { name: string; description: string; required: boolean; - - // TODO @Georg: need to check how to store the class in typescript, can use instance.constructor.name to get the name probably -> is stored as string type: unknown; constructor({ diff --git a/adapter/src/adapter/interpreter/Interpreter.ts b/adapter/src/adapter/interpreter/Interpreter.ts index 229cb8a19..d62d55c0f 100644 --- a/adapter/src/adapter/interpreter/Interpreter.ts +++ b/adapter/src/adapter/interpreter/Interpreter.ts @@ -45,8 +45,8 @@ export abstract class Interpreter { illegalArgumentsMessage += argument + ' is not needed by importer \n'; } } - - for (const requiredParameter of this.getAvailableParameters()) { + const requiredParameters = this.getAvailableParameters(); + for (const requiredParameter of requiredParameters) { const param = inputParameters[ requiredParameter.name ] as InterpreterParameterDescription; @@ -57,6 +57,18 @@ export abstract class Interpreter { illegalArgumentsMessage += 'interpreter requires parameter '; illegalArgumentsMessage += requiredParameter.name; illegalArgumentsMessage += '\n'; + break; + } + const checkType = param.constructor.name; + if (checkType.toLowerCase() !== requiredParameter.type) { + illegalArguments = true; + illegalArgumentsMessage += this.type; + illegalArgumentsMessage += ' interpreter requires parameter '; + illegalArgumentsMessage += requiredParameter.name; + illegalArgumentsMessage += ' to be type '; + illegalArgumentsMessage += requiredParameter.type; + illegalArgumentsMessage += '\n'; + break; } } if (illegalArguments) { diff --git a/adapter/src/tests/adapter/CsvInterpreter.spec.ts b/adapter/src/tests/adapter/CsvInterpreter.spec.ts index ef638ab19..0b18d3c88 100644 --- a/adapter/src/tests/adapter/CsvInterpreter.spec.ts +++ b/adapter/src/tests/adapter/CsvInterpreter.spec.ts @@ -201,6 +201,54 @@ describe('validateParameters from CSV Interpreter', () => { interpreter.validateParameters(parameters); }); + test('validateParameters with Parameter misses columnSeperator throws Exception', () => { + const interpreter: Interpreter = Format.CSV; + const parameters = { + lineSeparator: '\n', + skipFirstDataRow: true, + firstRowAsHeader: false, + }; + expect(() => { + interpreter.validateParameters(parameters); + }).toThrow(InterpreterParameterError); + }); + + test('validateParameters with Parameter misses lineSeperator throws Exception', () => { + const interpreter: Interpreter = Format.CSV; + const parameters = { + columnSeparator: ',', + skipFirstDataRow: true, + firstRowAsHeader: false, + }; + expect(() => { + interpreter.validateParameters(parameters); + }).toThrow(InterpreterParameterError); + }); + + test('validateParameters with Parameter misses skipFirstDataRow throws Exception', () => { + const interpreter: Interpreter = Format.CSV; + const parameters = { + columnSeparator: ',', + lineSeparator: '\n', + firstRowAsHeader: false, + }; + expect(() => { + interpreter.validateParameters(parameters); + }).toThrow(InterpreterParameterError); + }); + + test('validateParameters with Parameter misses firstRowAsHeader throws Exception', () => { + const interpreter: Interpreter = Format.CSV; + const parameters = { + columnSeparator: ',', + lineSeparator: '\n', + skipFirstDataRow: true, + }; + expect(() => { + interpreter.validateParameters(parameters); + }).toThrow(InterpreterParameterError); + }); + test('validateParameters with too many Parameters throws Error', () => { const interpreter: Interpreter = Format.CSV; const parameters = { From 90dbde74d08b50c73d94fcc91a3a0e003ef3b492 Mon Sep 17 00:00:00 2001 From: = <=> Date: Fri, 22 Apr 2022 16:05:15 +0200 Subject: [PATCH 35/40] adapted tests for new xml parser with @PVahldiek --- .../src/adapter-stateless.test.js | 4 ++- .../src/adapter/interpreter/XmlInterpreter.ts | 9 +---- .../src/tests/adapter/XmlInterpreter.spec.ts | 34 +++++++++++-------- 3 files changed, 24 insertions(+), 23 deletions(-) diff --git a/adapter/integration-test/src/adapter-stateless.test.js b/adapter/integration-test/src/adapter-stateless.test.js index f64efb43c..c58272434 100644 --- a/adapter/integration-test/src/adapter-stateless.test.js +++ b/adapter/integration-test/src/adapter-stateless.test.js @@ -133,7 +133,9 @@ describe('Stateless data import', () => { .send(reqBody); expect(response.status).toEqual(200); const importedData = response.body.data; - expect(JSON.parse(importedData)).toEqual({ from: 'Rick', to: 'Morty' }); + expect(JSON.parse(importedData)).toEqual({ + root: { from: 'Rick', to: 'Morty' }, + }); }, TIMEOUT, ); diff --git a/adapter/src/adapter/interpreter/XmlInterpreter.ts b/adapter/src/adapter/interpreter/XmlInterpreter.ts index 3fe471203..77b224e89 100644 --- a/adapter/src/adapter/interpreter/XmlInterpreter.ts +++ b/adapter/src/adapter/interpreter/XmlInterpreter.ts @@ -31,15 +31,8 @@ export class XmlInterpreter extends Interpreter { const parser = new XMLParser(options); const result = parser.parse(data) as Record; - const updatedResult: Record = {}; - const resultKey: Record = result[ - Object.keys(result)[0] - ] as Record; - for (const [key, value] of Object.entries(resultKey)) { - updatedResult[key] = value; - } return new Promise(function (resolve) { - resolve(JSON.stringify(updatedResult)); + resolve(JSON.stringify(result)); }); } } diff --git a/adapter/src/tests/adapter/XmlInterpreter.spec.ts b/adapter/src/tests/adapter/XmlInterpreter.spec.ts index c6a8e3cc4..e5c748e70 100644 --- a/adapter/src/tests/adapter/XmlInterpreter.spec.ts +++ b/adapter/src/tests/adapter/XmlInterpreter.spec.ts @@ -7,10 +7,12 @@ describe('doInterpret XML Format Returns valid JSON', () => { 'ToveJaniReminderDon\'t forget me this weekend!'; return xmlFormat.doInterpret(data, {}).then((res) => { expect(JSON.parse(res)).toEqual({ - to: 'Tove', - from: 'Jani', - heading: 'Reminder', - body: "Don't forget me this weekend!", + root: { + to: 'Tove', + from: 'Jani', + heading: 'Reminder', + body: "Don't forget me this weekend!", + }, }); }); }); @@ -21,10 +23,12 @@ describe('doInterpret XML Format Returns valid JSON', () => { 'ToveJaniReminderSubheadingReminderDon\'t forget me this weekend!'; return xmlFormat.doInterpret(data, {}).then((res) => { expect(JSON.parse(res)).toEqual({ - to: 'Tove', - from: 'Jani', - heading: { subheading: 'ReminderSubheading', Reminder: 'Reminder' }, - body: "Don't forget me this weekend!", + root: { + to: 'Tove', + from: 'Jani', + heading: { subheading: 'ReminderSubheading', Reminder: 'Reminder' }, + body: "Don't forget me this weekend!", + }, }); }); }); @@ -35,13 +39,15 @@ describe('doInterpret XML Format Returns valid JSON', () => { 'ToveJaniReminderSubheadingReminder1Reminder2Reminder3Don\'t forget me this weekend!'; return xmlFormat.doInterpret(data, {}).then((res) => { expect(JSON.parse(res)).toEqual({ - to: 'Tove', - from: 'Jani', - heading: { - subheading: 'ReminderSubheading', - Reminder: ['Reminder1', 'Reminder2', 'Reminder3'], + root: { + to: 'Tove', + from: 'Jani', + heading: { + subheading: 'ReminderSubheading', + Reminder: ['Reminder1', 'Reminder2', 'Reminder3'], + }, + body: "Don't forget me this weekend!", }, - body: "Don't forget me this weekend!", }); }); }); From f26b75ce30cc85967f56e91b0cb25f12935ac6fe Mon Sep 17 00:00:00 2001 From: = <=> Date: Fri, 22 Apr 2022 16:20:47 +0200 Subject: [PATCH 36/40] interpreter when there are no parameters present with @PVahldiek --- adapter/src/adapter/interpreter/Interpreter.ts | 3 +++ 1 file changed, 3 insertions(+) diff --git a/adapter/src/adapter/interpreter/Interpreter.ts b/adapter/src/adapter/interpreter/Interpreter.ts index d62d55c0f..2d57b7ebf 100644 --- a/adapter/src/adapter/interpreter/Interpreter.ts +++ b/adapter/src/adapter/interpreter/Interpreter.ts @@ -23,6 +23,9 @@ export abstract class Interpreter { abstract getAvailableParameters(): Array; validateParameters(inputParameters: Record): void { + if (inputParameters === undefined) { + return; + } let illegalArguments = false; let illegalArgumentsMessage = ''; From 59683b8c70df623ee811a7b83c66e44fc9c39cd3 Mon Sep 17 00:00:00 2001 From: = <=> Date: Fri, 22 Apr 2022 16:25:17 +0200 Subject: [PATCH 37/40] interpreter bug fix with @PVahldiek --- adapter/src/adapter/interpreter/Interpreter.ts | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/adapter/src/adapter/interpreter/Interpreter.ts b/adapter/src/adapter/interpreter/Interpreter.ts index 2d57b7ebf..cd9365338 100644 --- a/adapter/src/adapter/interpreter/Interpreter.ts +++ b/adapter/src/adapter/interpreter/Interpreter.ts @@ -23,15 +23,16 @@ export abstract class Interpreter { abstract getAvailableParameters(): Array; validateParameters(inputParameters: Record): void { - if (inputParameters === undefined) { - return; - } let illegalArguments = false; let illegalArgumentsMessage = ''; const possibleParameters: Array = this.getAvailableParameters(); + if (possibleParameters.length === 0) { + return; + } + const unnecessaryArguments = []; const names = possibleParameters.map((a) => a.name); const keys = Object.keys(inputParameters); From 78b4b6de673ec0008866ed669c67365ade8f217c Mon Sep 17 00:00:00 2001 From: Pascal Vahldiek Date: Fri, 22 Apr 2022 16:40:12 +0200 Subject: [PATCH 38/40] fixed integration test with "root" element with @MarcoDoell --- adapter/integration-test/src/adapter-stateless.test.js | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/adapter/integration-test/src/adapter-stateless.test.js b/adapter/integration-test/src/adapter-stateless.test.js index c58272434..426d2d8fa 100644 --- a/adapter/integration-test/src/adapter-stateless.test.js +++ b/adapter/integration-test/src/adapter-stateless.test.js @@ -162,9 +162,11 @@ describe('Stateless data import', () => { expect(response.status).toEqual(200); const importedData = response.body.data; expect(JSON.parse(importedData)).toEqual({ - from: 'Rick', - to: 'Morty', - test: { hello: 'hello', servus: 'servus' }, + root: { + from: 'Rick', + to: 'Morty', + test: { hello: 'hello', servus: 'servus' }, + }, }); }, TIMEOUT, From bb275ddfde1af8114e10a2c720ac5a6adcde61f2 Mon Sep 17 00:00:00 2001 From: = <=> Date: Sun, 24 Apr 2022 20:45:17 +0200 Subject: [PATCH 39/40] todo raus with @PVahldiek --- adapter/src/adapter/importer/HttpImporter.ts | 1 - 1 file changed, 1 deletion(-) diff --git a/adapter/src/adapter/importer/HttpImporter.ts b/adapter/src/adapter/importer/HttpImporter.ts index 062126c9a..201b61037 100644 --- a/adapter/src/adapter/importer/HttpImporter.ts +++ b/adapter/src/adapter/importer/HttpImporter.ts @@ -20,7 +20,6 @@ export class HttpImporter extends Importer { 'Encoding of the source. Available encodings: ISO-8859-1, US-ASCII, UTF-8', type: 'string', }), - // TODO RuntimeParameters type is probably wrong new ImporterParameterDescription({ name: 'defaultParameters', description: 'Default values for open parameters in the URI', From 50b197395646758cc558e18cd145f0ef0cc6bd2f Mon Sep 17 00:00:00 2001 From: = <=> Date: Sun, 24 Apr 2022 20:53:23 +0200 Subject: [PATCH 40/40] method descriptions for adapterService with @PVahldiek --- adapter/src/adapter/services/adapterService.ts | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/adapter/src/adapter/services/adapterService.ts b/adapter/src/adapter/services/adapterService.ts index eda25a384..bcded2897 100644 --- a/adapter/src/adapter/services/adapterService.ts +++ b/adapter/src/adapter/services/adapterService.ts @@ -29,6 +29,15 @@ export class AdapterService { return [Protocol.HTTP]; } + /** + * Executes an adapter configuration + * + * @param _adapterConfig the adapter configuration + * @return the imported and interpreted data + * @throws ImporterParameterError on errors in the interpreter config (e.g. missing parameters, ...) + * @throws InterpreterParameterError on errors in the interpreter config (e.g. missing parameters, ...) + * @throws Error on response errors when importing the data + */ async executeJob(_adapterConfig: AdapterConfig): Promise { const rawData = await this.executeProtocol(_adapterConfig.protocolConfig); const result = await this.executeFormat( @@ -39,6 +48,14 @@ export class AdapterService { return returnValue; } + /** + * Executes an protocol configuration + * + * @param _protocolConfig the protocol configuration + * @return the imported and interpreted data + * @throws ImporterParameterError on errors in the interpreter config (e.g. missing parameters, ...) + * @throws Error on response errors when importing the data + */ async executeRawJob( _protocolConfig: ProtocolConfig, ): Promise {