diff --git a/autodoc/src/main/java/com/bakdata/conquery/Constants.java b/autodoc/src/main/java/com/bakdata/conquery/Constants.java index 30fc07cd0a..1dc115aacf 100644 --- a/autodoc/src/main/java/com/bakdata/conquery/Constants.java +++ b/autodoc/src/main/java/com/bakdata/conquery/Constants.java @@ -38,7 +38,7 @@ import com.bakdata.conquery.models.config.CSVConfig; import com.bakdata.conquery.models.config.ClusterConfig; import com.bakdata.conquery.models.config.ConqueryConfig; -import com.bakdata.conquery.models.config.DatabaseConfig; +import com.bakdata.conquery.models.config.DatabaseConnection; import com.bakdata.conquery.models.config.Dialect; import com.bakdata.conquery.models.config.FrontendConfig; import com.bakdata.conquery.models.config.LocaleConfig; @@ -129,7 +129,7 @@ public class Constants { .otherClass(MinaConfig.class) .otherClass(FrontendConfig.CurrencyConfig.class) .otherClass(XodusConfig.class) - .otherClasses(List.of(SqlConnectorConfig.class, DatabaseConfig.class, Dialect.class)) + .otherClasses(List.of(SqlConnectorConfig.class, DatabaseConnection.class, Dialect.class)) .hide(Charset.class) .hide(Currency.class) .hide(InetAddress.class) diff --git a/backend/src/main/java/com/bakdata/conquery/commands/ManagerNode.java b/backend/src/main/java/com/bakdata/conquery/commands/ManagerNode.java index 94ac5437b8..ca02070c6d 100644 --- a/backend/src/main/java/com/bakdata/conquery/commands/ManagerNode.java +++ b/backend/src/main/java/com/bakdata/conquery/commands/ManagerNode.java @@ -84,6 +84,10 @@ public void run(Manager manager) throws InterruptedException { // FormScanner needs to be instantiated before plugins are initialized formScanner = new FormScanner(config); + // TODO this could be implemented as a plugin tbh + if(config.getSqlConnectorConfig() != null) { + config.getSqlConnectorConfig().initialize(environment); + } // Init all plugins config.getPlugins().forEach(pluginConfig -> pluginConfig.initialize(this)); diff --git a/backend/src/main/java/com/bakdata/conquery/mode/local/LocalNamespaceHandler.java b/backend/src/main/java/com/bakdata/conquery/mode/local/LocalNamespaceHandler.java index 86e349e5bb..e0e1890d80 100644 --- a/backend/src/main/java/com/bakdata/conquery/mode/local/LocalNamespaceHandler.java +++ b/backend/src/main/java/com/bakdata/conquery/mode/local/LocalNamespaceHandler.java @@ -6,15 +6,13 @@ import com.bakdata.conquery.mode.NamespaceSetupData; import com.bakdata.conquery.mode.cluster.InternalMapperFactory; import com.bakdata.conquery.models.config.ConqueryConfig; -import com.bakdata.conquery.models.config.DatabaseConfig; +import com.bakdata.conquery.models.config.DatabaseConnection; import com.bakdata.conquery.models.config.IdColumnConfig; import com.bakdata.conquery.models.config.SqlConnectorConfig; import com.bakdata.conquery.models.identifiable.ids.specific.DatasetId; import com.bakdata.conquery.models.query.ExecutionManager; import com.bakdata.conquery.models.worker.DatasetRegistry; import com.bakdata.conquery.models.worker.LocalNamespace; -import com.bakdata.conquery.sql.DSLContextWrapper; -import com.bakdata.conquery.sql.DslContextFactory; import com.bakdata.conquery.sql.conquery.SqlExecutionManager; import com.bakdata.conquery.sql.conversion.NodeConversions; import com.bakdata.conquery.sql.conversion.SqlConverter; @@ -37,16 +35,19 @@ public class LocalNamespaceHandler implements NamespaceHandler { private final SqlDialectFactory dialectFactory; @Override - public LocalNamespace createNamespace(NamespaceStorage namespaceStorage, MetaStorage metaStorage, DatasetRegistry datasetRegistry, Environment environment) { + public LocalNamespace createNamespace( + NamespaceStorage namespaceStorage, + MetaStorage metaStorage, + DatasetRegistry datasetRegistry, + Environment environment) { NamespaceSetupData namespaceData = NamespaceHandler.createNamespaceSetup(namespaceStorage, config, internalMapperFactory, datasetRegistry, environment); IdColumnConfig idColumns = config.getIdColumns(); SqlConnectorConfig sqlConnectorConfig = config.getSqlConnectorConfig(); - DatabaseConfig databaseConfig = sqlConnectorConfig.getDatabaseConfig(namespaceStorage.getDataset()); + DatabaseConnection databaseConfig = sqlConnectorConfig.getDatabaseConfig(namespaceStorage.getDataset()); - DSLContextWrapper dslContextWrapper = DslContextFactory.create(databaseConfig, sqlConnectorConfig, environment.healthChecks()); - DSLContext dslContext = dslContextWrapper.getDslContext(); + DSLContext dslContext = databaseConfig.connect(sqlConnectorConfig); SqlDialect sqlDialect = dialectFactory.createSqlDialect(databaseConfig.getDialect()); boolean valid = dslContext.connectionResult(connection -> connection.isValid(1)); @@ -68,8 +69,7 @@ public LocalNamespace createNamespace(NamespaceStorage namespaceStorage, MetaSto namespaceData.preprocessMapper(), namespaceStorage, executionManager, - dslContextWrapper, - sqlStorageHandler, + dslContext, sqlStorageHandler, namespaceData.jobManager(), namespaceData.filterSearch(), sqlEntityResolver diff --git a/backend/src/main/java/com/bakdata/conquery/models/config/DatabaseConfig.java b/backend/src/main/java/com/bakdata/conquery/models/config/DatabaseConfig.java deleted file mode 100644 index d2138d21d2..0000000000 --- a/backend/src/main/java/com/bakdata/conquery/models/config/DatabaseConfig.java +++ /dev/null @@ -1,52 +0,0 @@ -package com.bakdata.conquery.models.config; - -import lombok.AllArgsConstructor; -import lombok.Builder; -import lombok.Data; -import lombok.NoArgsConstructor; -import lombok.ToString; -import lombok.extern.jackson.Jacksonized; - -/** - * Connection properties for a SQL database. - *

- * Currently supported are HANA and Prostgres databases, see {@link DatabaseConfig#dialect}. - */ -@Data -@Builder -@Jacksonized -@NoArgsConstructor -@AllArgsConstructor -public class DatabaseConfig { - - private static final String DEFAULT_PRIMARY_COLUMN = "pid"; - - /** - * SQL vendor specific dialect used to transform queries to SQL - */ - private Dialect dialect; - - /** - * Username used to connect to the database. - */ - private String databaseUsername; - - - /** - * Password used to connect to the database. - */ - @ToString.Exclude - private String databasePassword; - - /** - * Connections url in JDBC notation. - */ - private String jdbcConnectionUrl; - - /** - * Name of the column which is shared among the table and all aggregations are grouped by. - */ - @Builder.Default - private String primaryColumn = DEFAULT_PRIMARY_COLUMN; - -} diff --git a/backend/src/main/java/com/bakdata/conquery/models/config/DatabaseConnection.java b/backend/src/main/java/com/bakdata/conquery/models/config/DatabaseConnection.java new file mode 100644 index 0000000000..ed9092ba39 --- /dev/null +++ b/backend/src/main/java/com/bakdata/conquery/models/config/DatabaseConnection.java @@ -0,0 +1,134 @@ +package com.bakdata.conquery.models.config; + +import java.io.IOException; +import java.sql.SQLException; + +import com.codahale.metrics.health.HealthCheckRegistry; +import com.fasterxml.jackson.annotation.JsonIgnore; +import com.google.common.base.Preconditions; +import com.zaxxer.hikari.HikariConfig; +import com.zaxxer.hikari.HikariDataSource; +import io.dropwizard.lifecycle.Managed; +import io.dropwizard.util.Duration; +import lombok.AllArgsConstructor; +import lombok.Builder; +import lombok.Data; +import lombok.NoArgsConstructor; +import lombok.ToString; +import lombok.extern.jackson.Jacksonized; +import lombok.extern.slf4j.Slf4j; +import org.jooq.DSLContext; +import org.jooq.conf.RenderOptionalKeyword; +import org.jooq.conf.RenderQuotedNames; +import org.jooq.conf.Settings; +import org.jooq.impl.DSL; + +/** + * Connection properties for a SQL database. + *

+ * Currently supported are HANA and Prostgres databases, see {@link DatabaseConnection#dialect}. + */ +@Data +@Builder +@Jacksonized +@NoArgsConstructor +@AllArgsConstructor +@Slf4j +public class DatabaseConnection implements Managed { + + private static final String DEFAULT_PRIMARY_COLUMN = "pid"; + + /** + * SQL vendor specific dialect used to transform queries to SQL + */ + private Dialect dialect; + + /** + * Username used to connect to the database. + */ + private String databaseUsername; + + + /** + * Password used to connect to the database. + */ + @ToString.Exclude + private String databasePassword; + + /** + * Connections url in JDBC notation. + */ + private String jdbcConnectionUrl; + + private Duration connectivityTimeout; + + /** + * Name of the column which is shared among the table and all aggregations are grouped by. + */ + @Builder.Default + private String primaryColumn = DEFAULT_PRIMARY_COLUMN; + + @JsonIgnore + private HikariDataSource dataSource; + + @JsonIgnore + private HealthCheckRegistry healthCheckRegistry; + + @Override + public void start() throws Exception { + initializeDataSource(); + } + + public void initializeDataSource() { + HikariConfig hikariConfig = new HikariConfig(); + hikariConfig.setJdbcUrl(getJdbcConnectionUrl()); + hikariConfig.setUsername(getDatabaseUsername()); + hikariConfig.setPassword(getDatabasePassword()); + + if (healthCheckRegistry != null) { + hikariConfig.setHealthCheckRegistry(healthCheckRegistry); + if (getConnectivityTimeout() != null) { + long connectivityTimeoutMs = getConnectivityTimeout().toMilliseconds(); + hikariConfig.addHealthCheckProperty("connectivityCheckTimeoutMs", Long.toString(connectivityTimeoutMs)); + } + } + + dataSource = new HikariDataSource(hikariConfig); + + try { + log.debug("TEST connecting to {}", getJdbcConnectionUrl()); + if (dataSource.getConnection().isValid(100)) { + log.info("SUCCESS connecting to {}", getJdbcConnectionUrl()); + } + else { + log.error("FAILED connecting to {}. Connection did not become valid.", getJdbcConnectionUrl()); + } + } + catch (SQLException exception) { + log.error("FAILED connecting to {}", getJdbcConnectionUrl(), exception); + } + } + + public DSLContext connect(SqlConnectorConfig connectorConfig) { + Preconditions.checkNotNull(this.dataSource, "dataSource has not been initialized yet."); + + Settings settings = new Settings() + .withRenderFormatted(connectorConfig.isWithPrettyPrinting()) + // enforces all identifiers to be quoted if not explicitly unquoted via DSL.unquotedName() + // to prevent any lowercase/uppercase SQL dialect specific identifier naming issues + .withRenderQuotedNames(RenderQuotedNames.EXPLICIT_DEFAULT_QUOTED) + // always render "as" keyword for field aliases + .withRenderOptionalAsKeywordForFieldAliases(RenderOptionalKeyword.ON); + + return DSL.using( + this.dataSource, + getDialect().getJooqDialect(), + settings + ); + } + + @Override + public void stop() throws IOException { + dataSource.close(); + } +} diff --git a/backend/src/main/java/com/bakdata/conquery/models/config/Dialect.java b/backend/src/main/java/com/bakdata/conquery/models/config/Dialect.java index edd431f90e..fe1ac49877 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/config/Dialect.java +++ b/backend/src/main/java/com/bakdata/conquery/models/config/Dialect.java @@ -1,6 +1,7 @@ package com.bakdata.conquery.models.config; import lombok.Getter; +import lombok.RequiredArgsConstructor; import org.jooq.SQLDialect; /** @@ -8,17 +9,18 @@ *

* There is no fallback dialect, so the dialect must fit the targeted database. */ +@RequiredArgsConstructor @Getter public enum Dialect { /** * Dialect for PostgreSQL database */ - POSTGRESQL(SQLDialect.POSTGRES, 63), + POSTGRESQL(SQLDialect.POSTGRES, 63, "SELECT 1"), /** * Dialect for SAP HANA database */ - HANA(SQLDialect.DEFAULT, 127); + HANA(SQLDialect.DEFAULT, 127, "SELECT 1 FROM DUMMY"); private final SQLDialect jooqDialect; @@ -26,10 +28,5 @@ public enum Dialect { * Set's the max length of database identifiers (column names, qualifiers, etc.). */ private final int nameMaxLength; - - Dialect(SQLDialect jooqDialect, int nameMaxLength) { - this.jooqDialect = jooqDialect; - this.nameMaxLength = nameMaxLength; - } - + private final String testConnection; } diff --git a/backend/src/main/java/com/bakdata/conquery/models/config/SqlConnectorConfig.java b/backend/src/main/java/com/bakdata/conquery/models/config/SqlConnectorConfig.java index ef66c3ca8a..4860dc4967 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/config/SqlConnectorConfig.java +++ b/backend/src/main/java/com/bakdata/conquery/models/config/SqlConnectorConfig.java @@ -4,14 +4,17 @@ import jakarta.validation.Valid; import com.bakdata.conquery.models.datasets.Dataset; +import com.bakdata.conquery.models.identifiable.ids.specific.DatasetId; import com.fasterxml.jackson.annotation.JsonIgnore; -import io.dropwizard.util.Duration; +import io.dropwizard.core.setup.Environment; +import io.dropwizard.lifecycle.Managed; import io.dropwizard.validation.ValidationMethod; import lombok.AllArgsConstructor; import lombok.Builder; import lombok.Data; import lombok.NoArgsConstructor; import lombok.extern.jackson.Jacksonized; +import lombok.extern.slf4j.Slf4j; /** * Configuration for SQL databases to send dataset queries to. @@ -25,7 +28,8 @@ @Jacksonized @NoArgsConstructor @AllArgsConstructor -public class SqlConnectorConfig { +@Slf4j +public class SqlConnectorConfig implements Managed { private boolean enabled; @@ -37,15 +41,23 @@ public class SqlConnectorConfig { /** * Keys must match the name of existing {@link Dataset}s. */ - private Map databaseConfigs; + private Map databaseConfigs; - /** - * Timeout duration after which a database connection is considered unhealthy (defaults to connection timeout) - */ - private Duration connectivityCheckTimeout; - public DatabaseConfig getDatabaseConfig(Dataset dataset) { - return databaseConfigs.get(dataset.getName()); + public DatabaseConnection getDatabaseConfig(Dataset dataset) { + return databaseConfigs.get(dataset.getId()); + } + + + public void initialize(Environment environment) { + if(databaseConfigs == null || !enabled){ + return; + } + + for (DatabaseConnection connection : databaseConfigs.values()) { + connection.setHealthCheckRegistry(environment.healthChecks()); + environment.lifecycle().manage(connection); + } } @JsonIgnore diff --git a/backend/src/main/java/com/bakdata/conquery/models/datasets/Table.java b/backend/src/main/java/com/bakdata/conquery/models/datasets/Table.java index 4a022cd8cb..882558b987 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/datasets/Table.java +++ b/backend/src/main/java/com/bakdata/conquery/models/datasets/Table.java @@ -13,7 +13,7 @@ import com.bakdata.conquery.io.jackson.View; import com.bakdata.conquery.io.storage.NamespacedStorage; import com.bakdata.conquery.mode.ValidationMode; -import com.bakdata.conquery.models.config.DatabaseConfig; +import com.bakdata.conquery.models.config.DatabaseConnection; import com.bakdata.conquery.models.identifiable.LabeledNamespaceIdentifiable; import com.bakdata.conquery.models.identifiable.NamespacedStorageProvider; import com.bakdata.conquery.models.identifiable.ids.specific.DatasetId; @@ -65,7 +65,7 @@ public class Table extends LabeledNamespaceIdentifiable implements Init private Column[] columns = new Column[0]; /** * Defines the primary key/column of this table. Only required for SQL mode. - * If unset {@link DatabaseConfig#getPrimaryColumn()} is assumed. + * If unset {@link DatabaseConnection#getPrimaryColumn()} is assumed. */ @Nullable @JsonManagedReference diff --git a/backend/src/main/java/com/bakdata/conquery/models/worker/LocalNamespace.java b/backend/src/main/java/com/bakdata/conquery/models/worker/LocalNamespace.java index 78c43ecc5c..fb6329ef8b 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/worker/LocalNamespace.java +++ b/backend/src/main/java/com/bakdata/conquery/models/worker/LocalNamespace.java @@ -1,6 +1,5 @@ package com.bakdata.conquery.models.worker; -import java.io.IOException; import java.util.Set; import java.util.stream.Collectors; import java.util.stream.Stream; @@ -11,19 +10,19 @@ import com.bakdata.conquery.models.datasets.Column; import com.bakdata.conquery.models.jobs.JobManager; import com.bakdata.conquery.models.query.ExecutionManager; -import com.bakdata.conquery.sql.DSLContextWrapper; import com.bakdata.conquery.sql.conversion.dialect.SqlDialect; import com.bakdata.conquery.util.search.SearchProcessor; import com.fasterxml.jackson.databind.ObjectMapper; import lombok.Getter; import lombok.extern.slf4j.Slf4j; +import org.jooq.DSLContext; @Getter @Slf4j public class LocalNamespace extends Namespace { private final SqlDialect dialect; - private final DSLContextWrapper dslContextWrapper; + private final DSLContext dslContext; private final SqlStorageHandler storageHandler; public LocalNamespace( @@ -31,14 +30,14 @@ public LocalNamespace( ObjectMapper preprocessMapper, NamespaceStorage storage, ExecutionManager executionManager, - DSLContextWrapper dslContextWrapper, + DSLContext dslContext, SqlStorageHandler storageHandler, JobManager jobManager, SearchProcessor filterSearch, SqlEntityResolver sqlEntityResolver ) { super(preprocessMapper, storage, executionManager, jobManager, filterSearch, sqlEntityResolver); - this.dslContextWrapper = dslContextWrapper; + this.dslContext = dslContext; this.storageHandler = storageHandler; this.dialect = dialect; } @@ -63,22 +62,13 @@ void registerColumnValuesInSearch(Set columns) { @Override public void close() { - closeDslContextWrapper(); + //TODO do we even need to shutdown the connection? super.close(); } - private void closeDslContextWrapper() { - try { - dslContextWrapper.close(); - } - catch (IOException e) { - log.warn("Could not close namespace's {} DSLContext/Datasource directly", getDataset().getId(), e); - } - } - @Override public void remove() { - closeDslContextWrapper(); + super.remove(); } diff --git a/backend/src/main/java/com/bakdata/conquery/sql/DSLContextWrapper.java b/backend/src/main/java/com/bakdata/conquery/sql/DSLContextWrapper.java deleted file mode 100644 index cf031678ad..0000000000 --- a/backend/src/main/java/com/bakdata/conquery/sql/DSLContextWrapper.java +++ /dev/null @@ -1,29 +0,0 @@ -package com.bakdata.conquery.sql; - -import java.io.Closeable; -import java.io.IOException; - -import com.zaxxer.hikari.HikariDataSource; -import lombok.Getter; -import lombok.RequiredArgsConstructor; -import org.jooq.DSLContext; - -/** - * Provides access to a configured {@link DSLContext} and enables closing the underlying connection pool. - */ -@RequiredArgsConstructor -public class DSLContextWrapper implements Closeable { - - @Getter - private final DSLContext dslContext; - - private final HikariDataSource dataSource; - - @Override - public void close() throws IOException { - // Hikari opens a connection pool under the hood which we won't be able to close after passing it to the DSLContext. - // That's why we keep the HikariDataSource reference. - dataSource.close(); - } - -} diff --git a/backend/src/main/java/com/bakdata/conquery/sql/DslContextFactory.java b/backend/src/main/java/com/bakdata/conquery/sql/DslContextFactory.java deleted file mode 100644 index 59f13756c6..0000000000 --- a/backend/src/main/java/com/bakdata/conquery/sql/DslContextFactory.java +++ /dev/null @@ -1,52 +0,0 @@ -package com.bakdata.conquery.sql; - -import javax.annotation.Nullable; - -import com.bakdata.conquery.models.config.DatabaseConfig; -import com.bakdata.conquery.models.config.SqlConnectorConfig; -import com.codahale.metrics.health.HealthCheckRegistry; -import com.zaxxer.hikari.HikariConfig; -import com.zaxxer.hikari.HikariDataSource; -import org.jooq.DSLContext; -import org.jooq.conf.RenderOptionalKeyword; -import org.jooq.conf.RenderQuotedNames; -import org.jooq.conf.Settings; -import org.jooq.impl.DSL; - -public class DslContextFactory { - - public static DSLContextWrapper create(DatabaseConfig config, SqlConnectorConfig connectorConfig, @Nullable HealthCheckRegistry healthCheckRegistry) { - - HikariConfig hikariConfig = new HikariConfig(); - hikariConfig.setJdbcUrl(config.getJdbcConnectionUrl()); - hikariConfig.setUsername(config.getDatabaseUsername()); - hikariConfig.setPassword(config.getDatabasePassword()); - - if (healthCheckRegistry != null) { - hikariConfig.setHealthCheckRegistry(healthCheckRegistry); - if (connectorConfig.getConnectivityCheckTimeout() != null) { - long connectivityTimeoutMs = connectorConfig.getConnectivityCheckTimeout().toMilliseconds(); - hikariConfig.addHealthCheckProperty("connectivityCheckTimeoutMs", Long.toString(connectivityTimeoutMs)); - } - } - - HikariDataSource hikariDataSource = new HikariDataSource(hikariConfig); - - Settings settings = new Settings() - .withRenderFormatted(connectorConfig.isWithPrettyPrinting()) - // enforces all identifiers to be quoted if not explicitly unquoted via DSL.unquotedName() - // to prevent any lowercase/uppercase SQL dialect specific identifier naming issues - .withRenderQuotedNames(RenderQuotedNames.EXPLICIT_DEFAULT_QUOTED) - // always render "as" keyword for field aliases - .withRenderOptionalAsKeywordForFieldAliases(RenderOptionalKeyword.ON); - - DSLContext dslContext = DSL.using( - hikariDataSource, - config.getDialect().getJooqDialect(), - settings - ); - - return new DSLContextWrapper(dslContext, hikariDataSource); - } - -} diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/NodeConversions.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/NodeConversions.java index be5564974b..158871efdd 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/NodeConversions.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/NodeConversions.java @@ -4,7 +4,7 @@ import com.bakdata.conquery.apiv1.query.QueryDescription; import com.bakdata.conquery.models.config.ConqueryConfig; -import com.bakdata.conquery.models.config.DatabaseConfig; +import com.bakdata.conquery.models.config.DatabaseConnection; import com.bakdata.conquery.models.config.IdColumnConfig; import com.bakdata.conquery.models.query.PrintSettings; import com.bakdata.conquery.models.query.Visitable; @@ -22,7 +22,7 @@ public class NodeConversions extends Conversions> List customize(List defaults, List substitutes) { + Map, C> substituteMap = getSubstituteMap(substitutes); + return defaults.stream() + .map(converter -> substituteMap.getOrDefault(converter.getConversionClass(), converter)) + .toList(); + } + + private static > Map, C> getSubstituteMap(List substitutes) { + return substitutes.stream() + .collect(Collectors.toMap( + Converter::getConversionClass, + Function.identity() + )); + } + + boolean isTypeCompatible(Field field, MajorTypeId type); SqlFunctionProvider getFunctionProvider(); @@ -78,19 +95,4 @@ default List> getDefaultNodeConverters(DSLCon ); } - private static > List customize(List defaults, List substitutes) { - Map, C> substituteMap = getSubstituteMap(substitutes); - return defaults.stream() - .map(converter -> substituteMap.getOrDefault(converter.getConversionClass(), converter)) - .toList(); - } - - private static > Map, C> getSubstituteMap(List substitutes) { - return substitutes.stream() - .collect(Collectors.toMap( - Converter::getConversionClass, - Function.identity() - )); - } - } diff --git a/backend/src/main/java/com/bakdata/conquery/util/TablePrimaryColumnUtil.java b/backend/src/main/java/com/bakdata/conquery/util/TablePrimaryColumnUtil.java index 685bdf254f..06af1087e3 100644 --- a/backend/src/main/java/com/bakdata/conquery/util/TablePrimaryColumnUtil.java +++ b/backend/src/main/java/com/bakdata/conquery/util/TablePrimaryColumnUtil.java @@ -1,13 +1,13 @@ package com.bakdata.conquery.util; -import com.bakdata.conquery.models.config.DatabaseConfig; +import com.bakdata.conquery.models.config.DatabaseConnection; import com.bakdata.conquery.models.datasets.Table; import org.jooq.Field; import org.jooq.impl.DSL; public class TablePrimaryColumnUtil { - public static Field findPrimaryColumn(Table table, DatabaseConfig databaseConfig) { + public static Field findPrimaryColumn(Table table, DatabaseConnection databaseConfig) { String primaryColumnName = table.getPrimaryColumn() == null ? databaseConfig.getPrimaryColumn() : table.getPrimaryColumn().getName(); diff --git a/backend/src/main/java/com/bakdata/conquery/util/validation/SqlTableValidator.java b/backend/src/main/java/com/bakdata/conquery/util/validation/SqlTableValidator.java index db15e56fb1..a12812d81e 100644 --- a/backend/src/main/java/com/bakdata/conquery/util/validation/SqlTableValidator.java +++ b/backend/src/main/java/com/bakdata/conquery/util/validation/SqlTableValidator.java @@ -29,7 +29,7 @@ public boolean isValid(Table value, ConstraintValidatorContext context) { final Stopwatch stopwatch = Stopwatch.createStarted(); final LocalNamespace localNamespace = (LocalNamespace) value.getNamespace(); - final DSLContext dslContext = localNamespace.getDslContextWrapper().getDslContext(); + final DSLContext dslContext = localNamespace.getDslContext(); final SqlDialect dialect = localNamespace.getDialect(); final Result result; diff --git a/backend/src/test/java/com/bakdata/conquery/integration/IntegrationTests.java b/backend/src/test/java/com/bakdata/conquery/integration/IntegrationTests.java index e9a1c038cc..8cdd3f96e6 100644 --- a/backend/src/test/java/com/bakdata/conquery/integration/IntegrationTests.java +++ b/backend/src/test/java/com/bakdata/conquery/integration/IntegrationTests.java @@ -4,7 +4,6 @@ import java.io.File; import java.io.IOException; -import java.io.InputStream; import java.net.URI; import java.nio.file.Files; import java.util.ArrayList; @@ -30,7 +29,7 @@ import com.bakdata.conquery.io.jackson.Jackson; import com.bakdata.conquery.io.jackson.View; import com.bakdata.conquery.models.config.ConqueryConfig; -import com.bakdata.conquery.models.config.DatabaseConfig; +import com.bakdata.conquery.models.config.DatabaseConnection; import com.bakdata.conquery.models.config.Dialect; import com.bakdata.conquery.models.config.SqlConnectorConfig; import com.bakdata.conquery.models.config.XodusStoreFactory; @@ -118,7 +117,7 @@ public List jsonTests() { } @SneakyThrows - public Stream sqlProgrammaticTests(DatabaseConfig databaseConfig, TestSqlConnectorConfig sqlConfig, TestDataImporter testDataImporter) { + public Stream sqlProgrammaticTests(DatabaseConnection databaseConfig, TestSqlConnectorConfig sqlConfig, TestDataImporter testDataImporter) { this.config.setSqlConnectorConfig(sqlConfig); return programmaticTests(testDataImporter, StandaloneSupport.Mode.SQL); } @@ -169,7 +168,7 @@ private DynamicTest createDynamicProgrammaticTestNode(ProgrammaticIntegrationTes } @SneakyThrows - public List sqlQueryTests(DatabaseConfig databaseConfig, TestSqlConnectorConfig sqlConfig, TestDataImporter testDataImporter) { + public List sqlQueryTests(DatabaseConnection databaseConfig, TestSqlConnectorConfig sqlConfig, TestDataImporter testDataImporter) { this.config.setSqlConnectorConfig(sqlConfig); final String testRoot = Objects.requireNonNullElse(System.getenv(TestTags.SQL_BACKEND_TEST_DIRECTORY_ENVIRONMENT_VARIABLE), defaultTestRoot); ResourceTree tree = scanForResources(testRoot, SQL_TEST_PATTERN); diff --git a/backend/src/test/java/com/bakdata/conquery/integration/sql/CsvTableImporter.java b/backend/src/test/java/com/bakdata/conquery/integration/sql/CsvTableImporter.java index 8c3ab27b3d..f0f5d0cb63 100644 --- a/backend/src/test/java/com/bakdata/conquery/integration/sql/CsvTableImporter.java +++ b/backend/src/test/java/com/bakdata/conquery/integration/sql/CsvTableImporter.java @@ -17,7 +17,7 @@ import com.bakdata.conquery.models.common.daterange.CDateRange; import com.bakdata.conquery.models.config.CSVConfig; import com.bakdata.conquery.models.config.ConqueryConfig; -import com.bakdata.conquery.models.config.DatabaseConfig; +import com.bakdata.conquery.models.config.DatabaseConnection; import com.bakdata.conquery.models.events.MajorTypeId; import com.bakdata.conquery.models.preproc.parser.specific.DateRangeParser; import com.google.common.base.Strings; @@ -43,9 +43,9 @@ public class CsvTableImporter { private final DateRangeParser dateRangeParser; private final CsvParser csvReader; private final TestSqlDialect testSqlDialect; - private final DatabaseConfig databaseConfig; + private final DatabaseConnection databaseConfig; - public CsvTableImporter(DSLContext dslContext, TestSqlDialect testSqlDialect, DatabaseConfig databaseConfig) { + public CsvTableImporter(DSLContext dslContext, TestSqlDialect testSqlDialect, DatabaseConnection databaseConfig) { this.dslContext = dslContext; this.dateRangeParser = new DateRangeParser(new ConqueryConfig()); this.csvReader = new CSVConfig().withSkipHeader(true).createParser(); diff --git a/backend/src/test/java/com/bakdata/conquery/integration/sql/dialect/HanaSqlIntegrationTests.java b/backend/src/test/java/com/bakdata/conquery/integration/sql/dialect/HanaSqlIntegrationTests.java index 553f0e681c..4b1b9a992e 100644 --- a/backend/src/test/java/com/bakdata/conquery/integration/sql/dialect/HanaSqlIntegrationTests.java +++ b/backend/src/test/java/com/bakdata/conquery/integration/sql/dialect/HanaSqlIntegrationTests.java @@ -21,10 +21,8 @@ import com.bakdata.conquery.integration.json.TestDataImporter; import com.bakdata.conquery.integration.sql.CsvTableImporter; import com.bakdata.conquery.integration.sql.testcontainer.hana.HanaContainer; -import com.bakdata.conquery.models.config.DatabaseConfig; +import com.bakdata.conquery.models.config.DatabaseConnection; import com.bakdata.conquery.models.config.Dialect; -import com.bakdata.conquery.sql.DSLContextWrapper; -import com.bakdata.conquery.sql.DslContextFactory; import com.bakdata.conquery.sql.conversion.dialect.HanaSqlDialect; import com.bakdata.conquery.sql.conversion.supplier.DateNowSupplier; import com.google.common.base.Strings; @@ -52,7 +50,7 @@ public class HanaSqlIntegrationTests extends IntegrationTests { private final static DockerImageName HANA_IMAGE = DockerImageName.parse("saplabs/hanaexpress:latest"); private static final Path TMP_HANA_MOUNT_DIR = Paths.get("/tmp/data/hana"); private static boolean useLocalHanaDb = true; - private static DSLContextWrapper dslContextWrapper; + private static DSLContext dslContext; static { final String USE_LOCAL_HANA_DB = System.getenv("USE_LOCAL_HANA_DB"); @@ -84,9 +82,6 @@ public static void prepareTmpHanaDir() { @SneakyThrows @AfterAll public static void tearDownClass() { - - dslContextWrapper.close(); - if (!Files.exists(TMP_HANA_MOUNT_DIR)) { return; } @@ -105,13 +100,13 @@ public Stream sqlBackendTests() { ? new HanaTestcontainerContextProvider() : new RemoteHanaContextProvider(); - log.info("Running HANA tests with %s.".formatted(provider.getClass().getSimpleName())); + log.info("Running HANA tests with {}.", provider.getClass().getSimpleName()); - dslContextWrapper = provider.getDslContextWrapper(); - DatabaseConfig databaseConfig = provider.getDatabaseConfig(); + dslContext = provider.getDslContext(); + DatabaseConnection databaseConfig = provider.getDatabaseConfig(); TestSqlConnectorConfig config = provider.getSqlConnectorConfig(); TestHanaDialect testHanaDialect = new TestHanaDialect(); - TestDataImporter testDataImporter = new SqlTestDataImporter(new CsvTableImporter(dslContextWrapper.getDslContext(), testHanaDialect, databaseConfig)); + TestDataImporter testDataImporter = new SqlTestDataImporter(new CsvTableImporter(dslContext, testHanaDialect, databaseConfig)); return Stream.concat( super.sqlProgrammaticTests(databaseConfig, config, testDataImporter), @@ -159,8 +154,8 @@ public String createDropTableStatement(Table table, DSLContext dslContex @Getter private static class HanaTestcontainerContextProvider implements TestContextProvider { - private final DSLContextWrapper dslContextWrapper; - private final DatabaseConfig databaseConfig; + private final DSLContext dslContext; + private final DatabaseConnection databaseConfig; private final TestSqlConnectorConfig sqlConnectorConfig; @Container @@ -170,14 +165,15 @@ public HanaTestcontainerContextProvider() { this.hanaContainer = new HanaContainer<>(HANA_IMAGE) .withFileSystemBind(TMP_HANA_MOUNT_DIR.toString(), "/home/secrets"); this.hanaContainer.start(); - this.databaseConfig = DatabaseConfig.builder() - .dialect(Dialect.HANA) - .jdbcConnectionUrl(hanaContainer.getJdbcUrl()) - .databaseUsername(hanaContainer.getUsername()) - .databasePassword(hanaContainer.getPassword()) - .build(); + this.databaseConfig = DatabaseConnection.builder() + .dialect(Dialect.HANA) + .jdbcConnectionUrl(hanaContainer.getJdbcUrl()) + .databaseUsername(hanaContainer.getUsername()) + .databasePassword(hanaContainer.getPassword()) + .build(); + this.databaseConfig.initializeDataSource(); this.sqlConnectorConfig = new TestSqlConnectorConfig(databaseConfig); - this.dslContextWrapper = DslContextFactory.create(this.databaseConfig, sqlConnectorConfig, null); + this.dslContext = this.databaseConfig.connect(sqlConnectorConfig); } } @@ -190,19 +186,19 @@ private static class RemoteHanaContextProvider implements TestContextProvider { private final static String CONNECTION_URL = "jdbc:sap://%s:%s/databaseName=HXE&encrypt=true&validateCertificate=false".formatted(HOST, PORT); private final static String USERNAME = Objects.requireNonNullElse(System.getenv("CONQUERY_SQL_USER"), "SYSTEM"); private final static String PASSWORD = System.getenv("CONQUERY_SQL_PASSWORD"); - private final DSLContextWrapper dslContextWrapper; - private final DatabaseConfig databaseConfig; + private final DSLContext dslContext; + private final DatabaseConnection databaseConfig; private final TestSqlConnectorConfig sqlConnectorConfig; public RemoteHanaContextProvider() { - this.databaseConfig = DatabaseConfig.builder() - .dialect(Dialect.HANA) - .jdbcConnectionUrl(CONNECTION_URL) - .databaseUsername(USERNAME) - .databasePassword(PASSWORD) - .build(); + this.databaseConfig = DatabaseConnection.builder() + .dialect(Dialect.HANA) + .jdbcConnectionUrl(CONNECTION_URL) + .databaseUsername(USERNAME) + .databasePassword(PASSWORD) + .build(); this.sqlConnectorConfig = new TestSqlConnectorConfig(databaseConfig); - this.dslContextWrapper = DslContextFactory.create(databaseConfig, sqlConnectorConfig, null); + this.dslContext = databaseConfig.connect(sqlConnectorConfig); } } diff --git a/backend/src/test/java/com/bakdata/conquery/integration/sql/dialect/PostgreSqlIntegrationTests.java b/backend/src/test/java/com/bakdata/conquery/integration/sql/dialect/PostgreSqlIntegrationTests.java index 33d6e28105..358bf84f2e 100644 --- a/backend/src/test/java/com/bakdata/conquery/integration/sql/dialect/PostgreSqlIntegrationTests.java +++ b/backend/src/test/java/com/bakdata/conquery/integration/sql/dialect/PostgreSqlIntegrationTests.java @@ -9,12 +9,10 @@ import com.bakdata.conquery.integration.IntegrationTests; import com.bakdata.conquery.integration.json.SqlTestDataImporter; import com.bakdata.conquery.integration.sql.CsvTableImporter; -import com.bakdata.conquery.models.config.DatabaseConfig; +import com.bakdata.conquery.models.config.DatabaseConnection; import com.bakdata.conquery.models.config.Dialect; import com.bakdata.conquery.models.error.ConqueryError; import com.bakdata.conquery.models.i18n.I18n; -import com.bakdata.conquery.sql.DSLContextWrapper; -import com.bakdata.conquery.sql.DslContextFactory; import com.bakdata.conquery.sql.conversion.dialect.PostgreSqlDialect; import com.bakdata.conquery.sql.conversion.model.SqlQuery; import com.bakdata.conquery.sql.conversion.supplier.DateNowSupplier; @@ -25,9 +23,11 @@ import lombok.Getter; import lombok.extern.slf4j.Slf4j; import org.assertj.core.api.Assertions; +import org.jooq.DSLContext; import org.junit.jupiter.api.AfterAll; import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.DynamicNode; +import org.junit.jupiter.api.Order; import org.junit.jupiter.api.Tag; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.TestFactory; @@ -45,8 +45,8 @@ public class PostgreSqlIntegrationTests extends IntegrationTests { private static final String USERNAME = "user"; private static final String PASSWORD = "pass"; - private static DSLContextWrapper dslContextWrapper; - private static DatabaseConfig databaseConfig; + private static DSLContext dslContext; + private static DatabaseConnection databaseConfig; private static TestSqlConnectorConfig sqlConfig; private static TestSqlDialect testSqlDialect; private static SqlTestDataImporter testDataImporter; @@ -71,25 +71,28 @@ static void before() { : new RemotePostgresContextProvider(); databaseConfig = provider.getDatabaseConfig(); + databaseConfig.initializeDataSource(); + sqlConfig = new TestSqlConnectorConfig(databaseConfig); - dslContextWrapper = DslContextFactory.create(databaseConfig, sqlConfig, null); + dslContext = databaseConfig.connect(sqlConfig); testSqlDialect = new TestPostgreSqlDialect(); - testDataImporter = new SqlTestDataImporter(new CsvTableImporter(dslContextWrapper.getDslContext(), testSqlDialect, databaseConfig)); + testDataImporter = new SqlTestDataImporter(new CsvTableImporter(dslContext, testSqlDialect, databaseConfig)); } @AfterAll static void after() throws IOException { - dslContextWrapper.close(); + databaseConfig.stop(); } @Test @Tag(TestTags.INTEGRATION_SQL_BACKEND) + @Order(0) public void shouldThrowException() { // This can be removed as soon as we switch to a full integration test including the REST API I18n.init(); ResultSetProcessor resultSetProcessor = ResultSetProcessorFactory.create(config, testSqlDialect); - SqlExecutionService executionService = new SqlExecutionService(dslContextWrapper.getDslContext(), resultSetProcessor); - SqlQuery validQuery = new TestSqlQuery("SELECT 1"); + SqlExecutionService executionService = new SqlExecutionService(dslContext, resultSetProcessor); + SqlQuery validQuery = new TestSqlQuery(Dialect.POSTGRESQL.getTestConnection()); Assertions.assertThatNoException().isThrownBy(() -> executionService.execute(validQuery)); // executing an empty query should throw an SQL error @@ -101,6 +104,7 @@ public void shouldThrowException() { @TestFactory @Tag(TestTags.INTEGRATION_SQL_BACKEND) + @Order(1) public Stream sqlBackendTests() { return Stream.concat( super.sqlProgrammaticTests(databaseConfig, sqlConfig, testDataImporter), @@ -136,8 +140,8 @@ protected TestSqlQuery(String sql) { @Getter private static class PortgresTestcontainerContextProvider implements TestContextProvider { - private final DSLContextWrapper dslContextWrapper; - private final DatabaseConfig databaseConfig; + private final DSLContext dslContext; + private final DatabaseConnection databaseConfig; private final TestSqlConnectorConfig sqlConnectorConfig; @Container @@ -149,15 +153,16 @@ public PortgresTestcontainerContextProvider() { .withUsername(USERNAME) .withPassword(PASSWORD); this.postgreSQLContainer.start(); - this.databaseConfig = DatabaseConfig.builder() - .dialect(Dialect.POSTGRESQL) - .jdbcConnectionUrl(postgreSQLContainer.getJdbcUrl()) - .databaseUsername(USERNAME) - .databasePassword(PASSWORD) - .build(); + this.databaseConfig = DatabaseConnection.builder() + .dialect(Dialect.POSTGRESQL) + .jdbcConnectionUrl(postgreSQLContainer.getJdbcUrl()) + .databaseUsername(USERNAME) + .databasePassword(PASSWORD) + .build(); + this.databaseConfig.initializeDataSource(); this.sqlConnectorConfig = new TestSqlConnectorConfig(databaseConfig); - this.dslContextWrapper = DslContextFactory.create(this.databaseConfig, sqlConnectorConfig, null); + this.dslContext = this.databaseConfig.connect(sqlConnectorConfig); } } @@ -172,19 +177,21 @@ private static class RemotePostgresContextProvider implements TestContextProvide private final static String CONNECTION_URL = "jdbc:postgresql://%s:%s/%s".formatted(HOST, PORT, DATABASE); private final static String USERNAME = Objects.requireNonNullElse(System.getenv("CONQUERY_SQL_USER"), PostgreSqlIntegrationTests.USERNAME); private final static String PASSWORD = Objects.requireNonNullElse(System.getenv("CONQUERY_SQL_PASSWORD"), PostgreSqlIntegrationTests.PASSWORD); - private final DSLContextWrapper dslContextWrapper; - private final DatabaseConfig databaseConfig; + private final DSLContext dslContext; + private final DatabaseConnection databaseConfig; private final TestSqlConnectorConfig sqlConnectorConfig; public RemotePostgresContextProvider() { - this.databaseConfig = DatabaseConfig.builder() - .dialect(Dialect.POSTGRESQL) - .jdbcConnectionUrl(CONNECTION_URL) - .databaseUsername(USERNAME) - .databasePassword(PASSWORD) - .build(); + this.databaseConfig = DatabaseConnection.builder() + .dialect(Dialect.POSTGRESQL) + .jdbcConnectionUrl(CONNECTION_URL) + .databaseUsername(USERNAME) + .databasePassword(PASSWORD) + .build(); + this.databaseConfig.initializeDataSource(); + this.sqlConnectorConfig = new TestSqlConnectorConfig(databaseConfig); - this.dslContextWrapper = DslContextFactory.create(databaseConfig, sqlConnectorConfig, null); + this.dslContext = databaseConfig.connect(sqlConnectorConfig); } } diff --git a/backend/src/test/java/com/bakdata/conquery/integration/sql/dialect/TestContextProvider.java b/backend/src/test/java/com/bakdata/conquery/integration/sql/dialect/TestContextProvider.java index d5cf8eb140..887c146c67 100644 --- a/backend/src/test/java/com/bakdata/conquery/integration/sql/dialect/TestContextProvider.java +++ b/backend/src/test/java/com/bakdata/conquery/integration/sql/dialect/TestContextProvider.java @@ -1,14 +1,14 @@ package com.bakdata.conquery.integration.sql.dialect; -import com.bakdata.conquery.models.config.DatabaseConfig; -import com.bakdata.conquery.sql.DSLContextWrapper; +import com.bakdata.conquery.models.config.DatabaseConnection; +import org.jooq.DSLContext; public interface TestContextProvider { - DatabaseConfig getDatabaseConfig(); + DatabaseConnection getDatabaseConfig(); TestSqlConnectorConfig getSqlConnectorConfig(); - DSLContextWrapper getDslContextWrapper(); + DSLContext getDslContext(); } diff --git a/backend/src/test/java/com/bakdata/conquery/integration/sql/dialect/TestSqlConnectorConfig.java b/backend/src/test/java/com/bakdata/conquery/integration/sql/dialect/TestSqlConnectorConfig.java index bf3c1a5325..fbdb97e32f 100644 --- a/backend/src/test/java/com/bakdata/conquery/integration/sql/dialect/TestSqlConnectorConfig.java +++ b/backend/src/test/java/com/bakdata/conquery/integration/sql/dialect/TestSqlConnectorConfig.java @@ -2,9 +2,10 @@ import java.util.Map; -import com.bakdata.conquery.models.config.DatabaseConfig; +import com.bakdata.conquery.models.config.DatabaseConnection; import com.bakdata.conquery.models.config.SqlConnectorConfig; import com.bakdata.conquery.models.datasets.Dataset; +import com.bakdata.conquery.models.identifiable.ids.specific.DatasetId; import com.fasterxml.jackson.databind.annotation.JsonDeserialize; import lombok.NoArgsConstructor; @@ -14,13 +15,13 @@ public class TestSqlConnectorConfig extends SqlConnectorConfig { private static final String TEST_DATASET = "test"; - public TestSqlConnectorConfig(DatabaseConfig databaseConfig) { - super(true, true, Map.of(TEST_DATASET, databaseConfig), null); + public TestSqlConnectorConfig(DatabaseConnection databaseConfig) { + super(true, true, Map.of(new DatasetId(TEST_DATASET), databaseConfig)); } @Override - public DatabaseConfig getDatabaseConfig(Dataset dataset) { - return getDatabaseConfigs().get(TEST_DATASET); + public DatabaseConnection getDatabaseConfig(Dataset dataset) { + return getDatabaseConfigs().get(new DatasetId(TEST_DATASET)); } } diff --git a/backend/src/test/java/com/bakdata/conquery/integration/sql/testcontainer/hana/HanaContainer.java b/backend/src/test/java/com/bakdata/conquery/integration/sql/testcontainer/hana/HanaContainer.java index b60a476ffe..9f9e69b7b0 100644 --- a/backend/src/test/java/com/bakdata/conquery/integration/sql/testcontainer/hana/HanaContainer.java +++ b/backend/src/test/java/com/bakdata/conquery/integration/sql/testcontainer/hana/HanaContainer.java @@ -2,6 +2,7 @@ import java.time.Duration; +import com.bakdata.conquery.models.config.Dialect; import org.testcontainers.containers.JdbcDatabaseContainer; import org.testcontainers.containers.wait.strategy.LogMessageWaitStrategy; import org.testcontainers.utility.DockerImageName; @@ -55,7 +56,7 @@ public String getDatabaseName() { @Override protected String getTestQueryString() { - return "SELECT 1"; + return Dialect.HANA.getTestConnection(); } @Override