diff --git a/morf-core/src/main/java/org/alfasoftware/morf/jdbc/DatabaseMetaDataProvider.java b/morf-core/src/main/java/org/alfasoftware/morf/jdbc/DatabaseMetaDataProvider.java index 1da493303..c75b44572 100755 --- a/morf-core/src/main/java/org/alfasoftware/morf/jdbc/DatabaseMetaDataProvider.java +++ b/morf-core/src/main/java/org/alfasoftware/morf/jdbc/DatabaseMetaDataProvider.java @@ -32,6 +32,7 @@ import java.util.LinkedHashMap; import java.util.List; import java.util.Map; +import java.util.Set; import java.util.stream.Collectors; import java.util.stream.IntStream; @@ -56,6 +57,7 @@ import com.google.common.cache.LoadingCache; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; +import com.google.common.collect.ImmutableSet; import com.google.common.collect.Maps; /** @@ -116,6 +118,8 @@ public abstract class DatabaseMetaDataProvider implements Schema { private final LoadingCache sequenceCache = CacheBuilder.newBuilder().build(CacheLoader.from(this::loadSequence)); private final Supplier> databaseInformation = Suppliers.memoize(this::loadDatabaseInformation); + protected Supplier> ignoredPartitionTables = Suppliers.memoize(this::loadIgnoredPartitionTables); + protected Supplier> partitionedTables = Suppliers.memoize(this::loadPartitionedTables); /** * @param connection The database connection from which meta data should be provided. @@ -148,6 +152,9 @@ private Map loadDatabaseInformation() { } } + protected Set loadIgnoredPartitionTables() { return ImmutableSet.of(); } + + protected Set loadPartitionedTables() { return ImmutableSet.of(); } /** * @see org.alfasoftware.morf.metadata.Schema#isEmptyDatabase() @@ -306,6 +313,10 @@ protected Map loadAllTableNames() { throw new RuntimeSqlException("Error reading metadata for table ["+tableName+"]", e); } } + // add partitioned tables to list + partitionedTables.get().forEach(table -> { + tableNameMappings.put(table, table); + }); long end = System.currentTimeMillis(); Map tableNameMap = tableNameMappings.build(); @@ -389,8 +400,8 @@ protected boolean isSystemSequence(@SuppressWarnings("unused") RealName sequence * @param tableName The table which we are accessing. * @return true if the table should be ignored, false otherwise. */ - protected boolean isIgnoredTable(@SuppressWarnings("unused") RealName tableName) { - return false; + protected boolean isIgnoredTable(RealName tableName) { + return ignoredPartitionTables.get().contains(tableName); } @@ -1129,22 +1140,16 @@ protected static AName named(String name) { protected void runSQL(String sql, String schemaName, ResultSetHandler handler) { if (log.isTraceEnabled()) log.trace("runSQL: " + sql); try { - PreparedStatement statement = connection.prepareStatement(sql); - try { + try (PreparedStatement statement = connection.prepareStatement(sql)) { // pass through the schema name if (schemaName != null && !schemaName.isBlank()) { statement.setString(1, schemaName); } - ResultSet resultSet = statement.executeQuery(); - try { + try (ResultSet resultSet = statement.executeQuery()) { handler.handle(resultSet); - } finally { - resultSet.close(); } - } finally { - statement.close(); } } catch (SQLException sqle) { throw new RuntimeSqlException("Error running SQL: " + sql, sqle); @@ -1196,7 +1201,7 @@ protected interface ResultSetHandler { * @return {@link RealName} instance holding the two name versions. * Can also be used as a key in the lookup maps, like {@link AName}. */ - protected static RealName createRealName(String dbName, String realName) { + public static RealName createRealName(String dbName, String realName) { return new RealName(dbName, realName); } @@ -1210,7 +1215,7 @@ protected static RealName createRealName(String dbName, String realName) { * see {@link DatabaseMetaDataProvider#named(String)} * and {@link DatabaseMetaDataProvider#createRealName(String, String)} */ - protected static class AName { + public static class AName { private final String aName; private final int hashCode; @@ -1255,11 +1260,11 @@ public final boolean equals(Object obj) { // final intentional! * see {@link DatabaseMetaDataProvider#named(String)} * and {@link DatabaseMetaDataProvider#createRealName(String, String)} */ - protected static final class RealName extends AName { + public static final class RealName extends AName { private final String realName; - private RealName(String dbName, String realName) { + public RealName(String dbName, String realName) { super(dbName); this.realName = realName; } diff --git a/morf-core/src/main/java/org/alfasoftware/morf/metadata/AdditionalMetadata.java b/morf-core/src/main/java/org/alfasoftware/morf/metadata/AdditionalMetadata.java index 916d13f42..5a88fcdee 100644 --- a/morf-core/src/main/java/org/alfasoftware/morf/metadata/AdditionalMetadata.java +++ b/morf-core/src/main/java/org/alfasoftware/morf/metadata/AdditionalMetadata.java @@ -2,7 +2,9 @@ import java.util.List; import java.util.Map; +import java.util.Set; +import org.alfasoftware.morf.jdbc.DatabaseMetaDataProvider; import org.apache.commons.lang3.NotImplementedException; /** @@ -18,6 +20,26 @@ default Map primaryKeyIndexNames() { throw new NotImplementedException("Not implemented yet."); } + /** + * Provides the names of all partitioned tables in the database. This applies for now for postgres. Note that the order of + * the tables in the result is not specified. The case of the + * table names may be preserved when logging progress, but should not be relied on for schema + * processing. A partitioned table is a table that has partitions. + * + * @return A collection of all partitioned table names available in the database. + */ + default Set partitionedTableNames() { throw new NotImplementedException("Not implemented yet."); } + + /** + * Provides the names of all partition tables in the database. This applies for now for postgres. Note that the order of + * the tables in the result is not specified. The case of the + * table names may be preserved when logging progress, but should not be relied on for schema + * processing. A partition table is a table that is a partition of a partitioned table. + * + * @return A collection of all partition table names available in the database. + */ + default Set partitionTableNames() { throw new NotImplementedException("Not implemented yet."); } + default Map> ignoredIndexes() { return Map.of(); } diff --git a/morf-h2/src/test/java/org/alfasoftware/morf/jdbc/h2/TestH2MetaDataProvider.java b/morf-h2/src/test/java/org/alfasoftware/morf/jdbc/h2/TestH2MetaDataProvider.java index ea8890ce8..74da6f61d 100644 --- a/morf-h2/src/test/java/org/alfasoftware/morf/jdbc/h2/TestH2MetaDataProvider.java +++ b/morf-h2/src/test/java/org/alfasoftware/morf/jdbc/h2/TestH2MetaDataProvider.java @@ -33,7 +33,6 @@ import static org.mockito.Mockito.*; import static org.mockito.Mockito.when; - /** * Test class for {@link H2MetaDataProvider} * diff --git a/morf-integration-test/src/test/java/org/alfasoftware/morf/integration/TestDatabaseUpgradeIntegration.java b/morf-integration-test/src/test/java/org/alfasoftware/morf/integration/TestDatabaseUpgradeIntegration.java index dc1ac5b3f..f67477dfb 100755 --- a/morf-integration-test/src/test/java/org/alfasoftware/morf/integration/TestDatabaseUpgradeIntegration.java +++ b/morf-integration-test/src/test/java/org/alfasoftware/morf/integration/TestDatabaseUpgradeIntegration.java @@ -1398,7 +1398,8 @@ public void testUpdateOfMissingFieldThrowingException() { () -> verifyUpgrade(expected, List.of(UpdateId.class, DropPrimaryKey.class, UpdateMissingField.class, AddColumn.class, UpdateField.class))); assertThat(exception.getMessage(), containsString("Error executing SQL")); - assertThat(exception.getMessage(), containsString("UPDATE WithDefaultValue SET missingColumn")); + assertThat(exception.getMessage(), containsString("UPDATE")); + assertThat(exception.getMessage(), containsString("WithDefaultValue SET missingColumn")); } diff --git a/morf-integration-test/src/test/java/org/alfasoftware/morf/integration/TestSqlStatements.java b/morf-integration-test/src/test/java/org/alfasoftware/morf/integration/TestSqlStatements.java index 59d51de89..a20f9031e 100755 --- a/morf-integration-test/src/test/java/org/alfasoftware/morf/integration/TestSqlStatements.java +++ b/morf-integration-test/src/test/java/org/alfasoftware/morf/integration/TestSqlStatements.java @@ -117,6 +117,7 @@ import java.util.List; import java.util.Random; import java.util.Set; +import java.util.concurrent.atomic.AtomicBoolean; import java.util.function.BiConsumer; import java.util.function.UnaryOperator; @@ -155,12 +156,15 @@ import org.alfasoftware.morf.sql.element.FieldLiteral; import org.alfasoftware.morf.sql.element.FieldReference; import org.alfasoftware.morf.sql.element.Function; +import org.alfasoftware.morf.sql.element.PortableSqlFunction; import org.alfasoftware.morf.sql.element.SqlParameter; import org.alfasoftware.morf.sql.element.TableReference; import org.alfasoftware.morf.testing.DatabaseSchemaManager; import org.alfasoftware.morf.testing.DatabaseSchemaManager.TruncationBehavior; import org.alfasoftware.morf.testing.TestingDataSourceModule; import org.alfasoftware.morf.upgrade.LoggingSqlScriptVisitor; +import org.apache.commons.codec.DecoderException; +import org.apache.commons.codec.binary.Hex; import org.apache.commons.lang3.mutable.MutableBoolean; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; @@ -211,6 +215,24 @@ public class TestSqlStatements { //CHECKSTYLE:OFF private static final String BLOB1_VALUE = "A Blob named One"; private static final String BLOB2_VALUE = "A Blob named Two"; + private static final byte[] BLOB3_VALUE = new byte[] { + (byte)0x00, (byte)0x01, (byte)0x02, (byte)0x03, (byte)0x04, (byte)0x05, (byte)0x06, (byte)0x07, (byte)0x08, (byte)0x09, (byte)0x0A, (byte)0x0B, (byte)0x0C, (byte)0x0D, (byte)0x0E, (byte)0x0F, (byte)0x10, + (byte)0x11, (byte)0x12, (byte)0x13, (byte)0x14, (byte)0x15, (byte)0x16, (byte)0x17, (byte)0x18, (byte)0x19, (byte)0x1A, (byte)0x1B, (byte)0x1C, (byte)0x1D, (byte)0x1E, (byte)0x1F, (byte)0x20, (byte)0x21, + (byte)0x22, (byte)0x23, (byte)0x24, (byte)0x25, (byte)0x26, (byte)0x27, (byte)0x28, (byte)0x29, (byte)0x2A, (byte)0x2B, (byte)0x2C, (byte)0x2D, (byte)0x2E, (byte)0x2F, (byte)0x30, (byte)0x31, (byte)0x32, + (byte)0x33, (byte)0x34, (byte)0x35, (byte)0x36, (byte)0x37, (byte)0x38, (byte)0x39, (byte)0x3A, (byte)0x3B, (byte)0x3C, (byte)0x3D, (byte)0x3E, (byte)0x3F, (byte)0x40, (byte)0x41, (byte)0x42, (byte)0x43, + (byte)0x44, (byte)0x45, (byte)0x46, (byte)0x47, (byte)0x48, (byte)0x49, (byte)0x4A, (byte)0x4B, (byte)0x4C, (byte)0x4D, (byte)0x4E, (byte)0x4F, (byte)0x50, (byte)0x51, (byte)0x52, (byte)0x53, (byte)0x54, + (byte)0x55, (byte)0x56, (byte)0x57, (byte)0x58, (byte)0x59, (byte)0x5A, (byte)0x5B, (byte)0x5C, (byte)0x5D, (byte)0x5E, (byte)0x5F, (byte)0x60, (byte)0x61, (byte)0x62, (byte)0x63, (byte)0x64, (byte)0x65, + (byte)0x66, (byte)0x67, (byte)0x68, (byte)0x69, (byte)0x6A, (byte)0x6B, (byte)0x6C, (byte)0x6D, (byte)0x6E, (byte)0x6F, (byte)0x70, (byte)0x71, (byte)0x72, (byte)0x73, (byte)0x74, (byte)0x75, (byte)0x76, + (byte)0x77, (byte)0x78, (byte)0x79, (byte)0x7A, (byte)0x7B, (byte)0x7C, (byte)0x7D, (byte)0x7E, (byte)0x7F, (byte)0x80, (byte)0x81, (byte)0x82, (byte)0x83, (byte)0x84, (byte)0x85, (byte)0x86, (byte)0x87, + (byte)0x88, (byte)0x89, (byte)0x8A, (byte)0x8B, (byte)0x8C, (byte)0x8D, (byte)0x8E, (byte)0x8F, (byte)0x90, (byte)0x91, (byte)0x92, (byte)0x93, (byte)0x94, (byte)0x95, (byte)0x96, (byte)0x97, (byte)0x98, + (byte)0x99, (byte)0x9A, (byte)0x9B, (byte)0x9C, (byte)0x9D, (byte)0x9E, (byte)0x9F, (byte)0xA0, (byte)0xA1, (byte)0xA2, (byte)0xA3, (byte)0xA4, (byte)0xA5, (byte)0xA6, (byte)0xA7, (byte)0xA8, (byte)0xA9, + (byte)0xAA, (byte)0xAB, (byte)0xAC, (byte)0xAD, (byte)0xAE, (byte)0xAF, (byte)0xB0, (byte)0xB1, (byte)0xB2, (byte)0xB3, (byte)0xB4, (byte)0xB5, (byte)0xB6, (byte)0xB7, (byte)0xB8, (byte)0xB9, (byte)0xBA, + (byte)0xBB, (byte)0xBC, (byte)0xBD, (byte)0xBE, (byte)0xBF, (byte)0xC0, (byte)0xC1, (byte)0xC2, (byte)0xC3, (byte)0xC4, (byte)0xC5, (byte)0xC6, (byte)0xC7, (byte)0xC8, (byte)0xC9, (byte)0xCA, (byte)0xCB, + (byte)0xCC, (byte)0xCD, (byte)0xCE, (byte)0xCF, (byte)0xD0, (byte)0xD1, (byte)0xD2, (byte)0xD3, (byte)0xD4, (byte)0xD5, (byte)0xD6, (byte)0xD7, (byte)0xD8, (byte)0xD9, (byte)0xDA, (byte)0xDB, (byte)0xDC, + (byte)0xDD, (byte)0xDE, (byte)0xDF, (byte)0xE0, (byte)0xE1, (byte)0xE2, (byte)0xE3, (byte)0xE4, (byte)0xE5, (byte)0xE6, (byte)0xE7, (byte)0xE8, (byte)0xE9, (byte)0xEA, (byte)0xEB, (byte)0xEC, (byte)0xED, + (byte)0xEE, (byte)0xEF, (byte)0xF0, (byte)0xF1, (byte)0xF2, (byte)0xF3, (byte)0xF4, (byte)0xF5, (byte)0xF6, (byte)0xF7, (byte)0xF8, (byte)0xF9, (byte)0xFA, (byte)0xFB, (byte)0xFC, (byte)0xFD, (byte)0xFE, + (byte)0xFF + }; private static final String DATABASE_TYPE_MYSQL = "MY_SQL"; private static final String DATABASE_TYPE_SQL_SERVER = "SQL_SERVER"; @@ -1918,8 +1940,12 @@ public void testBlobFields() throws SQLException { field("column1").eq(blobLiteral(BLOB1_VALUE.getBytes())), field("column1").eq(blobLiteral(BLOB1_VALUE)) )); + // this update fails to work as an update without a WHERE clause - it strangely inserts a duplicate row on Postgres without a where clause UpdateStatement updateStatement = update(tableRef("BlobTable")) - .set(blobLiteral(BLOB1_VALUE + " Updated").as("column1"), blobLiteral((BLOB2_VALUE + " Updated").getBytes()).as("column2")); + .set(blobLiteral(BLOB1_VALUE + " Updated").as("column1"), blobLiteral((BLOB2_VALUE + " Updated").getBytes()).as("column2")) + .where( + field("column1").eq(blobLiteral((BLOB1_VALUE).getBytes())) + ); SelectStatement selectStatementAfterUpdate = select(field("column1"), field("column2")) .from(tableRef("BlobTable")) .where(or( @@ -1933,19 +1959,32 @@ public void testBlobFields() throws SQLException { // Check result - note that this is deliberately not tidy - we are making sure that results get // passed back up to this scope correctly. String sql = convertStatementToSQL(selectStatementAfterInsert); + AtomicBoolean isFirstValueHex = new AtomicBoolean(false); Integer numberOfRecords = executor.executeQuery(sql, connection, new ResultSetProcessor() { @Override public Integer process(ResultSet resultSet) throws SQLException { int result = 0; while (resultSet.next()) { result++; - assertEquals("column1 blob value not correctly set/returned after insert", BLOB1_VALUE, new String(resultSet.getBytes(1))); - assertEquals("column2 blob value not correctly set/returned after insert", BLOB2_VALUE, new String(resultSet.getBytes(2))); + byte[] bytesFromFirst = resultSet.getBytes("column1"); + + if (bytesFromFirst[1] == 32) { // if second char is a space then it isn't hex encoded + assertEquals("column1 blob value not correctly set/returned after insert", BLOB1_VALUE, new String(resultSet.getBytes(1))); + assertEquals("column2 blob value not correctly set/returned after insert", BLOB2_VALUE, new String(resultSet.getBytes(2))); + } else { + isFirstValueHex.set(true); + assertEquals("column1 blob value not correctly set/returned after insert", BLOB1_VALUE, decodeBlobHexFromBytesToText(resultSet.getBytes(1))); + assertEquals("column2 blob value not correctly set/returned after insert", BLOB2_VALUE, decodeBlobHexFromBytesToText(resultSet.getBytes(2))); + } } return result; } }); - assertEquals("Should be exactly two records", 2, numberOfRecords.intValue()); + if (isFirstValueHex.get()) { + assertEquals("Should be exactly one record", 1, numberOfRecords.intValue()); + } else { + assertEquals("Should be exactly two records", 2, numberOfRecords.intValue()); + } // Update executor.execute(ImmutableList.of(convertStatementToSQL(updateStatement)), connection); @@ -1953,21 +1992,215 @@ public Integer process(ResultSet resultSet) throws SQLException { // Check result- note that this is deliberately not tidy - we are making sure that results get // passed back up to this scope correctly. sql = convertStatementToSQL(selectStatementAfterUpdate); + AtomicBoolean isUpdateFirstValueHex = new AtomicBoolean(false); numberOfRecords = executor.executeQuery(sql, connection, new ResultSetProcessor() { @Override public Integer process(ResultSet resultSet) throws SQLException { int result = 0; while (resultSet.next()) { result++; + byte[] bytesFromFirst = resultSet.getBytes("column1"); + if (bytesFromFirst[1] == 32) { // if second char is a space then it isn't hex encoded assertEquals("column1 blob value not correctly set/returned after update", BLOB1_VALUE + " Updated", new String(resultSet.getBytes(1))); assertEquals("column2 blob value not correctly set/returned after update", BLOB2_VALUE + " Updated", new String(resultSet.getBytes(2))); + } else { + isUpdateFirstValueHex.set(true); + assertEquals("column1 blob value not correctly set/returned after update", BLOB1_VALUE + " Updated", decodeBlobHexFromBytesToText(resultSet.getBytes(1))); + assertEquals("column2 blob value not correctly set/returned after update", BLOB2_VALUE + " Updated", decodeBlobHexFromBytesToText(resultSet.getBytes(2))); + } } return result; } }); - assertEquals("Should be exactly two records", 2, numberOfRecords.intValue()); + if (isUpdateFirstValueHex.get()) { + assertEquals("Should be exactly one records", 1, numberOfRecords.intValue()); + } else { + assertEquals("Should be exactly two records", 2, numberOfRecords.intValue()); + } + } + + + /** + * Test the behaviour of SELECTs, INSERTs and UPDATEs of blob fields. In the process + * we test a lot of {@link SqlScriptExecutor}'s statement handling capabilities + * + * @throws SQLException if something goes wrong. + */ + @Test + public void testBlobFieldsRealBinary() { // throws SQLException + SqlScriptExecutor executor = sqlScriptExecutorProvider.get(new LoggingSqlScriptVisitor()); + + // Set up queries + InsertStatement insertStatement = insert() + .into(tableRef("BlobTable")) + .fields(field("column1"), field("column2")) + .values(blobLiteral(BLOB3_VALUE).as("column1"), blobLiteral(BLOB3_VALUE).as("column2")); + SelectStatement selectStatementAfterInsert = select(field("column1"), field("column2")) + .from(tableRef("BlobTable")) + .where(or( + field("column1").eq(blobLiteral(BLOB3_VALUE)), + field("column1").eq(blobLiteral(BLOB3_VALUE)) + )); + + byte[] bytUpdated = Arrays.copyOf(BLOB3_VALUE, 256+3); + bytUpdated[256] = 1; + bytUpdated[257] = 2; + bytUpdated[258] = 3; + // this update fails to work as an update without a WHERE clause - it strangely inserts a duplicate row on Postgres without a where clause + UpdateStatement updateStatement = update(tableRef("BlobTable")) + .set(blobLiteral(bytUpdated).as("column1"), blobLiteral(bytUpdated).as("column2")) + .where( + field("column1").eq(blobLiteral(BLOB3_VALUE)) + ); + SelectStatement selectStatementAfterUpdate = select(field("column1"), field("column2")) + .from(tableRef("BlobTable")) + .where(or( + field("column1").eq(blobLiteral(bytUpdated)), + field("column1").eq(blobLiteral(bytUpdated)) + )); + + // Insert + executor.execute(convertStatementToSQL(insertStatement, schema, null), connection); + + boolean isOracle = false; + + try { + String databaseProductName = this.dataSource.getConnection().getMetaData().getDatabaseProductName(); + isOracle = databaseProductName.contains("Oracle"); + } catch (SQLException e) { + // ignore SQLException + } + + if (isOracle) { + // for Oracle need to compare BLOB's with DBMS_LOB.INSTR + AliasedField compareFunctionBlob = PortableSqlFunction.builder() + .withFunctionForDatabaseType("ORACLE", + "DBMS_LOB.INSTR", + new FieldReference("column1"), + blobLiteral(BLOB3_VALUE), + new FieldLiteral("1"), + new FieldLiteral("1") + ) + .build(); + + AliasedField compareFunctionUpdated = PortableSqlFunction.builder() + .withFunctionForDatabaseType("ORACLE", + "DBMS_LOB.INSTR", + new FieldReference("column1"), + blobLiteral(bytUpdated), + new FieldLiteral("1"), + new FieldLiteral("1") + ) + .build(); + + selectStatementAfterInsert = select(field("column1"), field("column2")) + .from(tableRef("BlobTable")) + .where( + compareFunctionBlob.greaterThan(0) + ); + updateStatement = update(tableRef("BlobTable")) + .set(blobLiteral(bytUpdated).as("column1"), blobLiteral(bytUpdated).as("column2")) + .where( + compareFunctionBlob.greaterThan(0) + ); + selectStatementAfterUpdate = select(field("column1"), field("column2")) + .from(tableRef("BlobTable")) + .where( + compareFunctionUpdated.greaterThan(0) + ); } + // Check result - note that this is deliberately not tidy - we are making sure that results get + // passed back up to this scope correctly. + String sql = convertStatementToSQL(selectStatementAfterInsert); + AtomicBoolean isFirstValueHex = new AtomicBoolean(false); + Integer numberOfRecords = executor.executeQuery(sql, connection, new ResultSetProcessor() { + @Override + public Integer process(ResultSet resultSet) throws SQLException { + int result = 0; + while (resultSet.next()) { + result++; + byte[] bytesFromFirst = resultSet.getBytes("column1"); + + if (bytesFromFirst[3] == 0x03) { // if 4th char is 0x03 then it isn't hex encoded like in Postgres + assertEquals("column1 blob value not correctly set/returned after insert", 0, Arrays.compare(BLOB3_VALUE, resultSet.getBytes(1))); + assertEquals("column2 blob value not correctly set/returned after insert", 0, Arrays.compare(BLOB3_VALUE, resultSet.getBytes(2))); + } else { + isFirstValueHex.set(true); + assertEquals("column1 blob value not correctly set/returned after insert", 0, Arrays.compare(BLOB3_VALUE, decodeBlobHexFromBytesToByteArray(resultSet.getBytes(1)))); + assertEquals("column2 blob value not correctly set/returned after insert", 0, Arrays.compare(BLOB3_VALUE, decodeBlobHexFromBytesToByteArray(resultSet.getBytes(2)))); + } + } + return result; + } + }); + + assertEquals("Should be exactly one record", 1, numberOfRecords.intValue()); + + // Update + executor.execute(ImmutableList.of(convertStatementToSQL(updateStatement)), connection); + + // Check result- note that this is deliberately not tidy - we are making sure that results get + // passed back up to this scope correctly. + sql = convertStatementToSQL(selectStatementAfterUpdate); + AtomicBoolean isUpdateFirstValueHex = new AtomicBoolean(false); + numberOfRecords = executor.executeQuery(sql, connection, new ResultSetProcessor() { + @Override + public Integer process(ResultSet resultSet) throws SQLException { + int result = 0; + while (resultSet.next()) { + result++; + byte[] bytesFromFirst = resultSet.getBytes("column1"); + if (bytesFromFirst[3] == 0x03) { // if second char is a space then it isn't hex encoded + assertEquals("column1 blob value not correctly set/returned after update", 0, Arrays.compare(bytUpdated, resultSet.getBytes(1))); + assertEquals("column2 blob value not correctly set/returned after update", 0, Arrays.compare(bytUpdated, resultSet.getBytes(2))); + } else { + isUpdateFirstValueHex.set(true); + assertEquals("column1 blob value not correctly set/returned after update", 0, Arrays.compare(bytUpdated, decodeBlobHexFromBytesToByteArray(resultSet.getBytes(1)))); + assertEquals("column2 blob value not correctly set/returned after update", 0, Arrays.compare(bytUpdated, decodeBlobHexFromBytesToByteArray(resultSet.getBytes(2)))); + } + } + return result; + } + }); + assertEquals("Should be exactly one records", 1, numberOfRecords.intValue()); + } + + private static byte[] decodeBlobHexFromBytesToByteArray(byte[] bytSrc) { + Hex hexUtil = new Hex(); + int lenSrc = bytSrc.length; + char[] charBlob = new char[lenSrc]; + byte[] bytBlob = new byte[charBlob.length >> 1]; + try { + for (int i = 0; i < bytSrc.length; i++) { + charBlob[i] = (char) bytSrc[i]; + } + hexUtil.decodeHex(charBlob, bytBlob, 0); + } catch (DecoderException e) { + throw new RuntimeException(e); + } + return bytBlob; + } + + private static String decodeBlobHexFromBytesToText(byte[] bytSrc) { + String blobStringResult; + Hex hexUtil = new Hex(); + try { + int lenSrc = bytSrc.length; + char[] charBlob = new char[lenSrc]; + byte[] bytBlob = new byte[charBlob.length >> 1]; + for (int i = 0; i < bytSrc.length; i++) { + charBlob[i] = (char) bytSrc[i]; + } + hexUtil.decodeHex(charBlob, bytBlob, 0); + + blobStringResult = new String(bytBlob); + } catch (DecoderException e) { + throw new RuntimeException(e); + } + return blobStringResult; + } + /** * Asserts that the number of records in the table are as expected. diff --git a/morf-integration-test/src/test/java/org/alfasoftware/morf/jdbc/TestDatabaseMetaDataProvider.java b/morf-integration-test/src/test/java/org/alfasoftware/morf/jdbc/TestDatabaseMetaDataProvider.java index 8bde7df40..ec06cb008 100755 --- a/morf-integration-test/src/test/java/org/alfasoftware/morf/jdbc/TestDatabaseMetaDataProvider.java +++ b/morf-integration-test/src/test/java/org/alfasoftware/morf/jdbc/TestDatabaseMetaDataProvider.java @@ -33,7 +33,9 @@ import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.equalToIgnoringCase; import static org.hamcrest.Matchers.is; +import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertThrows; import static org.junit.Assert.assertTrue; import static org.junit.Assert.fail; @@ -67,6 +69,7 @@ import com.google.common.base.Strings; import com.google.common.collect.ImmutableList; import com.google.common.collect.Iterables; +import com.google.common.util.concurrent.UncheckedExecutionException; import com.google.inject.Inject; import net.jcip.annotations.NotThreadSafe; @@ -131,7 +134,12 @@ public class TestDatabaseMetaDataProvider { column("bigIntegerCol", DataType.BIG_INTEGER).defaultValue("8"), column("booleanCol", DataType.BOOLEAN).defaultValue("1"), column("integerTenCol", DataType.INTEGER).defaultValue("17"), - column("dateCol", DataType.DATE).defaultValue("2020-01-01")) + column("dateCol", DataType.DATE).defaultValue("2020-01-01")), + table("WithPartition") + .columns( + SchemaUtils.idColumn(), + column("stringCol", DataType.STRING, 20) + ) ), schema( view("ViewWithTypes", select(field("primaryStringCol"), field("id")).from("WithTypes").crossJoin(tableRef("WithDefaults"))), @@ -173,7 +181,6 @@ public void after() throws SQLException { String schema = Strings.isNullOrEmpty(database.getSchemaName()) ? "" : database.getSchemaName() + "."; connection.createStatement().executeUpdate("DROP TABLE " + schema + "WithTimestamp"); } - schemaManager.invalidateCache(); } @@ -208,6 +215,7 @@ public void testViewsAndTables() throws SQLException { tableNameEqualTo("WithTypes"), tableNameEqualTo("WithDefaults"), tableNameEqualTo("WithLobs"), + tableNameEqualTo("WithPartition"), equalToIgnoringCase("WithTimestamp") // can read table names even if they contain unsupported columns ))); @@ -216,6 +224,7 @@ public void testViewsAndTables() throws SQLException { tableNameMatcher("WithTypes"), tableNameMatcher("WithDefaults"), tableNameMatcher("WithLobs"), + tableNameMatcher("WithPartition"), propertyMatcher(Table::getName, "name", equalToIgnoringCase("WithTimestamp")) // can read table names even if they contain unsupported columns ))); } @@ -284,7 +293,7 @@ public void testTableWithTypes() throws SQLException { )); schemaResource.getAdditionalMetadata().ifPresent(additionalMetadata -> - assertThat(additionalMetadata.ignoredIndexes().get("WithTypes".toLowerCase()), containsInAnyOrder(ImmutableList.of( + assertThat(additionalMetadata.ignoredIndexes().get("withtypes"), containsInAnyOrder(ImmutableList.of( indexMatcher(index("WithTypes_PRF1").columns("decimalNineFiveCol", "bigIntegerCol")) )))); } @@ -319,6 +328,39 @@ public void testTableWithLobs() throws SQLException { } + @Test + public void testTableWithPartition() throws SQLException { + boolean isPostgres = databaseType.equals("PGSQL"); + // RE-CREATE table with two partitions on table WithPartition + try (Connection connection = database.getDataSource().getConnection()) { + if (isPostgres) { + String tableSchema = Strings.isNullOrEmpty(database.getSchemaName()) ? "" : database.getSchemaName() + "."; + connection.createStatement().executeUpdate("DROP TABLE " + tableSchema + "WithPartition"); + connection.createStatement().executeUpdate("CREATE TABLE " + tableSchema + "WithPartition(id numeric(19) NOT NULL, stringCol VARCHAR(20)) PARTITION BY RANGE (id)"); + connection.createStatement().executeUpdate("COMMENT ON TABLE "+ tableSchema + "WithPartition IS 'REALNAME:[WithPartition]'"); + connection.createStatement().executeUpdate("CREATE TABLE " + tableSchema + "WithPartition_p0 PARTITION OF " + tableSchema + "WithPartition FOR VALUES FROM (0) TO (10000)"); + connection.createStatement().executeUpdate("COMMENT ON TABLE "+ tableSchema + "WithPartition_p0 IS 'REALNAME:[WithPartition_p0]'"); + connection.createStatement().executeUpdate("CREATE TABLE " + tableSchema + "WithPartition_p1 PARTITION OF " + tableSchema + "WithPartition FOR VALUES FROM (10000) TO (99999)"); + connection.createStatement().executeUpdate("COMMENT ON TABLE "+ tableSchema + "WithPartition_p1 IS 'REALNAME:[WithPartition_p1]'"); + } + } + + try(SchemaResource schemaResource = database.openSchemaResource()) { + assertTrue(schemaResource.tableExists("WithPartition")); + + if (isPostgres) { + UncheckedExecutionException uncheckedExecutionException = assertThrows(UncheckedExecutionException.class, () -> schemaResource.getTable("WithPartition_p0")); + assertTrue("partition must not be found on getTable", uncheckedExecutionException.getMessage().contains("Table [WithPartition_p0/*] not found.")); + + Table table = schemaResource.getTable("WithPartition"); + assertEquals("table must have 2 columns", 2, table.columns().size()); + assertEquals("first column must match", "id", table.columns().get(0).getName()); + assertEquals("second column column must match", "stringcol", table.columns().get(1).getName()); + } + } + } + + @Test public void testTableWithDefaults() throws SQLException { try(SchemaResource schemaResource = database.openSchemaResource()) { diff --git a/morf-integration-test/src/test/java/org/alfasoftware/morf/upgrade/TestDatabaseUpgradePathValidationService.java b/morf-integration-test/src/test/java/org/alfasoftware/morf/upgrade/TestDatabaseUpgradePathValidationService.java index 08183865d..1502a9a6a 100644 --- a/morf-integration-test/src/test/java/org/alfasoftware/morf/upgrade/TestDatabaseUpgradePathValidationService.java +++ b/morf-integration-test/src/test/java/org/alfasoftware/morf/upgrade/TestDatabaseUpgradePathValidationService.java @@ -82,6 +82,7 @@ public void setup() { public void tearDown() { dropUpgradeStatusTable(); schemaManager.invalidateCache(); + schemaManager.dropAllTables(); } diff --git a/morf-oracle/src/main/java/org/alfasoftware/morf/jdbc/oracle/OracleMetaDataProvider.java b/morf-oracle/src/main/java/org/alfasoftware/morf/jdbc/oracle/OracleMetaDataProvider.java index 817145484..d6d7e1ac5 100755 --- a/morf-oracle/src/main/java/org/alfasoftware/morf/jdbc/oracle/OracleMetaDataProvider.java +++ b/morf-oracle/src/main/java/org/alfasoftware/morf/jdbc/oracle/OracleMetaDataProvider.java @@ -29,6 +29,7 @@ import java.util.HashMap; import java.util.HashSet; import java.util.List; +import java.util.Locale; import java.util.Map; import java.util.Map.Entry; import java.util.Set; @@ -382,7 +383,7 @@ public void handle(ResultSet resultSet) throws SQLException { if (DatabaseMetaDataProviderUtils.shouldIgnoreIndex(indexName)) { Index ignoredIndex = getAssembledIndex(unique, indexNameFinal); - String currentTableName = currentTable.getName().toUpperCase(); + String currentTableName = currentTable.getName().toLowerCase(Locale.ROOT); if (ignoredIndexes.containsKey(currentTableName)) { ignoredIndexes.compute(currentTableName, (k, tableIgnoredIndexes) -> { List newList = tableIgnoredIndexes == null ? new ArrayList<>() : new ArrayList<>(tableIgnoredIndexes); @@ -471,7 +472,7 @@ public void handle(ResultSet resultSet) throws SQLException { if (DatabaseMetaDataProviderUtils.shouldIgnoreIndex(indexName)) { Index lastIndex = null; - for (Index currentIndex : ignoredIndexes.get(currentTable.getName().toUpperCase())) { + for (Index currentIndex : ignoredIndexes.get(currentTable.getName().toLowerCase())) { if (currentIndex.getName().equalsIgnoreCase(indexName)) { lastIndex = currentIndex; break; @@ -788,22 +789,16 @@ private interface ResultSetHandler { */ private void runSQL(String sql, ResultSetHandler handler) { try { - PreparedStatement statement = connection.prepareStatement(sql); - try { + try (PreparedStatement statement = connection.prepareStatement(sql)) { // We'll inevitably need a lot of meta data so may as well get it in big chunks. statement.setFetchSize(100); // pass through the schema name statement.setString(1, schemaName); - ResultSet resultSet = statement.executeQuery(); - try { + try (ResultSet resultSet = statement.executeQuery()){ handler.handle(resultSet); - } finally { - resultSet.close(); } - } finally { - statement.close(); } } catch (SQLException sqle) { throw new RuntimeSqlException("Error running SQL: " + sql, sqle); diff --git a/morf-oracle/src/test/java/org/alfasoftware/morf/jdbc/oracle/TestOracleMetaDataProvider.java b/morf-oracle/src/test/java/org/alfasoftware/morf/jdbc/oracle/TestOracleMetaDataProvider.java index 4fd59b786..78ea7b0db 100755 --- a/morf-oracle/src/test/java/org/alfasoftware/morf/jdbc/oracle/TestOracleMetaDataProvider.java +++ b/morf-oracle/src/test/java/org/alfasoftware/morf/jdbc/oracle/TestOracleMetaDataProvider.java @@ -33,6 +33,7 @@ import java.sql.ResultSet; import java.sql.SQLException; import java.util.List; +import java.util.Locale; import java.util.Map; import javax.sql.DataSource; @@ -155,7 +156,7 @@ public void testPrimaryKeyIndexNames() throws SQLException { /** * Checks the building of the collection of primary key index names. - * @throws SQLException + * @throws SQLException with error */ @Test public void testIgnoredIndexes() throws SQLException { @@ -211,7 +212,7 @@ public void testIgnoredIndexes() throws SQLException { // When final AdditionalMetadata oracleMetaDataProvider = (AdditionalMetadata) oracle.openSchema(connection, "TESTDATABASE", "TESTSCHEMA"); - List actualIgnoredIndexes = oracleMetaDataProvider.ignoredIndexes().get("AREALTABLE"); + List actualIgnoredIndexes = oracleMetaDataProvider.ignoredIndexes().get("AREALTABLE".toLowerCase(Locale.ROOT)); assertEquals("Ignored indexes size.", 2, actualIgnoredIndexes.size()); assertEquals("Ignored AREALTABLE table indexes size.", 2, actualIgnoredIndexes.size()); Index index = actualIgnoredIndexes.get(0); diff --git a/morf-postgresql/src/main/java/org/alfasoftware/morf/jdbc/postgresql/PostgreSQLMetaDataProvider.java b/morf-postgresql/src/main/java/org/alfasoftware/morf/jdbc/postgresql/PostgreSQLMetaDataProvider.java index 1295800d7..95a23df64 100644 --- a/morf-postgresql/src/main/java/org/alfasoftware/morf/jdbc/postgresql/PostgreSQLMetaDataProvider.java +++ b/morf-postgresql/src/main/java/org/alfasoftware/morf/jdbc/postgresql/PostgreSQLMetaDataProvider.java @@ -30,6 +30,7 @@ import com.google.common.base.Suppliers; import com.google.common.collect.ImmutableMap; +import com.google.common.collect.ImmutableSet; /** * Provides meta data from a PostgreSQL database connection. @@ -51,6 +52,53 @@ public PostgreSQLMetaDataProvider(Connection connection, String schemaName) { } + @Override + protected Set loadIgnoredPartitionTables() { + ImmutableSet.Builder ignoredTables = new ImmutableSet.Builder<>(); + try(Statement ignoredTablesStmt = connection.createStatement()) { + // distinguish partitioned tables from regular ones: relkind = 'p' (partition) or 'r' (regular) also can use boolean col relispartition + // a partition table attached has (r, true) -- CREATE TABLE MEASURE_P1 (id, a, b) FOR VALUES ('0) TO ('37000'); + // a partitioned table has (p, false) -- CREATE TABLE MEASURE(id, a, b) PARTITION by ID; + try (ResultSet ignoredTablesRs = ignoredTablesStmt.executeQuery("select par.relname, d.description\n" + + "from pg_class par \n" + + "join pg_namespace n on n.oid = par.relnamespace\n" + + "join pg_description d ON d.objoid = par.oid and d.objsubid = 0\n" + + " where par.relispartition and par.relkind = 'r'")) { + while (ignoredTablesRs.next()) { + ignoredTables.add(createRealName(ignoredTablesRs.getString(1), ignoredTablesRs.getString(2))); + } + } + } catch (SQLException e) { + // ignore exception, if it fails then incompatible Postgres version + log.info("The loading of ignored partitions failed, probably because it is a version before 11"); + } + return ignoredTables.build(); + } + + @Override + protected Set loadPartitionedTables() { + ImmutableSet.Builder partitionedTables = new ImmutableSet.Builder<>(); + try(Statement partitionedTablesStmt = connection.createStatement()) { + // distinguish partitioned tables from regular ones: relkind = 'p' (partition) or 'r' (regular) also can use boolean col relispartition + // a partition table attached has (r, true) -- CREATE TABLE MEASURE_P1 (id, a, b) FOR VALUES ('0) TO ('37000'); + // a partitioned table has (p, false) -- CREATE TABLE MEASURE(id, a, b) PARTITION by ID; + try (ResultSet ignoredTablesRs = partitionedTablesStmt.executeQuery("select par.relname as tableName, d.description\n" + + "from pg_class par \n" + + "join pg_namespace n on n.oid = par.relnamespace \n" + + "join pg_description d ON d.objoid = par.oid and d.objsubid = 0\n" + + "where not par.relispartition and par.relkind = 'p'")) { + while (ignoredTablesRs.next()) { + partitionedTables.add(createRealName(ignoredTablesRs.getString(1), matchComment(ignoredTablesRs.getString(2)))); + } + } + } catch (SQLException e) { + // ignore exception, if it fails then incompatible Postgres version + log.info("The loading of ignored partitions failed, probably because it is a version before 11"); + } + return partitionedTables.build(); + } + + @Override protected boolean isPrimaryKeyIndex(RealName indexName) { return indexName.getDbName().endsWith("_pk"); @@ -233,4 +281,15 @@ protected String buildSequenceSql(String schemaName) { return sequenceSqlBuilder.toString(); } + + + @Override + public Set partitionedTableNames() { + return super.partitionedTables.get(); + } + + @Override + public Set partitionTableNames() { + return super.ignoredPartitionTables.get(); + } } diff --git a/morf-postgresql/src/test/java/org/alfasoftware/morf/jdbc/postgresql/TestPostgreSQLDialect.java b/morf-postgresql/src/test/java/org/alfasoftware/morf/jdbc/postgresql/TestPostgreSQLDialect.java index 2d676854e..f782a753e 100644 --- a/morf-postgresql/src/test/java/org/alfasoftware/morf/jdbc/postgresql/TestPostgreSQLDialect.java +++ b/morf-postgresql/src/test/java/org/alfasoftware/morf/jdbc/postgresql/TestPostgreSQLDialect.java @@ -1642,7 +1642,7 @@ protected String expectedSelectWithJoinAndLimit() { return "SELECT Test.id, Alternate.stringField FROM " + tableName(TEST_TABLE) + " INNER JOIN " + tableName("Alternate") + " ON (Test.id = Alternate.id) LIMIT 25"; } - + /** * @see AbstractSqlDialectTest#expectedPortableSqlExpression() */ @@ -1650,8 +1650,8 @@ protected String expectedSelectWithJoinAndLimit() { protected String expectedPortableSqlExpression() { return "SELECT CONCAT(first_name, ' ', last_name, ' (', params->>'role', ')') FROM testschema.Test"; } - - + + @Override protected String expectedSelectWithOrderByWhereAndLimit() { return "SELECT id, stringField FROM " + tableName(TEST_TABLE) + " WHERE (stringField IS NOT NULL) ORDER BY id DESC LIMIT 10"; diff --git a/morf-postgresql/src/test/java/org/alfasoftware/morf/jdbc/postgresql/TestPostgreSqlMetaDataProvider.java b/morf-postgresql/src/test/java/org/alfasoftware/morf/jdbc/postgresql/TestPostgreSqlMetaDataProvider.java index b98d3598b..7ea6783eb 100644 --- a/morf-postgresql/src/test/java/org/alfasoftware/morf/jdbc/postgresql/TestPostgreSqlMetaDataProvider.java +++ b/morf-postgresql/src/test/java/org/alfasoftware/morf/jdbc/postgresql/TestPostgreSqlMetaDataProvider.java @@ -1,218 +1,378 @@ -/* Copyright 2017 Alfa Financial Software - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.alfasoftware.morf.jdbc.postgresql; - -import static org.hamcrest.MatcherAssert.assertThat; -import static org.hamcrest.Matchers.contains; -import static org.junit.Assert.assertEquals; -import static org.mockito.ArgumentMatchers.anyString; -import static org.mockito.ArgumentMatchers.eq; -import static org.mockito.Mockito.RETURNS_SMART_NULLS; -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.verify; -import static org.mockito.Mockito.when; - -import java.sql.Connection; -import java.sql.DatabaseMetaData; -import java.sql.PreparedStatement; -import java.sql.ResultSet; -import java.sql.SQLException; -import java.sql.Statement; -import java.util.List; -import java.util.Map; - -import javax.sql.DataSource; - -import org.alfasoftware.morf.jdbc.DatabaseType; -import org.alfasoftware.morf.metadata.AdditionalMetadata; -import org.alfasoftware.morf.metadata.Index; -import org.alfasoftware.morf.metadata.Schema; -import org.alfasoftware.morf.metadata.Sequence; -import org.junit.Before; -import org.junit.Test; -import org.mockito.invocation.InvocationOnMock; -import org.mockito.stubbing.Answer; - - -/** - * Test class for {@link PostgreSQLMetaDataProvider} - * - * @author Copyright (c) Alfa Financial Software Ltd. 2024 - */ -public class TestPostgreSqlMetaDataProvider { - private static final String TABLE_NAME = "AREALTABLE"; - private static final String TEST_SCHEMA = "TestSchema"; - - private final DataSource dataSource = mock(DataSource.class, RETURNS_SMART_NULLS); - private final Connection connection = mock(Connection.class, RETURNS_SMART_NULLS); - private DatabaseType postgres; - - @Before - public void setup() { - postgres = DatabaseType.Registry.findByIdentifier(PostgreSQL.IDENTIFIER); - } - - - @Before - public void before() throws SQLException { - when(dataSource.getConnection()).thenReturn(connection); - } - - - /** - * Checks the SQL run for retrieving sequences information - * - * @throws SQLException exception - */ - @Test - public void testLoadSequences() throws SQLException { - // Given - final PreparedStatement statement = mock(PreparedStatement.class, RETURNS_SMART_NULLS); - when(connection.prepareStatement("SELECT S.relname FROM pg_class S LEFT JOIN pg_depend D ON " + - "(S.oid = D.objid AND D.deptype = 'a') LEFT JOIN pg_namespace N on (N.oid = S.relnamespace) WHERE S.relkind = " + - "'S' AND D.objid IS NULL AND N.nspname=?")).thenReturn(statement); - when(statement.executeQuery()).thenAnswer(new ReturnMockResultSetWithSequence(1)); - - // When - final Schema postgresMetaDataProvider = postgres.openSchema(connection, "TestDatabase", TEST_SCHEMA); - assertEquals("Sequence names", "[Sequence1]", postgresMetaDataProvider.sequenceNames().toString()); - Sequence sequence = postgresMetaDataProvider.sequences().iterator().next(); - assertEquals("Sequence name", "Sequence1", sequence.getName()); - - verify(statement).setString(1, TEST_SCHEMA); - } - - /** - * Checks the SQL run for retrieving sequences information - * - * @throws SQLException exception - */ - @Test - public void testLoadAllIgnoredIndexes() throws SQLException { - // Given - Statement statement = mock(Statement.class, RETURNS_SMART_NULLS); - when(connection.createStatement(eq(ResultSet.TYPE_FORWARD_ONLY), eq(ResultSet.CONCUR_READ_ONLY))).thenReturn(statement); - when(statement.executeQuery(anyString())).thenAnswer(answer -> { - ResultSet resultSet = mock(ResultSet.class, RETURNS_SMART_NULLS); - when(resultSet.next()).thenReturn(true, true, true, false); - when(resultSet.getString(1)).thenReturn("AREALTABLE_1", "AREALTABLE_PRF1", "AREALTABLE_PRF2"); - when(resultSet.getString(2)).thenReturn("REALNAME:[AREALTABLE_1]", "REALNAME:[AREALTABLE_PRF1]", "REALNAME:[AREALTABLE_PRF2]"); - when(resultSet.getString(3)).thenReturn("AREALTABLE", "AREALTABLE", "AREALTABLE"); - when(resultSet.getString(4)).thenReturn("REALNAME:[ARealTable]", "REALNAME:[ARealTable]", "REALNAME:[ARealTable]"); - return resultSet; - }); - - DatabaseMetaData databaseMetaData = mock(DatabaseMetaData.class, RETURNS_SMART_NULLS); - when(connection.getMetaData()).thenReturn(databaseMetaData); - - // mock getTables - when(databaseMetaData.getTables(null, TEST_SCHEMA, null, new String[] { "TABLE" })) - .thenAnswer(answer -> { - ResultSet resultSet = mock(ResultSet.class, RETURNS_SMART_NULLS); - when(resultSet.next()).thenReturn(true, false); - when(resultSet.getString(3)).thenReturn(TABLE_NAME); // // 3 - TABLE_NAME - when(resultSet.getString(2)).thenReturn(TEST_SCHEMA); // 2 - TABLE_SCHEM - when(resultSet.getString(5)).thenReturn("REALNAME:[AREALTABLE]"); // 5 - TABLE_REMARKS - when(resultSet.getString(4)).thenReturn("REGULAR"); // 4 - TABLE_TYPE - - return resultSet; - }); - - // mock getColumns - when(databaseMetaData.getColumns(null, TEST_SCHEMA, null, null)) - .thenAnswer(answer -> { - ResultSet resultSet = mock(ResultSet.class, RETURNS_SMART_NULLS); - when(resultSet.next()).thenReturn(true, false); - when(resultSet.getString(3)).thenReturn(TABLE_NAME); // 3 - COLUMN_TABLE_NAME - when(resultSet.getString(4)).thenReturn("column1"); // 4 - COLUMN_NAME - when(resultSet.getString(6)).thenReturn("VARCHAR"); // 6 - COLUMN_TYPE_NAME - when(resultSet.getInt(5)).thenReturn(12); // 5 - COLUMN_DATA_TYPE - VARCHAR 12 - when(resultSet.getInt(7)).thenReturn(1); // 7 - COLUMN_SIZE - width - when(resultSet.getInt(9)).thenReturn(0); // 9 - COLUMN_DECIMAL_DIGITS - scale - when(resultSet.getString(12)).thenReturn("REALNAME:[column1]/TYPE:[STRING]"); // 12 - TABLE_REMARKS - when(resultSet.getString(18)).thenReturn("NO"); // 18 - COLUMN_IS_NULLABLE - when(resultSet.getString(23)).thenReturn("NO"); // 23 - COLUMN_IS_AUTOINCREMENT - when(resultSet.getString(23)).thenReturn(null); // 13 - COLUMN_DEFAULT_EXPR - - return resultSet; - }); - - // mock getIndexInfo - when(databaseMetaData.getIndexInfo(null, TEST_SCHEMA, "AREALTABLE", false, false)) - .thenAnswer(answer -> { - ResultSet resultSet = mock(ResultSet.class, RETURNS_SMART_NULLS); - when(resultSet.next()).thenReturn(true, true, true, false, true, true, true, false); - when(resultSet.getString(6)).thenReturn("AREALTABLE_1", "AREALTABLE_PRF1", "AREALTABLE_PRF2", "AREALTABLE_1", "AREALTABLE_PRF1", "AREALTABLE_PRF2"); // 6 - INDEX_NAME - when(resultSet.getString(9)).thenReturn("column1", "column1", "column1", "column1", "column1", "column1"); - when(resultSet.getBoolean(4)).thenReturn(true, true, true, true, true, true); // 4 - INDEX_NON_UNIQUE - return resultSet; - }); - - // When - final Schema postgresMetaDataProvider = postgres.openSchema(connection, "TestDatabase", TEST_SCHEMA); - Map> ignoredIndexesMap = ((AdditionalMetadata)postgresMetaDataProvider).ignoredIndexes(); - // test loading the cached version: - Map> ignoredIndexesMap1 = ((AdditionalMetadata)postgresMetaDataProvider).ignoredIndexes(); - - // Then - assertEquals("map size must match", 1, ignoredIndexesMap.size()); - assertEquals("map size must match", 1, ignoredIndexesMap1.size()); - String tableNameLowerCase = TABLE_NAME.toLowerCase(); - assertEquals("table ignored indexes size must match", 2, ignoredIndexesMap.get(tableNameLowerCase).size()); - Index indexPrf1 = ignoredIndexesMap.get(tableNameLowerCase).get(0); - Index indexPrf2 = ignoredIndexesMap.get(tableNameLowerCase).get(1); - assertEquals("index prf1 name", "AREALTABLE_PRF1", indexPrf1.getName()); - assertThat("index prf1 columns", indexPrf1.columnNames(), contains("column1")); - assertEquals("index prf2 name", "AREALTABLE_PRF2", indexPrf2.getName()); - assertThat("index prf2 columns", indexPrf2.columnNames(), contains("column1")); - } - - - /** - * Mockito {@link Answer} that returns a mock result set with a given number of resultRows. - */ - private static final class ReturnMockResultSetWithSequence implements Answer { - - private final int numberOfResultRows; - - - /** - * @param numberOfResultRows - */ - private ReturnMockResultSetWithSequence(int numberOfResultRows) { - super(); - this.numberOfResultRows = numberOfResultRows; - } - - @Override - public ResultSet answer(final InvocationOnMock invocation) throws Throwable { - final ResultSet resultSet = mock(ResultSet.class, RETURNS_SMART_NULLS); - when(resultSet.next()).thenAnswer(new Answer() { - private int counter; - - @Override - public Boolean answer(InvocationOnMock invocation) throws Throwable { - return counter++ < numberOfResultRows; - } - }); - - when(resultSet.getString(1)).thenReturn("Sequence1"); - - return resultSet; - } - } - -} +/* Copyright 2017 Alfa Financial Software + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.alfasoftware.morf.jdbc.postgresql; + +import static org.hamcrest.MatcherAssert.assertThat; +import static org.hamcrest.Matchers.contains; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyString; +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.Mockito.RETURNS_SMART_NULLS; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +import java.sql.Connection; +import java.sql.DatabaseMetaData; +import java.sql.PreparedStatement; +import java.sql.ResultSet; +import java.sql.SQLException; +import java.sql.Statement; +import java.util.List; +import java.util.Map; + +import javax.sql.DataSource; + +import org.alfasoftware.morf.jdbc.DatabaseMetaDataProvider; +import org.alfasoftware.morf.jdbc.DatabaseType; +import org.alfasoftware.morf.metadata.AdditionalMetadata; +import org.alfasoftware.morf.metadata.Index; +import org.alfasoftware.morf.metadata.Schema; +import org.alfasoftware.morf.metadata.Sequence; +import org.junit.Before; +import org.junit.Test; +import org.mockito.invocation.InvocationOnMock; +import org.mockito.stubbing.Answer; + + +/** + * Test class for {@link PostgreSQLMetaDataProvider} + * + * @author Copyright (c) Alfa Financial Software Ltd. 2024 + */ +public class TestPostgreSqlMetaDataProvider { + private static final String TABLE_NAME = "AREALTABLE"; + private static final String TEST_SCHEMA = "TestSchema"; + + private final DataSource dataSource = mock(DataSource.class, RETURNS_SMART_NULLS); + private final Connection connection = mock(Connection.class, RETURNS_SMART_NULLS); + private DatabaseType postgres; + + @Before + public void setup() { + postgres = DatabaseType.Registry.findByIdentifier(PostgreSQL.IDENTIFIER); + } + + + @Before + public void before() throws SQLException { + when(dataSource.getConnection()).thenReturn(connection); + } + + + /** + * Checks the SQL run for retrieving sequences information + * + * @throws SQLException exception + */ + @Test + public void testLoadSequences() throws SQLException { + // Given + final PreparedStatement statement = mock(PreparedStatement.class, RETURNS_SMART_NULLS); + when(connection.prepareStatement("SELECT S.relname FROM pg_class S LEFT JOIN pg_depend D ON " + + "(S.oid = D.objid AND D.deptype = 'a') LEFT JOIN pg_namespace N on (N.oid = S.relnamespace) WHERE S.relkind = " + + "'S' AND D.objid IS NULL AND N.nspname=?")).thenReturn(statement); + when(statement.executeQuery()).thenAnswer(new ReturnMockResultSetWithSequence(1)); + + // When + final Schema postgresMetaDataProvider = postgres.openSchema(connection, "TestDatabase", TEST_SCHEMA); + assertEquals("Sequence names", "[Sequence1]", postgresMetaDataProvider.sequenceNames().toString()); + Sequence sequence = postgresMetaDataProvider.sequences().iterator().next(); + assertEquals("Sequence name", "Sequence1", sequence.getName()); + + verify(statement).setString(1, TEST_SCHEMA); + } + + /** + * Checks the SQL run for retrieving sequences information + * + * @throws SQLException exception + */ + @Test + public void testLoadAllIgnoredIndexes() throws SQLException { + // Given + final Statement statement0 = mock(Statement.class, RETURNS_SMART_NULLS); + final Statement statement1 = mock(Statement.class, RETURNS_SMART_NULLS); + when(connection.createStatement()).thenReturn(statement0, statement1); + when(statement0.executeQuery("select par.relname, d.description\n" + + "from pg_class par \n" + + "join pg_namespace n on n.oid = par.relnamespace\n" + + "join pg_description d ON d.objoid = par.oid and d.objsubid = 0\n" + + " where par.relispartition and par.relkind = 'r'")) + .thenAnswer(new ReturnMockResultSetWithPartitionTables(0, "", "")); + when(statement1.executeQuery("select par.relname as tableName, d.description\n" + + "from pg_class par \n" + + "join pg_namespace n on n.oid = par.relnamespace \n" + + "join pg_description d ON d.objoid = par.oid and d.objsubid = 0\n" + + "where not par.relispartition and par.relkind = 'p'")) + .thenAnswer(new ReturnMockResultSetWithPartitionTables(0, "", "")); + + Statement statement = mock(Statement.class, RETURNS_SMART_NULLS); + when(connection.createStatement(eq(ResultSet.TYPE_FORWARD_ONLY), eq(ResultSet.CONCUR_READ_ONLY))).thenReturn(statement); + when(statement.executeQuery(anyString())).thenAnswer(answer -> { + ResultSet resultSet = mock(ResultSet.class, RETURNS_SMART_NULLS); + when(resultSet.next()).thenReturn(true, true, true, false); + when(resultSet.getString(1)).thenReturn("AREALTABLE_1", "AREALTABLE_PRF1", "AREALTABLE_PRF2"); + when(resultSet.getString(2)).thenReturn("REALNAME:[AREALTABLE_1]", "REALNAME:[AREALTABLE_PRF1]", "REALNAME:[AREALTABLE_PRF2]"); + when(resultSet.getString(3)).thenReturn("AREALTABLE", "AREALTABLE", "AREALTABLE"); + when(resultSet.getString(4)).thenReturn("REALNAME:[ARealTable]", "REALNAME:[ARealTable]", "REALNAME:[ARealTable]"); + return resultSet; + }); + + DatabaseMetaData databaseMetaData = mock(DatabaseMetaData.class, RETURNS_SMART_NULLS); + when(connection.getMetaData()).thenReturn(databaseMetaData); + + // mock getTables + when(databaseMetaData.getTables(null, TEST_SCHEMA, null, new String[] { "TABLE" })) + .thenAnswer(answer -> { + ResultSet resultSet = mock(ResultSet.class, RETURNS_SMART_NULLS); + when(resultSet.next()).thenReturn(true, false); + when(resultSet.getString(3)).thenReturn(TABLE_NAME); // // 3 - TABLE_NAME + when(resultSet.getString(2)).thenReturn(TEST_SCHEMA); // 2 - TABLE_SCHEM + when(resultSet.getString(5)).thenReturn("REALNAME:[AREALTABLE]"); // 5 - TABLE_REMARKS + when(resultSet.getString(4)).thenReturn("REGULAR"); // 4 - TABLE_TYPE + + return resultSet; + }); + + // mock getColumns + when(databaseMetaData.getColumns(null, TEST_SCHEMA, null, null)) + .thenAnswer(answer -> { + ResultSet resultSet = mock(ResultSet.class, RETURNS_SMART_NULLS); + when(resultSet.next()).thenReturn(true, false); + when(resultSet.getString(3)).thenReturn(TABLE_NAME); // 3 - COLUMN_TABLE_NAME + when(resultSet.getString(4)).thenReturn("column1"); // 4 - COLUMN_NAME + when(resultSet.getString(6)).thenReturn("VARCHAR"); // 6 - COLUMN_TYPE_NAME + when(resultSet.getInt(5)).thenReturn(12); // 5 - COLUMN_DATA_TYPE - VARCHAR 12 + when(resultSet.getInt(7)).thenReturn(1); // 7 - COLUMN_SIZE - width + when(resultSet.getInt(9)).thenReturn(0); // 9 - COLUMN_DECIMAL_DIGITS - scale + when(resultSet.getString(12)).thenReturn("REALNAME:[column1]/TYPE:[STRING]"); // 12 - TABLE_REMARKS + when(resultSet.getString(18)).thenReturn("NO"); // 18 - COLUMN_IS_NULLABLE + when(resultSet.getString(23)).thenReturn("NO"); // 23 - COLUMN_IS_AUTOINCREMENT + when(resultSet.getString(23)).thenReturn(null); // 13 - COLUMN_DEFAULT_EXPR + + return resultSet; + }); + + // mock getIndexInfo + when(databaseMetaData.getIndexInfo(null, TEST_SCHEMA, "AREALTABLE", false, false)) + .thenAnswer(answer -> { + ResultSet resultSet = mock(ResultSet.class, RETURNS_SMART_NULLS); + when(resultSet.next()).thenReturn(true, true, true, false, true, true, true, false); + when(resultSet.getString(6)).thenReturn("AREALTABLE_1", "AREALTABLE_PRF1", "AREALTABLE_PRF2", "AREALTABLE_1", "AREALTABLE_PRF1", "AREALTABLE_PRF2"); // 6 - INDEX_NAME + when(resultSet.getString(9)).thenReturn("column1", "column1", "column1", "column1", "column1", "column1"); + when(resultSet.getBoolean(4)).thenReturn(true, true, true, true, true, true); // 4 - INDEX_NON_UNIQUE + return resultSet; + }); + + // When + final Schema postgresMetaDataProvider = postgres.openSchema(connection, "TestDatabase", TEST_SCHEMA); + Map> ignoredIndexesMap = ((AdditionalMetadata)postgresMetaDataProvider).ignoredIndexes(); + // test loading the cached version: + Map> ignoredIndexesMap1 = ((AdditionalMetadata)postgresMetaDataProvider).ignoredIndexes(); + + // Then + assertEquals("map size must match", 1, ignoredIndexesMap.size()); + assertEquals("map size must match", 1, ignoredIndexesMap1.size()); + String tableNameLowerCase = TABLE_NAME.toLowerCase(); + assertEquals("table ignored indexes size must match", 2, ignoredIndexesMap.get(tableNameLowerCase).size()); + Index indexPrf1 = ignoredIndexesMap.get(tableNameLowerCase).get(0); + Index indexPrf2 = ignoredIndexesMap.get(tableNameLowerCase).get(1); + assertEquals("index prf1 name", "AREALTABLE_PRF1", indexPrf1.getName()); + assertThat("index prf1 columns", indexPrf1.columnNames(), contains("column1")); + assertEquals("index prf2 name", "AREALTABLE_PRF2", indexPrf2.getName()); + assertThat("index prf2 columns", indexPrf2.columnNames(), contains("column1")); + } + + + /** + * Checks the SQL run for retrieving partitioned tables information + * + * @throws SQLException exception + */ + @Test + public void testLoadPartitionedTables() throws SQLException { + // Given + final Statement statement = mock(PreparedStatement.class, RETURNS_SMART_NULLS); + when(connection.createStatement()).thenReturn(statement); + when(statement.executeQuery("select par.relname as tableName, d.description\n" + + "from pg_class par \n" + + "join pg_namespace n on n.oid = par.relnamespace \n" + + "join pg_description d ON d.objoid = par.oid and d.objsubid = 0\n" + + "where not par.relispartition and par.relkind = 'p'")) + .thenAnswer(new ReturnMockResultSetWithPartitionTables(1, "partition", "REALNAME:[Partition]")); + when(statement.executeQuery("select par.relname, d.description\n" + + "from pg_class par \n" + + "join pg_namespace n on n.oid = par.relnamespace\n" + + "join pg_description d ON d.objoid = par.oid and d.objsubid = 0\n" + + " where par.relispartition and par.relkind = 'r'")) + .thenAnswer(new ReturnMockResultSetWithPartitionTables(1, "partition_p0", "REALNAME:[Partition_p0]")); + + // When + final AdditionalMetadata postgresMetaDataProvider = (AdditionalMetadata)postgres.openSchema(connection, "TestDatabase", "TestSchema"); + assertEquals("Partition Table name", "[partition/Partition]", postgresMetaDataProvider.partitionedTableNames().toString()); + DatabaseMetaDataProvider.RealName partitionTable = postgresMetaDataProvider.partitionedTableNames().iterator().next(); + assertEquals("Partition Table name", "partition", partitionTable.getDbName()); + } + + + /** + * Checks the SQL run for retrieving partition table information + * + * @throws SQLException exception + */ + @Test + public void testLoadPartitionTables() throws SQLException { + // Given + final Statement statement = mock(PreparedStatement.class, RETURNS_SMART_NULLS); + when(connection.createStatement()).thenReturn(statement); + when(statement.executeQuery("select par.relname as tableName, d.description\n" + + "from pg_class par \n" + + "join pg_namespace n on n.oid = par.relnamespace \n" + + "join pg_description d ON d.objoid = par.oid and d.objsubid = 0\n" + + "where not par.relispartition and par.relkind = 'p'")) + .thenAnswer(new ReturnMockResultSetWithPartitionTables(1, "partition", "Partition")); + when(statement.executeQuery("select par.relname, d.description\n" + + "from pg_class par \n" + + "join pg_namespace n on n.oid = par.relnamespace\n" + + "join pg_description d ON d.objoid = par.oid and d.objsubid = 0\n" + + " where par.relispartition and par.relkind = 'r'")) + .thenAnswer(new ReturnMockResultSetWithPartitionTables(1, "partition_p0", "Partition_p0")); + + // When + final AdditionalMetadata postgresMetaDataProvider = (AdditionalMetadata)postgres.openSchema(connection, "TestDatabase", "TestSchema"); + + assertEquals("Partition Table name", "[partition_p0/Partition_p0]", postgresMetaDataProvider.partitionTableNames().toString()); + DatabaseMetaDataProvider.RealName partitionTable = postgresMetaDataProvider.partitionTableNames().iterator().next(); + assertEquals("Partition Table name", "partition_p0", partitionTable.getDbName()); + } + + + /** + * Checks the SQL run for retrieving partition table information + * + * @throws SQLException exception + */ + @Test + public void testIgnoredTables() throws SQLException { + // Given + final Statement statement = mock(PreparedStatement.class, RETURNS_SMART_NULLS); + + final PreparedStatement statement1 = mock(PreparedStatement.class, RETURNS_SMART_NULLS); + when(connection.prepareStatement(anyString())).thenReturn(statement1); + + when(connection.createStatement()).thenReturn(statement); + when(statement.executeQuery("select par.relname as tableName, d.description\n" + + "from pg_class par \n" + + "join pg_namespace n on n.oid = par.relnamespace \n" + + "join pg_description d ON d.objoid = par.oid and d.objsubid = 0\n" + + "where not par.relispartition and par.relkind = 'p'")) + .thenAnswer(new ReturnMockResultSetWithPartitionTables(1, "partition", "REALNAME:[Partition]")); + when(statement.executeQuery("select par.relname, d.description\n" + + "from pg_class par \n" + + "join pg_namespace n on n.oid = par.relnamespace\n" + + "join pg_description d ON d.objoid = par.oid and d.objsubid = 0\n" + + " where par.relispartition and par.relkind = 'r'")) + .thenAnswer(new ReturnMockResultSetWithPartitionTables(1, "partition_p0", "REALNAME:[Partition_p0]")); + DatabaseMetaData postgreSQLMetaDataMock = mock(DatabaseMetaData.class); + when(connection.getMetaData()).thenReturn(postgreSQLMetaDataMock); + when(postgreSQLMetaDataMock.getTables(any(), any(), any(), any())) + .thenAnswer(new ReturnMockResultSetWithSequence(0)); + + // When + final Schema postgresMetaDataProvider = postgres.openSchema(connection, "TestDatabase", "TestSchema"); + // Then + assertEquals("Partition Table name", "[Partition]", postgresMetaDataProvider.tableNames().toString()); + assertFalse("Table names", postgresMetaDataProvider.tableNames().toString().contains("partition_p0")); + } + + + /** + * Mockito {@link Answer} that returns a mock result set with a given number of resultRows. + */ + private static final class ReturnMockResultSetWithSequence implements Answer { + + private final int numberOfResultRows; + + + /** + * @param numberOfResultRows number of result rows + */ + private ReturnMockResultSetWithSequence(int numberOfResultRows) { + super(); + this.numberOfResultRows = numberOfResultRows; + } + + @Override + public ResultSet answer(final InvocationOnMock invocation) throws Throwable { + final ResultSet resultSet = mock(ResultSet.class, RETURNS_SMART_NULLS); + when(resultSet.next()).thenAnswer(new Answer() { + private int counter; + + @Override + public Boolean answer(InvocationOnMock invocation) throws Throwable { + return counter++ < numberOfResultRows; + } + }); + + when(resultSet.getString(1)).thenReturn("Sequence1"); + + return resultSet; + } + } + + /** + * Mockito {@link Answer} that returns a mock result set with a given number of resultRows for partition tables. + */ + private static final class ReturnMockResultSetWithPartitionTables implements Answer { + + private final int numberOfResultRows; + private final String partitionResult; + private final String partitionName; + + + /** + * @param numberOfResultRows number of result rows + */ + private ReturnMockResultSetWithPartitionTables(int numberOfResultRows, String partitionResult, String partitionName) { + super(); + this.numberOfResultRows = numberOfResultRows; + // class is rigged for just one value + this.partitionResult = partitionResult; + this.partitionName = partitionName; + } + + @Override + public ResultSet answer(final InvocationOnMock invocation) throws Throwable { + final ResultSet resultSet = mock(ResultSet.class, RETURNS_SMART_NULLS); + when(resultSet.next()).thenAnswer(new Answer() { + private int counter; + + @Override + public Boolean answer(InvocationOnMock invocation) throws Throwable { + return counter++ < numberOfResultRows; + } + }); + + when(resultSet.getString(1)).thenReturn(partitionResult); + when(resultSet.getString(2)).thenReturn(partitionName); + + return resultSet; + } + } +}