Skip to content
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
12 changes: 12 additions & 0 deletions libs/input-mapping/src/Configuration/Table/Manifest.php
Original file line number Diff line number Diff line change
Expand Up @@ -27,6 +27,7 @@ public static function configureNode(ArrayNodeDefinition $node): void
->scalarNode('id')->isRequired()->end()
->scalarNode('name')->end()
->scalarNode('uri')->end()
->scalarNode('description')->end()
->arrayNode('primary_key')->prototype('scalar')->end()->end()
->arrayNode('distribution_key')->prototype('scalar')->end()->end()
->scalarNode('created')->end()
Expand Down Expand Up @@ -80,6 +81,17 @@ public static function configureNode(ArrayNodeDefinition $node): void
->end()
->defaultValue([])
->end()
->arrayNode('schema')
->prototype('array')
->children()
->scalarNode('name')->isRequired()->end()
->variableNode('data_type')->end()
->booleanNode('nullable')->end()
->booleanNode('primary_key')->end()
->scalarNode('description')->end()
->end()
->end()
->end()
->end()
;
// BEFORE MODIFICATION OF THIS CONFIGURATION, READ AND UNDERSTAND
Expand Down
81 changes: 81 additions & 0 deletions libs/input-mapping/src/Helper/ManifestCreator.php
Original file line number Diff line number Diff line change
Expand Up @@ -27,6 +27,9 @@ public function writeTableManifest(
'last_change_date' => $tableInfo['lastChangeDate'],
'last_import_date' => $tableInfo['lastImportDate'],
];
if (isset($tableInfo['definition']['description']) && $tableInfo['definition']['description'] !== '') {
$manifest['description'] = $tableInfo['definition']['description'];
}
if (isset($tableInfo['s3'])) {
$manifest['s3'] = $tableInfo['s3'];
}
Expand All @@ -44,6 +47,11 @@ public function writeTableManifest(
$manifest['column_metadata'][$column] = $columnMetadata;
}

$schema = $this->buildSchema($tableInfo, $columns);
if ($schema !== null) {
$manifest['schema'] = $schema;
}

$adapter = new TableAdapter($format);
try {
$adapter->setConfig($manifest);
Expand All @@ -61,6 +69,79 @@ public function writeTableManifest(
}
}

/**
* Builds the `schema` node describing each selected column's data type, nullability, primary-key flag and
* description from the table definition. Returns null when the table has no definition.
*
* @param string[] $columns
*/
private function buildSchema(array $tableInfo, array $columns): ?array
{
if (empty($tableInfo['definition']['columns'])) {
return null;
}

$backend = $tableInfo['bucket']['backend'] ?? null;
$primaryKeysNames = $tableInfo['definition']['primaryKeysNames'] ?? [];

$definitionColumns = [];
foreach ($tableInfo['definition']['columns'] as $definitionColumn) {
$definitionColumns[$definitionColumn['name']] = $definitionColumn;
}

$schema = [];
foreach ($columns as $columnName) {
if (!isset($definitionColumns[$columnName])) {
continue;
}
$definitionColumn = $definitionColumns[$columnName];
$definition = $definitionColumn['definition'] ?? [];

$column = ['name' => $columnName];
$dataType = $this->buildColumnDataType($definitionColumn, $backend);
if ($dataType !== []) {
$column['data_type'] = $dataType;
}
if (isset($definition['nullable'])) {
$column['nullable'] = $definition['nullable'];
}
$column['primary_key'] = in_array($columnName, $primaryKeysNames, true);
if (isset($definition['description']) && $definition['description'] !== '') {
$column['description'] = $definition['description'];
}
$schema[] = $column;
}

return $schema;
}

/**
* Builds the `data_type` node for a single column: the backend-agnostic `base` type plus the type as it exists
* on the table's actual backend (length and default are included only when present in the definition). Returns
* an empty array for non-typed columns, whose definition carries no type information.
*/
private function buildColumnDataType(array $definitionColumn, ?string $backend): array
{
$definition = $definitionColumn['definition'] ?? [];

$dataType = [];
if (isset($definitionColumn['basetype'])) {
$dataType['base'] = ['type' => $definitionColumn['basetype']];
}
if ($backend !== null && isset($definition['type'])) {
$backendType = ['type' => $definition['type']];
if (isset($definition['length'])) {
$backendType['length'] = $definition['length'];
}
if (isset($definition['default'])) {
$backendType['default'] = $definition['default'];
}
$dataType[$backend] = $backendType;
}

return $dataType;
}

public function createFileManifest(array $fileInfo): array
{
return [
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -45,7 +45,8 @@ public function setConfigAndSerializeData(): Generator
"distribution_key": [],
"columns": [],
"metadata": [],
"column_metadata": []
"column_metadata": [],
"schema": []
}
EOF,
];
Expand All @@ -58,6 +59,7 @@ public function setConfigAndSerializeData(): Generator
columns: { }
metadata: { }
column_metadata: { }
schema: { }

EOF,
];
Expand Down Expand Up @@ -97,7 +99,8 @@ public function fileOperationsData(): Generator
"distribution_key": [],
"columns": [],
"metadata": [],
"column_metadata": []
"column_metadata": [],
"schema": []
}
EOF,
];
Expand All @@ -110,6 +113,7 @@ public function fileOperationsData(): Generator
columns: { }
metadata: { }
column_metadata: { }
schema: { }

EOF,
];
Expand Down Expand Up @@ -139,6 +143,7 @@ public function testFileOperations(
'columns' => [],
'metadata' => [],
'column_metadata' => [],
'schema' => [],
], $adapter->readFromFile($filePathname));
}

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -56,6 +56,7 @@ public function testConfiguration(
'column_metadata' => $columnsMetadata,
];
$expectedResponse = $config;
$expectedResponse['schema'] = [];
$processedConfiguration = (new Manifest())->parse(['config' => $config]);

self::assertEquals($expectedResponse, $processedConfiguration);
Expand Down
Loading
Loading