diff --git a/Apache-Arrow-Flight-Tester/protobuf/protocol.proto b/Apache-Arrow-Flight-Tester/protobuf/protocol.proto index d26b19e..58462ce 100644 --- a/Apache-Arrow-Flight-Tester/protobuf/protocol.proto +++ b/Apache-Arrow-Flight-Tester/protobuf/protocol.proto @@ -67,14 +67,20 @@ message Configuration { // The number of seconds between each transfer of data to the remote object store. optional uint64 transfer_time_in_seconds = 5; + // The approximate maximum size, in bytes, of a single WAL segment file before it is closed and a new one is started. + uint64 segment_size_threshold_in_bytes = 6; + // Number of threads to allocate for converting multivariate time series to univariate time series. - uint32 ingestion_threads = 6; + uint32 ingestion_threads = 7; // Number of threads to allocate for compressing univariate time series to segments. - uint32 compression_threads = 7; + uint32 compression_threads = 8; // Number of threads to allocate for writing segments to a local and/or remote data folder. - uint32 writer_threads = 8; + uint32 writer_threads = 9; + + // Whether the write-ahead log is enabled. + bool wal_enabled = 10; } // Request to update the configuration of a ModelarDB node. @@ -85,6 +91,7 @@ message UpdateConfiguration { COMPRESSED_RESERVED_MEMORY_IN_BYTES = 2; TRANSFER_BATCH_SIZE_IN_BYTES = 3; TRANSFER_TIME_IN_SECONDS = 4; + SEGMENT_SIZE_THRESHOLD_IN_BYTES = 5; } // Setting to update in the configuration. diff --git a/Apache-Arrow-Flight-Tester/protobuf/protocol_pb2.py b/Apache-Arrow-Flight-Tester/protobuf/protocol_pb2.py index 8ba9068..6aaa638 100644 --- a/Apache-Arrow-Flight-Tester/protobuf/protocol_pb2.py +++ b/Apache-Arrow-Flight-Tester/protobuf/protocol_pb2.py @@ -24,7 +24,7 @@ -DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n\x0eprotocol.proto\x12\x19modelardb.flight.protocol\"\xfe\x04\n\rTableMetadata\x12T\n\x0cnormal_table\x18\x01 \x01(\x0b\x32<.modelardb.flight.protocol.TableMetadata.NormalTableMetadataH\x00\x12]\n\x11time_series_table\x18\x02 \x01(\x0b\x32@.modelardb.flight.protocol.TableMetadata.TimeSeriesTableMetadataH\x00\x1a\x33\n\x13NormalTableMetadata\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\x0e\n\x06schema\x18\x02 \x01(\x0c\x1a\xf0\x02\n\x17TimeSeriesTableMetadata\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\x0e\n\x06schema\x18\x02 \x01(\x0c\x12\x61\n\x0c\x65rror_bounds\x18\x03 \x03(\x0b\x32K.modelardb.flight.protocol.TableMetadata.TimeSeriesTableMetadata.ErrorBound\x12$\n\x1cgenerated_column_expressions\x18\x04 \x03(\x0c\x1a\xad\x01\n\nErrorBound\x12^\n\x04type\x18\x01 \x01(\x0e\x32P.modelardb.flight.protocol.TableMetadata.TimeSeriesTableMetadata.ErrorBound.Type\x12\r\n\x05value\x18\x02 \x01(\x02\"0\n\x04Type\x12\x0c\n\x08\x41\x42SOLUTE\x10\x00\x12\x0c\n\x08RELATIVE\x10\x01\x12\x0c\n\x08LOSSLESS\x10\x02\x42\x10\n\x0etable_metadata\"\xfa\x02\n\rConfiguration\x12-\n%multivariate_reserved_memory_in_bytes\x18\x01 \x01(\x04\x12-\n%uncompressed_reserved_memory_in_bytes\x18\x02 \x01(\x04\x12+\n#compressed_reserved_memory_in_bytes\x18\x03 \x01(\x04\x12)\n\x1ctransfer_batch_size_in_bytes\x18\x04 \x01(\x04H\x00\x88\x01\x01\x12%\n\x18transfer_time_in_seconds\x18\x05 \x01(\x04H\x01\x88\x01\x01\x12\x19\n\x11ingestion_threads\x18\x06 \x01(\r\x12\x1b\n\x13\x63ompression_threads\x18\x07 \x01(\r\x12\x16\n\x0ewriter_threads\x18\x08 \x01(\rB\x1f\n\x1d_transfer_batch_size_in_bytesB\x1b\n\x19_transfer_time_in_seconds\"\xcf\x02\n\x13UpdateConfiguration\x12G\n\x07setting\x18\x01 \x01(\x0e\x32\x36.modelardb.flight.protocol.UpdateConfiguration.Setting\x12\x16\n\tnew_value\x18\x02 \x01(\x04H\x00\x88\x01\x01\"\xc8\x01\n\x07Setting\x12)\n%MULTIVARIATE_RESERVED_MEMORY_IN_BYTES\x10\x00\x12)\n%UNCOMPRESSED_RESERVED_MEMORY_IN_BYTES\x10\x01\x12\'\n#COMPRESSED_RESERVED_MEMORY_IN_BYTES\x10\x02\x12 \n\x1cTRANSFER_BATCH_SIZE_IN_BYTES\x10\x03\x12\x1c\n\x18TRANSFER_TIME_IN_SECONDS\x10\x04\x42\x0c\n\n_new_valueb\x06proto3') +DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n\x0eprotocol.proto\x12\x19modelardb.flight.protocol\"\xfe\x04\n\rTableMetadata\x12T\n\x0cnormal_table\x18\x01 \x01(\x0b\x32<.modelardb.flight.protocol.TableMetadata.NormalTableMetadataH\x00\x12]\n\x11time_series_table\x18\x02 \x01(\x0b\x32@.modelardb.flight.protocol.TableMetadata.TimeSeriesTableMetadataH\x00\x1a\x33\n\x13NormalTableMetadata\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\x0e\n\x06schema\x18\x02 \x01(\x0c\x1a\xf0\x02\n\x17TimeSeriesTableMetadata\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\x0e\n\x06schema\x18\x02 \x01(\x0c\x12\x61\n\x0c\x65rror_bounds\x18\x03 \x03(\x0b\x32K.modelardb.flight.protocol.TableMetadata.TimeSeriesTableMetadata.ErrorBound\x12$\n\x1cgenerated_column_expressions\x18\x04 \x03(\x0c\x1a\xad\x01\n\nErrorBound\x12^\n\x04type\x18\x01 \x01(\x0e\x32P.modelardb.flight.protocol.TableMetadata.TimeSeriesTableMetadata.ErrorBound.Type\x12\r\n\x05value\x18\x02 \x01(\x02\"0\n\x04Type\x12\x0c\n\x08\x41\x42SOLUTE\x10\x00\x12\x0c\n\x08RELATIVE\x10\x01\x12\x0c\n\x08LOSSLESS\x10\x02\x42\x10\n\x0etable_metadata\"\xb8\x03\n\rConfiguration\x12-\n%multivariate_reserved_memory_in_bytes\x18\x01 \x01(\x04\x12-\n%uncompressed_reserved_memory_in_bytes\x18\x02 \x01(\x04\x12+\n#compressed_reserved_memory_in_bytes\x18\x03 \x01(\x04\x12)\n\x1ctransfer_batch_size_in_bytes\x18\x04 \x01(\x04H\x00\x88\x01\x01\x12%\n\x18transfer_time_in_seconds\x18\x05 \x01(\x04H\x01\x88\x01\x01\x12\'\n\x1fsegment_size_threshold_in_bytes\x18\x06 \x01(\x04\x12\x19\n\x11ingestion_threads\x18\x07 \x01(\r\x12\x1b\n\x13\x63ompression_threads\x18\x08 \x01(\r\x12\x16\n\x0ewriter_threads\x18\t \x01(\r\x12\x13\n\x0bwal_enabled\x18\n \x01(\x08\x42\x1f\n\x1d_transfer_batch_size_in_bytesB\x1b\n\x19_transfer_time_in_seconds\"\xf4\x02\n\x13UpdateConfiguration\x12G\n\x07setting\x18\x01 \x01(\x0e\x32\x36.modelardb.flight.protocol.UpdateConfiguration.Setting\x12\x16\n\tnew_value\x18\x02 \x01(\x04H\x00\x88\x01\x01\"\xed\x01\n\x07Setting\x12)\n%MULTIVARIATE_RESERVED_MEMORY_IN_BYTES\x10\x00\x12)\n%UNCOMPRESSED_RESERVED_MEMORY_IN_BYTES\x10\x01\x12\'\n#COMPRESSED_RESERVED_MEMORY_IN_BYTES\x10\x02\x12 \n\x1cTRANSFER_BATCH_SIZE_IN_BYTES\x10\x03\x12\x1c\n\x18TRANSFER_TIME_IN_SECONDS\x10\x04\x12#\n\x1fSEGMENT_SIZE_THRESHOLD_IN_BYTES\x10\x05\x42\x0c\n\n_new_valueb\x06proto3') _globals = globals() _builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, _globals) @@ -42,9 +42,9 @@ _globals['_TABLEMETADATA_TIMESERIESTABLEMETADATA_ERRORBOUND_TYPE']._serialized_start=618 _globals['_TABLEMETADATA_TIMESERIESTABLEMETADATA_ERRORBOUND_TYPE']._serialized_end=666 _globals['_CONFIGURATION']._serialized_start=687 - _globals['_CONFIGURATION']._serialized_end=1065 - _globals['_UPDATECONFIGURATION']._serialized_start=1068 - _globals['_UPDATECONFIGURATION']._serialized_end=1403 - _globals['_UPDATECONFIGURATION_SETTING']._serialized_start=1189 - _globals['_UPDATECONFIGURATION_SETTING']._serialized_end=1389 + _globals['_CONFIGURATION']._serialized_end=1127 + _globals['_UPDATECONFIGURATION']._serialized_start=1130 + _globals['_UPDATECONFIGURATION']._serialized_end=1502 + _globals['_UPDATECONFIGURATION_SETTING']._serialized_start=1251 + _globals['_UPDATECONFIGURATION_SETTING']._serialized_end=1488 # @@protoc_insertion_point(module_scope) diff --git a/Apache-Arrow-Flight-Tester/protobuf/protocol_pb2.pyi b/Apache-Arrow-Flight-Tester/protobuf/protocol_pb2.pyi index 01f2fc5..5a61d93 100644 --- a/Apache-Arrow-Flight-Tester/protobuf/protocol_pb2.pyi +++ b/Apache-Arrow-Flight-Tester/protobuf/protocol_pb2.pyi @@ -49,24 +49,28 @@ class TableMetadata(_message.Message): def __init__(self, normal_table: _Optional[_Union[TableMetadata.NormalTableMetadata, _Mapping]] = ..., time_series_table: _Optional[_Union[TableMetadata.TimeSeriesTableMetadata, _Mapping]] = ...) -> None: ... class Configuration(_message.Message): - __slots__ = ("multivariate_reserved_memory_in_bytes", "uncompressed_reserved_memory_in_bytes", "compressed_reserved_memory_in_bytes", "transfer_batch_size_in_bytes", "transfer_time_in_seconds", "ingestion_threads", "compression_threads", "writer_threads") + __slots__ = ("multivariate_reserved_memory_in_bytes", "uncompressed_reserved_memory_in_bytes", "compressed_reserved_memory_in_bytes", "transfer_batch_size_in_bytes", "transfer_time_in_seconds", "segment_size_threshold_in_bytes", "ingestion_threads", "compression_threads", "writer_threads", "wal_enabled") MULTIVARIATE_RESERVED_MEMORY_IN_BYTES_FIELD_NUMBER: _ClassVar[int] UNCOMPRESSED_RESERVED_MEMORY_IN_BYTES_FIELD_NUMBER: _ClassVar[int] COMPRESSED_RESERVED_MEMORY_IN_BYTES_FIELD_NUMBER: _ClassVar[int] TRANSFER_BATCH_SIZE_IN_BYTES_FIELD_NUMBER: _ClassVar[int] TRANSFER_TIME_IN_SECONDS_FIELD_NUMBER: _ClassVar[int] + SEGMENT_SIZE_THRESHOLD_IN_BYTES_FIELD_NUMBER: _ClassVar[int] INGESTION_THREADS_FIELD_NUMBER: _ClassVar[int] COMPRESSION_THREADS_FIELD_NUMBER: _ClassVar[int] WRITER_THREADS_FIELD_NUMBER: _ClassVar[int] + WAL_ENABLED_FIELD_NUMBER: _ClassVar[int] multivariate_reserved_memory_in_bytes: int uncompressed_reserved_memory_in_bytes: int compressed_reserved_memory_in_bytes: int transfer_batch_size_in_bytes: int transfer_time_in_seconds: int + segment_size_threshold_in_bytes: int ingestion_threads: int compression_threads: int writer_threads: int - def __init__(self, multivariate_reserved_memory_in_bytes: _Optional[int] = ..., uncompressed_reserved_memory_in_bytes: _Optional[int] = ..., compressed_reserved_memory_in_bytes: _Optional[int] = ..., transfer_batch_size_in_bytes: _Optional[int] = ..., transfer_time_in_seconds: _Optional[int] = ..., ingestion_threads: _Optional[int] = ..., compression_threads: _Optional[int] = ..., writer_threads: _Optional[int] = ...) -> None: ... + wal_enabled: bool + def __init__(self, multivariate_reserved_memory_in_bytes: _Optional[int] = ..., uncompressed_reserved_memory_in_bytes: _Optional[int] = ..., compressed_reserved_memory_in_bytes: _Optional[int] = ..., transfer_batch_size_in_bytes: _Optional[int] = ..., transfer_time_in_seconds: _Optional[int] = ..., segment_size_threshold_in_bytes: _Optional[int] = ..., ingestion_threads: _Optional[int] = ..., compression_threads: _Optional[int] = ..., writer_threads: _Optional[int] = ..., wal_enabled: bool = ...) -> None: ... class UpdateConfiguration(_message.Message): __slots__ = ("setting", "new_value") @@ -77,11 +81,13 @@ class UpdateConfiguration(_message.Message): COMPRESSED_RESERVED_MEMORY_IN_BYTES: _ClassVar[UpdateConfiguration.Setting] TRANSFER_BATCH_SIZE_IN_BYTES: _ClassVar[UpdateConfiguration.Setting] TRANSFER_TIME_IN_SECONDS: _ClassVar[UpdateConfiguration.Setting] + SEGMENT_SIZE_THRESHOLD_IN_BYTES: _ClassVar[UpdateConfiguration.Setting] MULTIVARIATE_RESERVED_MEMORY_IN_BYTES: UpdateConfiguration.Setting UNCOMPRESSED_RESERVED_MEMORY_IN_BYTES: UpdateConfiguration.Setting COMPRESSED_RESERVED_MEMORY_IN_BYTES: UpdateConfiguration.Setting TRANSFER_BATCH_SIZE_IN_BYTES: UpdateConfiguration.Setting TRANSFER_TIME_IN_SECONDS: UpdateConfiguration.Setting + SEGMENT_SIZE_THRESHOLD_IN_BYTES: UpdateConfiguration.Setting SETTING_FIELD_NUMBER: _ClassVar[int] NEW_VALUE_FIELD_NUMBER: _ClassVar[int] setting: UpdateConfiguration.Setting diff --git a/Apache-Arrow-Flight-Tester/requirements.txt b/Apache-Arrow-Flight-Tester/requirements.txt index 9db9c04..bd353fd 100644 Binary files a/Apache-Arrow-Flight-Tester/requirements.txt and b/Apache-Arrow-Flight-Tester/requirements.txt differ diff --git a/Apache-Arrow-Flight-Tester/server.py b/Apache-Arrow-Flight-Tester/server.py index 035520e..438247f 100644 --- a/Apache-Arrow-Flight-Tester/server.py +++ b/Apache-Arrow-Flight-Tester/server.py @@ -1,3 +1,5 @@ +import os + import pyarrow from pyarrow import flight from pyarrow._flight import Result, Ticket @@ -28,7 +30,7 @@ def workload_balanced_query(self, query: str) -> None: cloud_node_url = endpoint.locations[0] print(f"Executing query on {cloud_node_url}...") - cloud_client = ModelarDBServerFlightClient(cloud_node_url) + cloud_client = ModelarDBServerFlightClient(cloud_node_url, token=self._token) cloud_client.do_get(endpoint.ticket) def create_table(self, table_name: str, columns: list[tuple[str, str]], time_series_table=False) -> None: @@ -55,6 +57,10 @@ def vacuum_tables(self, table_names: list[str]) -> None: """Vacuum the given tables in the server.""" self.do_get(Ticket(f"VACUUM {', '.join(table_names)}")) + def optimize_tables(self, table_names: list[str]) -> None: + """Optimize the given tables in the server.""" + self.do_get(Ticket(f"OPTIMIZE {', '.join(table_names)}")) + def create_normal_table_from_metadata(self, table_name: str, schema: pyarrow.Schema) -> None: """Create a normal table using the table name and schema.""" normal_table_metadata = protocol_pb2.TableMetadata.NormalTableMetadata() @@ -107,7 +113,9 @@ def node_type(self) -> str: if __name__ == "__main__": - server_client = ModelarDBServerFlightClient("grpc://127.0.0.1:9999") + token = os.environ.get("MODELARDB_TOKEN") + server_client = ModelarDBServerFlightClient("grpc://127.0.0.1:9999", token=token) + print(f"Node type: {server_client.node_type()}\n") util.create_test_tables(server_client) diff --git a/Apache-Arrow-Flight-Tester/util.py b/Apache-Arrow-Flight-Tester/util.py index 758562a..ec190a0 100644 --- a/Apache-Arrow-Flight-Tester/util.py +++ b/Apache-Arrow-Flight-Tester/util.py @@ -42,9 +42,9 @@ def create_record_batch(num_rows: int) -> pyarrow.RecordBatch: def get_time_series_table_schema() -> pyarrow.Schema: """Return a schema for a time series table with one timestamp column, three tag columns, and three field columns.""" return pyarrow.schema([ - ("location", pyarrow.utf8()), - ("install_year", pyarrow.utf8()), - ("model", pyarrow.utf8()), + ("location", pyarrow.string_view()), + ("install_year", pyarrow.string_view()), + ("model", pyarrow.string_view()), ("timestamp", pyarrow.timestamp("us")), ("power_output", pyarrow.float32()), ("wind_speed", pyarrow.float32()), @@ -93,7 +93,7 @@ def create_test_tables_from_metadata(server_client: ModelarDBServerFlightClient) normal_table_schema = pyarrow.schema([ ("timestamp", pyarrow.timestamp("us")), ("values", pyarrow.float32()), - ("metadata", pyarrow.utf8()) + ("metadata", pyarrow.string_view()) ]) server_client.create_normal_table_from_metadata("test_table_1", normal_table_schema) diff --git a/Apache-Arrow-Flight-Tester/wrapper.py b/Apache-Arrow-Flight-Tester/wrapper.py index c8f2142..70e4b60 100644 --- a/Apache-Arrow-Flight-Tester/wrapper.py +++ b/Apache-Arrow-Flight-Tester/wrapper.py @@ -5,11 +5,36 @@ from pyarrow._flight import FlightInfo, ActionType, Result, Ticket +class _BearerTokenMiddlewareFactory(flight.ClientMiddlewareFactory): + """Creates middleware that attaches a Bearer token to every outgoing call.""" + + def __init__(self, token: str, *args, **kwargs): + super().__init__(*args, **kwargs) + self._token = token + + def start_call(self, _info) -> flight.ClientMiddleware: + return _BearerTokenMiddleware(self._token) + + +class _BearerTokenMiddleware(flight.ClientMiddleware): + """Adds 'authorization: Bearer ' to the outgoing request headers.""" + + def __init__(self, token: str, *args, **kwargs): + super().__init__(*args, **kwargs) + self._token = token + + def sending_headers(self) -> dict[str, str]: + return {"authorization": f"Bearer {self._token}"} + + class FlightClientWrapper: """Wrapper around the FlightClient class to simplify interaction with an Apache Arrow Flight server.""" - def __init__(self, location: str): - self.flight_client = flight.FlightClient(location) + def __init__(self, location: str, token: str | None = None): + self._token = token + + middleware = [_BearerTokenMiddlewareFactory(token)] if token else [] + self.flight_client = flight.FlightClient(location, middleware=middleware) def list_flights(self) -> list[FlightInfo]: """Wrapper around the list_flights method of the FlightClient class."""