From c0dd17695bd03c1269c47964da58da0983962b61 Mon Sep 17 00:00:00 2001 From: Marco Aurelio da Costa Date: Mon, 17 Mar 2025 09:39:53 +0100 Subject: [PATCH] Support for local tables in separated directory from root dir --- pybufrkit/__init__.py | 11 +++++++++++ pybufrkit/bufr.py | 4 +++- pybufrkit/coder.py | 5 ++++- pybufrkit/commands.py | 23 ++++++++++++++++++----- pybufrkit/decoder.py | 5 +++-- pybufrkit/encoder.py | 5 +++-- pybufrkit/tables.py | 19 ++++++++++++++----- 7 files changed, 56 insertions(+), 16 deletions(-) diff --git a/pybufrkit/__init__.py b/pybufrkit/__init__.py index 6f15bcc..9e94b5e 100644 --- a/pybufrkit/__init__.py +++ b/pybufrkit/__init__.py @@ -16,6 +16,7 @@ import argparse import logging +import os import sys from pybufrkit.commands import (command_compile, @@ -62,6 +63,12 @@ def main(): ap.add_argument('-t', '--tables-root-directory', help='The directory to locate BUFR tables') + ap.add_argument('-l', '--tables-local-directory', + help='The directory to locate local BUFR tables') + + ap.add_argument('-p', '--tables-local-provider', + help='The provider for local BUFR tables') + subparsers = ap.add_subparsers( dest='command', title='List of commands', @@ -261,6 +268,10 @@ def main(): ns = ap.parse_args() + ns.tables_local_directory = ns.tables_local_directory or ns.tables_root_directory + if ns.tables_local_provider: + ns.tables_local_directory = os.path.join(ns.tables_local_directory, ns.tables_local_provider) + if ns.info: logging_level = logging.INFO elif ns.debug: diff --git a/pybufrkit/bufr.py b/pybufrkit/bufr.py index e6680cb..07d8948 100644 --- a/pybufrkit/bufr.py +++ b/pybufrkit/bufr.py @@ -353,18 +353,20 @@ def add_section(self, section): """ self.sections.append(section) - def build_template(self, tables_root_dir, normalize=1): + def build_template(self, tables_root_dir, tables_local_dir=None, normalize=1): """ Build the BufrTemplate object using the list of unexpanded descriptors and corresponding table group. :param tables_root_dir: The root directory to find BUFR tables + :param tables_local_dir: The root directory to find local BUFR tables :param normalize: Whether to use some default table group if the specific one is not available. :return: A tuple of BufrTemplate and the associated TableGroup """ table_group = TableGroupCacheManager.get_table_group( tables_root_dir=tables_root_dir, + tables_local_dir=tables_local_dir, master_table_number=self.master_table_number.value, originating_centre=self.originating_centre.value, originating_subcentre=self.originating_subcentre.value, diff --git a/pybufrkit/coder.py b/pybufrkit/coder.py index dd3d4ab..371c145 100644 --- a/pybufrkit/coder.py +++ b/pybufrkit/coder.py @@ -260,14 +260,17 @@ class Coder(object): :param definitions_dir: Where to find the BPCL definition files. :param tables_root_dir: Where to find the BUFR table files. + :param tables_local_dir: Where to find the local BUFR table files. """ def __init__(self, definitions_dir=None, - tables_root_dir=None): + tables_root_dir=None, + tables_local_dir=None): self.section_configurer = SectionConfigurer(definitions_dir=definitions_dir) self.tables_root_dir = tables_root_dir or DEFAULT_TABLES_DIR + self.tables_local_dir = tables_local_dir or self.tables_root_dir @abc.abstractmethod def process(self, *args, **kwargs): diff --git a/pybufrkit/commands.py b/pybufrkit/commands.py index 28a63fa..d9c683a 100644 --- a/pybufrkit/commands.py +++ b/pybufrkit/commands.py @@ -36,6 +36,7 @@ def command_decode(ns): """ decoder = Decoder(definitions_dir=ns.definitions_directory, tables_root_dir=ns.tables_root_directory, + tables_local_dir=ns.tables_local_directory, compiled_template_cache_max=ns.compiled_template_cache_max) def show_message(m): @@ -78,11 +79,13 @@ def command_info(ns): """ flat_text_render = FlatTextRenderer() decoder = Decoder(definitions_dir=ns.definitions_directory, - tables_root_dir=ns.tables_root_directory) + tables_root_dir=ns.tables_root_directory, + tables_local_dir=ns.tables_local_directory) def show_message_info(m): bufr_template, table_group = m.build_template( - ns.tables_root_directory, normalize=1) + ns.tables_root_directory, + ns.tables_local_directory, normalize=1) print(flat_text_render.render(m)) if ns.template: @@ -114,6 +117,7 @@ def command_encode(ns): """ encoder = Encoder(definitions_dir=ns.definitions_directory, tables_root_dir=ns.tables_root_directory, + tables_local_dir=ns.tables_local_directory, compiled_template_cache_max=ns.compiled_template_cache_max, master_table_version=ns.master_table_version) if ns.filename != '-': @@ -159,7 +163,8 @@ def command_split(ns): BufrMessage. """ decoder = Decoder(definitions_dir=ns.definitions_directory, - tables_root_dir=ns.tables_root_directory) + tables_root_dir=ns.tables_root_directory, + tables_local_dir=ns.tables_local_directory) for filename in ns.filenames: with open(filename, 'rb') as ins: @@ -179,6 +184,7 @@ def command_lookup(ns): Command to lookup the given descriptors from command line """ table_group = TableGroupCacheManager.get_table_group(ns.tables_root_directory, + ns.tables_local_directory, ns.master_table_number, ns.originating_centre, ns.originating_subcentre, @@ -223,12 +229,15 @@ def command_compile(ns): if os.path.exists(ns.input): decoder = Decoder(definitions_dir=ns.definitions_directory, - tables_root_dir=ns.tables_root_directory) + tables_root_dir=ns.tables_root_directory, + tables_local_dir=ns.tables_local_directory) with open(ns.input, 'rb') as ins: bufr_message = decoder.process(ins.read(), file_path=ns.input, info_only=True) - template, table_group = bufr_message.build_template(ns.tables_root_directory, normalize=1) + template, table_group = bufr_message.build_template(ns.tables_root_directory, + ns.tables_local_directory, normalize=1) else: table_group = TableGroupCacheManager.get_table_group(ns.tables_root_directory, + ns.tables_local_directory, ns.master_table_number, ns.originating_centre, ns.originating_subcentre, @@ -247,9 +256,11 @@ def command_subset(ns): """ decoder = Decoder(definitions_dir=ns.definitions_directory, tables_root_dir=ns.tables_root_directory, + tables_local_dir=ns.tables_local_directory, compiled_template_cache_max=ns.compiled_template_cache_max) encoder = Encoder(definitions_dir=ns.definitions_directory, tables_root_dir=ns.tables_root_directory, + tables_local_dir=ns.tables_local_directory, compiled_template_cache_max=ns.compiled_template_cache_max) subset_indices = [int(x) for x in ns.subset_indices.split(',')] @@ -272,6 +283,7 @@ def command_query(ns): """ decoder = Decoder(definitions_dir=ns.definitions_directory, tables_root_dir=ns.tables_root_directory, + tables_local_dir=ns.tables_local_directory, compiled_template_cache_max=ns.compiled_template_cache_max) for filename in ns.filenames: @@ -322,6 +334,7 @@ def command_script(ns): decoder = Decoder(definitions_dir=ns.definitions_directory, tables_root_dir=ns.tables_root_directory, + tables_local_dir=ns.tables_local_directory, compiled_template_cache_max=ns.compiled_template_cache_max) for filename in ns.filenames: diff --git a/pybufrkit/decoder.py b/pybufrkit/decoder.py index a8dea18..133bc64 100644 --- a/pybufrkit/decoder.py +++ b/pybufrkit/decoder.py @@ -44,9 +44,10 @@ class Decoder(Coder): def __init__(self, definitions_dir=None, tables_root_dir=None, + tables_local_dir=None, compiled_template_cache_max=None): - super(Decoder, self).__init__(definitions_dir, tables_root_dir) + super(Decoder, self).__init__(definitions_dir, tables_root_dir, tables_local_dir) # Only enable template compilation if cache is requested if compiled_template_cache_max is not None: @@ -185,7 +186,7 @@ def process_template_data(self, bufr_message, bit_reader): :return: TemplateData decoded from the bit stream. """ # TODO: Parametrise the "normalize" argument - bufr_template, table_group = bufr_message.build_template(self.tables_root_dir, normalize=1) + bufr_template, table_group = bufr_message.build_template(self.tables_root_dir, self.tables_local_dir, normalize=1) state = CoderState(bufr_message.is_compressed.value, bufr_message.n_subsets.value) diff --git a/pybufrkit/encoder.py b/pybufrkit/encoder.py index 75168f1..885ec7c 100644 --- a/pybufrkit/encoder.py +++ b/pybufrkit/encoder.py @@ -56,12 +56,13 @@ class Encoder(Coder): def __init__(self, definitions_dir=None, tables_root_dir=None, + tables_local_dir=None, ignore_declared_length=True, compiled_template_cache_max=None, master_table_number=None, master_table_version=None): - super(Encoder, self).__init__(definitions_dir, tables_root_dir) + super(Encoder, self).__init__(definitions_dir, tables_root_dir, tables_local_dir) self.ignore_declared_length = ignore_declared_length self.overrides = {} if master_table_number: @@ -223,7 +224,7 @@ def process_template_data(self, bufr_message, bit_writer, section_parameter): :return: """ # TODO: Parametrise the "normalize" argument - bufr_template, table_group = bufr_message.build_template(self.tables_root_dir, normalize=0) + bufr_template, table_group = bufr_message.build_template(self.tables_root_dir, self.tables_local_dir, normalize=0) state = CoderState(bufr_message.is_compressed.value, bufr_message.n_subsets.value, section_parameter.value) diff --git a/pybufrkit/tables.py b/pybufrkit/tables.py index a2d8bc3..cd25438 100644 --- a/pybufrkit/tables.py +++ b/pybufrkit/tables.py @@ -45,6 +45,7 @@ log = logging.getLogger(__file__) TableGroupKey = namedtuple('TableGroupKey', ['tables_root_dir', + 'tables_local_dir', 'wmo_tables_sn', 'local_tables_sn']) @@ -76,6 +77,7 @@ def get_tables_sn(master_table_number, def normalize_tables_sn(tables_root_dir, + tables_local_dir, master_table_number, originating_centre, originating_subcentre, @@ -93,6 +95,7 @@ def normalize_tables_sn(tables_root_dir, :param local_table_version: :param master_table_number: :param tables_root_dir: + :param tables_local_dir: :return: The directory and SN for which tables can actually be found :rtype: (str, str) """ @@ -129,7 +132,7 @@ def normalize_tables_sn(tables_root_dir, '{}_{}'.format(originating_centre, DEFAULT_ORIGINATING_SUBCENTRE), ] for idx, centres in enumerate(centres_candidates): - if os.path.isdir(os.path.join(tables_root_dir, + if os.path.isdir(os.path.join(tables_local_dir, master_table_number_string, centres, local_table_version_string)): @@ -160,10 +163,11 @@ def __eq__(self, other): return type(self) is type(other) and self.table_group_key == other.table_group_key def __str__(self): - return '{}: {} - {}, {}'.format( + return '{}: {} - {}, {} - {}'.format( self.__class__.__name__, self.table_group_key.tables_root_dir, self.table_group_key.wmo_tables_sn, + self.table_group_key.tables_local_dir, self.table_group_key.local_tables_sn ) @@ -178,7 +182,7 @@ def tables_dir_wmo(self): @property def tables_dir_local(self): return ( - os.path.join(os.path.join(self.table_group_key.tables_root_dir, *self.table_group_key.local_tables_sn)) + os.path.join(os.path.join(self.table_group_key.tables_local_dir, *self.table_group_key.local_tables_sn)) if self.table_group_key.local_tables_sn else None ) @@ -366,10 +370,11 @@ def __eq__(self, other): ) def __str__(self): - return '<{}: {!r} - {}, {}>'.format( + return '<{}: {!r} - {}, {!r} - {}>'.format( self.__class__.__name__, self.A.tables_root_dir, self.A.wmo_tables_sn, + self.A.tables_local_dir, self.A.local_tables_sn, ) @@ -523,6 +528,7 @@ def get_table_group_by_key(cls, table_group_key): @classmethod def get_table_group(cls, tables_root_dir=None, + tables_local_dir=None, master_table_number=None, originating_centre=None, originating_subcentre=None, @@ -539,6 +545,7 @@ def get_table_group(cls, :param master_table_version: :param local_table_version: :param str tables_root_dir: Root directory to read the BUFR tables + :param str tables_local_dir: Root directory to read the local BUFR tables :param int|bool normalize: Whether the program tries to fix non-exist tables SN by using default values. This is generally useful for decoding. But could be misleading when encoding. @@ -546,10 +553,12 @@ def get_table_group(cls, """ tables_root_dir = tables_root_dir or DEFAULT_TABLES_DIR + tables_local_dir = tables_local_dir or tables_root_dir if normalize: wmo_tables_sn, local_tables_sn = normalize_tables_sn( tables_root_dir, + tables_local_dir, master_table_number or DEFAULT_MASTER_TABLE_NUMBER, originating_centre or DEFAULT_ORIGINATING_CENTRE, originating_subcentre or DEFAULT_ORIGINATING_SUBCENTRE, @@ -565,7 +574,7 @@ def get_table_group(cls, local_table_version ) - table_group_key = TableGroupKey(tables_root_dir, wmo_tables_sn, local_tables_sn) + table_group_key = TableGroupKey(tables_root_dir, tables_local_dir, wmo_tables_sn, local_tables_sn) # TODO: catch error on file reading? return TableGroupCacheManager.get_table_group_by_key(table_group_key)