Skip to content

Support for local tables in separated directory from root dir #41

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
ywangd merged 1 commit into from
Mar 22, 2025
Merged

Support for local tables in separated directory from root dir #41

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
11 changes: 11 additions & 0 deletions pybufrkit/__init__.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,7 @@

import argparse
import logging
import os
import sys

from pybufrkit.commands import (command_compile,
Expand Down Expand Up @@ -62,6 +63,12 @@ def main():
ap.add_argument('-t', '--tables-root-directory',
help='The directory to locate BUFR tables')

ap.add_argument('-l', '--tables-local-directory',
help='The directory to locate local BUFR tables')

ap.add_argument('-p', '--tables-local-provider',
help='The provider for local BUFR tables')

subparsers = ap.add_subparsers(
dest='command',
title='List of commands',
Expand Down Expand Up @@ -261,6 +268,10 @@ def main():

ns = ap.parse_args()

ns.tables_local_directory = ns.tables_local_directory or ns.tables_root_directory
if ns.tables_local_provider:
ns.tables_local_directory = os.path.join(ns.tables_local_directory, ns.tables_local_provider)

if ns.info:
logging_level = logging.INFO
elif ns.debug:
Expand Down
4 changes: 3 additions & 1 deletion pybufrkit/bufr.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -353,18 +353,20 @@ def add_section(self, section):
"""
self.sections.append(section)

def build_template(self, tables_root_dir, normalize=1):
def build_template(self, tables_root_dir, tables_local_dir=None, normalize=1):
"""
Build the BufrTemplate object using the list of unexpanded descriptors
and corresponding table group.

:param tables_root_dir: The root directory to find BUFR tables
:param tables_local_dir: The root directory to find local BUFR tables
:param normalize: Whether to use some default table group if the specific
one is not available.
:return: A tuple of BufrTemplate and the associated TableGroup
"""
table_group = TableGroupCacheManager.get_table_group(
tables_root_dir=tables_root_dir,
tables_local_dir=tables_local_dir,
master_table_number=self.master_table_number.value,
originating_centre=self.originating_centre.value,
originating_subcentre=self.originating_subcentre.value,
Expand Down
5 changes: 4 additions & 1 deletion pybufrkit/coder.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -260,14 +260,17 @@ class Coder(object):

:param definitions_dir: Where to find the BPCL definition files.
:param tables_root_dir: Where to find the BUFR table files.
:param tables_local_dir: Where to find the local BUFR table files.
"""

def __init__(self,
definitions_dir=None,
tables_root_dir=None):
tables_root_dir=None,
tables_local_dir=None):

self.section_configurer = SectionConfigurer(definitions_dir=definitions_dir)
self.tables_root_dir = tables_root_dir or DEFAULT_TABLES_DIR
self.tables_local_dir = tables_local_dir or self.tables_root_dir

@abc.abstractmethod
def process(self, *args, **kwargs):
Expand Down
23 changes: 18 additions & 5 deletions pybufrkit/commands.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -36,6 +36,7 @@ def command_decode(ns):
"""
decoder = Decoder(definitions_dir=ns.definitions_directory,
tables_root_dir=ns.tables_root_directory,
tables_local_dir=ns.tables_local_directory,
compiled_template_cache_max=ns.compiled_template_cache_max)

def show_message(m):
Expand Down Expand Up @@ -78,11 +79,13 @@ def command_info(ns):
"""
flat_text_render = FlatTextRenderer()
decoder = Decoder(definitions_dir=ns.definitions_directory,
tables_root_dir=ns.tables_root_directory)
tables_root_dir=ns.tables_root_directory,
tables_local_dir=ns.tables_local_directory)

def show_message_info(m):
bufr_template, table_group = m.build_template(
ns.tables_root_directory, normalize=1)
ns.tables_root_directory,
ns.tables_local_directory, normalize=1)

print(flat_text_render.render(m))
if ns.template:
Expand Down Expand Up @@ -114,6 +117,7 @@ def command_encode(ns):
"""
encoder = Encoder(definitions_dir=ns.definitions_directory,
tables_root_dir=ns.tables_root_directory,
tables_local_dir=ns.tables_local_directory,
compiled_template_cache_max=ns.compiled_template_cache_max,
master_table_version=ns.master_table_version)
if ns.filename != '-':
Expand Down Expand Up @@ -159,7 +163,8 @@ def command_split(ns):
BufrMessage.
"""
decoder = Decoder(definitions_dir=ns.definitions_directory,
tables_root_dir=ns.tables_root_directory)
tables_root_dir=ns.tables_root_directory,
tables_local_dir=ns.tables_local_directory)

for filename in ns.filenames:
with open(filename, 'rb') as ins:
Expand All @@ -179,6 +184,7 @@ def command_lookup(ns):
Command to lookup the given descriptors from command line
"""
table_group = TableGroupCacheManager.get_table_group(ns.tables_root_directory,
ns.tables_local_directory,
ns.master_table_number,
ns.originating_centre,
ns.originating_subcentre,
Expand Down Expand Up @@ -223,12 +229,15 @@ def command_compile(ns):

if os.path.exists(ns.input):
decoder = Decoder(definitions_dir=ns.definitions_directory,
tables_root_dir=ns.tables_root_directory)
tables_root_dir=ns.tables_root_directory,
tables_local_dir=ns.tables_local_directory)
with open(ns.input, 'rb') as ins:
bufr_message = decoder.process(ins.read(), file_path=ns.input, info_only=True)
template, table_group = bufr_message.build_template(ns.tables_root_directory, normalize=1)
template, table_group = bufr_message.build_template(ns.tables_root_directory,
ns.tables_local_directory, normalize=1)
else:
table_group = TableGroupCacheManager.get_table_group(ns.tables_root_directory,
ns.tables_local_directory,
ns.master_table_number,
ns.originating_centre,
ns.originating_subcentre,
Expand All @@ -247,9 +256,11 @@ def command_subset(ns):
"""
decoder = Decoder(definitions_dir=ns.definitions_directory,
tables_root_dir=ns.tables_root_directory,
tables_local_dir=ns.tables_local_directory,
compiled_template_cache_max=ns.compiled_template_cache_max)
encoder = Encoder(definitions_dir=ns.definitions_directory,
tables_root_dir=ns.tables_root_directory,
tables_local_dir=ns.tables_local_directory,
compiled_template_cache_max=ns.compiled_template_cache_max)

subset_indices = [int(x) for x in ns.subset_indices.split(',')]
Expand All @@ -272,6 +283,7 @@ def command_query(ns):
"""
decoder = Decoder(definitions_dir=ns.definitions_directory,
tables_root_dir=ns.tables_root_directory,
tables_local_dir=ns.tables_local_directory,
compiled_template_cache_max=ns.compiled_template_cache_max)

for filename in ns.filenames:
Expand Down Expand Up @@ -322,6 +334,7 @@ def command_script(ns):

decoder = Decoder(definitions_dir=ns.definitions_directory,
tables_root_dir=ns.tables_root_directory,
tables_local_dir=ns.tables_local_directory,
compiled_template_cache_max=ns.compiled_template_cache_max)

for filename in ns.filenames:
Expand Down
5 changes: 3 additions & 2 deletions pybufrkit/decoder.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -44,9 +44,10 @@ class Decoder(Coder):
def __init__(self,
definitions_dir=None,
tables_root_dir=None,
tables_local_dir=None,
compiled_template_cache_max=None):

super(Decoder, self).__init__(definitions_dir, tables_root_dir)
super(Decoder, self).__init__(definitions_dir, tables_root_dir, tables_local_dir)

# Only enable template compilation if cache is requested
if compiled_template_cache_max is not None:
Expand Down Expand Up @@ -185,7 +186,7 @@ def process_template_data(self, bufr_message, bit_reader):
:return: TemplateData decoded from the bit stream.
"""
# TODO: Parametrise the "normalize" argument
bufr_template, table_group = bufr_message.build_template(self.tables_root_dir, normalize=1)
bufr_template, table_group = bufr_message.build_template(self.tables_root_dir, self.tables_local_dir, normalize=1)

state = CoderState(bufr_message.is_compressed.value, bufr_message.n_subsets.value)

Expand Down
5 changes: 3 additions & 2 deletions pybufrkit/encoder.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -56,12 +56,13 @@ class Encoder(Coder):
def __init__(self,
definitions_dir=None,
tables_root_dir=None,
tables_local_dir=None,
ignore_declared_length=True,
compiled_template_cache_max=None,
master_table_number=None,
master_table_version=None):

super(Encoder, self).__init__(definitions_dir, tables_root_dir)
super(Encoder, self).__init__(definitions_dir, tables_root_dir, tables_local_dir)
self.ignore_declared_length = ignore_declared_length
self.overrides = {}
if master_table_number:
Expand Down Expand Up @@ -223,7 +224,7 @@ def process_template_data(self, bufr_message, bit_writer, section_parameter):
:return:
"""
# TODO: Parametrise the "normalize" argument
bufr_template, table_group = bufr_message.build_template(self.tables_root_dir, normalize=0)
bufr_template, table_group = bufr_message.build_template(self.tables_root_dir, self.tables_local_dir, normalize=0)

state = CoderState(bufr_message.is_compressed.value, bufr_message.n_subsets.value, section_parameter.value)

Expand Down
19 changes: 14 additions & 5 deletions pybufrkit/tables.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -45,6 +45,7 @@
log = logging.getLogger(__file__)

TableGroupKey = namedtuple('TableGroupKey', ['tables_root_dir',
'tables_local_dir',
'wmo_tables_sn',
'local_tables_sn'])

Expand Down Expand Up @@ -76,6 +77,7 @@ def get_tables_sn(master_table_number,


def normalize_tables_sn(tables_root_dir,
tables_local_dir,
master_table_number,
originating_centre,
originating_subcentre,
Expand All @@ -93,6 +95,7 @@ def normalize_tables_sn(tables_root_dir,
:param local_table_version:
:param master_table_number:
:param tables_root_dir:
:param tables_local_dir:
:return: The directory and SN for which tables can actually be found
:rtype: (str, str)
"""
Expand Down Expand Up @@ -129,7 +132,7 @@ def normalize_tables_sn(tables_root_dir,
'{}_{}'.format(originating_centre, DEFAULT_ORIGINATING_SUBCENTRE),
]
for idx, centres in enumerate(centres_candidates):
if os.path.isdir(os.path.join(tables_root_dir,
if os.path.isdir(os.path.join(tables_local_dir,
master_table_number_string,
centres,
local_table_version_string)):
Expand Down Expand Up @@ -160,10 +163,11 @@ def __eq__(self, other):
return type(self) is type(other) and self.table_group_key == other.table_group_key

def __str__(self):
return '{}: {} - {}, {}'.format(
return '{}: {} - {}, {} - {}'.format(
self.__class__.__name__,
self.table_group_key.tables_root_dir,
self.table_group_key.wmo_tables_sn,
self.table_group_key.tables_local_dir,
self.table_group_key.local_tables_sn
)

Expand All @@ -178,7 +182,7 @@ def tables_dir_wmo(self):
@property
def tables_dir_local(self):
return (
os.path.join(os.path.join(self.table_group_key.tables_root_dir, *self.table_group_key.local_tables_sn))
os.path.join(os.path.join(self.table_group_key.tables_local_dir, *self.table_group_key.local_tables_sn))
if self.table_group_key.local_tables_sn else None
)

Expand Down Expand Up @@ -366,10 +370,11 @@ def __eq__(self, other):
)

def __str__(self):
return '<{}: {!r} - {}, {}>'.format(
return '<{}: {!r} - {}, {!r} - {}>'.format(
self.__class__.__name__,
self.A.tables_root_dir,
self.A.wmo_tables_sn,
self.A.tables_local_dir,
self.A.local_tables_sn,
)

Expand Down Expand Up @@ -523,6 +528,7 @@ def get_table_group_by_key(cls, table_group_key):
@classmethod
def get_table_group(cls,
tables_root_dir=None,
tables_local_dir=None,
master_table_number=None,
originating_centre=None,
originating_subcentre=None,
Expand All @@ -539,17 +545,20 @@ def get_table_group(cls,
:param master_table_version:
:param local_table_version:
:param str tables_root_dir: Root directory to read the BUFR tables
:param str tables_local_dir: Root directory to read the local BUFR tables
:param int|bool normalize: Whether the program tries to fix non-exist tables SN by
using default values. This is generally useful for
decoding. But could be misleading when encoding.
:rtype: BufrTableGroup
"""

tables_root_dir = tables_root_dir or DEFAULT_TABLES_DIR
tables_local_dir = tables_local_dir or tables_root_dir

if normalize:
wmo_tables_sn, local_tables_sn = normalize_tables_sn(
tables_root_dir,
tables_local_dir,
master_table_number or DEFAULT_MASTER_TABLE_NUMBER,
originating_centre or DEFAULT_ORIGINATING_CENTRE,
originating_subcentre or DEFAULT_ORIGINATING_SUBCENTRE,
Expand All @@ -565,7 +574,7 @@ def get_table_group(cls,
local_table_version
)

table_group_key = TableGroupKey(tables_root_dir, wmo_tables_sn, local_tables_sn)
table_group_key = TableGroupKey(tables_root_dir, tables_local_dir, wmo_tables_sn, local_tables_sn)

# TODO: catch error on file reading?
return TableGroupCacheManager.get_table_group_by_key(table_group_key)