diff --git a/wiitility/BDLSections/__init__.py b/wiitility/BDLSections/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/wiitility/BDLSections/bdl_section.py b/wiitility/BDLSections/bdl_section.py new file mode 100644 index 0000000..98801d3 --- /dev/null +++ b/wiitility/BDLSections/bdl_section.py @@ -0,0 +1,30 @@ +from io import BytesIO +from typing import Self + +PADDING = "This is padding data to alignment" + +class BDLSection: + magic: str + data: BytesIO + + def __init__(self, magic: str): + self.magic = magic + + @classmethod + def import_section(cls, raw_bytes: BytesIO) -> Self: + """ + Import a section from raw bytes. + This method must be overridden in subclasses to provide proper implementation. + Raises AttributeError: If not properly overridden in a subclass. + """ + raise AttributeError("Import section is not properly overwritten") + return cls() + + def export_section(self) -> BytesIO: + """ + Export a section from raw bytes. + This method must be overridden in subclasses to provide proper implementation. + Raises AttributeError: If not properly overridden in a subclass. + """ + raise AttributeError("Export section is not properly overwritten") + return BytesIO() diff --git a/wiitility/BDLSections/inf1.py b/wiitility/BDLSections/inf1.py new file mode 100644 index 0000000..81673f2 --- /dev/null +++ b/wiitility/BDLSections/inf1.py @@ -0,0 +1,101 @@ +from enum import IntEnum +from io import BytesIO + +from wiitility import bytes_helpers as bh +from wiitility.BDLSections.bdl_section import BDLSection, PADDING + +INF1_MAGIC = "INF1" + +class HierarchyNodeType(IntEnum): + FINISH = 0x00 + NEW_NODE = 0x01 + END_NODE = 0x02 + JOINT = 0x10 + MATERIAL = 0x11 + SHAPE = 0x12 + +class HierarchyNode: + type: HierarchyNodeType + data: int + + def __init__(self, type: HierarchyNodeType, data: int): + self.type = type + self.data = data + +class INF1Section(BDLSection): + hierarchy_nodes: list[HierarchyNode] + + flags: int + matrix_group_count: int + vertex_count: int + + def __init__(self, hierarchy_nodes: list[HierarchyNode] = None, flags: int = 0, matrix_group_count: int = 0, vertex_count: int = 0): + super().__init__(INF1_MAGIC) + + if hierarchy_nodes: + assert isinstance(hierarchy_nodes[0], HierarchyNode) + else: + hierarchy_nodes = [] + + self.hierarchy_nodes = hierarchy_nodes + + self.flags = flags + self.matrix_group_count = matrix_group_count + self.vertex_count = vertex_count + + def export_section_with_vertex_count(self, vertex_count: int) -> BytesIO: + data = self.export_section() + bh.write_u32(data, 0x10, vertex_count) + return data + + @classmethod + def import_section(cls, raw_bytes: BytesIO): + magic = bh.read_str(raw_bytes, 0x0, 4) + assert magic == INF1_MAGIC + + size = bh.read_u32(raw_bytes, 0x4) + flags = bh.read_u16(raw_bytes, 0x8) + # 2 bytes of padding + matrix_group_count = bh.read_u32(raw_bytes, 0xC) + vertex_count = bh.read_u32(raw_bytes, 0x10) + hierarchy_data_offset = bh.read_u32(raw_bytes, 0x14) + + hierarchy_nodes: list[HierarchyNode] = [] + + offset = hierarchy_data_offset + while offset < size: + node_type = bh.read_u16(raw_bytes, offset + 0x0) + node_data = bh.read_u16(raw_bytes, offset + 0x2) + + node: HierarchyNode = HierarchyNode(node_type, node_data) + hierarchy_nodes.append(node) + + if node.type == HierarchyNodeType.FINISH: + break + + offset += 0x4 + + return cls(hierarchy_nodes, flags, matrix_group_count, vertex_count) + + def export_section(self) -> BytesIO: + data = BytesIO() + + bh.write_u16(data, 0x8, self.flags) + bh.write_s16(data, 0xA, -1) # 2 bytes of padding + + shape_nodes: list[HierarchyNode] = [node for node in self.hierarchy_nodes + if node.type == HierarchyNodeType.SHAPE] + + bh.write_u32(data, 0xC, len(shape_nodes)) + bh.write_u32(data, 0x10, 0) # Vertex count, written in VTX1 + + hierarchy_data_offset = 0x18 + bh.write_u32(data, 0x14, hierarchy_data_offset) + + offset = hierarchy_data_offset + for hierarchy_node in self.hierarchy_nodes: + bh.write_u16(data, offset + 0x0, hierarchy_node.type) + bh.write_u16(data, offset + 0x2, hierarchy_node.data) + offset += 0x4 + + return data diff --git a/wiitility/BDLSections/vtx1.py b/wiitility/BDLSections/vtx1.py new file mode 100644 index 0000000..23a565e --- /dev/null +++ b/wiitility/BDLSections/vtx1.py @@ -0,0 +1,284 @@ +from enum import IntEnum +from io import BytesIO +from typing import NamedTuple + +from wiitility import bytes_helpers as bh +from wiitility.BDLSections.bdl_section import BDLSection, PADDING + +VTX1_MAGIC = "VTX1" + +class AttributeType(IntEnum): + POSITION_MATRIX = 0x0 + TEX_MATRIX_0 = 0x1 + TEX_MATRIX_1 = 0x2 + TEX_MATRIX_2 = 0x3 + TEX_MATRIX_3 = 0x4 + TEX_MATRIX_4 = 0x5 + TEX_MATRIX_5 = 0x6 + TEX_MATRIX_6 = 0x7 + TEX_MATRIX_7 = 0x8 + POSITION = 0x9 + NORMAL = 0xA + COLOR_0 = 0xB + COLOR_1 = 0xC + TEX_0 = 0xD + TEX_1 = 0xE + TEX_2 = 0xF + TEX_3 = 0x10 + TEX_4 = 0x11 + TEX_5 = 0x12 + TEX_6 = 0x13 + TEX_7 = 0x14 + TANGENT = 0x19 + NULL = 0xFF + +class PrimitiveDataType(IntEnum): + UNSIGNED_BYTE = 0x0 + SIGNED_BYTE = 0x1 + UNSIGNED_SHORT = 0x2 + SIGNED_SHORT = 0x3 + FLOAT = 0x4 + +class ColorDataType(IntEnum): + RGB565 = 0x0 + RGB8 = 0x1 + RGBX8 = 0x2 + RGBA4 = 0x3 + RGBA6 = 0x4 + RGBA8 = 0x5 + +class Vec3: + x: int | float + y: int | float + z: int | float + + def __init__(self, x: int | float = 0, y: int | float = 0, z: int | float = 0): + self.x = x + self.y = y + self.z = z + +AttributeToOffset = { + AttributeType.POSITION: 0xC, + AttributeType.NORMAL: 0x10, + AttributeType.TANGENT: 0x14, + AttributeType.COLOR_0: 0x18, + AttributeType.COLOR_1: 0x1C, + AttributeType.TEX_0: 0x20, + AttributeType.TEX_1: 0x24, + AttributeType.TEX_2: 0x28, + AttributeType.TEX_3: 0x2C, + AttributeType.TEX_4: 0x30, + AttributeType.TEX_5: 0x34, + AttributeType.TEX_6: 0x38, + AttributeType.TEX_7: 0x3C, +} + +class VertexFormat(NamedTuple): + attribute_type: AttributeType + component_count: int + component_type: PrimitiveDataType | ColorDataType + component_shift: int + +class Vertex: + vertex_format: VertexFormat + points: list[Vec3] + + def __init__(self, vertex_format: VertexFormat, points: list[Vec3]): + if points: + assert isinstance(points[0], Vec3) + else: + points = [] + + self.vertex_format = vertex_format + self.points = points + +class VTX1Section(BDLSection): + vertices: list[Vertex] + vertex_count: int + + def __init__(self, vertices: list[Vertex] = None): + super().__init__(VTX1_MAGIC) + + if vertices: + assert isinstance(vertices[0], Vertex) + else: + vertices = [] + + self.vertices = vertices + self.vertex_count = 0 + + @classmethod + def import_section(cls, raw_bytes: BytesIO): + magic = bh.read_str(raw_bytes, 0x0, 4) + assert magic == VTX1_MAGIC + + size = bh.read_u32(raw_bytes, 0x4) + vertex_format_offset = bh.read_u32(raw_bytes, 0x8) + position_data_array_offset = bh.read_u32(raw_bytes, 0xC) + normal_data_array_offset = bh.read_u32(raw_bytes, 0x10) + nbt_data_array_offset = bh.read_u32(raw_bytes, 0x14) + + color_data_array_offset = [] + for i in range(2): + color_data_array_offset.append(bh.read_u32(raw_bytes, 0x18 + 0x4 * i)) + + texcoord_data_array_offset = [] + for i in range(8): + texcoord_data_array_offset.append(bh.read_u32(raw_bytes, 0x20 + 0x4 * i)) + + vertex_formats: list[VertexFormat] = [] + + # Read the vertex formats + offset = vertex_format_offset + while True: + attribute_type = AttributeType(bh.read_u32(raw_bytes, offset + 0x0)) + component_count = bh.read_u32(raw_bytes, offset + 0x4) + + if attribute_type == AttributeType.COLOR_0 or attribute_type == AttributeType.COLOR_1: + component_type = ColorDataType(bh.read_u32(raw_bytes, offset + 0x8)) + else: + component_type = PrimitiveDataType(bh.read_u32(raw_bytes, offset + 0x8)) + + component_shift = bh.read_u8(raw_bytes, offset + 0xC) + + vertex_format = VertexFormat(attribute_type, component_count, component_type, component_shift) + vertex_formats.append(vertex_format) + + offset += 0x10 + + if vertex_format.attribute_type == AttributeType.NULL: + break + + # Read the vertex data + offsets: list[int] = [position_data_array_offset, normal_data_array_offset, nbt_data_array_offset] + color_data_array_offset + texcoord_data_array_offset + offsets = [offset for offset in offsets if offset != 0] + + vertices: list[Vertex] = [] + + for offset, next_offset, vertex_format in zip(offsets, offsets[1:] + [size], vertex_formats[:-1]): + points: list[Vec3] = [] + + match vertex_format.component_type: + case PrimitiveDataType.UNSIGNED_BYTE: + read_callback = bh.read_u8 + size = 1 + case PrimitiveDataType.SIGNED_BYTE: + read_callback = bh.read_s8 + size = 1 + case PrimitiveDataType.UNSIGNED_SHORT: + read_callback = bh.read_u16 + size = 2 + case PrimitiveDataType.SIGNED_SHORT: + read_callback = bh.read_s16 + size = 2 + case PrimitiveDataType.FLOAT: + read_callback = bh.read_float + size = 4 + + while offset < next_offset: + # Make sure we're not hitting the padding + if offset % 0x20: + alignment_length = 0x20 - offset % 0x20 + try: + string = bh.read_str(raw_bytes, offset, alignment_length) + if PADDING.startswith(string) and len(string) != 0: + offset = next_offset + continue + except: + pass + + x = read_callback(raw_bytes, offset) / 2 ** (size * 8 - vertex_format.component_shift) + y = read_callback(raw_bytes, offset + size) / 2 ** (size * 8 - vertex_format.component_shift) + z = read_callback(raw_bytes, offset + 2 * size) / 2 ** (size * 8 - vertex_format.component_shift) + + points.append(Vec3(x,y,z)) + + offset += 3 * size + + vertex = Vertex(vertex_format, points) + vertices.append(vertex) + + return cls(vertices) + + def export_section(self) -> BytesIO: + data = BytesIO() + + # Initialise all the offsets to 0 + bh.write_u32(data, 0x8, 0) + bh.write_u32(data, 0xC, 0) + bh.write_u32(data, 0x10, 0) + bh.write_u32(data, 0x14, 0) + bh.write_u64(data, 0x18, 0) + bh.write_u64(data, 0x20, 0) + bh.write_u64(data, 0x28, 0) + + offset = 0x40 + bh.write_u32(data, 0x8, offset) + for vertex_format in [vertex.vertex_format for vertex in self.vertices]: + bh.write_u32(data, offset + 0x0, vertex_format.attribute_type) + bh.write_u32(data, offset + 0x4, vertex_format.component_count) + bh.write_u32(data, offset + 0x8, vertex_format.component_type) + bh.write_u8(data, offset + 0xC, vertex_format.component_shift) + bh.write_bytes(data, offset + 0xD, b'\xFF' * 3) + + offset += 0x10 + + # Write the null vertex format + bh.write_u32(data, offset + 0x0, AttributeType.NULL) + bh.write_u32(data, offset + 0x4, 1) + bh.write_u32(data, offset + 0x8, 0) + bh.write_u8(data, offset + 0xC, 0) + bh.write_bytes(data, offset + 0xD, b'\xFF' * 3) + + position_vertices: int = 0 + + offset += 0x10 + + for vertex in self.vertices: + vertex_format = vertex.vertex_format + if vertex_format.attribute_type == AttributeType.NULL: + continue + + bh.write_u32(data, AttributeToOffset[vertex_format.attribute_type], offset) + + for point in vertex.points: + match vertex_format.component_type: + case PrimitiveDataType.UNSIGNED_BYTE: + write_callback = bh.write_u8 + size = 1 + cast = int + case PrimitiveDataType.SIGNED_BYTE: + write_callback = bh.write_s8 + size = 1 + cast = int + case PrimitiveDataType.UNSIGNED_SHORT: + write_callback = bh.write_u16 + size = 2 + cast = int + case PrimitiveDataType.SIGNED_SHORT: + write_callback = bh.write_s16 + size = 2 + cast = int + case PrimitiveDataType.FLOAT: + write_callback = bh.write_float + size = 4 + cast = float + + x = point.x * (2 ** (size * 8 - vertex_format.component_shift)) + y = point.y * (2 ** (size * 8 - vertex_format.component_shift)) + z = point.z * (2 ** (size * 8 - vertex_format.component_shift)) + + write_callback(data, offset, cast(x)) + write_callback(data, offset + size, cast(y)) + write_callback(data, offset + 2 * size, cast(z)) + + offset += 3 * size + + offset = bh.align(data, 0x20, PADDING) + + if vertex_format.attribute_type == AttributeType.POSITION: + position_vertices = len(vertex.points) + + self.vertex_count = position_vertices + + return data diff --git a/wiitility/__init__.py b/wiitility/__init__.py index 68b2253..69d0dd0 100644 --- a/wiitility/__init__.py +++ b/wiitility/__init__.py @@ -2,4 +2,8 @@ from wiitility.BMGSections.dat1 import DAT1Section, Message, Tag, TagIdentifier from wiitility.BMGSections.fli1 import FLI1Section, FLI1Entry from wiitility.BMGSections.flw1 import FLW1Section, NodeType, FLWTextNode, FLWEventNode, FLWConditionNode -from wiitility.BMGSections.inf1 import INF1Section, INF1Entry, BalloonType, CameraType, TalkType \ No newline at end of file +from wiitility.BMGSections.inf1 import INF1Section, INF1Entry, BalloonType, CameraType, TalkType + +from wiitility.BDLSections.bdl_section import BDLSection +from wiitility.BDLSections.inf1 import INF1Section, HierarchyNode, HierarchyNodeType +from wiitility.BDLSections.vtx1 import VTX1Section, Vertex, VertexFormat, Vec3, AttributeType, PrimitiveDataType, ColorDataType, AttributeToOffset diff --git a/wiitility/bdl.py b/wiitility/bdl.py new file mode 100644 index 0000000..1e3d80e --- /dev/null +++ b/wiitility/bdl.py @@ -0,0 +1,100 @@ +from io import BytesIO + +from wiitility import bytes_helpers as bh +from wiitility.BDLSections.bdl_section import BDLSection, PADDING +from wiitility.BDLSections.inf1 import INF1Section +from wiitility.BDLSections.vtx1 import VTX1Section + +class BDL: + section_count: int + sections: dict[str, BDLSection] + + def __init__(self, raw_bytes: BytesIO): + data_magic = bh.read_str(raw_bytes, 0x0, 4) + assert data_magic == "J3D2" + + file_magic = bh.read_str(raw_bytes, 0x4, 4) + assert file_magic == "bdl4" + + self.size = bh.read_u32(raw_bytes, 0x8) + self.section_count = bh.read_u32(raw_bytes, 0xC) + magic = bh.read_str(raw_bytes, 0x10, 4) + assert magic == "SVR3" + # 12 bytes of padding + + self.sections = {} + + offset = 0x20 + for section in range(self.section_count): + section_magic = bh.read_str(raw_bytes, offset, 4) + section_size = bh.read_u32(raw_bytes, offset + 0x4) + + raw_bytes.seek(offset, 0) + section_bytes = raw_bytes.read(section_size) + section_data = BytesIO(section_bytes) + + match section_magic: + case "INF1": + section = INF1Section.import_section(section_data) + case "VTX1": + section = VTX1Section.import_section(section_data) + case _: + section = BDLSection(section_magic) + section.data = section_data + + + self.sections[section.magic] = section + offset += section_size + + def add_header_to_section(self, section: BDLSection) -> BytesIO: + try: + section_data = section.export_section() + except: + section_data = section.data + section_size = section_data.seek(0, 2) + + padding = 0 + if section_size % 32: + padding = 32 - section_size % 32 + section_size += padding + + bh.write_str(section_data, 0x0, section.magic, 4) + bh.write_u32(section_data, 0x4, section_size) + bh.align(section_data, 0x20, PADDING) + + return section_data + + def export_bdl(self) -> BytesIO: + data = BytesIO() + + bh.write_str(data, 0x0, "J3D2", 4) + bh.write_str(data, 0x4, "bdl4", 4) + bh.write_u32(data, 0x8, 0) # Write the file size later + bh.write_u32(data, 0xC, len(self.sections.keys())) + bh.write_str(data, 0x10, "SVR3", 4) + bh.write_bytes(data, 0x14, b'\xff' * 12) + + vertex_count: int = 0 + vertex_count_position: int = 0 + + offset = 0x20 + for magic, section in self.sections.items(): + section_data = self.add_header_to_section(section) + section_size = section_data.seek(0, 2) + + if section.magic == "INF1": + vertex_count_position = offset + 0x10 + elif section.magic == "VTX1": + vertex_count = section.vertex_count + + bh.write_bytes(data, offset, section_data.getvalue()) + bh.write_str(data, offset, magic, 4) + bh.write_u32(data, offset + 0x4, section_size) + offset += section_size + + bh.write_u32(data, vertex_count_position, vertex_count) + + data_size = data.seek(0, 2) + bh.write_u32(data, 0x8, data_size) + + return data diff --git a/wiitility/bytes_helpers.py b/wiitility/bytes_helpers.py index f8f4884..a43d9ed 100644 --- a/wiitility/bytes_helpers.py +++ b/wiitility/bytes_helpers.py @@ -8,6 +8,19 @@ class ByteHelperError(Exception): GC_ENCODING_STR = "shift_jis" +def read_bitfield(data: BytesIO, offset: int, length: int, shift: int) -> bool: + data_length = data.seek(offset, 2) + if offset + length > data_length: + raise ByteHelperError(f"Offset {str(offset)} + Length {str(length)} is longer than the data size {str(data_length)}.") + value = int.from_bytes(read_bytes(data, offset, length)) + return bool((value >> shift) & 1) + +def read_bool(data: BytesIO, offset: int) -> bool: + data_length = data.seek(offset, 2) + if offset > data_length: + raise ByteHelperError(f"Offset {str(offset)} is longer than the data size {str(data_length)}.") + return read_bitfield(data, offset, 1, 0) + def read_u8(data: BytesIO, offset: int) -> int: data_length = data.seek(offset, 2) length = 1 @@ -32,6 +45,14 @@ def read_u32(data: BytesIO, offset: int) -> int: data.seek(offset) return struct.unpack(">I", data.read(length))[0] +def read_u64(data: BytesIO, offset: int) -> int: + data_length = data.seek(offset, 2) + length = 8 + if offset + length > data_length: + raise ByteHelperError(f"Offset {str(offset)} + Length {str(length)} is longer than the data size {str(data_length)}.") + data.seek(offset) + return struct.unpack(">Q", data.read(length))[0] + def read_s8(data: BytesIO, offset: int) -> int: data_length = data.seek(offset, 2) length = 1 @@ -56,6 +77,14 @@ def read_s32(data: BytesIO, offset: int) -> int: data.seek(offset) return struct.unpack(">i", data.read(length))[0] +def read_s64(data: BytesIO, offset: int) -> int: + data_length = data.seek(offset, 2) + length = 8 + if offset + length > data_length: + raise ByteHelperError(f"Offset {str(offset)} + Length {str(length)} is longer than the data size {str(data_length)}.") + data.seek(offset) + return struct.unpack(">q", data.read(length))[0] + def read_bytes(data: BytesIO, offset: int, size: int = -1) -> bytes: data_length = data.seek(offset, 2) if offset + size > data_length: @@ -71,7 +100,6 @@ def read_float(data: BytesIO, offset: int) -> int: data.seek(offset) return struct.unpack(">f", data.read(length))[0] - def write_u8(data: BytesIO, offset: int, new_value: int): new_bytes = struct.pack(">B", new_value) data.seek(offset) @@ -87,6 +115,11 @@ def write_u32(data: BytesIO, offset: int, new_value: int): data.seek(offset) data.write(new_bytes) +def write_u64(data: BytesIO, offset: int, new_value: int): + new_bytes = struct.pack(">Q", new_value) + data.seek(offset) + data.write(new_bytes) + def write_s8(data: BytesIO, offset: int, new_value: int): new_bytes = struct.pack(">b", new_value) data.seek(offset) @@ -102,6 +135,11 @@ def write_s32(data: BytesIO, offset: int, new_value: int): data.seek(offset) data.write(new_bytes) +def write_s64(data: BytesIO, offset: int, new_value: int): + new_bytes = struct.pack(">q", new_value) + data.seek(offset) + data.write(new_bytes) + def write_bytes(data: BytesIO, offset: int, new_bytes: bytes): data.seek(offset) data.write(new_bytes) @@ -111,6 +149,20 @@ def write_float(data: BytesIO, offset: int, new_value: float): data.seek(offset) data.write(new_bytes) +def align(data: BytesIO, alignment: int, padding: str | bytes) -> int: + data_length = data.seek(0, 2) + padding_length = 0 + if data_length % alignment != 0: + padding_length = alignment - data_length % alignment + if isinstance(padding, bytes): + assert len(padding) == 1 + write_bytes(data, data_length, padding * padding_length) + elif isinstance(padding, str): + assert len(padding) >= padding_length + write_str(data, data_length, padding[:padding_length], padding_length) + return data_length + padding_length + + def read_str(data: BytesIO, offset: int, max_length: int = -1) -> str: data_length = data.seek(offset, 2) if (offset + max_length) > data_length: @@ -140,4 +192,4 @@ def write_str(data: BytesIO, offset: int, new_string: str, max_length: int, padd new_value = encoded_string + (padding_byte * padding_length) data.seek(offset) - data.write(new_value) \ No newline at end of file + data.write(new_value)