From 9db9c5513cc5904679acdf3fe7cac55e2a4974d4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lo=C3=AFc=20Pinsard?= Date: Mon, 5 Jul 2021 21:39:15 +0200 Subject: [PATCH 1/5] add parse_header_ctags --- scripts/parse_header_ctags.py | 591 ++++++++++++++++++++++++++++++++++ 1 file changed, 591 insertions(+) create mode 100644 scripts/parse_header_ctags.py diff --git a/scripts/parse_header_ctags.py b/scripts/parse_header_ctags.py new file mode 100644 index 0000000..2f6e07f --- /dev/null +++ b/scripts/parse_header_ctags.py @@ -0,0 +1,591 @@ +""" +This script generates a Python binding stub from a Maya devkit header. + +pre-requisites: +1. universal ctags installed and in the PATH. + universal-ctags is available in most package managers, + including choco or scoop (in the extras repo) on Windows + pre-built binaries for windows can also be found here: https://github.com/universal-ctags/ctags-win32/releases +3. $DEVKIT_LOCATION has to be set to the root of the devkit + +To run the script: +>>> mayapy parse_headers.py MDagPath + +Notes: +- The script runs on python 3 only +- It needs to run from mayapy as it retrieves the docstrings from OpenMaya 2 + and compares the classes content with OpenMaya 1 + +If you wish to manually run ctags: +>>> ctags --C++-kinds=+p --format=2 --sort=no $DEVKIT_LOCATION/include/maya/MDagPath.h +""" + +from __future__ import annotations + +import argparse +import logging +import os +import re +import shutil +import subprocess +import textwrap +from dataclasses import dataclass, field +from enum import Enum +from pathlib import Path +from typing import Any, Callable, List, Optional, Type + + +try: + import maya.standalone +except ImportError: + raise RuntimeError("This script needs to run from Mayapy") +else: + maya.standalone.initialize() + + import maya + + +logging.getLogger().setLevel(logging.DEBUG) +logger = logging.getLogger("parse_header") +# logger.setLevel(logging.DEBUG) + + +MODULE_TEMPLATE = """\ +{docstring_definitions} + +{classes}""" + + +CLASS_TEMPLATE = """\ +py::class_<{maya_class_name}>(m, "{class_name}") +{body};""" + + +METHOD_TEMPLATE = """\ + +.def( + "{name}", []({arguments}){return_type} + {{ {body} }}, + {py_args}{docstring_variable}) +""" + +CONSTRUCTOR_TEMPLATE = """\ + +.def(py::init<{arguments}>()) +""" + +DOCSTRING_TEMPLATE = """\ +#define {variable_name} \\ +{docstring} +""" + + +@dataclass +class Module: + classes: List[Class] = field(default_factory=list) + + def __str__(self) -> str: + docstrings = [] + for cls in self.classes: + docstrings.extend(cls.unique_docstrings()) + + docstrings = map(str, docstrings) + classes = map(str, self.classes) + + docstrings_str = "\n".join(docstrings) + classes_str = "\n".join(classes) + + return MODULE_TEMPLATE.format( + docstring_definitions=docstrings_str, + classes=classes_str, + ) + + +@dataclass +class Class: + maya_class_name: str + name: str = field(init=False) + methods: List[Method] = field(default_factory=list) + + maya_api1_class: Optional[Type] = None + maya_api2_class: Optional[Type] = None + + def __post_init__(self) -> None: + self.name = self.maya_class_name.strip("M") + + self.maya_api1_class = self.find_maya_api1_class() + self.maya_api2_class = self.find_maya_api2_class() + + def find_maya_api1_class(self): + for module in ( + maya.OpenMaya, + maya.OpenMayaAnim, + maya.OpenMayaRender, + # maya.OpenMayaFX, # not available in mayapy? + maya.OpenMayaUI, + maya.OpenMayaMPx, + # maya.OpenMayaCloth, # not available in mayapy? + ): + if hasattr(module, self.maya_class_name): + return getattr(module, self.maya_class_name) + + def find_maya_api2_class(self): + for module in ( + maya.api.OpenMaya, + # maya.api.OpenMayaAnim, # not available in mayapy? + maya.api.OpenMayaRender, + maya.api.OpenMayaUI, + ): + if hasattr(module, self.maya_class_name): + return getattr(module, self.maya_class_name) + + def add_method(self, method: Method): + if not method.is_destructor and method.maya_api1_method: + # if not method.is_destructor: + self.methods.append(method) + + @property + def body_str(self) -> str: + body_str = "" + + for method in self.methods: + body_str += str(method) + + return body_str + + def unique_docstrings(self) -> List[Docstring]: + docstrings = [] + for method in self.methods: + if method.docstring not in docstrings: + docstrings.append(method.docstring) + return docstrings + + def __str__(self) -> str: + return CLASS_TEMPLATE.format( + maya_class_name=self.maya_class_name, + class_name=self.name, + body=textwrap.indent(self.body_str, " "), + ) + + +@dataclass +class Method: + + arguments: List[Argument] + klass: Class + name: str + return_type: Optional[str] + body = 'throw std::logic_error{{"Function not yet implemented."}};' + is_constructor = False + is_destructor = False + is_operator = False + docstring: Docstring = field(init=False) + maya_api1_method: Optional[Callable[[], Any]] = field(default=None, init=False) + maya_api2_method: Optional[Callable[[], Any]] = field(default=None, init=False) + + def __post_init__(self): + if self.name == self.klass.maya_class_name: + self.is_constructor = True + + if self.name.startswith("~"): + self.is_destructor = True + + try: + api1_method = getattr(self.klass.maya_api1_class, self.name) + except: + self.maya_api1_method = None + else: + self.maya_api1_method = api1_method + + try: + api2_method = getattr(self.klass.maya_api2_class, self.name) + except: + self.maya_api2_method = None + else: + self.maya_api2_method = api2_method + + if self.name in [ + "=", + "+", + "+=", + "-", + "-=", + "*", + "*=", + "/", + "/=", + "==", + "!=", + ]: + self.is_operator = True + + if not self.is_constructor: + self.arguments.insert( + 0, Argument.from_string(f"{self.klass.maya_class_name} & self") + ) + + docstring_str = "MISSING DOCSTRING" + if hasattr(self.klass.maya_api2_class, self.name): + maya_method = getattr(self.klass.maya_api2_class, self.name) + if maya_method.__doc__: + docstring_str = maya_method.__doc__ + self.docstring = Docstring(self, docstring_str) + + self._filter_mstatus() + + def _filter_mstatus(self): + arguments = self.arguments + for argument in arguments: + if "MStatus" in argument.type: + self.arguments.remove(argument) + + if self.return_type and "MStatus" in self.return_type: + self.return_type = None + + @classmethod + def new( + cls, + klass: Class, + name: str, + arguments_str: str, + return_type: Optional[str], + ) -> Method: + + arguments = [] + for argument_str in arguments_str.split(","): + if not argument_str: + continue + + try: + argument = Argument.from_string(argument_str.strip()) + except RuntimeError as e: + logger.warning(e) + else: + arguments.append(argument) + + return Method( + arguments=arguments, + klass=klass, + name=name, + return_type=return_type, + ) + + @property + def arguments_str(self) -> str: + return ", ".join([arg.source_string for arg in self.arguments]) + + @property + def pyargs_str(self) -> str: + # self is the first argument, we don't want it in the pybind arguments + arguments = self.arguments[1:] + + if not arguments: + return "" + + return ",\n ".join([arg.pyarg_str for arg in arguments if arg.name]) + ",\n " + + @property + def return_type_str(self) -> str: + if self.return_type: + return f" -> {self.return_type} " + else: + return " " + + def __str__(self) -> str: + if self.is_constructor: + return CONSTRUCTOR_TEMPLATE.format(arguments=self.arguments_str) + + if self.is_operator: + # not supported yet + return "" + + return METHOD_TEMPLATE.format( + name=self.name, + arguments=self.arguments_str, + py_args=self.pyargs_str, + return_type=self.return_type_str, + body=self.body, + docstring_variable=self.docstring.variable_name, + ) + + +@dataclass(eq=False) +class Docstring: + method: Method + docstring: str + + def __post_init__(self) -> None: + self._process_docstring() + + def _process_docstring(self): + self.strip_signature() + self.process_multilines() + + def strip_signature(self) -> None: + signature_regex = re.compile(r".*\(.*\) -> .*") + + lines = self.docstring.splitlines() + + while lines: + line = lines[0] + + # Exclude signatures and empty lines + if signature_regex.match(line) or not line: + lines.pop(0) + continue + + # we've reached the first documentation line. + if line: + break + + filtered_docstring = "\n".join(lines) + + self.docstring = filtered_docstring + + def process_multilines(self) -> None: + lines = self.docstring.splitlines() + line_template = '"{line}\\n"\\\n' + last_line_template = '"{line}"' + + multiline_docstring = "" + for i, line in enumerate(lines): + if i < len(lines) - 1: + formatted_line = line_template.format(line=line) + else: + formatted_line = last_line_template.format(line=line) + multiline_docstring += formatted_line + + self.docstring = multiline_docstring + + @property + def variable_name(self) -> str: + return f"_doc_{self.method.klass.name}_{self.method.name}" + + @property + def define_statement(self) -> str: + return DOCSTRING_TEMPLATE.format( + variable_name=self.variable_name, + docstring=textwrap.indent(self.docstring, " "), + ) + + def __str__(self) -> str: + return self.define_statement + + def __eq__(self, o: object) -> bool: + if isinstance(o, Docstring): + return self.variable_name == o.variable_name + else: + return False + + +@dataclass +class Argument: + # To make sense of the regex: https://regex101.com/r/7O6yVV/1 + _regex_pattern = re.compile( + r"^(?P.+?(\s?(\*|&)\s?)*)(?P\S+)?( = (?P\S+))?$" + ) + + name: Optional[str] + source_string: str + type: str + value: Optional[str] + + @classmethod + def from_string(cls, string: str) -> Argument: + string = string.strip() + match = cls._regex_pattern.match(string) + + if not match: + raise RuntimeError(f"Invalid argument pattern: {string}") + + groups = match.groupdict() + + name = groups["name"] + type = groups["type"] + value = groups.get("value", None) + + return Argument( + name=name, + source_string=string, + type=type, + value=value, + ) + + @property + def pyarg_str(self) -> Optional[str]: + if self.name: + pyarg_str = f'py::arg("{self.name}")' + + if self.value: + pyarg_str += f" = {self.value}" + + return pyarg_str + else: + return None + + @property + def is_reference(self): + return "&" in self.type + + @property + def is_pointer(self): + return "*" in self.type + + @property + def is_const(self): + return "const" in self.type + + + +class EntryKind(Enum): + Class = "c" + Member = "m" + Prototype = "p" # A prototype method is a method with no implementation + + +def find_header_file(header_name: str) -> Path: + logger.debug(f"Searching header file for {header_name}") + + devkit_path = Path(os.environ["DEVKIT_LOCATION"]) + + header_file = devkit_path / "include" / "maya" / header_name + + if not header_file.exists(): + raise LookupError(f"No '{header_name}' header in the devkit.") + else: + logger.info(f"Header file found: {header_file}") + + return header_file + + +def generate_tags(header_file: Path) -> Path: + logger.debug(f"Generating Tags from Header file: {header_file}") + + if shutil.which("ctags") is None: + raise RuntimeError( + "ctags not found.\n" + " universal-ctags is available in most package managers.\n" + " binaries for windows can also be found here: https://github.com/universal-ctags/ctags-win32/releases" + ) + + tags_dir = Path(__file__).parent.resolve() / "tags" + tags_dir.mkdir(exist_ok=True) + tags_file = tags_dir / header_file.name.replace(".h", ".tags") + + args = [ + "ctags", + "--C++-kinds=+p-d", + "--sort=no", + f"-f {tags_file}", + "--fields=+S", + "--excmd=number", + "--output-format=json", + str(header_file), + ] + subprocess.check_call(args) + + logger.info(f"Generated tag file: {tags_file}") + + return tags_file + + +def parse_tags(tag_file: Path) -> Module: + logger.debug(f"Parsing tag file: {tag_file}") + + with tag_file.open("r") as f: + tag_file_content = f.read() + + module = Module() + + # https://regex101.com/r/dPC7VE/1 + # I'm sorry... + line_re = re.compile( + r"^(?P[^!]\S+)\s*(?P\S+)\s*(?P\d+);\"\s(?Pd|c|p)((\sclass:(?P\S+))?(\styperef:typename:(?P\S+))?(\ssignature:\((?P.*)\))?)?" + ) + + current_class = None + for line in tag_file_content.splitlines(): + match = line_re.match(line) + + if not match: + continue + + entry = match.groupdict() + + name = entry["name"] + kind = EntryKind(entry["kind"]) + + if kind is EntryKind.Class: + current_class = Class(name) + module.classes.append(current_class) + elif kind is EntryKind.Prototype: + + cls = entry["class"] + if cls is None: + continue + + arguments = entry["arguments"] or "" + return_type = entry["return_type"] + try: + method = Method.new(current_class, name, arguments, return_type) + except RuntimeError as e: + logger.info(e) + else: + current_class.add_method(method) + else: + continue + + return module + + +def write_inl_file(header_name: str, module: Module, overwrite: bool): + src_dir = Path(__file__).parent.parent / "src" + filename = header_name.replace(".h", ".inl") + inl_file = (src_dir / filename).resolve() + + if inl_file.exists() and not overwrite: + raise OSError( + f"File '{inl_file}' already exist. " + "Use --overwrite if you want to overwrite the existing file." + ) + + with inl_file.open("w") as f: + f.write(str(module)) + + logger.info(f"Generated inl file: {inl_file}") + + +def parse_header(header_name: str, overwrite: bool): + if not header_name.endswith(".h"): + header_name = f"{header_name}.h" + + header_file = find_header_file(header_name) + tag_file = generate_tags(header_file) + module = parse_tags(tag_file) + write_inl_file(header_name, module, overwrite) + + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument("header", help="Name of Maya header, e.g. MPlug.h") + parser.add_argument( + "--overwrite", + type=bool, + nargs="?", + default=False, + const=True, + help="Whether to overwrite the existing inl file.", + ) + + opts = parser.parse_args() + try: + parse_header(opts.header, opts.overwrite) + except LookupError as e: + logger.error(str(e)) + except OSError as e: + logger.error(str(e)) + + +if __name__ == "__main__": + main() + maya.standalone.uninitialize() From 817b5f42cedf020f79e4f438c3e88f9ffe28055a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lo=C3=AFc=20Pinsard?= Date: Thu, 8 Jul 2021 00:32:44 +0200 Subject: [PATCH 2/5] escape quotes in docstrings --- scripts/parse_header_ctags.py | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/scripts/parse_header_ctags.py b/scripts/parse_header_ctags.py index 2f6e07f..29dc0a4 100644 --- a/scripts/parse_header_ctags.py +++ b/scripts/parse_header_ctags.py @@ -318,10 +318,11 @@ def __post_init__(self) -> None: self._process_docstring() def _process_docstring(self): - self.strip_signature() - self.process_multilines() + self._strip_signature() + self._process_quotes() + self._process_multilines() - def strip_signature(self) -> None: + def _strip_signature(self) -> None: signature_regex = re.compile(r".*\(.*\) -> .*") lines = self.docstring.splitlines() @@ -342,7 +343,10 @@ def strip_signature(self) -> None: self.docstring = filtered_docstring - def process_multilines(self) -> None: + def _process_quotes(self) -> None: + self.docstring = self.docstring.replace('"', '\\"') + + def _process_multilines(self) -> None: lines = self.docstring.splitlines() line_template = '"{line}\\n"\\\n' last_line_template = '"{line}"' From ef66ad25bb587c8f8cf9a7aeb090a633a78989fc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lo=C3=AFc=20Pinsard?= Date: Thu, 8 Jul 2021 00:33:57 +0200 Subject: [PATCH 3/5] output ctags result as json --- scripts/parse_header_ctags.py | 190 ++++++++++++++++++---------------- 1 file changed, 101 insertions(+), 89 deletions(-) diff --git a/scripts/parse_header_ctags.py b/scripts/parse_header_ctags.py index 29dc0a4..ce1aa4c 100644 --- a/scripts/parse_header_ctags.py +++ b/scripts/parse_header_ctags.py @@ -23,6 +23,7 @@ from __future__ import annotations import argparse +import json import logging import os import re @@ -32,8 +33,7 @@ from dataclasses import dataclass, field from enum import Enum from pathlib import Path -from typing import Any, Callable, List, Optional, Type - +from typing import Any, Callable, Dict, List, Optional, Type try: import maya.standalone @@ -80,17 +80,22 @@ """ +class UnnamedArgumentError(Exception): + """Raised when one or more signatures contain unnamed arguments.""" + + @dataclass class Module: - classes: List[Class] = field(default_factory=list) + classes: Dict[str, Class] = field(default_factory=dict) def __str__(self) -> str: docstrings = [] - for cls in self.classes: + classes = self.classes.values() + for cls in classes: docstrings.extend(cls.unique_docstrings()) docstrings = map(str, docstrings) - classes = map(str, self.classes) + classes = map(str, classes) docstrings_str = "\n".join(docstrings) classes_str = "\n".join(classes) @@ -171,34 +176,35 @@ def __str__(self) -> str: @dataclass class Method: - arguments: List[Argument] - klass: Class + cls: Class name: str + arguments: List[Argument] return_type: Optional[str] body = 'throw std::logic_error{{"Function not yet implemented."}};' - is_constructor = False - is_destructor = False - is_operator = False + + is_constructor: bool = field(init=False, default=False) + is_destructor: bool = field(init=False, default=False) + is_operator: bool = field(init=False, default=False) docstring: Docstring = field(init=False) maya_api1_method: Optional[Callable[[], Any]] = field(default=None, init=False) maya_api2_method: Optional[Callable[[], Any]] = field(default=None, init=False) def __post_init__(self): - if self.name == self.klass.maya_class_name: + if self.name == self.cls.maya_class_name: self.is_constructor = True if self.name.startswith("~"): self.is_destructor = True try: - api1_method = getattr(self.klass.maya_api1_class, self.name) + api1_method = getattr(self.cls.maya_api1_class, self.name) except: self.maya_api1_method = None else: self.maya_api1_method = api1_method try: - api2_method = getattr(self.klass.maya_api2_class, self.name) + api2_method = getattr(self.cls.maya_api2_class, self.name) except: self.maya_api2_method = None else: @@ -221,12 +227,12 @@ def __post_init__(self): if not self.is_constructor: self.arguments.insert( - 0, Argument.from_string(f"{self.klass.maya_class_name} & self") + 0, Argument.from_string(f"{self.cls.maya_class_name} & self") ) docstring_str = "MISSING DOCSTRING" - if hasattr(self.klass.maya_api2_class, self.name): - maya_method = getattr(self.klass.maya_api2_class, self.name) + if hasattr(self.cls.maya_api2_class, self.name): + maya_method = getattr(self.cls.maya_api2_class, self.name) if maya_method.__doc__: docstring_str = maya_method.__doc__ self.docstring = Docstring(self, docstring_str) @@ -234,6 +240,7 @@ def __post_init__(self): self._filter_mstatus() def _filter_mstatus(self): + """Remove any MStatus arguments.""" arguments = self.arguments for argument in arguments: if "MStatus" in argument.type: @@ -243,15 +250,35 @@ def _filter_mstatus(self): self.return_type = None @classmethod - def new( - cls, - klass: Class, - name: str, - arguments_str: str, - return_type: Optional[str], - ) -> Method: + def from_entry(cls, klass: Type, entry: Dict[str, str]) -> Method: + name = entry["name"] + signature = entry["signature"] + + return_type = entry.get("typeref") + if return_type: + return_type = return_type.replace("typename:", "") + + arguments = cls._parse_signature(signature) + + return Method( + arguments=arguments, + cls=klass, + name=name, + return_type=return_type, + ) + + @classmethod + def _parse_signature(cls, signature: str) -> Tuple[List[Argument], str]: + signature_re = re.compile(r"\((?P.*)\).*") + + match = signature_re.match(signature) + + if not match: + raise ValueError(f"Invalid signature format: {signature}") arguments = [] + + arguments_str = match["arguments_str"] for argument_str in arguments_str.split(","): if not argument_str: continue @@ -260,29 +287,26 @@ def new( argument = Argument.from_string(argument_str.strip()) except RuntimeError as e: logger.warning(e) + except UnnamedArgumentError: + pass else: arguments.append(argument) - return Method( - arguments=arguments, - klass=klass, - name=name, - return_type=return_type, - ) + return arguments @property def arguments_str(self) -> str: - return ", ".join([arg.source_string for arg in self.arguments]) + return ", ".join([arg.source_string for arg in self.arguments if arg.name]) @property def pyargs_str(self) -> str: # self is the first argument, we don't want it in the pybind arguments - arguments = self.arguments[1:] + arguments = [arg for arg in self.arguments[1:] if arg.name] if not arguments: return "" - return ",\n ".join([arg.pyarg_str for arg in arguments if arg.name]) + ",\n " + return ",\n ".join([arg.pyarg_str for arg in arguments]) + ",\n " @property def return_type_str(self) -> str: @@ -363,7 +387,7 @@ def _process_multilines(self) -> None: @property def variable_name(self) -> str: - return f"_doc_{self.method.klass.name}_{self.method.name}" + return f"_doc_{self.method.cls.name}_{self.method.name}" @property def define_statement(self) -> str: @@ -384,35 +408,53 @@ def __eq__(self, o: object) -> bool: @dataclass class Argument: - # To make sense of the regex: https://regex101.com/r/7O6yVV/1 - _regex_pattern = re.compile( - r"^(?P.+?(\s?(\*|&)\s?)*)(?P\S+)?( = (?P\S+))?$" - ) - name: Optional[str] source_string: str + name: Optional[str] type: str value: Optional[str] + is_pointer: bool + is_reference: bool + is_const: bool @classmethod - def from_string(cls, string: str) -> Argument: - string = string.strip() - match = cls._regex_pattern.match(string) + def from_string(cls, argument_str: str) -> Argument: + + # https://regex101.com/r/StbpY9/1 + arguement_re = re.compile( + r"^(?Pconst)? ?(?P.+?) ?(?P\*|&)? ?(?P\S+)?( = (?P\S+))?$" + ) + argument_str = argument_str.strip() + match = arguement_re.match(argument_str) if not match: - raise RuntimeError(f"Invalid argument pattern: {string}") + raise RuntimeError(f"Invalid argument pattern: {argument_str}") groups = match.groupdict() - name = groups["name"] + name = groups.get("name") + type = groups["type"] value = groups.get("value", None) + is_const = bool(groups.get("is_const")) + + passed_by = groups.get("passed_by") + is_reference = False + is_pointer = False + if passed_by == "*": + is_reference = True + if passed_by == "&": + is_pointer = False + return Argument( + source_string=argument_str, name=name, - source_string=string, type=type, value=value, + is_pointer=is_pointer, + is_reference=is_reference, + is_const=is_const, ) @property @@ -426,25 +468,6 @@ def pyarg_str(self) -> Optional[str]: return pyarg_str else: return None - - @property - def is_reference(self): - return "&" in self.type - - @property - def is_pointer(self): - return "*" in self.type - - @property - def is_const(self): - return "const" in self.type - - - -class EntryKind(Enum): - Class = "c" - Member = "m" - Prototype = "p" # A prototype method is a method with no implementation def find_header_file(header_name: str) -> Path: @@ -455,7 +478,7 @@ def find_header_file(header_name: str) -> Path: header_file = devkit_path / "include" / "maya" / header_name if not header_file.exists(): - raise LookupError(f"No '{header_name}' header in the devkit.") + raise FileNotFoundError(f"No '{header_name}' header in the devkit.") else: logger.info(f"Header file found: {header_file}") @@ -476,6 +499,9 @@ def generate_tags(header_file: Path) -> Path: tags_dir.mkdir(exist_ok=True) tags_file = tags_dir / header_file.name.replace(".h", ".tags") + if tags_file.exists(): + tags_file.unlink() + args = [ "ctags", "--C++-kinds=+p-d", @@ -501,42 +527,28 @@ def parse_tags(tag_file: Path) -> Module: module = Module() - # https://regex101.com/r/dPC7VE/1 - # I'm sorry... - line_re = re.compile( - r"^(?P[^!]\S+)\s*(?P\S+)\s*(?P\d+);\"\s(?Pd|c|p)((\sclass:(?P\S+))?(\styperef:typename:(?P\S+))?(\ssignature:\((?P.*)\))?)?" - ) - - current_class = None for line in tag_file_content.splitlines(): - match = line_re.match(line) + entry = json.loads(line) - if not match: + if entry["_type"] != "tag": continue - entry = match.groupdict() - name = entry["name"] - kind = EntryKind(entry["kind"]) - - if kind is EntryKind.Class: - current_class = Class(name) - module.classes.append(current_class) - elif kind is EntryKind.Prototype: + kind = entry["kind"] - cls = entry["class"] - if cls is None: - continue + if kind == "class": + module.classes[name] = Class(name) - arguments = entry["arguments"] or "" - return_type = entry["return_type"] + elif kind == "prototype": try: - method = Method.new(current_class, name, arguments, return_type) + cls = module.classes[entry["scope"]] + method = Method.from_entry(cls, entry) except RuntimeError as e: logger.info(e) else: - current_class.add_method(method) + cls.add_method(method) else: + # TODO: Support properties and probably more kinds continue return module @@ -584,7 +596,7 @@ def main(): opts = parser.parse_args() try: parse_header(opts.header, opts.overwrite) - except LookupError as e: + except FileNotFoundError as e: logger.error(str(e)) except OSError as e: logger.error(str(e)) From 474d803fd095061f0cac535d593dacc3198b83f7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lo=C3=AFc=20Pinsard?= Date: Sat, 10 Jul 2021 18:05:13 +0200 Subject: [PATCH 4/5] use "out arguments" as return types --- scripts/parse_header_ctags.py | 149 ++++++++++++++++++++++++---------- 1 file changed, 107 insertions(+), 42 deletions(-) diff --git a/scripts/parse_header_ctags.py b/scripts/parse_header_ctags.py index ce1aa4c..ae54a80 100644 --- a/scripts/parse_header_ctags.py +++ b/scripts/parse_header_ctags.py @@ -33,7 +33,7 @@ from dataclasses import dataclass, field from enum import Enum from pathlib import Path -from typing import Any, Callable, Dict, List, Optional, Type +from typing import Any, Callable, Dict, List, Optional, Type, Tuple try: import maya.standalone @@ -47,7 +47,10 @@ logging.getLogger().setLevel(logging.DEBUG) logger = logging.getLogger("parse_header") -# logger.setLevel(logging.DEBUG) + + +class DeprecationError(Exception): + """Raised when a method is deprecated.""" MODULE_TEMPLATE = """\ @@ -80,10 +83,6 @@ """ -class UnnamedArgumentError(Exception): - """Raised when one or more signatures contain unnamed arguments.""" - - @dataclass class Module: classes: Dict[str, Class] = field(default_factory=dict) @@ -237,20 +236,12 @@ def __post_init__(self): docstring_str = maya_method.__doc__ self.docstring = Docstring(self, docstring_str) - self._filter_mstatus() - - def _filter_mstatus(self): - """Remove any MStatus arguments.""" - arguments = self.arguments - for argument in arguments: - if "MStatus" in argument.type: - self.arguments.remove(argument) - - if self.return_type and "MStatus" in self.return_type: - self.return_type = None - @classmethod - def from_entry(cls, klass: Type, entry: Dict[str, str]) -> Method: + def from_entry( + cls, + klass: Class, + entry: Dict[str, str], + ) -> Method: name = entry["name"] signature = entry["signature"] @@ -258,17 +249,45 @@ def from_entry(cls, klass: Type, entry: Dict[str, str]) -> Method: if return_type: return_type = return_type.replace("typename:", "") - arguments = cls._parse_signature(signature) + if "OPENMAYA_DEPRECATED" in return_type: + raise DeprecationError( + f"Method {klass.name}.{name} is deprecated -- skipping." + ) + + if return_type == "void": + return_type = None + + if return_type == "MString": + return_type = "std::string" + + if return_type == "MStatus": + return_type = None + has_out_args = True + else: + has_out_args = False + + in_args, out_args = cls._parse_signature(signature, has_out_args) + + if out_args: + if len(out_args) == 1: + return_type = out_args[0].type + else: + out_args_types = [arg.type for arg in out_args] + return_type = "std::tuple<{}>".format(", ".join(out_args_types)) return Method( - arguments=arguments, + arguments=in_args, cls=klass, name=name, return_type=return_type, ) @classmethod - def _parse_signature(cls, signature: str) -> Tuple[List[Argument], str]: + def _parse_signature( + cls, + signature: str, + has_out_args: bool, + ) -> Tuple[List[Argument], List[Argument]]: signature_re = re.compile(r"\((?P.*)\).*") match = signature_re.match(signature) @@ -276,27 +295,36 @@ def _parse_signature(cls, signature: str) -> Tuple[List[Argument], str]: if not match: raise ValueError(f"Invalid signature format: {signature}") - arguments = [] + in_args: List[Argument] = [] + out_args: List[Argument] = [] arguments_str = match["arguments_str"] for argument_str in arguments_str.split(","): if not argument_str: continue - try: argument = Argument.from_string(argument_str.strip()) - except RuntimeError as e: + except re.error as e: logger.warning(e) - except UnnamedArgumentError: - pass else: - arguments.append(argument) + if not argument.name or "MStatus" in argument.type: + continue - return arguments + if argument.is_const: + in_args.append(argument) + elif argument.is_reference: + if has_out_args: + out_args.append(argument) + else: + in_args.append(argument) + else: + in_args.append(argument) + + return (in_args, out_args) @property def arguments_str(self) -> str: - return ", ".join([arg.source_string for arg in self.arguments if arg.name]) + return ", ".join([arg.source_string for arg in self.arguments]) @property def pyargs_str(self) -> str: @@ -343,7 +371,7 @@ def __post_init__(self) -> None: def _process_docstring(self): self._strip_signature() - self._process_quotes() + self._escape_quotes() self._process_multilines() def _strip_signature(self) -> None: @@ -367,7 +395,7 @@ def _strip_signature(self) -> None: self.docstring = filtered_docstring - def _process_quotes(self) -> None: + def _escape_quotes(self) -> None: self.docstring = self.docstring.replace('"', '\\"') def _process_multilines(self) -> None: @@ -420,21 +448,24 @@ class Argument: @classmethod def from_string(cls, argument_str: str) -> Argument: - # https://regex101.com/r/StbpY9/1 + # https://regex101.com/r/99q0hk/2 arguement_re = re.compile( - r"^(?Pconst)? ?(?P.+?) ?(?P\*|&)? ?(?P\S+)?( = (?P\S+))?$" + r"^(?Pconst)? ?(?P(unsigned )?\S+) ?(?P\*|&)? ?(?P\S+?)?( ?= ?(?P\S+))?$" ) argument_str = argument_str.strip() match = arguement_re.match(argument_str) if not match: - raise RuntimeError(f"Invalid argument pattern: {argument_str}") + raise re.error(f"Invalid argument pattern: {argument_str}") groups = match.groupdict() name = groups.get("name") type = groups["type"] + if type == "MString": + type = "std::string" + value = groups.get("value", None) is_const = bool(groups.get("is_const")) @@ -443,9 +474,9 @@ def from_string(cls, argument_str: str) -> Argument: is_reference = False is_pointer = False if passed_by == "*": - is_reference = True + is_pointer = True if passed_by == "&": - is_pointer = False + is_reference = True return Argument( source_string=argument_str, @@ -469,6 +500,17 @@ def pyarg_str(self) -> Optional[str]: else: return None + def __str__(self) -> str: + argument_str = self.type + + if self.name: + argument_str += f" {self.name}" + + if self.value: + argument_str += f" = {self.value}" + + return argument_str + def find_header_file(header_name: str) -> Path: logger.debug(f"Searching header file for {header_name}") @@ -527,6 +569,7 @@ def parse_tags(tag_file: Path) -> Module: module = Module() + # register all the classes first for line in tag_file_content.splitlines(): entry = json.loads(line) @@ -535,16 +578,38 @@ def parse_tags(tag_file: Path) -> Module: name = entry["name"] kind = entry["kind"] + scope = entry.get("scope") if kind == "class": module.classes[name] = Class(name) - elif kind == "prototype": + # then register all the methods. + for line in tag_file_content.splitlines(): + entry = json.loads(line) + + if entry["_type"] != "tag": + continue + + name = entry["name"] + kind = entry["kind"] + scope = entry.get("scope") + + if kind == "prototype": + if not scope: + # we likely have a function in our hands + # TODO: Support function definitions. + continue + try: - cls = module.classes[entry["scope"]] - method = Method.from_entry(cls, entry) - except RuntimeError as e: - logger.info(e) + cls = module.classes[scope] + if cls: + method = Method.from_entry(cls, entry) + except KeyError as e: + logger.warning(f"class {scope} not found for {name} -- skipping.") + except DeprecationError as e: + logger.warning(e) + except Exception as e: + raise e else: cls.add_method(method) else: From fd9eab2e65c485556638ff9915c9c7bbf8ec1945 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lo=C3=AFc=20Pinsard?= Date: Sat, 10 Jul 2021 18:05:59 +0200 Subject: [PATCH 5/5] ignore tag files --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index ce6e611..6f870cc 100644 --- a/.gitignore +++ b/.gitignore @@ -6,6 +6,7 @@ tmp/* MFn.Types.inl devkit.tgz devkitBase +*.tags # IDE stuff .vscode