diff options
author | Shaheed Haque <srhaque@theiet.org> | 2016-05-01 20:59:41 +0200 |
---|---|---|
committer | Stephen Kelly <steveire@gmail.com> | 2016-10-31 15:31:39 +0000 |
commit | 37f1b5c4f905a37ddb8b43001bf2e4408c2a703b (patch) | |
tree | 7325aa86ea1f29099b6834f696595ec7a3f7fd07 | |
parent | 9b0621fabe6867754dce7884df064ededaa6278a (diff) | |
download | extra-cmake-modules-37f1b5c4f905a37ddb8b43001bf2e4408c2a703b.tar.gz extra-cmake-modules-37f1b5c4f905a37ddb8b43001bf2e4408c2a703b.tar.bz2 |
Add the rules_engine and sip_generator
These files process C++ headers with libclang and python-clang bindings,
using the AST to generate SIP files for PyQt binding library generation.
The design allows for further extensions which can process other
artifacts found in headers. This iteration of the design is a simple
minimum which suffices to generate headers for KItemModels and some
other libraries.
-rw-r--r-- | CMakeLists.txt | 2 | ||||
-rwxr-xr-x | find-modules/rules_engine.py | 526 | ||||
-rw-r--r-- | find-modules/sip_generator.py | 700 |
3 files changed, 1227 insertions, 1 deletions
diff --git a/CMakeLists.txt b/CMakeLists.txt index 94e88306..fe7ebac5 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -117,7 +117,7 @@ install(FILES ${installModuleFiles} DESTINATION ${MODULES_INSTALL_DIR}) file(GLOB installKdeModuleFiles ${CMAKE_SOURCE_DIR}/kde-modules/*[^~]) install(FILES ${installKdeModuleFiles} DESTINATION ${KDE_MODULES_INSTALL_DIR}) -file(GLOB installFindModuleFiles ${CMAKE_SOURCE_DIR}/find-modules/*[^~]) +file(GLOB installFindModuleFiles ${CMAKE_SOURCE_DIR}/find-modules/*.cmake ${CMAKE_SOURCE_DIR}/find-modules/*.py) install(FILES ${installFindModuleFiles} DESTINATION ${FIND_MODULES_INSTALL_DIR}) file(GLOB installToolchainModuleFiles ${CMAKE_SOURCE_DIR}/toolchain/*[^~]) diff --git a/find-modules/rules_engine.py b/find-modules/rules_engine.py new file mode 100755 index 00000000..53c81413 --- /dev/null +++ b/find-modules/rules_engine.py @@ -0,0 +1,526 @@ +#!/usr/bin/env python +#============================================================================= +# Copyright 2016 by Shaheed Haque (srhaque@theiet.org) +# Copyright 2016 Stephen Kelly <steveire@gmail.com> +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions +# are met: +# +# 1. Redistributions of source code must retain the copyright +# notice, this list of conditions and the following disclaimer. +# 2. Redistributions in binary form must reproduce the copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution. +# 3. The name of the author may not be used to endorse or promote products +# derived from this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR +# IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES +# OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. +# IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, +# INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT +# NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF +# THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +#============================================================================= + +"""SIP file generation rules engine.""" + +from __future__ import print_function + +from abc import * +import argparse +import gettext +import inspect +import logging +import os +import re +import sys +import textwrap +import traceback +from copy import deepcopy +from clang.cindex import CursorKind + +from clang.cindex import AccessSpecifier + +class HelpFormatter(argparse.ArgumentDefaultsHelpFormatter, argparse.RawDescriptionHelpFormatter): + pass + + +logger = logging.getLogger(__name__) +gettext.install(__name__) +_SEPARATOR = "\x00" + +def _parents(container): + parents = [] + parent = container.semantic_parent + while parent and parent.kind != CursorKind.TRANSLATION_UNIT: + parents.append(parent.spelling) + parent = parent.semantic_parent + if parents: + parents = "::".join(reversed(parents)) + else: + parents = os.path.basename(container.translation_unit.spelling) + return parents + + +class Rule(object): + def __init__(self, db, rule_number, fn, pattern_zip): + self.db = db + self.rule_number = rule_number + self.fn = fn + self.usage = 0 + try: + groups = ["(?P<{}>{})".format(name, pattern) for pattern, name in pattern_zip] + groups = _SEPARATOR.join(groups) + self.matcher = re.compile(groups) + except Exception as e: + groups = ["{} '{}'".format(name, pattern) for pattern, name in pattern_zip] + groups = ", ".join(groups) + raise RuntimeError(_("Bad {}: {}: {}").format(self, groups, e)) + + def match(self, candidate): + return self.matcher.match(candidate) + + def trace_result(self, parents, item, original, modified): + fqn = parents + "::" + original["name"] + "[" + str(item.extent.start.line) + "]" + self._trace_result(fqn, original, modified) + + def _trace_result(self, fqn, original, modified): + if not modified["name"]: + logger.debug(_("Rule {} suppressed {}, {}").format(self, fqn, original)) + else: + delta = False + for k, v in original.iteritems(): + if v != modified[k]: + delta = True + break + if delta: + logger.debug(_("Rule {} modified {}, {}->{}").format(self, fqn, original, modified)) + else: + logger.warn(_("Rule {} did not modify {}, {}").format(self, fqn, original)) + + def __str__(self): + return "[{},{}]".format(self.rule_number, self.fn.__name__) + + +class AbstractCompiledRuleDb(object): + __metaclass__ = ABCMeta + + def __init__(self, db, parameter_names): + self.db = db + self.compiled_rules = [] + for i, raw_rule in enumerate(db()): + if len(raw_rule) != len(parameter_names) + 1: + raise RuntimeError(_("Bad raw rule {}: {}: {}").format(db.__name__, raw_rule, parameter_names)) + z = zip(raw_rule[:-1], parameter_names) + self.compiled_rules.append(Rule(db, i, raw_rule[-1], z)) + self.candidate_formatter = _SEPARATOR.join(["{}"] * len(parameter_names)) + + def _match(self, *args): + candidate = self.candidate_formatter.format(*args) + for rule in self.compiled_rules: + matcher = rule.match(candidate) + if matcher: + # + # Only use the first matching rule. + # + rule.usage += 1 + return matcher, rule + return None, None + + @abstractmethod + def apply(self, *args): + raise NotImplemented(_("Missing subclass")) + + def dump_usage(self, fn): + """ Dump the usage counts.""" + for rule in self.compiled_rules: + fn(self.__class__.__name__, str(rule), rule.usage) + + +class ContainerRuleDb(AbstractCompiledRuleDb): + """ + THE RULES FOR CONTAINERS. + + These are used to customise the behaviour of the SIP generator by allowing + the declaration for any container (class, namespace, struct, union) to be + customised, for example to add SIP compiler annotations. + + Each entry in the raw rule database must be a list with members as follows: + + 0. A regular expression which matches the fully-qualified name of the + "container" enclosing the container. + + 1. A regular expression which matches the container name. + + 2. A regular expression which matches any template parameters. + + 3. A regular expression which matches the container declaration. + + 4. A regular expression which matches any base specifiers. + + 5. A function. + + In use, the database is walked in order from the first entry. If the regular + expressions are matched, the function is called, and no further entries are + walked. The function is called with the following contract: + + def container_xxx(container, sip, matcher): + ''' + Return a modified declaration for the given container. + + :param container: The clang.cindex.Cursor for the container. + :param sip: A dict with the following keys: + + name The name of the container. + template_parameters Any template parameters. + decl The declaration. + base_specifiers Any base specifiers. + body The body, less the outer + pair of braces. + annotations Any SIP annotations. + + :param matcher: The re.Match object. This contains named + groups corresponding to the key names above + EXCEPT body and annotations. + + :return: An updated set of sip.xxx values. Setting sip.name to the + empty string will cause the container to be suppressed. + ''' + + :return: The compiled form of the rules. + """ + def __init__(self, db): + super(ContainerRuleDb, self).__init__(db, ["parents", "container", "template_parameters", "decl", "base_specifiers"]) + + def apply(self, container, sip): + """ + Walk over the rules database for functions, applying the first matching transformation. + + :param container: The clang.cindex.Cursor for the container. + :param sip: The SIP dict. + """ + parents = _parents(container) + matcher, rule = self._match(parents, sip["name"], sip["template_parameters"], sip["decl"], sip["base_specifiers"]) + if matcher: + before = deepcopy(sip) + rule.fn(container, sip, matcher) + rule.trace_result(parents, container, before, sip) + + +class FunctionRuleDb(AbstractCompiledRuleDb): + """ + THE RULES FOR FUNCTIONS. + + These are used to customise the behaviour of the SIP generator by allowing + the declaration for any function to be customised, for example to add SIP + compiler annotations. + + Each entry in the raw rule database must be a list with members as follows: + + 0. A regular expression which matches the fully-qualified name of the + "container" enclosing the function. + + 1. A regular expression which matches the function name. + + 2. A regular expression which matches any template parameters. + + 3. A regular expression which matches the function result. + + 4. A regular expression which matches the function parameters (e.g. + "int a, void *b" for "int foo(int a, void *b)"). + + 5. A function. + + In use, the database is walked in order from the first entry. If the regular + expressions are matched, the function is called, and no further entries are + walked. The function is called with the following contract: + + def function_xxx(container, function, sip, matcher): + ''' + Return a modified declaration for the given function. + + :param container: The clang.cindex.Cursor for the container. + :param function: The clang.cindex.Cursor for the function. + :param sip: A dict with the following keys: + + name The name of the function. + template_parameters Any template parameters. + fn_result Result, if not a constructor. + decl The declaration. + prefix Leading keyworks ("static"). Separated by space, + ends with a space. + suffix Trailing keywords ("const"). Separated by space, starts with + space. + annotations Any SIP annotations. + + :param matcher: The re.Match object. This contains named + groups corresponding to the key names above + EXCEPT annotations. + + :return: An updated set of sip.xxx values. Setting sip.name to the + empty string will cause the container to be suppressed. + ''' + + :return: The compiled form of the rules. + """ + def __init__(self, db): + super(FunctionRuleDb, self).__init__(db, ["container", "function", "template_parameters", "fn_result", "parameters"]) + + def apply(self, container, function, sip): + """ + Walk over the rules database for functions, applying the first matching transformation. + + :param container: The clang.cindex.Cursor for the container. + :param function: The clang.cindex.Cursor for the function. + :param sip: The SIP dict. + """ + parents = _parents(function) + matcher, rule = self._match(parents, sip["name"], ", ".join(sip["template_parameters"]), sip["fn_result"], ", ".join(sip["parameters"])) + if matcher: + before = deepcopy(sip) + rule.fn(container, function, sip, matcher) + rule.trace_result(parents, function, before, sip) + + +class ParameterRuleDb(AbstractCompiledRuleDb): + """ + THE RULES FOR FUNCTION PARAMETERS. + + These are used to customise the behaviour of the SIP generator by allowing + the declaration for any parameter in any function to be customised, for + example to add SIP compiler annotations. + + Each entry in the raw rule database must be a list with members as follows: + + 0. A regular expression which matches the fully-qualified name of the + "container" enclosing the function enclosing the parameter. + + 1. A regular expression which matches the function name enclosing the + parameter. + + 2. A regular expression which matches the parameter name. + + 3. A regular expression which matches the parameter declaration (e.g. + "int foo"). + + 4. A regular expression which matches the parameter initialiser (e.g. + "Xyz:MYCONST + 42"). + + 5. A function. + + In use, the database is walked in order from the first entry. If the regular + expressions are matched, the function is called, and no further entries are + walked. The function is called with the following contract: + + def parameter_xxx(container, function, parameter, sip, init, matcher): + ''' + Return a modified declaration and initialiser for the given parameter. + + :param container: The clang.cindex.Cursor for the container. + :param function: The clang.cindex.Cursor for the function. + :param parameter: The clang.cindex.Cursor for the parameter. + :param sip: A dict with the following keys: + + name The name of the function. + decl The declaration. + init Any initialiser. + annotations Any SIP annotations. + + :param matcher: The re.Match object. This contains named + groups corresponding to the key names above + EXCEPT annotations. + + :return: An updated set of sip.xxx values. + ''' + + :return: The compiled form of the rules. + """ + def __init__(self, db): + super(ParameterRuleDb, self).__init__(db, ["container", "function", "parameter", "decl", "init"]) + + def apply(self, container, function, parameter, sip): + """ + Walk over the rules database for parameters, applying the first matching transformation. + + :param container: The clang.cindex.Cursor for the container. + :param function: The clang.cindex.Cursor for the function. + :param parameter: The clang.cindex.Cursor for the parameter. + :param sip: The SIP dict. + """ + parents = _parents(function) + matcher, rule = self._match(parents, function.spelling, sip["name"], sip["decl"], sip["init"]) + if matcher: + before = deepcopy(sip) + rule.fn(container, function, parameter, sip, matcher) + rule.trace_result(parents, parameter, before, sip) + + +class VariableRuleDb(AbstractCompiledRuleDb): + """ + THE RULES FOR VARIABLES. + + These are used to customise the behaviour of the SIP generator by allowing + the declaration for any variable to be customised, for example to add SIP + compiler annotations. + + Each entry in the raw rule database must be a list with members as follows: + + 0. A regular expression which matches the fully-qualified name of the + "container" enclosing the variable. + + 1. A regular expression which matches the variable name. + + 2. A regular expression which matches the variable declaration (e.g. + "int foo"). + + 3. A function. + + In use, the database is walked in order from the first entry. If the regular + expressions are matched, the function is called, and no further entries are + walked. The function is called with the following contract: + + def variable_xxx(container, variable, sip, matcher): + ''' + Return a modified declaration for the given variable. + + :param container: The clang.cindex.Cursor for the container. + :param variable: The clang.cindex.Cursor for the variable. + :param sip: A dict with the following keys: + + name The name of the variable. + decl The declaration. + annotations Any SIP annotations. + + :param matcher: The re.Match object. This contains named + groups corresponding to the key names above + EXCEPT annotations. + + :return: An updated set of sip.xxx values. Setting sip.name to the + empty string will cause the container to be suppressed. + ''' + + :return: The compiled form of the rules. + """ + def __init__(self, db): + super(VariableRuleDb, self).__init__(db, ["container", "variable", "decl"]) + + def apply(self, container, variable, sip): + """ + Walk over the rules database for variables, applying the first matching transformation. + + :param container: The clang.cindex.Cursor for the container. + :param variable: The clang.cindex.Cursor for the variable. + :param sip: The SIP dict. + """ + parents = _parents(variable) + matcher, rule = self._match(parents, sip["name"], sip["decl"]) + if matcher: + before = deepcopy(sip) + rule.fn(container, variable, sip, matcher) + rule.trace_result(parents, variable, before, sip) + + +class RuleSet(object): + """ + To implement your own binding, create a subclass of RuleSet, also called + RuleSet in your own Python module. Your subclass will expose the raw rules + along with other ancilliary data exposed through the subclass methods. + + You then simply run the SIP generation and SIP compilation programs passing + in the name of your rules file + """ + __metaclass__ = ABCMeta + + @abstractmethod + def container_rules(self): + """ + Return a compiled list of rules for containers. + + :return: A ContainerRuleDb instance + """ + raise NotImplemented(_("Missing subclass implementation")) + + @abstractmethod + def function_rules(self): + """ + Return a compiled list of rules for functions. + + :return: A FunctionRuleDb instance + """ + raise NotImplemented(_("Missing subclass implementation")) + + @abstractmethod + def parameter_rules(self): + """ + Return a compiled list of rules for function parameters. + + :return: A ParameterRuleDb instance + """ + raise NotImplemented(_("Missing subclass implementation")) + + @abstractmethod + def variable_rules(self): + """ + Return a compiled list of rules for variables. + + :return: A VariableRuleDb instance + """ + raise NotImplemented(_("Missing subclass implementation")) + + def dump_unused(self): + """Usage statistics, to identify unused rules.""" + def dumper(db_name, rule, usage): + if usage: + logger.info(_("Rule {}::{} used {} times".format(db_name, rule, usage))) + else: + logger.warn(_("Rule {}::{} unused".format(db_name, rule))) + for db in [self.container_rules(), self.function_rules(), self.parameter_rules(), + self.variable_rules()]: + db.dump_usage(dumper) + + +def container_discard(container, sip, matcher): + sip["name"] = "" + +def function_discard(container, function, sip, matcher): + sip["name"] = "" + +def parameter_transfer_to_parent(container, function, parameter, sip, matcher): + if function.is_static_method(): + sip["annotations"].add("Transfer") + else: + sip["annotations"].add("TransferThis") + +def param_rewrite_mode_t_as_int(container, function, parameter, sip, matcher): + sip["decl"] = sip["decl"].replace("mode_t", "unsigned int") + +def return_rewrite_mode_t_as_int(container, function, sip, matcher): + sip["fn_result"] = "unsigned int" + +def variable_discard(container, variable, sip, matcher): + sip["name"] = "" + +def parameter_strip_class_enum(container, function, parameter, sip, matcher): + sip["decl"] = sip["decl"].replace("class ", "").replace("enum ", "") + +def function_discard_impl(container, function, sip, matcher): + if function.extent.start.column == 1: + sip["name"] = "" + +def rules(project_rules): + """ + Constructor. + + :param project_rules: The rules file for the project. + """ + import imp + imp.load_source("project_rules", project_rules) + # + # Statically prepare the rule logic. This takes the rules provided by the user and turns them into code. + # + return getattr(sys.modules["project_rules"], "RuleSet")() diff --git a/find-modules/sip_generator.py b/find-modules/sip_generator.py new file mode 100644 index 00000000..10be1477 --- /dev/null +++ b/find-modules/sip_generator.py @@ -0,0 +1,700 @@ +#!/usr/bin/env python +#============================================================================= +# Copyright 2016 by Shaheed Haque (srhaque@theiet.org) +# Copyright 2016 by Stephen Kelly (steveire@gmail.com) +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions +# are met: +# +# 1. Redistributions of source code must retain the copyright +# notice, this list of conditions and the following disclaimer. +# 2. Redistributions in binary form must reproduce the copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution. +# 3. The name of the author may not be used to endorse or promote products +# derived from this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR +# IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES +# OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. +# IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, +# INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT +# NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF +# THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +#============================================================================= + +"""SIP file generator for PyQt.""" + +from __future__ import print_function +import argparse +import gettext +import inspect +import logging +import os +import re +import subprocess +import sys +import traceback +from clang import cindex +from clang.cindex import AccessSpecifier, CursorKind, SourceRange, StorageClass, TokenKind, TypeKind, TranslationUnit + +import rules_engine + + +class HelpFormatter(argparse.ArgumentDefaultsHelpFormatter, argparse.RawDescriptionHelpFormatter): + pass + + +logger = logging.getLogger(__name__) +gettext.install(__name__) + +EXPR_KINDS = [ + CursorKind.UNEXPOSED_EXPR, + CursorKind.CONDITIONAL_OPERATOR, CursorKind.UNARY_OPERATOR, CursorKind.BINARY_OPERATOR, + CursorKind.INTEGER_LITERAL, CursorKind.FLOATING_LITERAL, CursorKind.STRING_LITERAL, + CursorKind.CXX_BOOL_LITERAL_EXPR, CursorKind.CXX_STATIC_CAST_EXPR, CursorKind.DECL_REF_EXPR +] +TEMPLATE_KINDS = [ + CursorKind.TYPE_REF, CursorKind.TEMPLATE_REF, CursorKind.NAMESPACE_REF + ] + EXPR_KINDS + + +class SipGenerator(object): + def __init__(self, project_rules, compile_flags, verbose=False, dump_includes=False, dump_privates=False): + """ + Constructor. + + :param project_rules: The rules for the file. + :param compile_flags: The compile flags for the file. + :param dump_includes: Turn on diagnostics for include files. + :param dump_privates: Turn on diagnostics for omitted private items. + """ + self.rules = project_rules + self.compile_flags = compile_flags + self.verbose = verbose + self.dump_includes = dump_includes + self.dump_privates = dump_privates + self.diagnostics = set() + self.tu = None + self.unpreprocessed_source = None + + @staticmethod + def describe(cursor, text=None): + if not text: + text = cursor.spelling + return "{} on line {} '{}'".format(cursor.kind.name, cursor.extent.start.line, text) + + def create_sip(self, h_file, include_filename): + """ + Actually convert the given source header file into its SIP equivalent. + + :param h_file: The source (header) file of interest. + :param include_filename: The (header) to generate in the sip file. + """ + + # + # Read in the original file. + # + source = h_file + self.unpreprocessed_source = [] + with open(source, "rU") as f: + for line in f: + self.unpreprocessed_source.append(line) + + index = cindex.Index.create() + self.tu = index.parse(source, ["-x", "c++"] + self.compile_flags) + for diag in self.tu.diagnostics: + # + # We expect to be run over hundreds of files. Any parsing issues are likely to be very repetitive. + # So, to avoid bothering the user, we suppress duplicates. + # + loc = diag.location + msg = "{}:{}[{}] {}".format(loc.file, loc.line, loc.column, diag.spelling) + if msg in self.diagnostics: + continue + self.diagnostics.add(msg) + logger.log(diag.severity, "Parse error {}".format(msg)) + if self.dump_includes: + for include in sorted(set(self.tu.get_includes())): + logger.debug(_("Used includes {}").format(include.include.name)) + # + # Run through the top level children in the translation unit. + # + body = self._container_get(self.tu.cursor, -1, h_file, include_filename) + return body, self.tu.get_includes + + CONTAINER_SKIPPABLE_UNEXPOSED_DECL = re.compile("_DECLARE_PRIVATE|friend|;") + FN_SKIPPABLE_ATTR = re.compile("_EXPORT|Q_REQUIRED_RESULT|format\(printf") + VAR_SKIPPABLE_ATTR = re.compile("_EXPORT") + TYPEDEF_SKIPPABLE_ATTR = re.compile("_EXPORT") + + def _container_get(self, container, level, h_file, include_filename): + """ + Generate the (recursive) translation for a class or namespace. + + :param container: A class or namespace. + :param h_file: Name of header file being processed. + :param level: Recursion level controls indentation. + :return: A string. + """ + + def skippable_attribute(member, text): + """ + We don't seem to have access to the __attribute__(())s, but at least we can look for stuff we care about. + + :param member: The attribute. + :param text: The raw source corresponding to the region of member. + """ + if text.find("_DEPRECATED") != -1: + sip["annotations"].add("Deprecated") + return True + SipGenerator._report_ignoring(container, member, text) + + sip = { + "name": container.displayname, + "annotations": set() + } + name = container.displayname + if container.access_specifier == AccessSpecifier.PRIVATE: + if self.dump_privates: + logger.debug("Ignoring private {}".format(SipGenerator.describe(container))) + return "" + body = "" + base_specifiers = [] + template_type_parameters = [] + had_copy_constructor = False + had_const_member = False + for member in container.get_children(): + # + # Only emit items in the translation unit. + # + if member.location.file.name != self.tu.spelling: + continue + decl = "" + if member.kind in [CursorKind.CXX_METHOD, CursorKind.FUNCTION_DECL, CursorKind.FUNCTION_TEMPLATE, + CursorKind.CONSTRUCTOR, CursorKind.DESTRUCTOR, CursorKind.CONVERSION_FUNCTION]: + decl = self._fn_get(container, member, level + 1) + elif member.kind == CursorKind.ENUM_DECL: + decl = self._enum_get(container, member, level + 1) + ";\n" + elif member.kind == CursorKind.CXX_ACCESS_SPEC_DECL: + decl = self._get_access_specifier(member, level + 1) + elif member.kind == CursorKind.TYPEDEF_DECL: + decl = self._typedef_get(member, level + 1) + elif member.kind == CursorKind.CXX_BASE_SPECIFIER: + # + # Strip off the leading "class". Except for TypeKind.UNEXPOSED... + # + base_specifiers.append(member.displayname.split(None, 2)[-1]) + elif member.kind == CursorKind.TEMPLATE_TYPE_PARAMETER: + template_type_parameters.append(member.displayname) + elif member.kind == CursorKind.TEMPLATE_NON_TYPE_PARAMETER: + template_type_parameters.append(member.type.spelling + " " + member.displayname) + elif member.kind in [CursorKind.VAR_DECL, CursorKind.FIELD_DECL]: + had_const_member = had_const_member or member.type.is_const_qualified() + decl = self._var_get(container, member, level + 1) + elif member.kind in [CursorKind.NAMESPACE, CursorKind.CLASS_DECL, + CursorKind.CLASS_TEMPLATE, CursorKind.CLASS_TEMPLATE_PARTIAL_SPECIALIZATION, + CursorKind.STRUCT_DECL, CursorKind.UNION_DECL]: + decl = self._container_get(member, level + 1, h_file, include_filename) + elif member.kind in TEMPLATE_KINDS + [CursorKind.USING_DECLARATION, CursorKind.USING_DIRECTIVE, + CursorKind.CXX_FINAL_ATTR]: + # + # Ignore: + # + # TEMPLATE_KINDS: Template type parameter. + # CursorKind.USING_DECLARATION, CursorKind.USING_DIRECTIVE: Using? Pah! + # CursorKind.CXX_FINAL_ATTR: Again, not much to be done with this. + # + pass + else: + SipGenerator._report_ignoring(container, member) + + def is_copy_constructor(member): + if member.kind != CursorKind.CONSTRUCTOR: + return False + numParams = 0 + hasSelfType = False + for child in member.get_children(): + numParams += 1 + if child.kind == CursorKind.PARM_DECL: + paramType = child.type.spelling + paramType = paramType.split("::")[-1] + paramType = paramType.replace("const", "").replace("&", "").strip() + hasSelfType = paramType == container.displayname + return numParams == 1 and hasSelfType + + def has_parameter_default(parameter): + for member in parameter.get_children(): + if member.kind.is_expression(): + return True + return False + + def is_default_constructor(member): + if member.kind != CursorKind.CONSTRUCTOR: + return False + numParams = 0 + for parameter in member.get_children(): + if (has_parameter_default(parameter)): + break + numParams += 1 + return numParams == 0 + + had_copy_constructor = had_copy_constructor or is_copy_constructor(member) + # + # Discard almost anything which is private. + # + if member.access_specifier == AccessSpecifier.PRIVATE: + if member.kind == CursorKind.CXX_ACCESS_SPEC_DECL: + # + # We need these because... + # + pass + elif is_copy_constructor(member) or is_default_constructor(member): + # + # ...we need to pass private copy contructors to the SIP compiler. + # + pass + else: + if self.dump_privates: + logger.debug("Ignoring private {}".format(SipGenerator.describe(member))) + continue + + if decl: + if self.verbose: + pad = " " * ((level + 1) * 4) + body += pad + "// {}\n".format(SipGenerator.describe(member)) + body += decl + # + # Empty containers are still useful if they provide namespaces or forward declarations. + # + if not body and level >= 0: + body = "\n" + text = self._read_source(container.extent) + if not text.endswith("}"): + # + # Forward declaration. + # + sip["annotations"].add("External") + if body and level >= 0: + if container.kind == CursorKind.NAMESPACE: + container_type = "namespace " + name + elif container.kind in [CursorKind.CLASS_DECL, CursorKind.CLASS_TEMPLATE, + CursorKind.CLASS_TEMPLATE_PARTIAL_SPECIALIZATION]: + container_type = "class " + name + elif container.kind == CursorKind.STRUCT_DECL: + container_type = "struct " + name + elif container.kind == CursorKind.UNION_DECL: + container_type = "union " + name + else: + raise AssertionError( + _("Unexpected container {}: {}[{}]").format(container.kind, name, container.extent.start.line)) + # + # Generate private copy constructor for non-copyable types. + # + if had_const_member and not had_copy_constructor: + body += " private:\n {}(const {} &); // Generated\n".format(name, container.type.get_canonical().spelling) + # + # Flesh out the SIP context for the rules engine. + # + sip["template_parameters"] = ", ".join(template_type_parameters) + sip["decl"] = container_type + sip["base_specifiers"] = ", ".join(base_specifiers) + sip["body"] = body + self.rules.container_rules().apply(container, sip) + pad = " " * (level * 4) + if sip["name"]: + decl = pad + sip["decl"] + if "External" in sip["annotations"]: + # + # SIP /External/ does not seem to work as one might wish. Suppress. + # + body = decl + " /External/;\n" + body = pad + "// Discarded {}\n".format(SipGenerator.describe(container)) + else: + if sip["base_specifiers"]: + decl += ": " + sip["base_specifiers"] + if sip["annotations"]: + decl += " /" + ",".join(sip["annotations"]) + "/" + if sip["template_parameters"]: + decl = pad + "template <" + sip["template_parameters"] + ">\n" + decl + decl += "\n" + pad + "{\n" + decl += "%TypeHeaderCode\n#include <{}>\n%End\n".format(include_filename) + body = decl + sip["body"] + pad + "};\n" + else: + body = pad + "// Discarded {}\n".format(SipGenerator.describe(container)) + return body + + def _get_access_specifier(self, member, level): + """ + Skip access specifiers embedded in the Q_OBJECT macro. + """ + access_specifier = self._read_source(member.extent) + if access_specifier == "Q_OBJECT": + return "" + pad = " " * ((level - 1) * 4) + decl = pad + access_specifier + "\n" + return decl + + def _enum_get(self, container, enum, level): + pad = " " * (level * 4) + decl = pad + "enum {} {{\n".format(enum.displayname) + enumerations = [] + for enum in enum.get_children(): + enumerations.append(pad + " {}".format(enum.displayname)) + assert enum.kind == CursorKind.ENUM_CONSTANT_DECL + decl += ",\n".join(enumerations) + "\n" + decl += pad + "}" + return decl + + def _fn_get(self, container, function, level): + """ + Generate the translation for a function. + + :param container: A class or namespace. + :param function: The function object. + :param level: Recursion level controls indentation. + :return: A string. + """ + + def skippable_attribute(member, text): + """ + We don't seem to have access to the __attribute__(())s, but at least we can look for stuff we care about. + + :param member: The attribute. + :param text: The raw source corresponding to the region of member. + """ + if SipGenerator.FN_SKIPPABLE_ATTR.search(text): + return True + SipGenerator._report_ignoring(function, member, text) + + sip = { + "name": function.spelling, + } + parameters = [] + template_parameters = [] + for child in function.get_children(): + if child.kind == CursorKind.PARM_DECL: + parameter = child.displayname or "__{}".format(len(parameters)) + theType = child.type.get_canonical() + typeSpelling = theType.spelling + if theType.kind == TypeKind.POINTER: + typeSpelling = theType.get_pointee().spelling + "* " + + decl = "{} {}".format(typeSpelling, parameter) + child_sip = { + "name": parameter, + "decl": decl, + "init": self._fn_get_parameter_default(function, child), + "annotations": set() + } + self.rules.parameter_rules().apply(container, function, child, child_sip) + decl = child_sip["decl"] + if child_sip["annotations"]: + decl += " /" + ",".join(child_sip["annotations"]) + "/" + if child_sip["init"]: + decl += " = " + child_sip["init"] + parameters.append(decl) + elif child.kind in [CursorKind.COMPOUND_STMT, CursorKind.CXX_OVERRIDE_ATTR, + CursorKind.MEMBER_REF, CursorKind.DECL_REF_EXPR, CursorKind.CALL_EXPR] + TEMPLATE_KINDS: + # + # Ignore: + # + # CursorKind.COMPOUND_STMT: Function body. + # CursorKind.CXX_OVERRIDE_ATTR: The "override" keyword. + # CursorKind.MEMBER_REF, CursorKind.DECL_REF_EXPR, CursorKind.CALL_EXPR: Constructor initialisers. + # TEMPLATE_KINDS: The result type. + # + pass + elif child.kind == CursorKind.TEMPLATE_TYPE_PARAMETER: + template_parameters.append(child.displayname) + elif child.kind == CursorKind.TEMPLATE_NON_TYPE_PARAMETER: + template_parameters.append(child.type.spelling + " " + child.displayname) + else: + text = self._read_source(child.extent) + if child.kind in [CursorKind.UNEXPOSED_ATTR, CursorKind.VISIBILITY_ATTR] and skippable_attribute(child, + text): + pass + else: + SipGenerator._report_ignoring(function, child) + # + # Flesh out the SIP context for the rules engine. + # + sip["template_parameters"] = template_parameters + if function.kind in [CursorKind.CONSTRUCTOR, CursorKind.DESTRUCTOR]: + sip["fn_result"] = "" + else: + sip["fn_result"] = function.result_type.spelling + sip["parameters"] = parameters + sip["prefix"], sip["suffix"] = self._fn_get_decorators(function) + self.rules.function_rules().apply(container, function, sip) + pad = " " * (level * 4) + if sip["name"]: + sip["template_parameters"] = ", ".join(sip["template_parameters"]) + decl = sip["name"] + "(" + ", ".join(sip["parameters"]) + ")" + if sip["fn_result"]: + decl = sip["fn_result"] + " " + decl + decl = pad + sip["prefix"] + decl + sip["suffix"] + if sip["template_parameters"]: + decl = pad + "template <" + sip["template_parameters"] + ">\n" + decl + decl += ";\n" + else: + decl = pad + "// Discarded {}\n".format(SipGenerator.describe(function)) + return decl + + def _fn_get_decorators(self, function): + """ + The parser does not provide direct access to the complete keywords (explicit, const, static, etc) of a function + in the displayname. It would be nice to get these from the AST, but I cannot find where they are hiding. + + Now, we could resort to using the original source. That does not bode well if you have macros (QOBJECT, + xxxDEPRECATED?), inlined bodies and the like, using the rule engine could be used to patch corner cases... + + ...or we can try to guess what SIP cares about, i.e static and maybe const. Luckily (?), we have those to hand! + + :param function: The function object. + :return: prefix, suffix String containing any prefix or suffix keywords. + """ + suffix = "" + if function.is_const_method(): + suffix += " const" + prefix = "" + if function.is_static_method(): + prefix += "static " + if function.is_virtual_method(): + prefix += "virtual " + if function.is_pure_virtual_method(): + suffix += " = 0" + return prefix, suffix + + def _fn_get_parameter_default(self, function, parameter): + """ + The parser does not seem to provide access to the complete text of a parameter. + This makes it hard to find any default values, so we: + + 1. Run the lexer from "here" to the end of the file, bailing out when we see the "," + or a ")" marking the end. + 2. Watch for the assignment. + """ + def _get_param_type(parameter): + result = parameter.type.get_declaration().type + + if (parameter.type.get_declaration().type.kind == TypeKind.TYPEDEF): + isQFlags = False + for member in parameter.type.get_declaration().get_children(): + if member.kind == CursorKind.TEMPLATE_REF and member.spelling == "QFlags": + isQFlags = True + if isQFlags and member.kind == CursorKind.TYPE_REF: + result = member.type + break + + return result + + def _get_param_value(text, parameterType): + if text == "0": + return text + if not "::" in parameterType.spelling: + return text + try: + typeText, typeInit = text.split("(") + typeInit = "(" + typeInit + except: + typeText = text + typeInit = "" + + prefix = parameterType.spelling.rsplit("::", 1)[0] + if "::" in typeText: + typeText = typeText.rsplit("::", 1)[1] + return prefix + "::" + typeText + typeInit + + + for member in parameter.get_children(): + if member.kind.is_expression(): + + possible_extent = SourceRange.from_locations(parameter.extent.start, function.extent.end) + text = "" + bracket_level = 0 + found_start = False + found_end = False + for token in self.tu.get_tokens(extent=possible_extent): + if (token.spelling == "="): + found_start = True + continue + if token.spelling == "," and bracket_level == 0: + found_end = True + break + elif token.spelling == "(": + bracket_level += 1 + text += token.spelling + elif token.spelling == ")": + if bracket_level == 0: + found_end = True + break + bracket_level -= 1 + text += token.spelling + if bracket_level == 0: + found_end = True + break + elif found_start: + text += token.spelling + if not found_end and text: + RuntimeError(_("No end found for {}::{}, '{}'").format(function.spelling, parameter.spelling, text)) + + parameterType = _get_param_type(parameter) + + return _get_param_value(text, parameterType) + return "" + + def _typedef_get(self, typedef, level): + """ + Generate the translation for a typedef. + + :param container: A class or namespace. + :param typedef: The typedef object. + :param level: Recursion level controls indentation. + :return: A string. + """ + + pad = " " * (level * 4) + + decl = pad + "typedef {} {}".format(typedef.underlying_typedef_type.spelling, typedef.displayname) + decl += ";\n" + return decl + + def _var_get(self, container, variable, level): + """ + Generate the translation for a variable. + + :param container: A class or namespace. + :param variable: The variable object. + :param level: Recursion level controls indentation. + :return: A string. + """ + + def skippable_attribute(member, text): + """ + We don't seem to have access to the __attribute__(())s, but at least we can look for stuff we care about. + + :param member: The attribute. + :param text: The raw source corresponding to the region of member. + """ + if SipGenerator.VAR_SKIPPABLE_ATTR.search(text): + return True + SipGenerator._report_ignoring(container, member, text) + + sip = { + "name": variable.spelling + } + for child in variable.get_children(): + if child.kind in TEMPLATE_KINDS + [CursorKind.STRUCT_DECL, CursorKind.UNION_DECL]: + # + # Ignore: + # + # TEMPLATE_KINDS, CursorKind.STRUCT_DECL, CursorKind.UNION_DECL: : The variable type. + # + pass + else: + text = self._read_source(child.extent) + if child.kind == CursorKind.VISIBILITY_ATTR and skippable_attribute(child, text): + pass + else: + SipGenerator._report_ignoring(variable, child) + # + # Flesh out the SIP context for the rules engine. + # + decl = "{} {}".format(variable.type.spelling, variable.spelling) + sip["decl"] = decl + self.rules.variable_rules().apply(container, variable, sip) + + pad = " " * (level * 4) + if sip["name"]: + decl = sip["decl"] + # + # SIP does not support protected variables, so we ignore them. + # + if variable.access_specifier == AccessSpecifier.PROTECTED: + decl = pad + "// Discarded {}\n".format(SipGenerator.describe(variable)) + else: + decl = pad + decl + ";\n" + else: + decl = pad + "// Discarded {}\n".format(SipGenerator.describe(variable)) + return decl + + def _read_source(self, extent): + """ + Read the given range from the unpre-processed source. + + :param extent: The range of text required. + """ + extract = self.unpreprocessed_source[extent.start.line - 1:extent.end.line] + if extent.start.line == extent.end.line: + extract[0] = extract[0][extent.start.column - 1:extent.end.column - 1] + else: + extract[0] = extract[0][extent.start.column - 1:] + extract[-1] = extract[-1][:extent.end.column - 1] + # + # Return a single line of text. + # + return "".join(extract).replace("\n", " ") + + @staticmethod + def _report_ignoring(parent, child, text=None): + if not text: + text = child.displayname or child.spelling + logger.debug(_("Ignoring {} {} child {}").format(parent.kind.name, parent.spelling, SipGenerator.describe(child, text))) + + +def main(argv=None): + """ + Take a single C++ header file and generate the corresponding SIP file. + Beyond simple generation of the SIP file from the corresponding C++ + header file, a set of rules can be used to customise the generated + SIP file. + + Examples: + + sip_generator.py /usr/include/KF5/KItemModels/kselectionproxymodel.h + """ + if argv is None: + argv = sys.argv + parser = argparse.ArgumentParser(epilog=inspect.getdoc(main), + formatter_class=HelpFormatter) + parser.add_argument("-v", "--verbose", action="store_true", default=False, help=_("Enable verbose output")) + parser.add_argument("--flags", + help=_("Semicolon-separated C++ compile flags to use")) + parser.add_argument("--include_filename", help=_("C++ header include to compile")) + parser.add_argument("libclang", help=_("libclang library to use for parsing")) + parser.add_argument("project_rules", help=_("Project rules")) + parser.add_argument("source", help=_("C++ header to process")) + parser.add_argument("output", help=_("output filename to write")) + try: + args = parser.parse_args(argv[1:]) + if args.verbose: + logging.basicConfig(level=logging.DEBUG, format='%(asctime)s %(name)s %(levelname)s: %(message)s') + else: + logging.basicConfig(level=logging.INFO, format='%(levelname)s: %(message)s') + # + # Generate! + # + + cindex.Config.set_library_file(args.libclang) + + rules = rules_engine.rules(args.project_rules) + g = SipGenerator(rules, args.flags.lstrip().split(";"), args.verbose) + body, includes = g.create_sip(args.source, args.include_filename) + with open(args.output, "w") as outputFile: + outputFile.write(body) + except Exception as e: + tbk = traceback.format_exc() + print(tbk) + return -1 + + +if __name__ == "__main__": + if sys.argv[1] != "--self-check": + sys.exit(main()) + else: + cindex.Config.set_library_file(sys.argv[2]) |