diff options
| -rw-r--r-- | CMakeLists.txt | 2 | ||||
| -rwxr-xr-x | find-modules/rules_engine.py | 526 | ||||
| -rw-r--r-- | find-modules/sip_generator.py | 700 |
3 files changed, 1227 insertions, 1 deletions
diff --git a/CMakeLists.txt b/CMakeLists.txt index 94e88306..fe7ebac5 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -117,7 +117,7 @@ install(FILES ${installModuleFiles} DESTINATION ${MODULES_INSTALL_DIR}) file(GLOB installKdeModuleFiles ${CMAKE_SOURCE_DIR}/kde-modules/*[^~]) install(FILES ${installKdeModuleFiles} DESTINATION ${KDE_MODULES_INSTALL_DIR}) -file(GLOB installFindModuleFiles ${CMAKE_SOURCE_DIR}/find-modules/*[^~]) +file(GLOB installFindModuleFiles ${CMAKE_SOURCE_DIR}/find-modules/*.cmake ${CMAKE_SOURCE_DIR}/find-modules/*.py) install(FILES ${installFindModuleFiles} DESTINATION ${FIND_MODULES_INSTALL_DIR}) file(GLOB installToolchainModuleFiles ${CMAKE_SOURCE_DIR}/toolchain/*[^~]) diff --git a/find-modules/rules_engine.py b/find-modules/rules_engine.py new file mode 100755 index 00000000..53c81413 --- /dev/null +++ b/find-modules/rules_engine.py @@ -0,0 +1,526 @@ +#!/usr/bin/env python +#============================================================================= +# Copyright 2016 by Shaheed Haque (srhaque@theiet.org) +# Copyright 2016 Stephen Kelly <steveire@gmail.com> +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions +# are met: +# +# 1. Redistributions of source code must retain the copyright +# notice, this list of conditions and the following disclaimer. +# 2. Redistributions in binary form must reproduce the copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution. +# 3. The name of the author may not be used to endorse or promote products +# derived from this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR +# IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES +# OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. +# IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, +# INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT +# NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF +# THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +#============================================================================= + +"""SIP file generation rules engine.""" + +from __future__ import print_function + +from abc import * +import argparse +import gettext +import inspect +import logging +import os +import re +import sys +import textwrap +import traceback +from copy import deepcopy +from clang.cindex import CursorKind + +from clang.cindex import AccessSpecifier + +class HelpFormatter(argparse.ArgumentDefaultsHelpFormatter, argparse.RawDescriptionHelpFormatter): + pass + + +logger = logging.getLogger(__name__) +gettext.install(__name__) +_SEPARATOR = "\x00" + +def _parents(container): + parents = [] + parent = container.semantic_parent + while parent and parent.kind != CursorKind.TRANSLATION_UNIT: + parents.append(parent.spelling) + parent = parent.semantic_parent + if parents: + parents = "::".join(reversed(parents)) + else: + parents = os.path.basename(container.translation_unit.spelling) + return parents + + +class Rule(object): + def __init__(self, db, rule_number, fn, pattern_zip): + self.db = db + self.rule_number = rule_number + self.fn = fn + self.usage = 0 + try: + groups = ["(?P<{}>{})".format(name, pattern) for pattern, name in pattern_zip] + groups = _SEPARATOR.join(groups) + self.matcher = re.compile(groups) + except Exception as e: + groups = ["{} '{}'".format(name, pattern) for pattern, name in pattern_zip] + groups = ", ".join(groups) + raise RuntimeError(_("Bad {}: {}: {}").format(self, groups, e)) + + def match(self, candidate): + return self.matcher.match(candidate) + + def trace_result(self, parents, item, original, modified): + fqn = parents + "::" + original["name"] + "[" + str(item.extent.start.line) + "]" + self._trace_result(fqn, original, modified) + + def _trace_result(self, fqn, original, modified): + if not modified["name"]: + logger.debug(_("Rule {} suppressed {}, {}").format(self, fqn, original)) + else: + delta = False + for k, v in original.iteritems(): + if v != modified[k]: + delta = True + break + if delta: + logger.debug(_("Rule {} modified {}, {}->{}").format(self, fqn, original, modified)) + else: + logger.warn(_("Rule {} did not modify {}, {}").format(self, fqn, original)) + + def __str__(self): + return "[{},{}]".format(self.rule_number, self.fn.__name__) + + +class AbstractCompiledRuleDb(object): + __metaclass__ = ABCMeta + + def __init__(self, db, parameter_names): + self.db = db + self.compiled_rules = [] + for i, raw_rule in enumerate(db()): + if len(raw_rule) != len(parameter_names) + 1: + raise RuntimeError(_("Bad raw rule {}: {}: {}").format(db.__name__, raw_rule, parameter_names)) + z = zip(raw_rule[:-1], parameter_names) + self.compiled_rules.append(Rule(db, i, raw_rule[-1], z)) + self.candidate_formatter = _SEPARATOR.join(["{}"] * len(parameter_names)) + + def _match(self, *args): + candidate = self.candidate_formatter.format(*args) + for rule in self.compiled_rules: + matcher = rule.match(candidate) + if matcher: + # + # Only use the first matching rule. + # + rule.usage += 1 + return matcher, rule + return None, None + + @abstractmethod + def apply(self, *args): + raise NotImplemented(_("Missing subclass")) + + def dump_usage(self, fn): + """ Dump the usage counts.""" + for rule in self.compiled_rules: + fn(self.__class__.__name__, str(rule), rule.usage) + + +class ContainerRuleDb(AbstractCompiledRuleDb): + """ + THE RULES FOR CONTAINERS. + + These are used to customise the behaviour of the SIP generator by allowing + the declaration for any container (class, namespace, struct, union) to be + customised, for example to add SIP compiler annotations. + + Each entry in the raw rule database must be a list with members as follows: + + 0. A regular expression which matches the fully-qualified name of the + "container" enclosing the container. + + 1. A regular expression which matches the container name. + + 2. A regular expression which matches any template parameters. + + 3. A regular expression which matches the container declaration. + + 4. A regular expression which matches any base specifiers. + + 5. A function. + + In use, the database is walked in order from the first entry. If the regular + expressions are matched, the function is called, and no further entries are + walked. The function is called with the following contract: + + def container_xxx(container, sip, matcher): + ''' + Return a modified declaration for the given container. + + :param container: The clang.cindex.Cursor for the container. + :param sip: A dict with the following keys: + + name The name of the container. + template_parameters Any template parameters. + decl The declaration. + base_specifiers Any base specifiers. + body The body, less the outer + pair of braces. + annotations Any SIP annotations. + + :param matcher: The re.Match object. This contains named + groups corresponding to the key names above + EXCEPT body and annotations. + + :return: An updated set of sip.xxx values. Setting sip.name to the + empty string will cause the container to be suppressed. + ''' + + :return: The compiled form of the rules. + """ + def __init__(self, db): + super(ContainerRuleDb, self).__init__(db, ["parents", "container", "template_parameters", "decl", "base_specifiers"]) + + def apply(self, container, sip): + """ + Walk over the rules database for functions, applying the first matching transformation. + + :param container: The clang.cindex.Cursor for the container. + :param sip: The SIP dict. + """ + parents = _parents(container) + matcher, rule = self._match(parents, sip["name"], sip["template_parameters"], sip["decl"], sip["base_specifiers"]) + if matcher: + before = deepcopy(sip) + rule.fn(container, sip, matcher) + rule.trace_result(parents, container, before, sip) + + +class FunctionRuleDb(AbstractCompiledRuleDb): + """ + THE RULES FOR FUNCTIONS. + + These are used to customise the behaviour of the SIP generator by allowing + the declaration for any function to be customised, for example to add SIP + compiler annotations. + + Each entry in the raw rule database must be a list with members as follows: + + 0. A regular expression which matches the fully-qualified name of the + "container" enclosing the function. + + 1. A regular expression which matches the function name. + + 2. A regular expression which matches any template parameters. + + 3. A regular expression which matches the function result. + + 4. A regular expression which matches the function parameters (e.g. + "int a, void *b" for "int foo(int a, void *b)"). + + 5. A function. + + In use, the database is walked in order from the first entry. If the regular + expressions are matched, the function is called, and no further entries are + walked. The function is called with the following contract: + + def function_xxx(container, function, sip, matcher): + ''' + Return a modified declaration for the given function. + + :param container: The clang.cindex.Cursor for the container. + :param function: The clang.cindex.Cursor for the function. + :param sip: A dict with the following keys: + + name The name of the function. + template_parameters Any template parameters. + fn_result Result, if not a constructor. + decl The declaration. + prefix Leading keyworks ("static"). Separated by space, + ends with a space. + suffix Trailing keywords ("const"). Separated by space, starts with + space. + annotations Any SIP annotations. + + :param matcher: The re.Match object. This contains named + groups corresponding to the key names above + EXCEPT annotations. + + :return: An updated set of sip.xxx values. Setting sip.name to the + empty string will cause the container to be suppressed. + ''' + + :return: The compiled form of the rules. + """ + def __init__(self, db): + super(FunctionRuleDb, self).__init__(db, ["container", "function", "template_parameters", "fn_result", "parameters"]) + + def apply(self, container, function, sip): + """ + Walk over the rules database for functions, applying the first matching transformation. + + :param container: The clang.cindex.Cursor for the container. + :param function: The clang.cindex.Cursor for the function. + :param sip: The SIP dict. + """ + parents = _parents(function) + matcher, rule = self._match(parents, sip["name"], ", ".join(sip["template_parameters"]), sip["fn_result"], ", ".join(sip["parameters"])) + if matcher: + before = deepcopy(sip) + rule.fn(container, function, sip, matcher) + rule.trace_result(parents, function, before, sip) + + +class ParameterRuleDb(AbstractCompiledRuleDb): + """ + THE RULES FOR FUNCTION PARAMETERS. + + These are used to customise the behaviour of the SIP generator by allowing + the declaration for any parameter in any function to be customised, for + example to add SIP compiler annotations. + + Each entry in the raw rule database must be a list with members as follows: + + 0. A regular expression which matches the fully-qualified name of the + "container" enclosing the function enclosing the parameter. + + 1. A regular expression which matches the function name enclosing the + parameter. + + 2. A regular expression which matches the parameter name. + + 3. A regular expression which matches the parameter declaration (e.g. + "int foo"). + + 4. A regular expression which matches the parameter initialiser (e.g. + "Xyz:MYCONST + 42"). + + 5. A function. + + In use, the database is walked in order from the first entry. If the regular + expressions are matched, the function is called, and no further entries are + walked. The function is called with the following contract: + + def parameter_xxx(container, function, parameter, sip, init, matcher): + ''' + Return a modified declaration and initialiser for the given parameter. + + :param container: The clang.cindex.Cursor for the container. + :param function: The clang.cindex.Cursor for the function. + :param parameter: The clang.cindex.Cursor for the parameter. + :param sip: A dict with the following keys: + + name The name of the function. + decl The declaration. + init Any initialiser. + annotations Any SIP annotations. + + :param matcher: The re.Match object. This contains named + groups corresponding to the key names above + EXCEPT annotations. + + :return: An updated set of sip.xxx values. + ''' + + :return: The compiled form of the rules. + """ + def __init__(self, db): + super(ParameterRuleDb, self).__init__(db, ["container", "function", "parameter", "decl", "init"]) + + def apply(self, container, function, parameter, sip): + """ + Walk over the rules database for parameters, applying the first matching transformation. + + :param container: The clang.cindex.Cursor for the container. + :param function: The clang.cindex.Cursor for the function. + :param parameter: The clang.cindex.Cursor for the parameter. + :param sip: The SIP dict. + """ + parents = _parents(function) + matcher, rule = self._match(parents, function.spelling, sip["name"], sip["decl"], sip["init"]) + if matcher: + before = deepcopy(sip) + rule.fn(container, function, parameter, sip, matcher) + rule.trace_result(parents, parameter, before, sip) + + +class VariableRuleDb(AbstractCompiledRuleDb): + """ + THE RULES FOR VARIABLES. + + These are used to customise the behaviour of the SIP generator by allowing + the declaration for any variable to be customised, for example to add SIP + compiler annotations. + + Each entry in the raw rule database must be a list with members as follows: + + 0. A regular expression which matches the fully-qualified name of the + "container" enclosing the variable. + + 1. A regular expression which matches the variable name. + + 2. A regular expression which matches the variable declaration (e.g. + "int foo"). + + 3. A function. + + In use, the database is walked in order from the first entry. If the regular + expressions are matched, the function is called, and no further entries are + walked. The function is called with the following contract: + + def variable_xxx(container, variable, sip, matcher): + ''' + Return a modified declaration for the given variable. + + :param container: The clang.cindex.Cursor for the container. + :param variable: The clang.cindex.Cursor for the variable. + :param sip: A dict with the following keys: + + name The name of the variable. + decl The declaration. + annotations Any SIP annotations. + + :param matcher: The re.Match object. This contains named + groups corresponding to the key names above + EXCEPT annotations. + + :return: An updated set of sip.xxx values. Setting sip.name to the + empty string will cause the container to be suppressed. + ''' + + :return: The compiled form of the rules. + """ + def __init__(self, db): + super(VariableRuleDb, self).__init__(db, ["container", "variable", "decl"]) + + def apply(self, container, variable, sip): + """ + Walk over the rules database for variables, applying the first matching transformation. + + :param container: The clang.cindex.Cursor for the container. + :param variable: The clang.cindex.Cursor for the variable. + :param sip: The SIP dict. + """ + parents = _parents(variable) + matcher, rule = self._match(parents, sip["name"], sip["decl"]) + if matcher: + before = deepcopy(sip) + rule.fn(container, variable, sip, matcher) + rule.trace_result(parents, variable, before, sip) + + +class RuleSet(object): + """ + To implement your own binding, create a subclass of RuleSet, also called + RuleSet in your own Python module. Your subclass will expose the raw rules + along with other ancilliary data exposed through the subclass methods. + + You then simply run the SIP generation and SIP compilation programs passing + in the name of your rules file + """ + __metaclass__ = ABCMeta + + @abstractmethod + def container_rules(self): + """ + Return a compiled list of rules for containers. + + :return: A ContainerRuleDb instance + """ + raise NotImplemented(_("Missing subclass implementation")) + + @abstractmethod + def function_rules(self): + """ + Return a compiled list of rules for functions. + + :return: A FunctionRuleDb instance + """ + raise NotImplemented(_("Missing subclass implementation")) + + @abstractmethod + def parameter_rules(self): + """ + Return a compiled list of rules for function parameters. + + :return: A ParameterRuleDb instance + """ + raise NotImplemented(_("Missing subclass implementation")) + + @abstractmethod + def variable_rules(self): + """ + Return a compiled list of rules for variables. + + :return: A VariableRuleDb instance + """ + raise NotImplemented(_("Missing subclass implementation")) + + def dump_unused(self): + """Usage statistics, to identify unused rules.""" + def dumper(db_name, rule, usage): + if usage: + logger.info(_("Rule {}::{} used {} times".format(db_name, rule, usage))) + else: + logger.warn(_("Rule {}::{} unused".format(db_name, rule))) + for db in [self.container_rules(), self.function_rules(), self.parameter_rules(), + self.variable_rules()]: + db.dump_usage(dumper) + + +def container_discard(container, sip, matcher): + sip["name"] = "" + +def function_discard(container, function, sip, matcher): + sip["name"] = "" + +def parameter_transfer_to_parent(container, function, parameter, sip, matcher): + if function.is_static_method(): + sip["annotations"].add("Transfer") + else: + sip["annotations"].add("TransferThis") + +def param_rewrite_mode_t_as_int(container, function, parameter, sip, matcher): + sip["decl"] = sip["decl"].replace("mode_t", "unsigned int") + +def return_rewrite_mode_t_as_int(container, function, sip, matcher): + sip["fn_result"] = "unsigned int" + +def variable_discard(container, variable, sip, matcher): + sip["name"] = "" + +def parameter_strip_class_enum(container, function, parameter, sip, matcher): + sip["decl"] = sip["decl"].replace("class ", "").replace("enum ", "") + +def function_discard_impl(container, function, sip, matcher): + if function.extent.start.column == 1: + sip["name"] = "" + +def rules(project_rules): + """ + Constructor. + + :param project_rules: The rules file for the project. + """ + import imp + imp.load_source("project_rules", project_rules) + # + # Statically prepare the rule logic. This takes the rules provided by the user and turns them into code. + # + return getattr(sys.modules["project_rules"], "RuleSet")() diff --git a/find-modules/sip_generator.py b/find-modules/sip_generator.py new file mode 100644 index 00000000..10be1477 --- /dev/null +++ b/find-modules/sip_generator.py @@ -0,0 +1,700 @@ +#!/usr/bin/env python +#============================================================================= +# Copyright 2016 by Shaheed Haque (srhaque@theiet.org) +# Copyright 2016 by Stephen Kelly (steveire@gmail.com) +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions +# are met: +# +# 1. Redistributions of source code must retain the copyright +# notice, this list of conditions and the following disclaimer. +# 2. Redistributions in binary form must reproduce the copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution. +# 3. The name of the author may not be used to endorse or promote products +# derived from this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR +# IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES +# OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. +# IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, +# INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT +# NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF +# THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +#============================================================================= + +"""SIP file generator for PyQt.""" + +from __future__ import print_function +import argparse +import gettext +import inspect +import logging +import os +import re +import subprocess +import sys +import traceback +from clang import cindex +from clang.cindex import AccessSpecifier, CursorKind, SourceRange, StorageClass, TokenKind, TypeKind, TranslationUnit + +import rules_engine + + +class HelpFormatter(argparse.ArgumentDefaultsHelpFormatter, argparse.RawDescriptionHelpFormatter): + pass + + +logger = logging.getLogger(__name__) +gettext.install(__name__) + +EXPR_KINDS = [ + CursorKind.UNEXPOSED_EXPR, + CursorKind.CONDITIONAL_OPERATOR, CursorKind.UNARY_OPERATOR, CursorKind.BINARY_OPERATOR, + CursorKind.INTEGER_LITERAL, CursorKind.FLOATING_LITERAL, CursorKind.STRING_LITERAL, + CursorKind.CXX_BOOL_LITERAL_EXPR, CursorKind.CXX_STATIC_CAST_EXPR, CursorKind.DECL_REF_EXPR +] +TEMPLATE_KINDS = [ + CursorKind.TYPE_REF, CursorKind.TEMPLATE_REF, CursorKind.NAMESPACE_REF + ] + EXPR_KINDS + + +class SipGenerator(object): + def __init__(self, project_rules, compile_flags, verbose=False, dump_includes=False, dump_privates=False): + """ + Constructor. + + :param project_rules: The rules for the file. + :param compile_flags: The compile flags for the file. + :param dump_includes: Turn on diagnostics for include files. + :param dump_privates: Turn on diagnostics for omitted private items. + """ + self.rules = project_rules + self.compile_flags = compile_flags + self.verbose = verbose + self.dump_includes = dump_includes + self.dump_privates = dump_privates + self.diagnostics = set() + self.tu = None + self.unpreprocessed_source = None + + @staticmethod + def describe(cursor, text=None): + if not text: + text = cursor.spelling + return "{} on line {} '{}'".format(cursor.kind.name, cursor.extent.start.line, text) + + def create_sip(self, h_file, include_filename): + """ + Actually convert the given source header file into its SIP equivalent. + + :param h_file: The source (header) file of interest. + :param include_filename: The (header) to generate in the sip file. + """ + + # + # Read in the original file. + # + source = h_file + self.unpreprocessed_source = [] + with open(source, "rU") as f: + for line in f: + self.unpreprocessed_source.append(line) + + index = cindex.Index.create() + self.tu = index.parse(source, ["-x", "c++"] + self.compile_flags) + for diag in self.tu.diagnostics: + # + # We expect to be run over hundreds of files. Any parsing issues are likely to be very repetitive. + # So, to avoid bothering the user, we suppress duplicates. + # + loc = diag.location + msg = "{}:{}[{}] {}".format(loc.file, loc.line, loc.column, diag.spelling) + if msg in self.diagnostics: + continue + self.diagnostics.add(msg) + logger.log(diag.severity, "Parse error {}".format(msg)) + if self.dump_includes: + for include in sorted(set(self.tu.get_includes())): + logger.debug(_("Used includes {}").format(include.include.name)) + # + # Run through the top level children in the translation unit. + # + body = self._container_get(self.tu.cursor, -1, h_file, include_filename) + return body, self.tu.get_includes + + CONTAINER_SKIPPABLE_UNEXPOSED_DECL = re.compile("_DECLARE_PRIVATE|friend|;") + FN_SKIPPABLE_ATTR = re.compile("_EXPORT|Q_REQUIRED_RESULT|format\(printf") + VAR_SKIPPABLE_ATTR = re.compile("_EXPORT") + TYPEDEF_SKIPPABLE_ATTR = re.compile("_EXPORT") + + def _container_get(self, container, level, h_file, include_filename): + """ + Generate the (recursive) translation for a class or namespace. + + :param container: A class or namespace. + :param h_file: Name of header file being processed. + :param level: Recursion level controls indentation. + :return: A string. + """ + + def skippable_attribute(member, text): + """ + We don't seem to have access to the __attribute__(())s, but at least we can look for stuff we care about. + + :param member: The attribute. + :param text: The raw source corresponding to the region of member. + """ + if text.find("_DEPRECATED") != -1: + sip["annotations"].add("Deprecated") + return True + SipGenerator._report_ignoring(container, member, text) + + sip = { + "name": container.displayname, + "annotations": set() + } + name = container.displayname + if container.access_specifier == AccessSpecifier.PRIVATE: + if self.dump_privates: + logger.debug("Ignoring private {}".format(SipGenerator.describe(container))) + return "" + body = "" + base_specifiers = [] + template_type_parameters = [] + had_copy_constructor = False + had_const_member = False + for member in container.get_children(): + # + # Only emit items in the translation unit. + # + if member.location.file.name != self.tu.spelling: + continue + decl = "" + if member.kind in [CursorKind.CXX_METHOD, CursorKind.FUNCTION_DECL, CursorKind.FUNCTION_TEMPLATE, + CursorKind.CONSTRUCTOR, CursorKind.DESTRUCTOR, CursorKind.CONVERSION_FUNCTION]: + decl = self._fn_get(container, member, level + 1) + elif member.kind == CursorKind.ENUM_DECL: + decl = self._enum_get(container, member, level + 1) + ";\n" + elif member.kind == CursorKind.CXX_ACCESS_SPEC_DECL: + decl = self._get_access_specifier(member, level + 1) + elif member.kind == CursorKind.TYPEDEF_DECL: + decl = self._typedef_get(member, level + 1) + elif member.kind == CursorKind.CXX_BASE_SPECIFIER: + # + # Strip off the leading "class". Except for TypeKind.UNEXPOSED... + # + base_specifiers.append(member.displayname.split(None, 2)[-1]) + elif member.kind == CursorKind.TEMPLATE_TYPE_PARAMETER: + template_type_parameters.append(member.displayname) + elif member.kind == CursorKind.TEMPLATE_NON_TYPE_PARAMETER: + template_type_parameters.append(member.type.spelling + " " + member.displayname) + elif member.kind in [CursorKind.VAR_DECL, CursorKind.FIELD_DECL]: + had_const_member = had_const_member or member.type.is_const_qualified() + decl = self._var_get(container, member, level + 1) + elif member.kind in [CursorKind.NAMESPACE, CursorKind.CLASS_DECL, + CursorKind.CLASS_TEMPLATE, CursorKind.CLASS_TEMPLATE_PARTIAL_SPECIALIZATION, + CursorKind.STRUCT_DECL, CursorKind.UNION_DECL]: + decl = self._container_get(member, level + 1, h_file, include_filename) + elif member.kind in TEMPLATE_KINDS + [CursorKind.USING_DECLARATION, CursorKind.USING_DIRECTIVE, + CursorKind.CXX_FINAL_ATTR]: + # + # Ignore: + # + # TEMPLATE_KINDS: Template type parameter. + # CursorKind.USING_DECLARATION, CursorKind.USING_DIRECTIVE: Using? Pah! + # CursorKind.CXX_FINAL_ATTR: Again, not much to be done with this. + # + pass + else: + SipGenerator._report_ignoring(container, member) + + def is_copy_constructor(member): + if member.kind != CursorKind.CONSTRUCTOR: + return False + numParams = 0 + hasSelfType = False + for child in member.get_children(): + numParams += 1 + if child.kind == CursorKind.PARM_DECL: + paramType = child.type.spelling + paramType = paramType.split("::")[-1] + paramType = paramType.replace("const", "").replace("&", "").strip() + hasSelfType = paramType == container.displayname + return numParams == 1 and hasSelfType + + def has_parameter_default(parameter): + for member in parameter.get_children(): + if member.kind.is_expression(): + return True + return False + + def is_default_constructor(member): + if member.kind != CursorKind.CONSTRUCTOR: + return False + numParams = 0 + for parameter in member.get_children(): + if (has_parameter_default(parameter)): + break + numParams += 1 + return numParams == 0 + + had_copy_constructor = had_copy_constructor or is_copy_constructor(member) + # + # Discard almost anything which is private. + # + if member.access_specifier == AccessSpecifier.PRIVATE: + if member.kind == CursorKind.CXX_ACCESS_SPEC_DECL: + # + # We need these because... + # + pass + elif is_copy_constructor(member) or is_default_constructor(member): + # + # ...we need to pass private copy contructors to the SIP compiler. + # + pass + else: + if self.dump_privates: + logger.debug("Ignoring private {}".format(SipGenerator.describe(member))) + continue + + if decl: + if self.verbose: + pad = " " * ((level + 1) * 4) + body += pad + "// {}\n".format(SipGenerator.describe(member)) + body += decl + # + # Empty containers are still useful if they provide namespaces or forward declarations. + # + if not body and level >= 0: + body = "\n" + text = self._read_source(container.extent) + if not text.endswith("}"): + # + # Forward declaration. + # + sip["annotations"].add("External") + if body and level >= 0: + if container.kind == CursorKind.NAMESPACE: + container_type = "namespace " + name + elif container.kind in [CursorKind.CLASS_DECL, CursorKind.CLASS_TEMPLATE, + CursorKind.CLASS_TEMPLATE_PARTIAL_SPECIALIZATION]: + container_type = "class " + name + elif container.kind == CursorKind.STRUCT_DECL: + container_type = "struct " + name + elif container.kind == CursorKind.UNION_DECL: + container_type = "union " + name + else: + raise AssertionError( + _("Unexpected container {}: {}[{}]").format(container.kind, name, container.extent.start.line)) + # + # Generate private copy constructor for non-copyable types. + # + if had_const_member and not had_copy_constructor: + body += " private:\n {}(const {} &); // Generated\n".format(name, container.type.get_canonical().spelling) + # + # Flesh out the SIP context for the rules engine. + # + sip["template_parameters"] = ", ".join(template_type_parameters) + sip["decl"] = container_type + sip["base_specifiers"] = ", ".join(base_specifiers) + sip["body"] = body + self.rules.container_rules().apply(container, sip) + pad = " " * (level * 4) + if sip["name"]: + decl = pad + sip["decl"] + if "External" in sip["annotations"]: + # + # SIP /External/ does not seem to work as one might wish. Suppress. + # + body = decl + " /External/;\n" + body = pad + "// Discarded {}\n".format(SipGenerator.describe(container)) + else: + if sip["base_specifiers"]: + decl += ": " + sip["base_specifiers"] + if sip["annotations"]: + decl += " /" + ",".join(sip["annotations"]) + "/" + if sip["template_parameters"]: + decl = pad + "template <" + sip["template_parameters"] + ">\n" + decl + decl += "\n" + pad + "{\n" + decl += "%TypeHeaderCode\n#include <{}>\n%End\n".format(include_filename) + body = decl + sip["body"] + pad + "};\n" + else: + body = pad + "// Discarded {}\n".format(SipGenerator.describe(container)) + return body + + def _get_access_specifier(self, member, level): + """ + Skip access specifiers embedded in the Q_OBJECT macro. + """ + access_specifier = self._read_source(member.extent) + if access_specifier == "Q_OBJECT": + return "" + pad = " " * ((level - 1) * 4) + decl = pad + access_specifier + "\n" + return decl + + def _enum_get(self, container, enum, level): + pad = " " * (level * 4) + decl = pad + "enum {} {{\n".format(enum.displayname) + enumerations = [] + for enum in enum.get_children(): + enumerations.append(pad + " {}".format(enum.displayname)) + assert enum.kind == CursorKind.ENUM_CONSTANT_DECL + decl += ",\n".join(enumerations) + "\n" + decl += pad + "}" + return decl + + def _fn_get(self, container, function, level): + """ + Generate the translation for a function. + + :param container: A class or namespace. + :param function: The function object. + :param level: Recursion level controls indentation. + :return: A string. + """ + + def skippable_attribute(member, text): + """ + We don't seem to have access to the __attribute__(())s, but at least we can look for stuff we care about. + + :param member: The attribute. + :param text: The raw source corresponding to the region of member. + """ + if SipGenerator.FN_SKIPPABLE_ATTR.search(text): + return True + SipGenerator._report_ignoring(function, member, text) + + sip = { + "name": function.spelling, + } + parameters = [] + template_parameters = [] + for child in function.get_children(): + if child.kind == CursorKind.PARM_DECL: + parameter = child.displayname or "__{}".format(len(parameters)) + theType = child.type.get_canonical() + typeSpelling = theType.spelling + if theType.kind == TypeKind.POINTER: + typeSpelling = theType.get_pointee().spelling + "* " + + decl = "{} {}".format(typeSpelling, parameter) + child_sip = { + "name": parameter, + "decl": decl, + "init": self._fn_get_parameter_default(function, child), + "annotations": set() + } + self.rules.parameter_rules().apply(container, function, child, child_sip) + decl = child_sip["decl"] + if child_sip["annotations"]: + decl += " /" + ",".join(child_sip["annotations"]) + "/" + if child_sip["init"]: + decl += " = " + child_sip["init"] + parameters.append(decl) + elif child.kind in [CursorKind.COMPOUND_STMT, CursorKind.CXX_OVERRIDE_ATTR, + CursorKind.MEMBER_REF, CursorKind.DECL_REF_EXPR, CursorKind.CALL_EXPR] + TEMPLATE_KINDS: + # + # Ignore: + # + # CursorKind.COMPOUND_STMT: Function body. + # CursorKind.CXX_OVERRIDE_ATTR: The "override" keyword. + # CursorKind.MEMBER_REF, CursorKind.DECL_REF_EXPR, CursorKind.CALL_EXPR: Constructor initialisers. + # TEMPLATE_KINDS: The result type. + # + pass + elif child.kind == CursorKind.TEMPLATE_TYPE_PARAMETER: + template_parameters.append(child.displayname) + elif child.kind == CursorKind.TEMPLATE_NON_TYPE_PARAMETER: + template_parameters.append(child.type.spelling + " " + child.displayname) + else: + text = self._read_source(child.extent) + if child.kind in [CursorKind.UNEXPOSED_ATTR, CursorKind.VISIBILITY_ATTR] and skippable_attribute(child, + text): + pass + else: + SipGenerator._report_ignoring(function, child) + # + # Flesh out the SIP context for the rules engine. + # + sip["template_parameters"] = template_parameters + if function.kind in [CursorKind.CONSTRUCTOR, CursorKind.DESTRUCTOR]: + sip["fn_result"] = "" + else: + sip["fn_result"] = function.result_type.spelling + sip["parameters"] = parameters + sip["prefix"], sip["suffix"] = self._fn_get_decorators(function) + self.rules.function_rules().apply(container, function, sip) + pad = " " * (level * 4) + if sip["name"]: + sip["template_parameters"] = ", ".join(sip["template_parameters"]) + decl = sip["name"] + "(" + ", ".join(sip["parameters"]) + ")" + if sip["fn_result"]: + decl = sip["fn_result"] + " " + decl + decl = pad + sip["prefix"] + decl + sip["suffix"] + if sip["template_parameters"]: + decl = pad + "template <" + sip["template_parameters"] + ">\n" + decl + decl += ";\n" + else: + decl = pad + "// Discarded {}\n".format(SipGenerator.describe(function)) + return decl + + def _fn_get_decorators(self, function): + """ + The parser does not provide direct access to the complete keywords (explicit, const, static, etc) of a function + in the displayname. It would be nice to get these from the AST, but I cannot find where they are hiding. + + Now, we could resort to using the original source. That does not bode well if you have macros (QOBJECT, + xxxDEPRECATED?), inlined bodies and the like, using the rule engine could be used to patch corner cases... + + ...or we can try to guess what SIP cares about, i.e static and maybe const. Luckily (?), we have those to hand! + + :param function: The function object. + :return: prefix, suffix String containing any prefix or suffix keywords. + """ + suffix = "" + if function.is_const_method(): + suffix += " const" + prefix = "" + if function.is_static_method(): + prefix += "static " + if function.is_virtual_method(): + prefix += "virtual " + if function.is_pure_virtual_method(): + suffix += " = 0" + return prefix, suffix + + def _fn_get_parameter_default(self, function, parameter): + """ + The parser does not seem to provide access to the complete text of a parameter. + This makes it hard to find any default values, so we: + + 1. Run the lexer from "here" to the end of the file, bailing out when we see the "," + or a ")" marking the end. + 2. Watch for the assignment. + """ + def _get_param_type(parameter): + result = parameter.type.get_declaration().type + + if (parameter.type.get_declaration().type.kind == TypeKind.TYPEDEF): + isQFlags = False + for member in parameter.type.get_declaration().get_children(): + if member.kind == CursorKind.TEMPLATE_REF and member.spelling == "QFlags": + isQFlags = True + if isQFlags and member.kind == CursorKind.TYPE_REF: + result = member.type + break + + return result + + def _get_param_value(text, parameterType): + if text == "0": + return text + if not "::" in parameterType.spelling: + return text + try: + typeText, typeInit = text.split("(") + typeInit = "(" + typeInit + except: + typeText = text + typeInit = "" + + prefix = parameterType.spelling.rsplit("::", 1)[0] + if "::" in typeText: + typeText = typeText.rsplit("::", 1)[1] + return prefix + "::" + typeText + typeInit + + + for member in parameter.get_children(): + if member.kind.is_expression(): + + possible_extent = SourceRange.from_locations(parameter.extent.start, function.extent.end) + text = "" + bracket_level = 0 + found_start = False + found_end = False + for token in self.tu.get_tokens(extent=possible_extent): + if (token.spelling == "="): + found_start = True + continue + if token.spelling == "," and bracket_level == 0: + found_end = True + break + elif token.spelling == "(": + bracket_level += 1 + text += token.spelling + elif token.spelling == ")": + if bracket_level == 0: + found_end = True + break + bracket_level -= 1 + text += token.spelling + if bracket_level == 0: + found_end = True + break + elif found_start: + text += token.spelling + if not found_end and text: + RuntimeError(_("No end found for {}::{}, '{}'").format(function.spelling, parameter.spelling, text)) + + parameterType = _get_param_type(parameter) + + return _get_param_value(text, parameterType) + return "" + + def _typedef_get(self, typedef, level): + """ + Generate the translation for a typedef. + + :param container: A class or namespace. + :param typedef: The typedef object. + :param level: Recursion level controls indentation. + :return: A string. + """ + + pad = " " * (level * 4) + + decl = pad + "typedef {} {}".format(typedef.underlying_typedef_type.spelling, typedef.displayname) + decl += ";\n" + return decl + + def _var_get(self, container, variable, level): + """ + Generate the translation for a variable. + + :param container: A class or namespace. + :param variable: The variable object. + :param level: Recursion level controls indentation. + :return: A string. + """ + + def skippable_attribute(member, text): + """ + We don't seem to have access to the __attribute__(())s, but at least we can look for stuff we care about. + + :param member: The attribute. + :param text: The raw source corresponding to the region of member. + """ + if SipGenerator.VAR_SKIPPABLE_ATTR.search(text): + return True + SipGenerator._report_ignoring(container, member, text) + + sip = { + "name": variable.spelling + } + for child in variable.get_children(): + if child.kind in TEMPLATE_KINDS + [CursorKind.STRUCT_DECL, CursorKind.UNION_DECL]: + # + # Ignore: + # + # TEMPLATE_KINDS, CursorKind.STRUCT_DECL, CursorKind.UNION_DECL: : The variable type. + # + pass + else: + text = self._read_source(child.extent) + if child.kind == CursorKind.VISIBILITY_ATTR and skippable_attribute(child, text): + pass + else: + SipGenerator._report_ignoring(variable, child) + # + # Flesh out the SIP context for the rules engine. + # + decl = "{} {}".format(variable.type.spelling, variable.spelling) + sip["decl"] = decl + self.rules.variable_rules().apply(container, variable, sip) + + pad = " " * (level * 4) + if sip["name"]: + decl = sip["decl"] + # + # SIP does not support protected variables, so we ignore them. + # + if variable.access_specifier == AccessSpecifier.PROTECTED: + decl = pad + "// Discarded {}\n".format(SipGenerator.describe(variable)) + else: + decl = pad + decl + ";\n" + else: + decl = pad + "// Discarded {}\n".format(SipGenerator.describe(variable)) + return decl + + def _read_source(self, extent): + """ + Read the given range from the unpre-processed source. + + :param extent: The range of text required. + """ + extract = self.unpreprocessed_source[extent.start.line - 1:extent.end.line] + if extent.start.line == extent.end.line: + extract[0] = extract[0][extent.start.column - 1:extent.end.column - 1] + else: + extract[0] = extract[0][extent.start.column - 1:] + extract[-1] = extract[-1][:extent.end.column - 1] + # + # Return a single line of text. + # + return "".join(extract).replace("\n", " ") + + @staticmethod + def _report_ignoring(parent, child, text=None): + if not text: + text = child.displayname or child.spelling + logger.debug(_("Ignoring {} {} child {}").format(parent.kind.name, parent.spelling, SipGenerator.describe(child, text))) + + +def main(argv=None): + """ + Take a single C++ header file and generate the corresponding SIP file. + Beyond simple generation of the SIP file from the corresponding C++ + header file, a set of rules can be used to customise the generated + SIP file. + + Examples: + + sip_generator.py /usr/include/KF5/KItemModels/kselectionproxymodel.h + """ + if argv is None: + argv = sys.argv + parser = argparse.ArgumentParser(epilog=inspect.getdoc(main), + formatter_class=HelpFormatter) + parser.add_argument("-v", "--verbose", action="store_true", default=False, help=_("Enable verbose output")) + parser.add_argument("--flags", + help=_("Semicolon-separated C++ compile flags to use")) + parser.add_argument("--include_filename", help=_("C++ header include to compile")) + parser.add_argument("libclang", help=_("libclang library to use for parsing")) + parser.add_argument("project_rules", help=_("Project rules")) + parser.add_argument("source", help=_("C++ header to process")) + parser.add_argument("output", help=_("output filename to write")) + try: + args = parser.parse_args(argv[1:]) + if args.verbose: + logging.basicConfig(level=logging.DEBUG, format='%(asctime)s %(name)s %(levelname)s: %(message)s') + else: + logging.basicConfig(level=logging.INFO, format='%(levelname)s: %(message)s') + # + # Generate! + # + + cindex.Config.set_library_file(args.libclang) + + rules = rules_engine.rules(args.project_rules) + g = SipGenerator(rules, args.flags.lstrip().split(";"), args.verbose) + body, includes = g.create_sip(args.source, args.include_filename) + with open(args.output, "w") as outputFile: + outputFile.write(body) + except Exception as e: + tbk = traceback.format_exc() + print(tbk) + return -1 + + +if __name__ == "__main__": + if sys.argv[1] != "--self-check": + sys.exit(main()) + else: + cindex.Config.set_library_file(sys.argv[2]) |