From 961c15a3a4319808d3acc9723e36542cf730c04f Mon Sep 17 00:00:00 2001 From: Bruno Santos Date: Fri, 29 Nov 2024 00:22:12 +0000 Subject: [PATCH 1/6] util: make compiler helper language aware Include paths differ between C and C++ domains (typically C++ has additional search directories), so we need to take that into account. Furthermore, it makes sense to specify `-nostdinc` if we are including the standard include directories explicitly already. Not only we prevent potential duplication of search paths, we ensure that the resulting search is strictly confined to the paths the given compiler would search and not the disjunction of those and libclang's own default search paths. Since this is going to be called from within the directives now, it also makes sense to improve error handling, which we do. --- src/hawkmoth/util/compiler.py | 39 ++++++++++++++++++++++++++--------- 1 file changed, 29 insertions(+), 10 deletions(-) diff --git a/src/hawkmoth/util/compiler.py b/src/hawkmoth/util/compiler.py index ea35370f..bc17b762 100644 --- a/src/hawkmoth/util/compiler.py +++ b/src/hawkmoth/util/compiler.py @@ -9,6 +9,9 @@ """ import subprocess +from sphinx.util import logging + +logger = logging.getLogger(__name__) def _removesuffix(s, suffix): if suffix and s.endswith(suffix): @@ -32,22 +35,38 @@ def _get_paths_from_output(output): yield line.strip() -def _get_include_paths(cc_path): - result = subprocess.run([cc_path, '-E', '-Wp,-v', '-'], - stdin=subprocess.DEVNULL, - capture_output=True, - check=True, - text=True) +def _get_include_paths(cpath, lang): + try: + result = subprocess.run([cpath, '-x', lang, '-E', '-Wp,-v', '-'], + stdin=subprocess.DEVNULL, + capture_output=True, + check=True, + text=True) + except FileNotFoundError: + logger.warning(f"get_include_args: compiler not found ('{cpath}')") + return [] + + except subprocess.CalledProcessError: + logger.warning(f"get_include_args: incompatible compiler ('{cpath}')") + return [] + + if result.returncode != 0: + logger.warning(f"get_include_args: incompatible compiler ('{cpath}')") + return [] return _get_paths_from_output(result.stderr) -def get_include_args(cc_path='clang'): - return [f'-I{path}' for path in _get_include_paths(cc_path)] +def get_include_args(cpath='clang', lang='c', cc_path=None): + if cc_path is not None: + cpath = cc_path + logger.warning('get_include_args: `cc_path` argument has been deprecated; use `cpath` instead') # noqa: E501 + + return ['-nostdinc'] + [f'-I{path}' for path in _get_include_paths(cpath, lang)] if __name__ == '__main__': import pprint import sys - cc_path = sys.argv[1] if len(sys.argv) > 1 else 'clang' + compiler = sys.argv[1] if len(sys.argv) > 1 else 'clang' - pprint.pprint(get_include_args(cc_path)) + pprint.pprint(get_include_args(compiler)) From d979c64c9710dc7a404fce18cb4263686a5b26d7 Mon Sep 17 00:00:00 2001 From: Bruno Santos Date: Fri, 29 Nov 2024 00:11:00 +0000 Subject: [PATCH 2/6] cfg: add compiler and autoconf options It's a common problem to want hawkmoth to parse the sources with the exact headers a given code is meant to be compiled with and doing so requires some verbosity upfront which can easily be avoided. Provide a couple of options to do just that. The defaults are chosen in a technically backwards incompatible way, but it would take a frankly adversarial case to break it. In fact the new defaults should yield the default behaviour everyone expected from the beginning: we all expected libclang's search paths to be an exact match to clang's, and indeed they are in some distributions. Finally, the new search paths are added after other user's options since the headers are searched in a left to right manner, giving the user the possibility of overriding specific headers despite using this option. For maximum flexibility, this functionality is gated on the 'autoconf' settings, which can easily be extended later to support additional features. --- doc/extension.rst | 42 +++++++++++++++++++++++++++------------- src/hawkmoth/__init__.py | 31 +++++++++++++++++++++++++++-- 2 files changed, 58 insertions(+), 15 deletions(-) diff --git a/doc/extension.rst b/doc/extension.rst index c2df8aac..0a58ba1b 100644 --- a/doc/extension.rst +++ b/doc/extension.rst @@ -56,10 +56,38 @@ See also additional configuration options in the :ref:`built-in extensions :type: str The default transform parameter to be passed to the - :event:`hawkmoth-process-docstring` event. It can be overriden with the + :event:`hawkmoth-process-docstring` event. It can be overridden with the ``transform`` option of the :ref:`directives `. Defaults to ``None``. +.. py:data:: hawkmoth_compiler + :type: str + + The (path to the) default compiler used by the project. This is used to + determine the exact options needed to parse the code files by libclang + provided the relevant options are enabled in :data:`hawkmoth_autoconf`. + + Notably, it allows hawkmoth to override libclang's default search path for + system headers with those of the specified compiler. + + This presumes the compiler supports being called as + `` -x -E -Wp,-v /dev/null``. + + Defaults to ``clang``, which may differ from libclang's own default includes. + It will use libclang's defaults if set to ``None`` though. + + This is a shortcut to specify ``-nostdinc -I ... -I`` in + :data:`hawkmoth_clang` with the search directories of the specified compiler. + +.. py:data:: hawkmoth_autoconf + :type: list + + List of options that control the automatic configuration features of + hawkmoth. Currently supported options: + + * ``'stdinc'``: override the standard include paths of libclang with those of + the specified compiler (see :data:`hawkmoth_compiler`). + .. py:data:: hawkmoth_clang :type: list @@ -73,18 +101,6 @@ See also additional configuration options in the :ref:`built-in extensions hawkmoth_clang = ['-I/path/to/include', '-DHAWKMOTH'] - Hawkmoth provides a convenience helper for querying the include path from the - compiler, and providing them as ``-I`` options: - - .. code-block:: python - - from hawkmoth.util import compiler - - hawkmoth_clang = compiler.get_include_args() - - You can also pass in the compiler to use, for example - ``get_include_args('gcc')``. - .. py:data:: hawkmoth_clang_c :type: list diff --git a/src/hawkmoth/__init__.py b/src/hawkmoth/__init__.py index e6e2aac7..702fe244 100644 --- a/src/hawkmoth/__init__.py +++ b/src/hawkmoth/__init__.py @@ -19,6 +19,7 @@ from sphinx.util.docutils import switch_source_input, SphinxDirective from sphinx.util import logging +from hawkmoth.util import compiler from hawkmoth.parser import parse, ErrorLevel from hawkmoth.util import strutil from hawkmoth import docstring @@ -39,6 +40,31 @@ class _AutoBaseDirective(SphinxDirective): _domain: Optional[str] = None _docstring_types: Optional[list[type[docstring.Docstring]]] = None + def __init__(self, name, arguments, options, + content, lineno, content_offset, + block_text, state, state_machine): + + super().__init__(name, arguments, options, + content, lineno, content_offset, + block_text, state, state_machine) + + cpath = self.env.config.hawkmoth_compiler + autoconf = self.env.config.hawkmoth_autoconf + + ignored_options = [x for x in autoconf if x not in ['stdinc']] + if len(ignored_options) > 0: + self.logger.warning(f'autoconf: {ignored_options} unsupported option(s) ignored') + + self._clang_args_post = [] + if 'stdinc' in autoconf: + if cpath: + if self._domain == 'c': + self._clang_args_post = compiler.get_include_args(cpath, 'c') + else: + self._clang_args_post = compiler.get_include_args(cpath, 'c++') + else: + self.logger.warning('autoconf: \'stdinc\' option ignored (missing compiler)') + def __display_parser_diagnostics(self, errors): # Map parser diagnostic level to Sphinx level name log_level_map = { @@ -56,8 +82,6 @@ def __display_parser_diagnostics(self, errors): def __get_clang_args(self): clang_args = [] - clang_args.extend(self.env.config.hawkmoth_clang.copy()) - if self._domain == 'c': clang_args.extend(self.env.config.hawkmoth_clang_c.copy()) else: @@ -65,6 +89,7 @@ def __get_clang_args(self): clang_args.extend(self.options.get('clang', [])) + clang_args.extend(self._clang_args_post) return clang_args def __parse(self, filename): @@ -358,6 +383,8 @@ def setup(app): app.require_sphinx('3.0') app.add_config_value('hawkmoth_root', app.confdir, 'env', [str]) + app.add_config_value('hawkmoth_compiler', 'clang', 'env', [str, type(None)]) + app.add_config_value('hawkmoth_autoconf', ['stdinc'], 'env', [list]) app.add_config_value('hawkmoth_clang', [], 'env', [list]) app.add_config_value('hawkmoth_clang_c', [], 'env', [list]) app.add_config_value('hawkmoth_clang_cpp', [], 'env', [list]) From 66748da014bf8e7562e7f6f4a089aef5af970c1d Mon Sep 17 00:00:00 2001 From: Bruno Santos Date: Fri, 29 Nov 2024 00:28:32 +0000 Subject: [PATCH 3/6] test: fix header search paths in tests This reverts the effects of commit 1243852, while fixing the inconsistent handling of C/C++ domains since libclang can't always figure out the right search paths for its own version of Clang despite prior claims. It is still unclear whether the upstream issue is an actual issue or if this is the intended behaviour (https://github.com/llvm/llvm-project/issues/18150), though it _is_ an issue and simply using the fixed helper function and configuration parameter comes at a suitably low cost to maintenance. Fixes #262. --- test/testenv.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/test/testenv.py b/test/testenv.py index 77f6e88a..ec7404c4 100644 --- a/test/testenv.py +++ b/test/testenv.py @@ -7,6 +7,7 @@ import pytest import strictyaml +from hawkmoth.util import compiler from test import conf testext = '.yaml' @@ -49,8 +50,10 @@ def get_clang_args(self): if self.domain == 'c': clang_args.extend(getattr(conf, 'hawkmoth_clang_c', [])) + clang_args.extend(compiler.get_include_args('clang', 'c')) else: clang_args.extend(getattr(conf, 'hawkmoth_clang_cpp', [])) + clang_args.extend(compiler.get_include_args('clang', 'c++')) clang_args.extend(self.options.get('clang', [])) From d27cc9685e6fd5f9579ef8c04002541626395a04 Mon Sep 17 00:00:00 2001 From: Bruno Santos Date: Fri, 29 Nov 2024 01:33:45 +0000 Subject: [PATCH 4/6] ci: examples: install dependencies --- .github/workflows/makefile.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/makefile.yml b/.github/workflows/makefile.yml index 91ea9f64..94be632f 100644 --- a/.github/workflows/makefile.yml +++ b/.github/workflows/makefile.yml @@ -46,6 +46,8 @@ jobs: steps: - uses: actions/checkout@v4 - uses: actions/setup-python@v5 + - name: Install dependencies (apt) + run: sudo apt-get install -y python3-clang python3-pip - name: Examples check run: | . venv From b6c57dea454273f261236b031b59017ea672c004 Mon Sep 17 00:00:00 2001 From: Bruno Santos Date: Sat, 28 Dec 2024 23:41:37 +0000 Subject: [PATCH 5/6] test: optionally select active tests for a test case Allow each test case to select which of all the supported tests are active. By default, all of them are, but sometimes it's impossible to create tests that will pass as they target something specific to those tests. The motivation here is the expanding set of configuration variables, which have not been tested so far, but which only affect extension tests. Creating a test case for a breaking configuration would then not work since the parser / cli tests would not see the same breaking configuration. Technically, it's unlikely a test case will ever need to be _just_ for cli or parser, but not extension: anything that breaks those would break the extension ones. But we do it in a general way anyway for simplicity. --- test/test_cli.py | 8 +++++++- test/test_extension.py | 10 ++++++++-- test/test_parser.py | 7 ++++++- test/testenv.py | 9 +++++++++ 4 files changed, 30 insertions(+), 4 deletions(-) diff --git a/test/test_cli.py b/test/test_cli.py index 554559d3..ef83719f 100644 --- a/test/test_cli.py +++ b/test/test_cli.py @@ -32,6 +32,10 @@ def _stderr_basename(errors_str): class CliTestcase(testenv.Testcase): + + def valid(self): + return 'cli' in self.options.get('test', ['cli']) + def set_monkeypatch(self, monkeypatch): self.monkeypatch = monkeypatch self.mock_args([]) @@ -88,7 +92,9 @@ def get_expected(self): def _get_cli_testcases(path): for f in testenv.get_testcase_filenames(path): - yield CliTestcase(f) + testcase = CliTestcase(f) + if testcase.valid(): + yield testcase @pytest.mark.full @pytest.mark.parametrize('testcase', _get_cli_testcases(testenv.testdir), diff --git a/test/test_extension.py b/test/test_extension.py index 4ee74dc6..121c6377 100755 --- a/test/test_extension.py +++ b/test/test_extension.py @@ -20,6 +20,9 @@ def __init__(self, filename, buildername): super().__init__(filename) self._buildername = buildername + def valid(self): + return 'extension' in self.options.get('test', ['extension']) + def _get_suffix(self): return 'txt' if self._buildername == 'text' else self._buildername @@ -27,6 +30,7 @@ def _sphinx_build(self, srcdir): outdir = os.path.join(srcdir, self._buildername) doctreedir = os.path.join(srcdir, 'doctrees') confdir = testenv.testdir + confoverrides = self.get_conf_overrides() # Don't emit color codes in Sphinx status/warning output console.nocolor() @@ -36,7 +40,7 @@ def _sphinx_build(self, srcdir): with patch_docutils(confdir), docutils_namespace(): app = Sphinx(srcdir=srcdir, confdir=confdir, outdir=outdir, doctreedir=doctreedir, buildername=self._buildername, - warning=warning) + confoverrides=confoverrides, warning=warning) # Ensure there are no errors with app creation. assert warning.getvalue() == '' @@ -83,7 +87,9 @@ def get_expected(self): def _get_extension_testcases(path, buildername): for f in testenv.get_testcase_filenames(path): - yield ExtensionTestcase(f, buildername) + testcase = ExtensionTestcase(f, buildername) + if testcase.valid(): + yield testcase # Test using Sphinx plain text builder @pytest.mark.parametrize('testcase', _get_extension_testcases(testenv.testdir, 'text'), diff --git a/test/test_parser.py b/test/test_parser.py index 43bc5003..af80e262 100755 --- a/test/test_parser.py +++ b/test/test_parser.py @@ -58,6 +58,9 @@ def _filter_members(directive): return members class ParserTestcase(testenv.Testcase): + def valid(self): + return 'parser' in self.options.get('test', ['parser']) + def get_output(self): roots = {} docs_str = '' @@ -114,7 +117,9 @@ def get_expected(self): def _get_parser_testcases(path): for f in testenv.get_testcase_filenames(path): - yield ParserTestcase(f) + testcase = ParserTestcase(f) + if testcase.valid(): + yield testcase @pytest.mark.parametrize('testcase', _get_parser_testcases(testenv.testdir), ids=testenv.get_testid) diff --git a/test/testenv.py b/test/testenv.py index ec7404c4..868bf31d 100644 --- a/test/testenv.py +++ b/test/testenv.py @@ -61,6 +61,7 @@ def get_clang_args(self): class Testcase: _options_schema = strictyaml.Map({ + strictyaml.Optional('test'): strictyaml.Seq(strictyaml.Str()), 'directives': strictyaml.Seq(strictyaml.Map({ 'domain': strictyaml.Enum(['c', 'cpp']), 'directive': strictyaml.Str(), @@ -73,6 +74,9 @@ class Testcase: strictyaml.Optional('transform'): strictyaml.Str(), }), })), + strictyaml.Optional('conf-overrides'): strictyaml.MapPattern( + strictyaml.Str(), strictyaml.NullNone() | strictyaml.EmptyList() | strictyaml.Any(), + ), strictyaml.Optional('expected-failure'): strictyaml.Bool(), strictyaml.Optional('example-use-namespace'): strictyaml.Bool(), strictyaml.Optional('example-title'): strictyaml.Str(), @@ -85,6 +89,8 @@ def __init__(self, filename): self.filename = filename with open(filename) as f: self.options = strictyaml.load(f.read(), self._options_schema).data + if self.options.get('test', None) is None: + self.options['test'] = ['cli', 'parser', 'extension'] self.testid = os.path.splitext(os.path.relpath(self.filename, testdir))[0] self.directives = [Directive(self, directive_config) for @@ -105,6 +111,9 @@ def get_expected_filename(self): def get_stderr_filename(self): return self.get_relative_filename(self.options.get('errors')) + def get_conf_overrides(self): + return self.options.get('conf-overrides', {}) + def run_test(self): if self.options.get('expected-failure'): pytest.xfail() From ad093b8cd574bd3b4159fd3156d047b89e30c8df Mon Sep 17 00:00:00 2001 From: Bruno Santos Date: Mon, 30 Dec 2024 12:21:31 +0000 Subject: [PATCH 6/6] test: add compiler/autoconf option tests --- test/c/autoconf-invalid.stderr | 1 + test/c/autoconf-invalid.yaml | 12 +++++++++ test/c/autoconf.yaml | 13 ++++++++++ test/c/compiler-autoconf-mismatch.stderr | 1 + test/c/compiler-autoconf-mismatch.yaml | 13 ++++++++++ test/c/compiler-not-found.stderr | 1 + test/c/compiler-not-found.yaml | 11 ++++++++ test/c/compiler-unknown.stderr | 1 + test/c/compiler-unknown.yaml | 11 ++++++++ test/c/no-autoconf.rst | 32 ++++++++++++++++++++++++ test/c/no-autoconf.stderr | 1 + test/c/no-autoconf.yaml | 12 +++++++++ 12 files changed, 109 insertions(+) create mode 100644 test/c/autoconf-invalid.stderr create mode 100644 test/c/autoconf-invalid.yaml create mode 100644 test/c/autoconf.yaml create mode 100644 test/c/compiler-autoconf-mismatch.stderr create mode 100644 test/c/compiler-autoconf-mismatch.yaml create mode 100644 test/c/compiler-not-found.stderr create mode 100644 test/c/compiler-not-found.yaml create mode 100644 test/c/compiler-unknown.stderr create mode 100644 test/c/compiler-unknown.yaml create mode 100644 test/c/no-autoconf.rst create mode 100644 test/c/no-autoconf.stderr create mode 100644 test/c/no-autoconf.yaml diff --git a/test/c/autoconf-invalid.stderr b/test/c/autoconf-invalid.stderr new file mode 100644 index 00000000..2a565ee7 --- /dev/null +++ b/test/c/autoconf-invalid.stderr @@ -0,0 +1 @@ +WARNING: autoconf: ['invalid'] unsupported option(s) ignored diff --git a/test/c/autoconf-invalid.yaml b/test/c/autoconf-invalid.yaml new file mode 100644 index 00000000..6d2303fe --- /dev/null +++ b/test/c/autoconf-invalid.yaml @@ -0,0 +1,12 @@ +test: +- extension +directives: +- domain: c + directive: autodoc + arguments: + - doc.c +conf-overrides: + hawkmoth_autoconf: + - 'invalid' +errors: autoconf-invalid.stderr +expected: doc.rst diff --git a/test/c/autoconf.yaml b/test/c/autoconf.yaml new file mode 100644 index 00000000..5e3194be --- /dev/null +++ b/test/c/autoconf.yaml @@ -0,0 +1,13 @@ +test: +- extension +directives: +- domain: c + directive: autodoc + arguments: + - bool.c +conf-overrides: + hawkmoth_clang: -nostdinc + hawkmoth_compiler: clang + hawkmoth_autoconf: + - 'stdinc' +expected: bool.rst diff --git a/test/c/compiler-autoconf-mismatch.stderr b/test/c/compiler-autoconf-mismatch.stderr new file mode 100644 index 00000000..3e3bc1b0 --- /dev/null +++ b/test/c/compiler-autoconf-mismatch.stderr @@ -0,0 +1 @@ +WARNING: autoconf: 'stdinc' option ignored (missing compiler) diff --git a/test/c/compiler-autoconf-mismatch.yaml b/test/c/compiler-autoconf-mismatch.yaml new file mode 100644 index 00000000..87b63d4a --- /dev/null +++ b/test/c/compiler-autoconf-mismatch.yaml @@ -0,0 +1,13 @@ +test: +- extension +directives: +- domain: c + directive: autodoc + arguments: + - doc.c +conf-overrides: + hawkmoth_compiler: null + hawkmoth_autoconf: + - 'stdinc' +errors: compiler-autoconf-mismatch.stderr +expected: doc.rst diff --git a/test/c/compiler-not-found.stderr b/test/c/compiler-not-found.stderr new file mode 100644 index 00000000..0eaab939 --- /dev/null +++ b/test/c/compiler-not-found.stderr @@ -0,0 +1 @@ +WARNING: get_include_args: compiler not found ('invalid') diff --git a/test/c/compiler-not-found.yaml b/test/c/compiler-not-found.yaml new file mode 100644 index 00000000..00ed457e --- /dev/null +++ b/test/c/compiler-not-found.yaml @@ -0,0 +1,11 @@ +test: +- extension +directives: +- domain: c + directive: autodoc + arguments: + - doc.c +conf-overrides: + hawkmoth_compiler: invalid +errors: compiler-not-found.stderr +expected: doc.rst diff --git a/test/c/compiler-unknown.stderr b/test/c/compiler-unknown.stderr new file mode 100644 index 00000000..1e51666a --- /dev/null +++ b/test/c/compiler-unknown.stderr @@ -0,0 +1 @@ +WARNING: get_include_args: incompatible compiler ('false') diff --git a/test/c/compiler-unknown.yaml b/test/c/compiler-unknown.yaml new file mode 100644 index 00000000..f86ee65f --- /dev/null +++ b/test/c/compiler-unknown.yaml @@ -0,0 +1,11 @@ +test: +- extension +directives: +- domain: c + directive: autodoc + arguments: + - doc.c +conf-overrides: + hawkmoth_compiler: false +errors: compiler-unknown.stderr +expected: doc.rst diff --git a/test/c/no-autoconf.rst b/test/c/no-autoconf.rst new file mode 100644 index 00000000..ec0aefc7 --- /dev/null +++ b/test/c/no-autoconf.rst @@ -0,0 +1,32 @@ + +.. c:var:: static int convert_bool + + Retain bool instead of using _Bool. + + +.. c:var:: static bool convert_Bool + + Also convert _Bool to bool. + + +.. c:function:: int boolean(int bar, bool baz) + + Bool function. + + +.. c:struct:: sample_struct + + This is a sample struct + + Woohoo. + + + .. c:member:: int bool_member + + bool member + + + .. c:member:: bool _Bool_member + + _Bool member + diff --git a/test/c/no-autoconf.stderr b/test/c/no-autoconf.stderr new file mode 100644 index 00000000..bd295082 --- /dev/null +++ b/test/c/no-autoconf.stderr @@ -0,0 +1 @@ +ERROR: bool.c:1: 'stdbool.h' file not found diff --git a/test/c/no-autoconf.yaml b/test/c/no-autoconf.yaml new file mode 100644 index 00000000..e5c26343 --- /dev/null +++ b/test/c/no-autoconf.yaml @@ -0,0 +1,12 @@ +test: +- extension +directives: +- domain: c + directive: autodoc + arguments: + - bool.c +conf-overrides: + hawkmoth_clang: -nostdinc + hawkmoth_autoconf: +errors: no-autoconf.stderr +expected: no-autoconf.rst