From: Vladimír Vondruš Date: Fri, 12 Jul 2019 20:18:13 +0000 (+0200) Subject: documentation/python: robust linking also for nested/generic types. X-Git-Url: https://www.chiark.greenend.org.uk/ucgi/~cjwatson/git?a=commitdiff_plain;h=6d8d55d5952fb8dce5d78660a022c35e1fc056d6;p=blog.git documentation/python: robust linking also for nested/generic types. Wow of course this again took me ten times more than originally anticipated, because there are NO FREAKING DOCS AT ALL for the typing module. The major advantage of the new (hard to implement) approach is that string annotations are parsed and dereferenced correctly as well. That also means the previous way I did them (fully qualified names) was mostly wrong. --- diff --git a/documentation/python.py b/documentation/python.py index f4382583..75815e39 100755 --- a/documentation/python.py +++ b/documentation/python.py @@ -39,6 +39,7 @@ import os import re import sys import shutil +import typing from enum import Enum from types import SimpleNamespace as Empty @@ -665,27 +666,47 @@ def extract_summary(state: State, external_docs, path: List[str], doc: str) -> s def extract_type(type) -> str: # For types we concatenate the type name with its module unless it's - # builtins (i.e., we want re.Match but not builtins.int). - return (type.__module__ + '.' if type.__module__ != 'builtins' else '') + type.__name__ + # builtins (i.e., we want re.Match but not builtins.int). We need to use + # __qualname__ instead of __name__ because __name__ doesn't take nested + # classes into account. + return (type.__module__ + '.' if type.__module__ != 'builtins' else '') + type.__qualname__ + +def get_type_hints_or_nothing(path: List[str], object): + try: + return typing.get_type_hints(object) + except Exception as e: + # Gracefully handle an invalid name or a missing attribute, give up on + # everything else (syntax error and so) + if not isinstance(e, (AttributeError, NameError)): raise e + logging.warning("failed to dereference type hints for %s (%s), falling back to non-dereferenced", '.'.join(path), e.__class__.__name__) + return {} def extract_annotation(state: State, referrer_path: List[str], annotation) -> str: # TODO: why this is not None directly? if annotation is inspect.Signature.empty: return None - # Annotations can be strings, also https://stackoverflow.com/a/33533514 - if type(annotation) == str: out = annotation - - # To avoid getting for types (and getting foo.bar - # instead) but getting the actual type for types annotated with e.g. - # List[int], we need to check if the annotation is actually from the - # typing module or it's directly a type. In Python 3.7 this worked with - # inspect.isclass(annotation), but on 3.6 that gives True for annotations - # as well and then we would get just List instead of List[int]. - elif annotation.__module__ == 'typing': out = str(annotation) - else: out = extract_type(annotation) + # If dereferencing with typing.get_type_hints() failed, we might end up + # with forward-referenced types being plain strings. Keep them as is, since + # those are most probably an error. + if type(annotation) == str: return annotation + + # Or the plain strings might be inside (e.g. List['Foo']), which gets + # converted by Python to ForwardRef. Hammer out the actual string and again + # leave it as-is, since it's most probably an error. + elif isinstance(annotation, typing.ForwardRef): + return annotation.__forward_arg__ + + # If the annotation is from the typing module, it could be a "bracketed" + # type, in which case we want to recurse to its types as well. Otherwise + # just get its name. + elif annotation.__module__ == 'typing': + if hasattr(annotation, '__args__'): + return 'typing.{}[{}]'.format(annotation._name, ', '.join([extract_annotation(state, referrer_path, i) for i in annotation.__args__])) + else: + return 'typing.' + annotation._name - # Map name prefix, add links to the result - return make_type_link(state, referrer_path, map_name_prefix(state, out)) + # Otherwise it's a plain type. Turn it into a link. + return make_type_link(state, referrer_path, map_name_prefix(state, extract_type(annotation))) def extract_module_doc(state: State, path: List[str], module): assert inspect.ismodule(module) @@ -878,13 +899,26 @@ def extract_function_doc(state: State, parent, path: List[str], function) -> Lis out.is_classmethod = inspect.ismethod(function) out.is_staticmethod = out.name in parent.__dict__ and isinstance(parent.__dict__[out.name], staticmethod) + # First try to get fully dereferenced type hints (with strings + # converted to actual annotations). If that fails (e.g. because a type + # doesn't exist), we'll take the non-dereferenced annotations from + # inspect instead. + type_hints = get_type_hints_or_nothing(path, function) + try: signature = inspect.signature(function) - out.type = extract_annotation(state, path, signature.return_annotation) + + if 'return' in type_hints: + out.type = extract_annotation(state, path, type_hints['return']) + else: + out.type = extract_annotation(state, path, signature.return_annotation) for i in signature.parameters.values(): param = Empty() param.name = i.name - param.type = extract_annotation(state, path, i.annotation) + if i.name in type_hints: + param.type = extract_annotation(state, path, type_hints[i.name]) + else: + param.type = extract_annotation(state, path, i.annotation) if param.type: out.has_complex_params = True if i.default is inspect.Signature.empty: @@ -921,7 +955,20 @@ def extract_property_doc(state: State, path: List[str], property): try: signature = inspect.signature(property.fget) - out.type = extract_annotation(state, path, signature.return_annotation) + + # First try to get fully dereferenced type hints (with strings + # converted to actual annotations). If that fails (e.g. because a type + # doesn't exist), we'll take the non-dereferenced annotations from + # inspect instead. This is deliberately done *after* inspecting the + # signature because pybind11 properties would throw TypeError from + # typing.get_type_hints(). This way they throw ValueError from inspect + # and we don't need to handle TypeError in get_type_hints_or_nothing(). + if property.fget: type_hints = get_type_hints_or_nothing(path, property.fget) + + if 'return' in type_hints: + out.type = extract_annotation(state, path, type_hints['return']) + else: + out.type = extract_annotation(state, path, signature.return_annotation) except ValueError: # pybind11 properties have the type in the docstring if state.config['PYBIND11_COMPATIBILITY']: @@ -940,10 +987,19 @@ def extract_data_doc(state: State, parent, path: List[str], data): # Welp. https://stackoverflow.com/questions/8820276/docstring-for-variable out.summary = '' out.has_details = False - if hasattr(parent, '__annotations__') and out.name in parent.__annotations__: + + # First try to get fully dereferenced type hints (with strings converted to + # actual annotations). If that fails (e.g. because a type doesn't exist), + # we'll take the non-dereferenced annotations instead. + type_hints = get_type_hints_or_nothing(path, parent) + + if out.name in type_hints: + out.type = extract_annotation(state, path, type_hints[out.name]) + elif hasattr(parent, '__annotations__') and out.name in parent.__annotations__: out.type = extract_annotation(state, path, parent.__annotations__[out.name]) else: out.type = None + # The autogenerated is useless, so provide the value # only if __repr__ is implemented for given type if '__repr__' in type(data).__dict__: diff --git a/documentation/test_python/inspect_annotations/inspect_annotations.py b/documentation/test_python/inspect_annotations/inspect_annotations.py index 1f645227..fbf143fc 100644 --- a/documentation/test_python/inspect_annotations/inspect_annotations.py +++ b/documentation/test_python/inspect_annotations/inspect_annotations.py @@ -10,7 +10,7 @@ class Foo: # Self-reference is only possible with a string in Py3 # https://stackoverflow.com/a/33533514 - def string_annotation(self: 'inspect_annotations.Foo'): + def string_annotation(self: 'Foo'): """String annotations""" pass diff --git a/documentation/test_python/inspect_name_mapping/inspect_name_mapping/_sub/__init__.py b/documentation/test_python/inspect_name_mapping/inspect_name_mapping/_sub/__init__.py index dbb7f59d..08791868 100644 --- a/documentation/test_python/inspect_name_mapping/inspect_name_mapping/_sub/__init__.py +++ b/documentation/test_python/inspect_name_mapping/inspect_name_mapping/_sub/__init__.py @@ -1,6 +1,6 @@ class Foo: """A class""" # https://stackoverflow.com/a/33533514, have to use a string in Py3 - def a_thing(self) -> 'inspect_name_mapping._sub.Foo': + def a_thing(self) -> 'Foo': """A method""" pass diff --git a/documentation/test_python/inspect_type_links/inspect_type_links.second.Foo.html b/documentation/test_python/inspect_type_links/inspect_type_links.second.Foo.html index d57e0541..42c1277d 100644 --- a/documentation/test_python/inspect_type_links/inspect_type_links.second.Foo.html +++ b/documentation/test_python/inspect_type_links/inspect_type_links.second.Foo.html @@ -30,6 +30,7 @@ Reference @@ -41,6 +42,23 @@ type_property: Enum get
A property
+
+ type_property_string_invalid: FooBar get +
+
A property
+
+ type_property_string_nested: typing.Tuple[Foo, typing.List[Enum], typing.Any] get +
+
A property
+ + +
+

Data

+
+
+ TYPE_DATA_STRING_INVALID: Foo.Bar = 3 +
+
diff --git a/documentation/test_python/inspect_type_links/inspect_type_links.second.html b/documentation/test_python/inspect_type_links/inspect_type_links.second.html index 801fed85..5e4b3a0b 100644 --- a/documentation/test_python/inspect_type_links/inspect_type_links.second.html +++ b/documentation/test_python/inspect_type_links/inspect_type_links.second.html @@ -61,10 +61,42 @@ def type_enum(a: Enum)
Function referencing an enum
+
+ def type_nested(a: typing.Tuple[Foo, typing.List[Enum], typing.Any]) +
+
A function with nested type annotation
+
+ def type_nested_string(a: typing.Tuple[Foo, typing.List[Enum], typing.Any]) +
+
A function with nested string type annotation
+
+ def type_nested_string_invalid(a: typing.Tuple[FooBar, List[Enum], Any]) +
+
A function with invalid nested string type annotation
def type_return() -> Foo
A function with a return type annotation
+
+ def type_return_string_invalid(a: Foo) -> FooBar +
+
A function with invalid return string type annotation
+
+ def type_return_string_nested() -> typing.Tuple[Foo, typing.List[Enum], typing.Any] +
+
A function with a string nested return type
+
+ def type_string(a: Foo) +
+
A function with string type annotation
+
+ def type_string_invalid(a: Foo.Bar) +
+
A function with invalid string type annotation
+
+ def type_string_nested(a: typing.Tuple[Foo, typing.List[Enum], typing.Any]) +
+
A function with string nested type annotation
@@ -74,6 +106,10 @@ TYPE_DATA: Foo
+
+ TYPE_DATA_STRING_NESTED: typing.Tuple[Foo, typing.List[Enum], typing.Any] = {} +
+
diff --git a/documentation/test_python/inspect_type_links/inspect_type_links/first/__init__.py b/documentation/test_python/inspect_type_links/inspect_type_links/first/__init__.py index 30b9afdd..b5a52c71 100644 --- a/documentation/test_python/inspect_type_links/inspect_type_links/first/__init__.py +++ b/documentation/test_python/inspect_type_links/inspect_type_links/first/__init__.py @@ -6,10 +6,10 @@ from inspect_type_links import second class Foo: """A class in the first module""" - def reference_self(self, a: 'inspect_type_links.first.Foo'): + def reference_self(self, a: 'first.Foo'): """A method referencing its wrapper class. Due to the inner Foo this is quite a pathological case and I'm not sure if first.Foo or Foo is better.""" - def reference_inner(self, a: 'inspect_type_links.first.Foo.Foo'): + def reference_inner(self, a: 'first.Foo.Foo'): """A method referencing an inner class. This is quite a pathological case and I'm not sure if Foo or Foo.Foo is better.""" def reference_other(self, a: second.Foo): @@ -18,10 +18,10 @@ class Foo: class Foo: """An inner class in the first module""" - def reference_self(self, a: 'inspect_type_links.first.Foo.Foo'): + def reference_self(self, a: 'first.Foo.Foo'): """A method referencing its wrapper class""" - def reference_parent(self, a: 'inspect_type_links.first.Foo'): + def reference_parent(self, a: 'first.Foo'): """A method referencing its parent wrapper class""" def reference_self(a: Foo, b: first.Foo): diff --git a/documentation/test_python/inspect_type_links/inspect_type_links/first/sub.py b/documentation/test_python/inspect_type_links/inspect_type_links/first/sub.py index 77954056..52288909 100644 --- a/documentation/test_python/inspect_type_links/inspect_type_links/first/sub.py +++ b/documentation/test_python/inspect_type_links/inspect_type_links/first/sub.py @@ -5,13 +5,13 @@ from inspect_type_links import first class Foo: """A class in the submodule""" - def reference_self(a: 'inspect_type_links.first.sub.Foo'): + def reference_self(a: 'first.sub.Foo'): """A method referencing a type in this submodule""" def reference_parent(a: first.Foo, b: first.Foo): """A method referencing a type in a parent module""" -def reference_self(a: Foo, b: 'inspect_type_links.first.sub.Foo'): +def reference_self(a: Foo, b: 'first.sub.Foo'): """A function referencing a type in this submodule""" def reference_parent(a: first.Foo, b: first.Foo): diff --git a/documentation/test_python/inspect_type_links/inspect_type_links/second.py b/documentation/test_python/inspect_type_links/inspect_type_links/second.py index 3ca7908d..31db9f1f 100644 --- a/documentation/test_python/inspect_type_links/inspect_type_links/second.py +++ b/documentation/test_python/inspect_type_links/inspect_type_links/second.py @@ -1,5 +1,7 @@ """Second module""" +from typing import Tuple, List, Any + import enum class Enum(enum.Enum): @@ -18,7 +20,45 @@ class Foo: def type_property(self) -> Enum: """A property""" + @property + def type_property_string_nested(self) -> 'Tuple[Foo, List[Enum], Any]': + """A property""" + + @property + def type_property_string_invalid(self) -> 'FooBar': + """A property""" + + # Has to be here, because if it would be globally, it would prevent all + # other data annotations from being retrieved + TYPE_DATA_STRING_INVALID: 'Foo.Bar' = 3 + +def type_string(a: 'Foo'): + """A function with string type annotation""" + +def type_nested(a: Tuple[Foo, List[Enum], Any]): + """A function with nested type annotation""" + +def type_string_nested(a: 'Tuple[Foo, List[Enum], Any]'): + """A function with string nested type annotation""" + +def type_nested_string(a: Tuple['Foo', 'List[Enum]', 'Any']): + """A function with nested string type annotation""" + +def type_string_invalid(a: 'Foo.Bar'): + """A function with invalid string type annotation""" + +def type_nested_string_invalid(a: Tuple['FooBar', 'List[Enum]', 'Any']): + """A function with invalid nested string type annotation""" + def type_return() -> Foo: """A function with a return type annotation""" +def type_return_string_nested() -> 'Tuple[Foo, List[Enum], Any]': + """A function with a string nested return type""" + +def type_return_string_invalid(a: Foo) -> 'FooBar': + """A function with invalid return string type annotation""" + TYPE_DATA: Foo = Foo() + +TYPE_DATA_STRING_NESTED: 'Tuple[Foo, List[Enum], Any]' = {} diff --git a/documentation/test_python/pybind_type_links/pybind_type_links.cpp b/documentation/test_python/pybind_type_links/pybind_type_links.cpp index fb996185..54d0c432 100644 --- a/documentation/test_python/pybind_type_links/pybind_type_links.cpp +++ b/documentation/test_python/pybind_type_links/pybind_type_links.cpp @@ -1,4 +1,5 @@ #include +#include namespace py = pybind11; @@ -16,6 +17,8 @@ struct Foo { Foo typeReturn() { return {}; } +void typeNested(const std::pair>&) {} + } PYBIND11_MODULE(pybind_type_links, m) { @@ -31,5 +34,6 @@ PYBIND11_MODULE(pybind_type_links, m) { m .def("type_enum", &typeEnum, "A function taking an enum") - .def("type_return", &typeReturn, "A function returning a type"); + .def("type_return", &typeReturn, "A function returning a type") + .def("type_nested", &typeNested, "A function with nested type annotation"); } diff --git a/documentation/test_python/pybind_type_links/pybind_type_links.html b/documentation/test_python/pybind_type_links/pybind_type_links.html index 57cf5886..85e6e06e 100644 --- a/documentation/test_python/pybind_type_links/pybind_type_links.html +++ b/documentation/test_python/pybind_type_links/pybind_type_links.html @@ -60,6 +60,10 @@ def type_enum(arg0: Enum, /)
A function taking an enum
+
+ def type_nested(arg0: Tuple[Foo, List[Enum]], /) +
+
A function with nested type annotation
def type_return() -> Foo