1 //===-- PythonDataObjects.h--------------------------------------*- C++ -*-===//
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
7 //===----------------------------------------------------------------------===//
10 // !! FIXME FIXME FIXME !!
12 // Python APIs nearly all can return an exception. They do this
13 // by returning NULL, or -1, or some such value and setting
14 // the exception state with PyErr_Set*(). Exceptions must be
15 // handled before further python API functions are called. Failure
16 // to do so will result in asserts on debug builds of python.
17 // It will also sometimes, but not usually result in crashes of
20 // Nearly all the code in this header does not handle python exceptions
21 // correctly. It should all be converted to return Expected<> or
22 // Error types to capture the exception.
24 // Everything in this file except functions that return Error or
25 // Expected<> is considered deprecated and should not be
26 // used in new code. If you need to use it, fix it first.
29 // TODOs for this file
31 // * Make all methods safe for exceptions.
33 // * Eliminate method signatures that must translate exceptions into
34 // empty objects or NULLs. Almost everything here should return
35 // Expected<>. It should be acceptable for certain operations that
36 // can never fail to assert instead, such as the creation of
37 // PythonString from a string literal.
39 // * Eliminate Reset(), and make all non-default constructors private.
40 // Python objects should be created with Retain<> or Take<>, and they
41 // should be assigned with operator=
43 // * Eliminate default constructors, make python objects always
44 // nonnull, and use optionals where necessary.
48 #ifndef LLDB_PLUGINS_SCRIPTINTERPRETER_PYTHON_PYTHONDATAOBJECTS_H
49 #define LLDB_PLUGINS_SCRIPTINTERPRETER_PYTHON_PYTHONDATAOBJECTS_H
51 #include "lldb/Host/Config.h"
53 #if LLDB_ENABLE_PYTHON
55 // LLDB Python header must be included first
56 #include "lldb-python.h"
58 #include "lldb/Host/File.h"
59 #include "lldb/Utility/StructuredData.h"
61 #include "llvm/ADT/ArrayRef.h"
63 namespace lldb_private
{
70 class PythonDictionary
;
72 class PythonException
;
77 m_state
= PyGILState_Ensure();
78 assert(!PyErr_Occurred());
80 ~GIL() { PyGILState_Release(m_state
); }
83 PyGILState_STATE m_state
;
86 enum class PyObjectType
{
102 enum class PyRefType
{
103 Borrowed
, // We are not given ownership of the incoming PyObject.
104 // We cannot safely hold it without calling Py_INCREF.
105 Owned
// We have ownership of the incoming PyObject. We should
106 // not call Py_INCREF.
110 // Take a reference that you already own, and turn it into
113 // Most python API methods will return a +1 reference
114 // if they succeed or NULL if and only if
115 // they set an exception. Use this to collect such return
116 // values, after checking for NULL.
118 // If T is not just PythonObject, then obj must be already be
119 // checked to be of the correct type.
120 template <typename T
> T
Take(PyObject
*obj
) {
122 assert(!PyErr_Occurred());
123 T
thing(PyRefType::Owned
, obj
);
124 assert(thing
.IsValid());
128 // Retain a reference you have borrowed, and turn it into
131 // A minority of python APIs return a borrowed reference
132 // instead of a +1. They will also return NULL if and only
133 // if they set an exception. Use this to collect such return
134 // values, after checking for NULL.
136 // If T is not just PythonObject, then obj must be already be
137 // checked to be of the correct type.
138 template <typename T
> T
Retain(PyObject
*obj
) {
140 assert(!PyErr_Occurred());
141 T
thing(PyRefType::Borrowed
, obj
);
142 assert(thing
.IsValid());
146 // This class can be used like a utility function to convert from
147 // a llvm-friendly Twine into a null-terminated const char *,
148 // which is the form python C APIs want their strings in.
151 // const llvm::Twine &some_twine;
152 // PyFoo_Bar(x, y, z, NullTerminated(some_twine));
154 // Why a class instead of a function? If the twine isn't already null
155 // terminated, it will need a temporary buffer to copy the string
156 // into. We need that buffer to stick around for the lifetime of the
158 class NullTerminated
{
160 llvm::SmallString
<32> storage
;
163 NullTerminated(const llvm::Twine
&twine
) {
164 llvm::StringRef ref
= twine
.toNullTerminatedStringRef(storage
);
167 operator const char *() { return str
; }
170 inline llvm::Error
nullDeref() {
171 return llvm::createStringError(llvm::inconvertibleErrorCode(),
172 "A NULL PyObject* was dereferenced");
175 inline llvm::Error
exception(const char *s
= nullptr) {
176 return llvm::make_error
<PythonException
>(s
);
179 inline llvm::Error
keyError() {
180 return llvm::createStringError(llvm::inconvertibleErrorCode(),
184 inline const char *py2_const_cast(const char *s
) { return s
; }
186 enum class PyInitialValue
{ Invalid
, Empty
};
188 // DOC: https://docs.python.org/3/c-api/arg.html#building-values
189 template <typename T
, typename Enable
= void> struct PythonFormat
;
191 template <typename T
, char F
> struct PassthroughFormat
{
192 static constexpr char format
= F
;
193 static constexpr T
get(T t
) { return t
; }
196 template <> struct PythonFormat
<char *> : PassthroughFormat
<char *, 's'> {};
197 template <> struct PythonFormat
<const char *> :
198 PassthroughFormat
<const char *, 's'> {};
199 template <> struct PythonFormat
<char> : PassthroughFormat
<char, 'b'> {};
201 struct PythonFormat
<unsigned char> : PassthroughFormat
<unsigned char, 'B'> {};
202 template <> struct PythonFormat
<short> : PassthroughFormat
<short, 'h'> {};
204 struct PythonFormat
<unsigned short> : PassthroughFormat
<unsigned short, 'H'> {};
205 template <> struct PythonFormat
<int> : PassthroughFormat
<int, 'i'> {};
206 template <> struct PythonFormat
<bool> : PassthroughFormat
<bool, 'p'> {};
208 struct PythonFormat
<unsigned int> : PassthroughFormat
<unsigned int, 'I'> {};
209 template <> struct PythonFormat
<long> : PassthroughFormat
<long, 'l'> {};
211 struct PythonFormat
<unsigned long> : PassthroughFormat
<unsigned long, 'k'> {};
213 struct PythonFormat
<long long> : PassthroughFormat
<long long, 'L'> {};
215 struct PythonFormat
<unsigned long long>
216 : PassthroughFormat
<unsigned long long, 'K'> {};
218 struct PythonFormat
<PyObject
*> : PassthroughFormat
<PyObject
*, 'O'> {};
220 template <typename T
>
222 T
, typename
std::enable_if
<std::is_base_of
<PythonObject
, T
>::value
>::type
> {
223 static constexpr char format
= 'O';
224 static auto get(const T
&value
) { return value
.get(); }
229 PythonObject() = default;
231 PythonObject(PyRefType type
, PyObject
*py_obj
) {
233 // If this is a borrowed reference, we need to convert it to
234 // an owned reference by incrementing it. If it is an owned
235 // reference (for example the caller allocated it with PyDict_New()
236 // then we must *not* increment it.
237 if (m_py_obj
&& Py_IsInitialized() && type
== PyRefType::Borrowed
)
238 Py_XINCREF(m_py_obj
);
241 PythonObject(const PythonObject
&rhs
)
242 : PythonObject(PyRefType::Borrowed
, rhs
.m_py_obj
) {}
244 PythonObject(PythonObject
&&rhs
) {
245 m_py_obj
= rhs
.m_py_obj
;
246 rhs
.m_py_obj
= nullptr;
249 ~PythonObject() { Reset(); }
255 _PyObject_Dump(m_py_obj
);
260 void Dump(Stream
&strm
) const;
262 PyObject
*get() const { return m_py_obj
; }
264 PyObject
*release() {
265 PyObject
*result
= m_py_obj
;
270 PythonObject
&operator=(PythonObject other
) {
272 m_py_obj
= std::exchange(other
.m_py_obj
, nullptr);
276 PyObjectType
GetObjectType() const;
278 PythonString
Repr() const;
280 PythonString
Str() const;
282 static PythonObject
ResolveNameWithDictionary(llvm::StringRef name
,
283 const PythonDictionary
&dict
);
285 template <typename T
>
286 static T
ResolveNameWithDictionary(llvm::StringRef name
,
287 const PythonDictionary
&dict
) {
288 return ResolveNameWithDictionary(name
, dict
).AsType
<T
>();
291 PythonObject
ResolveName(llvm::StringRef name
) const;
293 template <typename T
> T
ResolveName(llvm::StringRef name
) const {
294 return ResolveName(name
).AsType
<T
>();
297 bool HasAttribute(llvm::StringRef attribute
) const;
299 PythonObject
GetAttributeValue(llvm::StringRef attribute
) const;
301 bool IsNone() const { return m_py_obj
== Py_None
; }
303 bool IsValid() const { return m_py_obj
!= nullptr; }
305 bool IsAllocated() const { return IsValid() && !IsNone(); }
307 explicit operator bool() const { return IsValid() && !IsNone(); }
309 template <typename T
> T
AsType() const {
310 if (!T::Check(m_py_obj
))
312 return T(PyRefType::Borrowed
, m_py_obj
);
315 StructuredData::ObjectSP
CreateStructuredObject() const;
317 template <typename
... T
>
318 llvm::Expected
<PythonObject
> CallMethod(const char *name
,
319 const T
&... t
) const {
320 const char format
[] = {'(', PythonFormat
<T
>::format
..., ')', 0};
322 PyObject_CallMethod(m_py_obj
, py2_const_cast(name
),
323 py2_const_cast(format
), PythonFormat
<T
>::get(t
)...);
326 return python::Take
<PythonObject
>(obj
);
329 template <typename
... T
>
330 llvm::Expected
<PythonObject
> Call(const T
&... t
) const {
331 const char format
[] = {'(', PythonFormat
<T
>::format
..., ')', 0};
332 PyObject
*obj
= PyObject_CallFunction(m_py_obj
, py2_const_cast(format
),
333 PythonFormat
<T
>::get(t
)...);
336 return python::Take
<PythonObject
>(obj
);
339 llvm::Expected
<PythonObject
> GetAttribute(const llvm::Twine
&name
) const {
342 PyObject
*obj
= PyObject_GetAttrString(m_py_obj
, NullTerminated(name
));
345 return python::Take
<PythonObject
>(obj
);
348 llvm::Expected
<PythonObject
> GetType() const {
351 PyObject
*obj
= PyObject_Type(m_py_obj
);
354 return python::Take
<PythonObject
>(obj
);
357 llvm::Expected
<bool> IsTrue() {
360 int r
= PyObject_IsTrue(m_py_obj
);
366 llvm::Expected
<long long> AsLongLong() const;
368 llvm::Expected
<unsigned long long> AsUnsignedLongLong() const;
370 // wraps on overflow, instead of raising an error.
371 llvm::Expected
<unsigned long long> AsModuloUnsignedLongLong() const;
373 llvm::Expected
<bool> IsInstance(const PythonObject
&cls
) {
374 if (!m_py_obj
|| !cls
.IsValid())
376 int r
= PyObject_IsInstance(m_py_obj
, cls
.get());
383 PyObject
*m_py_obj
= nullptr;
387 // This is why C++ needs monads.
388 template <typename T
> llvm::Expected
<T
> As(llvm::Expected
<PythonObject
> &&obj
) {
390 return obj
.takeError();
391 if (!T::Check(obj
.get().get()))
392 return llvm::createStringError(llvm::inconvertibleErrorCode(),
394 return T(PyRefType::Borrowed
, std::move(obj
.get().get()));
397 template <> llvm::Expected
<bool> As
<bool>(llvm::Expected
<PythonObject
> &&obj
);
400 llvm::Expected
<long long> As
<long long>(llvm::Expected
<PythonObject
> &&obj
);
403 llvm::Expected
<unsigned long long>
404 As
<unsigned long long>(llvm::Expected
<PythonObject
> &&obj
);
407 llvm::Expected
<std::string
> As
<std::string
>(llvm::Expected
<PythonObject
> &&obj
);
410 template <class T
> class TypedPythonObject
: public PythonObject
{
412 TypedPythonObject(PyRefType type
, PyObject
*py_obj
) {
415 if (T::Check(py_obj
))
416 PythonObject::operator=(PythonObject(type
, py_obj
));
417 else if (type
== PyRefType::Owned
)
421 TypedPythonObject() = default;
424 class PythonBytes
: public TypedPythonObject
<PythonBytes
> {
426 using TypedPythonObject::TypedPythonObject
;
427 explicit PythonBytes(llvm::ArrayRef
<uint8_t> bytes
);
428 PythonBytes(const uint8_t *bytes
, size_t length
);
430 static bool Check(PyObject
*py_obj
);
432 llvm::ArrayRef
<uint8_t> GetBytes() const;
434 size_t GetSize() const;
436 void SetBytes(llvm::ArrayRef
<uint8_t> stringbytes
);
438 StructuredData::StringSP
CreateStructuredString() const;
441 class PythonByteArray
: public TypedPythonObject
<PythonByteArray
> {
443 using TypedPythonObject::TypedPythonObject
;
444 explicit PythonByteArray(llvm::ArrayRef
<uint8_t> bytes
);
445 PythonByteArray(const uint8_t *bytes
, size_t length
);
446 PythonByteArray(const PythonBytes
&object
);
448 static bool Check(PyObject
*py_obj
);
450 llvm::ArrayRef
<uint8_t> GetBytes() const;
452 size_t GetSize() const;
454 void SetBytes(llvm::ArrayRef
<uint8_t> stringbytes
);
456 StructuredData::StringSP
CreateStructuredString() const;
459 class PythonString
: public TypedPythonObject
<PythonString
> {
461 using TypedPythonObject::TypedPythonObject
;
462 static llvm::Expected
<PythonString
> FromUTF8(llvm::StringRef string
);
464 PythonString() : TypedPythonObject() {} // MSVC requires this for some reason
466 explicit PythonString(llvm::StringRef string
); // safe, null on error
468 static bool Check(PyObject
*py_obj
);
470 llvm::StringRef
GetString() const; // safe, empty string on error
472 llvm::Expected
<llvm::StringRef
> AsUTF8() const;
474 size_t GetSize() const;
476 void SetString(llvm::StringRef string
); // safe, null on error
478 StructuredData::StringSP
CreateStructuredString() const;
481 class PythonInteger
: public TypedPythonObject
<PythonInteger
> {
483 using TypedPythonObject::TypedPythonObject
;
485 PythonInteger() : TypedPythonObject() {} // MSVC requires this for some reason
487 explicit PythonInteger(int64_t value
);
489 static bool Check(PyObject
*py_obj
);
491 void SetInteger(int64_t value
);
493 StructuredData::IntegerSP
CreateStructuredInteger() const;
495 StructuredData::UnsignedIntegerSP
CreateStructuredUnsignedInteger() const;
497 StructuredData::SignedIntegerSP
CreateStructuredSignedInteger() const;
500 class PythonBoolean
: public TypedPythonObject
<PythonBoolean
> {
502 using TypedPythonObject::TypedPythonObject
;
504 explicit PythonBoolean(bool value
);
506 static bool Check(PyObject
*py_obj
);
508 bool GetValue() const;
510 void SetValue(bool value
);
512 StructuredData::BooleanSP
CreateStructuredBoolean() const;
515 class PythonList
: public TypedPythonObject
<PythonList
> {
517 using TypedPythonObject::TypedPythonObject
;
519 PythonList() : TypedPythonObject() {} // MSVC requires this for some reason
521 explicit PythonList(PyInitialValue value
);
522 explicit PythonList(int list_size
);
524 static bool Check(PyObject
*py_obj
);
526 uint32_t GetSize() const;
528 PythonObject
GetItemAtIndex(uint32_t index
) const;
530 void SetItemAtIndex(uint32_t index
, const PythonObject
&object
);
532 void AppendItem(const PythonObject
&object
);
534 StructuredData::ArraySP
CreateStructuredArray() const;
537 class PythonTuple
: public TypedPythonObject
<PythonTuple
> {
539 using TypedPythonObject::TypedPythonObject
;
541 explicit PythonTuple(PyInitialValue value
);
542 explicit PythonTuple(int tuple_size
);
543 PythonTuple(std::initializer_list
<PythonObject
> objects
);
544 PythonTuple(std::initializer_list
<PyObject
*> objects
);
546 static bool Check(PyObject
*py_obj
);
548 uint32_t GetSize() const;
550 PythonObject
GetItemAtIndex(uint32_t index
) const;
552 void SetItemAtIndex(uint32_t index
, const PythonObject
&object
);
554 StructuredData::ArraySP
CreateStructuredArray() const;
557 class PythonDictionary
: public TypedPythonObject
<PythonDictionary
> {
559 using TypedPythonObject::TypedPythonObject
;
561 PythonDictionary() : TypedPythonObject() {} // MSVC requires this for some reason
563 explicit PythonDictionary(PyInitialValue value
);
565 static bool Check(PyObject
*py_obj
);
567 bool HasKey(const llvm::Twine
&key
) const;
569 uint32_t GetSize() const;
571 PythonList
GetKeys() const;
573 PythonObject
GetItemForKey(const PythonObject
&key
) const; // DEPRECATED
574 void SetItemForKey(const PythonObject
&key
,
575 const PythonObject
&value
); // DEPRECATED
577 llvm::Expected
<PythonObject
> GetItem(const PythonObject
&key
) const;
578 llvm::Expected
<PythonObject
> GetItem(const llvm::Twine
&key
) const;
579 llvm::Error
SetItem(const PythonObject
&key
, const PythonObject
&value
) const;
580 llvm::Error
SetItem(const llvm::Twine
&key
, const PythonObject
&value
) const;
582 StructuredData::DictionarySP
CreateStructuredDictionary() const;
585 class PythonModule
: public TypedPythonObject
<PythonModule
> {
587 using TypedPythonObject::TypedPythonObject
;
589 static bool Check(PyObject
*py_obj
);
591 static PythonModule
BuiltinsModule();
593 static PythonModule
MainModule();
595 static PythonModule
AddModule(llvm::StringRef module
);
597 // safe, returns invalid on error;
598 static PythonModule
ImportModule(llvm::StringRef name
) {
599 std::string s
= std::string(name
);
600 auto mod
= Import(s
.c_str());
602 llvm::consumeError(mod
.takeError());
603 return PythonModule();
605 return std::move(mod
.get());
608 static llvm::Expected
<PythonModule
> Import(const llvm::Twine
&name
);
610 llvm::Expected
<PythonObject
> Get(const llvm::Twine
&name
);
612 PythonDictionary
GetDictionary() const;
615 class PythonCallable
: public TypedPythonObject
<PythonCallable
> {
617 using TypedPythonObject::TypedPythonObject
;
620 /* the largest number of positional arguments this callable
621 * can accept, or UNBOUNDED, ie UINT_MAX if it's a varargs
622 * function and can accept an arbitrary number */
623 unsigned max_positional_args
;
624 static constexpr unsigned UNBOUNDED
= UINT_MAX
; // FIXME c++17 inline
627 static bool Check(PyObject
*py_obj
);
629 llvm::Expected
<ArgInfo
> GetArgInfo() const;
631 PythonObject
operator()();
633 PythonObject
operator()(std::initializer_list
<PyObject
*> args
);
635 PythonObject
operator()(std::initializer_list
<PythonObject
> args
);
637 template <typename Arg
, typename
... Args
>
638 PythonObject
operator()(const Arg
&arg
, Args
... args
) {
639 return operator()({arg
, args
...});
643 class PythonFile
: public TypedPythonObject
<PythonFile
> {
645 using TypedPythonObject::TypedPythonObject
;
647 PythonFile() : TypedPythonObject() {} // MSVC requires this for some reason
649 static bool Check(PyObject
*py_obj
);
651 static llvm::Expected
<PythonFile
> FromFile(File
&file
,
652 const char *mode
= nullptr);
654 llvm::Expected
<lldb::FileSP
> ConvertToFile(bool borrowed
= false);
655 llvm::Expected
<lldb::FileSP
>
656 ConvertToFileForcingUseOfScriptingIOMethods(bool borrowed
= false);
659 class PythonException
: public llvm::ErrorInfo
<PythonException
> {
661 PyObject
*m_exception_type
, *m_exception
, *m_traceback
;
662 PyObject
*m_repr_bytes
;
666 const char *toCString() const;
667 PythonException(const char *caller
= nullptr);
669 ~PythonException() override
;
670 void log(llvm::raw_ostream
&OS
) const override
;
671 std::error_code
convertToErrorCode() const override
;
672 bool Matches(PyObject
*exc
) const;
673 std::string
ReadBacktrace() const;
676 // This extracts the underlying T out of an Expected<T> and returns it.
677 // If the Expected is an Error instead of a T, that error will be converted
678 // into a python exception, and this will return a default-constructed T.
680 // This is appropriate for use right at the boundary of python calling into
681 // C++, such as in a SWIG typemap. In such a context you should simply
682 // check if the returned T is valid, and if it is, return a NULL back
683 // to python. This will result in the Error being raised as an exception
684 // from python code's point of view.
688 // Expected<Foo *> efoop = some_cpp_function();
689 // Foo *foop = unwrapOrSetPythonException(efoop);
692 // do_something(*foop);
694 // If the Error returned was itself created because a python exception was
695 // raised when C++ code called into python, then the original exception
696 // will be restored. Otherwise a simple string exception will be raised.
697 template <typename T
> T
unwrapOrSetPythonException(llvm::Expected
<T
> expected
) {
699 return expected
.get();
700 llvm::handleAllErrors(
701 expected
.takeError(), [](PythonException
&E
) { E
.Restore(); },
702 [](const llvm::ErrorInfoBase
&E
) {
703 PyErr_SetString(PyExc_Exception
, E
.message().c_str());
708 // This is only here to help incrementally migrate old, exception-unsafe
710 template <typename T
> T
unwrapIgnoringErrors(llvm::Expected
<T
> expected
) {
712 return std::move(expected
.get());
713 llvm::consumeError(expected
.takeError());
717 llvm::Expected
<PythonObject
> runStringOneLine(const llvm::Twine
&string
,
718 const PythonDictionary
&globals
,
719 const PythonDictionary
&locals
);
721 llvm::Expected
<PythonObject
> runStringMultiLine(const llvm::Twine
&string
,
722 const PythonDictionary
&globals
,
723 const PythonDictionary
&locals
);
725 // Sometimes the best way to interact with a python interpreter is
726 // to run some python code. You construct a PythonScript with
727 // script string. The script assigns some function to `_function_`
728 // and you get a C++ callable object that calls the python function.
732 // const char script[] = R"(
737 // Expected<PythonObject> cpp_foo_wrapper(PythonObject x, PythonObject y) {
738 // // no need to synchronize access to this global, we already have the GIL
739 // static PythonScript foo(script)
744 PythonCallable function
;
749 PythonScript(const char *script
) : script(script
), function() {}
751 template <typename
... Args
>
752 llvm::Expected
<PythonObject
> operator()(Args
&&... args
) {
753 if (llvm::Error error
= Init())
754 return std::move(error
);
755 return function
.Call(std::forward
<Args
>(args
)...);
759 class StructuredPythonObject
: public StructuredData::Generic
{
761 StructuredPythonObject() : StructuredData::Generic() {}
763 // Take ownership of the object we received.
764 StructuredPythonObject(PythonObject obj
)
765 : StructuredData::Generic(obj
.release()) {}
767 ~StructuredPythonObject() override
{
768 // Hand ownership back to a (temporary) PythonObject instance and let it
769 // take care of releasing it.
770 PythonObject(PyRefType::Owned
, static_cast<PyObject
*>(GetValue()));
773 bool IsValid() const override
{ return GetValue() && GetValue() != Py_None
; }
775 void Serialize(llvm::json::OStream
&s
) const override
;
778 StructuredPythonObject(const StructuredPythonObject
&) = delete;
779 const StructuredPythonObject
&
780 operator=(const StructuredPythonObject
&) = delete;
783 } // namespace python
784 } // namespace lldb_private
788 #endif // LLDB_PLUGINS_SCRIPTINTERPRETER_PYTHON_PYTHONDATAOBJECTS_H