lldb/source/Plugins/ScriptInterpreter/Python/PythonDataObjects.cpp

   1 //===-- PythonDataObjects.cpp ---------------------------------------------===//
   2 //
   3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
   4 // See https://llvm.org/LICENSE.txt for license information.
   5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
   6 //
   7 //===----------------------------------------------------------------------===//
   8
   9 #include "lldb/Host/Config.h"
  10
  11 #if LLDB_ENABLE_PYTHON
  12
  13 #include "PythonDataObjects.h"
  14 #include "ScriptInterpreterPython.h"
  15
  16 #include "lldb/Host/File.h"
  17 #include "lldb/Host/FileSystem.h"
  18 #include "lldb/Interpreter/ScriptInterpreter.h"
  19 #include "lldb/Utility/LLDBLog.h"
  20 #include "lldb/Utility/Log.h"
  21 #include "lldb/Utility/Stream.h"
  22
  23 #include "llvm/Support/Casting.h"
  24 #include "llvm/Support/ConvertUTF.h"
  25 #include "llvm/Support/Errno.h"
  26
  27 #include <cstdio>
  28 #include <variant>
  29
  30 using namespace lldb_private;
  31 using namespace lldb;
  32 using namespace lldb_private::python;
  33 using llvm::cantFail;
  34 using llvm::Error;
  35 using llvm::Expected;
  36 using llvm::Twine;
  37
  38 template <> Expected<bool> python::As<bool>(Expected<PythonObject> &&obj) {
  39   if (!obj)
  40     return obj.takeError();
  41   return obj.get().IsTrue();
  42 }
  43
  44 template <>
  45 Expected<long long> python::As<long long>(Expected<PythonObject> &&obj) {
  46   if (!obj)
  47     return obj.takeError();
  48   return obj->AsLongLong();
  49 }
  50
  51 template <>
  52 Expected<unsigned long long>
  53 python::As<unsigned long long>(Expected<PythonObject> &&obj) {
  54   if (!obj)
  55     return obj.takeError();
  56   return obj->AsUnsignedLongLong();
  57 }
  58
  59 template <>
  60 Expected<std::string> python::As<std::string>(Expected<PythonObject> &&obj) {
  61   if (!obj)
  62     return obj.takeError();
  63   PyObject *str_obj = PyObject_Str(obj.get().get());
  64   if (!str_obj)
  65     return llvm::make_error<PythonException>();
  66   auto str = Take<PythonString>(str_obj);
  67   auto utf8 = str.AsUTF8();
  68   if (!utf8)
  69     return utf8.takeError();
  70   return std::string(utf8.get());
  71 }
  72
  73 static bool python_is_finalizing() {
  74 #if PY_VERSION_HEX >= 0x030d0000
  75   return Py_IsFinalizing();
  76 #elif PY_VERSION_HEX >= 0x03070000
  77   return _Py_IsFinalizing();
  78 #else
  79   return _Py_Finalizing != nullptr;
  80 #endif
  81 }
  82
  83 void PythonObject::Reset() {
  84   if (m_py_obj && Py_IsInitialized()) {
  85     if (python_is_finalizing()) {
  86       // Leak m_py_obj rather than crashing the process.
  87       // https://docs.python.org/3/c-api/init.html#c.PyGILState_Ensure
  88     } else {
  89       PyGILState_STATE state = PyGILState_Ensure();
  90       Py_DECREF(m_py_obj);
  91       PyGILState_Release(state);
  92     }
  93   }
  94   m_py_obj = nullptr;
  95 }
  96
  97 Expected<long long> PythonObject::AsLongLong() const {
  98   if (!m_py_obj)
  99     return nullDeref();
 100   assert(!PyErr_Occurred());
 101   long long r = PyLong_AsLongLong(m_py_obj);
 102   if (PyErr_Occurred())
 103     return exception();
 104   return r;
 105 }
 106
 107 Expected<unsigned long long> PythonObject::AsUnsignedLongLong() const {
 108   if (!m_py_obj)
 109     return nullDeref();
 110   assert(!PyErr_Occurred());
 111   long long r = PyLong_AsUnsignedLongLong(m_py_obj);
 112   if (PyErr_Occurred())
 113     return exception();
 114   return r;
 115 }
 116
 117 // wraps on overflow, instead of raising an error.
 118 Expected<unsigned long long> PythonObject::AsModuloUnsignedLongLong() const {
 119   if (!m_py_obj)
 120     return nullDeref();
 121   assert(!PyErr_Occurred());
 122   unsigned long long r = PyLong_AsUnsignedLongLongMask(m_py_obj);
 123   // FIXME: We should fetch the exception message and hoist it.
 124   if (PyErr_Occurred())
 125     return exception();
 126   return r;
 127 }
 128
 129 void StructuredPythonObject::Serialize(llvm::json::OStream &s) const {
 130   s.value(llvm::formatv("Python Obj: {0:X}", GetValue()).str());
 131 }
 132
 133 // PythonObject
 134
 135 void PythonObject::Dump(Stream &strm) const {
 136   if (m_py_obj) {
 137     FILE *file = llvm::sys::RetryAfterSignal(nullptr, ::tmpfile);
 138     if (file) {
 139       ::PyObject_Print(m_py_obj, file, 0);
 140       const long length = ftell(file);
 141       if (length) {
 142         ::rewind(file);
 143         std::vector<char> file_contents(length, '\0');
 144         const size_t length_read =
 145             ::fread(file_contents.data(), 1, file_contents.size(), file);
 146         if (length_read > 0)
 147           strm.Write(file_contents.data(), length_read);
 148       }
 149       ::fclose(file);
 150     }
 151   } else
 152     strm.PutCString("NULL");
 153 }
 154
 155 PyObjectType PythonObject::GetObjectType() const {
 156   if (!IsAllocated())
 157     return PyObjectType::None;
 158
 159   if (PythonModule::Check(m_py_obj))
 160     return PyObjectType::Module;
 161   if (PythonList::Check(m_py_obj))
 162     return PyObjectType::List;
 163   if (PythonTuple::Check(m_py_obj))
 164     return PyObjectType::Tuple;
 165   if (PythonDictionary::Check(m_py_obj))
 166     return PyObjectType::Dictionary;
 167   if (PythonString::Check(m_py_obj))
 168     return PyObjectType::String;
 169   if (PythonBytes::Check(m_py_obj))
 170     return PyObjectType::Bytes;
 171   if (PythonByteArray::Check(m_py_obj))
 172     return PyObjectType::ByteArray;
 173   if (PythonBoolean::Check(m_py_obj))
 174     return PyObjectType::Boolean;
 175   if (PythonInteger::Check(m_py_obj))
 176     return PyObjectType::Integer;
 177   if (PythonFile::Check(m_py_obj))
 178     return PyObjectType::File;
 179   if (PythonCallable::Check(m_py_obj))
 180     return PyObjectType::Callable;
 181   return PyObjectType::Unknown;
 182 }
 183
 184 PythonString PythonObject::Repr() const {
 185   if (!m_py_obj)
 186     return PythonString();
 187   PyObject *repr = PyObject_Repr(m_py_obj);
 188   if (!repr)
 189     return PythonString();
 190   return PythonString(PyRefType::Owned, repr);
 191 }
 192
 193 PythonString PythonObject::Str() const {
 194   if (!m_py_obj)
 195     return PythonString();
 196   PyObject *str = PyObject_Str(m_py_obj);
 197   if (!str)
 198     return PythonString();
 199   return PythonString(PyRefType::Owned, str);
 200 }
 201
 202 PythonObject
 203 PythonObject::ResolveNameWithDictionary(llvm::StringRef name,
 204                                         const PythonDictionary &dict) {
 205   size_t dot_pos = name.find('.');
 206   llvm::StringRef piece = name.substr(0, dot_pos);
 207   PythonObject result = dict.GetItemForKey(PythonString(piece));
 208   if (dot_pos == llvm::StringRef::npos) {
 209     // There was no dot, we're done.
 210     return result;
 211   }
 212
 213   // There was a dot.  The remaining portion of the name should be looked up in
 214   // the context of the object that was found in the dictionary.
 215   return result.ResolveName(name.substr(dot_pos + 1));
 216 }
 217
 218 PythonObject PythonObject::ResolveName(llvm::StringRef name) const {
 219   // Resolve the name in the context of the specified object.  If, for example,
 220   // `this` refers to a PyModule, then this will look for `name` in this
 221   // module.  If `this` refers to a PyType, then it will resolve `name` as an
 222   // attribute of that type.  If `this` refers to an instance of an object,
 223   // then it will resolve `name` as the value of the specified field.
 224   //
 225   // This function handles dotted names so that, for example, if `m_py_obj`
 226   // refers to the `sys` module, and `name` == "path.append", then it will find
 227   // the function `sys.path.append`.
 228
 229   size_t dot_pos = name.find('.');
 230   if (dot_pos == llvm::StringRef::npos) {
 231     // No dots in the name, we should be able to find the value immediately as
 232     // an attribute of `m_py_obj`.
 233     return GetAttributeValue(name);
 234   }
 235
 236   // Look up the first piece of the name, and resolve the rest as a child of
 237   // that.
 238   PythonObject parent = ResolveName(name.substr(0, dot_pos));
 239   if (!parent.IsAllocated())
 240     return PythonObject();
 241
 242   // Tail recursion.. should be optimized by the compiler
 243   return parent.ResolveName(name.substr(dot_pos + 1));
 244 }
 245
 246 bool PythonObject::HasAttribute(llvm::StringRef attr) const {
 247   if (!IsValid())
 248     return false;
 249   PythonString py_attr(attr);
 250   return !!PyObject_HasAttr(m_py_obj, py_attr.get());
 251 }
 252
 253 PythonObject PythonObject::GetAttributeValue(llvm::StringRef attr) const {
 254   if (!IsValid())
 255     return PythonObject();
 256
 257   PythonString py_attr(attr);
 258   if (!PyObject_HasAttr(m_py_obj, py_attr.get()))
 259     return PythonObject();
 260
 261   return PythonObject(PyRefType::Owned,
 262                       PyObject_GetAttr(m_py_obj, py_attr.get()));
 263 }
 264
 265 StructuredData::ObjectSP PythonObject::CreateStructuredObject() const {
 266   assert(PyGILState_Check());
 267   switch (GetObjectType()) {
 268   case PyObjectType::Dictionary:
 269     return PythonDictionary(PyRefType::Borrowed, m_py_obj)
 270         .CreateStructuredDictionary();
 271   case PyObjectType::Boolean:
 272     return PythonBoolean(PyRefType::Borrowed, m_py_obj)
 273         .CreateStructuredBoolean();
 274   case PyObjectType::Integer: {
 275     StructuredData::IntegerSP int_sp =
 276         PythonInteger(PyRefType::Borrowed, m_py_obj).CreateStructuredInteger();
 277     if (std::holds_alternative<StructuredData::UnsignedIntegerSP>(int_sp))
 278       return std::get<StructuredData::UnsignedIntegerSP>(int_sp);
 279     if (std::holds_alternative<StructuredData::SignedIntegerSP>(int_sp))
 280       return std::get<StructuredData::SignedIntegerSP>(int_sp);
 281     return nullptr;
 282   };
 283   case PyObjectType::List:
 284     return PythonList(PyRefType::Borrowed, m_py_obj).CreateStructuredArray();
 285   case PyObjectType::String:
 286     return PythonString(PyRefType::Borrowed, m_py_obj).CreateStructuredString();
 287   case PyObjectType::Bytes:
 288     return PythonBytes(PyRefType::Borrowed, m_py_obj).CreateStructuredString();
 289   case PyObjectType::ByteArray:
 290     return PythonByteArray(PyRefType::Borrowed, m_py_obj)
 291         .CreateStructuredString();
 292   case PyObjectType::None:
 293     return StructuredData::ObjectSP();
 294   default:
 295     return StructuredData::ObjectSP(new StructuredPythonObject(
 296         PythonObject(PyRefType::Borrowed, m_py_obj)));
 297   }
 298 }
 299
 300 // PythonString
 301
 302 PythonBytes::PythonBytes(llvm::ArrayRef<uint8_t> bytes) { SetBytes(bytes); }
 303
 304 PythonBytes::PythonBytes(const uint8_t *bytes, size_t length) {
 305   SetBytes(llvm::ArrayRef<uint8_t>(bytes, length));
 306 }
 307
 308 bool PythonBytes::Check(PyObject *py_obj) {
 309   if (!py_obj)
 310     return false;
 311   return PyBytes_Check(py_obj);
 312 }
 313
 314 llvm::ArrayRef<uint8_t> PythonBytes::GetBytes() const {
 315   if (!IsValid())
 316     return llvm::ArrayRef<uint8_t>();
 317
 318   Py_ssize_t size;
 319   char *c;
 320
 321   PyBytes_AsStringAndSize(m_py_obj, &c, &size);
 322   return llvm::ArrayRef<uint8_t>(reinterpret_cast<uint8_t *>(c), size);
 323 }
 324
 325 size_t PythonBytes::GetSize() const {
 326   if (!IsValid())
 327     return 0;
 328   return PyBytes_Size(m_py_obj);
 329 }
 330
 331 void PythonBytes::SetBytes(llvm::ArrayRef<uint8_t> bytes) {
 332   const char *data = reinterpret_cast<const char *>(bytes.data());
 333   *this = Take<PythonBytes>(PyBytes_FromStringAndSize(data, bytes.size()));
 334 }
 335
 336 StructuredData::StringSP PythonBytes::CreateStructuredString() const {
 337   StructuredData::StringSP result(new StructuredData::String);
 338   Py_ssize_t size;
 339   char *c;
 340   PyBytes_AsStringAndSize(m_py_obj, &c, &size);
 341   result->SetValue(std::string(c, size));
 342   return result;
 343 }
 344
 345 PythonByteArray::PythonByteArray(llvm::ArrayRef<uint8_t> bytes)
 346     : PythonByteArray(bytes.data(), bytes.size()) {}
 347
 348 PythonByteArray::PythonByteArray(const uint8_t *bytes, size_t length) {
 349   const char *str = reinterpret_cast<const char *>(bytes);
 350   *this = Take<PythonByteArray>(PyByteArray_FromStringAndSize(str, length));
 351 }
 352
 353 bool PythonByteArray::Check(PyObject *py_obj) {
 354   if (!py_obj)
 355     return false;
 356   return PyByteArray_Check(py_obj);
 357 }
 358
 359 llvm::ArrayRef<uint8_t> PythonByteArray::GetBytes() const {
 360   if (!IsValid())
 361     return llvm::ArrayRef<uint8_t>();
 362
 363   char *c = PyByteArray_AsString(m_py_obj);
 364   size_t size = GetSize();
 365   return llvm::ArrayRef<uint8_t>(reinterpret_cast<uint8_t *>(c), size);
 366 }
 367
 368 size_t PythonByteArray::GetSize() const {
 369   if (!IsValid())
 370     return 0;
 371
 372   return PyByteArray_Size(m_py_obj);
 373 }
 374
 375 StructuredData::StringSP PythonByteArray::CreateStructuredString() const {
 376   StructuredData::StringSP result(new StructuredData::String);
 377   llvm::ArrayRef<uint8_t> bytes = GetBytes();
 378   const char *str = reinterpret_cast<const char *>(bytes.data());
 379   result->SetValue(std::string(str, bytes.size()));
 380   return result;
 381 }
 382
 383 // PythonString
 384
 385 Expected<PythonString> PythonString::FromUTF8(llvm::StringRef string) {
 386   PyObject *str = PyUnicode_FromStringAndSize(string.data(), string.size());
 387   if (!str)
 388     return llvm::make_error<PythonException>();
 389   return Take<PythonString>(str);
 390 }
 391
 392 PythonString::PythonString(llvm::StringRef string) { SetString(string); }
 393
 394 bool PythonString::Check(PyObject *py_obj) {
 395   if (!py_obj)
 396     return false;
 397
 398   if (PyUnicode_Check(py_obj))
 399     return true;
 400   return false;
 401 }
 402
 403 llvm::StringRef PythonString::GetString() const {
 404   auto s = AsUTF8();
 405   if (!s) {
 406     llvm::consumeError(s.takeError());
 407     return llvm::StringRef("");
 408   }
 409   return s.get();
 410 }
 411
 412 Expected<llvm::StringRef> PythonString::AsUTF8() const {
 413   if (!IsValid())
 414     return nullDeref();
 415
 416   Py_ssize_t size;
 417   const char *data;
 418
 419   data = PyUnicode_AsUTF8AndSize(m_py_obj, &size);
 420
 421   if (!data)
 422     return exception();
 423
 424   return llvm::StringRef(data, size);
 425 }
 426
 427 size_t PythonString::GetSize() const {
 428   if (IsValid()) {
 429 #if PY_MINOR_VERSION >= 3
 430     return PyUnicode_GetLength(m_py_obj);
 431 #else
 432     return PyUnicode_GetSize(m_py_obj);
 433 #endif
 434   }
 435   return 0;
 436 }
 437
 438 void PythonString::SetString(llvm::StringRef string) {
 439   auto s = FromUTF8(string);
 440   if (!s) {
 441     llvm::consumeError(s.takeError());
 442     Reset();
 443   } else {
 444     *this = std::move(s.get());
 445   }
 446 }
 447
 448 StructuredData::StringSP PythonString::CreateStructuredString() const {
 449   StructuredData::StringSP result(new StructuredData::String);
 450   result->SetValue(GetString());
 451   return result;
 452 }
 453
 454 // PythonInteger
 455
 456 PythonInteger::PythonInteger(int64_t value) { SetInteger(value); }
 457
 458 bool PythonInteger::Check(PyObject *py_obj) {
 459   if (!py_obj)
 460     return false;
 461
 462   // Python 3 does not have PyInt_Check.  There is only one type of integral
 463   // value, long.
 464   return PyLong_Check(py_obj);
 465 }
 466
 467 void PythonInteger::SetInteger(int64_t value) {
 468   *this = Take<PythonInteger>(PyLong_FromLongLong(value));
 469 }
 470
 471 StructuredData::IntegerSP PythonInteger::CreateStructuredInteger() const {
 472   StructuredData::UnsignedIntegerSP uint_sp = CreateStructuredUnsignedInteger();
 473   return uint_sp ? StructuredData::IntegerSP(uint_sp)
 474                  : CreateStructuredSignedInteger();
 475 }
 476
 477 StructuredData::UnsignedIntegerSP
 478 PythonInteger::CreateStructuredUnsignedInteger() const {
 479   StructuredData::UnsignedIntegerSP result = nullptr;
 480   llvm::Expected<unsigned long long> value = AsUnsignedLongLong();
 481   if (!value)
 482     llvm::consumeError(value.takeError());
 483   else
 484     result = std::make_shared<StructuredData::UnsignedInteger>(value.get());
 485
 486   return result;
 487 }
 488
 489 StructuredData::SignedIntegerSP
 490 PythonInteger::CreateStructuredSignedInteger() const {
 491   StructuredData::SignedIntegerSP result = nullptr;
 492   llvm::Expected<long long> value = AsLongLong();
 493   if (!value)
 494     llvm::consumeError(value.takeError());
 495   else
 496     result = std::make_shared<StructuredData::SignedInteger>(value.get());
 497
 498   return result;
 499 }
 500
 501 // PythonBoolean
 502
 503 PythonBoolean::PythonBoolean(bool value) {
 504   SetValue(value);
 505 }
 506
 507 bool PythonBoolean::Check(PyObject *py_obj) {
 508   return py_obj ? PyBool_Check(py_obj) : false;
 509 }
 510
 511 bool PythonBoolean::GetValue() const {
 512   return m_py_obj ? PyObject_IsTrue(m_py_obj) : false;
 513 }
 514
 515 void PythonBoolean::SetValue(bool value) {
 516   *this = Take<PythonBoolean>(PyBool_FromLong(value));
 517 }
 518
 519 StructuredData::BooleanSP PythonBoolean::CreateStructuredBoolean() const {
 520   StructuredData::BooleanSP result(new StructuredData::Boolean);
 521   result->SetValue(GetValue());
 522   return result;
 523 }
 524
 525 // PythonList
 526
 527 PythonList::PythonList(PyInitialValue value) {
 528   if (value == PyInitialValue::Empty)
 529     *this = Take<PythonList>(PyList_New(0));
 530 }
 531
 532 PythonList::PythonList(int list_size) {
 533   *this = Take<PythonList>(PyList_New(list_size));
 534 }
 535
 536 bool PythonList::Check(PyObject *py_obj) {
 537   if (!py_obj)
 538     return false;
 539   return PyList_Check(py_obj);
 540 }
 541
 542 uint32_t PythonList::GetSize() const {
 543   if (IsValid())
 544     return PyList_GET_SIZE(m_py_obj);
 545   return 0;
 546 }
 547
 548 PythonObject PythonList::GetItemAtIndex(uint32_t index) const {
 549   if (IsValid())
 550     return PythonObject(PyRefType::Borrowed, PyList_GetItem(m_py_obj, index));
 551   return PythonObject();
 552 }
 553
 554 void PythonList::SetItemAtIndex(uint32_t index, const PythonObject &object) {
 555   if (IsAllocated() && object.IsValid()) {
 556     // PyList_SetItem is documented to "steal" a reference, so we need to
 557     // convert it to an owned reference by incrementing it.
 558     Py_INCREF(object.get());
 559     PyList_SetItem(m_py_obj, index, object.get());
 560   }
 561 }
 562
 563 void PythonList::AppendItem(const PythonObject &object) {
 564   if (IsAllocated() && object.IsValid()) {
 565     // `PyList_Append` does *not* steal a reference, so do not call `Py_INCREF`
 566     // here like we do with `PyList_SetItem`.
 567     PyList_Append(m_py_obj, object.get());
 568   }
 569 }
 570
 571 StructuredData::ArraySP PythonList::CreateStructuredArray() const {
 572   StructuredData::ArraySP result(new StructuredData::Array);
 573   uint32_t count = GetSize();
 574   for (uint32_t i = 0; i < count; ++i) {
 575     PythonObject obj = GetItemAtIndex(i);
 576     result->AddItem(obj.CreateStructuredObject());
 577   }
 578   return result;
 579 }
 580
 581 // PythonTuple
 582
 583 PythonTuple::PythonTuple(PyInitialValue value) {
 584   if (value == PyInitialValue::Empty)
 585     *this = Take<PythonTuple>(PyTuple_New(0));
 586 }
 587
 588 PythonTuple::PythonTuple(int tuple_size) {
 589   *this = Take<PythonTuple>(PyTuple_New(tuple_size));
 590 }
 591
 592 PythonTuple::PythonTuple(std::initializer_list<PythonObject> objects) {
 593   m_py_obj = PyTuple_New(objects.size());
 594
 595   uint32_t idx = 0;
 596   for (auto object : objects) {
 597     if (object.IsValid())
 598       SetItemAtIndex(idx, object);
 599     idx++;
 600   }
 601 }
 602
 603 PythonTuple::PythonTuple(std::initializer_list<PyObject *> objects) {
 604   m_py_obj = PyTuple_New(objects.size());
 605
 606   uint32_t idx = 0;
 607   for (auto py_object : objects) {
 608     PythonObject object(PyRefType::Borrowed, py_object);
 609     if (object.IsValid())
 610       SetItemAtIndex(idx, object);
 611     idx++;
 612   }
 613 }
 614
 615 bool PythonTuple::Check(PyObject *py_obj) {
 616   if (!py_obj)
 617     return false;
 618   return PyTuple_Check(py_obj);
 619 }
 620
 621 uint32_t PythonTuple::GetSize() const {
 622   if (IsValid())
 623     return PyTuple_GET_SIZE(m_py_obj);
 624   return 0;
 625 }
 626
 627 PythonObject PythonTuple::GetItemAtIndex(uint32_t index) const {
 628   if (IsValid())
 629     return PythonObject(PyRefType::Borrowed, PyTuple_GetItem(m_py_obj, index));
 630   return PythonObject();
 631 }
 632
 633 void PythonTuple::SetItemAtIndex(uint32_t index, const PythonObject &object) {
 634   if (IsAllocated() && object.IsValid()) {
 635     // PyTuple_SetItem is documented to "steal" a reference, so we need to
 636     // convert it to an owned reference by incrementing it.
 637     Py_INCREF(object.get());
 638     PyTuple_SetItem(m_py_obj, index, object.get());
 639   }
 640 }
 641
 642 StructuredData::ArraySP PythonTuple::CreateStructuredArray() const {
 643   StructuredData::ArraySP result(new StructuredData::Array);
 644   uint32_t count = GetSize();
 645   for (uint32_t i = 0; i < count; ++i) {
 646     PythonObject obj = GetItemAtIndex(i);
 647     result->AddItem(obj.CreateStructuredObject());
 648   }
 649   return result;
 650 }
 651
 652 // PythonDictionary
 653
 654 PythonDictionary::PythonDictionary(PyInitialValue value) {
 655   if (value == PyInitialValue::Empty)
 656     *this = Take<PythonDictionary>(PyDict_New());
 657 }
 658
 659 bool PythonDictionary::Check(PyObject *py_obj) {
 660   if (!py_obj)
 661     return false;
 662
 663   return PyDict_Check(py_obj);
 664 }
 665
 666 bool PythonDictionary::HasKey(const llvm::Twine &key) const {
 667   if (!IsValid())
 668     return false;
 669
 670   PythonString key_object(key.isSingleStringRef() ? key.getSingleStringRef()
 671                                                   : key.str());
 672
 673   if (int res = PyDict_Contains(m_py_obj, key_object.get()) > 0)
 674     return res;
 675
 676   PyErr_Print();
 677   return false;
 678 }
 679
 680 uint32_t PythonDictionary::GetSize() const {
 681   if (IsValid())
 682     return PyDict_Size(m_py_obj);
 683   return 0;
 684 }
 685
 686 PythonList PythonDictionary::GetKeys() const {
 687   if (IsValid())
 688     return PythonList(PyRefType::Owned, PyDict_Keys(m_py_obj));
 689   return PythonList(PyInitialValue::Invalid);
 690 }
 691
 692 PythonObject PythonDictionary::GetItemForKey(const PythonObject &key) const {
 693   auto item = GetItem(key);
 694   if (!item) {
 695     llvm::consumeError(item.takeError());
 696     return PythonObject();
 697   }
 698   return std::move(item.get());
 699 }
 700
 701 Expected<PythonObject>
 702 PythonDictionary::GetItem(const PythonObject &key) const {
 703   if (!IsValid())
 704     return nullDeref();
 705   PyObject *o = PyDict_GetItemWithError(m_py_obj, key.get());
 706   if (PyErr_Occurred())
 707     return exception();
 708   if (!o)
 709     return keyError();
 710   return Retain<PythonObject>(o);
 711 }
 712
 713 Expected<PythonObject> PythonDictionary::GetItem(const Twine &key) const {
 714   if (!IsValid())
 715     return nullDeref();
 716   PyObject *o = PyDict_GetItemString(m_py_obj, NullTerminated(key));
 717   if (PyErr_Occurred())
 718     return exception();
 719   if (!o)
 720     return keyError();
 721   return Retain<PythonObject>(o);
 722 }
 723
 724 Error PythonDictionary::SetItem(const PythonObject &key,
 725                                 const PythonObject &value) const {
 726   if (!IsValid() || !value.IsValid())
 727     return nullDeref();
 728   int r = PyDict_SetItem(m_py_obj, key.get(), value.get());
 729   if (r < 0)
 730     return exception();
 731   return Error::success();
 732 }
 733
 734 Error PythonDictionary::SetItem(const Twine &key,
 735                                 const PythonObject &value) const {
 736   if (!IsValid() || !value.IsValid())
 737     return nullDeref();
 738   int r = PyDict_SetItemString(m_py_obj, NullTerminated(key), value.get());
 739   if (r < 0)
 740     return exception();
 741   return Error::success();
 742 }
 743
 744 void PythonDictionary::SetItemForKey(const PythonObject &key,
 745                                      const PythonObject &value) {
 746   Error error = SetItem(key, value);
 747   if (error)
 748     llvm::consumeError(std::move(error));
 749 }
 750
 751 StructuredData::DictionarySP
 752 PythonDictionary::CreateStructuredDictionary() const {
 753   StructuredData::DictionarySP result(new StructuredData::Dictionary);
 754   PythonList keys(GetKeys());
 755   uint32_t num_keys = keys.GetSize();
 756   for (uint32_t i = 0; i < num_keys; ++i) {
 757     PythonObject key = keys.GetItemAtIndex(i);
 758     PythonObject value = GetItemForKey(key);
 759     StructuredData::ObjectSP structured_value = value.CreateStructuredObject();
 760     result->AddItem(key.Str().GetString(), structured_value);
 761   }
 762   return result;
 763 }
 764
 765 PythonModule PythonModule::BuiltinsModule() { return AddModule("builtins"); }
 766
 767 PythonModule PythonModule::MainModule() { return AddModule("__main__"); }
 768
 769 PythonModule PythonModule::AddModule(llvm::StringRef module) {
 770   std::string str = module.str();
 771   return PythonModule(PyRefType::Borrowed, PyImport_AddModule(str.c_str()));
 772 }
 773
 774 Expected<PythonModule> PythonModule::Import(const Twine &name) {
 775   PyObject *mod = PyImport_ImportModule(NullTerminated(name));
 776   if (!mod)
 777     return exception();
 778   return Take<PythonModule>(mod);
 779 }
 780
 781 Expected<PythonObject> PythonModule::Get(const Twine &name) {
 782   if (!IsValid())
 783     return nullDeref();
 784   PyObject *dict = PyModule_GetDict(m_py_obj);
 785   if (!dict)
 786     return exception();
 787   PyObject *item = PyDict_GetItemString(dict, NullTerminated(name));
 788   if (!item)
 789     return exception();
 790   return Retain<PythonObject>(item);
 791 }
 792
 793 bool PythonModule::Check(PyObject *py_obj) {
 794   if (!py_obj)
 795     return false;
 796
 797   return PyModule_Check(py_obj);
 798 }
 799
 800 PythonDictionary PythonModule::GetDictionary() const {
 801   if (!IsValid())
 802     return PythonDictionary();
 803   return Retain<PythonDictionary>(PyModule_GetDict(m_py_obj));
 804 }
 805
 806 bool PythonCallable::Check(PyObject *py_obj) {
 807   if (!py_obj)
 808     return false;
 809
 810   return PyCallable_Check(py_obj);
 811 }
 812
 813 #if PY_VERSION_HEX >= 0x03030000
 814 static const char get_arg_info_script[] = R"(
 815 from inspect import signature, Parameter, ismethod
 816 from collections import namedtuple
 817 ArgInfo = namedtuple('ArgInfo', ['count', 'has_varargs'])
 818 def main(f):
 819     count = 0
 820     varargs = False
 821     for parameter in signature(f).parameters.values():
 822         kind = parameter.kind
 823         if kind in (Parameter.POSITIONAL_ONLY,
 824                     Parameter.POSITIONAL_OR_KEYWORD):
 825             count += 1
 826         elif kind == Parameter.VAR_POSITIONAL:
 827             varargs = True
 828         elif kind in (Parameter.KEYWORD_ONLY,
 829                       Parameter.VAR_KEYWORD):
 830             pass
 831         else:
 832             raise Exception(f'unknown parameter kind: {kind}')
 833     return ArgInfo(count, varargs)
 834 )";
 835 #endif
 836
 837 Expected<PythonCallable::ArgInfo> PythonCallable::GetArgInfo() const {
 838   ArgInfo result = {};
 839   if (!IsValid())
 840     return nullDeref();
 841
 842 #if PY_VERSION_HEX >= 0x03030000
 843
 844   // no need to synchronize access to this global, we already have the GIL
 845   static PythonScript get_arg_info(get_arg_info_script);
 846   Expected<PythonObject> pyarginfo = get_arg_info(*this);
 847   if (!pyarginfo)
 848     return pyarginfo.takeError();
 849   long long count =
 850       cantFail(As<long long>(pyarginfo.get().GetAttribute("count")));
 851   bool has_varargs =
 852       cantFail(As<bool>(pyarginfo.get().GetAttribute("has_varargs")));
 853   result.max_positional_args = has_varargs ? ArgInfo::UNBOUNDED : count;
 854
 855 #else
 856   PyObject *py_func_obj;
 857   bool is_bound_method = false;
 858   bool is_class = false;
 859
 860   if (PyType_Check(m_py_obj) || PyClass_Check(m_py_obj)) {
 861     auto init = GetAttribute("__init__");
 862     if (!init)
 863       return init.takeError();
 864     py_func_obj = init.get().get();
 865     is_class = true;
 866   } else {
 867     py_func_obj = m_py_obj;
 868   }
 869
 870   if (PyMethod_Check(py_func_obj)) {
 871     py_func_obj = PyMethod_GET_FUNCTION(py_func_obj);
 872     PythonObject im_self = GetAttributeValue("im_self");
 873     if (im_self.IsValid() && !im_self.IsNone())
 874       is_bound_method = true;
 875   } else {
 876     // see if this is a callable object with an __call__ method
 877     if (!PyFunction_Check(py_func_obj)) {
 878       PythonObject __call__ = GetAttributeValue("__call__");
 879       if (__call__.IsValid()) {
 880         auto __callable__ = __call__.AsType<PythonCallable>();
 881         if (__callable__.IsValid()) {
 882           py_func_obj = PyMethod_GET_FUNCTION(__callable__.get());
 883           PythonObject im_self = __callable__.GetAttributeValue("im_self");
 884           if (im_self.IsValid() && !im_self.IsNone())
 885             is_bound_method = true;
 886         }
 887       }
 888     }
 889   }
 890
 891   if (!py_func_obj)
 892     return result;
 893
 894   PyCodeObject *code = (PyCodeObject *)PyFunction_GET_CODE(py_func_obj);
 895   if (!code)
 896     return result;
 897
 898   auto count = code->co_argcount;
 899   bool has_varargs = !!(code->co_flags & CO_VARARGS);
 900   result.max_positional_args =
 901       has_varargs ? ArgInfo::UNBOUNDED
 902                   : (count - (int)is_bound_method) - (int)is_class;
 903
 904 #endif
 905
 906   return result;
 907 }
 908
 909 constexpr unsigned
 910     PythonCallable::ArgInfo::UNBOUNDED; // FIXME delete after c++17
 911
 912 PythonObject PythonCallable::operator()() {
 913   return PythonObject(PyRefType::Owned, PyObject_CallObject(m_py_obj, nullptr));
 914 }
 915
 916 PythonObject PythonCallable::
 917 operator()(std::initializer_list<PyObject *> args) {
 918   PythonTuple arg_tuple(args);
 919   return PythonObject(PyRefType::Owned,
 920                       PyObject_CallObject(m_py_obj, arg_tuple.get()));
 921 }
 922
 923 PythonObject PythonCallable::
 924 operator()(std::initializer_list<PythonObject> args) {
 925   PythonTuple arg_tuple(args);
 926   return PythonObject(PyRefType::Owned,
 927                       PyObject_CallObject(m_py_obj, arg_tuple.get()));
 928 }
 929
 930 bool PythonFile::Check(PyObject *py_obj) {
 931   if (!py_obj)
 932     return false;
 933   // In Python 3, there is no `PyFile_Check`, and in fact PyFile is not even a
 934   // first-class object type anymore.  `PyFile_FromFd` is just a thin wrapper
 935   // over `io.open()`, which returns some object derived from `io.IOBase`. As a
 936   // result, the only way to detect a file in Python 3 is to check whether it
 937   // inherits from `io.IOBase`.
 938   auto io_module = PythonModule::Import("io");
 939   if (!io_module) {
 940     llvm::consumeError(io_module.takeError());
 941     return false;
 942   }
 943   auto iobase = io_module.get().Get("IOBase");
 944   if (!iobase) {
 945     llvm::consumeError(iobase.takeError());
 946     return false;
 947   }
 948   int r = PyObject_IsInstance(py_obj, iobase.get().get());
 949   if (r < 0) {
 950     llvm::consumeError(exception()); // clear the exception and log it.
 951     return false;
 952   }
 953   return !!r;
 954 }
 955
 956 const char *PythonException::toCString() const {
 957   if (!m_repr_bytes)
 958     return "unknown exception";
 959   return PyBytes_AS_STRING(m_repr_bytes);
 960 }
 961
 962 PythonException::PythonException(const char *caller) {
 963   assert(PyErr_Occurred());
 964   m_exception_type = m_exception = m_traceback = m_repr_bytes = nullptr;
 965   PyErr_Fetch(&m_exception_type, &m_exception, &m_traceback);
 966   PyErr_NormalizeException(&m_exception_type, &m_exception, &m_traceback);
 967   PyErr_Clear();
 968   if (m_exception) {
 969     PyObject *repr = PyObject_Repr(m_exception);
 970     if (repr) {
 971       m_repr_bytes = PyUnicode_AsEncodedString(repr, "utf-8", nullptr);
 972       if (!m_repr_bytes) {
 973         PyErr_Clear();
 974       }
 975       Py_XDECREF(repr);
 976     } else {
 977       PyErr_Clear();
 978     }
 979   }
 980   Log *log = GetLog(LLDBLog::Script);
 981   if (caller)
 982     LLDB_LOGF(log, "%s failed with exception: %s", caller, toCString());
 983   else
 984     LLDB_LOGF(log, "python exception: %s", toCString());
 985 }
 986 void PythonException::Restore() {
 987   if (m_exception_type && m_exception) {
 988     PyErr_Restore(m_exception_type, m_exception, m_traceback);
 989   } else {
 990     PyErr_SetString(PyExc_Exception, toCString());
 991   }
 992   m_exception_type = m_exception = m_traceback = nullptr;
 993 }
 994
 995 PythonException::~PythonException() {
 996   Py_XDECREF(m_exception_type);
 997   Py_XDECREF(m_exception);
 998   Py_XDECREF(m_traceback);
 999   Py_XDECREF(m_repr_bytes);
1000 }
1001
1002 void PythonException::log(llvm::raw_ostream &OS) const { OS << toCString(); }
1003
1004 std::error_code PythonException::convertToErrorCode() const {
1005   return llvm::inconvertibleErrorCode();
1006 }
1007
1008 bool PythonException::Matches(PyObject *exc) const {
1009   return PyErr_GivenExceptionMatches(m_exception_type, exc);
1010 }
1011
1012 const char read_exception_script[] = R"(
1013 import sys
1014 from traceback import print_exception
1015 if sys.version_info.major < 3:
1016   from StringIO import StringIO
1017 else:
1018   from io import StringIO
1019 def main(exc_type, exc_value, tb):
1020   f = StringIO()
1021   print_exception(exc_type, exc_value, tb, file=f)
1022   return f.getvalue()
1023 )";
1024
1025 std::string PythonException::ReadBacktrace() const {
1026
1027   if (!m_traceback)
1028     return toCString();
1029
1030   // no need to synchronize access to this global, we already have the GIL
1031   static PythonScript read_exception(read_exception_script);
1032
1033   Expected<std::string> backtrace = As<std::string>(
1034       read_exception(m_exception_type, m_exception, m_traceback));
1035
1036   if (!backtrace) {
1037     std::string message =
1038         std::string(toCString()) + "\n" +
1039         "Traceback unavailable, an error occurred while reading it:\n";
1040     return (message + llvm::toString(backtrace.takeError()));
1041   }
1042
1043   return std::move(backtrace.get());
1044 }
1045
1046 char PythonException::ID = 0;
1047
1048 llvm::Expected<File::OpenOptions>
1049 GetOptionsForPyObject(const PythonObject &obj) {
1050   auto options = File::OpenOptions(0);
1051   auto readable = As<bool>(obj.CallMethod("readable"));
1052   if (!readable)
1053     return readable.takeError();
1054   auto writable = As<bool>(obj.CallMethod("writable"));
1055   if (!writable)
1056     return writable.takeError();
1057   if (readable.get() && writable.get())
1058     options |= File::eOpenOptionReadWrite;
1059   else if (writable.get())
1060     options |= File::eOpenOptionWriteOnly;
1061   else if (readable.get())
1062     options |= File::eOpenOptionReadOnly;
1063   return options;
1064 }
1065
1066 // Base class template for python files.   All it knows how to do
1067 // is hold a reference to the python object and close or flush it
1068 // when the File is closed.
1069 namespace {
1070 template <typename Base> class OwnedPythonFile : public Base {
1071 public:
1072   template <typename... Args>
1073   OwnedPythonFile(const PythonFile &file, bool borrowed, Args... args)
1074       : Base(args...), m_py_obj(file), m_borrowed(borrowed) {
1075     assert(m_py_obj);
1076   }
1077
1078   ~OwnedPythonFile() override {
1079     assert(m_py_obj);
1080     GIL takeGIL;
1081     Close();
1082     // we need to ensure the python object is released while we still
1083     // hold the GIL
1084     m_py_obj.Reset();
1085   }
1086
1087   bool IsPythonSideValid() const {
1088     GIL takeGIL;
1089     auto closed = As<bool>(m_py_obj.GetAttribute("closed"));
1090     if (!closed) {
1091       llvm::consumeError(closed.takeError());
1092       return false;
1093     }
1094     return !closed.get();
1095   }
1096
1097   bool IsValid() const override {
1098     return IsPythonSideValid() && Base::IsValid();
1099   }
1100
1101   Status Close() override {
1102     assert(m_py_obj);
1103     Status py_error, base_error;
1104     GIL takeGIL;
1105     if (!m_borrowed) {
1106       auto r = m_py_obj.CallMethod("close");
1107       if (!r)
1108         py_error = Status::FromError(r.takeError());
1109     }
1110     base_error = Base::Close();
1111     // Cloning since the wrapped exception may still reference the PyThread.
1112     if (py_error.Fail())
1113       return py_error.Clone();
1114     return base_error.Clone();
1115   };
1116
1117   PyObject *GetPythonObject() const {
1118     assert(m_py_obj.IsValid());
1119     return m_py_obj.get();
1120   }
1121
1122   static bool classof(const File *file) = delete;
1123
1124 protected:
1125   PythonFile m_py_obj;
1126   bool m_borrowed;
1127 };
1128 } // namespace
1129
1130 // A SimplePythonFile is a OwnedPythonFile that just does all I/O as
1131 // a NativeFile
1132 namespace {
1133 class SimplePythonFile : public OwnedPythonFile<NativeFile> {
1134 public:
1135   SimplePythonFile(const PythonFile &file, bool borrowed, int fd,
1136                    File::OpenOptions options)
1137       : OwnedPythonFile(file, borrowed, fd, options, false) {}
1138
1139   static char ID;
1140   bool isA(const void *classID) const override {
1141     return classID == &ID || NativeFile::isA(classID);
1142   }
1143   static bool classof(const File *file) { return file->isA(&ID); }
1144 };
1145 char SimplePythonFile::ID = 0;
1146 } // namespace
1147
1148 namespace {
1149 class PythonBuffer {
1150 public:
1151   PythonBuffer &operator=(const PythonBuffer &) = delete;
1152   PythonBuffer(const PythonBuffer &) = delete;
1153
1154   static Expected<PythonBuffer> Create(PythonObject &obj,
1155                                        int flags = PyBUF_SIMPLE) {
1156     Py_buffer py_buffer = {};
1157     PyObject_GetBuffer(obj.get(), &py_buffer, flags);
1158     if (!py_buffer.obj)
1159       return llvm::make_error<PythonException>();
1160     return PythonBuffer(py_buffer);
1161   }
1162
1163   PythonBuffer(PythonBuffer &&other) {
1164     m_buffer = other.m_buffer;
1165     other.m_buffer.obj = nullptr;
1166   }
1167
1168   ~PythonBuffer() {
1169     if (m_buffer.obj)
1170       PyBuffer_Release(&m_buffer);
1171   }
1172
1173   Py_buffer &get() { return m_buffer; }
1174
1175 private:
1176   // takes ownership of the buffer.
1177   PythonBuffer(const Py_buffer &py_buffer) : m_buffer(py_buffer) {}
1178   Py_buffer m_buffer;
1179 };
1180 } // namespace
1181
1182 // Shared methods between TextPythonFile and BinaryPythonFile
1183 namespace {
1184 class PythonIOFile : public OwnedPythonFile<File> {
1185 public:
1186   PythonIOFile(const PythonFile &file, bool borrowed)
1187       : OwnedPythonFile(file, borrowed) {}
1188
1189   ~PythonIOFile() override { Close(); }
1190
1191   bool IsValid() const override { return IsPythonSideValid(); }
1192
1193   Status Close() override {
1194     assert(m_py_obj);
1195     GIL takeGIL;
1196     if (m_borrowed)
1197       return Flush();
1198     auto r = m_py_obj.CallMethod("close");
1199     if (!r)
1200       // Cloning since the wrapped exception may still reference the PyThread.
1201       return Status::FromError(r.takeError()).Clone();
1202     return Status();
1203   }
1204
1205   Status Flush() override {
1206     GIL takeGIL;
1207     auto r = m_py_obj.CallMethod("flush");
1208     if (!r)
1209       // Cloning since the wrapped exception may still reference the PyThread.
1210       return Status::FromError(r.takeError()).Clone();
1211     return Status();
1212   }
1213
1214   Expected<File::OpenOptions> GetOptions() const override {
1215     GIL takeGIL;
1216     return GetOptionsForPyObject(m_py_obj);
1217   }
1218
1219   static char ID;
1220   bool isA(const void *classID) const override {
1221     return classID == &ID || File::isA(classID);
1222   }
1223   static bool classof(const File *file) { return file->isA(&ID); }
1224 };
1225 char PythonIOFile::ID = 0;
1226 } // namespace
1227
1228 namespace {
1229 class BinaryPythonFile : public PythonIOFile {
1230 protected:
1231   int m_descriptor;
1232
1233 public:
1234   BinaryPythonFile(int fd, const PythonFile &file, bool borrowed)
1235       : PythonIOFile(file, borrowed),
1236         m_descriptor(File::DescriptorIsValid(fd) ? fd
1237                                                  : File::kInvalidDescriptor) {}
1238
1239   int GetDescriptor() const override { return m_descriptor; }
1240
1241   Status Write(const void *buf, size_t &num_bytes) override {
1242     GIL takeGIL;
1243     PyObject *pybuffer_p = PyMemoryView_FromMemory(
1244         const_cast<char *>((const char *)buf), num_bytes, PyBUF_READ);
1245     if (!pybuffer_p)
1246       // Cloning since the wrapped exception may still reference the PyThread.
1247       return Status::FromError(llvm::make_error<PythonException>()).Clone();
1248     auto pybuffer = Take<PythonObject>(pybuffer_p);
1249     num_bytes = 0;
1250     auto bytes_written = As<long long>(m_py_obj.CallMethod("write", pybuffer));
1251     if (!bytes_written)
1252       return Status::FromError(bytes_written.takeError());
1253     if (bytes_written.get() < 0)
1254       return Status::FromErrorString(
1255           ".write() method returned a negative number!");
1256     static_assert(sizeof(long long) >= sizeof(size_t), "overflow");
1257     num_bytes = bytes_written.get();
1258     return Status();
1259   }
1260
1261   Status Read(void *buf, size_t &num_bytes) override {
1262     GIL takeGIL;
1263     static_assert(sizeof(long long) >= sizeof(size_t), "overflow");
1264     auto pybuffer_obj =
1265         m_py_obj.CallMethod("read", (unsigned long long)num_bytes);
1266     if (!pybuffer_obj)
1267       // Cloning since the wrapped exception may still reference the PyThread.
1268       return Status::FromError(pybuffer_obj.takeError()).Clone();
1269     num_bytes = 0;
1270     if (pybuffer_obj.get().IsNone()) {
1271       // EOF
1272       num_bytes = 0;
1273       return Status();
1274     }
1275     auto pybuffer = PythonBuffer::Create(pybuffer_obj.get());
1276     if (!pybuffer)
1277       // Cloning since the wrapped exception may still reference the PyThread.
1278       return Status::FromError(pybuffer.takeError()).Clone();
1279     memcpy(buf, pybuffer.get().get().buf, pybuffer.get().get().len);
1280     num_bytes = pybuffer.get().get().len;
1281     return Status();
1282   }
1283 };
1284 } // namespace
1285
1286 namespace {
1287 class TextPythonFile : public PythonIOFile {
1288 protected:
1289   int m_descriptor;
1290
1291 public:
1292   TextPythonFile(int fd, const PythonFile &file, bool borrowed)
1293       : PythonIOFile(file, borrowed),
1294         m_descriptor(File::DescriptorIsValid(fd) ? fd
1295                                                  : File::kInvalidDescriptor) {}
1296
1297   int GetDescriptor() const override { return m_descriptor; }
1298
1299   Status Write(const void *buf, size_t &num_bytes) override {
1300     GIL takeGIL;
1301     auto pystring =
1302         PythonString::FromUTF8(llvm::StringRef((const char *)buf, num_bytes));
1303     if (!pystring)
1304       return Status::FromError(pystring.takeError());
1305     num_bytes = 0;
1306     auto bytes_written =
1307         As<long long>(m_py_obj.CallMethod("write", pystring.get()));
1308     if (!bytes_written)
1309       // Cloning since the wrapped exception may still reference the PyThread.
1310       return Status::FromError(bytes_written.takeError()).Clone();
1311     if (bytes_written.get() < 0)
1312       return Status::FromErrorString(
1313           ".write() method returned a negative number!");
1314     static_assert(sizeof(long long) >= sizeof(size_t), "overflow");
1315     num_bytes = bytes_written.get();
1316     return Status();
1317   }
1318
1319   Status Read(void *buf, size_t &num_bytes) override {
1320     GIL takeGIL;
1321     size_t num_chars = num_bytes / 6;
1322     size_t orig_num_bytes = num_bytes;
1323     num_bytes = 0;
1324     if (orig_num_bytes < 6) {
1325       return Status::FromErrorString(
1326           "can't read less than 6 bytes from a utf8 text stream");
1327     }
1328     auto pystring = As<PythonString>(
1329         m_py_obj.CallMethod("read", (unsigned long long)num_chars));
1330     if (!pystring)
1331       // Cloning since the wrapped exception may still reference the PyThread.
1332       return Status::FromError(pystring.takeError()).Clone();
1333     if (pystring.get().IsNone()) {
1334       // EOF
1335       return Status();
1336     }
1337     auto stringref = pystring.get().AsUTF8();
1338     if (!stringref)
1339       // Cloning since the wrapped exception may still reference the PyThread.
1340       return Status::FromError(stringref.takeError()).Clone();
1341     num_bytes = stringref.get().size();
1342     memcpy(buf, stringref.get().begin(), num_bytes);
1343     return Status();
1344   }
1345 };
1346 } // namespace
1347
1348 llvm::Expected<FileSP> PythonFile::ConvertToFile(bool borrowed) {
1349   if (!IsValid())
1350     return llvm::createStringError(llvm::inconvertibleErrorCode(),
1351                                    "invalid PythonFile");
1352
1353   int fd = PyObject_AsFileDescriptor(m_py_obj);
1354   if (fd < 0) {
1355     PyErr_Clear();
1356     return ConvertToFileForcingUseOfScriptingIOMethods(borrowed);
1357   }
1358   auto options = GetOptionsForPyObject(*this);
1359   if (!options)
1360     return options.takeError();
1361
1362   File::OpenOptions rw =
1363       options.get() & (File::eOpenOptionReadOnly | File::eOpenOptionWriteOnly |
1364                        File::eOpenOptionReadWrite);
1365   if (rw == File::eOpenOptionWriteOnly || rw == File::eOpenOptionReadWrite) {
1366     // LLDB and python will not share I/O buffers.  We should probably
1367     // flush the python buffers now.
1368     auto r = CallMethod("flush");
1369     if (!r)
1370       return r.takeError();
1371   }
1372
1373   FileSP file_sp;
1374   if (borrowed) {
1375     // In this case we don't need to retain the python
1376     // object at all.
1377     file_sp = std::make_shared<NativeFile>(fd, options.get(), false);
1378   } else {
1379     file_sp = std::static_pointer_cast<File>(
1380         std::make_shared<SimplePythonFile>(*this, borrowed, fd, options.get()));
1381   }
1382   if (!file_sp->IsValid())
1383     return llvm::createStringError(llvm::inconvertibleErrorCode(),
1384                                    "invalid File");
1385
1386   return file_sp;
1387 }
1388
1389 llvm::Expected<FileSP>
1390 PythonFile::ConvertToFileForcingUseOfScriptingIOMethods(bool borrowed) {
1391
1392   assert(!PyErr_Occurred());
1393
1394   if (!IsValid())
1395     return llvm::createStringError(llvm::inconvertibleErrorCode(),
1396                                    "invalid PythonFile");
1397
1398   int fd = PyObject_AsFileDescriptor(m_py_obj);
1399   if (fd < 0) {
1400     PyErr_Clear();
1401     fd = File::kInvalidDescriptor;
1402   }
1403
1404   auto io_module = PythonModule::Import("io");
1405   if (!io_module)
1406     return io_module.takeError();
1407   auto textIOBase = io_module.get().Get("TextIOBase");
1408   if (!textIOBase)
1409     return textIOBase.takeError();
1410   auto rawIOBase = io_module.get().Get("RawIOBase");
1411   if (!rawIOBase)
1412     return rawIOBase.takeError();
1413   auto bufferedIOBase = io_module.get().Get("BufferedIOBase");
1414   if (!bufferedIOBase)
1415     return bufferedIOBase.takeError();
1416
1417   FileSP file_sp;
1418
1419   auto isTextIO = IsInstance(textIOBase.get());
1420   if (!isTextIO)
1421     return isTextIO.takeError();
1422   if (isTextIO.get())
1423     file_sp = std::static_pointer_cast<File>(
1424         std::make_shared<TextPythonFile>(fd, *this, borrowed));
1425
1426   auto isRawIO = IsInstance(rawIOBase.get());
1427   if (!isRawIO)
1428     return isRawIO.takeError();
1429   auto isBufferedIO = IsInstance(bufferedIOBase.get());
1430   if (!isBufferedIO)
1431     return isBufferedIO.takeError();
1432
1433   if (isRawIO.get() || isBufferedIO.get()) {
1434     file_sp = std::static_pointer_cast<File>(
1435         std::make_shared<BinaryPythonFile>(fd, *this, borrowed));
1436   }
1437
1438   if (!file_sp)
1439     return llvm::createStringError(llvm::inconvertibleErrorCode(),
1440                                    "python file is neither text nor binary");
1441
1442   if (!file_sp->IsValid())
1443     return llvm::createStringError(llvm::inconvertibleErrorCode(),
1444                                    "invalid File");
1445
1446   return file_sp;
1447 }
1448
1449 Expected<PythonFile> PythonFile::FromFile(File &file, const char *mode) {
1450   if (!file.IsValid())
1451     return llvm::createStringError(llvm::inconvertibleErrorCode(),
1452                                    "invalid file");
1453
1454   if (auto *simple = llvm::dyn_cast<SimplePythonFile>(&file))
1455     return Retain<PythonFile>(simple->GetPythonObject());
1456   if (auto *pythonio = llvm::dyn_cast<PythonIOFile>(&file))
1457     return Retain<PythonFile>(pythonio->GetPythonObject());
1458
1459   if (!mode) {
1460     auto m = file.GetOpenMode();
1461     if (!m)
1462       return m.takeError();
1463     mode = m.get();
1464   }
1465
1466   PyObject *file_obj;
1467   file_obj = PyFile_FromFd(file.GetDescriptor(), nullptr, mode, -1, nullptr,
1468                            "ignore", nullptr, /*closefd=*/0);
1469
1470   if (!file_obj)
1471     return exception();
1472
1473   return Take<PythonFile>(file_obj);
1474 }
1475
1476 Error PythonScript::Init() {
1477   if (function.IsValid())
1478     return Error::success();
1479
1480   PythonDictionary globals(PyInitialValue::Empty);
1481   auto builtins = PythonModule::BuiltinsModule();
1482   if (Error error = globals.SetItem("__builtins__", builtins))
1483     return error;
1484   PyObject *o =
1485       PyRun_String(script, Py_file_input, globals.get(), globals.get());
1486   if (!o)
1487     return exception();
1488   Take<PythonObject>(o);
1489   auto f = As<PythonCallable>(globals.GetItem("main"));
1490   if (!f)
1491     return f.takeError();
1492   function = std::move(f.get());
1493
1494   return Error::success();
1495 }
1496
1497 llvm::Expected<PythonObject>
1498 python::runStringOneLine(const llvm::Twine &string,
1499                          const PythonDictionary &globals,
1500                          const PythonDictionary &locals) {
1501   if (!globals.IsValid() || !locals.IsValid())
1502     return nullDeref();
1503
1504   PyObject *code =
1505       Py_CompileString(NullTerminated(string), "<string>", Py_eval_input);
1506   if (!code) {
1507     PyErr_Clear();
1508     code =
1509         Py_CompileString(NullTerminated(string), "<string>", Py_single_input);
1510   }
1511   if (!code)
1512     return exception();
1513   auto code_ref = Take<PythonObject>(code);
1514
1515   PyObject *result = PyEval_EvalCode(code, globals.get(), locals.get());
1516
1517   if (!result)
1518     return exception();
1519
1520   return Take<PythonObject>(result);
1521 }
1522
1523 llvm::Expected<PythonObject>
1524 python::runStringMultiLine(const llvm::Twine &string,
1525                            const PythonDictionary &globals,
1526                            const PythonDictionary &locals) {
1527   if (!globals.IsValid() || !locals.IsValid())
1528     return nullDeref();
1529   PyObject *result = PyRun_String(NullTerminated(string), Py_file_input,
1530                                   globals.get(), locals.get());
1531   if (!result)
1532     return exception();
1533   return Take<PythonObject>(result);
1534 }
1535
1536 #endif