src/pl/plpython/plpy_typeio.c

   1 /*
   2  * transforming Datums to Python objects and vice versa
   3  *
   4  * src/pl/plpython/plpy_typeio.c
   5  */
   6
   7 #include "postgres.h"
   8
   9 #include "access/htup_details.h"
  10 #include "catalog/pg_type.h"
  11 #include "funcapi.h"
  12 #include "mb/pg_wchar.h"
  13 #include "miscadmin.h"
  14 #include "plpy_elog.h"
  15 #include "plpy_main.h"
  16 #include "plpy_typeio.h"
  17 #include "plpython.h"
  18 #include "utils/array.h"
  19 #include "utils/builtins.h"
  20 #include "utils/fmgroids.h"
  21 #include "utils/lsyscache.h"
  22 #include "utils/memutils.h"
  23
  24 /* conversion from Datums to Python objects */
  25 static PyObject *PLyBool_FromBool(PLyDatumToOb *arg, Datum d);
  26 static PyObject *PLyFloat_FromFloat4(PLyDatumToOb *arg, Datum d);
  27 static PyObject *PLyFloat_FromFloat8(PLyDatumToOb *arg, Datum d);
  28 static PyObject *PLyDecimal_FromNumeric(PLyDatumToOb *arg, Datum d);
  29 static PyObject *PLyLong_FromInt16(PLyDatumToOb *arg, Datum d);
  30 static PyObject *PLyLong_FromInt32(PLyDatumToOb *arg, Datum d);
  31 static PyObject *PLyLong_FromInt64(PLyDatumToOb *arg, Datum d);
  32 static PyObject *PLyLong_FromOid(PLyDatumToOb *arg, Datum d);
  33 static PyObject *PLyBytes_FromBytea(PLyDatumToOb *arg, Datum d);
  34 static PyObject *PLyUnicode_FromScalar(PLyDatumToOb *arg, Datum d);
  35 static PyObject *PLyObject_FromTransform(PLyDatumToOb *arg, Datum d);
  36 static PyObject *PLyList_FromArray(PLyDatumToOb *arg, Datum d);
  37 static PyObject *PLyList_FromArray_recurse(PLyDatumToOb *elm, int *dims, int ndim, int dim,
  38                                                                                    char **dataptr_p, bits8 **bitmap_p, int *bitmask_p);
  39 static PyObject *PLyDict_FromComposite(PLyDatumToOb *arg, Datum d);
  40 static PyObject *PLyDict_FromTuple(PLyDatumToOb *arg, HeapTuple tuple, TupleDesc desc, bool include_generated);
  41
  42 /* conversion from Python objects to Datums */
  43 static Datum PLyObject_ToBool(PLyObToDatum *arg, PyObject *plrv,
  44                                                           bool *isnull, bool inarray);
  45 static Datum PLyObject_ToBytea(PLyObToDatum *arg, PyObject *plrv,
  46                                                            bool *isnull, bool inarray);
  47 static Datum PLyObject_ToComposite(PLyObToDatum *arg, PyObject *plrv,
  48                                                                    bool *isnull, bool inarray);
  49 static Datum PLyObject_ToScalar(PLyObToDatum *arg, PyObject *plrv,
  50                                                                 bool *isnull, bool inarray);
  51 static Datum PLyObject_ToDomain(PLyObToDatum *arg, PyObject *plrv,
  52                                                                 bool *isnull, bool inarray);
  53 static Datum PLyObject_ToTransform(PLyObToDatum *arg, PyObject *plrv,
  54                                                                    bool *isnull, bool inarray);
  55 static Datum PLySequence_ToArray(PLyObToDatum *arg, PyObject *plrv,
  56                                                                  bool *isnull, bool inarray);
  57 static void PLySequence_ToArray_recurse(PyObject *obj,
  58                                                                                 ArrayBuildState **astatep,
  59                                                                                 int *ndims, int *dims, int cur_depth,
  60                                                                                 PLyObToDatum *elm, Oid elmbasetype);
  61
  62 /* conversion from Python objects to composite Datums */
  63 static Datum PLyUnicode_ToComposite(PLyObToDatum *arg, PyObject *string, bool inarray);
  64 static Datum PLyMapping_ToComposite(PLyObToDatum *arg, TupleDesc desc, PyObject *mapping);
  65 static Datum PLySequence_ToComposite(PLyObToDatum *arg, TupleDesc desc, PyObject *sequence);
  66 static Datum PLyGenericObject_ToComposite(PLyObToDatum *arg, TupleDesc desc, PyObject *object, bool inarray);
  67
  68
  69 /*
  70  * Conversion functions.  Remember output from Python is input to
  71  * PostgreSQL, and vice versa.
  72  */
  73
  74 /*
  75  * Perform input conversion, given correctly-set-up state information.
  76  *
  77  * This is the outer-level entry point for any input conversion.  Internally,
  78  * the conversion functions recurse directly to each other.
  79  */
  80 PyObject *
  81 PLy_input_convert(PLyDatumToOb *arg, Datum val)
  82 {
  83         PyObject   *result;
  84         PLyExecutionContext *exec_ctx = PLy_current_execution_context();
  85         MemoryContext scratch_context = PLy_get_scratch_context(exec_ctx);
  86         MemoryContext oldcontext;
  87
  88         /*
  89          * Do the work in the scratch context to avoid leaking memory from the
  90          * datatype output function calls.  (The individual PLyDatumToObFunc
  91          * functions can't reset the scratch context, because they recurse and an
  92          * inner one might clobber data an outer one still needs.  So we do it
  93          * once at the outermost recursion level.)
  94          *
  95          * We reset the scratch context before, not after, each conversion cycle.
  96          * This way we aren't on the hook to release a Python refcount on the
  97          * result object in case MemoryContextReset throws an error.
  98          */
  99         MemoryContextReset(scratch_context);
 100
 101         oldcontext = MemoryContextSwitchTo(scratch_context);
 102
 103         result = arg->func(arg, val);
 104
 105         MemoryContextSwitchTo(oldcontext);
 106
 107         return result;
 108 }
 109
 110 /*
 111  * Perform output conversion, given correctly-set-up state information.
 112  *
 113  * This is the outer-level entry point for any output conversion.  Internally,
 114  * the conversion functions recurse directly to each other.
 115  *
 116  * The result, as well as any cruft generated along the way, are in the
 117  * current memory context.  Caller is responsible for cleanup.
 118  */
 119 Datum
 120 PLy_output_convert(PLyObToDatum *arg, PyObject *val, bool *isnull)
 121 {
 122         /* at outer level, we are not considering an array element */
 123         return arg->func(arg, val, isnull, false);
 124 }
 125
 126 /*
 127  * Transform a tuple into a Python dict object.
 128  *
 129  * Note: the tupdesc must match the one used to set up *arg.  We could
 130  * insist that this function lookup the tupdesc from what is in *arg,
 131  * but in practice all callers have the right tupdesc available.
 132  */
 133 PyObject *
 134 PLy_input_from_tuple(PLyDatumToOb *arg, HeapTuple tuple, TupleDesc desc, bool include_generated)
 135 {
 136         PyObject   *dict;
 137         PLyExecutionContext *exec_ctx = PLy_current_execution_context();
 138         MemoryContext scratch_context = PLy_get_scratch_context(exec_ctx);
 139         MemoryContext oldcontext;
 140
 141         /*
 142          * As in PLy_input_convert, do the work in the scratch context.
 143          */
 144         MemoryContextReset(scratch_context);
 145
 146         oldcontext = MemoryContextSwitchTo(scratch_context);
 147
 148         dict = PLyDict_FromTuple(arg, tuple, desc, include_generated);
 149
 150         MemoryContextSwitchTo(oldcontext);
 151
 152         return dict;
 153 }
 154
 155 /*
 156  * Initialize, or re-initialize, per-column input info for a composite type.
 157  *
 158  * This is separate from PLy_input_setup_func() because in cases involving
 159  * anonymous record types, we need to be passed the tupdesc explicitly.
 160  * It's caller's responsibility that the tupdesc has adequate lifespan
 161  * in such cases.  If the tupdesc is for a named composite or registered
 162  * record type, it does not need to be long-lived.
 163  */
 164 void
 165 PLy_input_setup_tuple(PLyDatumToOb *arg, TupleDesc desc, PLyProcedure *proc)
 166 {
 167         int                     i;
 168
 169         /* We should be working on a previously-set-up struct */
 170         Assert(arg->func == PLyDict_FromComposite);
 171
 172         /* Save pointer to tupdesc, but only if this is an anonymous record type */
 173         if (arg->typoid == RECORDOID && arg->typmod < 0)
 174                 arg->u.tuple.recdesc = desc;
 175
 176         /* (Re)allocate atts array as needed */
 177         if (arg->u.tuple.natts != desc->natts)
 178         {
 179                 if (arg->u.tuple.atts)
 180                         pfree(arg->u.tuple.atts);
 181                 arg->u.tuple.natts = desc->natts;
 182                 arg->u.tuple.atts = (PLyDatumToOb *)
 183                         MemoryContextAllocZero(arg->mcxt,
 184                                                                    desc->natts * sizeof(PLyDatumToOb));
 185         }
 186
 187         /* Fill the atts entries, except for dropped columns */
 188         for (i = 0; i < desc->natts; i++)
 189         {
 190                 Form_pg_attribute attr = TupleDescAttr(desc, i);
 191                 PLyDatumToOb *att = &arg->u.tuple.atts[i];
 192
 193                 if (attr->attisdropped)
 194                         continue;
 195
 196                 if (att->typoid == attr->atttypid && att->typmod == attr->atttypmod)
 197                         continue;                       /* already set up this entry */
 198
 199                 PLy_input_setup_func(att, arg->mcxt,
 200                                                          attr->atttypid, attr->atttypmod,
 201                                                          proc);
 202         }
 203 }
 204
 205 /*
 206  * Initialize, or re-initialize, per-column output info for a composite type.
 207  *
 208  * This is separate from PLy_output_setup_func() because in cases involving
 209  * anonymous record types, we need to be passed the tupdesc explicitly.
 210  * It's caller's responsibility that the tupdesc has adequate lifespan
 211  * in such cases.  If the tupdesc is for a named composite or registered
 212  * record type, it does not need to be long-lived.
 213  */
 214 void
 215 PLy_output_setup_tuple(PLyObToDatum *arg, TupleDesc desc, PLyProcedure *proc)
 216 {
 217         int                     i;
 218
 219         /* We should be working on a previously-set-up struct */
 220         Assert(arg->func == PLyObject_ToComposite);
 221
 222         /* Save pointer to tupdesc, but only if this is an anonymous record type */
 223         if (arg->typoid == RECORDOID && arg->typmod < 0)
 224                 arg->u.tuple.recdesc = desc;
 225
 226         /* (Re)allocate atts array as needed */
 227         if (arg->u.tuple.natts != desc->natts)
 228         {
 229                 if (arg->u.tuple.atts)
 230                         pfree(arg->u.tuple.atts);
 231                 arg->u.tuple.natts = desc->natts;
 232                 arg->u.tuple.atts = (PLyObToDatum *)
 233                         MemoryContextAllocZero(arg->mcxt,
 234                                                                    desc->natts * sizeof(PLyObToDatum));
 235         }
 236
 237         /* Fill the atts entries, except for dropped columns */
 238         for (i = 0; i < desc->natts; i++)
 239         {
 240                 Form_pg_attribute attr = TupleDescAttr(desc, i);
 241                 PLyObToDatum *att = &arg->u.tuple.atts[i];
 242
 243                 if (attr->attisdropped)
 244                         continue;
 245
 246                 if (att->typoid == attr->atttypid && att->typmod == attr->atttypmod)
 247                         continue;                       /* already set up this entry */
 248
 249                 PLy_output_setup_func(att, arg->mcxt,
 250                                                           attr->atttypid, attr->atttypmod,
 251                                                           proc);
 252         }
 253 }
 254
 255 /*
 256  * Set up output info for a PL/Python function returning record.
 257  *
 258  * Note: the given tupdesc is not necessarily long-lived.
 259  */
 260 void
 261 PLy_output_setup_record(PLyObToDatum *arg, TupleDesc desc, PLyProcedure *proc)
 262 {
 263         /* Makes no sense unless RECORD */
 264         Assert(arg->typoid == RECORDOID);
 265         Assert(desc->tdtypeid == RECORDOID);
 266
 267         /*
 268          * Bless the record type if not already done.  We'd have to do this anyway
 269          * to return a tuple, so we might as well force the issue so we can use
 270          * the known-record-type code path.
 271          */
 272         BlessTupleDesc(desc);
 273
 274         /*
 275          * Update arg->typmod, and clear the recdesc link if it's changed. The
 276          * next call of PLyObject_ToComposite will look up a long-lived tupdesc
 277          * for the record type.
 278          */
 279         arg->typmod = desc->tdtypmod;
 280         if (arg->u.tuple.recdesc &&
 281                 arg->u.tuple.recdesc->tdtypmod != arg->typmod)
 282                 arg->u.tuple.recdesc = NULL;
 283
 284         /* Update derived data if necessary */
 285         PLy_output_setup_tuple(arg, desc, proc);
 286 }
 287
 288 /*
 289  * Recursively initialize the PLyObToDatum structure(s) needed to construct
 290  * a SQL value of the specified typeOid/typmod from a Python value.
 291  * (But note that at this point we may have RECORDOID/-1, ie, an indeterminate
 292  * record type.)
 293  * proc is used to look up transform functions.
 294  */
 295 void
 296 PLy_output_setup_func(PLyObToDatum *arg, MemoryContext arg_mcxt,
 297                                           Oid typeOid, int32 typmod,
 298                                           PLyProcedure *proc)
 299 {
 300         TypeCacheEntry *typentry;
 301         char            typtype;
 302         Oid                     trfuncid;
 303         Oid                     typinput;
 304
 305         /* Since this is recursive, it could theoretically be driven to overflow */
 306         check_stack_depth();
 307
 308         arg->typoid = typeOid;
 309         arg->typmod = typmod;
 310         arg->mcxt = arg_mcxt;
 311
 312         /*
 313          * Fetch typcache entry for the target type, asking for whatever info
 314          * we'll need later.  RECORD is a special case: just treat it as composite
 315          * without bothering with the typcache entry.
 316          */
 317         if (typeOid != RECORDOID)
 318         {
 319                 typentry = lookup_type_cache(typeOid, TYPECACHE_DOMAIN_BASE_INFO);
 320                 typtype = typentry->typtype;
 321                 arg->typbyval = typentry->typbyval;
 322                 arg->typlen = typentry->typlen;
 323                 arg->typalign = typentry->typalign;
 324         }
 325         else
 326         {
 327                 typentry = NULL;
 328                 typtype = TYPTYPE_COMPOSITE;
 329                 /* hard-wired knowledge about type RECORD: */
 330                 arg->typbyval = false;
 331                 arg->typlen = -1;
 332                 arg->typalign = TYPALIGN_DOUBLE;
 333         }
 334
 335         /*
 336          * Choose conversion method.  Note that transform functions are checked
 337          * for composite and scalar types, but not for arrays or domains.  This is
 338          * somewhat historical, but we'd have a problem allowing them on domains,
 339          * since we drill down through all levels of a domain nest without looking
 340          * at the intermediate levels at all.
 341          */
 342         if (typtype == TYPTYPE_DOMAIN)
 343         {
 344                 /* Domain */
 345                 arg->func = PLyObject_ToDomain;
 346                 arg->u.domain.domain_info = NULL;
 347                 /* Recursively set up conversion info for the element type */
 348                 arg->u.domain.base = (PLyObToDatum *)
 349                         MemoryContextAllocZero(arg_mcxt, sizeof(PLyObToDatum));
 350                 PLy_output_setup_func(arg->u.domain.base, arg_mcxt,
 351                                                           typentry->domainBaseType,
 352                                                           typentry->domainBaseTypmod,
 353                                                           proc);
 354         }
 355         else if (typentry &&
 356                          IsTrueArrayType(typentry))
 357         {
 358                 /* Standard array */
 359                 arg->func = PLySequence_ToArray;
 360                 /* Get base type OID to insert into constructed array */
 361                 /* (note this might not be the same as the immediate child type) */
 362                 arg->u.array.elmbasetype = getBaseType(typentry->typelem);
 363                 /* Recursively set up conversion info for the element type */
 364                 arg->u.array.elm = (PLyObToDatum *)
 365                         MemoryContextAllocZero(arg_mcxt, sizeof(PLyObToDatum));
 366                 PLy_output_setup_func(arg->u.array.elm, arg_mcxt,
 367                                                           typentry->typelem, typmod,
 368                                                           proc);
 369         }
 370         else if ((trfuncid = get_transform_tosql(typeOid,
 371                                                                                          proc->langid,
 372                                                                                          proc->trftypes)))
 373         {
 374                 arg->func = PLyObject_ToTransform;
 375                 fmgr_info_cxt(trfuncid, &arg->u.transform.typtransform, arg_mcxt);
 376         }
 377         else if (typtype == TYPTYPE_COMPOSITE)
 378         {
 379                 /* Named composite type, or RECORD */
 380                 arg->func = PLyObject_ToComposite;
 381                 /* We'll set up the per-field data later */
 382                 arg->u.tuple.recdesc = NULL;
 383                 arg->u.tuple.typentry = typentry;
 384                 arg->u.tuple.tupdescid = INVALID_TUPLEDESC_IDENTIFIER;
 385                 arg->u.tuple.atts = NULL;
 386                 arg->u.tuple.natts = 0;
 387                 /* Mark this invalid till needed, too */
 388                 arg->u.tuple.recinfunc.fn_oid = InvalidOid;
 389         }
 390         else
 391         {
 392                 /* Scalar type, but we have a couple of special cases */
 393                 switch (typeOid)
 394                 {
 395                         case BOOLOID:
 396                                 arg->func = PLyObject_ToBool;
 397                                 break;
 398                         case BYTEAOID:
 399                                 arg->func = PLyObject_ToBytea;
 400                                 break;
 401                         default:
 402                                 arg->func = PLyObject_ToScalar;
 403                                 getTypeInputInfo(typeOid, &typinput, &arg->u.scalar.typioparam);
 404                                 fmgr_info_cxt(typinput, &arg->u.scalar.typfunc, arg_mcxt);
 405                                 break;
 406                 }
 407         }
 408 }
 409
 410 /*
 411  * Recursively initialize the PLyDatumToOb structure(s) needed to construct
 412  * a Python value from a SQL value of the specified typeOid/typmod.
 413  * (But note that at this point we may have RECORDOID/-1, ie, an indeterminate
 414  * record type.)
 415  * proc is used to look up transform functions.
 416  */
 417 void
 418 PLy_input_setup_func(PLyDatumToOb *arg, MemoryContext arg_mcxt,
 419                                          Oid typeOid, int32 typmod,
 420                                          PLyProcedure *proc)
 421 {
 422         TypeCacheEntry *typentry;
 423         char            typtype;
 424         Oid                     trfuncid;
 425         Oid                     typoutput;
 426         bool            typisvarlena;
 427
 428         /* Since this is recursive, it could theoretically be driven to overflow */
 429         check_stack_depth();
 430
 431         arg->typoid = typeOid;
 432         arg->typmod = typmod;
 433         arg->mcxt = arg_mcxt;
 434
 435         /*
 436          * Fetch typcache entry for the target type, asking for whatever info
 437          * we'll need later.  RECORD is a special case: just treat it as composite
 438          * without bothering with the typcache entry.
 439          */
 440         if (typeOid != RECORDOID)
 441         {
 442                 typentry = lookup_type_cache(typeOid, TYPECACHE_DOMAIN_BASE_INFO);
 443                 typtype = typentry->typtype;
 444                 arg->typbyval = typentry->typbyval;
 445                 arg->typlen = typentry->typlen;
 446                 arg->typalign = typentry->typalign;
 447         }
 448         else
 449         {
 450                 typentry = NULL;
 451                 typtype = TYPTYPE_COMPOSITE;
 452                 /* hard-wired knowledge about type RECORD: */
 453                 arg->typbyval = false;
 454                 arg->typlen = -1;
 455                 arg->typalign = TYPALIGN_DOUBLE;
 456         }
 457
 458         /*
 459          * Choose conversion method.  Note that transform functions are checked
 460          * for composite and scalar types, but not for arrays or domains.  This is
 461          * somewhat historical, but we'd have a problem allowing them on domains,
 462          * since we drill down through all levels of a domain nest without looking
 463          * at the intermediate levels at all.
 464          */
 465         if (typtype == TYPTYPE_DOMAIN)
 466         {
 467                 /* Domain --- we don't care, just recurse down to the base type */
 468                 PLy_input_setup_func(arg, arg_mcxt,
 469                                                          typentry->domainBaseType,
 470                                                          typentry->domainBaseTypmod,
 471                                                          proc);
 472         }
 473         else if (typentry &&
 474                          IsTrueArrayType(typentry))
 475         {
 476                 /* Standard array */
 477                 arg->func = PLyList_FromArray;
 478                 /* Recursively set up conversion info for the element type */
 479                 arg->u.array.elm = (PLyDatumToOb *)
 480                         MemoryContextAllocZero(arg_mcxt, sizeof(PLyDatumToOb));
 481                 PLy_input_setup_func(arg->u.array.elm, arg_mcxt,
 482                                                          typentry->typelem, typmod,
 483                                                          proc);
 484         }
 485         else if ((trfuncid = get_transform_fromsql(typeOid,
 486                                                                                            proc->langid,
 487                                                                                            proc->trftypes)))
 488         {
 489                 arg->func = PLyObject_FromTransform;
 490                 fmgr_info_cxt(trfuncid, &arg->u.transform.typtransform, arg_mcxt);
 491         }
 492         else if (typtype == TYPTYPE_COMPOSITE)
 493         {
 494                 /* Named composite type, or RECORD */
 495                 arg->func = PLyDict_FromComposite;
 496                 /* We'll set up the per-field data later */
 497                 arg->u.tuple.recdesc = NULL;
 498                 arg->u.tuple.typentry = typentry;
 499                 arg->u.tuple.tupdescid = INVALID_TUPLEDESC_IDENTIFIER;
 500                 arg->u.tuple.atts = NULL;
 501                 arg->u.tuple.natts = 0;
 502         }
 503         else
 504         {
 505                 /* Scalar type, but we have a couple of special cases */
 506                 switch (typeOid)
 507                 {
 508                         case BOOLOID:
 509                                 arg->func = PLyBool_FromBool;
 510                                 break;
 511                         case FLOAT4OID:
 512                                 arg->func = PLyFloat_FromFloat4;
 513                                 break;
 514                         case FLOAT8OID:
 515                                 arg->func = PLyFloat_FromFloat8;
 516                                 break;
 517                         case NUMERICOID:
 518                                 arg->func = PLyDecimal_FromNumeric;
 519                                 break;
 520                         case INT2OID:
 521                                 arg->func = PLyLong_FromInt16;
 522                                 break;
 523                         case INT4OID:
 524                                 arg->func = PLyLong_FromInt32;
 525                                 break;
 526                         case INT8OID:
 527                                 arg->func = PLyLong_FromInt64;
 528                                 break;
 529                         case OIDOID:
 530                                 arg->func = PLyLong_FromOid;
 531                                 break;
 532                         case BYTEAOID:
 533                                 arg->func = PLyBytes_FromBytea;
 534                                 break;
 535                         default:
 536                                 arg->func = PLyUnicode_FromScalar;
 537                                 getTypeOutputInfo(typeOid, &typoutput, &typisvarlena);
 538                                 fmgr_info_cxt(typoutput, &arg->u.scalar.typfunc, arg_mcxt);
 539                                 break;
 540                 }
 541         }
 542 }
 543
 544
 545 /*
 546  * Special-purpose input converters.
 547  */
 548
 549 static PyObject *
 550 PLyBool_FromBool(PLyDatumToOb *arg, Datum d)
 551 {
 552         if (DatumGetBool(d))
 553                 Py_RETURN_TRUE;
 554         Py_RETURN_FALSE;
 555 }
 556
 557 static PyObject *
 558 PLyFloat_FromFloat4(PLyDatumToOb *arg, Datum d)
 559 {
 560         return PyFloat_FromDouble(DatumGetFloat4(d));
 561 }
 562
 563 static PyObject *
 564 PLyFloat_FromFloat8(PLyDatumToOb *arg, Datum d)
 565 {
 566         return PyFloat_FromDouble(DatumGetFloat8(d));
 567 }
 568
 569 static PyObject *
 570 PLyDecimal_FromNumeric(PLyDatumToOb *arg, Datum d)
 571 {
 572         static PyObject *decimal_constructor;
 573         char       *str;
 574         PyObject   *pyvalue;
 575
 576         /* Try to import cdecimal.  If it doesn't exist, fall back to decimal. */
 577         if (!decimal_constructor)
 578         {
 579                 PyObject   *decimal_module;
 580
 581                 decimal_module = PyImport_ImportModule("cdecimal");
 582                 if (!decimal_module)
 583                 {
 584                         PyErr_Clear();
 585                         decimal_module = PyImport_ImportModule("decimal");
 586                 }
 587                 if (!decimal_module)
 588                         PLy_elog(ERROR, "could not import a module for Decimal constructor");
 589
 590                 decimal_constructor = PyObject_GetAttrString(decimal_module, "Decimal");
 591                 if (!decimal_constructor)
 592                         PLy_elog(ERROR, "no Decimal attribute in module");
 593         }
 594
 595         str = DatumGetCString(DirectFunctionCall1(numeric_out, d));
 596         pyvalue = PyObject_CallFunction(decimal_constructor, "s", str);
 597         if (!pyvalue)
 598                 PLy_elog(ERROR, "conversion from numeric to Decimal failed");
 599
 600         return pyvalue;
 601 }
 602
 603 static PyObject *
 604 PLyLong_FromInt16(PLyDatumToOb *arg, Datum d)
 605 {
 606         return PyLong_FromLong(DatumGetInt16(d));
 607 }
 608
 609 static PyObject *
 610 PLyLong_FromInt32(PLyDatumToOb *arg, Datum d)
 611 {
 612         return PyLong_FromLong(DatumGetInt32(d));
 613 }
 614
 615 static PyObject *
 616 PLyLong_FromInt64(PLyDatumToOb *arg, Datum d)
 617 {
 618         return PyLong_FromLongLong(DatumGetInt64(d));
 619 }
 620
 621 static PyObject *
 622 PLyLong_FromOid(PLyDatumToOb *arg, Datum d)
 623 {
 624         return PyLong_FromUnsignedLong(DatumGetObjectId(d));
 625 }
 626
 627 static PyObject *
 628 PLyBytes_FromBytea(PLyDatumToOb *arg, Datum d)
 629 {
 630         text       *txt = DatumGetByteaPP(d);
 631         char       *str = VARDATA_ANY(txt);
 632         size_t          size = VARSIZE_ANY_EXHDR(txt);
 633
 634         return PyBytes_FromStringAndSize(str, size);
 635 }
 636
 637
 638 /*
 639  * Generic input conversion using a SQL type's output function.
 640  */
 641 static PyObject *
 642 PLyUnicode_FromScalar(PLyDatumToOb *arg, Datum d)
 643 {
 644         char       *x = OutputFunctionCall(&arg->u.scalar.typfunc, d);
 645         PyObject   *r = PLyUnicode_FromString(x);
 646
 647         pfree(x);
 648         return r;
 649 }
 650
 651 /*
 652  * Convert using a from-SQL transform function.
 653  */
 654 static PyObject *
 655 PLyObject_FromTransform(PLyDatumToOb *arg, Datum d)
 656 {
 657         Datum           t;
 658
 659         t = FunctionCall1(&arg->u.transform.typtransform, d);
 660         return (PyObject *) DatumGetPointer(t);
 661 }
 662
 663 /*
 664  * Convert a SQL array to a Python list.
 665  */
 666 static PyObject *
 667 PLyList_FromArray(PLyDatumToOb *arg, Datum d)
 668 {
 669         ArrayType  *array = DatumGetArrayTypeP(d);
 670         PLyDatumToOb *elm = arg->u.array.elm;
 671         int                     ndim;
 672         int                *dims;
 673         char       *dataptr;
 674         bits8      *bitmap;
 675         int                     bitmask;
 676
 677         if (ARR_NDIM(array) == 0)
 678                 return PyList_New(0);
 679
 680         /* Array dimensions and left bounds */
 681         ndim = ARR_NDIM(array);
 682         dims = ARR_DIMS(array);
 683         Assert(ndim <= MAXDIM);
 684
 685         /*
 686          * We iterate the SQL array in the physical order it's stored in the
 687          * datum. For example, for a 3-dimensional array the order of iteration
 688          * would be the following: [0,0,0] elements through [0,0,k], then [0,1,0]
 689          * through [0,1,k] till [0,m,k], then [1,0,0] through [1,0,k] till
 690          * [1,m,k], and so on.
 691          *
 692          * In Python, there are no multi-dimensional lists as such, but they are
 693          * represented as a list of lists. So a 3-d array of [n,m,k] elements is a
 694          * list of n m-element arrays, each element of which is k-element array.
 695          * PLyList_FromArray_recurse() builds the Python list for a single
 696          * dimension, and recurses for the next inner dimension.
 697          */
 698         dataptr = ARR_DATA_PTR(array);
 699         bitmap = ARR_NULLBITMAP(array);
 700         bitmask = 1;
 701
 702         return PLyList_FromArray_recurse(elm, dims, ndim, 0,
 703                                                                          &dataptr, &bitmap, &bitmask);
 704 }
 705
 706 static PyObject *
 707 PLyList_FromArray_recurse(PLyDatumToOb *elm, int *dims, int ndim, int dim,
 708                                                   char **dataptr_p, bits8 **bitmap_p, int *bitmask_p)
 709 {
 710         int                     i;
 711         PyObject   *list;
 712
 713         list = PyList_New(dims[dim]);
 714         if (!list)
 715                 return NULL;
 716
 717         if (dim < ndim - 1)
 718         {
 719                 /* Outer dimension. Recurse for each inner slice. */
 720                 for (i = 0; i < dims[dim]; i++)
 721                 {
 722                         PyObject   *sublist;
 723
 724                         sublist = PLyList_FromArray_recurse(elm, dims, ndim, dim + 1,
 725                                                                                                 dataptr_p, bitmap_p, bitmask_p);
 726                         PyList_SET_ITEM(list, i, sublist);
 727                 }
 728         }
 729         else
 730         {
 731                 /*
 732                  * Innermost dimension. Fill the list with the values from the array
 733                  * for this slice.
 734                  */
 735                 char       *dataptr = *dataptr_p;
 736                 bits8      *bitmap = *bitmap_p;
 737                 int                     bitmask = *bitmask_p;
 738
 739                 for (i = 0; i < dims[dim]; i++)
 740                 {
 741                         /* checking for NULL */
 742                         if (bitmap && (*bitmap & bitmask) == 0)
 743                         {
 744                                 Py_INCREF(Py_None);
 745                                 PyList_SET_ITEM(list, i, Py_None);
 746                         }
 747                         else
 748                         {
 749                                 Datum           itemvalue;
 750
 751                                 itemvalue = fetch_att(dataptr, elm->typbyval, elm->typlen);
 752                                 PyList_SET_ITEM(list, i, elm->func(elm, itemvalue));
 753                                 dataptr = att_addlength_pointer(dataptr, elm->typlen, dataptr);
 754                                 dataptr = (char *) att_align_nominal(dataptr, elm->typalign);
 755                         }
 756
 757                         /* advance bitmap pointer if any */
 758                         if (bitmap)
 759                         {
 760                                 bitmask <<= 1;
 761                                 if (bitmask == 0x100 /* (1<<8) */ )
 762                                 {
 763                                         bitmap++;
 764                                         bitmask = 1;
 765                                 }
 766                         }
 767                 }
 768
 769                 *dataptr_p = dataptr;
 770                 *bitmap_p = bitmap;
 771                 *bitmask_p = bitmask;
 772         }
 773
 774         return list;
 775 }
 776
 777 /*
 778  * Convert a composite SQL value to a Python dict.
 779  */
 780 static PyObject *
 781 PLyDict_FromComposite(PLyDatumToOb *arg, Datum d)
 782 {
 783         PyObject   *dict;
 784         HeapTupleHeader td;
 785         Oid                     tupType;
 786         int32           tupTypmod;
 787         TupleDesc       tupdesc;
 788         HeapTupleData tmptup;
 789
 790         td = DatumGetHeapTupleHeader(d);
 791         /* Extract rowtype info and find a tupdesc */
 792         tupType = HeapTupleHeaderGetTypeId(td);
 793         tupTypmod = HeapTupleHeaderGetTypMod(td);
 794         tupdesc = lookup_rowtype_tupdesc(tupType, tupTypmod);
 795
 796         /* Set up I/O funcs if not done yet */
 797         PLy_input_setup_tuple(arg, tupdesc,
 798                                                   PLy_current_execution_context()->curr_proc);
 799
 800         /* Build a temporary HeapTuple control structure */
 801         tmptup.t_len = HeapTupleHeaderGetDatumLength(td);
 802         tmptup.t_data = td;
 803
 804         dict = PLyDict_FromTuple(arg, &tmptup, tupdesc, true);
 805
 806         ReleaseTupleDesc(tupdesc);
 807
 808         return dict;
 809 }
 810
 811 /*
 812  * Transform a tuple into a Python dict object.
 813  */
 814 static PyObject *
 815 PLyDict_FromTuple(PLyDatumToOb *arg, HeapTuple tuple, TupleDesc desc, bool include_generated)
 816 {
 817         PyObject   *volatile dict;
 818
 819         /* Simple sanity check that desc matches */
 820         Assert(desc->natts == arg->u.tuple.natts);
 821
 822         dict = PyDict_New();
 823         if (dict == NULL)
 824                 return NULL;
 825
 826         PG_TRY();
 827         {
 828                 int                     i;
 829
 830                 for (i = 0; i < arg->u.tuple.natts; i++)
 831                 {
 832                         PLyDatumToOb *att = &arg->u.tuple.atts[i];
 833                         Form_pg_attribute attr = TupleDescAttr(desc, i);
 834                         char       *key;
 835                         Datum           vattr;
 836                         bool            is_null;
 837                         PyObject   *value;
 838
 839                         if (attr->attisdropped)
 840                                 continue;
 841
 842                         if (attr->attgenerated)
 843                         {
 844                                 /* don't include unless requested */
 845                                 if (!include_generated)
 846                                         continue;
 847                         }
 848
 849                         key = NameStr(attr->attname);
 850                         vattr = heap_getattr(tuple, (i + 1), desc, &is_null);
 851
 852                         if (is_null)
 853                                 PyDict_SetItemString(dict, key, Py_None);
 854                         else
 855                         {
 856                                 value = att->func(att, vattr);
 857                                 PyDict_SetItemString(dict, key, value);
 858                                 Py_DECREF(value);
 859                         }
 860                 }
 861         }
 862         PG_CATCH();
 863         {
 864                 Py_DECREF(dict);
 865                 PG_RE_THROW();
 866         }
 867         PG_END_TRY();
 868
 869         return dict;
 870 }
 871
 872 /*
 873  * Convert a Python object to a PostgreSQL bool datum.  This can't go
 874  * through the generic conversion function, because Python attaches a
 875  * Boolean value to everything, more things than the PostgreSQL bool
 876  * type can parse.
 877  */
 878 static Datum
 879 PLyObject_ToBool(PLyObToDatum *arg, PyObject *plrv,
 880                                  bool *isnull, bool inarray)
 881 {
 882         if (plrv == Py_None)
 883         {
 884                 *isnull = true;
 885                 return (Datum) 0;
 886         }
 887         *isnull = false;
 888         return BoolGetDatum(PyObject_IsTrue(plrv));
 889 }
 890
 891 /*
 892  * Convert a Python object to a PostgreSQL bytea datum.  This doesn't
 893  * go through the generic conversion function to circumvent problems
 894  * with embedded nulls.  And it's faster this way.
 895  */
 896 static Datum
 897 PLyObject_ToBytea(PLyObToDatum *arg, PyObject *plrv,
 898                                   bool *isnull, bool inarray)
 899 {
 900         PyObject   *volatile plrv_so = NULL;
 901         Datum           rv = (Datum) 0;
 902
 903         if (plrv == Py_None)
 904         {
 905                 *isnull = true;
 906                 return (Datum) 0;
 907         }
 908         *isnull = false;
 909
 910         plrv_so = PyObject_Bytes(plrv);
 911         if (!plrv_so)
 912                 PLy_elog(ERROR, "could not create bytes representation of Python object");
 913
 914         PG_TRY();
 915         {
 916                 char       *plrv_sc = PyBytes_AsString(plrv_so);
 917                 size_t          len = PyBytes_Size(plrv_so);
 918                 size_t          size = len + VARHDRSZ;
 919                 bytea      *result = palloc(size);
 920
 921                 SET_VARSIZE(result, size);
 922                 memcpy(VARDATA(result), plrv_sc, len);
 923                 rv = PointerGetDatum(result);
 924         }
 925         PG_FINALLY();
 926         {
 927                 Py_XDECREF(plrv_so);
 928         }
 929         PG_END_TRY();
 930
 931         return rv;
 932 }
 933
 934
 935 /*
 936  * Convert a Python object to a composite type. First look up the type's
 937  * description, then route the Python object through the conversion function
 938  * for obtaining PostgreSQL tuples.
 939  */
 940 static Datum
 941 PLyObject_ToComposite(PLyObToDatum *arg, PyObject *plrv,
 942                                           bool *isnull, bool inarray)
 943 {
 944         Datum           rv;
 945         TupleDesc       desc;
 946
 947         if (plrv == Py_None)
 948         {
 949                 *isnull = true;
 950                 return (Datum) 0;
 951         }
 952         *isnull = false;
 953
 954         /*
 955          * The string conversion case doesn't require a tupdesc, nor per-field
 956          * conversion data, so just go for it if that's the case to use.
 957          */
 958         if (PyUnicode_Check(plrv))
 959                 return PLyUnicode_ToComposite(arg, plrv, inarray);
 960
 961         /*
 962          * If we're dealing with a named composite type, we must look up the
 963          * tupdesc every time, to protect against possible changes to the type.
 964          * RECORD types can't change between calls; but we must still be willing
 965          * to set up the info the first time, if nobody did yet.
 966          */
 967         if (arg->typoid != RECORDOID)
 968         {
 969                 desc = lookup_rowtype_tupdesc(arg->typoid, arg->typmod);
 970                 /* We should have the descriptor of the type's typcache entry */
 971                 Assert(desc == arg->u.tuple.typentry->tupDesc);
 972                 /* Detect change of descriptor, update cache if needed */
 973                 if (arg->u.tuple.tupdescid != arg->u.tuple.typentry->tupDesc_identifier)
 974                 {
 975                         PLy_output_setup_tuple(arg, desc,
 976                                                                    PLy_current_execution_context()->curr_proc);
 977                         arg->u.tuple.tupdescid = arg->u.tuple.typentry->tupDesc_identifier;
 978                 }
 979         }
 980         else
 981         {
 982                 desc = arg->u.tuple.recdesc;
 983                 if (desc == NULL)
 984                 {
 985                         desc = lookup_rowtype_tupdesc(arg->typoid, arg->typmod);
 986                         arg->u.tuple.recdesc = desc;
 987                 }
 988                 else
 989                 {
 990                         /* Pin descriptor to match unpin below */
 991                         PinTupleDesc(desc);
 992                 }
 993         }
 994
 995         /* Simple sanity check on our caching */
 996         Assert(desc->natts == arg->u.tuple.natts);
 997
 998         /*
 999          * Convert, using the appropriate method depending on the type of the
1000          * supplied Python object.
1001          */
1002         if (PySequence_Check(plrv))
1003                 /* composite type as sequence (tuple, list etc) */
1004                 rv = PLySequence_ToComposite(arg, desc, plrv);
1005         else if (PyMapping_Check(plrv))
1006                 /* composite type as mapping (currently only dict) */
1007                 rv = PLyMapping_ToComposite(arg, desc, plrv);
1008         else
1009                 /* returned as smth, must provide method __getattr__(name) */
1010                 rv = PLyGenericObject_ToComposite(arg, desc, plrv, inarray);
1011
1012         ReleaseTupleDesc(desc);
1013
1014         return rv;
1015 }
1016
1017
1018 /*
1019  * Convert Python object to C string in server encoding.
1020  *
1021  * Note: this is exported for use by add-on transform modules.
1022  */
1023 char *
1024 PLyObject_AsString(PyObject *plrv)
1025 {
1026         PyObject   *plrv_bo;
1027         char       *plrv_sc;
1028         size_t          plen;
1029         size_t          slen;
1030
1031         if (PyUnicode_Check(plrv))
1032                 plrv_bo = PLyUnicode_Bytes(plrv);
1033         else if (PyFloat_Check(plrv))
1034         {
1035                 /* use repr() for floats, str() is lossy */
1036                 PyObject   *s = PyObject_Repr(plrv);
1037
1038                 plrv_bo = PLyUnicode_Bytes(s);
1039                 Py_XDECREF(s);
1040         }
1041         else
1042         {
1043                 PyObject   *s = PyObject_Str(plrv);
1044
1045                 plrv_bo = PLyUnicode_Bytes(s);
1046                 Py_XDECREF(s);
1047         }
1048         if (!plrv_bo)
1049                 PLy_elog(ERROR, "could not create string representation of Python object");
1050
1051         plrv_sc = pstrdup(PyBytes_AsString(plrv_bo));
1052         plen = PyBytes_Size(plrv_bo);
1053         slen = strlen(plrv_sc);
1054
1055         Py_XDECREF(plrv_bo);
1056
1057         if (slen < plen)
1058                 ereport(ERROR,
1059                                 (errcode(ERRCODE_DATATYPE_MISMATCH),
1060                                  errmsg("could not convert Python object into cstring: Python string representation appears to contain null bytes")));
1061         else if (slen > plen)
1062                 elog(ERROR, "could not convert Python object into cstring: Python string longer than reported length");
1063         pg_verifymbstr(plrv_sc, slen, false);
1064
1065         return plrv_sc;
1066 }
1067
1068
1069 /*
1070  * Generic output conversion function: convert PyObject to cstring and
1071  * cstring into PostgreSQL type.
1072  */
1073 static Datum
1074 PLyObject_ToScalar(PLyObToDatum *arg, PyObject *plrv,
1075                                    bool *isnull, bool inarray)
1076 {
1077         char       *str;
1078
1079         if (plrv == Py_None)
1080         {
1081                 *isnull = true;
1082                 return (Datum) 0;
1083         }
1084         *isnull = false;
1085
1086         str = PLyObject_AsString(plrv);
1087
1088         return InputFunctionCall(&arg->u.scalar.typfunc,
1089                                                          str,
1090                                                          arg->u.scalar.typioparam,
1091                                                          arg->typmod);
1092 }
1093
1094
1095 /*
1096  * Convert to a domain type.
1097  */
1098 static Datum
1099 PLyObject_ToDomain(PLyObToDatum *arg, PyObject *plrv,
1100                                    bool *isnull, bool inarray)
1101 {
1102         Datum           result;
1103         PLyObToDatum *base = arg->u.domain.base;
1104
1105         result = base->func(base, plrv, isnull, inarray);
1106         domain_check(result, *isnull, arg->typoid,
1107                                  &arg->u.domain.domain_info, arg->mcxt);
1108         return result;
1109 }
1110
1111
1112 /*
1113  * Convert using a to-SQL transform function.
1114  */
1115 static Datum
1116 PLyObject_ToTransform(PLyObToDatum *arg, PyObject *plrv,
1117                                           bool *isnull, bool inarray)
1118 {
1119         if (plrv == Py_None)
1120         {
1121                 *isnull = true;
1122                 return (Datum) 0;
1123         }
1124         *isnull = false;
1125         return FunctionCall1(&arg->u.transform.typtransform, PointerGetDatum(plrv));
1126 }
1127
1128
1129 /*
1130  * Convert Python sequence (or list of lists) to SQL array.
1131  */
1132 static Datum
1133 PLySequence_ToArray(PLyObToDatum *arg, PyObject *plrv,
1134                                         bool *isnull, bool inarray)
1135 {
1136         ArrayBuildState *astate = NULL;
1137         int                     ndims = 1;
1138         int                     dims[MAXDIM];
1139         int                     lbs[MAXDIM];
1140
1141         if (plrv == Py_None)
1142         {
1143                 *isnull = true;
1144                 return (Datum) 0;
1145         }
1146         *isnull = false;
1147
1148         /*
1149          * For historical reasons, we allow any sequence (not only a list) at the
1150          * top level when converting a Python object to a SQL array.  However, a
1151          * multi-dimensional array is recognized only when the object contains
1152          * true lists.
1153          */
1154         if (!PySequence_Check(plrv))
1155                 ereport(ERROR,
1156                                 (errcode(ERRCODE_DATATYPE_MISMATCH),
1157                                  errmsg("return value of function with array return type is not a Python sequence")));
1158
1159         /* Initialize dimensionality info with first-level dimension */
1160         memset(dims, 0, sizeof(dims));
1161         dims[0] = PySequence_Length(plrv);
1162
1163         /*
1164          * Traverse the Python lists, in depth-first order, and collect all the
1165          * elements at the bottom level into an ArrayBuildState.
1166          */
1167         PLySequence_ToArray_recurse(plrv, &astate,
1168                                                                 &ndims, dims, 1,
1169                                                                 arg->u.array.elm,
1170                                                                 arg->u.array.elmbasetype);
1171
1172         /* ensure we get zero-D array for no inputs, as per PG convention */
1173         if (astate == NULL)
1174                 return PointerGetDatum(construct_empty_array(arg->u.array.elmbasetype));
1175
1176         for (int i = 0; i < ndims; i++)
1177                 lbs[i] = 1;
1178
1179         return makeMdArrayResult(astate, ndims, dims, lbs,
1180                                                          CurrentMemoryContext, true);
1181 }
1182
1183 /*
1184  * Helper function for PLySequence_ToArray. Traverse a Python list of lists in
1185  * depth-first order, storing the elements in *astatep.
1186  *
1187  * The ArrayBuildState is created only when we first find a scalar element;
1188  * if we didn't do it like that, we'd need some other convention for knowing
1189  * whether we'd already found any scalars (and thus the number of dimensions
1190  * is frozen).
1191  */
1192 static void
1193 PLySequence_ToArray_recurse(PyObject *obj, ArrayBuildState **astatep,
1194                                                         int *ndims, int *dims, int cur_depth,
1195                                                         PLyObToDatum *elm, Oid elmbasetype)
1196 {
1197         int                     i;
1198         int                     len = PySequence_Length(obj);
1199
1200         /* We should not get here with a non-sequence object */
1201         if (len < 0)
1202                 PLy_elog(ERROR, "could not determine sequence length for function return value");
1203
1204         for (i = 0; i < len; i++)
1205         {
1206                 /* fetch the array element */
1207                 PyObject   *subobj = PySequence_GetItem(obj, i);
1208
1209                 /* need PG_TRY to ensure we release the subobj's refcount */
1210                 PG_TRY();
1211                 {
1212                         /* multi-dimensional array? */
1213                         if (PyList_Check(subobj))
1214                         {
1215                                 /* set size when at first element in this level, else compare */
1216                                 if (i == 0 && *ndims == cur_depth)
1217                                 {
1218                                         /* array after some scalars at same level? */
1219                                         if (*astatep != NULL)
1220                                                 ereport(ERROR,
1221                                                                 (errcode(ERRCODE_INVALID_TEXT_REPRESENTATION),
1222                                                                  errmsg("multidimensional arrays must have array expressions with matching dimensions")));
1223                                         /* too many dimensions? */
1224                                         if (cur_depth >= MAXDIM)
1225                                                 ereport(ERROR,
1226                                                                 (errcode(ERRCODE_PROGRAM_LIMIT_EXCEEDED),
1227                                                                  errmsg("number of array dimensions exceeds the maximum allowed (%d)",
1228                                                                                 MAXDIM)));
1229                                         /* OK, add a dimension */
1230                                         dims[*ndims] = PySequence_Length(subobj);
1231                                         (*ndims)++;
1232                                 }
1233                                 else if (cur_depth >= *ndims ||
1234                                                  PySequence_Length(subobj) != dims[cur_depth])
1235                                         ereport(ERROR,
1236                                                         (errcode(ERRCODE_INVALID_TEXT_REPRESENTATION),
1237                                                          errmsg("multidimensional arrays must have array expressions with matching dimensions")));
1238
1239                                 /* recurse to fetch elements of this sub-array */
1240                                 PLySequence_ToArray_recurse(subobj, astatep,
1241                                                                                         ndims, dims, cur_depth + 1,
1242                                                                                         elm, elmbasetype);
1243                         }
1244                         else
1245                         {
1246                                 Datum           dat;
1247                                 bool            isnull;
1248
1249                                 /* scalar after some sub-arrays at same level? */
1250                                 if (*ndims != cur_depth)
1251                                         ereport(ERROR,
1252                                                         (errcode(ERRCODE_INVALID_TEXT_REPRESENTATION),
1253                                                          errmsg("multidimensional arrays must have array expressions with matching dimensions")));
1254
1255                                 /* convert non-list object to Datum */
1256                                 dat = elm->func(elm, subobj, &isnull, true);
1257
1258                                 /* create ArrayBuildState if we didn't already */
1259                                 if (*astatep == NULL)
1260                                         *astatep = initArrayResult(elmbasetype,
1261                                                                                            CurrentMemoryContext, true);
1262
1263                                 /* ... and save the element value in it */
1264                                 (void) accumArrayResult(*astatep, dat, isnull,
1265                                                                                 elmbasetype, CurrentMemoryContext);
1266                         }
1267                 }
1268                 PG_FINALLY();
1269                 {
1270                         Py_XDECREF(subobj);
1271                 }
1272                 PG_END_TRY();
1273         }
1274 }
1275
1276
1277 /*
1278  * Convert a Python string to composite, using record_in.
1279  */
1280 static Datum
1281 PLyUnicode_ToComposite(PLyObToDatum *arg, PyObject *string, bool inarray)
1282 {
1283         char       *str;
1284
1285         /*
1286          * Set up call data for record_in, if we didn't already.  (We can't just
1287          * use DirectFunctionCall, because record_in needs a fn_extra field.)
1288          */
1289         if (!OidIsValid(arg->u.tuple.recinfunc.fn_oid))
1290                 fmgr_info_cxt(F_RECORD_IN, &arg->u.tuple.recinfunc, arg->mcxt);
1291
1292         str = PLyObject_AsString(string);
1293
1294         /*
1295          * If we are parsing a composite type within an array, and the string
1296          * isn't a valid record literal, there's a high chance that the function
1297          * did something like:
1298          *
1299          * CREATE FUNCTION .. RETURNS comptype[] AS $$ return [['foo', 'bar']] $$
1300          * LANGUAGE plpython;
1301          *
1302          * Before PostgreSQL 10, that was interpreted as a single-dimensional
1303          * array, containing record ('foo', 'bar'). PostgreSQL 10 added support
1304          * for multi-dimensional arrays, and it is now interpreted as a
1305          * two-dimensional array, containing two records, 'foo', and 'bar'.
1306          * record_in() will throw an error, because "foo" is not a valid record
1307          * literal.
1308          *
1309          * To make that less confusing to users who are upgrading from older
1310          * versions, try to give a hint in the typical instances of that. If we
1311          * are parsing an array of composite types, and we see a string literal
1312          * that is not a valid record literal, give a hint. We only want to give
1313          * the hint in the narrow case of a malformed string literal, not any
1314          * error from record_in(), so check for that case here specifically.
1315          *
1316          * This check better match the one in record_in(), so that we don't forbid
1317          * literals that are actually valid!
1318          */
1319         if (inarray)
1320         {
1321                 char       *ptr = str;
1322
1323                 /* Allow leading whitespace */
1324                 while (*ptr && isspace((unsigned char) *ptr))
1325                         ptr++;
1326                 if (*ptr++ != '(')
1327                         ereport(ERROR,
1328                                         (errcode(ERRCODE_INVALID_TEXT_REPRESENTATION),
1329                                          errmsg("malformed record literal: \"%s\"", str),
1330                                          errdetail("Missing left parenthesis."),
1331                                          errhint("To return a composite type in an array, return the composite type as a Python tuple, e.g., \"[('foo',)]\".")));
1332         }
1333
1334         return InputFunctionCall(&arg->u.tuple.recinfunc,
1335                                                          str,
1336                                                          arg->typoid,
1337                                                          arg->typmod);
1338 }
1339
1340
1341 static Datum
1342 PLyMapping_ToComposite(PLyObToDatum *arg, TupleDesc desc, PyObject *mapping)
1343 {
1344         Datum           result;
1345         HeapTuple       tuple;
1346         Datum      *values;
1347         bool       *nulls;
1348         volatile int i;
1349
1350         Assert(PyMapping_Check(mapping));
1351
1352         /* Build tuple */
1353         values = palloc(sizeof(Datum) * desc->natts);
1354         nulls = palloc(sizeof(bool) * desc->natts);
1355         for (i = 0; i < desc->natts; ++i)
1356         {
1357                 char       *key;
1358                 PyObject   *volatile value;
1359                 PLyObToDatum *att;
1360                 Form_pg_attribute attr = TupleDescAttr(desc, i);
1361
1362                 if (attr->attisdropped)
1363                 {
1364                         values[i] = (Datum) 0;
1365                         nulls[i] = true;
1366                         continue;
1367                 }
1368
1369                 key = NameStr(attr->attname);
1370                 value = NULL;
1371                 att = &arg->u.tuple.atts[i];
1372                 PG_TRY();
1373                 {
1374                         value = PyMapping_GetItemString(mapping, key);
1375                         if (!value)
1376                                 ereport(ERROR,
1377                                                 (errcode(ERRCODE_UNDEFINED_COLUMN),
1378                                                  errmsg("key \"%s\" not found in mapping", key),
1379                                                  errhint("To return null in a column, "
1380                                                                  "add the value None to the mapping with the key named after the column.")));
1381
1382                         values[i] = att->func(att, value, &nulls[i], false);
1383
1384                         Py_XDECREF(value);
1385                         value = NULL;
1386                 }
1387                 PG_CATCH();
1388                 {
1389                         Py_XDECREF(value);
1390                         PG_RE_THROW();
1391                 }
1392                 PG_END_TRY();
1393         }
1394
1395         tuple = heap_form_tuple(desc, values, nulls);
1396         result = heap_copy_tuple_as_datum(tuple, desc);
1397         heap_freetuple(tuple);
1398
1399         pfree(values);
1400         pfree(nulls);
1401
1402         return result;
1403 }
1404
1405
1406 static Datum
1407 PLySequence_ToComposite(PLyObToDatum *arg, TupleDesc desc, PyObject *sequence)
1408 {
1409         Datum           result;
1410         HeapTuple       tuple;
1411         Datum      *values;
1412         bool       *nulls;
1413         volatile int idx;
1414         volatile int i;
1415
1416         Assert(PySequence_Check(sequence));
1417
1418         /*
1419          * Check that sequence length is exactly same as PG tuple's. We actually
1420          * can ignore exceeding items or assume missing ones as null but to avoid
1421          * plpython developer's errors we are strict here
1422          */
1423         idx = 0;
1424         for (i = 0; i < desc->natts; i++)
1425         {
1426                 if (!TupleDescAttr(desc, i)->attisdropped)
1427                         idx++;
1428         }
1429         if (PySequence_Length(sequence) != idx)
1430                 ereport(ERROR,
1431                                 (errcode(ERRCODE_DATATYPE_MISMATCH),
1432                                  errmsg("length of returned sequence did not match number of columns in row")));
1433
1434         /* Build tuple */
1435         values = palloc(sizeof(Datum) * desc->natts);
1436         nulls = palloc(sizeof(bool) * desc->natts);
1437         idx = 0;
1438         for (i = 0; i < desc->natts; ++i)
1439         {
1440                 PyObject   *volatile value;
1441                 PLyObToDatum *att;
1442
1443                 if (TupleDescAttr(desc, i)->attisdropped)
1444                 {
1445                         values[i] = (Datum) 0;
1446                         nulls[i] = true;
1447                         continue;
1448                 }
1449
1450                 value = NULL;
1451                 att = &arg->u.tuple.atts[i];
1452                 PG_TRY();
1453                 {
1454                         value = PySequence_GetItem(sequence, idx);
1455                         Assert(value);
1456
1457                         values[i] = att->func(att, value, &nulls[i], false);
1458
1459                         Py_XDECREF(value);
1460                         value = NULL;
1461                 }
1462                 PG_CATCH();
1463                 {
1464                         Py_XDECREF(value);
1465                         PG_RE_THROW();
1466                 }
1467                 PG_END_TRY();
1468
1469                 idx++;
1470         }
1471
1472         tuple = heap_form_tuple(desc, values, nulls);
1473         result = heap_copy_tuple_as_datum(tuple, desc);
1474         heap_freetuple(tuple);
1475
1476         pfree(values);
1477         pfree(nulls);
1478
1479         return result;
1480 }
1481
1482
1483 static Datum
1484 PLyGenericObject_ToComposite(PLyObToDatum *arg, TupleDesc desc, PyObject *object, bool inarray)
1485 {
1486         Datum           result;
1487         HeapTuple       tuple;
1488         Datum      *values;
1489         bool       *nulls;
1490         volatile int i;
1491
1492         /* Build tuple */
1493         values = palloc(sizeof(Datum) * desc->natts);
1494         nulls = palloc(sizeof(bool) * desc->natts);
1495         for (i = 0; i < desc->natts; ++i)
1496         {
1497                 char       *key;
1498                 PyObject   *volatile value;
1499                 PLyObToDatum *att;
1500                 Form_pg_attribute attr = TupleDescAttr(desc, i);
1501
1502                 if (attr->attisdropped)
1503                 {
1504                         values[i] = (Datum) 0;
1505                         nulls[i] = true;
1506                         continue;
1507                 }
1508
1509                 key = NameStr(attr->attname);
1510                 value = NULL;
1511                 att = &arg->u.tuple.atts[i];
1512                 PG_TRY();
1513                 {
1514                         value = PyObject_GetAttrString(object, key);
1515                         if (!value)
1516                         {
1517                                 /*
1518                                  * No attribute for this column in the object.
1519                                  *
1520                                  * If we are parsing a composite type in an array, a likely
1521                                  * cause is that the function contained something like "[[123,
1522                                  * 'foo']]". Before PostgreSQL 10, that was interpreted as an
1523                                  * array, with a composite type (123, 'foo') in it. But now
1524                                  * it's interpreted as a two-dimensional array, and we try to
1525                                  * interpret "123" as the composite type. See also similar
1526                                  * heuristic in PLyObject_ToScalar().
1527                                  */
1528                                 ereport(ERROR,
1529                                                 (errcode(ERRCODE_UNDEFINED_COLUMN),
1530                                                  errmsg("attribute \"%s\" does not exist in Python object", key),
1531                                                  inarray ?
1532                                                  errhint("To return a composite type in an array, return the composite type as a Python tuple, e.g., \"[('foo',)]\".") :
1533                                                  errhint("To return null in a column, let the returned object have an attribute named after column with value None.")));
1534                         }
1535
1536                         values[i] = att->func(att, value, &nulls[i], false);
1537
1538                         Py_XDECREF(value);
1539                         value = NULL;
1540                 }
1541                 PG_CATCH();
1542                 {
1543                         Py_XDECREF(value);
1544                         PG_RE_THROW();
1545                 }
1546                 PG_END_TRY();
1547         }
1548
1549         tuple = heap_form_tuple(desc, values, nulls);
1550         result = heap_copy_tuple_as_datum(tuple, desc);
1551         heap_freetuple(tuple);
1552
1553         pfree(values);
1554         pfree(nulls);
1555
1556         return result;
1557 }