2 * transforming Datums to Python objects and vice versa
4 * src/pl/plpython/plpy_typeio.c
9 #include "access/htup_details.h"
10 #include "catalog/pg_type.h"
12 #include "mb/pg_wchar.h"
13 #include "miscadmin.h"
14 #include "plpy_elog.h"
15 #include "plpy_main.h"
16 #include "plpy_typeio.h"
18 #include "utils/array.h"
19 #include "utils/builtins.h"
20 #include "utils/fmgroids.h"
21 #include "utils/lsyscache.h"
22 #include "utils/memutils.h"
24 /* conversion from Datums to Python objects */
25 static PyObject
*PLyBool_FromBool(PLyDatumToOb
*arg
, Datum d
);
26 static PyObject
*PLyFloat_FromFloat4(PLyDatumToOb
*arg
, Datum d
);
27 static PyObject
*PLyFloat_FromFloat8(PLyDatumToOb
*arg
, Datum d
);
28 static PyObject
*PLyDecimal_FromNumeric(PLyDatumToOb
*arg
, Datum d
);
29 static PyObject
*PLyLong_FromInt16(PLyDatumToOb
*arg
, Datum d
);
30 static PyObject
*PLyLong_FromInt32(PLyDatumToOb
*arg
, Datum d
);
31 static PyObject
*PLyLong_FromInt64(PLyDatumToOb
*arg
, Datum d
);
32 static PyObject
*PLyLong_FromOid(PLyDatumToOb
*arg
, Datum d
);
33 static PyObject
*PLyBytes_FromBytea(PLyDatumToOb
*arg
, Datum d
);
34 static PyObject
*PLyUnicode_FromScalar(PLyDatumToOb
*arg
, Datum d
);
35 static PyObject
*PLyObject_FromTransform(PLyDatumToOb
*arg
, Datum d
);
36 static PyObject
*PLyList_FromArray(PLyDatumToOb
*arg
, Datum d
);
37 static PyObject
*PLyList_FromArray_recurse(PLyDatumToOb
*elm
, int *dims
, int ndim
, int dim
,
38 char **dataptr_p
, bits8
**bitmap_p
, int *bitmask_p
);
39 static PyObject
*PLyDict_FromComposite(PLyDatumToOb
*arg
, Datum d
);
40 static PyObject
*PLyDict_FromTuple(PLyDatumToOb
*arg
, HeapTuple tuple
, TupleDesc desc
, bool include_generated
);
42 /* conversion from Python objects to Datums */
43 static Datum
PLyObject_ToBool(PLyObToDatum
*arg
, PyObject
*plrv
,
44 bool *isnull
, bool inarray
);
45 static Datum
PLyObject_ToBytea(PLyObToDatum
*arg
, PyObject
*plrv
,
46 bool *isnull
, bool inarray
);
47 static Datum
PLyObject_ToComposite(PLyObToDatum
*arg
, PyObject
*plrv
,
48 bool *isnull
, bool inarray
);
49 static Datum
PLyObject_ToScalar(PLyObToDatum
*arg
, PyObject
*plrv
,
50 bool *isnull
, bool inarray
);
51 static Datum
PLyObject_ToDomain(PLyObToDatum
*arg
, PyObject
*plrv
,
52 bool *isnull
, bool inarray
);
53 static Datum
PLyObject_ToTransform(PLyObToDatum
*arg
, PyObject
*plrv
,
54 bool *isnull
, bool inarray
);
55 static Datum
PLySequence_ToArray(PLyObToDatum
*arg
, PyObject
*plrv
,
56 bool *isnull
, bool inarray
);
57 static void PLySequence_ToArray_recurse(PyObject
*obj
,
58 ArrayBuildState
**astatep
,
59 int *ndims
, int *dims
, int cur_depth
,
60 PLyObToDatum
*elm
, Oid elmbasetype
);
62 /* conversion from Python objects to composite Datums */
63 static Datum
PLyUnicode_ToComposite(PLyObToDatum
*arg
, PyObject
*string
, bool inarray
);
64 static Datum
PLyMapping_ToComposite(PLyObToDatum
*arg
, TupleDesc desc
, PyObject
*mapping
);
65 static Datum
PLySequence_ToComposite(PLyObToDatum
*arg
, TupleDesc desc
, PyObject
*sequence
);
66 static Datum
PLyGenericObject_ToComposite(PLyObToDatum
*arg
, TupleDesc desc
, PyObject
*object
, bool inarray
);
70 * Conversion functions. Remember output from Python is input to
71 * PostgreSQL, and vice versa.
75 * Perform input conversion, given correctly-set-up state information.
77 * This is the outer-level entry point for any input conversion. Internally,
78 * the conversion functions recurse directly to each other.
81 PLy_input_convert(PLyDatumToOb
*arg
, Datum val
)
84 PLyExecutionContext
*exec_ctx
= PLy_current_execution_context();
85 MemoryContext scratch_context
= PLy_get_scratch_context(exec_ctx
);
86 MemoryContext oldcontext
;
89 * Do the work in the scratch context to avoid leaking memory from the
90 * datatype output function calls. (The individual PLyDatumToObFunc
91 * functions can't reset the scratch context, because they recurse and an
92 * inner one might clobber data an outer one still needs. So we do it
93 * once at the outermost recursion level.)
95 * We reset the scratch context before, not after, each conversion cycle.
96 * This way we aren't on the hook to release a Python refcount on the
97 * result object in case MemoryContextReset throws an error.
99 MemoryContextReset(scratch_context
);
101 oldcontext
= MemoryContextSwitchTo(scratch_context
);
103 result
= arg
->func(arg
, val
);
105 MemoryContextSwitchTo(oldcontext
);
111 * Perform output conversion, given correctly-set-up state information.
113 * This is the outer-level entry point for any output conversion. Internally,
114 * the conversion functions recurse directly to each other.
116 * The result, as well as any cruft generated along the way, are in the
117 * current memory context. Caller is responsible for cleanup.
120 PLy_output_convert(PLyObToDatum
*arg
, PyObject
*val
, bool *isnull
)
122 /* at outer level, we are not considering an array element */
123 return arg
->func(arg
, val
, isnull
, false);
127 * Transform a tuple into a Python dict object.
129 * Note: the tupdesc must match the one used to set up *arg. We could
130 * insist that this function lookup the tupdesc from what is in *arg,
131 * but in practice all callers have the right tupdesc available.
134 PLy_input_from_tuple(PLyDatumToOb
*arg
, HeapTuple tuple
, TupleDesc desc
, bool include_generated
)
137 PLyExecutionContext
*exec_ctx
= PLy_current_execution_context();
138 MemoryContext scratch_context
= PLy_get_scratch_context(exec_ctx
);
139 MemoryContext oldcontext
;
142 * As in PLy_input_convert, do the work in the scratch context.
144 MemoryContextReset(scratch_context
);
146 oldcontext
= MemoryContextSwitchTo(scratch_context
);
148 dict
= PLyDict_FromTuple(arg
, tuple
, desc
, include_generated
);
150 MemoryContextSwitchTo(oldcontext
);
156 * Initialize, or re-initialize, per-column input info for a composite type.
158 * This is separate from PLy_input_setup_func() because in cases involving
159 * anonymous record types, we need to be passed the tupdesc explicitly.
160 * It's caller's responsibility that the tupdesc has adequate lifespan
161 * in such cases. If the tupdesc is for a named composite or registered
162 * record type, it does not need to be long-lived.
165 PLy_input_setup_tuple(PLyDatumToOb
*arg
, TupleDesc desc
, PLyProcedure
*proc
)
169 /* We should be working on a previously-set-up struct */
170 Assert(arg
->func
== PLyDict_FromComposite
);
172 /* Save pointer to tupdesc, but only if this is an anonymous record type */
173 if (arg
->typoid
== RECORDOID
&& arg
->typmod
< 0)
174 arg
->u
.tuple
.recdesc
= desc
;
176 /* (Re)allocate atts array as needed */
177 if (arg
->u
.tuple
.natts
!= desc
->natts
)
179 if (arg
->u
.tuple
.atts
)
180 pfree(arg
->u
.tuple
.atts
);
181 arg
->u
.tuple
.natts
= desc
->natts
;
182 arg
->u
.tuple
.atts
= (PLyDatumToOb
*)
183 MemoryContextAllocZero(arg
->mcxt
,
184 desc
->natts
* sizeof(PLyDatumToOb
));
187 /* Fill the atts entries, except for dropped columns */
188 for (i
= 0; i
< desc
->natts
; i
++)
190 Form_pg_attribute attr
= TupleDescAttr(desc
, i
);
191 PLyDatumToOb
*att
= &arg
->u
.tuple
.atts
[i
];
193 if (attr
->attisdropped
)
196 if (att
->typoid
== attr
->atttypid
&& att
->typmod
== attr
->atttypmod
)
197 continue; /* already set up this entry */
199 PLy_input_setup_func(att
, arg
->mcxt
,
200 attr
->atttypid
, attr
->atttypmod
,
206 * Initialize, or re-initialize, per-column output info for a composite type.
208 * This is separate from PLy_output_setup_func() because in cases involving
209 * anonymous record types, we need to be passed the tupdesc explicitly.
210 * It's caller's responsibility that the tupdesc has adequate lifespan
211 * in such cases. If the tupdesc is for a named composite or registered
212 * record type, it does not need to be long-lived.
215 PLy_output_setup_tuple(PLyObToDatum
*arg
, TupleDesc desc
, PLyProcedure
*proc
)
219 /* We should be working on a previously-set-up struct */
220 Assert(arg
->func
== PLyObject_ToComposite
);
222 /* Save pointer to tupdesc, but only if this is an anonymous record type */
223 if (arg
->typoid
== RECORDOID
&& arg
->typmod
< 0)
224 arg
->u
.tuple
.recdesc
= desc
;
226 /* (Re)allocate atts array as needed */
227 if (arg
->u
.tuple
.natts
!= desc
->natts
)
229 if (arg
->u
.tuple
.atts
)
230 pfree(arg
->u
.tuple
.atts
);
231 arg
->u
.tuple
.natts
= desc
->natts
;
232 arg
->u
.tuple
.atts
= (PLyObToDatum
*)
233 MemoryContextAllocZero(arg
->mcxt
,
234 desc
->natts
* sizeof(PLyObToDatum
));
237 /* Fill the atts entries, except for dropped columns */
238 for (i
= 0; i
< desc
->natts
; i
++)
240 Form_pg_attribute attr
= TupleDescAttr(desc
, i
);
241 PLyObToDatum
*att
= &arg
->u
.tuple
.atts
[i
];
243 if (attr
->attisdropped
)
246 if (att
->typoid
== attr
->atttypid
&& att
->typmod
== attr
->atttypmod
)
247 continue; /* already set up this entry */
249 PLy_output_setup_func(att
, arg
->mcxt
,
250 attr
->atttypid
, attr
->atttypmod
,
256 * Set up output info for a PL/Python function returning record.
258 * Note: the given tupdesc is not necessarily long-lived.
261 PLy_output_setup_record(PLyObToDatum
*arg
, TupleDesc desc
, PLyProcedure
*proc
)
263 /* Makes no sense unless RECORD */
264 Assert(arg
->typoid
== RECORDOID
);
265 Assert(desc
->tdtypeid
== RECORDOID
);
268 * Bless the record type if not already done. We'd have to do this anyway
269 * to return a tuple, so we might as well force the issue so we can use
270 * the known-record-type code path.
272 BlessTupleDesc(desc
);
275 * Update arg->typmod, and clear the recdesc link if it's changed. The
276 * next call of PLyObject_ToComposite will look up a long-lived tupdesc
277 * for the record type.
279 arg
->typmod
= desc
->tdtypmod
;
280 if (arg
->u
.tuple
.recdesc
&&
281 arg
->u
.tuple
.recdesc
->tdtypmod
!= arg
->typmod
)
282 arg
->u
.tuple
.recdesc
= NULL
;
284 /* Update derived data if necessary */
285 PLy_output_setup_tuple(arg
, desc
, proc
);
289 * Recursively initialize the PLyObToDatum structure(s) needed to construct
290 * a SQL value of the specified typeOid/typmod from a Python value.
291 * (But note that at this point we may have RECORDOID/-1, ie, an indeterminate
293 * proc is used to look up transform functions.
296 PLy_output_setup_func(PLyObToDatum
*arg
, MemoryContext arg_mcxt
,
297 Oid typeOid
, int32 typmod
,
300 TypeCacheEntry
*typentry
;
305 /* Since this is recursive, it could theoretically be driven to overflow */
308 arg
->typoid
= typeOid
;
309 arg
->typmod
= typmod
;
310 arg
->mcxt
= arg_mcxt
;
313 * Fetch typcache entry for the target type, asking for whatever info
314 * we'll need later. RECORD is a special case: just treat it as composite
315 * without bothering with the typcache entry.
317 if (typeOid
!= RECORDOID
)
319 typentry
= lookup_type_cache(typeOid
, TYPECACHE_DOMAIN_BASE_INFO
);
320 typtype
= typentry
->typtype
;
321 arg
->typbyval
= typentry
->typbyval
;
322 arg
->typlen
= typentry
->typlen
;
323 arg
->typalign
= typentry
->typalign
;
328 typtype
= TYPTYPE_COMPOSITE
;
329 /* hard-wired knowledge about type RECORD: */
330 arg
->typbyval
= false;
332 arg
->typalign
= TYPALIGN_DOUBLE
;
336 * Choose conversion method. Note that transform functions are checked
337 * for composite and scalar types, but not for arrays or domains. This is
338 * somewhat historical, but we'd have a problem allowing them on domains,
339 * since we drill down through all levels of a domain nest without looking
340 * at the intermediate levels at all.
342 if (typtype
== TYPTYPE_DOMAIN
)
345 arg
->func
= PLyObject_ToDomain
;
346 arg
->u
.domain
.domain_info
= NULL
;
347 /* Recursively set up conversion info for the element type */
348 arg
->u
.domain
.base
= (PLyObToDatum
*)
349 MemoryContextAllocZero(arg_mcxt
, sizeof(PLyObToDatum
));
350 PLy_output_setup_func(arg
->u
.domain
.base
, arg_mcxt
,
351 typentry
->domainBaseType
,
352 typentry
->domainBaseTypmod
,
356 IsTrueArrayType(typentry
))
359 arg
->func
= PLySequence_ToArray
;
360 /* Get base type OID to insert into constructed array */
361 /* (note this might not be the same as the immediate child type) */
362 arg
->u
.array
.elmbasetype
= getBaseType(typentry
->typelem
);
363 /* Recursively set up conversion info for the element type */
364 arg
->u
.array
.elm
= (PLyObToDatum
*)
365 MemoryContextAllocZero(arg_mcxt
, sizeof(PLyObToDatum
));
366 PLy_output_setup_func(arg
->u
.array
.elm
, arg_mcxt
,
367 typentry
->typelem
, typmod
,
370 else if ((trfuncid
= get_transform_tosql(typeOid
,
374 arg
->func
= PLyObject_ToTransform
;
375 fmgr_info_cxt(trfuncid
, &arg
->u
.transform
.typtransform
, arg_mcxt
);
377 else if (typtype
== TYPTYPE_COMPOSITE
)
379 /* Named composite type, or RECORD */
380 arg
->func
= PLyObject_ToComposite
;
381 /* We'll set up the per-field data later */
382 arg
->u
.tuple
.recdesc
= NULL
;
383 arg
->u
.tuple
.typentry
= typentry
;
384 arg
->u
.tuple
.tupdescid
= INVALID_TUPLEDESC_IDENTIFIER
;
385 arg
->u
.tuple
.atts
= NULL
;
386 arg
->u
.tuple
.natts
= 0;
387 /* Mark this invalid till needed, too */
388 arg
->u
.tuple
.recinfunc
.fn_oid
= InvalidOid
;
392 /* Scalar type, but we have a couple of special cases */
396 arg
->func
= PLyObject_ToBool
;
399 arg
->func
= PLyObject_ToBytea
;
402 arg
->func
= PLyObject_ToScalar
;
403 getTypeInputInfo(typeOid
, &typinput
, &arg
->u
.scalar
.typioparam
);
404 fmgr_info_cxt(typinput
, &arg
->u
.scalar
.typfunc
, arg_mcxt
);
411 * Recursively initialize the PLyDatumToOb structure(s) needed to construct
412 * a Python value from a SQL value of the specified typeOid/typmod.
413 * (But note that at this point we may have RECORDOID/-1, ie, an indeterminate
415 * proc is used to look up transform functions.
418 PLy_input_setup_func(PLyDatumToOb
*arg
, MemoryContext arg_mcxt
,
419 Oid typeOid
, int32 typmod
,
422 TypeCacheEntry
*typentry
;
428 /* Since this is recursive, it could theoretically be driven to overflow */
431 arg
->typoid
= typeOid
;
432 arg
->typmod
= typmod
;
433 arg
->mcxt
= arg_mcxt
;
436 * Fetch typcache entry for the target type, asking for whatever info
437 * we'll need later. RECORD is a special case: just treat it as composite
438 * without bothering with the typcache entry.
440 if (typeOid
!= RECORDOID
)
442 typentry
= lookup_type_cache(typeOid
, TYPECACHE_DOMAIN_BASE_INFO
);
443 typtype
= typentry
->typtype
;
444 arg
->typbyval
= typentry
->typbyval
;
445 arg
->typlen
= typentry
->typlen
;
446 arg
->typalign
= typentry
->typalign
;
451 typtype
= TYPTYPE_COMPOSITE
;
452 /* hard-wired knowledge about type RECORD: */
453 arg
->typbyval
= false;
455 arg
->typalign
= TYPALIGN_DOUBLE
;
459 * Choose conversion method. Note that transform functions are checked
460 * for composite and scalar types, but not for arrays or domains. This is
461 * somewhat historical, but we'd have a problem allowing them on domains,
462 * since we drill down through all levels of a domain nest without looking
463 * at the intermediate levels at all.
465 if (typtype
== TYPTYPE_DOMAIN
)
467 /* Domain --- we don't care, just recurse down to the base type */
468 PLy_input_setup_func(arg
, arg_mcxt
,
469 typentry
->domainBaseType
,
470 typentry
->domainBaseTypmod
,
474 IsTrueArrayType(typentry
))
477 arg
->func
= PLyList_FromArray
;
478 /* Recursively set up conversion info for the element type */
479 arg
->u
.array
.elm
= (PLyDatumToOb
*)
480 MemoryContextAllocZero(arg_mcxt
, sizeof(PLyDatumToOb
));
481 PLy_input_setup_func(arg
->u
.array
.elm
, arg_mcxt
,
482 typentry
->typelem
, typmod
,
485 else if ((trfuncid
= get_transform_fromsql(typeOid
,
489 arg
->func
= PLyObject_FromTransform
;
490 fmgr_info_cxt(trfuncid
, &arg
->u
.transform
.typtransform
, arg_mcxt
);
492 else if (typtype
== TYPTYPE_COMPOSITE
)
494 /* Named composite type, or RECORD */
495 arg
->func
= PLyDict_FromComposite
;
496 /* We'll set up the per-field data later */
497 arg
->u
.tuple
.recdesc
= NULL
;
498 arg
->u
.tuple
.typentry
= typentry
;
499 arg
->u
.tuple
.tupdescid
= INVALID_TUPLEDESC_IDENTIFIER
;
500 arg
->u
.tuple
.atts
= NULL
;
501 arg
->u
.tuple
.natts
= 0;
505 /* Scalar type, but we have a couple of special cases */
509 arg
->func
= PLyBool_FromBool
;
512 arg
->func
= PLyFloat_FromFloat4
;
515 arg
->func
= PLyFloat_FromFloat8
;
518 arg
->func
= PLyDecimal_FromNumeric
;
521 arg
->func
= PLyLong_FromInt16
;
524 arg
->func
= PLyLong_FromInt32
;
527 arg
->func
= PLyLong_FromInt64
;
530 arg
->func
= PLyLong_FromOid
;
533 arg
->func
= PLyBytes_FromBytea
;
536 arg
->func
= PLyUnicode_FromScalar
;
537 getTypeOutputInfo(typeOid
, &typoutput
, &typisvarlena
);
538 fmgr_info_cxt(typoutput
, &arg
->u
.scalar
.typfunc
, arg_mcxt
);
546 * Special-purpose input converters.
550 PLyBool_FromBool(PLyDatumToOb
*arg
, Datum d
)
558 PLyFloat_FromFloat4(PLyDatumToOb
*arg
, Datum d
)
560 return PyFloat_FromDouble(DatumGetFloat4(d
));
564 PLyFloat_FromFloat8(PLyDatumToOb
*arg
, Datum d
)
566 return PyFloat_FromDouble(DatumGetFloat8(d
));
570 PLyDecimal_FromNumeric(PLyDatumToOb
*arg
, Datum d
)
572 static PyObject
*decimal_constructor
;
576 /* Try to import cdecimal. If it doesn't exist, fall back to decimal. */
577 if (!decimal_constructor
)
579 PyObject
*decimal_module
;
581 decimal_module
= PyImport_ImportModule("cdecimal");
585 decimal_module
= PyImport_ImportModule("decimal");
588 PLy_elog(ERROR
, "could not import a module for Decimal constructor");
590 decimal_constructor
= PyObject_GetAttrString(decimal_module
, "Decimal");
591 if (!decimal_constructor
)
592 PLy_elog(ERROR
, "no Decimal attribute in module");
595 str
= DatumGetCString(DirectFunctionCall1(numeric_out
, d
));
596 pyvalue
= PyObject_CallFunction(decimal_constructor
, "s", str
);
598 PLy_elog(ERROR
, "conversion from numeric to Decimal failed");
604 PLyLong_FromInt16(PLyDatumToOb
*arg
, Datum d
)
606 return PyLong_FromLong(DatumGetInt16(d
));
610 PLyLong_FromInt32(PLyDatumToOb
*arg
, Datum d
)
612 return PyLong_FromLong(DatumGetInt32(d
));
616 PLyLong_FromInt64(PLyDatumToOb
*arg
, Datum d
)
618 return PyLong_FromLongLong(DatumGetInt64(d
));
622 PLyLong_FromOid(PLyDatumToOb
*arg
, Datum d
)
624 return PyLong_FromUnsignedLong(DatumGetObjectId(d
));
628 PLyBytes_FromBytea(PLyDatumToOb
*arg
, Datum d
)
630 text
*txt
= DatumGetByteaPP(d
);
631 char *str
= VARDATA_ANY(txt
);
632 size_t size
= VARSIZE_ANY_EXHDR(txt
);
634 return PyBytes_FromStringAndSize(str
, size
);
639 * Generic input conversion using a SQL type's output function.
642 PLyUnicode_FromScalar(PLyDatumToOb
*arg
, Datum d
)
644 char *x
= OutputFunctionCall(&arg
->u
.scalar
.typfunc
, d
);
645 PyObject
*r
= PLyUnicode_FromString(x
);
652 * Convert using a from-SQL transform function.
655 PLyObject_FromTransform(PLyDatumToOb
*arg
, Datum d
)
659 t
= FunctionCall1(&arg
->u
.transform
.typtransform
, d
);
660 return (PyObject
*) DatumGetPointer(t
);
664 * Convert a SQL array to a Python list.
667 PLyList_FromArray(PLyDatumToOb
*arg
, Datum d
)
669 ArrayType
*array
= DatumGetArrayTypeP(d
);
670 PLyDatumToOb
*elm
= arg
->u
.array
.elm
;
677 if (ARR_NDIM(array
) == 0)
678 return PyList_New(0);
680 /* Array dimensions and left bounds */
681 ndim
= ARR_NDIM(array
);
682 dims
= ARR_DIMS(array
);
683 Assert(ndim
<= MAXDIM
);
686 * We iterate the SQL array in the physical order it's stored in the
687 * datum. For example, for a 3-dimensional array the order of iteration
688 * would be the following: [0,0,0] elements through [0,0,k], then [0,1,0]
689 * through [0,1,k] till [0,m,k], then [1,0,0] through [1,0,k] till
690 * [1,m,k], and so on.
692 * In Python, there are no multi-dimensional lists as such, but they are
693 * represented as a list of lists. So a 3-d array of [n,m,k] elements is a
694 * list of n m-element arrays, each element of which is k-element array.
695 * PLyList_FromArray_recurse() builds the Python list for a single
696 * dimension, and recurses for the next inner dimension.
698 dataptr
= ARR_DATA_PTR(array
);
699 bitmap
= ARR_NULLBITMAP(array
);
702 return PLyList_FromArray_recurse(elm
, dims
, ndim
, 0,
703 &dataptr
, &bitmap
, &bitmask
);
707 PLyList_FromArray_recurse(PLyDatumToOb
*elm
, int *dims
, int ndim
, int dim
,
708 char **dataptr_p
, bits8
**bitmap_p
, int *bitmask_p
)
713 list
= PyList_New(dims
[dim
]);
719 /* Outer dimension. Recurse for each inner slice. */
720 for (i
= 0; i
< dims
[dim
]; i
++)
724 sublist
= PLyList_FromArray_recurse(elm
, dims
, ndim
, dim
+ 1,
725 dataptr_p
, bitmap_p
, bitmask_p
);
726 PyList_SET_ITEM(list
, i
, sublist
);
732 * Innermost dimension. Fill the list with the values from the array
735 char *dataptr
= *dataptr_p
;
736 bits8
*bitmap
= *bitmap_p
;
737 int bitmask
= *bitmask_p
;
739 for (i
= 0; i
< dims
[dim
]; i
++)
741 /* checking for NULL */
742 if (bitmap
&& (*bitmap
& bitmask
) == 0)
745 PyList_SET_ITEM(list
, i
, Py_None
);
751 itemvalue
= fetch_att(dataptr
, elm
->typbyval
, elm
->typlen
);
752 PyList_SET_ITEM(list
, i
, elm
->func(elm
, itemvalue
));
753 dataptr
= att_addlength_pointer(dataptr
, elm
->typlen
, dataptr
);
754 dataptr
= (char *) att_align_nominal(dataptr
, elm
->typalign
);
757 /* advance bitmap pointer if any */
761 if (bitmask
== 0x100 /* (1<<8) */ )
769 *dataptr_p
= dataptr
;
771 *bitmask_p
= bitmask
;
778 * Convert a composite SQL value to a Python dict.
781 PLyDict_FromComposite(PLyDatumToOb
*arg
, Datum d
)
788 HeapTupleData tmptup
;
790 td
= DatumGetHeapTupleHeader(d
);
791 /* Extract rowtype info and find a tupdesc */
792 tupType
= HeapTupleHeaderGetTypeId(td
);
793 tupTypmod
= HeapTupleHeaderGetTypMod(td
);
794 tupdesc
= lookup_rowtype_tupdesc(tupType
, tupTypmod
);
796 /* Set up I/O funcs if not done yet */
797 PLy_input_setup_tuple(arg
, tupdesc
,
798 PLy_current_execution_context()->curr_proc
);
800 /* Build a temporary HeapTuple control structure */
801 tmptup
.t_len
= HeapTupleHeaderGetDatumLength(td
);
804 dict
= PLyDict_FromTuple(arg
, &tmptup
, tupdesc
, true);
806 ReleaseTupleDesc(tupdesc
);
812 * Transform a tuple into a Python dict object.
815 PLyDict_FromTuple(PLyDatumToOb
*arg
, HeapTuple tuple
, TupleDesc desc
, bool include_generated
)
817 PyObject
*volatile dict
;
819 /* Simple sanity check that desc matches */
820 Assert(desc
->natts
== arg
->u
.tuple
.natts
);
830 for (i
= 0; i
< arg
->u
.tuple
.natts
; i
++)
832 PLyDatumToOb
*att
= &arg
->u
.tuple
.atts
[i
];
833 Form_pg_attribute attr
= TupleDescAttr(desc
, i
);
839 if (attr
->attisdropped
)
842 if (attr
->attgenerated
)
844 /* don't include unless requested */
845 if (!include_generated
)
849 key
= NameStr(attr
->attname
);
850 vattr
= heap_getattr(tuple
, (i
+ 1), desc
, &is_null
);
853 PyDict_SetItemString(dict
, key
, Py_None
);
856 value
= att
->func(att
, vattr
);
857 PyDict_SetItemString(dict
, key
, value
);
873 * Convert a Python object to a PostgreSQL bool datum. This can't go
874 * through the generic conversion function, because Python attaches a
875 * Boolean value to everything, more things than the PostgreSQL bool
879 PLyObject_ToBool(PLyObToDatum
*arg
, PyObject
*plrv
,
880 bool *isnull
, bool inarray
)
888 return BoolGetDatum(PyObject_IsTrue(plrv
));
892 * Convert a Python object to a PostgreSQL bytea datum. This doesn't
893 * go through the generic conversion function to circumvent problems
894 * with embedded nulls. And it's faster this way.
897 PLyObject_ToBytea(PLyObToDatum
*arg
, PyObject
*plrv
,
898 bool *isnull
, bool inarray
)
900 PyObject
*volatile plrv_so
= NULL
;
901 Datum rv
= (Datum
) 0;
910 plrv_so
= PyObject_Bytes(plrv
);
912 PLy_elog(ERROR
, "could not create bytes representation of Python object");
916 char *plrv_sc
= PyBytes_AsString(plrv_so
);
917 size_t len
= PyBytes_Size(plrv_so
);
918 size_t size
= len
+ VARHDRSZ
;
919 bytea
*result
= palloc(size
);
921 SET_VARSIZE(result
, size
);
922 memcpy(VARDATA(result
), plrv_sc
, len
);
923 rv
= PointerGetDatum(result
);
936 * Convert a Python object to a composite type. First look up the type's
937 * description, then route the Python object through the conversion function
938 * for obtaining PostgreSQL tuples.
941 PLyObject_ToComposite(PLyObToDatum
*arg
, PyObject
*plrv
,
942 bool *isnull
, bool inarray
)
955 * The string conversion case doesn't require a tupdesc, nor per-field
956 * conversion data, so just go for it if that's the case to use.
958 if (PyUnicode_Check(plrv
))
959 return PLyUnicode_ToComposite(arg
, plrv
, inarray
);
962 * If we're dealing with a named composite type, we must look up the
963 * tupdesc every time, to protect against possible changes to the type.
964 * RECORD types can't change between calls; but we must still be willing
965 * to set up the info the first time, if nobody did yet.
967 if (arg
->typoid
!= RECORDOID
)
969 desc
= lookup_rowtype_tupdesc(arg
->typoid
, arg
->typmod
);
970 /* We should have the descriptor of the type's typcache entry */
971 Assert(desc
== arg
->u
.tuple
.typentry
->tupDesc
);
972 /* Detect change of descriptor, update cache if needed */
973 if (arg
->u
.tuple
.tupdescid
!= arg
->u
.tuple
.typentry
->tupDesc_identifier
)
975 PLy_output_setup_tuple(arg
, desc
,
976 PLy_current_execution_context()->curr_proc
);
977 arg
->u
.tuple
.tupdescid
= arg
->u
.tuple
.typentry
->tupDesc_identifier
;
982 desc
= arg
->u
.tuple
.recdesc
;
985 desc
= lookup_rowtype_tupdesc(arg
->typoid
, arg
->typmod
);
986 arg
->u
.tuple
.recdesc
= desc
;
990 /* Pin descriptor to match unpin below */
995 /* Simple sanity check on our caching */
996 Assert(desc
->natts
== arg
->u
.tuple
.natts
);
999 * Convert, using the appropriate method depending on the type of the
1000 * supplied Python object.
1002 if (PySequence_Check(plrv
))
1003 /* composite type as sequence (tuple, list etc) */
1004 rv
= PLySequence_ToComposite(arg
, desc
, plrv
);
1005 else if (PyMapping_Check(plrv
))
1006 /* composite type as mapping (currently only dict) */
1007 rv
= PLyMapping_ToComposite(arg
, desc
, plrv
);
1009 /* returned as smth, must provide method __getattr__(name) */
1010 rv
= PLyGenericObject_ToComposite(arg
, desc
, plrv
, inarray
);
1012 ReleaseTupleDesc(desc
);
1019 * Convert Python object to C string in server encoding.
1021 * Note: this is exported for use by add-on transform modules.
1024 PLyObject_AsString(PyObject
*plrv
)
1031 if (PyUnicode_Check(plrv
))
1032 plrv_bo
= PLyUnicode_Bytes(plrv
);
1033 else if (PyFloat_Check(plrv
))
1035 /* use repr() for floats, str() is lossy */
1036 PyObject
*s
= PyObject_Repr(plrv
);
1038 plrv_bo
= PLyUnicode_Bytes(s
);
1043 PyObject
*s
= PyObject_Str(plrv
);
1045 plrv_bo
= PLyUnicode_Bytes(s
);
1049 PLy_elog(ERROR
, "could not create string representation of Python object");
1051 plrv_sc
= pstrdup(PyBytes_AsString(plrv_bo
));
1052 plen
= PyBytes_Size(plrv_bo
);
1053 slen
= strlen(plrv_sc
);
1055 Py_XDECREF(plrv_bo
);
1059 (errcode(ERRCODE_DATATYPE_MISMATCH
),
1060 errmsg("could not convert Python object into cstring: Python string representation appears to contain null bytes")));
1061 else if (slen
> plen
)
1062 elog(ERROR
, "could not convert Python object into cstring: Python string longer than reported length");
1063 pg_verifymbstr(plrv_sc
, slen
, false);
1070 * Generic output conversion function: convert PyObject to cstring and
1071 * cstring into PostgreSQL type.
1074 PLyObject_ToScalar(PLyObToDatum
*arg
, PyObject
*plrv
,
1075 bool *isnull
, bool inarray
)
1079 if (plrv
== Py_None
)
1086 str
= PLyObject_AsString(plrv
);
1088 return InputFunctionCall(&arg
->u
.scalar
.typfunc
,
1090 arg
->u
.scalar
.typioparam
,
1096 * Convert to a domain type.
1099 PLyObject_ToDomain(PLyObToDatum
*arg
, PyObject
*plrv
,
1100 bool *isnull
, bool inarray
)
1103 PLyObToDatum
*base
= arg
->u
.domain
.base
;
1105 result
= base
->func(base
, plrv
, isnull
, inarray
);
1106 domain_check(result
, *isnull
, arg
->typoid
,
1107 &arg
->u
.domain
.domain_info
, arg
->mcxt
);
1113 * Convert using a to-SQL transform function.
1116 PLyObject_ToTransform(PLyObToDatum
*arg
, PyObject
*plrv
,
1117 bool *isnull
, bool inarray
)
1119 if (plrv
== Py_None
)
1125 return FunctionCall1(&arg
->u
.transform
.typtransform
, PointerGetDatum(plrv
));
1130 * Convert Python sequence (or list of lists) to SQL array.
1133 PLySequence_ToArray(PLyObToDatum
*arg
, PyObject
*plrv
,
1134 bool *isnull
, bool inarray
)
1136 ArrayBuildState
*astate
= NULL
;
1141 if (plrv
== Py_None
)
1149 * For historical reasons, we allow any sequence (not only a list) at the
1150 * top level when converting a Python object to a SQL array. However, a
1151 * multi-dimensional array is recognized only when the object contains
1154 if (!PySequence_Check(plrv
))
1156 (errcode(ERRCODE_DATATYPE_MISMATCH
),
1157 errmsg("return value of function with array return type is not a Python sequence")));
1159 /* Initialize dimensionality info with first-level dimension */
1160 memset(dims
, 0, sizeof(dims
));
1161 dims
[0] = PySequence_Length(plrv
);
1164 * Traverse the Python lists, in depth-first order, and collect all the
1165 * elements at the bottom level into an ArrayBuildState.
1167 PLySequence_ToArray_recurse(plrv
, &astate
,
1170 arg
->u
.array
.elmbasetype
);
1172 /* ensure we get zero-D array for no inputs, as per PG convention */
1174 return PointerGetDatum(construct_empty_array(arg
->u
.array
.elmbasetype
));
1176 for (int i
= 0; i
< ndims
; i
++)
1179 return makeMdArrayResult(astate
, ndims
, dims
, lbs
,
1180 CurrentMemoryContext
, true);
1184 * Helper function for PLySequence_ToArray. Traverse a Python list of lists in
1185 * depth-first order, storing the elements in *astatep.
1187 * The ArrayBuildState is created only when we first find a scalar element;
1188 * if we didn't do it like that, we'd need some other convention for knowing
1189 * whether we'd already found any scalars (and thus the number of dimensions
1193 PLySequence_ToArray_recurse(PyObject
*obj
, ArrayBuildState
**astatep
,
1194 int *ndims
, int *dims
, int cur_depth
,
1195 PLyObToDatum
*elm
, Oid elmbasetype
)
1198 int len
= PySequence_Length(obj
);
1200 /* We should not get here with a non-sequence object */
1202 PLy_elog(ERROR
, "could not determine sequence length for function return value");
1204 for (i
= 0; i
< len
; i
++)
1206 /* fetch the array element */
1207 PyObject
*subobj
= PySequence_GetItem(obj
, i
);
1209 /* need PG_TRY to ensure we release the subobj's refcount */
1212 /* multi-dimensional array? */
1213 if (PyList_Check(subobj
))
1215 /* set size when at first element in this level, else compare */
1216 if (i
== 0 && *ndims
== cur_depth
)
1218 /* array after some scalars at same level? */
1219 if (*astatep
!= NULL
)
1221 (errcode(ERRCODE_INVALID_TEXT_REPRESENTATION
),
1222 errmsg("multidimensional arrays must have array expressions with matching dimensions")));
1223 /* too many dimensions? */
1224 if (cur_depth
>= MAXDIM
)
1226 (errcode(ERRCODE_PROGRAM_LIMIT_EXCEEDED
),
1227 errmsg("number of array dimensions exceeds the maximum allowed (%d)",
1229 /* OK, add a dimension */
1230 dims
[*ndims
] = PySequence_Length(subobj
);
1233 else if (cur_depth
>= *ndims
||
1234 PySequence_Length(subobj
) != dims
[cur_depth
])
1236 (errcode(ERRCODE_INVALID_TEXT_REPRESENTATION
),
1237 errmsg("multidimensional arrays must have array expressions with matching dimensions")));
1239 /* recurse to fetch elements of this sub-array */
1240 PLySequence_ToArray_recurse(subobj
, astatep
,
1241 ndims
, dims
, cur_depth
+ 1,
1249 /* scalar after some sub-arrays at same level? */
1250 if (*ndims
!= cur_depth
)
1252 (errcode(ERRCODE_INVALID_TEXT_REPRESENTATION
),
1253 errmsg("multidimensional arrays must have array expressions with matching dimensions")));
1255 /* convert non-list object to Datum */
1256 dat
= elm
->func(elm
, subobj
, &isnull
, true);
1258 /* create ArrayBuildState if we didn't already */
1259 if (*astatep
== NULL
)
1260 *astatep
= initArrayResult(elmbasetype
,
1261 CurrentMemoryContext
, true);
1263 /* ... and save the element value in it */
1264 (void) accumArrayResult(*astatep
, dat
, isnull
,
1265 elmbasetype
, CurrentMemoryContext
);
1278 * Convert a Python string to composite, using record_in.
1281 PLyUnicode_ToComposite(PLyObToDatum
*arg
, PyObject
*string
, bool inarray
)
1286 * Set up call data for record_in, if we didn't already. (We can't just
1287 * use DirectFunctionCall, because record_in needs a fn_extra field.)
1289 if (!OidIsValid(arg
->u
.tuple
.recinfunc
.fn_oid
))
1290 fmgr_info_cxt(F_RECORD_IN
, &arg
->u
.tuple
.recinfunc
, arg
->mcxt
);
1292 str
= PLyObject_AsString(string
);
1295 * If we are parsing a composite type within an array, and the string
1296 * isn't a valid record literal, there's a high chance that the function
1297 * did something like:
1299 * CREATE FUNCTION .. RETURNS comptype[] AS $$ return [['foo', 'bar']] $$
1300 * LANGUAGE plpython;
1302 * Before PostgreSQL 10, that was interpreted as a single-dimensional
1303 * array, containing record ('foo', 'bar'). PostgreSQL 10 added support
1304 * for multi-dimensional arrays, and it is now interpreted as a
1305 * two-dimensional array, containing two records, 'foo', and 'bar'.
1306 * record_in() will throw an error, because "foo" is not a valid record
1309 * To make that less confusing to users who are upgrading from older
1310 * versions, try to give a hint in the typical instances of that. If we
1311 * are parsing an array of composite types, and we see a string literal
1312 * that is not a valid record literal, give a hint. We only want to give
1313 * the hint in the narrow case of a malformed string literal, not any
1314 * error from record_in(), so check for that case here specifically.
1316 * This check better match the one in record_in(), so that we don't forbid
1317 * literals that are actually valid!
1323 /* Allow leading whitespace */
1324 while (*ptr
&& isspace((unsigned char) *ptr
))
1328 (errcode(ERRCODE_INVALID_TEXT_REPRESENTATION
),
1329 errmsg("malformed record literal: \"%s\"", str
),
1330 errdetail("Missing left parenthesis."),
1331 errhint("To return a composite type in an array, return the composite type as a Python tuple, e.g., \"[('foo',)]\".")));
1334 return InputFunctionCall(&arg
->u
.tuple
.recinfunc
,
1342 PLyMapping_ToComposite(PLyObToDatum
*arg
, TupleDesc desc
, PyObject
*mapping
)
1350 Assert(PyMapping_Check(mapping
));
1353 values
= palloc(sizeof(Datum
) * desc
->natts
);
1354 nulls
= palloc(sizeof(bool) * desc
->natts
);
1355 for (i
= 0; i
< desc
->natts
; ++i
)
1358 PyObject
*volatile value
;
1360 Form_pg_attribute attr
= TupleDescAttr(desc
, i
);
1362 if (attr
->attisdropped
)
1364 values
[i
] = (Datum
) 0;
1369 key
= NameStr(attr
->attname
);
1371 att
= &arg
->u
.tuple
.atts
[i
];
1374 value
= PyMapping_GetItemString(mapping
, key
);
1377 (errcode(ERRCODE_UNDEFINED_COLUMN
),
1378 errmsg("key \"%s\" not found in mapping", key
),
1379 errhint("To return null in a column, "
1380 "add the value None to the mapping with the key named after the column.")));
1382 values
[i
] = att
->func(att
, value
, &nulls
[i
], false);
1395 tuple
= heap_form_tuple(desc
, values
, nulls
);
1396 result
= heap_copy_tuple_as_datum(tuple
, desc
);
1397 heap_freetuple(tuple
);
1407 PLySequence_ToComposite(PLyObToDatum
*arg
, TupleDesc desc
, PyObject
*sequence
)
1416 Assert(PySequence_Check(sequence
));
1419 * Check that sequence length is exactly same as PG tuple's. We actually
1420 * can ignore exceeding items or assume missing ones as null but to avoid
1421 * plpython developer's errors we are strict here
1424 for (i
= 0; i
< desc
->natts
; i
++)
1426 if (!TupleDescAttr(desc
, i
)->attisdropped
)
1429 if (PySequence_Length(sequence
) != idx
)
1431 (errcode(ERRCODE_DATATYPE_MISMATCH
),
1432 errmsg("length of returned sequence did not match number of columns in row")));
1435 values
= palloc(sizeof(Datum
) * desc
->natts
);
1436 nulls
= palloc(sizeof(bool) * desc
->natts
);
1438 for (i
= 0; i
< desc
->natts
; ++i
)
1440 PyObject
*volatile value
;
1443 if (TupleDescAttr(desc
, i
)->attisdropped
)
1445 values
[i
] = (Datum
) 0;
1451 att
= &arg
->u
.tuple
.atts
[i
];
1454 value
= PySequence_GetItem(sequence
, idx
);
1457 values
[i
] = att
->func(att
, value
, &nulls
[i
], false);
1472 tuple
= heap_form_tuple(desc
, values
, nulls
);
1473 result
= heap_copy_tuple_as_datum(tuple
, desc
);
1474 heap_freetuple(tuple
);
1484 PLyGenericObject_ToComposite(PLyObToDatum
*arg
, TupleDesc desc
, PyObject
*object
, bool inarray
)
1493 values
= palloc(sizeof(Datum
) * desc
->natts
);
1494 nulls
= palloc(sizeof(bool) * desc
->natts
);
1495 for (i
= 0; i
< desc
->natts
; ++i
)
1498 PyObject
*volatile value
;
1500 Form_pg_attribute attr
= TupleDescAttr(desc
, i
);
1502 if (attr
->attisdropped
)
1504 values
[i
] = (Datum
) 0;
1509 key
= NameStr(attr
->attname
);
1511 att
= &arg
->u
.tuple
.atts
[i
];
1514 value
= PyObject_GetAttrString(object
, key
);
1518 * No attribute for this column in the object.
1520 * If we are parsing a composite type in an array, a likely
1521 * cause is that the function contained something like "[[123,
1522 * 'foo']]". Before PostgreSQL 10, that was interpreted as an
1523 * array, with a composite type (123, 'foo') in it. But now
1524 * it's interpreted as a two-dimensional array, and we try to
1525 * interpret "123" as the composite type. See also similar
1526 * heuristic in PLyObject_ToScalar().
1529 (errcode(ERRCODE_UNDEFINED_COLUMN
),
1530 errmsg("attribute \"%s\" does not exist in Python object", key
),
1532 errhint("To return a composite type in an array, return the composite type as a Python tuple, e.g., \"[('foo',)]\".") :
1533 errhint("To return null in a column, let the returned object have an attribute named after column with value None.")));
1536 values
[i
] = att
->func(att
, value
, &nulls
[i
], false);
1549 tuple
= heap_form_tuple(desc
, values
, nulls
);
1550 result
= heap_copy_tuple_as_datum(tuple
, desc
);
1551 heap_freetuple(tuple
);