1 /*--------------------------------------------------------------------------
3 * header file for postgres inverted index access method implementation.
5 * Copyright (c) 2006-2009, PostgreSQL Global Development Group
8 *--------------------------------------------------------------------------
13 #include "access/genam.h"
14 #include "access/itup.h"
15 #include "access/xlog.h"
20 * amproc indexes for inverted indexes.
22 #define GIN_COMPARE_PROC 1
23 #define GIN_EXTRACTVALUE_PROC 2
24 #define GIN_EXTRACTQUERY_PROC 3
25 #define GIN_CONSISTENT_PROC 4
26 #define GIN_COMPARE_PARTIAL_PROC 5
30 * Max depth allowed in search tree during bulk inserts. This is to keep from
31 * degenerating to O(N^2) behavior when the tree is unbalanced due to sorted
32 * or nearly-sorted input. (Perhaps it would be better to use a balanced-tree
33 * algorithm, but in common cases that would only add useless overhead.)
35 #define GIN_MAX_TREE_DEPTH 100
38 * Page opaque data in a inverted index page.
40 * Note: GIN does not include a page ID word as do the other index types.
41 * This is OK because the opaque data is only 8 bytes and so can be reliably
42 * distinguished by size. Revisit this if the size ever increases.
44 typedef struct GinPageOpaqueData
46 BlockNumber rightlink
; /* next page if any */
47 OffsetNumber maxoff
; /* number entries on GIN_DATA page; number of
48 * heap ItemPointer on GIN_DATA|GIN_LEAF page
49 * and number of records on GIN_DATA &
50 * ~GIN_LEAF page. On GIN_LIST page, number of
52 uint16 flags
; /* see bit definitions below */
55 typedef GinPageOpaqueData
*GinPageOpaque
;
57 #define GIN_DATA (1 << 0)
58 #define GIN_LEAF (1 << 1)
59 #define GIN_DELETED (1 << 2)
60 #define GIN_META (1 << 3)
61 #define GIN_LIST (1 << 4)
62 #define GIN_LIST_FULLROW (1 << 5) /* makes sense only on GIN_LIST page */
64 /* Page numbers of fixed-location pages */
65 #define GIN_METAPAGE_BLKNO (0)
66 #define GIN_ROOT_BLKNO (1)
68 typedef struct GinMetaPageData
71 * Pointers to head and tail of pending list, which consists of GIN_LIST
72 * pages. These store fast-inserted entries that haven't yet been moved
73 * into the regular GIN structure.
79 * Free space in bytes in the pending list's tail page.
84 * We store both number of pages and number of heap tuples that are in the
87 BlockNumber nPendingPages
;
88 int64 nPendingHeapTuples
;
91 #define GinPageGetMeta(p) \
92 ((GinMetaPageData *) PageGetContents(p))
97 #define GinPageGetOpaque(page) ( (GinPageOpaque) PageGetSpecialPointer(page) )
99 #define GinPageIsLeaf(page) ( GinPageGetOpaque(page)->flags & GIN_LEAF )
100 #define GinPageSetLeaf(page) ( GinPageGetOpaque(page)->flags |= GIN_LEAF )
101 #define GinPageSetNonLeaf(page) ( GinPageGetOpaque(page)->flags &= ~GIN_LEAF )
102 #define GinPageIsData(page) ( GinPageGetOpaque(page)->flags & GIN_DATA )
103 #define GinPageSetData(page) ( GinPageGetOpaque(page)->flags |= GIN_DATA )
104 #define GinPageHasFullRow(page) ( GinPageGetOpaque(page)->flags & GIN_LIST_FULLROW )
105 #define GinPageSetFullRow(page) ( GinPageGetOpaque(page)->flags |= GIN_LIST_FULLROW )
107 #define GinPageIsDeleted(page) ( GinPageGetOpaque(page)->flags & GIN_DELETED)
108 #define GinPageSetDeleted(page) ( GinPageGetOpaque(page)->flags |= GIN_DELETED)
109 #define GinPageSetNonDeleted(page) ( GinPageGetOpaque(page)->flags &= ~GIN_DELETED)
111 #define GinPageRightMost(page) ( GinPageGetOpaque(page)->rightlink == InvalidBlockNumber)
114 * We use our own ItemPointerGet(BlockNumber|GetOffsetNumber)
115 * to avoid Asserts, since sometimes the ip_posid isn't "valid"
118 #define GinItemPointerGetBlockNumber(pointer) \
119 BlockIdGetBlockNumber(&(pointer)->ip_blkid)
121 #define GinItemPointerGetOffsetNumber(pointer) \
122 ((pointer)->ip_posid)
124 #define ItemPointerSetMin(p) \
125 ItemPointerSet((p), (BlockNumber)0, (OffsetNumber)0)
126 #define ItemPointerIsMin(p) \
127 (GinItemPointerGetOffsetNumber(p) == (OffsetNumber)0 && \
128 GinItemPointerGetBlockNumber(p) == (BlockNumber)0)
129 #define ItemPointerSetMax(p) \
130 ItemPointerSet((p), InvalidBlockNumber, (OffsetNumber)0xffff)
131 #define ItemPointerIsMax(p) \
132 (GinItemPointerGetOffsetNumber(p) == (OffsetNumber)0xffff && \
133 GinItemPointerGetBlockNumber(p) == InvalidBlockNumber)
134 #define ItemPointerSetLossyPage(p, b) \
135 ItemPointerSet((p), (b), (OffsetNumber)0xffff)
136 #define ItemPointerIsLossyPage(p) \
137 (GinItemPointerGetOffsetNumber(p) == (OffsetNumber)0xffff && \
138 GinItemPointerGetBlockNumber(p) != InvalidBlockNumber)
142 BlockIdData child_blkno
; /* use it instead of BlockNumber to save space
147 #define PostingItemGetBlockNumber(pointer) \
148 BlockIdGetBlockNumber(&(pointer)->child_blkno)
150 #define PostingItemSetBlockNumber(pointer, blockNumber) \
151 BlockIdSet(&((pointer)->child_blkno), (blockNumber))
154 * Support work on IndexTuple on leaf pages
156 #define GinGetNPosting(itup) GinItemPointerGetOffsetNumber(&(itup)->t_tid)
157 #define GinSetNPosting(itup,n) ItemPointerSetOffsetNumber(&(itup)->t_tid,(n))
158 #define GIN_TREE_POSTING ((OffsetNumber)0xffff)
159 #define GinIsPostingTree(itup) ( GinGetNPosting(itup)==GIN_TREE_POSTING )
160 #define GinSetPostingTree(itup, blkno) ( GinSetNPosting((itup),GIN_TREE_POSTING ), ItemPointerSetBlockNumber(&(itup)->t_tid, blkno) )
161 #define GinGetPostingTree(itup) GinItemPointerGetBlockNumber(&(itup)->t_tid)
163 #define GinGetOrigSizePosting(itup) GinItemPointerGetBlockNumber(&(itup)->t_tid)
164 #define GinSetOrigSizePosting(itup,n) ItemPointerSetBlockNumber(&(itup)->t_tid,(n))
165 #define GinGetPosting(itup) ( (ItemPointer)(( ((char*)(itup)) + SHORTALIGN(GinGetOrigSizePosting(itup)) )) )
167 #define GinMaxItemSize \
168 ((BLCKSZ - SizeOfPageHeaderData - \
169 MAXALIGN(sizeof(GinPageOpaqueData))) / 3 - sizeof(ItemIdData))
173 * Data (posting tree) pages
175 #define GinDataPageGetRightBound(page) ((ItemPointer) PageGetContents(page))
176 #define GinDataPageGetData(page) \
177 (PageGetContents(page) + MAXALIGN(sizeof(ItemPointerData)))
178 #define GinSizeOfItem(page) \
179 (GinPageIsLeaf(page) ? sizeof(ItemPointerData) : sizeof(PostingItem))
180 #define GinDataPageGetItem(page,i) \
181 (GinDataPageGetData(page) + ((i)-1) * GinSizeOfItem(page))
183 #define GinDataPageGetFreeSpace(page) \
184 (BLCKSZ - MAXALIGN(SizeOfPageHeaderData) \
185 - MAXALIGN(sizeof(ItemPointerData)) \
186 - GinPageGetOpaque(page)->maxoff * GinSizeOfItem(page) \
187 - MAXALIGN(sizeof(GinPageOpaqueData)))
192 #define GinListPageSize \
193 ( BLCKSZ - SizeOfPageHeaderData - MAXALIGN(sizeof(GinPageOpaqueData)) )
196 * Storage type for GIN's reloptions
198 typedef struct GinOptions
200 int32 vl_len_
; /* varlena header (do not touch directly!) */
201 bool useFastUpdate
; /* use fast updates? */
204 #define GIN_DEFAULT_USE_FASTUPDATE true
205 #define GinGetUseFastUpdate(relation) \
206 ((relation)->rd_options ? \
207 ((GinOptions *) (relation)->rd_options)->useFastUpdate : GIN_DEFAULT_USE_FASTUPDATE)
210 #define GIN_UNLOCK BUFFER_LOCK_UNLOCK
211 #define GIN_SHARE BUFFER_LOCK_SHARE
212 #define GIN_EXCLUSIVE BUFFER_LOCK_EXCLUSIVE
214 typedef struct GinState
216 FmgrInfo compareFn
[INDEX_MAX_KEYS
];
217 FmgrInfo extractValueFn
[INDEX_MAX_KEYS
];
218 FmgrInfo extractQueryFn
[INDEX_MAX_KEYS
];
219 FmgrInfo consistentFn
[INDEX_MAX_KEYS
];
220 FmgrInfo comparePartialFn
[INDEX_MAX_KEYS
]; /* optional method */
222 bool canPartialMatch
[INDEX_MAX_KEYS
]; /* can opclass perform
223 * partial match (prefix
226 TupleDesc tupdesc
[INDEX_MAX_KEYS
];
227 TupleDesc origTupdesc
;
233 #define XLOG_GIN_CREATE_INDEX 0x00
235 #define XLOG_GIN_CREATE_PTREE 0x10
237 typedef struct ginxlogCreatePostingTree
242 /* follows list of heap's ItemPointer */
243 } ginxlogCreatePostingTree
;
245 #define XLOG_GIN_INSERT 0x20
247 typedef struct ginxlogInsert
251 BlockNumber updateBlkno
;
259 * follows: tuples or ItemPointerData or PostingItem or list of
264 #define XLOG_GIN_SPLIT 0x30
266 typedef struct ginxlogSplit
270 BlockNumber rootBlkno
;
273 OffsetNumber separator
;
280 BlockNumber leftChildBlkno
;
281 BlockNumber updateBlkno
;
283 ItemPointerData rightbound
; /* used only in posting tree */
284 /* follows: list of tuple or ItemPointerData or PostingItem */
287 #define XLOG_GIN_VACUUM_PAGE 0x40
289 typedef struct ginxlogVacuumPage
294 /* follows content of page */
297 #define XLOG_GIN_DELETE_PAGE 0x50
299 typedef struct ginxlogDeletePage
303 BlockNumber parentBlkno
;
304 OffsetNumber parentOffset
;
305 BlockNumber leftBlkno
;
306 BlockNumber rightLink
;
309 #define XLOG_GIN_UPDATE_META_PAGE 0x60
311 typedef struct ginxlogUpdateMeta
314 GinMetaPageData metadata
;
315 BlockNumber prevTail
;
316 BlockNumber newRightlink
;
317 int32 ntuples
; /* if ntuples > 0 then metadata.tail was
318 * updated with that many tuples; else new sub
319 * list was inserted */
320 /* array of inserted tuples follows */
323 #define XLOG_GIN_INSERT_LISTPAGE 0x70
325 typedef struct ginxlogInsertListPage
329 BlockNumber rightlink
;
331 /* array of inserted tuples follows */
332 } ginxlogInsertListPage
;
334 #define XLOG_GIN_DELETE_LISTPAGE 0x80
336 #define GIN_NDELETE_AT_ONCE 16
337 typedef struct ginxlogDeleteListPages
340 GinMetaPageData metadata
;
342 BlockNumber toDelete
[GIN_NDELETE_AT_ONCE
];
343 } ginxlogDeleteListPages
;
347 extern Datum
ginoptions(PG_FUNCTION_ARGS
);
348 extern void initGinState(GinState
*state
, Relation index
);
349 extern Buffer
GinNewBuffer(Relation index
);
350 extern void GinInitBuffer(Buffer b
, uint32 f
);
351 extern void GinInitPage(Page page
, uint32 f
, Size pageSize
);
352 extern void GinInitMetabuffer(Buffer b
);
353 extern int compareEntries(GinState
*ginstate
, OffsetNumber attnum
, Datum a
, Datum b
);
354 extern int compareAttEntries(GinState
*ginstate
, OffsetNumber attnum_a
, Datum a
,
355 OffsetNumber attnum_b
, Datum b
);
356 extern Datum
*extractEntriesS(GinState
*ginstate
, OffsetNumber attnum
, Datum value
,
357 int32
*nentries
, bool *needUnique
);
358 extern Datum
*extractEntriesSU(GinState
*ginstate
, OffsetNumber attnum
, Datum value
, int32
*nentries
);
360 extern Datum
gin_index_getattr(GinState
*ginstate
, IndexTuple tuple
);
361 extern OffsetNumber
gintuple_get_attrnum(GinState
*ginstate
, IndexTuple tuple
);
364 extern Datum
ginbuild(PG_FUNCTION_ARGS
);
365 extern Datum
gininsert(PG_FUNCTION_ARGS
);
366 extern void ginEntryInsert(Relation index
, GinState
*ginstate
,
367 OffsetNumber attnum
, Datum value
,
368 ItemPointerData
*items
, uint32 nitem
,
372 extern void gin_redo(XLogRecPtr lsn
, XLogRecord
*record
);
373 extern void gin_desc(StringInfo buf
, uint8 xl_info
, char *rec
);
374 extern void gin_xlog_startup(void);
375 extern void gin_xlog_cleanup(void);
376 extern bool gin_safe_restartpoint(void);
380 typedef struct GinBtreeStack
385 /* predictNumber contains prediction number of pages on current level */
386 uint32 predictNumber
;
387 struct GinBtreeStack
*parent
;
390 typedef struct GinBtreeData
*GinBtree
;
392 typedef struct GinBtreeData
395 BlockNumber (*findChildPage
) (GinBtree
, GinBtreeStack
*);
396 bool (*isMoveRight
) (GinBtree
, Page
);
397 bool (*findItem
) (GinBtree
, GinBtreeStack
*);
400 OffsetNumber (*findChildPtr
) (GinBtree
, Page
, BlockNumber
, OffsetNumber
);
401 BlockNumber (*getLeftMostPage
) (GinBtree
, Page
);
402 bool (*isEnoughSpace
) (GinBtree
, Buffer
, OffsetNumber
);
403 void (*placeToPage
) (GinBtree
, Buffer
, OffsetNumber
, XLogRecData
**);
404 Page (*splitPage
) (GinBtree
, Buffer
, Buffer
, OffsetNumber
, XLogRecData
**);
405 void (*fillRoot
) (GinBtree
, Buffer
, Buffer
, Buffer
);
414 BlockNumber rightblkno
;
417 OffsetNumber entryAttnum
;
422 /* Data (posting tree) option */
423 ItemPointerData
*items
;
430 extern GinBtreeStack
*ginPrepareFindLeafPage(GinBtree btree
, BlockNumber blkno
);
431 extern GinBtreeStack
*ginFindLeafPage(GinBtree btree
, GinBtreeStack
*stack
);
432 extern void freeGinBtreeStack(GinBtreeStack
*stack
);
433 extern void ginInsertValue(GinBtree btree
, GinBtreeStack
*stack
);
434 extern void findParents(GinBtree btree
, GinBtreeStack
*stack
, BlockNumber rootBlkno
);
437 extern IndexTuple
GinFormTuple(GinState
*ginstate
, OffsetNumber attnum
, Datum key
,
438 ItemPointerData
*ipd
, uint32 nipd
);
439 extern void GinShortenTuple(IndexTuple itup
, uint32 nipd
);
440 extern void prepareEntryScan(GinBtree btree
, Relation index
, OffsetNumber attnum
,
441 Datum value
, GinState
*ginstate
);
442 extern void entryFillRoot(GinBtree btree
, Buffer root
, Buffer lbuf
, Buffer rbuf
);
443 extern IndexTuple
ginPageGetLinkItup(Buffer buf
);
446 extern int compareItemPointers(ItemPointer a
, ItemPointer b
);
447 extern uint32
MergeItemPointers(ItemPointerData
*dst
,
448 ItemPointerData
*a
, uint32 na
,
449 ItemPointerData
*b
, uint32 nb
);
451 extern void GinDataPageAddItem(Page page
, void *data
, OffsetNumber offset
);
452 extern void PageDeletePostingItem(Page page
, OffsetNumber offset
);
457 GinBtreeStack
*stack
;
458 } GinPostingTreeScan
;
460 extern GinPostingTreeScan
*prepareScanPostingTree(Relation index
,
461 BlockNumber rootBlkno
, bool searchMode
);
462 extern void insertItemPointer(GinPostingTreeScan
*gdi
,
463 ItemPointerData
*items
, uint32 nitem
);
464 extern Buffer
scanBeginPostingTree(GinPostingTreeScan
*gdi
);
465 extern void dataFillRoot(GinBtree btree
, Buffer root
, Buffer lbuf
, Buffer rbuf
);
466 extern void prepareDataScan(GinBtree btree
, Relation index
);
470 typedef struct GinScanEntryData
*GinScanEntry
;
472 typedef struct GinScanEntryData
474 /* link to the equals entry in current scan key */
478 * link to values reported to consistentFn, points to
479 * GinScanKey->entryRes[i]
483 /* entry, got from extractQueryFn */
488 /* Current page in posting tree */
491 /* current ItemPointer to heap */
492 ItemPointerData curItem
;
494 /* partial match support */
496 TIDBitmap
*partialMatch
;
497 TBMIterator
*partialMatchIterator
;
498 TBMIterateResult
*partialMatchResult
;
499 StrategyNumber strategy
;
501 /* used for Posting list and one page in Posting tree */
502 ItemPointerData
*list
;
508 uint32 predictNumberResult
;
511 typedef struct GinScanKeyData
513 /* Number of entries in query (got by extractQueryFn) */
516 /* array of ItemPointer result, reported to consistentFn */
519 /* array of scans per entry */
520 GinScanEntry scanEntry
;
523 /* for calling consistentFn(GinScanKey->entryRes, strategy, query) */
524 StrategyNumber strategy
;
528 ItemPointerData curItem
;
533 typedef GinScanKeyData
*GinScanKey
;
535 typedef struct GinScanOpaqueData
537 MemoryContext tempCtx
;
542 bool isVoidRes
; /* true if ginstate.extractQueryFn guarantees
543 * that nothing will be found */
546 typedef GinScanOpaqueData
*GinScanOpaque
;
548 extern Datum
ginbeginscan(PG_FUNCTION_ARGS
);
549 extern Datum
ginendscan(PG_FUNCTION_ARGS
);
550 extern Datum
ginrescan(PG_FUNCTION_ARGS
);
551 extern Datum
ginmarkpos(PG_FUNCTION_ARGS
);
552 extern Datum
ginrestrpos(PG_FUNCTION_ARGS
);
553 extern void newScanKey(IndexScanDesc scan
);
556 extern PGDLLIMPORT
int GinFuzzySearchLimit
;
558 extern Datum
gingetbitmap(PG_FUNCTION_ARGS
);
561 extern Datum
ginbulkdelete(PG_FUNCTION_ARGS
);
562 extern Datum
ginvacuumcleanup(PG_FUNCTION_ARGS
);
565 extern Datum
ginarrayextract(PG_FUNCTION_ARGS
);
566 extern Datum
ginqueryarrayextract(PG_FUNCTION_ARGS
);
567 extern Datum
ginarrayconsistent(PG_FUNCTION_ARGS
);
570 typedef struct EntryAccumulator
576 ItemPointerData
*list
;
578 struct EntryAccumulator
*left
;
579 struct EntryAccumulator
*right
;
585 EntryAccumulator
*entries
;
587 EntryAccumulator
**stack
;
589 long allocatedMemory
;
592 EntryAccumulator
*entryallocator
;
595 extern void ginInitBA(BuildAccumulator
*accum
);
596 extern void ginInsertRecordBA(BuildAccumulator
*accum
,
598 OffsetNumber attnum
, Datum
*entries
, int32 nentry
);
599 extern ItemPointerData
*ginGetEntry(BuildAccumulator
*accum
, OffsetNumber
*attnum
, Datum
*entry
, uint32
*n
);
603 typedef struct GinTupleCollector
611 extern void ginHeapTupleFastInsert(Relation index
, GinState
*ginstate
,
612 GinTupleCollector
*collector
);
613 extern uint32
ginHeapTupleFastCollect(Relation index
, GinState
*ginstate
,
614 GinTupleCollector
*collector
,
615 OffsetNumber attnum
, Datum value
, ItemPointer item
);
616 extern void ginInsertCleanup(Relation index
, GinState
*ginstate
,
617 bool vac_delay
, IndexBulkDeleteResult
*stats
);