third_party/sqlite/patches/0014-fts2-Detect-and-handle-certain-corruption-cases.patch

   1 From c0a3343f2b446088f8f721baf8f53e20e8ce4fb5 Mon Sep 17 00:00:00 2001
   2 From: Scott Hess <shess@chromium.org>
   3 Date: Thu, 18 Sep 2008 17:00:30 -0700
   4 Subject: [PATCH 14/16] [fts2] Detect and handle certain corruption cases.
   5
   6 Detect and handle certain corruption cases for fts2,
   7 concentrating on high-level structural issues not trying to
   8 detect corruption when decoding, for now.  These cases handle
   9 all the in-the-wild corruption examples I have but one.
  10
  11 - During a query, detect when the fts index references a docid
  12   which doesn't exist in the content table.
  13
  14 - Detect when leavesReaderInit() receives an empty leaf node,
  15   or a leaf node which begins with a character other than nul.
  16   Similar mod to leavesReaderStep().
  17
  18 - Similar changes to interior-node handling in
  19   loadAndGetChildrenContaining(), and loadSegment().
  20
  21 - In various places detects if values of the wrong type are
  22   received from SQL queries.  I have audited the code, and all
  23   stored values are bound using a type appropriate to their
  24   column, so any mismatch means that we are either reading
  25   random data, or valid SQLite data from a page which wasn't
  26   previously part of an fts table (as if a page from another
  27   table were re-used).
  28
  29 Original Gears CL:
  30 https://code.google.com/p/gears/source/detail?r=2855&path=/trunk/third_party/sqlite_google/ext/fts2/fts2.c
  31 ---
  32  third_party/sqlite/src/ext/fts2/fts2.c | 152 ++++++++++++++++++++++++++++-----
  33  1 file changed, 132 insertions(+), 20 deletions(-)
  34
  35 diff --git a/third_party/sqlite/src/ext/fts2/fts2.c b/third_party/sqlite/src/ext/fts2/fts2.c
  36 index 1c68af5..5cb3fc6 100644
  37 --- a/third_party/sqlite/src/ext/fts2/fts2.c
  38 +++ b/third_party/sqlite/src/ext/fts2/fts2.c
  39 @@ -349,6 +349,16 @@
  40  # define TRACE(A)
  41  #endif
  42
  43 +#if 0
  44 +/* Useful to set breakpoints.  See main.c sqlite3Corrupt(). */
  45 +static int fts2Corrupt(void){
  46 +  return SQLITE_CORRUPT;
  47 +}
  48 +# define SQLITE_CORRUPT_BKPT fts2Corrupt()
  49 +#else
  50 +# define SQLITE_CORRUPT_BKPT SQLITE_CORRUPT
  51 +#endif
  52 +
  53  /* It is not safe to call isspace(), tolower(), or isalnum() on
  54  ** hi-bit-set characters.  This is the same solution used in the
  55  ** tokenizer.
  56 @@ -3435,8 +3445,11 @@ static int fulltextNext(sqlite3_vtab_cursor *pCursor){
  57        c->eof = 0;
  58        return SQLITE_OK;
  59      }
  60 -    /* an error occurred; abort */
  61 -    return rc==SQLITE_DONE ? SQLITE_ERROR : rc;
  62 +
  63 +    /* Corrupt if the index refers to missing document. */
  64 +    if( rc==SQLITE_DONE ) return SQLITE_CORRUPT_BKPT;
  65 +
  66 +    return rc;
  67    }
  68  }
  69
  70 @@ -5106,6 +5119,9 @@ static void leavesReaderDestroy(LeavesReader *pReader){
  71  ** the leaf data was entirely contained in the root), or from the
  72  ** stream of blocks between iStartBlockid and iEndBlockid, inclusive.
  73  */
  74 +/* TODO(shess): Figure out a means of indicating how many leaves are
  75 +** expected, for purposes of detecting corruption.
  76 +*/
  77  static int leavesReaderInit(fulltext_vtab *v,
  78                              int idx,
  79                              sqlite_int64 iStartBlockid,
  80 @@ -5117,6 +5133,10 @@ static int leavesReaderInit(fulltext_vtab *v,
  81
  82    dataBufferInit(&pReader->rootData, 0);
  83    if( iStartBlockid==0 ){
  84 +    /* Corrupt if this can't be a leaf node. */
  85 +    if( pRootData==NULL || nRootData<1 || pRootData[0]!='\0' ){
  86 +      return SQLITE_CORRUPT_BKPT;
  87 +    }
  88      /* Entire leaf level fit in root data. */
  89      dataBufferReplace(&pReader->rootData, pRootData, nRootData);
  90      leafReaderInit(pReader->rootData.pData, pReader->rootData.nData,
  91 @@ -5127,22 +5147,48 @@ static int leavesReaderInit(fulltext_vtab *v,
  92      if( rc!=SQLITE_OK ) return rc;
  93
  94      rc = sqlite3_bind_int64(s, 1, iStartBlockid);
  95 -    if( rc!=SQLITE_OK ) return rc;
  96 +    if( rc!=SQLITE_OK ) goto err;
  97
  98      rc = sqlite3_bind_int64(s, 2, iEndBlockid);
  99 -    if( rc!=SQLITE_OK ) return rc;
 100 +    if( rc!=SQLITE_OK ) goto err;
 101
 102      rc = sqlite3_step(s);
 103 +
 104 +    /* Corrupt if interior node referenced missing leaf node. */
 105      if( rc==SQLITE_DONE ){
 106 -      pReader->eof = 1;
 107 -      return SQLITE_OK;
 108 +      rc = SQLITE_CORRUPT_BKPT;
 109 +      goto err;
 110 +    }
 111 +
 112 +    if( rc!=SQLITE_ROW ) goto err;
 113 +    rc = SQLITE_OK;
 114 +
 115 +    /* Corrupt if leaf data isn't a blob. */
 116 +    if( sqlite3_column_type(s, 0)!=SQLITE_BLOB ){
 117 +      rc = SQLITE_CORRUPT_BKPT;
 118 +    }else{
 119 +      const char *pLeafData = sqlite3_column_blob(s, 0);
 120 +      int nLeafData = sqlite3_column_bytes(s, 0);
 121 +
 122 +      /* Corrupt if this can't be a leaf node. */
 123 +      if( pLeafData==NULL || nLeafData<1 || pLeafData[0]!='\0' ){
 124 +        rc = SQLITE_CORRUPT_BKPT;
 125 +      }else{
 126 +        leafReaderInit(pLeafData, nLeafData, &pReader->leafReader);
 127 +      }
 128 +    }
 129 +
 130 + err:
 131 +    if( rc!=SQLITE_OK ){
 132 +      if( idx==-1 ){
 133 +        sqlite3_finalize(s);
 134 +      }else{
 135 +        sqlite3_reset(s);
 136 +      }
 137 +      return rc;
 138      }
 139 -    if( rc!=SQLITE_ROW ) return rc;
 140
 141      pReader->pStmt = s;
 142 -    leafReaderInit(sqlite3_column_blob(pReader->pStmt, 0),
 143 -                   sqlite3_column_bytes(pReader->pStmt, 0),
 144 -                   &pReader->leafReader);
 145    }
 146    return SQLITE_OK;
 147  }
 148 @@ -5165,10 +5211,22 @@ static int leavesReaderStep(fulltext_vtab *v, LeavesReader *pReader){
 149        pReader->eof = 1;
 150        return rc==SQLITE_DONE ? SQLITE_OK : rc;
 151      }
 152 -    leafReaderDestroy(&pReader->leafReader);
 153 -    leafReaderInit(sqlite3_column_blob(pReader->pStmt, 0),
 154 -                   sqlite3_column_bytes(pReader->pStmt, 0),
 155 -                   &pReader->leafReader);
 156 +
 157 +    /* Corrupt if leaf data isn't a blob. */
 158 +    if( sqlite3_column_type(pReader->pStmt, 0)!=SQLITE_BLOB ){
 159 +      return SQLITE_CORRUPT_BKPT;
 160 +    }else{
 161 +      const char *pLeafData = sqlite3_column_blob(pReader->pStmt, 0);
 162 +      int nLeafData = sqlite3_column_bytes(pReader->pStmt, 0);
 163 +
 164 +      /* Corrupt if this can't be a leaf node. */
 165 +      if( pLeafData==NULL || nLeafData<1 || pLeafData[0]!='\0' ){
 166 +        return SQLITE_CORRUPT_BKPT;
 167 +      }
 168 +
 169 +      leafReaderDestroy(&pReader->leafReader);
 170 +      leafReaderInit(pLeafData, nLeafData, &pReader->leafReader);
 171 +    }
 172    }
 173    return SQLITE_OK;
 174  }
 175 @@ -5230,6 +5288,14 @@ static int leavesReadersInit(fulltext_vtab *v, int iLevel,
 176      const char *pRootData = sqlite3_column_blob(s, 2);
 177      int nRootData = sqlite3_column_bytes(s, 2);
 178
 179 +    /* Corrupt if we get back different types than we stored. */
 180 +    if( sqlite3_column_type(s, 0)!=SQLITE_INTEGER ||
 181 +        sqlite3_column_type(s, 1)!=SQLITE_INTEGER ||
 182 +        sqlite3_column_type(s, 2)!=SQLITE_BLOB ){
 183 +      rc = SQLITE_CORRUPT_BKPT;
 184 +      break;
 185 +    }
 186 +
 187      assert( i<MERGE_COUNT );
 188      rc = leavesReaderInit(v, i, iStart, iEnd, pRootData, nRootData,
 189                            &pReaders[i]);
 190 @@ -5241,6 +5307,7 @@ static int leavesReadersInit(fulltext_vtab *v, int iLevel,
 191      while( i-->0 ){
 192        leavesReaderDestroy(&pReaders[i]);
 193      }
 194 +    sqlite3_reset(s);          /* So we don't leave a lock. */
 195      return rc;
 196    }
 197
 198 @@ -5617,11 +5684,27 @@ static int loadAndGetChildrenContaining(
 199    if( rc!=SQLITE_OK ) return rc;
 200
 201    rc = sqlite3_step(s);
 202 -  if( rc==SQLITE_DONE ) return SQLITE_ERROR;
 203 +  /* Corrupt if interior node references missing child node. */
 204 +  if( rc==SQLITE_DONE ) return SQLITE_CORRUPT_BKPT;
 205    if( rc!=SQLITE_ROW ) return rc;
 206
 207 -  getChildrenContaining(sqlite3_column_blob(s, 0), sqlite3_column_bytes(s, 0),
 208 -                        pTerm, nTerm, isPrefix, piStartChild, piEndChild);
 209 +  /* Corrupt if child node isn't a blob. */
 210 +  if( sqlite3_column_type(s, 0)!=SQLITE_BLOB ){
 211 +    sqlite3_reset(s);         /* So we don't leave a lock. */
 212 +    return SQLITE_CORRUPT_BKPT;
 213 +  }else{
 214 +    const char *pData = sqlite3_column_blob(s, 0);
 215 +    int nData = sqlite3_column_bytes(s, 0);
 216 +
 217 +    /* Corrupt if child is not a valid interior node. */
 218 +    if( pData==NULL || nData<1 || pData[0]=='\0' ){
 219 +      sqlite3_reset(s);         /* So we don't leave a lock. */
 220 +      return SQLITE_CORRUPT_BKPT;
 221 +    }
 222 +
 223 +    getChildrenContaining(pData, nData, pTerm, nTerm,
 224 +                          isPrefix, piStartChild, piEndChild);
 225 +  }
 226
 227    /* We expect only one row.  We must execute another sqlite3_step()
 228     * to complete the iteration; otherwise the table will remain
 229 @@ -5704,7 +5787,8 @@ static int loadSegment(fulltext_vtab *v, const char *pData, int nData,
 230    DataBuffer result;
 231    int rc;
 232
 233 -  assert( nData>1 );
 234 +  /* Corrupt if segment root can't be valid. */
 235 +  if( pData==NULL || nData<1 ) return SQLITE_CORRUPT_BKPT;
 236
 237    /* This code should never be called with buffered updates. */
 238    assert( v->nPendingData<0 );
 239 @@ -5758,6 +5842,14 @@ static int termSelect(fulltext_vtab *v, int iColumn,
 240      const char *pData = sqlite3_column_blob(s, 2);
 241      const int nData = sqlite3_column_bytes(s, 2);
 242      const sqlite_int64 iLeavesEnd = sqlite3_column_int64(s, 1);
 243 +
 244 +    /* Corrupt if we get back different types than we stored. */
 245 +    if( sqlite3_column_type(s, 1)!=SQLITE_INTEGER ||
 246 +        sqlite3_column_type(s, 2)!=SQLITE_BLOB ){
 247 +      rc = SQLITE_CORRUPT_BKPT;
 248 +      goto err;
 249 +    }
 250 +
 251      rc = loadSegment(v, pData, nData, iLeavesEnd, pTerm, nTerm, isPrefix,
 252                       &doclist);
 253      if( rc!=SQLITE_OK ) goto err;
 254 @@ -5777,6 +5869,7 @@ static int termSelect(fulltext_vtab *v, int iColumn,
 255    }
 256
 257   err:
 258 +  sqlite3_reset(s);         /* So we don't leave a lock. */
 259    dataBufferDestroy(&doclist);
 260    return rc;
 261  }
 262 @@ -6269,6 +6362,14 @@ static void optimizeFunc(sqlite3_context *pContext,
 263        const char *pRootData = sqlite3_column_blob(s, 2);
 264        int nRootData = sqlite3_column_bytes(s, 2);
 265
 266 +      /* Corrupt if we get back different types than we stored. */
 267 +      if( sqlite3_column_type(s, 0)!=SQLITE_INTEGER ||
 268 +          sqlite3_column_type(s, 1)!=SQLITE_INTEGER ||
 269 +          sqlite3_column_type(s, 2)!=SQLITE_BLOB ){
 270 +        rc = SQLITE_CORRUPT_BKPT;
 271 +        break;
 272 +      }
 273 +
 274        assert( i<nReaders );
 275        rc = leavesReaderInit(v, -1, iStart, iEnd, pRootData, nRootData,
 276                              &readers[i].reader);
 277 @@ -6282,6 +6383,8 @@ static void optimizeFunc(sqlite3_context *pContext,
 278      if( rc==SQLITE_DONE ){
 279        assert( i==nReaders );
 280        rc = optimizeInternal(v, readers, nReaders, &writer);
 281 +    }else{
 282 +      sqlite3_reset(s);      /* So we don't leave a lock. */
 283      }
 284
 285      while( i-- > 0 ){
 286 @@ -6345,9 +6448,18 @@ static int collectSegmentTerms(fulltext_vtab *v, sqlite3_stmt *s,
 287    const sqlite_int64 iEndBlockid = sqlite3_column_int64(s, 1);
 288    const char *pRootData = sqlite3_column_blob(s, 2);
 289    const int nRootData = sqlite3_column_bytes(s, 2);
 290 +  int rc;
 291    LeavesReader reader;
 292 -  int rc = leavesReaderInit(v, 0, iStartBlockid, iEndBlockid,
 293 -                            pRootData, nRootData, &reader);
 294 +
 295 +  /* Corrupt if we get back different types than we stored. */
 296 +  if( sqlite3_column_type(s, 0)!=SQLITE_INTEGER ||
 297 +      sqlite3_column_type(s, 1)!=SQLITE_INTEGER ||
 298 +      sqlite3_column_type(s, 2)!=SQLITE_BLOB ){
 299 +    return SQLITE_CORRUPT_BKPT;
 300 +  }
 301 +
 302 +  rc = leavesReaderInit(v, 0, iStartBlockid, iEndBlockid,
 303 +                        pRootData, nRootData, &reader);
 304    if( rc!=SQLITE_OK ) return rc;
 305
 306    while( rc==SQLITE_OK && !leavesReaderAtEnd(&reader) ){
 307 --
 308 2.2.1
 309