ext/lsm1/lsm-test/lsmtest_main.c

   1
   2 #include "lsmtest.h"
   3 #include <sqlite3.h>
   4
   5 void test_failed(){
   6   assert( 0 );
   7   return;
   8 }
   9
  10 #define testSetError(rc) testSetErrorFunc(rc, pRc, __FILE__, __LINE__)
  11 static void testSetErrorFunc(int rc, int *pRc, const char *zFile, int iLine){
  12   if( rc ){
  13     *pRc = rc;
  14     fprintf(stderr, "FAILED (%s:%d) rc=%d ", zFile, iLine, rc);
  15     test_failed();
  16   }
  17 }
  18
  19 static int lsm_memcmp(u8 *a, u8 *b, int c){
  20   int i;
  21   for(i=0; i<c; i++){
  22     if( a[i]!=b[i] ) return a[i] - b[i];
  23   }
  24   return 0;
  25 }
  26
  27 /*
  28 ** A test utility function.
  29 */
  30 void testFetch(
  31   TestDb *pDb,                    /* Database handle */
  32   void *pKey, int nKey,           /* Key to query database for */
  33   void *pVal, int nVal,           /* Expected value */
  34   int *pRc                        /* IN/OUT: Error code */
  35 ){
  36   if( *pRc==0 ){
  37     void *pDbVal;
  38     int nDbVal;
  39     int rc;
  40
  41     static int nCall = 0; nCall++;
  42
  43     rc = tdb_fetch(pDb, pKey, nKey, &pDbVal, &nDbVal);
  44     testSetError(rc);
  45     if( rc==0 && (nVal!=nDbVal || (nVal>0 && lsm_memcmp(pVal, pDbVal, nVal))) ){
  46       testSetError(1);
  47     }
  48   }
  49 }
  50
  51 void testWrite(
  52   TestDb *pDb,                    /* Database handle */
  53   void *pKey, int nKey,           /* Key to query database for */
  54   void *pVal, int nVal,           /* Value to write */
  55   int *pRc                        /* IN/OUT: Error code */
  56 ){
  57   if( *pRc==0 ){
  58     int rc;
  59 static int nCall = 0;
  60 nCall++;
  61     rc = tdb_write(pDb, pKey, nKey, pVal, nVal);
  62     testSetError(rc);
  63   }
  64 }
  65 void testDelete(
  66   TestDb *pDb,                    /* Database handle */
  67   void *pKey, int nKey,           /* Key to query database for */
  68   int *pRc                        /* IN/OUT: Error code */
  69 ){
  70   if( *pRc==0 ){
  71     int rc;
  72     *pRc = rc = tdb_delete(pDb, pKey, nKey);
  73     testSetError(rc);
  74   }
  75 }
  76 void testDeleteRange(
  77   TestDb *pDb,                    /* Database handle */
  78   void *pKey1, int nKey1,
  79   void *pKey2, int nKey2,
  80   int *pRc                        /* IN/OUT: Error code */
  81 ){
  82   if( *pRc==0 ){
  83     int rc;
  84     *pRc = rc = tdb_delete_range(pDb, pKey1, nKey1, pKey2, nKey2);
  85     testSetError(rc);
  86   }
  87 }
  88
  89 void testBegin(TestDb *pDb, int iTrans, int *pRc){
  90   if( *pRc==0 ){
  91     int rc;
  92     rc = tdb_begin(pDb, iTrans);
  93     testSetError(rc);
  94   }
  95 }
  96 void testCommit(TestDb *pDb, int iTrans, int *pRc){
  97   if( *pRc==0 ){
  98     int rc;
  99     rc = tdb_commit(pDb, iTrans);
 100     testSetError(rc);
 101   }
 102 }
 103 #if 0 /* unused */
 104 static void testRollback(TestDb *pDb, int iTrans, int *pRc){
 105   if( *pRc==0 ){
 106     int rc;
 107     rc = tdb_rollback(pDb, iTrans);
 108     testSetError(rc);
 109   }
 110 }
 111 #endif
 112
 113 void testWriteStr(
 114   TestDb *pDb,                    /* Database handle */
 115   const char *zKey,               /* Key to query database for */
 116   const char *zVal,               /* Value to write */
 117   int *pRc                        /* IN/OUT: Error code */
 118 ){
 119   int nVal = (zVal ? strlen(zVal) : 0);
 120   testWrite(pDb, (void *)zKey, strlen(zKey), (void *)zVal, nVal, pRc);
 121 }
 122
 123 #if 0 /* unused */
 124 static void testDeleteStr(TestDb *pDb, const char *zKey, int *pRc){
 125   testDelete(pDb, (void *)zKey, strlen(zKey), pRc);
 126 }
 127 #endif
 128 void testFetchStr(
 129   TestDb *pDb,                    /* Database handle */
 130   const char *zKey,               /* Key to query database for */
 131   const char *zVal,               /* Value to write */
 132   int *pRc                        /* IN/OUT: Error code */
 133 ){
 134   int nVal = (zVal ? strlen(zVal) : 0);
 135   testFetch(pDb, (void *)zKey, strlen(zKey), (void *)zVal, nVal, pRc);
 136 }
 137
 138 void testFetchCompare(
 139   TestDb *pControl,
 140   TestDb *pDb,
 141   void *pKey, int nKey,
 142   int *pRc
 143 ){
 144   int rc;
 145   void *pDbVal1;
 146   void *pDbVal2;
 147   int nDbVal1;
 148   int nDbVal2;
 149
 150   static int nCall = 0;
 151   nCall++;
 152
 153   rc = tdb_fetch(pControl, pKey, nKey, &pDbVal1, &nDbVal1);
 154   testSetError(rc);
 155
 156   rc = tdb_fetch(pDb, pKey, nKey, &pDbVal2, &nDbVal2);
 157   testSetError(rc);
 158
 159   if( *pRc==0
 160    && (nDbVal1!=nDbVal2 || (nDbVal1>0 && memcmp(pDbVal1, pDbVal2, nDbVal1)))
 161   ){
 162     testSetError(1);
 163   }
 164 }
 165
 166 typedef struct ScanResult ScanResult;
 167 struct ScanResult {
 168   TestDb *pDb;
 169
 170   int nRow;
 171   u32 cksum1;
 172   u32 cksum2;
 173   void *pKey1; int nKey1;
 174   void *pKey2; int nKey2;
 175
 176   int bReverse;
 177   int nPrevKey;
 178   u8 aPrevKey[256];
 179 };
 180
 181 static int keyCompare(void *pKey1, int nKey1, void *pKey2, int nKey2){
 182   int res;
 183   res = memcmp(pKey1, pKey2, MIN(nKey1, nKey2));
 184   if( res==0 ){
 185     res = nKey1 - nKey2;
 186   }
 187   return res;
 188 }
 189
 190 int test_scan_debug = 0;
 191
 192 static void scanCompareCb(
 193   void *pCtx,
 194   void *pKey, int nKey,
 195   void *pVal, int nVal
 196 ){
 197   ScanResult *p = (ScanResult *)pCtx;
 198   u8 *aKey = (u8 *)pKey;
 199   u8 *aVal = (u8 *)pVal;
 200   int i;
 201
 202   if( test_scan_debug ){
 203     printf("%d: %.*s\n", p->nRow, nKey, (char *)pKey);
 204     fflush(stdout);
 205   }
 206 #if 0
 207   if( test_scan_debug ) printf("%.20s\n", (char *)pVal);
 208 #endif
 209
 210 #if 0
 211   /* Check tdb_fetch() matches */
 212   int rc = 0;
 213   testFetch(p->pDb, pKey, nKey, pVal, nVal, &rc);
 214   assert( rc==0 );
 215 #endif
 216
 217   /* Update the checksum data */
 218   p->nRow++;
 219   for(i=0; i<nKey; i++){
 220     p->cksum1 += ((int)aKey[i] << (i&0x0F));
 221     p->cksum2 += p->cksum1;
 222   }
 223   for(i=0; i<nVal; i++){
 224     p->cksum1 += ((int)aVal[i] << (i&0x0F));
 225     p->cksum2 += p->cksum1;
 226   }
 227
 228   /* Check that the delivered row is not out of order. */
 229   if( nKey<(int)sizeof(p->aPrevKey) ){
 230     if( p->nPrevKey ){
 231       int res = keyCompare(p->aPrevKey, p->nPrevKey, pKey, nKey);
 232       if( (res<0 && p->bReverse) || (res>0 && p->bReverse==0) ){
 233         testPrintError("Returned key out of order at %s:%d\n",
 234             __FILE__, __LINE__
 235         );
 236       }
 237     }
 238
 239     p->nPrevKey = nKey;
 240     memcpy(p->aPrevKey, pKey, MIN(p->nPrevKey, nKey));
 241   }
 242
 243   /* Check that the delivered row is within range. */
 244   if( p->pKey1 && (
 245       (memcmp(p->pKey1, pKey, MIN(p->nKey1, nKey))>0)
 246    || (memcmp(p->pKey1, pKey, MIN(p->nKey1, nKey))==0 && p->nKey1>nKey)
 247   )){
 248     testPrintError("Returned key too small at %s:%d\n", __FILE__, __LINE__);
 249   }
 250   if( p->pKey2 && (
 251       (memcmp(p->pKey2, pKey, MIN(p->nKey2, nKey))<0)
 252    || (memcmp(p->pKey2, pKey, MIN(p->nKey2, nKey))==0 && p->nKey2<nKey)
 253   )){
 254     testPrintError("Returned key too large at %s:%d\n", __FILE__, __LINE__);
 255   }
 256
 257 }
 258
 259 /*
 260 ** Scan the contents of the two databases. Check that they match.
 261 */
 262 void testScanCompare(
 263   TestDb *pDb1,                   /* Control (trusted) database */
 264   TestDb *pDb2,                   /* Database being tested */
 265   int bReverse,
 266   void *pKey1, int nKey1,
 267   void *pKey2, int nKey2,
 268   int *pRc
 269 ){
 270   static int nCall = 0; nCall++;
 271   if( *pRc==0 ){
 272     ScanResult res1;
 273     ScanResult res2;
 274     void *pRes1 = (void *)&res1;
 275     void *pRes2 = (void *)&res2;
 276
 277     memset(&res1, 0, sizeof(ScanResult));
 278     memset(&res2, 0, sizeof(ScanResult));
 279
 280     res1.pDb = pDb1;
 281     res1.nKey1 = nKey1; res1.pKey1 = pKey1;
 282     res1.nKey2 = nKey2; res1.pKey2 = pKey2;
 283     res1.bReverse = bReverse;
 284     res2.pDb = pDb2;
 285     res2.nKey1 = nKey1; res2.pKey1 = pKey1;
 286     res2.nKey2 = nKey2; res2.pKey2 = pKey2;
 287     res2.bReverse = bReverse;
 288
 289     tdb_scan(pDb1, pRes1, bReverse, pKey1, nKey1, pKey2, nKey2, scanCompareCb);
 290 if( test_scan_debug ) printf("\n\n\n");
 291     tdb_scan(pDb2, pRes2, bReverse, pKey1, nKey1, pKey2, nKey2, scanCompareCb);
 292 if( test_scan_debug ) printf("\n\n\n");
 293
 294     if( res1.nRow!=res2.nRow
 295      || res1.cksum1!=res2.cksum1
 296      || res1.cksum2!=res2.cksum2
 297     ){
 298       printf("expected: %d %X %X\n", res1.nRow, res1.cksum1, res1.cksum2);
 299       printf("got:      %d %X %X\n", res2.nRow, res2.cksum1, res2.cksum2);
 300       testSetError(1);
 301       *pRc = 1;
 302     }
 303   }
 304 }
 305
 306 void testClose(TestDb **ppDb){
 307   tdb_close(*ppDb);
 308   *ppDb = 0;
 309 }
 310
 311 TestDb *testOpen(const char *zSystem, int bClear, int *pRc){
 312   TestDb *pDb = 0;
 313   if( *pRc==0 ){
 314     int rc;
 315     rc = tdb_open(zSystem, 0, bClear, &pDb);
 316     if( rc!=0 ){
 317       testSetError(rc);
 318       *pRc = rc;
 319     }
 320   }
 321   return pDb;
 322 }
 323
 324 void testReopen(TestDb **ppDb, int *pRc){
 325   if( *pRc==0 ){
 326     const char *zLib;
 327     zLib = tdb_library_name(*ppDb);
 328     testClose(ppDb);
 329     *pRc = tdb_open(zLib, 0, 0, ppDb);
 330   }
 331 }
 332
 333
 334 #if 0 /* unused */
 335 static void testSystemSelect(const char *zSys, int *piSel, int *pRc){
 336   if( *pRc==0 ){
 337     struct SysName { const char *zName; } *aName;
 338     int nSys;
 339     int i;
 340
 341     for(nSys=0; tdb_system_name(nSys); nSys++);
 342     aName = malloc(sizeof(struct SysName) * (nSys+1));
 343     for(i=0; i<=nSys; i++){
 344       aName[i].zName = tdb_system_name(i);
 345     }
 346
 347     *pRc = testArgSelect(aName, "db", zSys, piSel);
 348     free(aName);
 349   }
 350 }
 351 #endif
 352
 353 char *testMallocVPrintf(const char *zFormat, va_list ap){
 354   int nByte;
 355   va_list copy;
 356   char *zRet;
 357
 358   __va_copy(copy, ap);
 359   nByte = vsnprintf(0, 0, zFormat, copy);
 360   va_end(copy);
 361
 362   assert( nByte>=0 );
 363   zRet = (char *)testMalloc(nByte+1);
 364   vsnprintf(zRet, nByte+1, zFormat, ap);
 365   return zRet;
 366 }
 367
 368 char *testMallocPrintf(const char *zFormat, ...){
 369   va_list ap;
 370   char *zRet;
 371
 372   va_start(ap, zFormat);
 373   zRet = testMallocVPrintf(zFormat, ap);
 374   va_end(ap);
 375
 376   return zRet;
 377 }
 378
 379
 380 /*
 381 ** A wrapper around malloc(3).
 382 **
 383 ** This function should be used for all allocations made by test procedures.
 384 ** It has the following properties:
 385 **
 386 **   * Test code may assume that allocations may not fail.
 387 **   * Returned memory is always zeroed.
 388 **
 389 ** Allocations made using testMalloc() should be freed using testFree().
 390 */
 391 void *testMalloc(int n){
 392   u8 *p = (u8*)malloc(n + 8);
 393   memset(p, 0, n+8);
 394   *(int*)p = n;
 395   return (void*)&p[8];
 396 }
 397
 398 void *testMallocCopy(void *pCopy, int nByte){
 399   void *pRet = testMalloc(nByte);
 400   memcpy(pRet, pCopy, nByte);
 401   return pRet;
 402 }
 403
 404 void *testRealloc(void *ptr, int n){
 405   if( ptr ){
 406     u8 *p = (u8*)ptr - 8;
 407     int nOrig =  *(int*)p;
 408     p = (u8*)realloc(p, n+8);
 409     if( nOrig<n ){
 410       memset(&p[8+nOrig], 0, n-nOrig);
 411     }
 412     *(int*)p = n;
 413     return (void*)&p[8];
 414   }
 415   return testMalloc(n);
 416 }
 417
 418 /*
 419 ** Free an allocation made by an earlier call to testMalloc().
 420 */
 421 void testFree(void *ptr){
 422   if( ptr ){
 423     u8 *p = (u8*)ptr - 8;
 424     memset(p, 0x55, *(int*)p + 8);
 425     free(p);
 426   }
 427 }
 428
 429 /*
 430 ** String zPattern contains a glob pattern. Return true if zStr matches
 431 ** the pattern, or false if it does not.
 432 */
 433 int testGlobMatch(const char *zPattern, const char *zStr){
 434   int i = 0;
 435   int j = 0;
 436
 437   while( zPattern[i] ){
 438     char p = zPattern[i];
 439
 440     if( p=='*' || p=='%' ){
 441       do {
 442         if( testGlobMatch(&zPattern[i+1], &zStr[j]) ) return 1;
 443       }while( zStr[j++] );
 444       return 0;
 445     }
 446
 447     if( zStr[j]==0 || (p!='?' && p!=zStr[j]) ){
 448       /* Match failed. */
 449       return 0;
 450     }
 451
 452     j++;
 453     i++;
 454   }
 455
 456   return (zPattern[i]==0 && zStr[j]==0);
 457 }
 458
 459 /*
 460 ** End of test utilities
 461 **************************************************************************/
 462
 463 int do_test(int nArg, char **azArg){
 464   int j;
 465   int rc;
 466   int nFail = 0;
 467   const char *zPattern = 0;
 468
 469   if( nArg>1 ){
 470     testPrintError("Usage: test ?PATTERN?\n");
 471     return 1;
 472   }
 473   if( nArg==1 ){
 474     zPattern = azArg[0];
 475   }
 476
 477   for(j=0; tdb_system_name(j); j++){
 478     rc = 0;
 479
 480     test_data_1(tdb_system_name(j), zPattern, &rc);
 481     test_data_2(tdb_system_name(j), zPattern, &rc);
 482     test_data_3(tdb_system_name(j), zPattern, &rc);
 483     test_data_4(tdb_system_name(j), zPattern, &rc);
 484     test_rollback(tdb_system_name(j), zPattern, &rc);
 485     test_mc(tdb_system_name(j), zPattern, &rc);
 486     test_mt(tdb_system_name(j), zPattern, &rc);
 487
 488     if( rc ) nFail++;
 489   }
 490
 491   rc = 0;
 492   test_oom(zPattern, &rc);
 493   if( rc ) nFail++;
 494
 495   rc = 0;
 496   test_api(zPattern, &rc);
 497   if( rc ) nFail++;
 498
 499   rc = 0;
 500   do_crash_test(zPattern, &rc);
 501   if( rc ) nFail++;
 502
 503   rc = 0;
 504   do_writer_crash_test(zPattern, &rc);
 505   if( rc ) nFail++;
 506
 507   return (nFail!=0);
 508 }
 509
 510 static lsm_db *configure_lsm_db(TestDb *pDb){
 511   lsm_db *pLsm;
 512   pLsm = tdb_lsm(pDb);
 513   if( pLsm ){
 514     tdb_lsm_config_str(pDb, "mmap=1 autowork=1 automerge=4 worker_automerge=4");
 515   }
 516   return pLsm;
 517 }
 518
 519 typedef struct WriteHookEvent WriteHookEvent;
 520 struct WriteHookEvent {
 521   i64 iOff;
 522   int nData;
 523   int nUs;
 524 };
 525 WriteHookEvent prev = {0, 0, 0};
 526
 527 static void flushPrev(FILE *pOut){
 528   if( prev.nData ){
 529     fprintf(pOut, "w %s %lld %d %d\n", "d", prev.iOff, prev.nData, prev.nUs);
 530     prev.nData = 0;
 531   }
 532 }
 533
 534 #if 0 /* unused */
 535 static void do_speed_write_hook2(
 536   void *pCtx,
 537   int bLog,
 538   i64 iOff,
 539   int nData,
 540   int nUs
 541 ){
 542   FILE *pOut = (FILE *)pCtx;
 543   if( bLog ) return;
 544
 545   if( prev.nData && nData && iOff==prev.iOff+prev.nData ){
 546     prev.nData += nData;
 547     prev.nUs += nUs;
 548   }else{
 549     flushPrev(pOut);
 550     if( nData==0 ){
 551       fprintf(pOut, "s %s 0 0 %d\n", (bLog ? "l" : "d"), nUs);
 552     }else{
 553       prev.iOff = iOff;
 554       prev.nData = nData;
 555       prev.nUs = nUs;
 556     }
 557   }
 558 }
 559 #endif
 560
 561 #define ST_REPEAT  0
 562 #define ST_WRITE   1
 563 #define ST_PAUSE   2
 564 #define ST_FETCH   3
 565 #define ST_SCAN    4
 566 #define ST_NSCAN   5
 567 #define ST_KEYSIZE 6
 568 #define ST_VALSIZE 7
 569 #define ST_TRANS   8
 570
 571
 572 static void print_speed_test_help(){
 573   printf(
 574 "\n"
 575 "Repeat the following $repeat times:\n"
 576 "  1. Insert $write key-value pairs. One transaction for each write op.\n"
 577 "  2. Pause for $pause ms.\n"
 578 "  3. Perform $fetch queries on the database.\n"
 579 "\n"
 580 "  Keys are $keysize bytes in size. Values are $valsize bytes in size\n"
 581 "  Both keys and values are pseudo-randomly generated\n"
 582 "\n"
 583 "Options are:\n"
 584 "  -repeat  $repeat                 (default value 10)\n"
 585 "  -write   $write                  (default value 10000)\n"
 586 "  -pause   $pause                  (default value 0)\n"
 587 "  -fetch   $fetch                  (default value 0)\n"
 588 "  -keysize $keysize                (default value 12)\n"
 589 "  -valsize $valsize                (default value 100)\n"
 590 "  -system  $system                 (default value \"lsm\")\n"
 591 "  -trans   $trans                  (default value 0)\n"
 592 "\n"
 593 );
 594 }
 595
 596 int do_speed_test2(int nArg, char **azArg){
 597   struct Option {
 598     const char *zOpt;
 599     int eVal;
 600     int iDefault;
 601   } aOpt[] = {
 602     { "-repeat",  ST_REPEAT,    10},
 603     { "-write",   ST_WRITE,  10000},
 604     { "-pause",   ST_PAUSE,      0},
 605     { "-fetch",   ST_FETCH,      0},
 606     { "-scan",    ST_SCAN,       0},
 607     { "-nscan",   ST_NSCAN,      0},
 608     { "-keysize", ST_KEYSIZE,   12},
 609     { "-valsize", ST_VALSIZE,  100},
 610     { "-trans",   ST_TRANS,      0},
 611     { "-system",  -1,            0},
 612     { "help",     -2,            0},
 613     {0, 0, 0}
 614   };
 615   int i;
 616   int aParam[9];
 617   int rc = 0;
 618   int bReadonly = 0;
 619   int nContent = 0;
 620
 621   TestDb *pDb;
 622   Datasource *pData;
 623   DatasourceDefn defn = { TEST_DATASOURCE_RANDOM, 0, 0, 0, 0 };
 624   char *zSystem = "";
 625   int bLsm = 1;
 626   FILE *pLog = 0;
 627
 628 #ifdef NDEBUG
 629   /* If NDEBUG is defined, disable the dynamic memory related checks in
 630   ** lsmtest_mem.c. They slow things down.  */
 631   testMallocUninstall(tdb_lsm_env());
 632 #endif
 633
 634   /* Initialize aParam[] with default values. */
 635   for(i=0; i<ArraySize(aOpt); i++){
 636     if( aOpt[i].zOpt ) aParam[aOpt[i].eVal] = aOpt[i].iDefault;
 637   }
 638
 639   /* Process the command line switches. */
 640   for(i=0; i<nArg; i+=2){
 641     int iSel;
 642     rc = testArgSelect(aOpt, "switch", azArg[i], &iSel);
 643     if( rc ){
 644       return rc;
 645     }
 646     if( aOpt[iSel].eVal==-2 ){
 647       print_speed_test_help();
 648       return 0;
 649     }
 650     if( i+1==nArg ){
 651       testPrintError("option %s requires an argument\n", aOpt[iSel].zOpt);
 652       return 1;
 653     }
 654     if( aOpt[iSel].eVal>=0 ){
 655       aParam[aOpt[iSel].eVal] = atoi(azArg[i+1]);
 656     }else{
 657       zSystem = azArg[i+1];
 658       bLsm = 0;
 659 #if 0
 660       for(j=0; zSystem[j]; j++){
 661         if( zSystem[j]=='=' ) bLsm = 1;
 662       }
 663 #endif
 664     }
 665   }
 666
 667   printf("#");
 668   for(i=0; i<ArraySize(aOpt); i++){
 669     if( aOpt[i].zOpt ){
 670       if( aOpt[i].eVal>=0 ){
 671         printf(" %s=%d", &aOpt[i].zOpt[1], aParam[aOpt[i].eVal]);
 672       }else if( aOpt[i].eVal==-1 ){
 673         printf(" %s=\"%s\"", &aOpt[i].zOpt[1], zSystem);
 674       }
 675     }
 676   }
 677   printf("\n");
 678
 679   defn.nMinKey = defn.nMaxKey = aParam[ST_KEYSIZE];
 680   defn.nMinVal = defn.nMaxVal = aParam[ST_VALSIZE];
 681   pData = testDatasourceNew(&defn);
 682
 683   if( aParam[ST_WRITE]==0 ){
 684     bReadonly = 1;
 685   }
 686
 687   if( bLsm ){
 688     rc = tdb_lsm_open(zSystem, "testdb.lsm", !bReadonly, &pDb);
 689   }else{
 690     pDb = testOpen(zSystem, !bReadonly, &rc);
 691   }
 692   if( rc!=0 ) return rc;
 693   if( bReadonly ){
 694     nContent = testCountDatabase(pDb);
 695   }
 696
 697 #if 0
 698   pLog = fopen("/tmp/speed.log", "w");
 699   tdb_lsm_write_hook(pDb, do_speed_write_hook2, (void *)pLog);
 700 #endif
 701
 702   for(i=0; i<aParam[ST_REPEAT] && rc==0; i++){
 703     int msWrite, msFetch;
 704     int iFetch;
 705     int nWrite = aParam[ST_WRITE];
 706
 707     if( bReadonly ){
 708       msWrite = 0;
 709     }else{
 710       testTimeInit();
 711
 712       if( aParam[ST_TRANS] ) testBegin(pDb, 2, &rc);
 713       testWriteDatasourceRange(pDb, pData, i*nWrite, nWrite, &rc);
 714       if( aParam[ST_TRANS] ) testCommit(pDb, 0, &rc);
 715
 716       msWrite = testTimeGet();
 717       nContent += nWrite;
 718     }
 719
 720     if( aParam[ST_PAUSE] ){
 721       if( aParam[ST_PAUSE]/1000 ) sleep(aParam[ST_PAUSE]/1000);
 722       if( aParam[ST_PAUSE]%1000 ) usleep(1000 * (aParam[ST_PAUSE]%1000));
 723     }
 724
 725     if( aParam[ST_FETCH] ){
 726       testTimeInit();
 727       if( aParam[ST_TRANS] ) testBegin(pDb, 1, &rc);
 728       for(iFetch=0; iFetch<aParam[ST_FETCH]; iFetch++){
 729         int iKey = testPrngValue(i*nWrite+iFetch) % nContent;
 730 #ifndef NDEBUG
 731         testDatasourceFetch(pDb, pData, iKey, &rc);
 732 #else
 733         void *pKey; int nKey;           /* Database key to query for */
 734         void *pVal; int nVal;           /* Result of query */
 735
 736         testDatasourceEntry(pData, iKey, &pKey, &nKey, 0, 0);
 737         rc = tdb_fetch(pDb, pKey, nKey, &pVal, &nVal);
 738         if( rc==0 && nVal<0 ) rc = 1;
 739         if( rc ) break;
 740 #endif
 741       }
 742       if( aParam[ST_TRANS] ) testCommit(pDb, 0, &rc);
 743       msFetch = testTimeGet();
 744     }else{
 745       msFetch = 0;
 746     }
 747
 748     if( i==(aParam[ST_REPEAT]-1) ){
 749       testTimeInit();
 750       testClose(&pDb);
 751       msWrite += testTimeGet();
 752     }
 753
 754     printf("%d %d %d\n", i, msWrite, msFetch);
 755     fflush(stdout);
 756   }
 757
 758   testClose(&pDb);
 759   testDatasourceFree(pData);
 760
 761   if( pLog ){
 762     flushPrev(pLog);
 763     fclose(pLog);
 764   }
 765   return rc;
 766 }
 767
 768 int do_speed_tests(int nArg, char **azArg){
 769
 770   struct DbSystem {
 771     const char *zLibrary;
 772     const char *zColor;
 773   } aSys[] = {
 774     { "sqlite3",      "black" },
 775     { "leveldb",      "blue" },
 776     { "lsm",          "red" },
 777     { "lsm_mt2",      "orange" },
 778     { "lsm_mt3",      "purple" },
 779     { "kyotocabinet", "green" },
 780     {0, 0}
 781   };
 782
 783   int i;
 784   int j;
 785   int rc;
 786   int nSleep = 0;                 /* ms of rest allowed between INSERT tests */
 787   int nRow = 0;                   /* Number of rows to insert into database */
 788   int nStep;                      /* Measure INSERT time after this many rows */
 789   int nSelStep;                   /* Measure SELECT time after this many rows */
 790   int nSelTest;                   /* Number of SELECTs to run for timing */
 791   int doReadTest = 1;
 792   int doWriteTest = 1;
 793
 794   int *aTime;                     /* INSERT timing data */
 795   int *aWrite;                    /* Writes per nStep inserts */
 796   int *aSelTime;                  /* SELECT timing data */
 797   int isFirst = 1;
 798   int bSleep = 0;
 799
 800   /* File to write gnuplot script to. */
 801   const char *zOut = "lsmtest_speed.gnuplot";
 802
 803   u32 sys_mask = 0;
 804
 805   testMallocUninstall(tdb_lsm_env());
 806
 807   for(i=0; i<nArg; i++){
 808     struct Opt {
 809       const char *zOpt;
 810       int isSwitch;
 811     } aOpt[] = {
 812       { "sqlite3" , 0},
 813       { "leveldb" , 0},
 814       { "lsm" , 0},
 815       { "lsm_mt2" , 0},
 816       { "lsm_mt3" , 0},
 817       { "kyotocabinet" , 0},
 818       { "-rows"     , 1},
 819       { "-sleep"    , 2},
 820       { "-testmode" , 3},
 821       { "-out"      , 4},
 822       { 0, 0}
 823     };
 824     int iSel;
 825
 826     rc = testArgSelect(aOpt, "argument", azArg[i], &iSel);
 827     if( rc ) return rc;
 828
 829     if( aOpt[iSel].isSwitch ){
 830       i++;
 831
 832       if( i>=nArg ){
 833         testPrintError("option %s requires an argument\n", aOpt[iSel].zOpt);
 834         return 1;
 835       }
 836       if( aOpt[iSel].isSwitch==1 ){
 837         nRow = atoi(azArg[i]);
 838       }
 839       if( aOpt[iSel].isSwitch==2 ){
 840         nSleep = atoi(azArg[i]);
 841       }
 842       if( aOpt[iSel].isSwitch==3 ){
 843         struct Mode {
 844           const char *zMode;
 845           int doReadTest;
 846           int doWriteTest;
 847         } aMode[] = {{"ro", 1, 0} , {"rw", 1, 1}, {"wo", 0, 1}, {0, 0, 0}};
 848         int iMode;
 849         rc = testArgSelect(aMode, "option", azArg[i], &iMode);
 850         if( rc ) return rc;
 851         doReadTest = aMode[iMode].doReadTest;
 852         doWriteTest = aMode[iMode].doWriteTest;
 853       }
 854       if( aOpt[iSel].isSwitch==4 ){
 855         /* The "-out FILE" switch. This option is used to specify a file to
 856         ** write the gnuplot script to. */
 857         zOut = azArg[i];
 858       }
 859     }else{
 860       /* A db name */
 861       rc = testArgSelect(aOpt, "system", azArg[i], &iSel);
 862       if( rc ) return rc;
 863       sys_mask |= (1<<iSel);
 864     }
 865   }
 866
 867   if( sys_mask==0 ) sys_mask = (1 << 0) | (1 << 1) | (1 << 2) | (1 << 3);
 868   nRow = MAX(nRow, 100000);
 869   nStep = nRow/100;
 870   nSelStep = nRow/10;
 871   nSelTest = (nSelStep > 100000) ? 100000 : nSelStep;
 872
 873   aTime = malloc(sizeof(int) * ArraySize(aSys) * nRow/nStep);
 874   aWrite = malloc(sizeof(int) * nRow/nStep);
 875   aSelTime = malloc(sizeof(int) * ArraySize(aSys) * nRow/nSelStep);
 876
 877   /* This loop collects the INSERT speed data. */
 878   if( doWriteTest ){
 879     printf("Writing output to file \"%s\".\n",  zOut);
 880
 881     for(j=0; aSys[j].zLibrary; j++){
 882       FILE *pLog = 0;
 883       TestDb *pDb;                  /* Database being tested */
 884       lsm_db *pLsm;
 885       int iDot = 0;
 886
 887       if( ((1<<j)&sys_mask)==0 ) continue;
 888       if( bSleep && nSleep ) sqlite3_sleep(nSleep);
 889       bSleep = 1;
 890
 891       testCaseBegin(&rc, 0, "speed.insert.%s", aSys[j].zLibrary);
 892
 893       rc = tdb_open(aSys[j].zLibrary, 0, 1, &pDb);
 894       if( rc ) return rc;
 895
 896       pLsm = configure_lsm_db(pDb);
 897 #if 0
 898       pLog = fopen("/tmp/speed.log", "w");
 899       tdb_lsm_write_hook(pDb, do_speed_write_hook2, (void *)pLog);
 900 #endif
 901
 902       testTimeInit();
 903       for(i=0; i<nRow; i+=nStep){
 904         int iStep;
 905         int nWrite1 = 0, nWrite2 = 0;
 906         testCaseProgress(i, nRow, testCaseNDot(), &iDot);
 907         if( pLsm ) lsm_info(pLsm, LSM_INFO_NWRITE, &nWrite1);
 908         for(iStep=0; iStep<nStep; iStep++){
 909           u32 aKey[4];                  /* 16-byte key */
 910           u32 aVal[25];                 /* 100 byte value */
 911           testPrngArray(i+iStep, aKey, ArraySize(aKey));
 912           testPrngArray(i+iStep, aVal, ArraySize(aVal));
 913           rc = tdb_write(pDb, aKey, sizeof(aKey), aVal, sizeof(aVal));
 914         }
 915         aTime[(j*nRow+i)/nStep] = testTimeGet();
 916         if( pLsm ) lsm_info(pLsm, LSM_INFO_NWRITE, &nWrite2);
 917         aWrite[i/nStep] = nWrite2 - nWrite1;
 918       }
 919
 920       tdb_close(pDb);
 921       if( pLog ) fclose(pLog);
 922       testCaseFinish(rc);
 923     }
 924   }
 925
 926   /* This loop collects the SELECT speed data. */
 927   if( doReadTest ){
 928     for(j=0; aSys[j].zLibrary; j++){
 929       int iDot = 0;
 930       TestDb *pDb;                  /* Database being tested */
 931
 932       if( ((1<<j)&sys_mask)==0 ) continue;
 933       if( bSleep && nSleep ) sqlite3_sleep(nSleep);
 934       bSleep = 1;
 935
 936       testCaseBegin(&rc, 0, "speed.select.%s", aSys[j].zLibrary);
 937
 938       if( doWriteTest ){
 939         rc = tdb_open(aSys[j].zLibrary, 0, 1, &pDb);
 940         if( rc ) return rc;
 941         configure_lsm_db(pDb);
 942
 943         for(i=0; i<nRow; i+=nSelStep){
 944           int iStep;
 945           int iSel;
 946           testCaseProgress(i, nRow, testCaseNDot(), &iDot);
 947           for(iStep=0; iStep<nSelStep; iStep++){
 948             u32 aKey[4];                  /* 16-byte key */
 949             u32 aVal[25];                 /* 100 byte value */
 950             testPrngArray(i+iStep, aKey, ArraySize(aKey));
 951             testPrngArray(i+iStep, aVal, ArraySize(aVal));
 952             rc = tdb_write(pDb, aKey, sizeof(aKey), aVal, sizeof(aVal));
 953           }
 954
 955           testTimeInit();
 956           for(iSel=0; iSel<nSelTest; iSel++){
 957             void *pDummy;
 958             int nDummy;
 959             u32 iKey;
 960             u32 aKey[4];                  /* 16-byte key */
 961
 962             iKey = testPrngValue(iSel) % (i+nSelStep);
 963             testPrngArray(iKey, aKey, ArraySize(aKey));
 964             rc = tdb_fetch(pDb, aKey, sizeof(aKey), &pDummy, &nDummy);
 965           }
 966           aSelTime[(j*nRow+i)/nSelStep] = testTimeGet();
 967           tdb_fetch(pDb, 0, 0, 0, 0);
 968         }
 969       }else{
 970         int t;
 971         int iSel;
 972
 973         rc = tdb_open(aSys[j].zLibrary, 0, 0, &pDb);
 974         configure_lsm_db(pDb);
 975
 976         testTimeInit();
 977         for(iSel=0; rc==LSM_OK && iSel<nSelTest; iSel++){
 978           void *pDummy;
 979           int nDummy;
 980           u32 iKey;
 981           u32 aKey[4];                  /* 16-byte key */
 982 #ifndef NDEBUG
 983           u32 aVal[25];                 /* 100 byte value */
 984 #endif
 985
 986           testCaseProgress(iSel, nSelTest, testCaseNDot(), &iDot);
 987
 988           iKey = testPrngValue(iSel) % nRow;
 989           testPrngArray(iKey, aKey, ArraySize(aKey));
 990           rc = tdb_fetch(pDb, aKey, sizeof(aKey), &pDummy, &nDummy);
 991
 992 #ifndef NDEBUG
 993           testPrngArray(iKey, aVal, ArraySize(aVal));
 994           assert( nDummy==100 && memcmp(aVal, pDummy, 100)==0 );
 995 #endif
 996         }
 997         if( rc!=LSM_OK ) return rc;
 998
 999         t = testTimeGet();
1000         tdb_fetch(pDb, 0, 0, 0, 0);
1001
1002         printf("%s: %d selects/second\n",
1003             aSys[j].zLibrary, (int)((double)nSelTest*1000.0/t)
1004         );
1005       }
1006
1007       tdb_close(pDb);
1008       testCaseFinish(rc);
1009     }
1010   }
1011
1012
1013   if( doWriteTest ){
1014     FILE *pOut = fopen(zOut, "w");
1015     if( !pOut ){
1016       printf("fopen(\"%s\", \"w\"): %s\n", zOut, strerror(errno));
1017       return 1;
1018     }
1019
1020     fprintf(pOut, "set xlabel \"Rows Inserted\"\n");
1021     fprintf(pOut, "set ylabel \"Inserts per second\"\n");
1022     if( doReadTest ){
1023       fprintf(pOut, "set y2label \"Selects per second\"\n");
1024     }else if( sys_mask==(1<<2) ){
1025       fprintf(pOut, "set y2label \"Page writes per insert\"\n");
1026     }
1027     fprintf(pOut, "set yrange [0:*]\n");
1028     fprintf(pOut, "set y2range [0:*]\n");
1029     fprintf(pOut, "set xrange [%d:*]\n", MAX(nStep, nRow/20) );
1030     fprintf(pOut, "set ytics nomirror\n");
1031     fprintf(pOut, "set y2tics nomirror\n");
1032     fprintf(pOut, "set key box lw 0.01\n");
1033     fprintf(pOut, "plot ");
1034
1035     for(j=0; aSys[j].zLibrary; j++){
1036       if( (1<<j)&sys_mask ){
1037         const char *zLib = aSys[j].zLibrary;
1038         fprintf(pOut, "%s\"-\" ti \"%s INSERT\" with lines lc rgb \"%s\" ",
1039             (isFirst?"":", "), zLib, aSys[j].zColor
1040         );
1041         if( doReadTest ){
1042           fprintf(pOut, ", \"-\" ti \"%s SELECT\" "
1043                  "axis x1y2 with points lw 3 lc rgb \"%s\""
1044               , zLib, aSys[j].zColor
1045           );
1046         }
1047         isFirst = 0;
1048       }
1049     }
1050
1051     assert( strcmp(aSys[2].zLibrary, "lsm")==0 );
1052     if( sys_mask==(1<<2) && !doReadTest ){
1053       fprintf(pOut, ", \"-\" ti \"lsm pages written\" "
1054         "axis x1y2 with boxes lw 1 lc rgb \"grey\""
1055       );
1056     }
1057
1058     fprintf(pOut, "\n");
1059
1060     for(j=0; aSys[j].zLibrary; j++){
1061       if( ((1<<j)&sys_mask)==0 ) continue;
1062       fprintf(pOut, "# Rows    Inserts per second\n");
1063       for(i=0; i<nRow; i+=nStep){
1064         int iTime = aTime[(j*nRow+i)/nStep];
1065         int ips = (int)((i+nStep)*1000.0 / (double)iTime);
1066         fprintf(pOut, "%d %d\n", i+nStep, ips);
1067       }
1068       fprintf(pOut, "end\n");
1069
1070       if( doReadTest ){
1071         fprintf(pOut, "# Rows    Selects per second\n");
1072         for(i=0; i<nRow; i+=nSelStep){
1073           int sps = (int)(nSelTest*1000.0/(double)aSelTime[(j*nRow+i)/nSelStep]);
1074           fprintf(pOut, "%d %d\n", i+nSelStep, sps);
1075         }
1076         fprintf(pOut, "end\n");
1077       }else if( sys_mask==(1<<2) ){
1078         for(i=0; i<(nRow/nStep); i++){
1079           fprintf(pOut, "%d %f\n", i*nStep, (double)aWrite[i] / (double)nStep);
1080         }
1081         fprintf(pOut, "end\n");
1082       }
1083     }
1084
1085     fprintf(pOut, "pause -1\n");
1086     fclose(pOut);
1087   }
1088
1089   free(aTime);
1090   free(aSelTime);
1091   free(aWrite);
1092   testMallocInstall(tdb_lsm_env());
1093   return 0;
1094 }
1095
1096 /*
1097 ** Usage: lsmtest random ?N?
1098 **
1099 ** This command prints a sequence of zero or more numbers from the PRNG
1100 ** system to stdout. If the "N" argument is missing, values the first 10
1101 ** values (i=0, i=1, ... i=9) are printed. Otherwise, the first N.
1102 **
1103 ** This was added to verify that the PRNG values do not change between
1104 ** runs of the lsmtest program.
1105 */
1106 int do_random_tests(int nArg, char **azArg){
1107   int i;
1108   int nRand;
1109   if( nArg==0 ){
1110     nRand = 10;
1111   }else if( nArg==1 ){
1112     nRand = atoi(azArg[0]);
1113   }else{
1114     testPrintError("Usage: random ?N?\n");
1115     return -1;
1116   }
1117   for(i=0; i<nRand; i++){
1118     printf("0x%x\n", testPrngValue(i));
1119   }
1120   return 0;
1121 }
1122
1123 static int testFormatSize(char *aBuf, int nBuf, i64 nByte){
1124   int res;
1125   if( nByte<(1<<10) ){
1126     res = snprintf(aBuf, nBuf, "%d byte", (int)nByte);
1127   }else if( nByte<(1<<20) ){
1128     res = snprintf(aBuf, nBuf, "%dK", (int)(nByte/(1<<10)));
1129   }else{
1130     res = snprintf(aBuf, nBuf, "%dM", (int)(nByte/(1<<20)));
1131   }
1132   return res;
1133 }
1134
1135 static i64 testReadSize(char *z){
1136   int n = strlen(z);
1137   char c = z[n-1];
1138   i64 nMul = 1;
1139
1140   switch( c ){
1141     case 'g': case 'G':
1142       nMul = (1<<30);
1143       break;
1144
1145     case 'm': case 'M':
1146       nMul = (1<<20);
1147       break;
1148
1149     case 'k': case 'K':
1150       nMul = (1<<10);
1151       break;
1152
1153     default:
1154       nMul = 1;
1155   }
1156
1157   return nMul * (i64)atoi(z);
1158 }
1159
1160 /*
1161 ** Usage: lsmtest writespeed FILESIZE BLOCKSIZE SYNCSIZE
1162 */
1163 static int do_writer_test(int nArg, char **azArg){
1164   int nBlock;
1165   int nSize;
1166   int i;
1167   int fd;
1168   int ms;
1169   char aFilesize[32];
1170   char aBlockSize[32];
1171
1172   char *aPage;
1173   int *aOrder;
1174   int nSync;
1175
1176   i64 filesize;
1177   i64 blocksize;
1178   i64 syncsize;
1179   int nPage = 4096;
1180
1181   /* How long to sleep before running a trial (in ms). */
1182 #if 0
1183   const int nSleep = 10000;
1184 #endif
1185   const int nSleep = 0;
1186
1187   if( nArg!=3 ){
1188     testPrintUsage("FILESIZE BLOCKSIZE SYNCSIZE");
1189     return -1;
1190   }
1191
1192   filesize = testReadSize(azArg[0]);
1193   blocksize = testReadSize(azArg[1]);
1194   syncsize = testReadSize(azArg[2]);
1195
1196   nBlock = (int)(filesize / blocksize);
1197   nSize = (int)blocksize;
1198   nSync = (int)(syncsize / blocksize);
1199
1200   aPage = (char *)malloc(4096);
1201   aOrder = (int *)malloc(nBlock * sizeof(int));
1202   for(i=0; i<nBlock; i++) aOrder[i] = i;
1203   for(i=0; i<(nBlock*25); i++){
1204     int tmp;
1205     u32 a = testPrngValue(i);
1206     u32 b = testPrngValue(a);
1207     a = a % nBlock;
1208     b = b % nBlock;
1209     tmp = aOrder[a];
1210     aOrder[a] = aOrder[b];
1211     aOrder[b] = tmp;
1212   }
1213
1214   testFormatSize(aFilesize, sizeof(aFilesize), (i64)nBlock * (i64)nSize);
1215   testFormatSize(aBlockSize, sizeof(aFilesize), nSize);
1216
1217   printf("Testing writing a %s file using %s blocks. ", aFilesize, aBlockSize);
1218   if( nSync==1 ){
1219     printf("Sync after each block.\n");
1220   }else{
1221     printf("Sync after each %d blocks.\n", nSync);
1222   }
1223
1224   printf("Preparing file... ");
1225   fflush(stdout);
1226   unlink("writer.out");
1227   fd = open("writer.out", O_RDWR|O_CREAT|_O_BINARY, 0664);
1228   if( fd<0 ){
1229     testPrintError("open(): %d - %s\n", errno, strerror(errno));
1230     return -1;
1231   }
1232   testTimeInit();
1233   for(i=0; i<nBlock; i++){
1234     int iPg;
1235     memset(aPage, i&0xFF, nPage);
1236     for(iPg=0; iPg<(nSize/nPage); iPg++){
1237       write(fd, aPage, nPage);
1238     }
1239   }
1240   fsync(fd);
1241   printf("ok (%d ms)\n", testTimeGet());
1242
1243   for(i=0; i<5; i++){
1244     int j;
1245
1246     sqlite3_sleep(nSleep);
1247     printf("Now writing sequentially...  ");
1248     fflush(stdout);
1249
1250     lseek(fd, 0, SEEK_SET);
1251     testTimeInit();
1252     for(j=0; j<nBlock; j++){
1253       int iPg;
1254       if( ((j+1)%nSync)==0 ) fdatasync(fd);
1255       memset(aPage, j&0xFF, nPage);
1256       for(iPg=0; iPg<(nSize/nPage); iPg++){
1257         write(fd, aPage, nPage);
1258       }
1259     }
1260     fdatasync(fd);
1261     ms = testTimeGet();
1262     printf("%d ms\n", ms);
1263     sqlite3_sleep(nSleep);
1264     printf("Now in an arbitrary order... ");
1265
1266     fflush(stdout);
1267     testTimeInit();
1268     for(j=0; j<nBlock; j++){
1269       int iPg;
1270       if( ((j+1)%nSync)==0 ) fdatasync(fd);
1271       lseek(fd, aOrder[j]*nSize, SEEK_SET);
1272       memset(aPage, j&0xFF, nPage);
1273       for(iPg=0; iPg<(nSize/nPage); iPg++){
1274         write(fd, aPage, nPage);
1275       }
1276     }
1277     fdatasync(fd);
1278     ms = testTimeGet();
1279     printf("%d ms\n", ms);
1280   }
1281
1282   close(fd);
1283   free(aPage);
1284   free(aOrder);
1285
1286   return 0;
1287 }
1288
1289 static void do_insert_work_hook(lsm_db *db, void *p){
1290   char *z = 0;
1291   lsm_info(db, LSM_INFO_DB_STRUCTURE, &z);
1292   if( z ){
1293     printf("%s\n", z);
1294     fflush(stdout);
1295     lsm_free(lsm_get_env(db), z);
1296   }
1297
1298   unused_parameter(p);
1299 }
1300
1301 typedef struct InsertWriteHook InsertWriteHook;
1302 struct InsertWriteHook {
1303   FILE *pOut;
1304   int bLog;
1305   i64 iOff;
1306   int nData;
1307 };
1308
1309 static void flushHook(InsertWriteHook *pHook){
1310   if( pHook->nData ){
1311     fprintf(pHook->pOut, "write %s %d %d\n",
1312         (pHook->bLog ? "log" : "db"), (int)pHook->iOff, pHook->nData
1313     );
1314     pHook->nData = 0;
1315     fflush(pHook->pOut);
1316   }
1317 }
1318
1319 static void do_insert_write_hook(
1320   void *pCtx,
1321   int bLog,
1322   i64 iOff,
1323   int nData,
1324   int nUs
1325 ){
1326   InsertWriteHook *pHook = (InsertWriteHook *)pCtx;
1327   if( bLog ) return;
1328
1329   if( nData==0 ){
1330     flushHook(pHook);
1331     fprintf(pHook->pOut, "sync %s\n", (bLog ? "log" : "db"));
1332   }else if( pHook->nData
1333          && bLog==pHook->bLog
1334          && iOff==(pHook->iOff+pHook->nData)
1335   ){
1336     pHook->nData += nData;
1337   }else{
1338     flushHook(pHook);
1339     pHook->bLog = bLog;
1340     pHook->iOff = iOff;
1341     pHook->nData = nData;
1342   }
1343 }
1344
1345 static int do_replay(int nArg, char **azArg){
1346   char aBuf[4096];
1347   FILE *pInput;
1348   FILE *pClose = 0;
1349   const char *zDb;
1350
1351   lsm_env *pEnv;
1352   lsm_file *pOut;
1353   int rc;
1354
1355   if( nArg!=2 ){
1356     testPrintError("Usage: replay WRITELOG FILE\n");
1357     return 1;
1358   }
1359
1360   if( strcmp(azArg[0], "-")==0 ){
1361     pInput = stdin;
1362   }else{
1363     pClose = pInput = fopen(azArg[0], "r");
1364   }
1365   zDb = azArg[1];
1366   pEnv = tdb_lsm_env();
1367   rc = pEnv->xOpen(pEnv, zDb, 0, &pOut);
1368   if( rc!=LSM_OK ) return rc;
1369
1370   while( feof(pInput)==0 ){
1371     char zLine[80];
1372     fgets(zLine, sizeof(zLine)-1, pInput);
1373     zLine[sizeof(zLine)-1] = '\0';
1374
1375     if( 0==memcmp("sync db", zLine, 7) ){
1376       rc = pEnv->xSync(pOut);
1377       if( rc!=0 ) break;
1378     }else{
1379       int iOff;
1380       int nData;
1381       int nMatch;
1382       nMatch = sscanf(zLine, "write db %d %d", &iOff, &nData);
1383       if( nMatch==2 ){
1384         int i;
1385         for(i=0; i<nData; i+=sizeof(aBuf)){
1386           memset(aBuf, i&0xFF, sizeof(aBuf));
1387           rc = pEnv->xWrite(pOut, iOff+i, aBuf, sizeof(aBuf));
1388           if( rc!=0 ) break;
1389         }
1390       }
1391     }
1392   }
1393   if( pClose ) fclose(pClose);
1394   pEnv->xClose(pOut);
1395
1396   return rc;
1397 }
1398
1399 static int do_insert(int nArg, char **azArg){
1400   const char *zDb = "lsm";
1401   TestDb *pDb = 0;
1402   int i;
1403   int rc;
1404   const int nRow = 1 * 1000 * 1000;
1405
1406   DatasourceDefn defn = { TEST_DATASOURCE_RANDOM, 8, 15, 80, 150 };
1407   Datasource *pData = 0;
1408
1409   if( nArg>1 ){
1410     testPrintError("Usage: insert ?DATABASE?\n");
1411     return 1;
1412   }
1413   if( nArg==1 ){ zDb = azArg[0]; }
1414
1415   testMallocUninstall(tdb_lsm_env());
1416   for(i=0; zDb[i] && zDb[i]!='='; i++);
1417   if( zDb[i] ){
1418     rc = tdb_lsm_open(zDb, "testdb.lsm", 1, &pDb);
1419   }else{
1420     rc = tdb_open(zDb, 0, 1, &pDb);
1421   }
1422
1423   if( rc!=0 ){
1424     testPrintError("Error opening db \"%s\": %d\n", zDb, rc);
1425   }else{
1426     InsertWriteHook hook;
1427     memset(&hook, 0, sizeof(hook));
1428     hook.pOut = fopen("writelog.txt", "w");
1429
1430     pData = testDatasourceNew(&defn);
1431     tdb_lsm_config_work_hook(pDb, do_insert_work_hook, 0);
1432     tdb_lsm_write_hook(pDb, do_insert_write_hook, (void *)&hook);
1433
1434     if( rc==0 ){
1435       for(i=0; i<nRow; i++){
1436         void *pKey; int nKey;     /* Database key to insert */
1437         void *pVal; int nVal;     /* Database value to insert */
1438         testDatasourceEntry(pData, i, &pKey, &nKey, &pVal, &nVal);
1439         tdb_write(pDb, pKey, nKey, pVal, nVal);
1440       }
1441     }
1442
1443     testDatasourceFree(pData);
1444     tdb_close(pDb);
1445     flushHook(&hook);
1446     fclose(hook.pOut);
1447   }
1448   testMallocInstall(tdb_lsm_env());
1449
1450   return rc;
1451 }
1452
1453 static int st_do_show(int a, char **b)      { return do_show(a, b); }
1454 static int st_do_work(int a, char **b)      { return do_work(a, b); }
1455 static int st_do_io(int a, char **b)        { return do_io(a, b); }
1456
1457 #ifdef __linux__
1458 #include <sys/time.h>
1459 #include <sys/resource.h>
1460
1461 static void lsmtest_rusage_report(void){
1462   struct rusage r;
1463   memset(&r, 0, sizeof(r));
1464
1465   getrusage(RUSAGE_SELF, &r);
1466   printf("# getrusage: { ru_maxrss %d ru_oublock %d ru_inblock %d }\n",
1467       (int)r.ru_maxrss, (int)r.ru_oublock, (int)r.ru_inblock
1468   );
1469 }
1470 #else
1471 static void lsmtest_rusage_report(void){
1472   /* no-op */
1473 }
1474 #endif
1475
1476 int main(int argc, char **argv){
1477   struct TestFunc {
1478     const char *zName;
1479     int bRusageReport;
1480     int (*xFunc)(int, char **);
1481   } aTest[] = {
1482     {"random",      1, do_random_tests},
1483     {"writespeed",  1, do_writer_test},
1484     {"io",          1, st_do_io},
1485
1486     {"insert",      1, do_insert},
1487     {"replay",      1, do_replay},
1488
1489     {"speed",       1, do_speed_tests},
1490     {"speed2",      1, do_speed_test2},
1491     {"show",        0, st_do_show},
1492     {"work",        1, st_do_work},
1493     {"test",        1, do_test},
1494
1495     {0, 0}
1496   };
1497   int rc;                         /* Return Code */
1498   int iFunc;                      /* Index into aTest[] */
1499
1500   int nLeakAlloc = 0;             /* Allocations leaked by lsm */
1501   int nLeakByte = 0;              /* Bytes leaked by lsm */
1502
1503 #ifdef LSM_DEBUG_MEM
1504   FILE *pReport = 0;              /* lsm malloc() report file */
1505   const char *zReport = "malloc.txt generated";
1506 #else
1507   const char *zReport = "malloc.txt NOT generated";
1508 #endif
1509
1510   testMallocInstall(tdb_lsm_env());
1511
1512   if( argc<2 ){
1513     testPrintError("Usage: %s sub-command ?args...?\n", argv[0]);
1514     return -1;
1515   }
1516
1517   /* Initialize error reporting */
1518   testErrorInit(argc, argv);
1519
1520   /* Initialize PRNG system */
1521   testPrngInit();
1522
1523   rc = testArgSelect(aTest, "sub-command", argv[1], &iFunc);
1524   if( rc==0 ){
1525     rc = aTest[iFunc].xFunc(argc-2, &argv[2]);
1526   }
1527
1528 #ifdef LSM_DEBUG_MEM
1529   pReport = fopen("malloc.txt", "w");
1530   testMallocCheck(tdb_lsm_env(), &nLeakAlloc, &nLeakByte, pReport);
1531   fclose(pReport);
1532 #else
1533   testMallocCheck(tdb_lsm_env(), &nLeakAlloc, &nLeakByte, 0);
1534 #endif
1535
1536   if( nLeakAlloc ){
1537     testPrintError("Leaked %d bytes in %d allocations (%s)\n",
1538         nLeakByte, nLeakAlloc, zReport
1539     );
1540     if( rc==0 ) rc = -1;
1541   }
1542   testMallocUninstall(tdb_lsm_env());
1543
1544   if( aTest[iFunc].bRusageReport ){
1545     lsmtest_rusage_report();
1546   }
1547   return rc;
1548 }