apps/tagcache.c

   1 /***************************************************************************
   2  *             __________               __   ___.
   3  *   Open      \______   \ ____   ____ |  | _\_ |__   _______  ___
   4  *   Source     |       _//  _ \_/ ___\|  |/ /| __ \ /  _ \  \/  /
   5  *   Jukebox    |    |   (  <_> )  \___|    < | \_\ (  <_> > <  <
   6  *   Firmware   |____|_  /\____/ \___  >__|_ \|___  /\____/__/\_ \
   7  *                     \/            \/     \/    \/            \/
   8  * $Id$
   9  *
  10  * Copyright (C) 2005 by Miika Pekkarinen
  11  *
  12  * This program is free software; you can redistribute it and/or
  13  * modify it under the terms of the GNU General Public License
  14  * as published by the Free Software Foundation; either version 2
  15  * of the License, or (at your option) any later version.
  16  *
  17  * This software is distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY
  18  * KIND, either express or implied.
  19  *
  20  ****************************************************************************/
  21
  22 /*
  23  *                    TagCache API
  24  *
  25  *       ----------x---------x------------------x-----
  26  *                 |         |                  |              External
  27  * +---------------x-------+ |       TagCache   |              Libraries
  28  * | Modification routines | |         Core     |
  29  * +-x---------x-----------+ |                  |
  30  *   | (R/W)   |             |                  |           |
  31  *   |  +------x-------------x-+  +-------------x-----+     |
  32  *   |  |                      x==x Filters & clauses |     |
  33  *   |  | Search routines      |  +-------------------+     |
  34  *   |  |                      x============================x DirCache
  35  *   |  +-x--------------------+                            | (optional)
  36  *   |    | (R)                                             |
  37  *   |    | +-------------------------------+  +---------+  |
  38  *   |    | | DB Commit (sort,unique,index) |  |         |  |
  39  *   |    | +-x--------------------------x--+  | Control |  |
  40  *   |    |   | (R/W)                    | (R) | Thread  |  |
  41  *   |    |   | +----------------------+ |     |         |  |
  42  *   |    |   | | TagCache DB Builder  | |     +---------+  |
  43  *   |    |   | +-x-------------x------+ |                  |
  44  *   |    |   |   | (R)         | (W)    |                  |
  45  *   |    |   |   |          +--x--------x---------+        |
  46  *   |    |   |   |          | Temporary Commit DB |        |
  47  *   |    |   |   |          +---------------------+        |
  48  * +-x----x-------x--+                                      |
  49  * | TagCache RAM DB x==\(W) +-----------------+            |
  50  * +-----------------+   \===x                 |            |
  51  *   |    |   |   |      (R) |  Ram DB Loader  x============x DirCache
  52  * +-x----x---x---x---+   /==x                 |            | (optional)
  53  * | Tagcache Disk DB x==/   +-----------------+            |
  54  * +------------------+                                     |
  55  *
  56  */
  57
  58 /* #define LOGF_ENABLE */
  59
  60 #include <stdio.h>
  61 #include <stdlib.h>
  62 #include <ctype.h>
  63 #include "config.h"
  64 #include "ata_idle_notify.h"
  65 #include "thread.h"
  66 #include "kernel.h"
  67 #include "system.h"
  68 #include "logf.h"
  69 #include "string.h"
  70 #include "usb.h"
  71 #include "metadata.h"
  72 #include "metadata.h"
  73 #include "tagcache.h"
  74 #include "buffer.h"
  75 #include "crc32.h"
  76 #include "misc.h"
  77 #include "settings.h"
  78 #include "dir.h"
  79 #include "structec.h"
  80 #include "tagcache.h"
  81
  82 #ifndef __PCTOOL__
  83 #include "lang.h"
  84 #include "eeprom_settings.h"
  85 #endif
  86
  87 #ifdef __PCTOOL__
  88 #define yield() do { } while(0)
  89 #define sim_sleep(timeout) do { } while(0)
  90 #define do_timed_yield() do { } while(0)
  91 #endif
  92
  93 #ifndef __PCTOOL__
  94 /* Tag Cache thread. */
  95 static struct event_queue tagcache_queue;
  96 static long tagcache_stack[(DEFAULT_STACK_SIZE + 0x4000)/sizeof(long)];
  97 static const char tagcache_thread_name[] = "tagcache";
  98 #endif
  99
 100 #define UNTAGGED "<Untagged>"
 101
 102 /* Previous path when scanning directory tree recursively. */
 103 static char curpath[TAG_MAXLEN+32];
 104
 105 /* Used when removing duplicates. */
 106 static char *tempbuf;     /* Allocated when needed. */
 107 static long tempbufidx;   /* Current location in buffer. */
 108 static long tempbuf_size; /* Buffer size (TEMPBUF_SIZE). */
 109 static long tempbuf_left; /* Buffer space left. */
 110 static long tempbuf_pos;
 111
 112 /* Tags we want to get sorted (loaded to the tempbuf). */
 113 static const int sorted_tags[] = { tag_artist, tag_album, tag_genre,
 114     tag_composer, tag_comment, tag_albumartist, tag_grouping, tag_title };
 115
 116 /* Uniqued tags (we can use these tags with filters and conditional clauses). */
 117 static const int unique_tags[] = { tag_artist, tag_album, tag_genre,
 118     tag_composer, tag_comment, tag_albumartist, tag_grouping };
 119
 120 /* Numeric tags (we can use these tags with conditional clauses). */
 121 static const int numeric_tags[] = { tag_year, tag_discnumber,
 122     tag_tracknumber, tag_length, tag_bitrate, tag_playcount, tag_rating,
 123     tag_playtime, tag_lastplayed, tag_commitid, tag_mtime,
 124     tag_virt_length_min, tag_virt_length_sec,
 125     tag_virt_playtime_min, tag_virt_playtime_sec,
 126     tag_virt_entryage, tag_virt_autoscore };
 127
 128 /* String presentation of the tags defined in tagcache.h. Must be in correct order! */
 129 static const char *tags_str[] = { "artist", "album", "genre", "title",
 130     "filename", "composer", "comment", "albumartist", "grouping", "year",
 131     "discnumber", "tracknumber", "bitrate", "length", "playcount", "rating",
 132     "playtime", "lastplayed", "commitid", "mtime" };
 133
 134 /* Status information of the tagcache. */
 135 static struct tagcache_stat tc_stat;
 136
 137 /* Queue commands. */
 138 enum tagcache_queue {
 139     Q_STOP_SCAN = 0,
 140     Q_START_SCAN,
 141     Q_IMPORT_CHANGELOG,
 142     Q_UPDATE,
 143     Q_REBUILD,
 144
 145     /* Internal tagcache command queue. */
 146     CMD_UPDATE_MASTER_HEADER,
 147     CMD_UPDATE_NUMERIC,
 148 };
 149
 150 struct tagcache_command_entry {
 151     int32_t command;
 152     int32_t idx_id;
 153     int32_t tag;
 154     int32_t data;
 155 };
 156
 157 #ifndef __PCTOOL__
 158 static struct tagcache_command_entry command_queue[TAGCACHE_COMMAND_QUEUE_LENGTH];
 159 static volatile int command_queue_widx = 0;
 160 static volatile int command_queue_ridx = 0;
 161 static struct mutex command_queue_mutex;
 162 #endif
 163
 164 /* Tag database structures. */
 165
 166 /* Variable-length tag entry in tag files. */
 167 struct tagfile_entry {
 168     int32_t tag_length;  /* Length of the data in bytes including '\0' */
 169     int32_t idx_id;      /* Corresponding entry location in index file of not unique tags */
 170     char tag_data[0];  /* Begin of the tag data */
 171 };
 172
 173 /* Fixed-size tag entry in master db index. */
 174 struct index_entry {
 175     int32_t tag_seek[TAG_COUNT]; /* Location of tag data or numeric tag data */
 176     int32_t flag;                /* Status flags */
 177 };
 178
 179 /* Header is the same in every file. */
 180 struct tagcache_header {
 181     int32_t magic;       /* Header version number */
 182     int32_t datasize;    /* Data size in bytes */
 183     int32_t entry_count; /* Number of entries in this file */
 184 };
 185
 186 struct master_header {
 187     struct tagcache_header tch;
 188     int32_t serial; /* Increasing counting number */
 189     int32_t commitid; /* Number of commits so far */
 190     int32_t dirty;
 191 };
 192
 193 /* For the endianess correction */
 194 static const char *tagfile_entry_ec   = "ll";
 195 /**
 196  Note: This should be (1 + TAG_COUNT) amount of l's.
 197  */
 198 static const char *index_entry_ec     = "lllllllllllllllllllll";
 199
 200 static const char *tagcache_header_ec = "lll";
 201 static const char *master_header_ec   = "llllll";
 202
 203 static struct master_header current_tcmh;
 204
 205 #ifdef HAVE_TC_RAMCACHE
 206 /* Header is created when loading database to ram. */
 207 struct ramcache_header {
 208     struct master_header h;      /* Header from the master index */
 209     struct index_entry *indices; /* Master index file content */
 210     char *tags[TAG_COUNT];       /* Tag file content (not including filename tag) */
 211     int entry_count[TAG_COUNT];  /* Number of entries in the indices. */
 212 };
 213
 214 # ifdef HAVE_EEPROM_SETTINGS
 215 struct statefile_header {
 216     struct ramcache_header *hdr;
 217     struct tagcache_stat tc_stat;
 218 };
 219 # endif
 220
 221 /* Pointer to allocated ramcache_header */
 222 static struct ramcache_header *hdr;
 223 #endif
 224
 225 /**
 226  * Full tag entries stored in a temporary file waiting
 227  * for commit to the cache. */
 228 struct temp_file_entry {
 229     long tag_offset[TAG_COUNT];
 230     short tag_length[TAG_COUNT];
 231     long flag;
 232
 233     long data_length;
 234 };
 235
 236 struct tempbuf_id_list {
 237     long id;
 238     struct tempbuf_id_list *next;
 239 };
 240
 241 struct tempbuf_searchidx {
 242     long idx_id;
 243     char *str;
 244     int seek;
 245     struct tempbuf_id_list idlist;
 246 };
 247
 248 /* Lookup buffer for fixing messed up index while after sorting. */
 249 static long commit_entry_count;
 250 static long lookup_buffer_depth;
 251 static struct tempbuf_searchidx **lookup;
 252
 253 /* Used when building the temporary file. */
 254 static int cachefd = -1, filenametag_fd;
 255 static int total_entry_count = 0;
 256 static int data_size = 0;
 257 static int processed_dir_count;
 258
 259 /* Thread safe locking */
 260 static volatile int write_lock;
 261 static volatile int read_lock;
 262
 263 static bool delete_entry(long idx_id);
 264
 265 const char* tagcache_tag_to_str(int tag)
 266 {
 267     return tags_str[tag];
 268 }
 269
 270 bool tagcache_is_numeric_tag(int type)
 271 {
 272     int i;
 273
 274     for (i = 0; i < (int)(sizeof(numeric_tags)/sizeof(numeric_tags[0])); i++)
 275     {
 276         if (type == numeric_tags[i])
 277             return true;
 278     }
 279
 280     return false;
 281 }
 282
 283 bool tagcache_is_unique_tag(int type)
 284 {
 285     int i;
 286
 287     for (i = 0; i < (int)(sizeof(unique_tags)/sizeof(unique_tags[0])); i++)
 288     {
 289         if (type == unique_tags[i])
 290             return true;
 291     }
 292
 293     return false;
 294 }
 295
 296 bool tagcache_is_sorted_tag(int type)
 297 {
 298     int i;
 299
 300     for (i = 0; i < (int)(sizeof(sorted_tags)/sizeof(sorted_tags[0])); i++)
 301     {
 302         if (type == sorted_tags[i])
 303             return true;
 304     }
 305
 306     return false;
 307 }
 308
 309 #ifdef HAVE_DIRCACHE
 310 /**
 311  * Returns true if specified flag is still present, i.e., dircache
 312  * has not been reloaded.
 313  */
 314 static bool is_dircache_intact(void)
 315 {
 316     return dircache_get_appflag(DIRCACHE_APPFLAG_TAGCACHE);
 317 }
 318 #endif
 319
 320 static int open_tag_fd(struct tagcache_header *hdr, int tag, bool write)
 321 {
 322     int fd;
 323     char buf[MAX_PATH];
 324     int rc;
 325
 326     if (tagcache_is_numeric_tag(tag) || tag < 0 || tag >= TAG_COUNT)
 327         return -1;
 328
 329     snprintf(buf, sizeof buf, TAGCACHE_FILE_INDEX, tag);
 330
 331     fd = open(buf, write ? O_RDWR : O_RDONLY);
 332     if (fd < 0)
 333     {
 334         logf("tag file open failed: tag=%d write=%d file=%s", tag, write, buf);
 335         tc_stat.ready = false;
 336         return fd;
 337     }
 338
 339     /* Check the header. */
 340     rc = ecread(fd, hdr, 1, tagcache_header_ec, tc_stat.econ);
 341     if (hdr->magic != TAGCACHE_MAGIC || rc != sizeof(struct tagcache_header))
 342     {
 343         logf("header error");
 344         tc_stat.ready = false;
 345         close(fd);
 346         return -2;
 347     }
 348
 349     return fd;
 350 }
 351
 352 static int open_master_fd(struct master_header *hdr, bool write)
 353 {
 354     int fd;
 355     int rc;
 356
 357     fd = open(TAGCACHE_FILE_MASTER, write ? O_RDWR : O_RDONLY);
 358     if (fd < 0)
 359     {
 360         logf("master file open failed for R/W");
 361         tc_stat.ready = false;
 362         return fd;
 363     }
 364
 365     tc_stat.econ = false;
 366
 367     /* Check the header. */
 368     rc = read(fd, hdr, sizeof(struct master_header));
 369     if (hdr->tch.magic == TAGCACHE_MAGIC && rc == sizeof(struct master_header))
 370     {
 371         /* Success. */
 372         return fd;
 373     }
 374
 375     /* Trying to read again, this time with endianess correction enabled. */
 376     lseek(fd, 0, SEEK_SET);
 377
 378     rc = ecread(fd, hdr, 1, master_header_ec, true);
 379     if (hdr->tch.magic != TAGCACHE_MAGIC || rc != sizeof(struct master_header))
 380     {
 381         logf("header error");
 382         tc_stat.ready = false;
 383         close(fd);
 384         return -2;
 385     }
 386
 387     tc_stat.econ = true;
 388
 389     return fd;
 390 }
 391
 392 #ifndef __PCTOOL__
 393 static bool do_timed_yield(void)
 394 {
 395     /* Sorting can lock up for quite a while, so yield occasionally */
 396     static long wakeup_tick = 0;
 397     if (current_tick >= wakeup_tick)
 398     {
 399         wakeup_tick = current_tick + (HZ/4);
 400         yield();
 401         return true;
 402     }
 403     return false;
 404 }
 405 #endif
 406
 407 #if defined(HAVE_TC_RAMCACHE) && defined(HAVE_DIRCACHE)
 408 static long find_entry_ram(const char *filename,
 409                            const struct dirent *dc)
 410 {
 411     static long last_pos = 0;
 412     int i;
 413
 414     /* Check if we tagcache is loaded into ram. */
 415     if (!tc_stat.ramcache)
 416         return -1;
 417
 418     if (dc == NULL)
 419         dc = dircache_get_entry_ptr(filename);
 420
 421     if (dc == NULL)
 422     {
 423         logf("tagcache: file not found.");
 424         return -1;
 425     }
 426
 427     try_again:
 428
 429     if (last_pos > 0)
 430         i = last_pos;
 431     else
 432         i = 0;
 433
 434     for (; i < hdr->h.tch.entry_count; i++)
 435     {
 436         if (hdr->indices[i].tag_seek[tag_filename] == (long)dc)
 437         {
 438             last_pos = MAX(0, i - 3);
 439             return i;
 440         }
 441
 442         do_timed_yield();
 443     }
 444
 445     if (last_pos > 0)
 446     {
 447         last_pos = 0;
 448         goto try_again;
 449     }
 450
 451     return -1;
 452 }
 453 #endif
 454
 455 static long find_entry_disk(const char *filename)
 456 {
 457     struct tagcache_header tch;
 458     static long last_pos = -1;
 459     long pos_history[POS_HISTORY_COUNT];
 460     long pos_history_idx = 0;
 461     bool found = false;
 462     struct tagfile_entry tfe;
 463     int fd;
 464     char buf[TAG_MAXLEN+32];
 465     int i;
 466     int pos = -1;
 467
 468     if (!tc_stat.ready)
 469         return -2;
 470
 471     fd = filenametag_fd;
 472     if (fd < 0)
 473     {
 474         last_pos = -1;
 475         if ( (fd = open_tag_fd(&tch, tag_filename, false)) < 0)
 476             return -1;
 477     }
 478
 479     check_again:
 480
 481     if (last_pos > 0)
 482         lseek(fd, last_pos, SEEK_SET);
 483     else
 484         lseek(fd, sizeof(struct tagcache_header), SEEK_SET);
 485
 486     while (true)
 487     {
 488         pos = lseek(fd, 0, SEEK_CUR);
 489         for (i = pos_history_idx-1; i >= 0; i--)
 490             pos_history[i+1] = pos_history[i];
 491         pos_history[0] = pos;
 492
 493         if (ecread(fd, &tfe, 1, tagfile_entry_ec, tc_stat.econ)
 494             != sizeof(struct tagfile_entry))
 495         {
 496             break ;
 497         }
 498
 499         if (tfe.tag_length >= (long)sizeof(buf))
 500         {
 501             logf("too long tag #1");
 502             close(fd);
 503             last_pos = -1;
 504             return -2;
 505         }
 506
 507         if (read(fd, buf, tfe.tag_length) != tfe.tag_length)
 508         {
 509             logf("read error #2");
 510             close(fd);
 511             last_pos = -1;
 512             return -3;
 513         }
 514
 515         if (!strcasecmp(filename, buf))
 516         {
 517             last_pos = pos_history[pos_history_idx];
 518             found = true;
 519             break ;
 520         }
 521
 522         if (pos_history_idx < POS_HISTORY_COUNT - 1)
 523             pos_history_idx++;
 524     }
 525
 526     /* Not found? */
 527     if (!found)
 528     {
 529         if (last_pos > 0)
 530         {
 531             last_pos = -1;
 532             logf("seek again");
 533             goto check_again;
 534         }
 535
 536         if (fd != filenametag_fd)
 537             close(fd);
 538         return -4;
 539     }
 540
 541     if (fd != filenametag_fd)
 542         close(fd);
 543
 544     return tfe.idx_id;
 545 }
 546
 547 static int find_index(const char *filename)
 548 {
 549     long idx_id = -1;
 550
 551 #if defined(HAVE_TC_RAMCACHE) && defined(HAVE_DIRCACHE)
 552     if (tc_stat.ramcache && is_dircache_intact())
 553         idx_id = find_entry_ram(filename, NULL);
 554 #endif
 555
 556     if (idx_id < 0)
 557         idx_id = find_entry_disk(filename);
 558
 559     return idx_id;
 560 }
 561
 562 bool tagcache_find_index(struct tagcache_search *tcs, const char *filename)
 563 {
 564     int idx_id;
 565
 566     if (!tc_stat.ready)
 567         return false;
 568
 569     idx_id = find_index(filename);
 570     if (idx_id < 0)
 571         return false;
 572
 573     if (!tagcache_search(tcs, tag_filename))
 574         return false;
 575
 576     tcs->entry_count = 0;
 577     tcs->idx_id = idx_id;
 578
 579     return true;
 580 }
 581
 582 static bool get_index(int masterfd, int idxid,
 583                       struct index_entry *idx, bool use_ram)
 584 {
 585     bool localfd = false;
 586
 587     if (idxid < 0)
 588     {
 589         logf("Incorrect idxid: %d", idxid);
 590         return false;
 591     }
 592
 593 #ifdef HAVE_TC_RAMCACHE
 594     if (tc_stat.ramcache && use_ram)
 595     {
 596         if (hdr->indices[idxid].flag & FLAG_DELETED)
 597             return false;
 598
 599         memcpy(idx, &hdr->indices[idxid], sizeof(struct index_entry));
 600         return true;
 601     }
 602 #else
 603     (void)use_ram;
 604 #endif
 605
 606     if (masterfd < 0)
 607     {
 608         struct master_header tcmh;
 609
 610         localfd = true;
 611         masterfd = open_master_fd(&tcmh, false);
 612         if (masterfd < 0)
 613             return false;
 614     }
 615
 616     lseek(masterfd, idxid * sizeof(struct index_entry)
 617           + sizeof(struct master_header), SEEK_SET);
 618     if (ecread(masterfd, idx, 1, index_entry_ec, tc_stat.econ)
 619         != sizeof(struct index_entry))
 620     {
 621         logf("read error #3");
 622         if (localfd)
 623             close(masterfd);
 624
 625         return false;
 626     }
 627
 628     if (localfd)
 629         close(masterfd);
 630
 631     if (idx->flag & FLAG_DELETED)
 632         return false;
 633
 634     return true;
 635 }
 636
 637 #ifndef __PCTOOL__
 638
 639 static bool write_index(int masterfd, int idxid, struct index_entry *idx)
 640 {
 641     /* We need to exclude all memory only flags & tags when writing to disk. */
 642     if (idx->flag & FLAG_DIRCACHE)
 643     {
 644         logf("memory only flags!");
 645         return false;
 646     }
 647
 648 #ifdef HAVE_TC_RAMCACHE
 649     /* Only update numeric data. Writing the whole index to RAM by memcpy
 650      * destroys dircache pointers!
 651      */
 652     if (tc_stat.ramcache)
 653     {
 654         int tag;
 655         struct index_entry *idx_ram = &hdr->indices[idxid];
 656
 657         for (tag = 0; tag < TAG_COUNT; tag++)
 658         {
 659             if (tagcache_is_numeric_tag(tag))
 660             {
 661                 idx_ram->tag_seek[tag] = idx->tag_seek[tag];
 662             }
 663         }
 664
 665         /* Don't touch the dircache flag or attributes. */
 666         idx_ram->flag = (idx->flag & 0x0000ffff)
 667             | (idx_ram->flag & (0xffff0000 | FLAG_DIRCACHE));
 668     }
 669 #endif
 670
 671     lseek(masterfd, idxid * sizeof(struct index_entry)
 672           + sizeof(struct master_header), SEEK_SET);
 673     if (ecwrite(masterfd, idx, 1, index_entry_ec, tc_stat.econ)
 674         != sizeof(struct index_entry))
 675     {
 676         logf("write error #3");
 677         logf("idxid: %d", idxid);
 678         return false;
 679     }
 680
 681     return true;
 682 }
 683
 684 #endif /* !__PCTOOL__ */
 685
 686 static bool open_files(struct tagcache_search *tcs, int tag)
 687 {
 688     if (tcs->idxfd[tag] < 0)
 689     {
 690         char fn[MAX_PATH];
 691
 692         snprintf(fn, sizeof fn, TAGCACHE_FILE_INDEX, tag);
 693         tcs->idxfd[tag] = open(fn, O_RDONLY);
 694     }
 695
 696     if (tcs->idxfd[tag] < 0)
 697     {
 698         logf("File not open!");
 699         return false;
 700     }
 701
 702     return true;
 703 }
 704
 705 static bool retrieve(struct tagcache_search *tcs, struct index_entry *idx,
 706                      int tag, char *buf, long size)
 707 {
 708     struct tagfile_entry tfe;
 709     long seek;
 710
 711     *buf = '\0';
 712
 713     if (tagcache_is_numeric_tag(tag))
 714         return false;
 715
 716     seek = idx->tag_seek[tag];
 717     if (seek < 0)
 718     {
 719         logf("Retrieve failed");
 720         return false;
 721     }
 722
 723 #ifdef HAVE_TC_RAMCACHE
 724     if (tcs->ramsearch)
 725     {
 726         struct tagfile_entry *ep;
 727
 728 # ifdef HAVE_DIRCACHE
 729         if (tag == tag_filename && (idx->flag & FLAG_DIRCACHE)
 730             && is_dircache_intact())
 731         {
 732             dircache_copy_path((struct dirent *)seek,
 733                                buf, size);
 734             return true;
 735         }
 736         else
 737 # endif
 738         if (tag != tag_filename)
 739         {
 740             ep = (struct tagfile_entry *)&hdr->tags[tag][seek];
 741             strncpy(buf, ep->tag_data, size-1);
 742
 743             return true;
 744         }
 745     }
 746 #endif
 747
 748     if (!open_files(tcs, tag))
 749         return false;
 750
 751     lseek(tcs->idxfd[tag], seek, SEEK_SET);
 752     if (ecread(tcs->idxfd[tag], &tfe, 1, tagfile_entry_ec, tc_stat.econ)
 753         != sizeof(struct tagfile_entry))
 754     {
 755         logf("read error #5");
 756         return false;
 757     }
 758
 759     if (tfe.tag_length >= size)
 760     {
 761         logf("too small buffer");
 762         return false;
 763     }
 764
 765     if (read(tcs->idxfd[tag], buf, tfe.tag_length) !=
 766         tfe.tag_length)
 767     {
 768         logf("read error #6");
 769         return false;
 770     }
 771
 772     buf[tfe.tag_length] = '\0';
 773
 774     return true;
 775 }
 776
 777 static long check_virtual_tags(int tag, const struct index_entry *idx)
 778 {
 779     long data = 0;
 780
 781     switch (tag)
 782     {
 783         case tag_virt_length_sec:
 784             data = (idx->tag_seek[tag_length]/1000) % 60;
 785             break;
 786
 787         case tag_virt_length_min:
 788             data = (idx->tag_seek[tag_length]/1000) / 60;
 789             break;
 790
 791         case tag_virt_playtime_sec:
 792             data = (idx->tag_seek[tag_playtime]/1000) % 60;
 793             break;
 794
 795         case tag_virt_playtime_min:
 796             data = (idx->tag_seek[tag_playtime]/1000) / 60;
 797             break;
 798
 799         case tag_virt_autoscore:
 800             if (idx->tag_seek[tag_length] == 0
 801                 || idx->tag_seek[tag_playcount] == 0)
 802             {
 803                 data = 0;
 804             }
 805             else
 806             {
 807                 data = 100 * idx->tag_seek[tag_playtime]
 808                     / idx->tag_seek[tag_length]
 809                     / idx->tag_seek[tag_playcount];
 810             }
 811             break;
 812
 813         /* How many commits before the file has been added to the DB. */
 814         case tag_virt_entryage:
 815             data = current_tcmh.commitid - idx->tag_seek[tag_commitid] - 1;
 816             break;
 817
 818         default:
 819             data = idx->tag_seek[tag];
 820     }
 821
 822     return data;
 823 }
 824
 825 long tagcache_get_numeric(const struct tagcache_search *tcs, int tag)
 826 {
 827     struct index_entry idx;
 828
 829     if (!tc_stat.ready)
 830         return false;
 831
 832     if (!tagcache_is_numeric_tag(tag))
 833         return -1;
 834
 835     if (!get_index(tcs->masterfd, tcs->idx_id, &idx, true))
 836         return -2;
 837
 838     return check_virtual_tags(tag, &idx);
 839 }
 840
 841 inline static bool str_ends_with(const char *str1, const char *str2)
 842 {
 843     int str_len = strlen(str1);
 844     int clause_len = strlen(str2);
 845
 846     if (clause_len > str_len)
 847         return false;
 848
 849     return !strcasecmp(&str1[str_len - clause_len], str2);
 850 }
 851
 852 inline static bool str_oneof(const char *str, const char *list)
 853 {
 854     const char *sep;
 855     int l, len = strlen(str);
 856
 857     while (*list)
 858     {
 859         sep = strchr(list, '|');
 860         l = sep ? (long)sep - (long)list : (int)strlen(list);
 861         if ((l==len) && !strncasecmp(str, list, len))
 862             return true;
 863         list += sep ? l + 1 : l;
 864     }
 865
 866     return false;
 867 }
 868
 869 static bool check_against_clause(long numeric, const char *str,
 870                                  const struct tagcache_search_clause *clause)
 871 {
 872     if (clause->numeric)
 873     {
 874         switch (clause->type)
 875         {
 876             case clause_is:
 877                 return numeric == clause->numeric_data;
 878             case clause_is_not:
 879                 return numeric != clause->numeric_data;
 880             case clause_gt:
 881                 return numeric > clause->numeric_data;
 882             case clause_gteq:
 883                 return numeric >= clause->numeric_data;
 884             case clause_lt:
 885                 return numeric < clause->numeric_data;
 886             case clause_lteq:
 887                 return numeric <= clause->numeric_data;
 888             default:
 889                 logf("Incorrect numeric tag: %d", clause->type);
 890         }
 891     }
 892     else
 893     {
 894         switch (clause->type)
 895         {
 896             case clause_is:
 897                 return !strcasecmp(clause->str, str);
 898             case clause_is_not:
 899                 return strcasecmp(clause->str, str);
 900             case clause_gt:
 901                 return 0>strcasecmp(clause->str, str);
 902             case clause_gteq:
 903                 return 0>=strcasecmp(clause->str, str);
 904             case clause_lt:
 905                 return 0<strcasecmp(clause->str, str);
 906             case clause_lteq:
 907                 return 0<=strcasecmp(clause->str, str);
 908             case clause_contains:
 909                 return (strcasestr(str, clause->str) != NULL);
 910             case clause_not_contains:
 911                 return (strcasestr(str, clause->str) == NULL);
 912             case clause_begins_with:
 913                 return (strcasestr(str, clause->str) == str);
 914             case clause_not_begins_with:
 915                 return (strcasestr(str, clause->str) != str);
 916             case clause_ends_with:
 917                 return str_ends_with(str, clause->str);
 918             case clause_not_ends_with:
 919                 return !str_ends_with(str, clause->str);
 920             case clause_oneof:
 921                 return str_oneof(str, clause->str);
 922
 923             default:
 924                 logf("Incorrect tag: %d", clause->type);
 925         }
 926     }
 927
 928     return false;
 929 }
 930
 931 static bool check_clauses(struct tagcache_search *tcs,
 932                           struct index_entry *idx,
 933                           struct tagcache_search_clause **clause, int count)
 934 {
 935     int i;
 936
 937 #ifdef HAVE_TC_RAMCACHE
 938     if (tcs->ramsearch)
 939     {
 940         /* Go through all conditional clauses. */
 941         for (i = 0; i < count; i++)
 942         {
 943             struct tagfile_entry *tfe;
 944             int seek;
 945             char buf[256];
 946             char *str = NULL;
 947
 948             seek = check_virtual_tags(clause[i]->tag, idx);
 949
 950             if (!tagcache_is_numeric_tag(clause[i]->tag))
 951             {
 952                 if (clause[i]->tag == tag_filename)
 953                 {
 954                     retrieve(tcs, idx, tag_filename, buf, sizeof buf);
 955                     str = buf;
 956                 }
 957                 else
 958                 {
 959                     tfe = (struct tagfile_entry *)&hdr->tags[clause[i]->tag][seek];
 960                     str = tfe->tag_data;
 961                 }
 962             }
 963
 964             if (!check_against_clause(seek, str, clause[i]))
 965                 return false;
 966         }
 967     }
 968     else
 969 #endif
 970     {
 971         /* Check for conditions. */
 972         for (i = 0; i < count; i++)
 973         {
 974             struct tagfile_entry tfe;
 975             int seek;
 976             char str[256];
 977
 978             seek = check_virtual_tags(clause[i]->tag, idx);
 979
 980             memset(str, 0, sizeof str);
 981             if (!tagcache_is_numeric_tag(clause[i]->tag))
 982             {
 983                 int fd = tcs->idxfd[clause[i]->tag];
 984                 lseek(fd, seek, SEEK_SET);
 985                 ecread(fd, &tfe, 1, tagfile_entry_ec, tc_stat.econ);
 986                 if (tfe.tag_length >= (int)sizeof(str))
 987                 {
 988                     logf("Too long tag read!");
 989                     break ;
 990                 }
 991
 992                 read(fd, str, tfe.tag_length);
 993
 994                 /* Check if entry has been deleted. */
 995                 if (str[0] == '\0')
 996                     break;
 997             }
 998
 999             if (!check_against_clause(seek, str, clause[i]))
1000                 return false;
1001         }
1002     }
1003
1004     return true;
1005 }
1006
1007 bool tagcache_check_clauses(struct tagcache_search *tcs,
1008                             struct tagcache_search_clause **clause, int count)
1009 {
1010     struct index_entry idx;
1011
1012     if (count == 0)
1013         return true;
1014
1015     if (!get_index(tcs->masterfd, tcs->idx_id, &idx, true))
1016         return false;
1017
1018     return check_clauses(tcs, &idx, clause, count);
1019 }
1020
1021 static bool add_uniqbuf(struct tagcache_search *tcs, unsigned long id)
1022 {
1023     int i;
1024
1025     /* If uniq buffer is not defined we must return true for search to work. */
1026     if (tcs->unique_list == NULL
1027         || (!tagcache_is_unique_tag(tcs->type)
1028             && !tagcache_is_numeric_tag(tcs->type)))
1029     {
1030         return true;
1031     }
1032
1033     for (i = 0; i < tcs->unique_list_count; i++)
1034     {
1035         /* Return false if entry is found. */
1036         if (tcs->unique_list[i] == id)
1037             return false;
1038     }
1039
1040     if (tcs->unique_list_count < tcs->unique_list_capacity)
1041     {
1042         tcs->unique_list[i] = id;
1043         tcs->unique_list_count++;
1044     }
1045
1046     return true;
1047 }
1048
1049 static bool build_lookup_list(struct tagcache_search *tcs)
1050 {
1051     struct index_entry entry;
1052     int i;
1053
1054     tcs->seek_list_count = 0;
1055
1056 #ifdef HAVE_TC_RAMCACHE
1057     if (tcs->ramsearch)
1058     {
1059         int j;
1060
1061         for (i = tcs->seek_pos; i < hdr->h.tch.entry_count; i++)
1062         {
1063             struct index_entry *idx = &hdr->indices[i];
1064             if (tcs->seek_list_count == SEEK_LIST_SIZE)
1065                 break ;
1066
1067             /* Skip deleted files. */
1068             if (idx->flag & FLAG_DELETED)
1069                 continue;
1070
1071             /* Go through all filters.. */
1072             for (j = 0; j < tcs->filter_count; j++)
1073             {
1074                 if (idx->tag_seek[tcs->filter_tag[j]] != tcs->filter_seek[j])
1075                 {
1076                     break ;
1077                 }
1078             }
1079
1080             if (j < tcs->filter_count)
1081                 continue ;
1082
1083             /* Check for conditions. */
1084             if (!check_clauses(tcs, idx, tcs->clause, tcs->clause_count))
1085                 continue;
1086
1087             /* Add to the seek list if not already in uniq buffer. */
1088             if (!add_uniqbuf(tcs, idx->tag_seek[tcs->type]))
1089                 continue;
1090
1091             /* Lets add it. */
1092             tcs->seek_list[tcs->seek_list_count] = idx->tag_seek[tcs->type];
1093             tcs->seek_flags[tcs->seek_list_count] = idx->flag;
1094             tcs->seek_list_count++;
1095         }
1096
1097         tcs->seek_pos = i;
1098
1099         return tcs->seek_list_count > 0;
1100     }
1101 #endif
1102
1103     lseek(tcs->masterfd, tcs->seek_pos * sizeof(struct index_entry) +
1104             sizeof(struct master_header), SEEK_SET);
1105
1106     while (ecread(tcs->masterfd, &entry, 1, index_entry_ec, tc_stat.econ)
1107            == sizeof(struct index_entry))
1108     {
1109         if (tcs->seek_list_count == SEEK_LIST_SIZE)
1110             break ;
1111
1112         tcs->seek_pos++;
1113
1114         /* Check if entry has been deleted. */
1115         if (entry.flag & FLAG_DELETED)
1116             continue;
1117
1118         /* Go through all filters.. */
1119         for (i = 0; i < tcs->filter_count; i++)
1120         {
1121             if (entry.tag_seek[tcs->filter_tag[i]] != tcs->filter_seek[i])
1122                 break ;
1123         }
1124
1125         if (i < tcs->filter_count)
1126             continue ;
1127
1128         /* Check for conditions. */
1129         if (!check_clauses(tcs, &entry, tcs->clause, tcs->clause_count))
1130             continue;
1131
1132         /* Add to the seek list if not already in uniq buffer. */
1133         if (!add_uniqbuf(tcs, entry.tag_seek[tcs->type]))
1134             continue;
1135
1136         /* Lets add it. */
1137         tcs->seek_list[tcs->seek_list_count] = entry.tag_seek[tcs->type];
1138         tcs->seek_flags[tcs->seek_list_count] = entry.flag;
1139         tcs->seek_list_count++;
1140
1141         yield();
1142     }
1143
1144     return tcs->seek_list_count > 0;
1145 }
1146
1147
1148 static void remove_files(void)
1149 {
1150     int i;
1151     char buf[MAX_PATH];
1152
1153     tc_stat.ready = false;
1154     tc_stat.ramcache = false;
1155     tc_stat.econ = false;
1156     remove(TAGCACHE_FILE_MASTER);
1157     for (i = 0; i < TAG_COUNT; i++)
1158     {
1159         if (tagcache_is_numeric_tag(i))
1160             continue;
1161
1162         snprintf(buf, sizeof buf, TAGCACHE_FILE_INDEX, i);
1163         remove(buf);
1164     }
1165 }
1166
1167
1168 static bool check_all_headers(void)
1169 {
1170     struct master_header myhdr;
1171     struct tagcache_header tch;
1172     int tag;
1173     int fd;
1174
1175     if ( (fd = open_master_fd(&myhdr, false)) < 0)
1176         return false;
1177
1178     close(fd);
1179     if (myhdr.dirty)
1180     {
1181         logf("tagcache is dirty!");
1182         return false;
1183     }
1184
1185     memcpy(&current_tcmh, &myhdr, sizeof(struct master_header));
1186
1187     for (tag = 0; tag < TAG_COUNT; tag++)
1188     {
1189         if (tagcache_is_numeric_tag(tag))
1190             continue;
1191
1192         if ( (fd = open_tag_fd(&tch, tag, false)) < 0)
1193             return false;
1194
1195         close(fd);
1196     }
1197
1198     return true;
1199 }
1200
1201 bool tagcache_search(struct tagcache_search *tcs, int tag)
1202 {
1203     struct tagcache_header tag_hdr;
1204     struct master_header   master_hdr;
1205     int i;
1206
1207     if (tcs->initialized)
1208         tagcache_search_finish(tcs);
1209
1210     while (read_lock)
1211         sleep(1);
1212
1213     memset(tcs, 0, sizeof(struct tagcache_search));
1214     if (tc_stat.commit_step > 0 || !tc_stat.ready)
1215         return false;
1216
1217     tcs->position = sizeof(struct tagcache_header);
1218     tcs->type = tag;
1219     tcs->seek_pos = 0;
1220     tcs->seek_list_count = 0;
1221     tcs->filter_count = 0;
1222     tcs->masterfd = -1;
1223
1224     for (i = 0; i < TAG_COUNT; i++)
1225         tcs->idxfd[i] = -1;
1226
1227 #ifndef HAVE_TC_RAMCACHE
1228     tcs->ramsearch = false;
1229 #else
1230     tcs->ramsearch = tc_stat.ramcache;
1231     if (tcs->ramsearch)
1232     {
1233         tcs->entry_count = hdr->entry_count[tcs->type];
1234     }
1235     else
1236 #endif
1237     {
1238         if (!tagcache_is_numeric_tag(tcs->type))
1239         {
1240             tcs->idxfd[tcs->type] = open_tag_fd(&tag_hdr, tcs->type, false);
1241             if (tcs->idxfd[tcs->type] < 0)
1242                 return false;
1243         }
1244
1245         /* Always open as R/W so we can pass tcs to functions that modify data also
1246          * without failing. */
1247         tcs->masterfd = open_master_fd(&master_hdr, true);
1248
1249         if (tcs->masterfd < 0)
1250             return false;
1251     }
1252
1253     tcs->valid = true;
1254     tcs->initialized = true;
1255     write_lock++;
1256
1257     return true;
1258 }
1259
1260 void tagcache_search_set_uniqbuf(struct tagcache_search *tcs,
1261                                  void *buffer, long length)
1262 {
1263     tcs->unique_list = (unsigned long *)buffer;
1264     tcs->unique_list_capacity = length / sizeof(*tcs->unique_list);
1265     tcs->unique_list_count = 0;
1266 }
1267
1268 bool tagcache_search_add_filter(struct tagcache_search *tcs,
1269                                 int tag, int seek)
1270 {
1271     if (tcs->filter_count == TAGCACHE_MAX_FILTERS)
1272         return false;
1273
1274     if (!tagcache_is_unique_tag(tag) || tagcache_is_numeric_tag(tag))
1275         return false;
1276
1277     tcs->filter_tag[tcs->filter_count] = tag;
1278     tcs->filter_seek[tcs->filter_count] = seek;
1279     tcs->filter_count++;
1280
1281     return true;
1282 }
1283
1284 bool tagcache_search_add_clause(struct tagcache_search *tcs,
1285                                 struct tagcache_search_clause *clause)
1286 {
1287     int i;
1288
1289     if (tcs->clause_count >= TAGCACHE_MAX_CLAUSES)
1290     {
1291         logf("Too many clauses");
1292         return false;
1293     }
1294
1295     /* Check if there is already a similar filter in present (filters are
1296      * much faster than clauses).
1297      */
1298     for (i = 0; i < tcs->filter_count; i++)
1299     {
1300         if (tcs->filter_tag[i] == clause->tag)
1301             return true;
1302     }
1303
1304     if (!tagcache_is_numeric_tag(clause->tag) && tcs->idxfd[clause->tag] < 0)
1305     {
1306         char buf[MAX_PATH];
1307
1308         snprintf(buf, sizeof buf, TAGCACHE_FILE_INDEX, clause->tag);
1309         tcs->idxfd[clause->tag] = open(buf, O_RDONLY);
1310     }
1311
1312     tcs->clause[tcs->clause_count] = clause;
1313     tcs->clause_count++;
1314
1315     return true;
1316 }
1317
1318 /* TODO: Remove this mess. */
1319 #ifdef HAVE_DIRCACHE
1320 #define TAG_FILENAME_RAM(tcs) ((tcs->type == tag_filename) \
1321     ? ((flag & FLAG_DIRCACHE) && is_dircache_intact()) : 1)
1322 #else
1323 #define TAG_FILENAME_RAM(tcs) (tcs->type != tag_filename)
1324 #endif
1325
1326 static bool get_next(struct tagcache_search *tcs)
1327 {
1328     static char buf[TAG_MAXLEN+32];
1329     struct tagfile_entry entry;
1330     long flag = 0;
1331
1332     if (!tcs->valid || !tc_stat.ready)
1333         return false;
1334
1335     if (tcs->idxfd[tcs->type] < 0 && !tagcache_is_numeric_tag(tcs->type)
1336 #ifdef HAVE_TC_RAMCACHE
1337         && !tcs->ramsearch
1338 #endif
1339         )
1340         return false;
1341
1342     /* Relative fetch. */
1343     if (tcs->filter_count > 0 || tcs->clause_count > 0
1344         || tagcache_is_numeric_tag(tcs->type))
1345     {
1346         /* Check for end of list. */
1347         if (tcs->seek_list_count == 0)
1348         {
1349             /* Try to fetch more. */
1350             if (!build_lookup_list(tcs))
1351             {
1352                 tcs->valid = false;
1353                 return false;
1354             }
1355         }
1356
1357         tcs->seek_list_count--;
1358         flag = tcs->seek_flags[tcs->seek_list_count];
1359
1360         /* Seek stream to the correct position and continue to direct fetch. */
1361         if ((!tcs->ramsearch || !TAG_FILENAME_RAM(tcs))
1362             && !tagcache_is_numeric_tag(tcs->type))
1363         {
1364             if (!open_files(tcs, tcs->type))
1365                 return false;
1366
1367             lseek(tcs->idxfd[tcs->type], tcs->seek_list[tcs->seek_list_count], SEEK_SET);
1368         }
1369         else
1370             tcs->position = tcs->seek_list[tcs->seek_list_count];
1371     }
1372
1373     if (tagcache_is_numeric_tag(tcs->type))
1374     {
1375         snprintf(buf, sizeof(buf), "%d", tcs->position);
1376         tcs->result_seek = tcs->position;
1377         tcs->result = buf;
1378         tcs->result_len = strlen(buf) + 1;
1379         return true;
1380     }
1381
1382     /* Direct fetch. */
1383 #ifdef HAVE_TC_RAMCACHE
1384     if (tcs->ramsearch && TAG_FILENAME_RAM(tcs))
1385     {
1386         struct tagfile_entry *ep;
1387
1388         if (tcs->entry_count == 0)
1389         {
1390             tcs->valid = false;
1391             return false;
1392         }
1393         tcs->entry_count--;
1394
1395         tcs->result_seek = tcs->position;
1396
1397 # ifdef HAVE_DIRCACHE
1398         if (tcs->type == tag_filename)
1399         {
1400             dircache_copy_path((struct dirent *)tcs->position,
1401                                buf, sizeof buf);
1402             tcs->result = buf;
1403             tcs->result_len = strlen(buf) + 1;
1404             tcs->idx_id = FLAG_GET_ATTR(flag);
1405             tcs->ramresult = false;
1406
1407             return true;
1408         }
1409 # endif
1410
1411         ep = (struct tagfile_entry *)&hdr->tags[tcs->type][tcs->position];
1412         tcs->position += sizeof(struct tagfile_entry) + ep->tag_length;
1413         tcs->result = ep->tag_data;
1414         tcs->result_len = strlen(tcs->result) + 1;
1415         tcs->idx_id = ep->idx_id;
1416         tcs->ramresult = true;
1417
1418         return true;
1419     }
1420     else
1421 #endif
1422     {
1423         if (!open_files(tcs, tcs->type))
1424             return false;
1425
1426         tcs->result_seek = lseek(tcs->idxfd[tcs->type], 0, SEEK_CUR);
1427         if (ecread(tcs->idxfd[tcs->type], &entry, 1,
1428                  tagfile_entry_ec, tc_stat.econ) != sizeof(struct tagfile_entry))
1429         {
1430             /* End of data. */
1431             tcs->valid = false;
1432             return false;
1433         }
1434     }
1435
1436     if (entry.tag_length > (long)sizeof(buf))
1437     {
1438         tcs->valid = false;
1439         logf("too long tag #2");
1440         return false;
1441     }
1442
1443     if (read(tcs->idxfd[tcs->type], buf, entry.tag_length) != entry.tag_length)
1444     {
1445         tcs->valid = false;
1446         logf("read error #4");
1447         return false;
1448     }
1449
1450     tcs->result = buf;
1451     tcs->result_len = strlen(tcs->result) + 1;
1452     tcs->idx_id = entry.idx_id;
1453     tcs->ramresult = false;
1454
1455     return true;
1456 }
1457
1458 bool tagcache_get_next(struct tagcache_search *tcs)
1459 {
1460     while (get_next(tcs))
1461     {
1462         if (tcs->result_len > 1)
1463             return true;
1464     }
1465
1466     return false;
1467 }
1468
1469 bool tagcache_retrieve(struct tagcache_search *tcs, int idxid,
1470                        int tag, char *buf, long size)
1471 {
1472     struct index_entry idx;
1473
1474     *buf = '\0';
1475     if (!get_index(tcs->masterfd, idxid, &idx, true))
1476         return false;
1477
1478     return retrieve(tcs, &idx, tag, buf, size);
1479 }
1480
1481 static bool update_master_header(void)
1482 {
1483     struct master_header myhdr;
1484     int fd;
1485
1486     if (!tc_stat.ready)
1487         return false;
1488
1489     if ( (fd = open_master_fd(&myhdr, true)) < 0)
1490         return false;
1491
1492     myhdr.serial = current_tcmh.serial;
1493     myhdr.commitid = current_tcmh.commitid;
1494
1495     /* Write it back */
1496     lseek(fd, 0, SEEK_SET);
1497     ecwrite(fd, &myhdr, 1, master_header_ec, tc_stat.econ);
1498     close(fd);
1499
1500 #ifdef HAVE_TC_RAMCACHE
1501     if (hdr)
1502     {
1503         hdr->h.serial = current_tcmh.serial;
1504         hdr->h.commitid = current_tcmh.commitid;
1505     }
1506 #endif
1507
1508     return true;
1509 }
1510
1511 #if 0
1512
1513 void tagcache_modify(struct tagcache_search *tcs, int type, const char *text)
1514 {
1515     struct tagentry *entry;
1516
1517     if (tcs->type != tag_title)
1518         return ;
1519
1520     /* We will need reserve buffer for this. */
1521     if (tcs->ramcache)
1522     {
1523         struct tagfile_entry *ep;
1524
1525         ep = (struct tagfile_entry *)&hdr->tags[tcs->type][tcs->result_seek];
1526         tcs->seek_list[tcs->seek_list_count];
1527     }
1528
1529     entry = find_entry_ram();
1530
1531 }
1532 #endif
1533
1534 void tagcache_search_finish(struct tagcache_search *tcs)
1535 {
1536     int i;
1537
1538     if (!tcs->initialized)
1539         return;
1540
1541     if (tcs->masterfd >= 0)
1542     {
1543         close(tcs->masterfd);
1544         tcs->masterfd = -1;
1545     }
1546
1547     for (i = 0; i < TAG_COUNT; i++)
1548     {
1549         if (tcs->idxfd[i] >= 0)
1550         {
1551             close(tcs->idxfd[i]);
1552             tcs->idxfd[i] = -1;
1553         }
1554     }
1555
1556     tcs->ramsearch = false;
1557     tcs->valid = false;
1558     tcs->initialized = 0;
1559     if (write_lock > 0)
1560         write_lock--;
1561 }
1562
1563 #if defined(HAVE_TC_RAMCACHE) && defined(HAVE_DIRCACHE)
1564 static struct tagfile_entry *get_tag(const struct index_entry *entry, int tag)
1565 {
1566     return (struct tagfile_entry *)&hdr->tags[tag][entry->tag_seek[tag]];
1567 }
1568
1569 static long get_tag_numeric(const struct index_entry *entry, int tag)
1570 {
1571     return check_virtual_tags(tag, entry);
1572 }
1573
1574 static char* get_tag_string(const struct index_entry *entry, int tag)
1575 {
1576     char* s = get_tag(entry, tag)->tag_data;
1577     return strcmp(s, UNTAGGED) ? s : NULL;
1578 }
1579
1580 bool tagcache_fill_tags(struct mp3entry *id3, const char *filename)
1581 {
1582     struct index_entry *entry;
1583     int idx_id;
1584
1585     if (!tc_stat.ready)
1586         return false;
1587
1588     /* Find the corresponding entry in tagcache. */
1589     idx_id = find_entry_ram(filename, NULL);
1590     if (idx_id < 0 || !tc_stat.ramcache)
1591         return false;
1592
1593     entry = &hdr->indices[idx_id];
1594
1595     id3->title        = get_tag_string(entry, tag_title);
1596     id3->artist       = get_tag_string(entry, tag_artist);
1597     id3->album        = get_tag_string(entry, tag_album);
1598     id3->genre_string = get_tag_string(entry, tag_genre);
1599     id3->composer     = get_tag_string(entry, tag_composer);
1600     id3->comment      = get_tag_string(entry, tag_comment);
1601     id3->albumartist  = get_tag_string(entry, tag_albumartist);
1602     id3->grouping     = get_tag_string(entry, tag_grouping);
1603
1604     id3->playcount  = get_tag_numeric(entry, tag_playcount);
1605     id3->rating     = get_tag_numeric(entry, tag_rating);
1606     id3->lastplayed = get_tag_numeric(entry, tag_lastplayed);
1607     id3->score      = get_tag_numeric(entry, tag_virt_autoscore) / 10;
1608     id3->year       = get_tag_numeric(entry, tag_year);
1609
1610     id3->discnum = get_tag_numeric(entry, tag_discnumber);
1611     id3->tracknum = get_tag_numeric(entry, tag_tracknumber);
1612     id3->bitrate = get_tag_numeric(entry, tag_bitrate);
1613     if (id3->bitrate == 0)
1614         id3->bitrate = 1;
1615
1616     return true;
1617 }
1618 #endif
1619
1620 static inline void write_item(const char *item)
1621 {
1622     int len = strlen(item) + 1;
1623
1624     data_size += len;
1625     write(cachefd, item, len);
1626 }
1627
1628 static int check_if_empty(char **tag)
1629 {
1630     int length;
1631
1632     if (*tag == NULL || **tag == '\0')
1633     {
1634         *tag = UNTAGGED;
1635         return sizeof(UNTAGGED); /* Tag length */
1636     }
1637
1638     length = strlen(*tag);
1639     if (length > TAG_MAXLEN)
1640     {
1641         logf("over length tag: %s", *tag);
1642         length = TAG_MAXLEN;
1643         (*tag)[length] = '\0';
1644     }
1645
1646     return length + 1;
1647 }
1648
1649 #define ADD_TAG(entry,tag,data) \
1650     /* Adding tag */ \
1651     entry.tag_offset[tag] = offset; \
1652     entry.tag_length[tag] = check_if_empty(data); \
1653     offset += entry.tag_length[tag]
1654
1655 static void add_tagcache(char *path, unsigned long mtime
1656 #if defined(HAVE_TC_RAMCACHE) && defined(HAVE_DIRCACHE)
1657                          ,const struct dirent *dc
1658 #endif
1659                          )
1660 {
1661     struct mp3entry id3;
1662     struct temp_file_entry entry;
1663     bool ret;
1664     int fd;
1665     int idx_id = -1;
1666     char tracknumfix[3];
1667     int offset = 0;
1668     int path_length = strlen(path);
1669     bool has_albumartist;
1670     bool has_grouping;
1671
1672     if (cachefd < 0)
1673         return ;
1674
1675     /* Check for overlength file path. */
1676     if (path_length > TAG_MAXLEN)
1677     {
1678         /* Path can't be shortened. */
1679         logf("Too long path: %s", path);
1680         return ;
1681     }
1682
1683     /* Check if the file is supported. */
1684     if (probe_file_format(path) == AFMT_UNKNOWN)
1685         return ;
1686
1687     /* Check if the file is already cached. */
1688 #if defined(HAVE_TC_RAMCACHE) && defined(HAVE_DIRCACHE)
1689     if (tc_stat.ramcache && is_dircache_intact())
1690     {
1691         idx_id = find_entry_ram(path, dc);
1692     }
1693     else
1694 #endif
1695     {
1696         if (filenametag_fd >= 0)
1697         {
1698             idx_id = find_entry_disk(path);
1699         }
1700     }
1701
1702     /* Check if file has been modified. */
1703     if (idx_id >= 0)
1704     {
1705         struct index_entry idx;
1706
1707         /* TODO: Mark that the index exists (for fast reverse scan) */
1708         //found_idx[idx_id/8] |= idx_id%8;
1709
1710         if (!get_index(-1, idx_id, &idx, true))
1711         {
1712             logf("failed to retrieve index entry");
1713             return ;
1714         }
1715
1716         if ((unsigned long)idx.tag_seek[tag_mtime] == mtime)
1717         {
1718             /* No changes to file. */
1719             return ;
1720         }
1721
1722         /* Metadata might have been changed. Delete the entry. */
1723         logf("Re-adding: %s", path);
1724         if (!delete_entry(idx_id))
1725         {
1726             logf("delete_entry failed: %d", idx_id);
1727             return ;
1728         }
1729     }
1730
1731     fd = open(path, O_RDONLY);
1732     if (fd < 0)
1733     {
1734         logf("open fail: %s", path);
1735         return ;
1736     }
1737
1738     memset(&id3, 0, sizeof(struct mp3entry));
1739     memset(&entry, 0, sizeof(struct temp_file_entry));
1740     memset(&tracknumfix, 0, sizeof(tracknumfix));
1741     ret = get_metadata(&id3, fd, path);
1742     close(fd);
1743
1744     if (!ret)
1745         return ;
1746
1747     logf("-> %s", path);
1748
1749     /* Generate track number if missing. */
1750     if (id3.tracknum <= 0)
1751     {
1752         const char *p = strrchr(path, '.');
1753
1754         if (p == NULL)
1755             p = &path[strlen(path)-1];
1756
1757         while (*p != '/')
1758         {
1759             if (isdigit(*p) && isdigit(*(p-1)))
1760             {
1761                 tracknumfix[1] = *p--;
1762                 tracknumfix[0] = *p;
1763                 break;
1764             }
1765             p--;
1766         }
1767
1768         if (tracknumfix[0] != '\0')
1769         {
1770             id3.tracknum = atoi(tracknumfix);
1771             /* Set a flag to indicate track number has been generated. */
1772             entry.flag |= FLAG_TRKNUMGEN;
1773         }
1774         else
1775         {
1776             /* Unable to generate track number. */
1777             id3.tracknum = -1;
1778         }
1779     }
1780
1781     /* Numeric tags */
1782     entry.tag_offset[tag_year] = id3.year;
1783     entry.tag_offset[tag_discnumber] = id3.discnum;
1784     entry.tag_offset[tag_tracknumber] = id3.tracknum;
1785     entry.tag_offset[tag_length] = id3.length;
1786     entry.tag_offset[tag_bitrate] = id3.bitrate;
1787     entry.tag_offset[tag_mtime] = mtime;
1788
1789     /* String tags. */
1790     has_albumartist = id3.albumartist != NULL
1791         && strlen(id3.albumartist) > 0;
1792     has_grouping = id3.grouping != NULL
1793         && strlen(id3.grouping) > 0;
1794
1795     ADD_TAG(entry, tag_filename, &path);
1796     ADD_TAG(entry, tag_title, &id3.title);
1797     ADD_TAG(entry, tag_artist, &id3.artist);
1798     ADD_TAG(entry, tag_album, &id3.album);
1799     ADD_TAG(entry, tag_genre, &id3.genre_string);
1800     ADD_TAG(entry, tag_composer, &id3.composer);
1801     ADD_TAG(entry, tag_comment, &id3.comment);
1802     if (has_albumartist)
1803     {
1804         ADD_TAG(entry, tag_albumartist, &id3.albumartist);
1805     }
1806     else
1807     {
1808         ADD_TAG(entry, tag_albumartist, &id3.artist);
1809     }
1810     if (has_grouping)
1811     {
1812         ADD_TAG(entry, tag_grouping, &id3.grouping);
1813     }
1814     else
1815     {
1816         ADD_TAG(entry, tag_grouping, &id3.title);
1817     }
1818     entry.data_length = offset;
1819
1820     /* Write the header */
1821     write(cachefd, &entry, sizeof(struct temp_file_entry));
1822
1823     /* And tags also... Correct order is critical */
1824     write_item(path);
1825     write_item(id3.title);
1826     write_item(id3.artist);
1827     write_item(id3.album);
1828     write_item(id3.genre_string);
1829     write_item(id3.composer);
1830     write_item(id3.comment);
1831     if (has_albumartist)
1832     {
1833         write_item(id3.albumartist);
1834     }
1835     else
1836     {
1837         write_item(id3.artist);
1838     }
1839     if (has_grouping)
1840     {
1841         write_item(id3.grouping);
1842     }
1843     else
1844     {
1845         write_item(id3.title);
1846     }
1847     total_entry_count++;
1848 }
1849
1850 static bool tempbuf_insert(char *str, int id, int idx_id, bool unique)
1851 {
1852     struct tempbuf_searchidx *index = (struct tempbuf_searchidx *)tempbuf;
1853     int len = strlen(str)+1;
1854     int i;
1855     unsigned crc32;
1856     unsigned *crcbuf = (unsigned *)&tempbuf[tempbuf_size-4];
1857     char buf[TAG_MAXLEN+32];
1858
1859     for (i = 0; str[i] != '\0' && i < (int)sizeof(buf)-1; i++)
1860         buf[i] = tolower(str[i]);
1861     buf[i] = '\0';
1862
1863     crc32 = crc_32(buf, i, 0xffffffff);
1864
1865     if (unique)
1866     {
1867         /* Check if the crc does not exist -> entry does not exist for sure. */
1868         for (i = 0; i < tempbufidx; i++)
1869         {
1870             if (crcbuf[-i] != crc32)
1871                 continue;
1872
1873             if (!strcasecmp(str, index[i].str))
1874             {
1875                 if (id < 0 || id >= lookup_buffer_depth)
1876                 {
1877                     logf("lookup buf overf.: %d", id);
1878                     return false;
1879                 }
1880
1881                 lookup[id] = &index[i];
1882                 return true;
1883             }
1884         }
1885     }
1886
1887     /* Insert to CRC buffer. */
1888     crcbuf[-tempbufidx] = crc32;
1889     tempbuf_left -= 4;
1890
1891     /* Insert it to the buffer. */
1892     tempbuf_left -= len;
1893     if (tempbuf_left - 4 < 0 || tempbufidx >= commit_entry_count-1)
1894         return false;
1895
1896     if (id >= lookup_buffer_depth)
1897     {
1898         logf("lookup buf overf. #2: %d", id);
1899         return false;
1900     }
1901
1902     if (id >= 0)
1903     {
1904         lookup[id] = &index[tempbufidx];
1905         index[tempbufidx].idlist.id = id;
1906     }
1907     else
1908         index[tempbufidx].idlist.id = -1;
1909
1910     index[tempbufidx].idlist.next = NULL;
1911     index[tempbufidx].idx_id = idx_id;
1912     index[tempbufidx].seek = -1;
1913     index[tempbufidx].str = &tempbuf[tempbuf_pos];
1914     memcpy(index[tempbufidx].str, str, len);
1915     tempbuf_pos += len;
1916     tempbufidx++;
1917
1918     return true;
1919 }
1920
1921 static int compare(const void *p1, const void *p2)
1922 {
1923     do_timed_yield();
1924
1925     struct tempbuf_searchidx *e1 = (struct tempbuf_searchidx *)p1;
1926     struct tempbuf_searchidx *e2 = (struct tempbuf_searchidx *)p2;
1927
1928     if (strcmp(e1->str, UNTAGGED) == 0)
1929     {
1930         if (strcmp(e2->str, UNTAGGED) == 0)
1931             return 0;
1932         return -1;
1933     }
1934     else if (strcmp(e2->str, UNTAGGED) == 0)
1935         return 1;
1936
1937     return strncasecmp(e1->str, e2->str, TAG_MAXLEN);
1938 }
1939
1940 static int tempbuf_sort(int fd)
1941 {
1942     struct tempbuf_searchidx *index = (struct tempbuf_searchidx *)tempbuf;
1943     struct tagfile_entry fe;
1944     int i;
1945     int length;
1946
1947     /* Generate reverse lookup entries. */
1948     for (i = 0; i < lookup_buffer_depth; i++)
1949     {
1950         struct tempbuf_id_list *idlist;
1951
1952         if (!lookup[i])
1953             continue;
1954
1955         if (lookup[i]->idlist.id == i)
1956             continue;
1957
1958         idlist = &lookup[i]->idlist;
1959         while (idlist->next != NULL)
1960             idlist = idlist->next;
1961
1962         tempbuf_left -= sizeof(struct tempbuf_id_list);
1963         if (tempbuf_left - 4 < 0)
1964             return -1;
1965
1966         idlist->next = (struct tempbuf_id_list *)&tempbuf[tempbuf_pos];
1967         if (tempbuf_pos & 0x03)
1968         {
1969             tempbuf_pos = (tempbuf_pos & ~0x03) + 0x04;
1970             tempbuf_left -= 3;
1971             idlist->next = (struct tempbuf_id_list *)&tempbuf[tempbuf_pos];
1972         }
1973         tempbuf_pos += sizeof(struct tempbuf_id_list);
1974
1975         idlist = idlist->next;
1976         idlist->id = i;
1977         idlist->next = NULL;
1978
1979         do_timed_yield();
1980     }
1981
1982     qsort(index, tempbufidx, sizeof(struct tempbuf_searchidx), compare);
1983     memset(lookup, 0, lookup_buffer_depth * sizeof(struct tempbuf_searchidx **));
1984
1985     for (i = 0; i < tempbufidx; i++)
1986     {
1987         struct tempbuf_id_list *idlist = &index[i].idlist;
1988
1989         /* Fix the lookup list. */
1990         while (idlist != NULL)
1991         {
1992             if (idlist->id >= 0)
1993                 lookup[idlist->id] = &index[i];
1994             idlist = idlist->next;
1995         }
1996
1997         index[i].seek = lseek(fd, 0, SEEK_CUR);
1998         length = strlen(index[i].str) + 1;
1999         fe.tag_length = length;
2000         fe.idx_id = index[i].idx_id;
2001
2002         /* Check the chunk alignment. */
2003         if ((fe.tag_length + sizeof(struct tagfile_entry))
2004             % TAGFILE_ENTRY_CHUNK_LENGTH)
2005         {
2006             fe.tag_length += TAGFILE_ENTRY_CHUNK_LENGTH -
2007                 ((fe.tag_length + sizeof(struct tagfile_entry))
2008                  % TAGFILE_ENTRY_CHUNK_LENGTH);
2009         }
2010
2011 #ifdef TAGCACHE_STRICT_ALIGN
2012         /* Make sure the entry is long aligned. */
2013         if (index[i].seek & 0x03)
2014         {
2015             logf("tempbuf_sort: alignment error!");
2016             return -3;
2017         }
2018 #endif
2019
2020         if (ecwrite(fd, &fe, 1, tagfile_entry_ec, tc_stat.econ) !=
2021             sizeof(struct tagfile_entry))
2022         {
2023             logf("tempbuf_sort: write error #1");
2024             return -1;
2025         }
2026
2027         if (write(fd, index[i].str, length) != length)
2028         {
2029             logf("tempbuf_sort: write error #2");
2030             return -2;
2031         }
2032
2033         /* Write some padding. */
2034         if (fe.tag_length - length > 0)
2035             write(fd, "XXXXXXXX", fe.tag_length - length);
2036     }
2037
2038     return i;
2039 }
2040
2041 inline static struct tempbuf_searchidx* tempbuf_locate(int id)
2042 {
2043     if (id < 0 || id >= lookup_buffer_depth)
2044         return NULL;
2045
2046     return lookup[id];
2047 }
2048
2049
2050 inline static int tempbuf_find_location(int id)
2051 {
2052     struct tempbuf_searchidx *entry;
2053
2054     entry = tempbuf_locate(id);
2055     if (entry == NULL)
2056         return -1;
2057
2058     return entry->seek;
2059 }
2060
2061 static bool build_numeric_indices(struct tagcache_header *h, int tmpfd)
2062 {
2063     struct master_header tcmh;
2064     struct index_entry idx;
2065     int masterfd;
2066     int masterfd_pos;
2067     struct temp_file_entry *entrybuf = (struct temp_file_entry *)tempbuf;
2068     int max_entries;
2069     int entries_processed = 0;
2070     int i, j;
2071     char buf[TAG_MAXLEN];
2072
2073     max_entries = tempbuf_size / sizeof(struct temp_file_entry) - 1;
2074
2075     logf("Building numeric indices...");
2076     lseek(tmpfd, sizeof(struct tagcache_header), SEEK_SET);
2077
2078     if ( (masterfd = open_master_fd(&tcmh, true)) < 0)
2079         return false;
2080
2081     masterfd_pos = lseek(masterfd, tcmh.tch.entry_count * sizeof(struct index_entry),
2082                          SEEK_CUR);
2083     if (masterfd_pos == filesize(masterfd))
2084     {
2085         logf("we can't append!");
2086         close(masterfd);
2087         return false;
2088     }
2089
2090     while (entries_processed < h->entry_count)
2091     {
2092         int count = MIN(h->entry_count - entries_processed, max_entries);
2093
2094         /* Read in as many entries as possible. */
2095         for (i = 0; i < count; i++)
2096         {
2097             struct temp_file_entry *tfe = &entrybuf[i];
2098             int datastart;
2099
2100             /* Read in numeric data. */
2101             if (read(tmpfd, tfe, sizeof(struct temp_file_entry)) !=
2102                 sizeof(struct temp_file_entry))
2103             {
2104                 logf("read fail #1");
2105                 close(masterfd);
2106                 return false;
2107             }
2108
2109             datastart = lseek(tmpfd, 0, SEEK_CUR);
2110
2111             /**
2112              * Read string data from the following tags:
2113              * - tag_filename
2114              * - tag_artist
2115              * - tag_album
2116              * - tag_title
2117              *
2118              * A crc32 hash is calculated from the read data
2119              * and stored back to the data offset field kept in memory.
2120              */
2121 #define tmpdb_read_string_tag(tag) \
2122     lseek(tmpfd, tfe->tag_offset[tag], SEEK_CUR); \
2123     if ((unsigned long)tfe->tag_length[tag] > sizeof buf) \
2124     { \
2125         logf("read fail: buffer overflow"); \
2126         close(masterfd); \
2127         return false; \
2128     } \
2129     \
2130     if (read(tmpfd, buf, tfe->tag_length[tag]) != \
2131         tfe->tag_length[tag]) \
2132     { \
2133         logf("read fail #2"); \
2134         close(masterfd); \
2135         return false; \
2136     } \
2137     \
2138     tfe->tag_offset[tag] = crc_32(buf, strlen(buf), 0xffffffff); \
2139     lseek(tmpfd, datastart, SEEK_SET)
2140
2141             tmpdb_read_string_tag(tag_filename);
2142             tmpdb_read_string_tag(tag_artist);
2143             tmpdb_read_string_tag(tag_album);
2144             tmpdb_read_string_tag(tag_title);
2145
2146             /* Seek to the end of the string data. */
2147             lseek(tmpfd, tfe->data_length, SEEK_CUR);
2148         }
2149
2150         /* Backup the master index position. */
2151         masterfd_pos = lseek(masterfd, 0, SEEK_CUR);
2152         lseek(masterfd, sizeof(struct master_header), SEEK_SET);
2153
2154         /* Check if we can resurrect some deleted runtime statistics data. */
2155         for (i = 0; i < tcmh.tch.entry_count; i++)
2156         {
2157             /* Read the index entry. */
2158             if (ecread(masterfd, &idx, 1, index_entry_ec, tc_stat.econ)
2159                 != sizeof(struct index_entry))
2160             {
2161                 logf("read fail #3");
2162                 close(masterfd);
2163                 return false;
2164             }
2165
2166             /**
2167              * Skip unless the entry is marked as being deleted
2168              * or the data has already been resurrected.
2169              */
2170             if (!(idx.flag & FLAG_DELETED) || idx.flag & FLAG_RESURRECTED)
2171                 continue;
2172
2173             /* Now try to match the entry. */
2174             /**
2175              * To succesfully match a song, the following conditions
2176              * must apply:
2177              *
2178              * For numeric fields: tag_length
2179              * - Full identical match is required
2180              *
2181              * If tag_filename matches, no further checking necessary.
2182              *
2183              * For string hashes: tag_artist, tag_album, tag_title
2184              * - Two of these must match
2185              */
2186             for (j = 0; j < count; j++)
2187             {
2188                 struct temp_file_entry *tfe = &entrybuf[j];
2189
2190                 /* Try to match numeric fields first. */
2191                 if (tfe->tag_offset[tag_length] != idx.tag_seek[tag_length])
2192                     continue;
2193
2194                 /* Now it's time to do the hash matching. */
2195                 if (tfe->tag_offset[tag_filename] != idx.tag_seek[tag_filename])
2196                 {
2197                     int match_count = 0;
2198
2199                     /* No filename match, check if we can match two other tags. */
2200 #define tmpdb_match(tag) \
2201     if (tfe->tag_offset[tag] == idx.tag_seek[tag]) \
2202         match_count++
2203
2204                     tmpdb_match(tag_artist);
2205                     tmpdb_match(tag_album);
2206                     tmpdb_match(tag_title);
2207
2208                     if (match_count < 2)
2209                     {
2210                         /* Still no match found, give up. */
2211                         continue;
2212                     }
2213                 }
2214
2215                 /* A match found, now copy & resurrect the statistical data. */
2216 #define tmpdb_copy_tag(tag) \
2217     tfe->tag_offset[tag] = idx.tag_seek[tag]
2218
2219                 tmpdb_copy_tag(tag_playcount);
2220                 tmpdb_copy_tag(tag_rating);
2221                 tmpdb_copy_tag(tag_playtime);
2222                 tmpdb_copy_tag(tag_lastplayed);
2223                 tmpdb_copy_tag(tag_commitid);
2224
2225                 /* Avoid processing this entry again. */
2226                 idx.flag |= FLAG_RESURRECTED;
2227
2228                 lseek(masterfd, -sizeof(struct index_entry), SEEK_CUR);
2229                 if (ecwrite(masterfd, &idx, 1, index_entry_ec, tc_stat.econ)
2230                     != sizeof(struct index_entry))
2231                 {
2232                     logf("masterfd writeback fail #1");
2233                     close(masterfd);
2234                     return false;
2235                 }
2236
2237                 logf("Entry resurrected");
2238             }
2239         }
2240
2241
2242         /* Restore the master index position. */
2243         lseek(masterfd, masterfd_pos, SEEK_SET);
2244
2245         /* Commit the data to the index. */
2246         for (i = 0; i < count; i++)
2247         {
2248             int loc = lseek(masterfd, 0, SEEK_CUR);
2249
2250             if (ecread(masterfd, &idx, 1, index_entry_ec, tc_stat.econ)
2251                 != sizeof(struct index_entry))
2252             {
2253                 logf("read fail #3");
2254                 close(masterfd);
2255                 return false;
2256             }
2257
2258             for (j = 0; j < TAG_COUNT; j++)
2259             {
2260                 if (!tagcache_is_numeric_tag(j))
2261                     continue;
2262
2263                 idx.tag_seek[j] = entrybuf[i].tag_offset[j];
2264             }
2265             idx.flag = entrybuf[i].flag;
2266
2267             if (idx.tag_seek[tag_commitid])
2268             {
2269                 /* Data has been resurrected. */
2270                 idx.flag |= FLAG_DIRTYNUM;
2271             }
2272             else if (tc_stat.ready && current_tcmh.commitid > 0)
2273             {
2274                 idx.tag_seek[tag_commitid] = current_tcmh.commitid;
2275                 idx.flag |= FLAG_DIRTYNUM;
2276             }
2277
2278             /* Write back the updated index. */
2279             lseek(masterfd, loc, SEEK_SET);
2280             if (ecwrite(masterfd, &idx, 1, index_entry_ec, tc_stat.econ)
2281                 != sizeof(struct index_entry))
2282             {
2283                 logf("write fail");
2284                 close(masterfd);
2285                 return false;
2286             }
2287         }
2288
2289         entries_processed += count;
2290         logf("%d/%ld entries processed", entries_processed, h->entry_count);
2291     }
2292
2293     close(masterfd);
2294
2295     return true;
2296 }
2297
2298 /**
2299  * Return values:
2300  *     > 0   success
2301  *    == 0   temporary failure
2302  *     < 0   fatal error
2303  */
2304 static int build_index(int index_type, struct tagcache_header *h, int tmpfd)
2305 {
2306     int i;
2307     struct tagcache_header tch;
2308     struct master_header   tcmh;
2309     struct index_entry idxbuf[IDX_BUF_DEPTH];
2310     int idxbuf_pos;
2311     char buf[TAG_MAXLEN+32];
2312     int fd = -1, masterfd;
2313     bool error = false;
2314     int init;
2315     int masterfd_pos;
2316
2317     logf("Building index: %d", index_type);
2318
2319     /* Check the number of entries we need to allocate ram for. */
2320     commit_entry_count = h->entry_count + 1;
2321
2322     masterfd = open_master_fd(&tcmh, false);
2323     if (masterfd >= 0)
2324     {
2325         commit_entry_count += tcmh.tch.entry_count;
2326         close(masterfd);
2327     }
2328     else
2329         remove_files(); /* Just to be sure we are clean. */
2330
2331     /* Open the index file, which contains the tag names. */
2332     fd = open_tag_fd(&tch, index_type, true);
2333     if (fd >= 0)
2334     {
2335         logf("tch.datasize=%ld", tch.datasize);
2336         lookup_buffer_depth = 1 +
2337         /* First part */ commit_entry_count +
2338         /* Second part */ (tch.datasize / TAGFILE_ENTRY_CHUNK_LENGTH);
2339     }
2340     else
2341     {
2342         lookup_buffer_depth = 1 +
2343         /* First part */ commit_entry_count +
2344         /* Second part */ 0;
2345     }
2346
2347     logf("lookup_buffer_depth=%ld", lookup_buffer_depth);
2348     logf("commit_entry_count=%ld", commit_entry_count);
2349
2350     /* Allocate buffer for all index entries from both old and new
2351      * tag files. */
2352     tempbufidx = 0;
2353     tempbuf_pos = commit_entry_count * sizeof(struct tempbuf_searchidx);
2354
2355     /* Allocate lookup buffer. The first portion of commit_entry_count
2356      * contains the new tags in the temporary file and the second
2357      * part for locating entries already in the db.
2358      *
2359      *  New tags  Old tags
2360      * +---------+---------------------------+
2361      * |  index  | position/ENTRY_CHUNK_SIZE |  lookup buffer
2362      * +---------+---------------------------+
2363      *
2364      * Old tags are inserted to a temporary buffer with position:
2365      *     tempbuf_insert(position/ENTRY_CHUNK_SIZE, ...);
2366      * And new tags with index:
2367      *     tempbuf_insert(idx, ...);
2368      *
2369      * The buffer is sorted and written into tag file:
2370      *     tempbuf_sort(...);
2371      * leaving master index locations messed up.
2372      *
2373      * That is fixed using the lookup buffer for old tags:
2374      *     new_seek = tempbuf_find_location(old_seek, ...);
2375      * and for new tags:
2376      *     new_seek = tempbuf_find_location(idx);
2377      */
2378     lookup = (struct tempbuf_searchidx **)&tempbuf[tempbuf_pos];
2379     tempbuf_pos += lookup_buffer_depth * sizeof(void **);
2380     memset(lookup, 0, lookup_buffer_depth * sizeof(void **));
2381
2382     /* And calculate the remaining data space used mainly for storing
2383      * tag data (strings). */
2384     tempbuf_left = tempbuf_size - tempbuf_pos - 8;
2385     if (tempbuf_left - TAGFILE_ENTRY_AVG_LENGTH * commit_entry_count < 0)
2386     {
2387         logf("Buffer way too small!");
2388         return 0;
2389     }
2390
2391     if (fd >= 0)
2392     {
2393         /**
2394          * If tag file contains unique tags (sorted index), we will load
2395          * it entirely into memory so we can resort it later for use with
2396          * chunked browsing.
2397          */
2398         if (tagcache_is_sorted_tag(index_type))
2399         {
2400             logf("loading tags...");
2401             for (i = 0; i < tch.entry_count; i++)
2402             {
2403                 struct tagfile_entry entry;
2404                 int loc = lseek(fd, 0, SEEK_CUR);
2405                 bool ret;
2406
2407                 if (ecread(fd, &entry, 1, tagfile_entry_ec, tc_stat.econ)
2408                     != sizeof(struct tagfile_entry))
2409                 {
2410                     logf("read error #7");
2411                     close(fd);
2412                     return -2;
2413                 }
2414
2415                 if (entry.tag_length >= (int)sizeof(buf))
2416                 {
2417                     logf("too long tag #3");
2418                     close(fd);
2419                     return -2;
2420                 }
2421
2422                 if (read(fd, buf, entry.tag_length) != entry.tag_length)
2423                 {
2424                     logf("read error #8");
2425                     close(fd);
2426                     return -2;
2427                 }
2428
2429                 /* Skip deleted entries. */
2430                 if (buf[0] == '\0')
2431                     continue;
2432
2433                 /**
2434                  * Save the tag and tag id in the memory buffer. Tag id
2435                  * is saved so we can later reindex the master lookup
2436                  * table when the index gets resorted.
2437                  */
2438                 ret = tempbuf_insert(buf, loc/TAGFILE_ENTRY_CHUNK_LENGTH
2439                                      + commit_entry_count, entry.idx_id,
2440                                      tagcache_is_unique_tag(index_type));
2441                 if (!ret)
2442                 {
2443                     close(fd);
2444                     return -3;
2445                 }
2446                 do_timed_yield();
2447             }
2448             logf("done");
2449         }
2450         else
2451             tempbufidx = tch.entry_count;
2452     }
2453     else
2454     {
2455         /**
2456          * Creating new index file to store the tags. No need to preload
2457          * anything whether the index type is sorted or not.
2458          */
2459         snprintf(buf, sizeof buf, TAGCACHE_FILE_INDEX, index_type);
2460         fd = open(buf, O_WRONLY | O_CREAT | O_TRUNC);
2461         if (fd < 0)
2462         {
2463             logf("%s open fail", buf);
2464             return -2;
2465         }
2466
2467         tch.magic = TAGCACHE_MAGIC;
2468         tch.entry_count = 0;
2469         tch.datasize = 0;
2470
2471         if (ecwrite(fd, &tch, 1, tagcache_header_ec, tc_stat.econ)
2472             != sizeof(struct tagcache_header))
2473         {
2474             logf("header write failed");
2475             close(fd);
2476             return -2;
2477         }
2478     }
2479
2480     /* Loading the tag lookup file as "master file". */
2481     logf("Loading index file");
2482     masterfd = open(TAGCACHE_FILE_MASTER, O_RDWR);
2483
2484     if (masterfd < 0)
2485     {
2486         logf("Creating new DB");
2487         masterfd = open(TAGCACHE_FILE_MASTER, O_WRONLY | O_CREAT | O_TRUNC);
2488
2489         if (masterfd < 0)
2490         {
2491             logf("Failure to create index file (%s)", TAGCACHE_FILE_MASTER);
2492             close(fd);
2493             return -2;
2494         }
2495
2496         /* Write the header (write real values later). */
2497         memset(&tcmh, 0, sizeof(struct master_header));
2498         tcmh.tch = *h;
2499         tcmh.tch.entry_count = 0;
2500         tcmh.tch.datasize = 0;
2501         tcmh.dirty = true;
2502         ecwrite(masterfd, &tcmh, 1, master_header_ec, tc_stat.econ);
2503         init = true;
2504         masterfd_pos = lseek(masterfd, 0, SEEK_CUR);
2505     }
2506     else
2507     {
2508         /**
2509          * Master file already exists so we need to process the current
2510          * file first.
2511          */
2512         init = false;
2513
2514         if (ecread(masterfd, &tcmh, 1, master_header_ec, tc_stat.econ) !=
2515             sizeof(struct master_header) || tcmh.tch.magic != TAGCACHE_MAGIC)
2516         {
2517             logf("header error");
2518             close(fd);
2519             close(masterfd);
2520             return -2;
2521         }
2522
2523         /**
2524          * If we reach end of the master file, we need to expand it to
2525          * hold new tags. If the current index is not sorted, we can
2526          * simply append new data to end of the file.
2527          * However, if the index is sorted, we need to update all tag
2528          * pointers in the master file for the current index.
2529          */
2530         masterfd_pos = lseek(masterfd, tcmh.tch.entry_count * sizeof(struct index_entry),
2531             SEEK_CUR);
2532         if (masterfd_pos == filesize(masterfd))
2533         {
2534             logf("appending...");
2535             init = true;
2536         }
2537     }
2538
2539     /**
2540      * Load new unique tags in memory to be sorted later and added
2541      * to the master lookup file.
2542      */
2543     if (tagcache_is_sorted_tag(index_type))
2544     {
2545         lseek(tmpfd, sizeof(struct tagcache_header), SEEK_SET);
2546         /* h is the header of the temporary file containing new tags. */
2547         logf("inserting new tags...");
2548         for (i = 0; i < h->entry_count; i++)
2549         {
2550             struct temp_file_entry entry;
2551
2552             if (read(tmpfd, &entry, sizeof(struct temp_file_entry)) !=
2553                 sizeof(struct temp_file_entry))
2554             {
2555                 logf("read fail #3");
2556                 error = true;
2557                 goto error_exit;
2558             }
2559
2560             /* Read data. */
2561             if (entry.tag_length[index_type] >= (long)sizeof(buf))
2562             {
2563                 logf("too long entry!");
2564                 error = true;
2565                 goto error_exit;
2566             }
2567
2568             lseek(tmpfd, entry.tag_offset[index_type], SEEK_CUR);
2569             if (read(tmpfd, buf, entry.tag_length[index_type]) !=
2570                 entry.tag_length[index_type])
2571             {
2572                 logf("read fail #4");
2573                 error = true;
2574                 goto error_exit;
2575             }
2576
2577             if (tagcache_is_unique_tag(index_type))
2578                 error = !tempbuf_insert(buf, i, -1, true);
2579             else
2580                 error = !tempbuf_insert(buf, i, tcmh.tch.entry_count + i, false);
2581
2582             if (error)
2583             {
2584                 logf("insert error");
2585                 goto error_exit;
2586             }
2587
2588             /* Skip to next. */
2589             lseek(tmpfd, entry.data_length - entry.tag_offset[index_type] -
2590                     entry.tag_length[index_type], SEEK_CUR);
2591             do_timed_yield();
2592         }
2593         logf("done");
2594
2595         /* Sort the buffer data and write it to the index file. */
2596         lseek(fd, sizeof(struct tagcache_header), SEEK_SET);
2597         i = tempbuf_sort(fd);
2598         if (i < 0)
2599             goto error_exit;
2600         logf("sorted %d tags", i);
2601
2602         /**
2603          * Now update all indexes in the master lookup file.
2604          */
2605         logf("updating indices...");
2606         lseek(masterfd, sizeof(struct master_header), SEEK_SET);
2607         for (i = 0; i < tcmh.tch.entry_count; i += idxbuf_pos)
2608         {
2609             int j;
2610             int loc = lseek(masterfd, 0, SEEK_CUR);
2611
2612             idxbuf_pos = MIN(tcmh.tch.entry_count - i, IDX_BUF_DEPTH);
2613
2614             if (ecread(masterfd, idxbuf, idxbuf_pos, index_entry_ec, tc_stat.econ)
2615                 != (int)sizeof(struct index_entry)*idxbuf_pos)
2616             {
2617                 logf("read fail #5");
2618                 error = true;
2619                 goto error_exit ;
2620             }
2621             lseek(masterfd, loc, SEEK_SET);
2622
2623             for (j = 0; j < idxbuf_pos; j++)
2624             {
2625                 if (idxbuf[j].flag & FLAG_DELETED)
2626                 {
2627                     /* We can just ignore deleted entries. */
2628                     // idxbuf[j].tag_seek[index_type] = 0;
2629                     continue;
2630                 }
2631
2632                 idxbuf[j].tag_seek[index_type] = tempbuf_find_location(
2633                     idxbuf[j].tag_seek[index_type]/TAGFILE_ENTRY_CHUNK_LENGTH
2634                     + commit_entry_count);
2635
2636                 if (idxbuf[j].tag_seek[index_type] < 0)
2637                 {
2638                     logf("update error: %d/%d/%d",
2639                          idxbuf[j].flag, i+j, tcmh.tch.entry_count);
2640                     error = true;
2641                     goto error_exit;
2642                 }
2643
2644                 do_timed_yield();
2645             }
2646
2647             /* Write back the updated index. */
2648             if (ecwrite(masterfd, idxbuf, idxbuf_pos,
2649                         index_entry_ec, tc_stat.econ) !=
2650                 (int)sizeof(struct index_entry)*idxbuf_pos)
2651             {
2652                 logf("write fail");
2653                 error = true;
2654                 goto error_exit;
2655             }
2656         }
2657         logf("done");
2658     }
2659
2660     /**
2661      * Walk through the temporary file containing the new tags.
2662      */
2663     // build_normal_index(h, tmpfd, masterfd, idx);
2664     logf("updating new indices...");
2665     lseek(masterfd, masterfd_pos, SEEK_SET);
2666     lseek(tmpfd, sizeof(struct tagcache_header), SEEK_SET);
2667     lseek(fd, 0, SEEK_END);
2668     for (i = 0; i < h->entry_count; i += idxbuf_pos)
2669     {
2670         int j;
2671
2672         idxbuf_pos = MIN(h->entry_count - i, IDX_BUF_DEPTH);
2673         if (init)
2674         {
2675             memset(idxbuf, 0, sizeof(struct index_entry)*IDX_BUF_DEPTH);
2676         }
2677         else
2678         {
2679             int loc = lseek(masterfd, 0, SEEK_CUR);
2680
2681             if (ecread(masterfd, idxbuf, idxbuf_pos, index_entry_ec, tc_stat.econ)
2682                 != (int)sizeof(struct index_entry)*idxbuf_pos)
2683             {
2684                 logf("read fail #6");
2685                 error = true;
2686                 break ;
2687             }
2688             lseek(masterfd, loc, SEEK_SET);
2689         }
2690
2691         /* Read entry headers. */
2692         for (j = 0; j < idxbuf_pos; j++)
2693         {
2694             if (!tagcache_is_sorted_tag(index_type))
2695             {
2696                 struct temp_file_entry entry;
2697                 struct tagfile_entry fe;
2698
2699                 if (read(tmpfd, &entry, sizeof(struct temp_file_entry)) !=
2700                     sizeof(struct temp_file_entry))
2701                 {
2702                     logf("read fail #7");
2703                     error = true;
2704                     break ;
2705                 }
2706
2707                 /* Read data. */
2708                 if (entry.tag_length[index_type] >= (int)sizeof(buf))
2709                 {
2710                     logf("too long entry!");
2711                     logf("length=%d", entry.tag_length[index_type]);
2712                     logf("pos=0x%02lx", lseek(tmpfd, 0, SEEK_CUR));
2713                     error = true;
2714                     break ;
2715                 }
2716
2717                 lseek(tmpfd, entry.tag_offset[index_type], SEEK_CUR);
2718                 if (read(tmpfd, buf, entry.tag_length[index_type]) !=
2719                     entry.tag_length[index_type])
2720                 {
2721                     logf("read fail #8");
2722                     logf("offset=0x%02lx", entry.tag_offset[index_type]);
2723                     logf("length=0x%02x", entry.tag_length[index_type]);
2724                     error = true;
2725                     break ;
2726                 }
2727
2728                 /* Write to index file. */
2729                 idxbuf[j].tag_seek[index_type] = lseek(fd, 0, SEEK_CUR);
2730                 fe.tag_length = entry.tag_length[index_type];
2731                 fe.idx_id = tcmh.tch.entry_count + i + j;
2732                 ecwrite(fd, &fe, 1, tagfile_entry_ec, tc_stat.econ);
2733                 write(fd, buf, fe.tag_length);
2734                 tempbufidx++;
2735
2736                 /* Skip to next. */
2737                 lseek(tmpfd, entry.data_length - entry.tag_offset[index_type] -
2738                       entry.tag_length[index_type], SEEK_CUR);
2739             }
2740             else
2741             {
2742                 /* Locate the correct entry from the sorted array. */
2743                 idxbuf[j].tag_seek[index_type] = tempbuf_find_location(i + j);
2744                 if (idxbuf[j].tag_seek[index_type] < 0)
2745                 {
2746                     logf("entry not found (%d)", j);
2747                     error = true;
2748                     break ;
2749                 }
2750             }
2751         }
2752
2753         /* Write index. */
2754         if (ecwrite(masterfd, idxbuf, idxbuf_pos,
2755                   index_entry_ec, tc_stat.econ) !=
2756             (int)sizeof(struct index_entry)*idxbuf_pos)
2757         {
2758             logf("tagcache: write fail #4");
2759             error = true;
2760             break ;
2761         }
2762
2763         do_timed_yield();
2764     }
2765     logf("done");
2766
2767     /* Finally write the header. */
2768     tch.magic = TAGCACHE_MAGIC;
2769     tch.entry_count = tempbufidx;
2770     tch.datasize = lseek(fd, 0, SEEK_END) - sizeof(struct tagcache_header);
2771     lseek(fd, 0, SEEK_SET);
2772     ecwrite(fd, &tch, 1, tagcache_header_ec, tc_stat.econ);
2773
2774     if (index_type != tag_filename)
2775         h->datasize += tch.datasize;
2776     logf("s:%d/%ld/%ld", index_type, tch.datasize, h->datasize);
2777     error_exit:
2778
2779     close(fd);
2780     close(masterfd);
2781
2782     if (error)
2783         return -2;
2784
2785     return 1;
2786 }
2787
2788 static bool commit(void)
2789 {
2790     struct tagcache_header tch;
2791     struct master_header   tcmh;
2792     int i, len, rc;
2793     int tmpfd;
2794     int masterfd;
2795 #ifdef HAVE_DIRCACHE
2796     bool dircache_buffer_stolen = false;
2797 #endif
2798     bool local_allocation = false;
2799
2800     logf("committing tagcache");
2801
2802     while (write_lock)
2803         sleep(1);
2804
2805     tmpfd = open(TAGCACHE_FILE_TEMP, O_RDONLY);
2806     if (tmpfd < 0)
2807     {
2808         logf("nothing to commit");
2809         return true;
2810     }
2811
2812
2813     /* Load the header. */
2814     len = sizeof(struct tagcache_header);
2815     rc = read(tmpfd, &tch, len);
2816
2817     if (tch.magic != TAGCACHE_MAGIC || rc != len)
2818     {
2819         logf("incorrect header");
2820         close(tmpfd);
2821         remove(TAGCACHE_FILE_TEMP);
2822         return false;
2823     }
2824
2825     if (tch.entry_count == 0)
2826     {
2827         logf("nothing to commit");
2828         close(tmpfd);
2829         remove(TAGCACHE_FILE_TEMP);
2830         return true;
2831     }
2832
2833 #ifdef HAVE_EEPROM_SETTINGS
2834     remove(TAGCACHE_STATEFILE);
2835 #endif
2836
2837     /* At first be sure to unload the ramcache! */
2838 #ifdef HAVE_TC_RAMCACHE
2839     tc_stat.ramcache = false;
2840 #endif
2841
2842     read_lock++;
2843
2844     /* Try to steal every buffer we can :) */
2845     if (tempbuf_size == 0)
2846         local_allocation = true;
2847
2848 #ifdef HAVE_DIRCACHE
2849     if (tempbuf_size == 0)
2850     {
2851         /* Try to steal the dircache buffer. */
2852         tempbuf = dircache_steal_buffer(&tempbuf_size);
2853         tempbuf_size &= ~0x03;
2854
2855         if (tempbuf_size > 0)
2856         {
2857             dircache_buffer_stolen = true;
2858         }
2859     }
2860 #endif
2861
2862 #ifdef HAVE_TC_RAMCACHE
2863     if (tempbuf_size == 0 && tc_stat.ramcache_allocated > 0)
2864     {
2865         tempbuf = (char *)(hdr + 1);
2866         tempbuf_size = tc_stat.ramcache_allocated - sizeof(struct ramcache_header) - 128;
2867         tempbuf_size &= ~0x03;
2868     }
2869 #endif
2870
2871     /* And finally fail if there are no buffers available. */
2872     if (tempbuf_size == 0)
2873     {
2874         logf("delaying commit until next boot");
2875         tc_stat.commit_delayed = true;
2876         close(tmpfd);
2877         read_lock--;
2878         return false;
2879     }
2880
2881     logf("commit %ld entries...", tch.entry_count);
2882
2883     /* Mark DB dirty so it will stay disabled if commit fails. */
2884     current_tcmh.dirty = true;
2885     update_master_header();
2886
2887     /* Now create the index files. */
2888     tc_stat.commit_step = 0;
2889     tch.datasize = 0;
2890     tc_stat.commit_delayed = false;
2891
2892     for (i = 0; i < TAG_COUNT; i++)
2893     {
2894         int ret;
2895
2896         if (tagcache_is_numeric_tag(i))
2897             continue;
2898
2899         tc_stat.commit_step++;
2900         ret = build_index(i, &tch, tmpfd);
2901         if (ret <= 0)
2902         {
2903             close(tmpfd);
2904             logf("tagcache failed init");
2905             if (ret < 0)
2906                 remove_files();
2907             else
2908                 tc_stat.commit_delayed = true;
2909             tc_stat.commit_step = 0;
2910             read_lock--;
2911             return false;
2912         }
2913     }
2914
2915     if (!build_numeric_indices(&tch, tmpfd))
2916     {
2917         logf("Failure to commit numeric indices");
2918         close(tmpfd);
2919         remove_files();
2920         tc_stat.commit_step = 0;
2921         read_lock--;
2922         return false;
2923     }
2924
2925     close(tmpfd);
2926     tc_stat.commit_step = 0;
2927
2928     /* Update the master index headers. */
2929     if ( (masterfd = open_master_fd(&tcmh, true)) < 0)
2930     {
2931         read_lock--;
2932         return false;
2933     }
2934
2935     tcmh.tch.entry_count += tch.entry_count;
2936     tcmh.tch.datasize = sizeof(struct master_header)
2937         + sizeof(struct index_entry) * tcmh.tch.entry_count
2938         + tch.datasize;
2939     tcmh.dirty = false;
2940     tcmh.commitid++;
2941
2942     lseek(masterfd, 0, SEEK_SET);
2943     ecwrite(masterfd, &tcmh, 1, master_header_ec, tc_stat.econ);
2944     close(masterfd);
2945
2946     logf("tagcache committed");
2947     remove(TAGCACHE_FILE_TEMP);
2948     tc_stat.ready = check_all_headers();
2949     tc_stat.readyvalid = true;
2950
2951     if (local_allocation)
2952     {
2953         tempbuf = NULL;
2954         tempbuf_size = 0;
2955     }
2956
2957 #ifdef HAVE_DIRCACHE
2958     /* Rebuild the dircache, if we stole the buffer. */
2959     if (dircache_buffer_stolen)
2960         dircache_build(0);
2961 #endif
2962
2963 #ifdef HAVE_TC_RAMCACHE
2964     /* Reload tagcache. */
2965     if (tc_stat.ramcache_allocated > 0)
2966         tagcache_start_scan();
2967 #endif
2968
2969     read_lock--;
2970
2971     return true;
2972 }
2973
2974 static void allocate_tempbuf(void)
2975 {
2976     /* Yeah, malloc would be really nice now :) */
2977 #ifdef __PCTOOL__
2978     tempbuf_size = 32*1024*1024;
2979     tempbuf = malloc(tempbuf_size);
2980 #else
2981     tempbuf = (char *)(((long)audiobuf & ~0x03) + 0x04);
2982     tempbuf_size = (long)audiobufend - (long)audiobuf - 4;
2983     audiobuf += tempbuf_size;
2984 #endif
2985 }
2986
2987 static void free_tempbuf(void)
2988 {
2989     if (tempbuf_size == 0)
2990         return ;
2991
2992 #ifdef __PCTOOL__
2993     free(tempbuf);
2994 #else
2995     audiobuf -= tempbuf_size;
2996 #endif
2997     tempbuf = NULL;
2998     tempbuf_size = 0;
2999 }
3000
3001 #ifndef __PCTOOL__
3002
3003 static bool modify_numeric_entry(int masterfd, int idx_id, int tag, long data)
3004 {
3005     struct index_entry idx;
3006
3007     if (!tc_stat.ready)
3008         return false;
3009
3010     if (!tagcache_is_numeric_tag(tag))
3011         return false;
3012
3013     if (!get_index(masterfd, idx_id, &idx, false))
3014         return false;
3015
3016     idx.tag_seek[tag] = data;
3017     idx.flag |= FLAG_DIRTYNUM;
3018
3019     return write_index(masterfd, idx_id, &idx);
3020 }
3021
3022 #if 0
3023 bool tagcache_modify_numeric_entry(struct tagcache_search *tcs,
3024                                    int tag, long data)
3025 {
3026     struct master_header myhdr;
3027
3028     if (tcs->masterfd < 0)
3029     {
3030         if ( (tcs->masterfd = open_master_fd(&myhdr, true)) < 0)
3031             return false;
3032     }
3033
3034     return modify_numeric_entry(tcs->masterfd, tcs->idx_id, tag, data);
3035 }
3036 #endif
3037
3038 #define COMMAND_QUEUE_IS_EMPTY (command_queue_ridx == command_queue_widx)
3039
3040 static bool command_queue_is_full(void)
3041 {
3042     int next;
3043
3044     next = command_queue_widx + 1;
3045     if (next >= TAGCACHE_COMMAND_QUEUE_LENGTH)
3046         next = 0;
3047
3048     return (next == command_queue_ridx);
3049 }
3050
3051 static bool command_queue_sync_callback(void)
3052 {
3053
3054     struct master_header myhdr;
3055     int masterfd;
3056
3057     mutex_lock(&command_queue_mutex);
3058
3059     if ( (masterfd = open_master_fd(&myhdr, true)) < 0)
3060         return false;
3061
3062     while (command_queue_ridx != command_queue_widx)
3063     {
3064         struct tagcache_command_entry *ce = &command_queue[command_queue_ridx];
3065
3066         switch (ce->command)
3067         {
3068             case CMD_UPDATE_MASTER_HEADER:
3069             {
3070                 close(masterfd);
3071                 update_master_header();
3072
3073                 /* Re-open the masterfd. */
3074                 if ( (masterfd = open_master_fd(&myhdr, true)) < 0)
3075                     return true;
3076
3077                 break;
3078             }
3079             case CMD_UPDATE_NUMERIC:
3080             {
3081                 modify_numeric_entry(masterfd, ce->idx_id, ce->tag, ce->data);
3082                 break;
3083             }
3084         }
3085
3086         if (++command_queue_ridx >= TAGCACHE_COMMAND_QUEUE_LENGTH)
3087             command_queue_ridx = 0;
3088     }
3089
3090     close(masterfd);
3091
3092     tc_stat.queue_length = 0;
3093     mutex_unlock(&command_queue_mutex);
3094     return true;
3095 }
3096
3097 static void run_command_queue(bool force)
3098 {
3099     if (COMMAND_QUEUE_IS_EMPTY)
3100         return;
3101
3102     if (force || command_queue_is_full())
3103         command_queue_sync_callback();
3104     else
3105         register_storage_idle_func(command_queue_sync_callback);
3106 }
3107
3108 static void queue_command(int cmd, long idx_id, int tag, long data)
3109 {
3110     while (1)
3111     {
3112         int next;
3113
3114         mutex_lock(&command_queue_mutex);
3115         next = command_queue_widx + 1;
3116         if (next >= TAGCACHE_COMMAND_QUEUE_LENGTH)
3117             next = 0;
3118
3119         /* Make sure queue is not full. */
3120         if (next != command_queue_ridx)
3121         {
3122             struct tagcache_command_entry *ce = &command_queue[command_queue_widx];
3123
3124             ce->command = cmd;
3125             ce->idx_id = idx_id;
3126             ce->tag = tag;
3127             ce->data = data;
3128
3129             command_queue_widx = next;
3130
3131             tc_stat.queue_length++;
3132
3133             mutex_unlock(&command_queue_mutex);
3134             break;
3135         }
3136
3137         /* Queue is full, try again later... */
3138         mutex_unlock(&command_queue_mutex);
3139         sleep(1);
3140     }
3141 }
3142
3143 long tagcache_increase_serial(void)
3144 {
3145     long old;
3146
3147     if (!tc_stat.ready)
3148         return -2;
3149
3150     while (read_lock)
3151         sleep(1);
3152
3153     old = current_tcmh.serial++;
3154     queue_command(CMD_UPDATE_MASTER_HEADER, 0, 0, 0);
3155
3156     return old;
3157 }
3158
3159 void tagcache_update_numeric(int idx_id, int tag, long data)
3160 {
3161     queue_command(CMD_UPDATE_NUMERIC, idx_id, tag, data);
3162 }
3163 #endif /* !__PCTOOL__ */
3164
3165 long tagcache_get_serial(void)
3166 {
3167     return current_tcmh.serial;
3168 }
3169
3170 long tagcache_get_commitid(void)
3171 {
3172     return current_tcmh.commitid;
3173 }
3174
3175 static bool write_tag(int fd, const char *tagstr, const char *datastr)
3176 {
3177     char buf[512];
3178     int i;
3179
3180     snprintf(buf, sizeof buf, "%s=\"", tagstr);
3181     for (i = strlen(buf); i < (long)sizeof(buf)-4; i++)
3182     {
3183         if (*datastr == '\0')
3184             break;
3185
3186         if (*datastr == '"' || *datastr == '\\')
3187             buf[i++] = '\\';
3188
3189         buf[i] = *(datastr++);
3190     }
3191
3192     strcpy(&buf[i], "\" ");
3193
3194     write(fd, buf, i + 2);
3195
3196     return true;
3197 }
3198
3199 #ifndef __PCTOOL__
3200
3201 static bool read_tag(char *dest, long size,
3202                      const char *src, const char *tagstr)
3203 {
3204     int pos;
3205     char current_tag[32];
3206
3207     while (*src != '\0')
3208     {
3209         /* Skip all whitespace */
3210         while (*src == ' ')
3211             src++;
3212
3213         if (*src == '\0')
3214             break;
3215
3216         pos = 0;
3217         /* Read in tag name */
3218         while (*src != '=' && *src != ' ')
3219         {
3220             current_tag[pos] = *src;
3221             src++;
3222             pos++;
3223
3224             if (*src == '\0' || pos >= (long)sizeof(current_tag))
3225                 return false;
3226         }
3227         current_tag[pos] = '\0';
3228
3229         /* Read in tag data */
3230
3231         /* Find the start. */
3232         while (*src != '"' && *src != '\0')
3233             src++;
3234
3235         if (*src == '\0' || *(++src) == '\0')
3236             return false;
3237
3238         /* Read the data. */
3239         for (pos = 0; pos < size; pos++)
3240         {
3241             if (*src == '\0')
3242                 break;
3243
3244             if (*src == '\\')
3245             {
3246                 dest[pos] = *(src+1);
3247                 src += 2;
3248                 continue;
3249             }
3250
3251             dest[pos] = *src;
3252
3253             if (*src == '"')
3254             {
3255                 src++;
3256                 break;
3257             }
3258
3259             if (*src == '\0')
3260                 break;
3261
3262             src++;
3263         }
3264         dest[pos] = '\0';
3265
3266         if (!strcasecmp(tagstr, current_tag))
3267             return true;
3268     }
3269
3270     return false;
3271 }
3272
3273 static int parse_changelog_line(int line_n, const char *buf, void *parameters)
3274 {
3275     struct index_entry idx;
3276     char tag_data[TAG_MAXLEN+32];
3277     int idx_id;
3278     long masterfd = (long)parameters;
3279     const int import_tags[] = { tag_playcount, tag_rating, tag_playtime, tag_lastplayed,
3280         tag_commitid };
3281     int i;
3282     (void)line_n;
3283
3284     if (*buf == '#')
3285         return 0;
3286
3287     logf("%d/%s", line_n, buf);
3288     if (!read_tag(tag_data, sizeof tag_data, buf, "filename"))
3289     {
3290         logf("filename missing");
3291         logf("-> %s", buf);
3292         return 0;
3293     }
3294
3295     idx_id = find_index(tag_data);
3296     if (idx_id < 0)
3297     {
3298         logf("entry not found");
3299         return 0;
3300     }
3301
3302     if (!get_index(masterfd, idx_id, &idx, false))
3303     {
3304         logf("failed to retrieve index entry");
3305         return 0;
3306     }
3307
3308     /* Stop if tag has already been modified. */
3309     if (idx.flag & FLAG_DIRTYNUM)
3310         return 0;
3311
3312     logf("import: %s", tag_data);
3313
3314     idx.flag |= FLAG_DIRTYNUM;
3315     for (i = 0; i < (long)(sizeof(import_tags)/sizeof(import_tags[0])); i++)
3316     {
3317         int data;
3318
3319         if (!read_tag(tag_data, sizeof tag_data, buf,
3320                       tagcache_tag_to_str(import_tags[i])))
3321         {
3322             continue;
3323         }
3324
3325         data = atoi(tag_data);
3326         if (data < 0)
3327             continue;
3328
3329         idx.tag_seek[import_tags[i]] = data;
3330
3331         if (import_tags[i] == tag_lastplayed && data >= current_tcmh.serial)
3332             current_tcmh.serial = data + 1;
3333         else if (import_tags[i] == tag_commitid && data >= current_tcmh.commitid)
3334             current_tcmh.commitid = data + 1;
3335     }
3336
3337     return write_index(masterfd, idx_id, &idx) ? 0 : -5;
3338 }
3339
3340 bool tagcache_import_changelog(void)
3341 {
3342     struct master_header myhdr;
3343     struct tagcache_header tch;
3344     int clfd;
3345     long masterfd;
3346     char buf[2048];
3347
3348     if (!tc_stat.ready)
3349         return false;
3350
3351     while (read_lock)
3352         sleep(1);
3353
3354     clfd = open(TAGCACHE_FILE_CHANGELOG, O_RDONLY);
3355     if (clfd < 0)
3356     {
3357         logf("failure to open changelog");
3358         return false;
3359     }
3360
3361     if ( (masterfd = open_master_fd(&myhdr, true)) < 0)
3362     {
3363         close(clfd);
3364         return false;
3365     }
3366
3367     write_lock++;
3368
3369     filenametag_fd = open_tag_fd(&tch, tag_filename, false);
3370
3371     fast_readline(clfd, buf, sizeof buf, (long *)masterfd,
3372                   parse_changelog_line);
3373
3374     close(clfd);
3375     close(masterfd);
3376
3377     if (filenametag_fd >= 0)
3378         close(filenametag_fd);
3379
3380     write_lock--;
3381
3382     update_master_header();
3383
3384     return true;
3385 }
3386
3387 #endif /* !__PCTOOL__ */
3388
3389 bool tagcache_create_changelog(struct tagcache_search *tcs)
3390 {
3391     struct master_header myhdr;
3392     struct index_entry idx;
3393     char buf[TAG_MAXLEN+32];
3394     char temp[32];
3395     int clfd;
3396     int i, j;
3397
3398     if (!tc_stat.ready)
3399         return false;
3400
3401     if (!tagcache_search(tcs, tag_filename))
3402         return false;
3403
3404     /* Initialize the changelog */
3405     clfd = open(TAGCACHE_FILE_CHANGELOG, O_WRONLY | O_CREAT | O_TRUNC);
3406     if (clfd < 0)
3407     {
3408         logf("failure to open changelog");
3409         return false;
3410     }
3411
3412     if (tcs->masterfd < 0)
3413     {
3414         if ( (tcs->masterfd = open_master_fd(&myhdr, false)) < 0)
3415             return false;
3416     }
3417     else
3418     {
3419         lseek(tcs->masterfd, 0, SEEK_SET);
3420         ecread(tcs->masterfd, &myhdr, 1, master_header_ec, tc_stat.econ);
3421     }
3422
3423     write(clfd, "## Changelog version 1\n", 23);
3424
3425     for (i = 0; i < myhdr.tch.entry_count; i++)
3426     {
3427         if (ecread(tcs->masterfd, &idx, 1, index_entry_ec, tc_stat.econ)
3428             != sizeof(struct index_entry))
3429         {
3430             logf("read error #9");
3431             tagcache_search_finish(tcs);
3432             close(clfd);
3433             return false;
3434         }
3435
3436         /* Skip until the entry found has been modified. */
3437         if (! (idx.flag & FLAG_DIRTYNUM) )
3438             continue;
3439
3440         /* Skip deleted entries too. */
3441         if (idx.flag & FLAG_DELETED)
3442             continue;
3443
3444         /* Now retrieve all tags. */
3445         for (j = 0; j < TAG_COUNT; j++)
3446         {
3447             if (tagcache_is_numeric_tag(j))
3448             {
3449                 snprintf(temp, sizeof temp, "%d", idx.tag_seek[j]);
3450                 write_tag(clfd, tagcache_tag_to_str(j), temp);
3451                 continue;
3452             }
3453
3454             tcs->type = j;
3455             tagcache_retrieve(tcs, i, tcs->type, buf, sizeof buf);
3456             write_tag(clfd, tagcache_tag_to_str(j), buf);
3457         }
3458
3459         write(clfd, "\n", 1);
3460         do_timed_yield();
3461     }
3462
3463     close(clfd);
3464
3465     tagcache_search_finish(tcs);
3466
3467     return true;
3468 }
3469
3470 static bool delete_entry(long idx_id)
3471 {
3472     int fd = -1;
3473     int masterfd = -1;
3474     int tag, i;
3475     struct index_entry idx, myidx;
3476     struct master_header myhdr;
3477     char buf[TAG_MAXLEN+32];
3478     int in_use[TAG_COUNT];
3479
3480     logf("delete_entry(): %ld", idx_id);
3481
3482 #ifdef HAVE_TC_RAMCACHE
3483     /* At first mark the entry removed from ram cache. */
3484     if (tc_stat.ramcache)
3485         hdr->indices[idx_id].flag |= FLAG_DELETED;
3486 #endif
3487
3488     if ( (masterfd = open_master_fd(&myhdr, true) ) < 0)
3489         return false;
3490
3491     lseek(masterfd, idx_id * sizeof(struct index_entry), SEEK_CUR);
3492     if (ecread(masterfd, &myidx, 1, index_entry_ec, tc_stat.econ)
3493         != sizeof(struct index_entry))
3494     {
3495         logf("delete_entry(): read error");
3496         goto cleanup;
3497     }
3498
3499     if (myidx.flag & FLAG_DELETED)
3500     {
3501         logf("delete_entry(): already deleted!");
3502         goto cleanup;
3503     }
3504
3505     myidx.flag |= FLAG_DELETED;
3506     lseek(masterfd, -sizeof(struct index_entry), SEEK_CUR);
3507     if (ecwrite(masterfd, &myidx, 1, index_entry_ec, tc_stat.econ)
3508         != sizeof(struct index_entry))
3509     {
3510         logf("delete_entry(): write_error #1");
3511         goto cleanup;
3512     }
3513
3514     /* Now check which tags are no longer in use (if any) */
3515     for (tag = 0; tag < TAG_COUNT; tag++)
3516         in_use[tag] = 0;
3517
3518     lseek(masterfd, sizeof(struct master_header), SEEK_SET);
3519     for (i = 0; i < myhdr.tch.entry_count; i++)
3520     {
3521         struct index_entry *idxp;
3522
3523 #ifdef HAVE_TC_RAMCACHE
3524         /* Use RAM DB if available for greater speed */
3525         if (tc_stat.ramcache)
3526             idxp = &hdr->indices[i];
3527         else
3528 #endif
3529         {
3530             if (ecread(masterfd, &idx, 1, index_entry_ec, tc_stat.econ)
3531                 != sizeof(struct index_entry))
3532             {
3533                 logf("delete_entry(): read error #2");
3534                 goto cleanup;
3535             }
3536             idxp = &idx;
3537         }
3538
3539         if (idxp->flag & FLAG_DELETED)
3540             continue;
3541
3542         for (tag = 0; tag < TAG_COUNT; tag++)
3543         {
3544             if (tagcache_is_numeric_tag(tag))
3545                 continue;
3546
3547             if (idxp->tag_seek[tag] == myidx.tag_seek[tag])
3548                 in_use[tag]++;
3549         }
3550     }
3551
3552     /* Now delete all tags no longer in use. */
3553     for (tag = 0; tag < TAG_COUNT; tag++)
3554     {
3555         struct tagcache_header tch;
3556         int oldseek = myidx.tag_seek[tag];
3557
3558         if (tagcache_is_numeric_tag(tag))
3559             continue;
3560
3561         /**
3562          * Replace tag seek with a hash value of the field string data.
3563          * That way runtime statistics of moved or altered files can be
3564          * resurrected.
3565          */
3566 #ifdef HAVE_TC_RAMCACHE
3567         if (tc_stat.ramcache && tag != tag_filename)
3568         {
3569             struct tagfile_entry *tfe;
3570             int32_t *seek = &hdr->indices[idx_id].tag_seek[tag];
3571
3572             tfe = (struct tagfile_entry *)&hdr->tags[tag][*seek];
3573             *seek = crc_32(tfe->tag_data, strlen(tfe->tag_data), 0xffffffff);
3574             myidx.tag_seek[tag] = *seek;
3575         }
3576         else
3577 #endif
3578         {
3579             struct tagfile_entry tfe;
3580
3581             /* Open the index file, which contains the tag names. */
3582             if ((fd = open_tag_fd(&tch, tag, true)) < 0)
3583                 goto cleanup;
3584
3585             /* Skip the header block */
3586             lseek(fd, myidx.tag_seek[tag], SEEK_SET);
3587             if (ecread(fd, &tfe, 1, tagfile_entry_ec, tc_stat.econ)
3588                 != sizeof(struct tagfile_entry))
3589             {
3590                 logf("delete_entry(): read error #3");
3591                 goto cleanup;
3592             }
3593
3594             if (read(fd, buf, tfe.tag_length) != tfe.tag_length)
3595             {
3596                 logf("delete_entry(): read error #4");
3597                 goto cleanup;
3598             }
3599
3600             myidx.tag_seek[tag] = crc_32(buf, strlen(buf), 0xffffffff);
3601         }
3602
3603         if (in_use[tag])
3604         {
3605             logf("in use: %d/%d", tag, in_use[tag]);
3606             if (fd >= 0)
3607             {
3608                 close(fd);
3609                 fd = -1;
3610             }
3611             continue;
3612         }
3613
3614 #ifdef HAVE_TC_RAMCACHE
3615         /* Delete from ram. */
3616         if (tc_stat.ramcache && tag != tag_filename)
3617         {
3618             struct tagfile_entry *tagentry = (struct tagfile_entry *)&hdr->tags[tag][oldseek];
3619             tagentry->tag_data[0] = '\0';
3620         }
3621 #endif
3622
3623         /* Open the index file, which contains the tag names. */
3624         if (fd < 0)
3625         {
3626             if ((fd = open_tag_fd(&tch, tag, true)) < 0)
3627                 goto cleanup;
3628         }
3629
3630         /* Skip the header block */
3631         lseek(fd, oldseek + sizeof(struct tagfile_entry), SEEK_SET);
3632
3633         /* Debug, print 10 first characters of the tag
3634         read(fd, buf, 10);
3635         buf[10]='\0';
3636         logf("TAG:%s", buf);
3637         lseek(fd, -10, SEEK_CUR);
3638         */
3639
3640         /* Write first data byte in tag as \0 */
3641         write(fd, "", 1);
3642
3643         /* Now tag data has been removed */
3644         close(fd);
3645         fd = -1;
3646     }
3647
3648     /* Write index entry back into master index. */
3649     lseek(masterfd, sizeof(struct master_header) +
3650           (idx_id * sizeof(struct index_entry)), SEEK_SET);
3651     if (ecwrite(masterfd, &myidx, 1, index_entry_ec, tc_stat.econ)
3652         != sizeof(struct index_entry))
3653     {
3654         logf("delete_entry(): write_error #2");
3655         goto cleanup;
3656     }
3657
3658     close(masterfd);
3659
3660     return true;
3661
3662     cleanup:
3663     if (fd >= 0)
3664         close(fd);
3665     if (masterfd >= 0)
3666         close(masterfd);
3667
3668     return false;
3669 }
3670
3671 #ifndef __PCTOOL__
3672 /**
3673  * Returns true if there is an event waiting in the queue
3674  * that requires the current operation to be aborted.
3675  */
3676 static bool check_event_queue(void)
3677 {
3678     struct queue_event ev;
3679
3680     queue_wait_w_tmo(&tagcache_queue, &ev, 0);
3681     switch (ev.id)
3682     {
3683         case Q_STOP_SCAN:
3684         case SYS_POWEROFF:
3685         case SYS_USB_CONNECTED:
3686             /* Put the event back into the queue. */
3687             queue_post(&tagcache_queue, ev.id, ev.data);
3688             return true;
3689     }
3690
3691     return false;
3692 }
3693 #endif
3694
3695 #ifdef HAVE_TC_RAMCACHE
3696 static bool allocate_tagcache(void)
3697 {
3698     struct master_header tcmh;
3699     int fd;
3700
3701     /* Load the header. */
3702     if ( (fd = open_master_fd(&tcmh, false)) < 0)
3703     {
3704         hdr = NULL;
3705         return false;
3706     }
3707
3708     close(fd);
3709
3710     /**
3711      * Now calculate the required cache size plus
3712      * some extra space for alignment fixes.
3713      */
3714     tc_stat.ramcache_allocated = tcmh.tch.datasize + 128 + TAGCACHE_RESERVE +
3715         sizeof(struct ramcache_header) + TAG_COUNT*sizeof(void *);
3716     hdr = buffer_alloc(tc_stat.ramcache_allocated + 128);
3717     memset(hdr, 0, sizeof(struct ramcache_header));
3718     memcpy(&hdr->h, &tcmh, sizeof(struct master_header));
3719     hdr->indices = (struct index_entry *)(hdr + 1);
3720     logf("tagcache: %d bytes allocated.", tc_stat.ramcache_allocated);
3721
3722     return true;
3723 }
3724
3725 # ifdef HAVE_EEPROM_SETTINGS
3726 static bool tagcache_dumpload(void)
3727 {
3728     struct statefile_header shdr;
3729     int fd, rc;
3730     long offpos;
3731     int i;
3732
3733     fd = open(TAGCACHE_STATEFILE, O_RDONLY);
3734     if (fd < 0)
3735     {
3736         logf("no tagcache statedump");
3737         return false;
3738     }
3739
3740     /* Check the statefile memory placement */
3741     hdr = buffer_alloc(0);
3742     rc = read(fd, &shdr, sizeof(struct statefile_header));
3743     if (rc != sizeof(struct statefile_header)
3744         /* || (long)hdr != (long)shdr.hdr */)
3745     {
3746         logf("incorrect statefile");
3747         hdr = NULL;
3748         close(fd);
3749         return false;
3750     }
3751
3752     offpos = (long)hdr - (long)shdr.hdr;
3753
3754     /* Lets allocate real memory and load it */
3755     hdr = buffer_alloc(shdr.tc_stat.ramcache_allocated);
3756     rc = read(fd, hdr, shdr.tc_stat.ramcache_allocated);
3757     close(fd);
3758
3759     if (rc != shdr.tc_stat.ramcache_allocated)
3760     {
3761         logf("read failure!");
3762         hdr = NULL;
3763         return false;
3764     }
3765
3766     memcpy(&tc_stat, &shdr.tc_stat, sizeof(struct tagcache_stat));
3767
3768     /* Now fix the pointers */
3769     hdr->indices = (struct index_entry *)((long)hdr->indices + offpos);
3770     for (i = 0; i < TAG_COUNT; i++)
3771         hdr->tags[i] += offpos;
3772
3773     return true;
3774 }
3775
3776 static bool tagcache_dumpsave(void)
3777 {
3778     struct statefile_header shdr;
3779     int fd;
3780
3781     if (!tc_stat.ramcache)
3782         return false;
3783
3784     fd = open(TAGCACHE_STATEFILE, O_WRONLY | O_CREAT | O_TRUNC);
3785     if (fd < 0)
3786     {
3787         logf("failed to create a statedump");
3788         return false;
3789     }
3790
3791     /* Create the header */
3792     shdr.hdr = hdr;
3793     memcpy(&shdr.tc_stat, &tc_stat, sizeof(struct tagcache_stat));
3794     write(fd, &shdr, sizeof(struct statefile_header));
3795
3796     /* And dump the data too */
3797     write(fd, hdr, tc_stat.ramcache_allocated);
3798     close(fd);
3799
3800     return true;
3801 }
3802 # endif
3803
3804 static bool load_tagcache(void)
3805 {
3806     struct tagcache_header *tch;
3807     long bytesleft = tc_stat.ramcache_allocated;
3808     struct index_entry *idx;
3809     int rc, fd;
3810     char *p;
3811     int i, tag;
3812
3813 # ifdef HAVE_DIRCACHE
3814     while (dircache_is_initializing())
3815         sleep(1);
3816
3817     dircache_set_appflag(DIRCACHE_APPFLAG_TAGCACHE);
3818 # endif
3819
3820     logf("loading tagcache to ram...");
3821
3822     fd = open(TAGCACHE_FILE_MASTER, O_RDONLY);
3823     if (fd < 0)
3824     {
3825         logf("tagcache open failed");
3826         return false;
3827     }
3828
3829     if (ecread(fd, &hdr->h, 1, master_header_ec, tc_stat.econ)
3830         != sizeof(struct master_header)
3831         || hdr->h.tch.magic != TAGCACHE_MAGIC)
3832     {
3833         logf("incorrect header");
3834         return false;
3835     }
3836
3837     idx = hdr->indices;
3838
3839     /* Load the master index table. */
3840     for (i = 0; i < hdr->h.tch.entry_count; i++)
3841     {
3842         rc = ecread(fd, idx, 1, index_entry_ec, tc_stat.econ);
3843         if (rc != sizeof(struct index_entry))
3844         {
3845             logf("read error #10");
3846             close(fd);
3847             return false;
3848         }
3849
3850         bytesleft -= sizeof(struct index_entry);
3851         if (bytesleft < 0 || ((long)idx - (long)hdr->indices) >= tc_stat.ramcache_allocated)
3852         {
3853             logf("too big tagcache.");
3854             close(fd);
3855             return false;
3856         }
3857
3858         idx++;
3859     }
3860
3861     close(fd);
3862
3863     /* Load the tags. */
3864     p = (char *)idx;
3865     for (tag = 0; tag < TAG_COUNT; tag++)
3866     {
3867         struct tagfile_entry *fe;
3868         char buf[TAG_MAXLEN+32];
3869
3870         if (tagcache_is_numeric_tag(tag))
3871             continue ;
3872
3873         //p = ((void *)p+1);
3874         p = (char *)((long)p & ~0x03) + 0x04;
3875         hdr->tags[tag] = p;
3876
3877         /* Check the header. */
3878         tch = (struct tagcache_header *)p;
3879         p += sizeof(struct tagcache_header);
3880
3881         if ( (fd = open_tag_fd(tch, tag, false)) < 0)
3882             return false;
3883
3884         for (hdr->entry_count[tag] = 0;
3885              hdr->entry_count[tag] < tch->entry_count;
3886              hdr->entry_count[tag]++)
3887         {
3888             long pos;
3889
3890             if (do_timed_yield())
3891             {
3892                 /* Abort if we got a critical event in queue */
3893                 if (check_event_queue())
3894                     return false;
3895             }
3896
3897             fe = (struct tagfile_entry *)p;
3898             pos = lseek(fd, 0, SEEK_CUR);
3899             rc = ecread(fd, fe, 1, tagfile_entry_ec, tc_stat.econ);
3900             if (rc != sizeof(struct tagfile_entry))
3901             {
3902                 /* End of lookup table. */
3903                 logf("read error #11");
3904                 close(fd);
3905                 return false;
3906             }
3907
3908             /* We have a special handling for the filename tags. */
3909             if (tag == tag_filename)
3910             {
3911 # ifdef HAVE_DIRCACHE
3912                 const struct dirent *dc;
3913 # endif
3914
3915                 // FIXME: This is wrong!
3916                 // idx = &hdr->indices[hdr->entry_count[i]];
3917                 idx = &hdr->indices[fe->idx_id];
3918
3919                 if (fe->tag_length >= (long)sizeof(buf)-1)
3920                 {
3921                     read(fd, buf, 10);
3922                     buf[10] = '\0';
3923                     logf("TAG:%s", buf);
3924                     logf("too long filename");
3925                     close(fd);
3926                     return false;
3927                 }
3928
3929                 rc = read(fd, buf, fe->tag_length);
3930                 if (rc != fe->tag_length)
3931                 {
3932                     logf("read error #12");
3933                     close(fd);
3934                     return false;
3935                 }
3936
3937                 /* Check if the entry has already been removed */
3938                 if (idx->flag & FLAG_DELETED)
3939                     continue;
3940
3941                 /* This flag must not be used yet. */
3942                 if (idx->flag & FLAG_DIRCACHE)
3943                 {
3944                     logf("internal error!");
3945                     close(fd);
3946                     return false;
3947                 }
3948
3949                 if (idx->tag_seek[tag] != pos)
3950                 {
3951                     logf("corrupt data structures!");
3952                     close(fd);
3953                     return false;
3954                 }
3955
3956 # ifdef HAVE_DIRCACHE
3957                 if (dircache_is_enabled())
3958                 {
3959                     dc = dircache_get_entry_ptr(buf);
3960                     if (dc == NULL)
3961                     {
3962                         logf("Entry no longer valid.");
3963                         logf("-> %s", buf);
3964                         delete_entry(fe->idx_id);
3965                         continue ;
3966                     }
3967
3968                     idx->flag |= FLAG_DIRCACHE;
3969                     FLAG_SET_ATTR(idx->flag, fe->idx_id);
3970                     idx->tag_seek[tag_filename] = (long)dc;
3971                 }
3972                 else
3973 # endif
3974                 {
3975                     /* This will be very slow unless dircache is enabled
3976                        or target is flash based, but do it anyway for
3977                        consistency. */
3978                     /* Check if entry has been removed. */
3979                     if (global_settings.tagcache_autoupdate)
3980                     {
3981                         if (!file_exists(buf))
3982                         {
3983                             logf("Entry no longer valid.");
3984                             logf("-> %s", buf);
3985                             delete_entry(fe->idx_id);
3986                             continue;
3987                         }
3988                     }
3989                 }
3990
3991                 continue ;
3992             }
3993
3994             bytesleft -= sizeof(struct tagfile_entry) + fe->tag_length;
3995             if (bytesleft < 0)
3996             {
3997                 logf("too big tagcache #2");
3998                 logf("tl: %d", fe->tag_length);
3999                 logf("bl: %ld", bytesleft);
4000                 close(fd);
4001                 return false;
4002             }
4003
4004             p = fe->tag_data;
4005             rc = read(fd, fe->tag_data, fe->tag_length);
4006             p += rc;
4007
4008             if (rc != fe->tag_length)
4009             {
4010                 logf("read error #13");
4011                 logf("rc=0x%04x", rc); // 0x431
4012                 logf("len=0x%04x", fe->tag_length); // 0x4000
4013                 logf("pos=0x%04lx", lseek(fd, 0, SEEK_CUR)); // 0x433
4014                 logf("tag=0x%02x", tag); // 0x00
4015                 close(fd);
4016                 return false;
4017             }
4018         }
4019         close(fd);
4020     }
4021
4022     tc_stat.ramcache_used = tc_stat.ramcache_allocated - bytesleft;
4023     logf("tagcache loaded into ram!");
4024
4025     return true;
4026 }
4027 #endif /* HAVE_TC_RAMCACHE */
4028
4029 static bool check_deleted_files(void)
4030 {
4031     int fd;
4032     char buf[TAG_MAXLEN+32];
4033     struct tagfile_entry tfe;
4034
4035     logf("reverse scan...");
4036     snprintf(buf, sizeof buf, TAGCACHE_FILE_INDEX, tag_filename);
4037     fd = open(buf, O_RDONLY);
4038
4039     if (fd < 0)
4040     {
4041         logf("%s open fail", buf);
4042         return false;
4043     }
4044
4045     lseek(fd, sizeof(struct tagcache_header), SEEK_SET);
4046     while (ecread(fd, &tfe, 1, tagfile_entry_ec, tc_stat.econ)
4047            == sizeof(struct tagfile_entry)
4048 #ifndef __PCTOOL__
4049            && !check_event_queue()
4050 #endif
4051            )
4052     {
4053         if (tfe.tag_length >= (long)sizeof(buf)-1)
4054         {
4055             logf("too long tag");
4056             close(fd);
4057             return false;
4058         }
4059
4060         if (read(fd, buf, tfe.tag_length) != tfe.tag_length)
4061         {
4062             logf("read error #14");
4063             close(fd);
4064             return false;
4065         }
4066
4067         /* Check if the file has already deleted from the db. */
4068         if (*buf == '\0')
4069             continue;
4070
4071         /* Now check if the file exists. */
4072         if (!file_exists(buf))
4073         {
4074             logf("Entry no longer valid.");
4075             logf("-> %s / %d", buf, tfe.tag_length);
4076             delete_entry(tfe.idx_id);
4077         }
4078     }
4079
4080     close(fd);
4081
4082     logf("done");
4083
4084     return true;
4085 }
4086
4087 static bool check_dir(const char *dirname, int add_files)
4088 {
4089     DIR *dir;
4090     int len;
4091     int success = false;
4092     int ignore, unignore;
4093     char newpath[MAX_PATH];
4094
4095     dir = opendir(dirname);
4096     if (!dir)
4097     {
4098         logf("tagcache: opendir() failed");
4099         return false;
4100     }
4101
4102     /* check for a database.ignore file */
4103     snprintf(newpath, MAX_PATH, "%s/database.ignore", dirname);
4104     ignore = file_exists(newpath);
4105     /* check for a database.unignore file */
4106     snprintf(newpath, MAX_PATH, "%s/database.unignore", dirname);
4107     unignore = file_exists(newpath);
4108
4109     /* don't do anything if both ignore and unignore are there */
4110     if (ignore != unignore)
4111         add_files = unignore;
4112
4113     /* Recursively scan the dir. */
4114 #ifdef __PCTOOL__
4115     while (1)
4116 #else
4117     while (!check_event_queue())
4118 #endif
4119     {
4120         struct dirent *entry;
4121
4122         entry = readdir(dir);
4123
4124         if (entry == NULL)
4125         {
4126             success = true;
4127             break ;
4128         }
4129
4130         if (!strcmp((char *)entry->d_name, ".") ||
4131             !strcmp((char *)entry->d_name, ".."))
4132             continue;
4133
4134         yield();
4135
4136         len = strlen(curpath);
4137         snprintf(&curpath[len], sizeof(curpath) - len, "/%s",
4138                  entry->d_name);
4139
4140         processed_dir_count++;
4141         if (entry->attribute & ATTR_DIRECTORY)
4142             check_dir(curpath, add_files);
4143         else if (add_files)
4144         {
4145             tc_stat.curentry = curpath;
4146
4147             /* Add a new entry to the temporary db file. */
4148             add_tagcache(curpath, (entry->wrtdate << 16) | entry->wrttime
4149 #if defined(HAVE_TC_RAMCACHE) && defined(HAVE_DIRCACHE)
4150                          , dir->internal_entry
4151 #endif
4152                          );
4153
4154             /* Wait until current path for debug screen is read and unset. */
4155             while (tc_stat.syncscreen && tc_stat.curentry != NULL)
4156                 yield();
4157
4158             tc_stat.curentry = NULL;
4159         }
4160
4161         curpath[len] = '\0';
4162     }
4163
4164     closedir(dir);
4165
4166     return success;
4167 }
4168
4169 void tagcache_screensync_event(void)
4170 {
4171     tc_stat.curentry = NULL;
4172 }
4173
4174 void tagcache_screensync_enable(bool state)
4175 {
4176     tc_stat.syncscreen = state;
4177 }
4178
4179 void tagcache_build(const char *path)
4180 {
4181     struct tagcache_header header;
4182     bool ret;
4183
4184     curpath[0] = '\0';
4185     data_size = 0;
4186     total_entry_count = 0;
4187     processed_dir_count = 0;
4188
4189 #ifdef HAVE_DIRCACHE
4190     while (dircache_is_initializing())
4191         sleep(1);
4192 #endif
4193
4194     logf("updating tagcache");
4195
4196     cachefd = open(TAGCACHE_FILE_TEMP, O_RDONLY);
4197     if (cachefd >= 0)
4198     {
4199         logf("skipping, cache already waiting for commit");
4200         close(cachefd);
4201         return ;
4202     }
4203
4204     cachefd = open(TAGCACHE_FILE_TEMP, O_RDWR | O_CREAT | O_TRUNC);
4205     if (cachefd < 0)
4206     {
4207         logf("master file open failed: %s", TAGCACHE_FILE_TEMP);
4208         return ;
4209     }
4210
4211     filenametag_fd = open_tag_fd(&header, tag_filename, false);
4212
4213     cpu_boost(true);
4214
4215     logf("Scanning files...");
4216     /* Scan for new files. */
4217     memset(&header, 0, sizeof(struct tagcache_header));
4218     write(cachefd, &header, sizeof(struct tagcache_header));
4219
4220     if (strcmp("/", path) != 0)
4221         strcpy(curpath, path);
4222     ret = check_dir(path, true);
4223
4224     /* Write the header. */
4225     header.magic = TAGCACHE_MAGIC;
4226     header.datasize = data_size;
4227     header.entry_count = total_entry_count;
4228     lseek(cachefd, 0, SEEK_SET);
4229     write(cachefd, &header, sizeof(struct tagcache_header));
4230     close(cachefd);
4231
4232     if (filenametag_fd >= 0)
4233     {
4234         close(filenametag_fd);
4235         filenametag_fd = -1;
4236     }
4237
4238     if (!ret)
4239     {
4240         logf("Aborted.");
4241         cpu_boost(false);
4242         return ;
4243     }
4244
4245     /* Commit changes to the database. */
4246 #ifdef __PCTOOL__
4247     allocate_tempbuf();
4248 #endif
4249     if (commit())
4250     {
4251         remove(TAGCACHE_FILE_TEMP);
4252         logf("tagcache built!");
4253     }
4254 #ifdef __PCTOOL__
4255     free_tempbuf();
4256 #endif
4257
4258 #ifdef HAVE_TC_RAMCACHE
4259     if (hdr)
4260     {
4261         /* Import runtime statistics if we just initialized the db. */
4262         if (hdr->h.serial == 0)
4263             queue_post(&tagcache_queue, Q_IMPORT_CHANGELOG, 0);
4264     }
4265 #endif
4266
4267     cpu_boost(false);
4268 }
4269
4270 #ifdef HAVE_TC_RAMCACHE
4271 static void load_ramcache(void)
4272 {
4273     if (!hdr)
4274         return ;
4275
4276     cpu_boost(true);
4277
4278     /* At first we should load the cache (if exists). */
4279     tc_stat.ramcache = load_tagcache();
4280
4281     if (!tc_stat.ramcache)
4282     {
4283         /* If loading failed, it must indicate some problem with the db
4284          * so disable it entirely to prevent further issues. */
4285         tc_stat.ready = false;
4286         hdr = NULL;
4287     }
4288
4289     cpu_boost(false);
4290 }
4291
4292 void tagcache_unload_ramcache(void)
4293 {
4294     tc_stat.ramcache = false;
4295     /* Just to make sure there is no statefile present. */
4296     // remove(TAGCACHE_STATEFILE);
4297 }
4298 #endif
4299
4300 #ifndef __PCTOOL__
4301 static void tagcache_thread(void)
4302 {
4303     struct queue_event ev;
4304     bool check_done = false;
4305
4306     /* If the previous cache build/update was interrupted, commit
4307      * the changes first in foreground. */
4308     cpu_boost(true);
4309     allocate_tempbuf();
4310     commit();
4311     free_tempbuf();
4312
4313 #ifdef HAVE_TC_RAMCACHE
4314 # ifdef HAVE_EEPROM_SETTINGS
4315     if (firmware_settings.initialized && firmware_settings.disk_clean)
4316         check_done = tagcache_dumpload();
4317
4318     remove(TAGCACHE_STATEFILE);
4319 # endif
4320
4321     /* Allocate space for the tagcache if found on disk. */
4322     if (global_settings.tagcache_ram && !tc_stat.ramcache)
4323         allocate_tagcache();
4324 #endif
4325
4326     cpu_boost(false);
4327     tc_stat.initialized = true;
4328
4329     /* Don't delay bootup with the header check but do it on background. */
4330     sleep(HZ);
4331     tc_stat.ready = check_all_headers();
4332     tc_stat.readyvalid = true;
4333
4334     while (1)
4335     {
4336         run_command_queue(false);
4337
4338         queue_wait_w_tmo(&tagcache_queue, &ev, HZ);
4339
4340         switch (ev.id)
4341         {
4342             case Q_IMPORT_CHANGELOG:
4343                 tagcache_import_changelog();
4344                 break;
4345
4346             case Q_REBUILD:
4347                 remove_files();
4348                 remove(TAGCACHE_FILE_TEMP);
4349                 tagcache_build("/");
4350                 break;
4351
4352             case Q_UPDATE:
4353                 tagcache_build("/");
4354 #ifdef HAVE_TC_RAMCACHE
4355                 load_ramcache();
4356 #endif
4357                 check_deleted_files();
4358                 break ;
4359
4360             case Q_START_SCAN:
4361                 check_done = false;
4362             case SYS_TIMEOUT:
4363                 if (check_done || !tc_stat.ready)
4364                     break ;
4365
4366 #ifdef HAVE_TC_RAMCACHE
4367                 if (!tc_stat.ramcache && global_settings.tagcache_ram)
4368                 {
4369                     load_ramcache();
4370                     if (tc_stat.ramcache && global_settings.tagcache_autoupdate)
4371                         tagcache_build("/");
4372                 }
4373                 else
4374 #endif
4375                 if (global_settings.tagcache_autoupdate)
4376                 {
4377                     tagcache_build("/");
4378
4379                     /* This will be very slow unless dircache is enabled
4380                        or target is flash based, but do it anyway for
4381                        consistency. */
4382                     check_deleted_files();
4383                 }
4384
4385                 logf("tagcache check done");
4386
4387                 check_done = true;
4388                 break ;
4389
4390             case Q_STOP_SCAN:
4391                 break ;
4392
4393             case SYS_POWEROFF:
4394                 break ;
4395
4396 #ifndef SIMULATOR
4397             case SYS_USB_CONNECTED:
4398                 logf("USB: TagCache");
4399                 usb_acknowledge(SYS_USB_CONNECTED_ACK);
4400                 usb_wait_for_disconnect(&tagcache_queue);
4401                 break ;
4402 #endif
4403         }
4404     }
4405 }
4406
4407 bool tagcache_prepare_shutdown(void)
4408 {
4409     if (tagcache_get_commit_step() > 0)
4410         return false;
4411
4412     tagcache_stop_scan();
4413     while (read_lock || write_lock)
4414         sleep(1);
4415
4416     return true;
4417 }
4418
4419 void tagcache_shutdown(void)
4420 {
4421     /* Flush the command queue. */
4422     run_command_queue(true);
4423
4424 #ifdef HAVE_EEPROM_SETTINGS
4425     if (tc_stat.ramcache)
4426         tagcache_dumpsave();
4427 #endif
4428 }
4429
4430 static int get_progress(void)
4431 {
4432     int total_count = -1;
4433
4434 #ifdef HAVE_DIRCACHE
4435     if (dircache_is_enabled())
4436     {
4437         total_count = dircache_get_entry_count();
4438     }
4439     else
4440 #endif
4441 #ifdef HAVE_TC_RAMCACHE
4442     {
4443         if (hdr && tc_stat.ramcache)
4444             total_count = hdr->h.tch.entry_count;
4445     }
4446 #endif
4447
4448     if (total_count < 0)
4449         return -1;
4450
4451     return processed_dir_count * 100 / total_count;
4452 }
4453
4454 struct tagcache_stat* tagcache_get_stat(void)
4455 {
4456     tc_stat.progress = get_progress();
4457     tc_stat.processed_entries = processed_dir_count;
4458
4459     return &tc_stat;
4460 }
4461
4462 void tagcache_start_scan(void)
4463 {
4464     queue_post(&tagcache_queue, Q_START_SCAN, 0);
4465 }
4466
4467 bool tagcache_update(void)
4468 {
4469     if (!tc_stat.ready)
4470         return false;
4471
4472     queue_post(&tagcache_queue, Q_UPDATE, 0);
4473     return false;
4474 }
4475
4476 bool tagcache_rebuild()
4477 {
4478     queue_post(&tagcache_queue, Q_REBUILD, 0);
4479     return false;
4480 }
4481
4482 void tagcache_stop_scan(void)
4483 {
4484     queue_post(&tagcache_queue, Q_STOP_SCAN, 0);
4485 }
4486
4487 #ifdef HAVE_TC_RAMCACHE
4488 bool tagcache_is_ramcache(void)
4489 {
4490     return tc_stat.ramcache;
4491 }
4492 #endif
4493
4494 #endif /* !__PCTOOL__ */
4495
4496
4497 void tagcache_init(void)
4498 {
4499     memset(&tc_stat, 0, sizeof(struct tagcache_stat));
4500     memset(&current_tcmh, 0, sizeof(struct master_header));
4501     filenametag_fd = -1;
4502     write_lock = read_lock = 0;
4503
4504 #ifndef __PCTOOL__
4505     mutex_init(&command_queue_mutex);
4506     queue_init(&tagcache_queue, true);
4507     create_thread(tagcache_thread, tagcache_stack,
4508                   sizeof(tagcache_stack), 0, tagcache_thread_name
4509                   IF_PRIO(, PRIORITY_BACKGROUND)
4510                           IF_COP(, CPU));
4511 #else
4512     tc_stat.initialized = true;
4513     allocate_tempbuf();
4514     commit();
4515     free_tempbuf();
4516     tc_stat.ready = check_all_headers();
4517 #endif
4518 }
4519
4520 #ifdef __PCTOOL__
4521 void tagcache_reverse_scan(void)
4522 {
4523     logf("Checking for deleted files");
4524     check_deleted_files();
4525 }
4526 #endif
4527
4528 bool tagcache_is_initialized(void)
4529 {
4530     return tc_stat.initialized;
4531 }
4532 bool tagcache_is_usable(void)
4533 {
4534     return tc_stat.initialized && tc_stat.ready;
4535 }
4536 int tagcache_get_commit_step(void)
4537 {
4538     return tc_stat.commit_step;
4539 }
4540 int tagcache_get_max_commit_step(void)
4541 {
4542     return (int)(sizeof(sorted_tags)/sizeof(sorted_tags[0]))+1;
4543 }
4544