beagled/FileAttributesStore_Sqlite.cs

   1 //
   2 // FileAttributesStore_Sqlite.cs
   3 //
   4 // Copyright (C) 2004 Novell, Inc.
   5 //
   6
   7 //
   8 // Permission is hereby granted, free of charge, to any person obtaining a copy
   9 // of this software and associated documentation files (the "Software"), to deal
  10 // in the Software without restriction, including without limitation the rights
  11 // to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
  12 // copies of the Software, and to permit persons to whom the Software is
  13 // furnished to do so, subject to the following conditions:
  14 //
  15 // The above copyright notice and this permission notice shall be included in all
  16 // copies or substantial portions of the Software.
  17 //
  18 // THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  19 // IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  20 // FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
  21 // AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
  22 // LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
  23 // OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
  24 // SOFTWARE.
  25 //
  26
  27 using System;
  28 using System.Collections;
  29 using System.IO;
  30 using System.Threading;
  31
  32 using Mono.Data.SqliteClient;
  33
  34 using Beagle.Util;
  35
  36 namespace Beagle.Daemon {
  37
  38         public class FileAttributesStore_Sqlite : IFileAttributesStore {
  39
  40                 // Version history:
  41                 // 1: Original version
  42                 // 2: Replaced LastIndexedTime with LastAttrTime
  43                 const int VERSION = 2;
  44
  45                 private SqliteConnection connection;
  46                 private BitArray path_flags;
  47                 private int transaction_count = 0;
  48
  49                 enum TransactionState {
  50                         None,
  51                         Requested,
  52                         Started
  53                 }
  54                 private TransactionState transaction_state;
  55
  56                 public FileAttributesStore_Sqlite (string directory, string index_fingerprint)
  57                 {
  58                         bool create_new_db = false;
  59                         path_flags = new BitArray (65536);
  60
  61                         if (! File.Exists (GetDbPath (directory))) {
  62                                 create_new_db = true;
  63                         } else {
  64
  65                                 // Funky logic here to deal with sqlite versions.
  66                                 //
  67                                 // When sqlite 3 tries to open an sqlite 2 database,
  68                                 // it will throw an SqliteException with SqliteError
  69                                 // NOTADB when trying to execute a command.
  70                                 //
  71                                 // When sqlite 2 tries to open an sqlite 3 database,
  72                                 // it will throw an ApplicationException when it
  73                                 // tries to open the database.
  74
  75                                 try {
  76                                         connection = Open (directory);
  77                                 } catch (ApplicationException) {
  78                                         Logger.Log.Warn ("Likely sqlite database version mismatch trying to open {0}.  Purging.", GetDbPath (directory));
  79                                         create_new_db = true;
  80                                 }
  81
  82                                 if (! create_new_db) {
  83                                         SqliteCommand command;
  84                                         SqliteDataReader reader = null;
  85                                         int stored_version = 0;
  86                                         string stored_fingerprint = null;
  87
  88
  89                                         command = new SqliteCommand ();
  90                                         command.Connection = connection;
  91                                         command.CommandText =
  92                                                 "SELECT version, fingerprint FROM db_info";
  93                                         try {
  94                                                 reader = ExecuteReaderOrWait (command);
  95                                         } catch (Exception ex) {
  96                                                 Logger.Log.Warn ("Likely sqlite database version mismatch trying to read from {0}.  Purging.", GetDbPath (directory));
  97                                                 create_new_db = true;
  98                                         }
  99                                         if (reader != null && ! create_new_db) {
 100                                                 if (ReadOrWait (reader)) {
 101                                                         stored_version = reader.GetInt32 (0);
 102                                                         stored_fingerprint = reader.GetString (1);
 103                                                 }
 104                                                 reader.Close ();
 105                                         }
 106                                         command.Dispose ();
 107
 108                                         if (VERSION != stored_version
 109                                             || (index_fingerprint != null && index_fingerprint != stored_fingerprint))
 110                                                 create_new_db = true;
 111                                 }
 112                         }
 113
 114                         if (create_new_db) {
 115                                 if (connection != null)
 116                                         connection.Dispose ();
 117                                 File.Delete (GetDbPath (directory));
 118                                 connection = Open (directory);
 119
 120                                 DoNonQuery ("CREATE TABLE db_info (             " +
 121                                             "  version       INTEGER NOT NULL,  " +
 122                                             "  fingerprint   STRING NOT NULL    " +
 123                                             ")");
 124
 125                                 DoNonQuery ("INSERT INTO db_info (version, fingerprint) VALUES ({0}, '{1}')",
 126                                             VERSION, index_fingerprint);
 127
 128                                 DoNonQuery ("CREATE TABLE file_attributes (           " +
 129                                             "  unique_id      STRING UNIQUE,          " +
 130                                             "  directory      STRING NOT NULL,        " +
 131                                             "  filename       STRING NOT NULL,        " +
 132                                             "  last_mtime     STRING NOT NULL,        " +
 133                                             "  last_attrtime  STRING NOT NULL,        " +
 134                                             "  filter_name    STRING NOT NULL,        " +
 135                                             "  filter_version STRING NOT NULL         " +
 136                                             ")");
 137
 138                                 DoNonQuery ("CREATE INDEX file_path on file_attributes (" +
 139                                             "  directory,      " +
 140                                             "  filename        " +
 141                                             ")");
 142                         } else {
 143                                 SqliteCommand command;
 144                                 SqliteDataReader reader;
 145                                 int count = 0;
 146
 147                                 DateTime dt1 = DateTime.Now;
 148
 149                                 // Select all of the files and use them to populate our bit-vector.
 150                                 command = new SqliteCommand ();
 151                                 command.Connection = connection;
 152                                 command.CommandText = "SELECT directory, filename FROM file_attributes";
 153
 154                                 reader = ExecuteReaderOrWait (command);
 155
 156                                 while (ReadOrWait (reader)) {
 157
 158                                         string dir = reader.GetString (0);
 159                                         string file = reader.GetString (1);
 160                                         string path = Path.Combine (dir, file);
 161                                         SetPathFlag (path);
 162                                         ++count;
 163                                 }
 164
 165                                 reader.Close ();
 166                                 command.Dispose ();
 167
 168                                 DateTime dt2 = DateTime.Now;
 169
 170                                 Logger.Log.Debug ("Loaded {0} records from {1} in {2:0.000}s",
 171                                                  count, GetDbPath (directory), (dt2 - dt1).TotalSeconds);
 172                         }
 173                 }
 174
 175                 ///////////////////////////////////////////////////////////////////
 176
 177                 private string GetDbPath (string directory)
 178                 {
 179                         return Path.Combine (directory, "FileAttributesStore.db");
 180                 }
 181
 182                 private SqliteConnection Open (string directory)
 183                 {
 184                         SqliteConnection c;
 185                         c = new SqliteConnection ();
 186                         c.ConnectionString = "version=" + ExternalStringsHack.SqliteVersion
 187                                 + ",encoding=UTF-8,URI=file:" + GetDbPath (directory);
 188                         c.Open ();
 189                         return c;
 190                 }
 191
 192                 private void DoNonQuery (string format, params object [] args)
 193                 {
 194                         SqliteCommand command;
 195                         command = new SqliteCommand ();
 196                         command.Connection = connection;
 197                         command.CommandText = String.Format (format, args);
 198
 199                         while (true) {
 200                                 try {
 201                                         command.ExecuteNonQuery ();
 202                                         break;
 203                                 } catch (SqliteBusyException ex) {
 204                                         Thread.Sleep (50);
 205                                 }
 206                         }
 207
 208                         command.Dispose ();
 209                 }
 210
 211                 private SqliteCommand QueryCommand (string where_format, params object [] where_args)
 212                 {
 213                         SqliteCommand command;
 214                         command = new SqliteCommand ();
 215                         command.Connection = connection;
 216                         command.CommandText =
 217                                 "SELECT unique_id, directory, filename, last_mtime, last_attrtime, filter_name, filter_version " +
 218                                 "FROM file_attributes WHERE " +
 219                                 String.Format (where_format, where_args);
 220                         return command;
 221                 }
 222
 223                 static private SqliteDataReader ExecuteReaderOrWait (SqliteCommand command)
 224                 {
 225                         SqliteDataReader reader = null;
 226                         while (reader == null) {
 227                                 try {
 228                                         reader = command.ExecuteReader ();
 229                                 } catch (SqliteBusyException ex) {
 230                                         Thread.Sleep (50);
 231                                 }
 232                         }
 233                         return reader;
 234                 }
 235
 236                 static private bool ReadOrWait (SqliteDataReader reader)
 237                 {
 238                         while (true) {
 239                                 try {
 240                                         return reader.Read ();
 241                                 } catch (SqliteBusyException ex) {
 242                                         Thread.Sleep (50);
 243                                 }
 244                         }
 245                 }
 246
 247                 private FileAttributes GetFromReader (SqliteDataReader reader)
 248                 {
 249                         FileAttributes attr = new FileAttributes ();
 250
 251                         attr.UniqueId = GuidFu.FromShortString (reader.GetString (0));
 252                         attr.Path = System.IO.Path.Combine (reader.GetString (1), reader.GetString (2));
 253                         attr.LastWriteTime = StringFu.StringToDateTime (reader.GetString (3));
 254                         attr.LastAttrTime = StringFu.StringToDateTime (reader.GetString (4));
 255                         attr.FilterName = reader.GetString (5);
 256                         attr.FilterVersion = int.Parse (reader.GetString (6));
 257
 258                         if (attr.FilterName == "")
 259                                 attr.FilterName = null;
 260
 261                         return attr;
 262                 }
 263
 264                 ///////////////////////////////////////////////////////////////////
 265
 266                 private int GetPathHash (string path)
 267                 {
 268                         uint hash = 0xdeadbeef;
 269                         foreach (char c in path)
 270                                 hash = 17 * hash + (uint) c;
 271                         // Fold the 32 bits in 16.
 272                         return (int) ((hash & 0xffff) ^ (hash >> 16));
 273                 }
 274
 275                 private bool GetPathFlag (string path)
 276                 {
 277                         int hash = GetPathHash (path);
 278                         return path_flags [hash];
 279                 }
 280
 281                 private void SetPathFlag (string path)
 282                 {
 283                         int hash = GetPathHash (path);
 284                         path_flags [hash] = true;
 285                 }
 286
 287                 ///////////////////////////////////////////////////////////////////
 288
 289                 public FileAttributes Read (string path)
 290                 {
 291                         // Sanitize the path; remove the last '/'
 292                         if (path != null && path != "/" && path.EndsWith ("/"))
 293                                 path = path.TrimEnd ('/');
 294
 295                         SqliteCommand command;
 296                         SqliteDataReader reader;
 297
 298                         if (! GetPathFlag (path))
 299                                 return null;
 300
 301                         FileAttributes attr = null;
 302                         bool found_too_many = false;
 303
 304                         // We need to quote any 's that appear in the strings
 305                         // (int particular, in the path)
 306                         string directory = FileSystem.GetDirectoryNameRootOk (path).Replace ("'", "''");
 307                         string filename = Path.GetFileName (path).Replace ("'", "''");
 308                         lock (connection) {
 309                                 command = QueryCommand ("directory='{0}' AND filename='{1}'",
 310                                                         directory, filename);
 311                                 reader = ExecuteReaderOrWait (command);
 312
 313                                 if (ReadOrWait (reader)) {
 314                                         attr = GetFromReader (reader);
 315
 316                                         if (ReadOrWait (reader))
 317                                                 found_too_many = true;
 318                                 }
 319                                 reader.Close ();
 320                                 command.Dispose ();
 321
 322                                 // If we found more than one matching record for a given
 323                                 // directory and filename, something has gone wrong.
 324                                 // Since we have no way of knowing which one is correct
 325                                 // and which isn't, we delete them all and return
 326                                 // null.  (Which in most cases will force a re-index.
 327                                 if (found_too_many) {
 328                                         DoNonQuery ("DELETE FROM file_attributes WHERE directory='{0}' AND filename='{1}'",
 329                                                     directory, filename);
 330                                 }
 331                         }
 332
 333                         return attr;
 334                 }
 335
 336                 public bool Write (FileAttributes fa)
 337                 {
 338                         SetPathFlag (fa.Path);
 339
 340                         // We need to quote any 's that appear in the strings
 341                         // (in particular, in the path)
 342                         lock (connection) {
 343
 344                                 // If a transaction has been requested, start it now.
 345                                 MaybeStartTransaction ();
 346
 347                                 string filter_name;
 348                                 filter_name = fa.FilterName;
 349                                 if (filter_name == null)
 350                                         filter_name = "";
 351                                 filter_name = filter_name.Replace ("'", "''");
 352
 353                                 DoNonQuery ("INSERT OR REPLACE INTO file_attributes " +
 354                                             " (unique_id, directory, filename, last_mtime, last_attrtime, filter_name, filter_version) " +
 355                                             " VALUES ('{0}', '{1}', '{2}', '{3}', '{4}', '{5}', '{6}')",
 356                                             GuidFu.ToShortString (fa.UniqueId),
 357                                             fa.Directory.Replace ("'", "''"), fa.Filename.Replace ("'", "''"),
 358                                             StringFu.DateTimeToString (fa.LastWriteTime),
 359                                             StringFu.DateTimeToString (fa.LastAttrTime),
 360                                             filter_name,
 361                                             fa.FilterVersion);
 362                         }
 363                         return true;
 364                 }
 365
 366                 public void Drop (string path)
 367                 {
 368                         // Sanitize the path; remove the last '/'
 369                         if (path != null && path != "/" && path.EndsWith ("/"))
 370                                 path = path.TrimEnd ('/');
 371
 372                         // We don't want to "UnSetPathFlag" here, since we have no way of knowing
 373                         // if another path hashes to the same value as this one.
 374
 375                         // We need to quote any 's that appear in the strings
 376                         // (in particular, in the path)
 377                         string directory = FileSystem.GetDirectoryNameRootOk (path).Replace ("'", "''");
 378                         string filename = Path.GetFileName (path).Replace ("'", "''");
 379                         lock (connection) {
 380
 381                                 // If a transaction has been requested, start it now.
 382                                 MaybeStartTransaction ();
 383
 384                                 DoNonQuery ("DELETE FROM file_attributes WHERE directory='{0}' AND filename='{1}'",
 385                                             directory, filename);
 386                         }
 387                 }
 388
 389                 private void MaybeStartTransaction ()
 390                 {
 391                         if (transaction_state == TransactionState.Requested) {
 392                                 DoNonQuery ("BEGIN");
 393                                 transaction_state = TransactionState.Started;
 394                         }
 395                 }
 396
 397                 public void BeginTransaction ()
 398                 {
 399                         if (transaction_state == TransactionState.None)
 400                                 transaction_state = TransactionState.Requested;
 401                 }
 402
 403                 public void CommitTransaction ()
 404                 {
 405                         if (transaction_state == TransactionState.Started) {
 406                                 lock (connection)
 407                                         DoNonQuery ("COMMIT");
 408                         }
 409                         transaction_state = TransactionState.None;
 410                 }
 411
 412                 public void Flush ()
 413                 {
 414                         lock (connection) {
 415                                 if (transaction_count > 0) {
 416                                         Logger.Log.Debug ("Flushing requested -- committing sqlite transaction");
 417                                         DoNonQuery ("COMMIT");
 418                                         transaction_count = 0;
 419                                 }
 420                         }
 421                 }
 422
 423                 ///////////////////////////////////////////////////////////////////
 424
 425                 // Return all attributes in the attributes database, used for merging
 426
 427                 private ICollection ReadAllAttributes ()
 428                 {
 429                         ArrayList attributes = new ArrayList ();
 430
 431                         SqliteCommand command;
 432                         SqliteDataReader reader;
 433
 434                         lock (connection) {
 435                                 command = new SqliteCommand ();
 436                                 command.Connection = connection;
 437                                 command.CommandText =
 438                                         "SELECT unique_id, directory, filename, last_mtime, last_attrtime, filter_name, filter_version " +
 439                                         "FROM file_attributes";
 440
 441                                 reader = ExecuteReaderOrWait (command);
 442
 443                                 while (ReadOrWait (reader)) {
 444                                         attributes.Add (GetFromReader (reader));
 445                                 }
 446                                 reader.Close ();
 447                                 command.Dispose ();
 448                         }
 449
 450                         return attributes;
 451                 }
 452
 453                 // FIXME: Might wanna do this a bit more intelligently
 454
 455                 public void Merge (FileAttributesStore_Sqlite fa_sqlite_store_to_merge)
 456                 {
 457                         ICollection attributes = fa_sqlite_store_to_merge.ReadAllAttributes ();
 458
 459                         BeginTransaction ();
 460
 461                         foreach (FileAttributes attribute in attributes)
 462                                 Write (attribute);
 463
 464                         CommitTransaction ();
 465                 }
 466         }
 467 }