beagled/FileAttributesStore_Sqlite.cs

   1 //
   2 // FileAttributesStore_Sqlite.cs
   3 //
   4 // Copyright (C) 2004 Novell, Inc.
   5 //
   6
   7 //
   8 // Permission is hereby granted, free of charge, to any person obtaining a copy
   9 // of this software and associated documentation files (the "Software"), to deal
  10 // in the Software without restriction, including without limitation the rights
  11 // to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
  12 // copies of the Software, and to permit persons to whom the Software is
  13 // furnished to do so, subject to the following conditions:
  14 //
  15 // The above copyright notice and this permission notice shall be included in all
  16 // copies or substantial portions of the Software.
  17 //
  18 // THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  19 // IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  20 // FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
  21 // AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
  22 // LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
  23 // OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
  24 // SOFTWARE.
  25 //
  26
  27 using System;
  28 using System.Collections;
  29 using System.IO;
  30 using System.Threading;
  31
  32 using Mono.Data.SqliteClient;
  33
  34 using Beagle.Util;
  35
  36 namespace Beagle.Daemon {
  37
  38         public class FileAttributesStore_Sqlite : IFileAttributesStore {
  39
  40                 const int VERSION = 1;
  41
  42                 private SqliteConnection connection;
  43                 private BitArray path_flags;
  44                 private int transaction_count = 0;
  45
  46                 public FileAttributesStore_Sqlite (string directory, string index_fingerprint)
  47                 {
  48                         bool create_new_db = false;
  49                         path_flags = new BitArray (65536);
  50
  51                         if (! File.Exists (GetDbPath (directory))) {
  52                                 create_new_db = true;
  53                         } else {
  54                                 connection = Open (directory);
  55
  56                                 SqliteCommand command;
  57                                 SqliteDataReader reader = null;
  58                                 int stored_version = 0;
  59                                 string stored_fingerprint = null;
  60
  61
  62                                 command = new SqliteCommand ();
  63                                 command.Connection = connection;
  64                                 command.CommandText =
  65                                         "SELECT version, fingerprint FROM db_info";
  66                                 try {
  67                                         reader = ExecuteReaderOrWait (command);
  68                                 } catch (Exception ex) {
  69                                         create_new_db = true;
  70                                 }
  71                                 if (reader != null && ! create_new_db) {
  72                                         if (ReadOrWait (reader)) {
  73                                                 stored_version = reader.GetInt32 (0);
  74                                                 stored_fingerprint = reader.GetString (1);
  75                                         }
  76                                         reader.Close ();
  77                                 }
  78                                 command.Dispose ();
  79
  80                                 if (VERSION != stored_version
  81                                     || (index_fingerprint != null && index_fingerprint != stored_fingerprint))
  82                                         create_new_db = true;
  83                         }
  84
  85                         if (create_new_db) {
  86                                 if (connection != null)
  87                                         connection.Dispose ();
  88                                 File.Delete (GetDbPath (directory));
  89                                 connection = Open (directory);
  90
  91                                 DoNonQuery ("CREATE TABLE db_info (             " +
  92                                             "  version       INTEGER NOT NULL,  " +
  93                                             "  fingerprint   STRING NOT NULL    " +
  94                                             ")");
  95
  96                                 DoNonQuery ("INSERT INTO db_info (version, fingerprint) VALUES ({0}, '{1}')",
  97                                             VERSION, index_fingerprint);
  98
  99                                 DoNonQuery ("CREATE TABLE file_attributes (           " +
 100                                             "  unique_id      STRING UNIQUE,          " +
 101                                             "  directory      STRING NOT NULL,        " +
 102                                             "  filename       STRING NOT NULL,        " +
 103                                             "  last_mtime     STRING NOT NULL,        " +
 104                                             "  last_indexed   STRING NOT NULL,        " +
 105                                             "  filter_name    STRING NOT NULL,        " +
 106                                             "  filter_version STRING NOT NULL         " +
 107                                             ")");
 108                         } else {
 109                                 SqliteCommand command;
 110                                 SqliteDataReader reader;
 111                                 int count = 0;
 112
 113                                 DateTime dt1 = DateTime.Now;
 114
 115                                 // Select all of the files and use them to populate our bit-vector.
 116                                 command = new SqliteCommand ();
 117                                 command.Connection = connection;
 118                                 command.CommandText = "SELECT directory, filename FROM file_attributes";
 119
 120                                 reader = ExecuteReaderOrWait (command);
 121
 122                                 while (ReadOrWait (reader)) {
 123
 124                                         string dir = reader.GetString (0);
 125                                         string file = reader.GetString (1);
 126                                         string path = Path.Combine (dir, file);
 127                                         SetPathFlag (path);
 128                                         ++count;
 129                                 }
 130
 131                                 reader.Close ();
 132                                 command.Dispose ();
 133
 134                                 DateTime dt2 = DateTime.Now;
 135
 136                                 Logger.Log.Debug ("Loaded {0} records from {1} in {2:0.000}s",
 137                                                  count, GetDbPath (directory), (dt2 - dt1).TotalSeconds);
 138                         }
 139
 140                         Shutdown.ShutdownEvent += Flush;
 141                 }
 142
 143                 ///////////////////////////////////////////////////////////////////
 144
 145                 private string GetDbPath (string directory)
 146                 {
 147                         return Path.Combine (directory, "FileAttributesStore.db");
 148                 }
 149
 150                 private SqliteConnection Open (string directory)
 151                 {
 152                         SqliteConnection c;
 153                         c = new SqliteConnection ();
 154                         c.ConnectionString = "URI=file:" + GetDbPath (directory);
 155                         c.Open ();
 156                         return c;
 157                 }
 158
 159                 private void DoNonQuery (string format, params object [] args)
 160                 {
 161                         SqliteCommand command;
 162                         command = new SqliteCommand ();
 163                         command.Connection = connection;
 164                         command.CommandText = String.Format (format, args);
 165
 166                         while (true) {
 167                                 try {
 168                                         command.ExecuteNonQuery ();
 169                                         break;
 170                                 } catch (SqliteException ex) {
 171                                         if (ex.SqliteError == SqliteError.BUSY)
 172                                                 Thread.Sleep (50);
 173                                         else
 174                                                 throw ex;
 175                                 }
 176                         }
 177
 178                         command.Dispose ();
 179                 }
 180
 181                 private SqliteCommand QueryCommand (string where_format, params object [] where_args)
 182                 {
 183                         SqliteCommand command;
 184                         command = new SqliteCommand ();
 185                         command.Connection = connection;
 186                         command.CommandText =
 187                                 "SELECT unique_id, directory, filename, last_mtime, last_indexed, filter_name, filter_version " +
 188                                 "FROM file_attributes WHERE " +
 189                                 String.Format (where_format, where_args);
 190                         return command;
 191                 }
 192
 193                 static private SqliteDataReader ExecuteReaderOrWait (SqliteCommand command)
 194                 {
 195                         SqliteDataReader reader = null;
 196                         while (reader == null) {
 197                                 try {
 198                                         reader = command.ExecuteReader ();
 199                                 } catch (SqliteException ex) {
 200                                         if (ex.SqliteError == SqliteError.BUSY)
 201                                                 Thread.Sleep (50);
 202                                         else
 203                                                 throw ex;
 204                                 }
 205                         }
 206                         return reader;
 207                 }
 208
 209                 static private bool ReadOrWait (SqliteDataReader reader)
 210                 {
 211                         while (true) {
 212                                 try {
 213                                         return reader.Read ();
 214                                 } catch (SqliteException ex) {
 215                                         if (ex.SqliteError == SqliteError.BUSY)
 216                                                 Thread.Sleep (50);
 217                                         else
 218                                                 throw ex;
 219                                 }
 220                         }
 221                 }
 222
 223                 private FileAttributes GetFromReader (SqliteDataReader reader)
 224                 {
 225                         FileAttributes attr = new FileAttributes ();
 226
 227                         attr.UniqueId = GuidFu.FromShortString (reader.GetString (0));
 228                         attr.Path = System.IO.Path.Combine (reader.GetString (1), reader.GetString (2));
 229                         attr.LastWriteTime = StringFu.StringToDateTime (reader.GetString (3));
 230                         attr.LastIndexedTime = StringFu.StringToDateTime (reader.GetString (4));
 231                         attr.FilterName = reader.GetString (5);
 232                         attr.FilterVersion = int.Parse (reader.GetString (6));
 233
 234                         return attr;
 235                 }
 236
 237                 ///////////////////////////////////////////////////////////////////
 238
 239                 private int GetPathHash (string path)
 240                 {
 241                         uint hash = 0xdeadbeef;
 242                         foreach (char c in path)
 243                                 hash = 17 * hash + (uint) c;
 244                         // Fold the 32 bits in 16.
 245                         return (int) ((hash & 0xffff) ^ (hash >> 16));
 246                 }
 247
 248                 private bool GetPathFlag (string path)
 249                 {
 250                         int hash = GetPathHash (path);
 251                         return path_flags [hash];
 252                 }
 253
 254                 private void SetPathFlag (string path)
 255                 {
 256                         int hash = GetPathHash (path);
 257                         path_flags [hash] = true;
 258                 }
 259
 260                 ///////////////////////////////////////////////////////////////////
 261
 262                 public FileAttributes Read (string path)
 263                 {
 264                         SqliteCommand command;
 265                         SqliteDataReader reader;
 266
 267                         if (! GetPathFlag (path))
 268                                 return null;
 269
 270                         FileAttributes attr = null;
 271                         bool found_too_many = false;
 272
 273                         // We need to quote any 's that appear in the strings
 274                         // (int particular, in the path)
 275                         string directory = Path.GetDirectoryName (path).Replace ("'", "''");
 276                         string filename = Path.GetFileName (path).Replace ("'", "''");
 277                         lock (connection) {
 278                                 command = QueryCommand ("directory='{0}' AND filename='{1}'",
 279                                                         directory, filename);
 280                                 reader = ExecuteReaderOrWait (command);
 281
 282                                 if (ReadOrWait (reader)) {
 283                                         attr = GetFromReader (reader);
 284
 285                                         if (ReadOrWait (reader))
 286                                                 found_too_many = true;
 287                                 }
 288                                 reader.Close ();
 289                                 command.Dispose ();
 290
 291                                 // If we found more than one matching record for a given
 292                                 // directory and filename, something has gone wrong.
 293                                 // Since we have no way of knowing which one is correct
 294                                 // and which isn't, we delete them all and return
 295                                 // null.  (Which in most cases will force a re-index.
 296                                 if (found_too_many) {
 297                                         DoNonQuery ("DELETE FROM file_attributes WHERE directory='{0}' AND filename='{1}'",
 298                                                     directory, filename);
 299                                 }
 300                         }
 301
 302                         return attr;
 303                 }
 304
 305                 public bool Write (FileAttributes fa)
 306                 {
 307                         SetPathFlag (fa.Path);
 308
 309                         // We need to quote any 's that appear in the strings
 310                         // (in particular, in the path)
 311                         lock (connection) {
 312                                 if (transaction_count == 0 && ! Shutdown.ShutdownRequested) {
 313                                         Logger.Log.Debug ("Beginning sqlite transaction");
 314                                         DoNonQuery ("BEGIN");
 315                                 }
 316
 317                                 DoNonQuery ("INSERT OR REPLACE INTO file_attributes " +
 318                                             " (unique_id, directory, filename, last_mtime, last_indexed, filter_name, filter_version) " +
 319                                             " VALUES ('{0}', '{1}', '{2}', '{3}', '{4}', '{5}', '{6}')",
 320                                             GuidFu.ToShortString (fa.UniqueId),
 321                                             fa.Directory.Replace ("'", "''"), fa.Filename.Replace ("'", "''"),
 322                                             StringFu.DateTimeToString (fa.LastWriteTime),
 323                                             StringFu.DateTimeToString (fa.LastIndexedTime),
 324                                             fa.FilterName,
 325                                             fa.FilterVersion);
 326
 327                                 if (! Shutdown.ShutdownRequested)
 328                                         ++transaction_count;
 329
 330                                 // 150 is a pretty arbitrary number
 331                                 if (transaction_count == 150) {
 332                                         Logger.Log.Debug ("Committing sqlite transaction");
 333                                         DoNonQuery ("COMMIT");
 334                                         transaction_count = 0;
 335                                 }
 336                         }
 337                         return true;
 338                 }
 339
 340                 public void Drop (string path)
 341                 {
 342                         // We don't want to "UnSetPathFlag" here, since we have no way of knowing
 343                         // if another path hashes to the same value as this one.
 344
 345                         // We need to quote any 's that appear in the strings
 346                         // (in particular, in the path)
 347                         string directory = Path.GetDirectoryName (path).Replace ("'", "''");
 348                         string filename = Path.GetFileName (path).Replace ("'", "''");
 349                         lock (connection) {
 350                                 DoNonQuery ("DELETE FROM file_attributes WHERE directory='{0}' AND filename='{1}'",
 351                                             directory, filename);
 352                         }
 353                 }
 354
 355                 public void Flush ()
 356                 {
 357                         lock (connection) {
 358                                 if (transaction_count > 0) {
 359                                         Logger.Log.Debug ("Flushing requested -- committing sqlite transaction");
 360                                         DoNonQuery ("COMMIT");
 361                                         transaction_count = 0;
 362                                 }
 363                         }
 364                 }
 365
 366                 ///////////////////////////////////////////////////////////////////
 367
 368                 // Return all attributes in the attributes database, used for merging
 369
 370                 private ICollection ReadAllAttributes ()
 371                 {
 372                         ArrayList attributes = new ArrayList ();
 373
 374                         SqliteCommand command;
 375                         SqliteDataReader reader;
 376
 377                         FileAttributes attr = null;
 378
 379                         lock (connection) {
 380                                 command = new SqliteCommand ();
 381                                 command.Connection = connection;
 382                                 command.CommandText =
 383                                         "SELECT unique_id, directory, filename, last_mtime, last_indexed, filter_name, filter_version " +
 384                                         "FROM file_attributes";
 385
 386                                 reader = ExecuteReaderOrWait (command);
 387
 388                                 while (ReadOrWait (reader)) {
 389                                         attributes.Add (GetFromReader (reader));
 390                                 }
 391                                 reader.Close ();
 392                                 command.Dispose ();
 393                         }
 394
 395                         return attributes;
 396                 }
 397
 398                 // FIXME: Might wanna do this a bit more intelligently
 399
 400                 public void Merge (FileAttributesStore_Sqlite fa_sqlite_store_to_merge)
 401                 {
 402                         ICollection attributes = fa_sqlite_store_to_merge.ReadAllAttributes ();
 403
 404                         foreach (FileAttributes attribute in attributes)
 405                                 Write (attribute);
 406                 }
 407         }
 408 }