beagled/FilterFactory.cs

   1 //
   2 // FilterFactory.cs
   3 //
   4 // Copyright (C) 2004 Novell, Inc.
   5 //
   6
   7 //
   8 // Permission is hereby granted, free of charge, to any person obtaining a
   9 // copy of this software and associated documentation files (the "Software"),
  10 // to deal in the Software without restriction, including without limitation
  11 // the rights to use, copy, modify, merge, publish, distribute, sublicense,
  12 // and/or sell copies of the Software, and to permit persons to whom the
  13 // Software is furnished to do so, subject to the following conditions:
  14 //
  15 // The above copyright notice and this permission notice shall be included in
  16 // all copies or substantial portions of the Software.
  17 //
  18 // THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  19 // IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  20 // FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
  21 // AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
  22 // LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
  23 // FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
  24 // DEALINGS IN THE SOFTWARE.
  25 //
  26
  27 using System;
  28 using System.Collections;
  29 using System.IO;
  30 using System.Reflection;
  31
  32 using System.Xml;
  33 using System.Xml.Serialization;
  34
  35 using Beagle.Util;
  36
  37 namespace Beagle.Daemon {
  38
  39         public class FilterFactory {
  40
  41                 static private bool Debug = false;
  42
  43                 static FilterFactory ()
  44                 {
  45                         ReflectionFu.ScanEnvironmentForAssemblies ("BEAGLE_FILTER_PATH", PathFinder.FilterDir,
  46                                                                    delegate (Assembly a) {
  47                                                                            int n = ScanAssemblyForFilters (a);
  48                                                                            Logger.Log.Debug ("Loaded {0} filter{1} from {2}",
  49                                                                                              n, n == 1 ? "" : "s", a.Location);
  50                                                                    });
  51                 }
  52
  53                 /////////////////////////////////////////////////////////////////////////
  54
  55
  56                 static private ICollection CreateFilters (Uri uri, string extension, string mime_type)
  57                 {
  58                         Hashtable matched_filters_by_flavor = FilterFlavor.NewHashtable ();
  59
  60                         foreach (FilterFlavor flavor in filter_types_by_flavor.Keys) {
  61                                 if (flavor.IsMatch (uri, extension, mime_type)) {
  62                                         Filter matched_filter = null;
  63
  64                                         try {
  65                                                 matched_filter = (Filter) Activator.CreateInstance ((Type) filter_types_by_flavor [flavor]);
  66
  67                                                 if (flavor.MimeType != null)
  68                                                         matched_filter.MimeType = flavor.MimeType;
  69                                                 if (flavor.Extension != null)
  70                                                         matched_filter.Extension = flavor.Extension;
  71
  72                                         } catch (Exception e) {
  73                                                 continue;
  74                                         }
  75                                         matched_filters_by_flavor [flavor] = matched_filter;
  76                                 }
  77                         }
  78
  79                         foreach (DictionaryEntry entry in matched_filters_by_flavor) {
  80                                 FilterFlavor flav = (FilterFlavor) entry.Key;
  81                                 Filter filter = (Filter) entry.Value;
  82
  83                                 if (Debug)
  84                                         Logger.Log.Debug ("Found matching filter: {0}, Weight: {1}", filter, flav.Weight);
  85                         }
  86
  87                         return matched_filters_by_flavor.Values;
  88                 }
  89
  90                 static public int GetFilterVersion (string filter_name)
  91                 {
  92                         if (filter_versions_by_name.Contains (filter_name)) {
  93                                 return (int) filter_versions_by_name [filter_name];
  94                         } else {
  95                                 return -1;
  96                         }
  97                 }
  98
  99                 /////////////////////////////////////////////////////////////////////////
 100
 101                 static public ICollection CreateFiltersFromMimeType (string mime_type)
 102                 {
 103                         return CreateFilters (null, null, mime_type);
 104                 }
 105
 106                 static public ICollection CreateFilterFromExtension (string extension)
 107                 {
 108                         return CreateFilters (null, extension, null);
 109                 }
 110
 111                 static public ICollection CreateFiltersFromPath (string path)
 112                 {
 113                         string guessed_mime_type = XdgMime.GetMimeType (path);
 114                         string extension = Path.GetExtension (path);
 115                         return CreateFilters (UriFu.PathToFileUri (path), extension, guessed_mime_type);
 116                 }
 117
 118                 static public ICollection CreateFiltersFromUri (Uri uri)
 119                 {
 120                         if (uri.IsFile)
 121                                 return CreateFiltersFromPath (uri.LocalPath);
 122                         else
 123                                 return CreateFilters (uri, null, null);
 124                 }
 125
 126                 static public ICollection CreateFiltersFromIndexable (Indexable indexable)
 127                 {
 128                         string path = indexable.ContentUri.LocalPath;
 129                         string extension = Path.GetExtension (path);
 130                         string mime_type = indexable.MimeType;
 131                         return CreateFilters (UriFu.PathToFileUri (path), extension, mime_type);
 132                 }
 133
 134                 /////////////////////////////////////////////////////////////////////////
 135
 136                 static public TextReader FilterFile (string path)
 137                 {
 138                         return FilterFile (path, null);
 139                 }
 140
 141                 static public TextReader FilterFile (string path, string mime_type)
 142                 {
 143                         if (mime_type == null)
 144                                 mime_type = XdgMime.GetMimeType (path);
 145                         else {
 146                                 // Mime types are all lower-case by convention
 147                                 // and all the filters expect them that way,
 148                                 // but we can't trust that they already are
 149                                 // when we get it from somewhere other than
 150                                 // xdgmime.  They are often capitalized in
 151                                 // emails, for example.
 152
 153                                 mime_type = mime_type.ToLower ();
 154                         }
 155
 156                         if (mime_type == null)
 157                                 return null;
 158
 159                         ICollection filters = CreateFilters (UriFu.PathToFileUri (path), Path.GetExtension (path), mime_type);
 160                         TextReader reader;
 161
 162                         foreach (Filter candidate_filter in filters) {
 163                                 if (Debug)
 164                                         Logger.Log.Debug ("Testing filter: {0}", candidate_filter);
 165
 166                                 // Open the filter, and hook up the TextReader.
 167                                 if (candidate_filter.Open (path)) {
 168                                         reader = candidate_filter.GetTextReader ();
 169
 170                                         if (Debug)
 171                                                 Logger.Log.Debug ("Successfully filtered {0} with {1}", path, candidate_filter);
 172
 173                                         return reader;
 174                                 } else if (Debug) {
 175                                         Logger.Log.Debug ("Unsuccessfully filtered {0} with {1}, falling back", path, candidate_filter);
 176                                 }
 177                         }
 178
 179                         return null;
 180                 }
 181
 182                 static private bool ShouldWeFilterThis (Indexable indexable)
 183                 {
 184                         if (indexable.Filtering == IndexableFiltering.Never
 185                             || indexable.NoContent)
 186                                 return false;
 187
 188                         if (indexable.Filtering == IndexableFiltering.Always)
 189                                 return true;
 190
 191                         // Our default behavior is to try to filter non-transient file
 192                         // indexable and indexables with a specific mime type attached.
 193                         if (indexable.IsNonTransient || indexable.MimeType != null)
 194                                 return true;
 195
 196                         return false;
 197                 }
 198
 199                 static public bool FilterIndexable (Indexable indexable, TextCache text_cache, out Filter filter)
 200                 {
 201                         filter = null;
 202                         ICollection filters = null;
 203
 204                         if (indexable.Filtering == IndexableFiltering.AlreadyFiltered)
 205                                 return false;
 206
 207                         if (! ShouldWeFilterThis (indexable)) {
 208                                 indexable.NoContent = true;
 209                                 return false;
 210                         }
 211
 212                         string path = null;
 213
 214                         // First, figure out which filter we should use to deal with
 215                         // the indexable.
 216
 217                         // If a specific mime type is specified, try to index as that type.
 218                         if (indexable.MimeType != null)
 219                                 filters = CreateFiltersFromMimeType (indexable.MimeType);
 220
 221                         if (indexable.ContentUri.IsFile) {
 222                                 path = indexable.ContentUri.LocalPath;
 223
 224                                 // Otherwise sniff the mime-type from the file
 225                                 if (indexable.MimeType == null)
 226                                         indexable.MimeType = XdgMime.GetMimeType (path);
 227
 228                                 if (filters == null || filters.Count == 0) {
 229                                         filters = CreateFiltersFromIndexable (indexable);
 230                                 }
 231
 232                                 if (Directory.Exists (path)) {
 233                                         indexable.MimeType = "inode/directory";
 234                                         indexable.NoContent = true;
 235                                         if (! indexable.ValidTimestamp)
 236                                                 indexable.Timestamp = Directory.GetLastWriteTimeUtc (path);
 237                                 } else if (File.Exists (path)) {
 238                                         // Set the timestamp to the best possible estimate (if no timestamp was set by the backend)
 239                                         if (! indexable.ValidTimestamp)
 240                                                 indexable.Timestamp = File.GetLastWriteTimeUtc (path);
 241                                 } else {
 242                                         Logger.Log.Warn ("No such file: {0}", path);
 243                                         return false;
 244                                 }
 245                         }
 246
 247                         // We don't know how to filter this, so there is nothing else to do.
 248                         if (filters.Count == 0) {
 249                                 if (! indexable.NoContent) {
 250                                         indexable.NoContent = true;
 251
 252                                         Logger.Log.Debug ("No filter for {0} ({1})", path != null ? path : indexable.Uri.ToString (), indexable.MimeType);
 253                                         return false;
 254                                 }
 255
 256                                 return true;
 257                         }
 258
 259                         foreach (Filter candidate_filter in filters) {
 260                                 if (Debug)
 261                                         Logger.Log.Debug ("Testing filter: {0}", candidate_filter);
 262
 263                                 // Hook up the snippet writer.
 264                                 if (candidate_filter.SnippetMode && text_cache != null) {
 265                                         if (candidate_filter.OriginalIsText && indexable.IsNonTransient) {
 266                                                 text_cache.MarkAsSelfCached (indexable.Uri);
 267                                         } else if (indexable.CacheContent) {
 268                                                 TextWriter writer = text_cache.GetWriter (indexable.Uri);
 269                                                 candidate_filter.AttachSnippetWriter (writer);
 270                                         }
 271                                 }
 272
 273                                 if (indexable.Crawled)
 274                                         candidate_filter.EnableCrawlMode ();
 275
 276                                 // Set the filter's URI
 277                                 candidate_filter.Uri = indexable.Uri;
 278
 279                                 // allow the filter access to the indexable's properties
 280                                 candidate_filter.IndexableProperties = indexable.Properties;
 281
 282                                 // Open the filter, copy the file's properties to the indexable,
 283                                 // and hook up the TextReaders.
 284
 285                                 bool succesful_open = false;
 286                                 TextReader text_reader;
 287                                 Stream binary_stream;
 288
 289                                 if (path != null)
 290                                         succesful_open = candidate_filter.Open (path);
 291                                 else if ((text_reader = indexable.GetTextReader ()) != null)
 292                                         succesful_open = candidate_filter.Open (text_reader);
 293                                 else if ((binary_stream = indexable.GetBinaryStream ()) != null)
 294                                         succesful_open = candidate_filter.Open (binary_stream);
 295
 296                                 if (succesful_open) {
 297                                         foreach (Property prop in candidate_filter.Properties)
 298                                                 indexable.AddProperty (prop);
 299                                         indexable.SetTextReader (candidate_filter.GetTextReader ());
 300                                         indexable.SetHotTextReader (candidate_filter.GetHotTextReader ());
 301
 302                                         if (Debug)
 303                                                 Logger.Log.Debug ("Successfully filtered {0} with {1}", path, candidate_filter);
 304
 305                                         filter = candidate_filter;
 306                                         return true;
 307                                 } else if (Debug) {
 308                                         Logger.Log.Debug ("Unsuccessfully filtered {0} with {1}, falling back", path, candidate_filter);
 309                                         candidate_filter.Cleanup ();
 310                                 }
 311                         }
 312
 313                         if (Debug)
 314                                 Logger.Log.Debug ("None of the matching filters could process the file: {0}", path);
 315
 316                         return false;
 317                 }
 318
 319                 static public bool FilterIndexable (Indexable indexable, out Filter filter)
 320                 {
 321                         return FilterIndexable (indexable, null, out filter);
 322                 }
 323
 324                 static public bool FilterIndexable (Indexable indexable)
 325                 {
 326                         Filter filter = null;
 327
 328                         return FilterIndexable (indexable, null, out filter);
 329                 }
 330
 331                 /////////////////////////////////////////////////////////////////////////
 332
 333                 private static Hashtable filter_types_by_flavor = new Hashtable ();
 334                 private static Hashtable filter_versions_by_name = new Hashtable ();
 335
 336                 static private int ScanAssemblyForFilters (Assembly assembly)
 337                 {
 338                         int count = 0;
 339
 340                         foreach (Type t in ReflectionFu.ScanAssemblyForClass (assembly, typeof (Filter))) {
 341                                 Filter filter = null;
 342
 343                                 try {
 344                                         filter = (Filter) Activator.CreateInstance (t);
 345                                 } catch (Exception ex) {
 346                                         Logger.Log.Error ("Caught exception while instantiating {0}", t);
 347                                         Logger.Log.Error (ex);
 348                                 }
 349
 350                                 if (filter == null)
 351                                         continue;
 352
 353                                 filter_versions_by_name [t.ToString ()] = filter.Version;
 354
 355                                 foreach (FilterFlavor flavor in filter.SupportedFlavors) {
 356                                         filter_types_by_flavor [flavor] = t;
 357                                         FilterFlavor.Flavors.Add (flavor);
 358                                 }
 359
 360                                 ++count;
 361                         }
 362
 363                         return count;
 364                 }
 365
 366         }
 367
 368         /////////////////////////////////////////////////////////////////////////
 369
 370         public class FilteredStatus
 371         {
 372                 private Uri uri;
 373                 private string filter_name;
 374                 private int filter_version;
 375
 376                 [XmlIgnore]
 377                 public Uri Uri {
 378                         get { return uri; }
 379                         set { uri = value; }
 380                 }
 381
 382                 [XmlAttribute ("Uri")]
 383                 public string UriAsString {
 384                         get {
 385                                 return UriFu.UriToSerializableString (uri);
 386                         }
 387
 388                         set {
 389                                 uri = UriFu.UriStringToUri (value);
 390                         }
 391                 }
 392
 393                 public string FilterName {
 394                         get { return filter_name; }
 395                         set { filter_name = value; }
 396                 }
 397
 398                 public int FilterVersion {
 399                         get { return filter_version; }
 400                         set { filter_version = value; }
 401                 }
 402
 403                 public static FilteredStatus New (Indexable indexable, Filter filter)
 404                 {
 405                         FilteredStatus status = new FilteredStatus ();
 406
 407                         status.Uri = indexable.Uri;
 408                         status.FilterName = filter.GetType ().ToString ();
 409                         status.FilterVersion = filter.Version;
 410
 411                         return status;
 412                 }
 413         }
 414 }