beagled/LifereaQueryable/LifereaQueryable.cs

   1 //
   2 // LifereaQueryable.cs
   3 //
   4 // Copyright (C) 2005 Carl-Emil Lagerstedt
   5 // Copyright (C) 2005 Novell, Inc.
   6 //
   7 //
   8 // Permission is hereby granted, free of charge, to any person obtaining a
   9 // copy of this software and associated documentation files (the "Software"),
  10 // to deal in the Software without restriction, including without limitation
  11 // the rights to use, copy, modify, merge, publish, distribute, sublicense,
  12 // and/or sell copies of the Software, and to permit persons to whom the
  13 // Software is furnished to do so, subject to the following conditions:
  14 //
  15 // The above copyright notice and this permission notice shall be included in
  16 // all copies or substantial portions of the Software.
  17 //
  18 // THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  19 // IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  20 // FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
  21 // AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
  22 // LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
  23 // FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
  24 // DEALINGS IN THE SOFTWARE.
  25 //
  26
  27 using System;
  28 using System.IO;
  29 using System.Collections;
  30 using System.Threading;
  31
  32 using System.Xml;
  33 using System.Xml.Serialization;
  34
  35 using Beagle.Daemon;
  36 using Beagle.Util;
  37
  38 namespace Beagle.Daemon.LifereaQueryable {
  39
  40         [QueryableFlavor (Name="Liferea", Domain=QueryDomain.Local, RequireInotify=false)]
  41         public class LifereaQueryable : LuceneFileQueryable, IIndexableGenerator {
  42
  43                 private static Logger log = Logger.Get ("LifereaQueryable");
  44
  45                 string liferea_dir;
  46
  47                 public LifereaQueryable () : base ("LifereaIndex")
  48                 {
  49                         liferea_dir = Path.Combine (PathFinder.HomeDir, ".liferea");
  50                         liferea_dir = Path.Combine (liferea_dir, "cache");
  51                         liferea_dir = Path.Combine (liferea_dir, "feeds");
  52                 }
  53
  54                 /////////////////////////////////////////////////
  55
  56                 public override void Start ()
  57                 {
  58                         base.Start ();
  59
  60                         ExceptionHandlingThread.Start (new ThreadStart (StartWorker));
  61                 }
  62
  63                 private void StartWorker ()
  64                 {
  65                         if (!Directory.Exists (liferea_dir)) {
  66                                 GLib.Timeout.Add (60000, new GLib.TimeoutHandler (CheckForExistence));
  67                                 return;
  68                         }
  69
  70                         if (Inotify.Enabled) {
  71                                 Inotify.EventType mask = Inotify.EventType.CloseWrite;
  72
  73                                 Inotify.Subscribe (liferea_dir, OnInotifyEvent, mask);
  74                         } else {
  75                                 FileSystemWatcher fsw = new FileSystemWatcher ();
  76                                 fsw.Path = liferea_dir;
  77
  78                                 fsw.Changed += new FileSystemEventHandler (OnChanged);
  79                                 fsw.Created += new FileSystemEventHandler (OnChanged);
  80
  81                                 fsw.EnableRaisingEvents = true;
  82                         }
  83
  84                         log.Info ("Scanning Liferea feeds...");
  85
  86                         Stopwatch stopwatch = new Stopwatch ();
  87                         stopwatch.Start ();
  88
  89                         DirectoryInfo dir = new DirectoryInfo (liferea_dir);
  90                         this.files_to_parse = dir.GetFiles ();
  91
  92                         Scheduler.Task task = NewAddTask (this);
  93                         task.Tag = "Liferea";
  94                         ThisScheduler.Add (task);
  95
  96                         stopwatch.Stop ();
  97                         log.Info ("{0} files will be parsed (scanned in {1})", this.files_to_parse.Count, stopwatch);
  98                 }
  99
 100                 private bool CheckForExistence ()
 101                 {
 102                         if (!Directory.Exists (liferea_dir))
 103                                 return true;
 104
 105                         this.Start ();
 106
 107                         return false;
 108                 }
 109
 110                 /////////////////////////////////////////////////
 111
 112                 // Modified/Created event using Inotify
 113
 114                 private void OnInotifyEvent (Inotify.Watch watch,
 115                                              string path,
 116                                              string subitem,
 117                                              string srcpath,
 118                                              Inotify.EventType type)
 119                 {
 120                         if (subitem == "")
 121                                 return;
 122
 123                         IndexSingleFeed (Path.Combine (path, subitem), Scheduler.Priority.Immediate);
 124                 }
 125
 126                 // Modified/Created event using FSW
 127
 128                 private void OnChanged (object o, FileSystemEventArgs args)
 129                 {
 130                         IndexSingleFeed (args.FullPath, Scheduler.Priority.Immediate);
 131                 }
 132
 133                 /////////////////////////////////////////////////
 134
 135                 private Indexable FeedItemToIndexable (Feed feed, Item item, FileInfo file)
 136                 {
 137                         Indexable indexable = new Indexable (new Uri (String.Format ("{0};item={1}", feed.Source, item.Source)));
 138                         indexable.ParentUri = UriFu.PathToFileUri (file.FullName);
 139                         indexable.MimeType = "text/html";
 140                         indexable.Type = "FeedItem";
 141
 142                         DateTime date = new DateTime (1970, 1, 1);
 143                         date = date.AddSeconds (item.Timestamp);
 144                         date = TimeZone.CurrentTimeZone.ToLocalTime (date);
 145
 146                         indexable.Timestamp = date;
 147
 148                         indexable.AddProperty (Property.New ("dc:title", item.Title));
 149                         indexable.AddProperty (Property.New ("fixme:author", item.Attribs.Author));
 150                         indexable.AddProperty (Property.NewDate ("fixme:published", date));
 151                         indexable.AddProperty (Property.NewKeyword ("fixme:itemuri", item.Source));
 152                         indexable.AddProperty (Property.NewKeyword ("fixme:webloguri", feed.Source));
 153
 154                         StringReader reader = new StringReader (item.Description);
 155                         indexable.SetTextReader (reader);
 156
 157                         return indexable;
 158                 }
 159
 160                 // Parse and index a single feed
 161
 162                 private int IndexSingleFeed (string filename, Scheduler.Priority priority)
 163                 {
 164                         FileInfo file = new FileInfo(filename);
 165
 166                         Feed feed;
 167                         int item_count = 0;
 168
 169                         if (IsUpToDate (file.FullName))
 170                                 return 0;
 171
 172                         feed = Feed.LoadFromFile (file.FullName);
 173
 174                         if (feed == null || feed.Items == null)
 175                                 return 0;
 176
 177                         foreach (Item item in feed.Items) {
 178                                 item_count++;
 179
 180                                 Indexable indexable = FeedItemToIndexable (feed, item, file);
 181
 182                                 Scheduler.Task task = NewAddTask (indexable);
 183                                 task.Priority = priority;
 184                                 task.SubPriority = 0;
 185                                 ThisScheduler.Add (task);
 186                         }
 187
 188                         return item_count;
 189                 }
 190
 191                 ////////////////////////////////////////////////
 192
 193                 // IIndexableGenerator implementation
 194
 195                 private ICollection files_to_parse;
 196                 private IEnumerator file_enumerator = null;
 197                 private IEnumerator item_enumerator = null;
 198                 private Feed current_feed;
 199
 200                 public Indexable GetNextIndexable ()
 201                 {
 202                         Item item = (Item) this.item_enumerator.Current;
 203                         FileInfo file = (FileInfo) this.file_enumerator.Current;
 204                         return FeedItemToIndexable (this.current_feed, item, file);
 205                 }
 206
 207                 public bool HasNextIndexable ()
 208                 {
 209                         if (this.files_to_parse.Count == 0)
 210                                 return false;
 211
 212                         while (this.item_enumerator == null || !this.item_enumerator.MoveNext ()) {
 213                                 if (this.file_enumerator == null)
 214                                         this.file_enumerator = this.files_to_parse.GetEnumerator ();
 215
 216                                 do {
 217                                         if (!this.file_enumerator.MoveNext ())
 218                                                 return false;
 219
 220                                         FileInfo file = (FileInfo) this.file_enumerator.Current;
 221
 222                                         if (IsUpToDate (file.FullName))
 223                                                 continue;
 224
 225                                         Feed feed = Feed.LoadFromFile (file.FullName);
 226
 227                                         if (feed == null || feed.Items == null)
 228                                                 continue;
 229
 230                                         this.current_feed = feed;
 231
 232                                 } while (this.current_feed == null);
 233
 234                                 this.item_enumerator = this.current_feed.Items.GetEnumerator ();
 235                         }
 236
 237                         return true;
 238                 }
 239
 240                 public string StatusName {
 241                         get { return null; }
 242                 }
 243
 244         }
 245
 246         ////////////////////////////////////////////////
 247
 248         // De-serialization classes
 249         // FIXME: Change to standard stream parsing for performance?
 250
 251         public class Item {
 252                 [XmlElement ("title")] public string Title = "";
 253                 [XmlElement ("description")] public string Description ="";
 254                 [XmlElement ("source")] public string Source="";
 255                 [XmlElement ("attributes")] public Attributes Attribs;
 256                 [XmlElement ("time")] public ulong Timestamp;
 257         }
 258
 259         public class Attributes{
 260                 [XmlAttribute ("author")] public string Author = "";
 261         }
 262
 263         public class Feed{
 264                 [XmlElement ("feedTitle")] public string Title="";
 265                 [XmlElement ("feedSource")] public string Source="";
 266                 [XmlElement ("feedDescription")] public string Description="";
 267
 268                 [XmlElement ("feedStatus")] public int Status;
 269                 [XmlElement ("feedUpdateInterval")] public int UpdateInterval;
 270                 [XmlElement ("feedDiscontinued")] public string Discontinued ="";
 271                 [XmlElement ("feedLastModified")] public string LastModified ="";
 272
 273                 [XmlElement ("item", typeof (Item))]
 274                 public ArrayList Items {
 275                         get { return mItems; }
 276                         set { mItems = value; }
 277                 }
 278
 279                 private ArrayList mItems = new ArrayList ();
 280
 281                 public static Feed LoadFromFile (string filename) {
 282                         Feed f;
 283                         XmlRootAttribute xRoot = new XmlRootAttribute();
 284                         xRoot.ElementName = "feed";
 285
 286                         XmlSerializer serializer = new XmlSerializer (typeof (Feed), xRoot);
 287                         Stream stream = new FileStream (filename,
 288                                                         FileMode.Open,
 289                                                         FileAccess.Read,
 290                                                         FileShare.Read);
 291                         XmlTextReader reader = new XmlTextReader (stream);
 292
 293                         if (!serializer.CanDeserialize(reader) )
 294                                 Console.WriteLine ("Muopp");
 295                         f = (Feed) serializer.Deserialize (reader);
 296
 297                         reader.Close ();
 298                         stream.Close ();
 299                         return f;
 300                 }
 301         }
 302 }