beagled/AkregatorQueryable/AkregatorQueryable.cs

   1 //
   2 // AkregatorQueryable.cs
   3 //
   4 // Copyright (C) 2005 Debajyoti Bera
   5 //
   6 //
   7 // Permission is hereby granted, free of charge, to any person obtaining a
   8 // copy of this software and associated documentation files (the "Software"),
   9 // to deal in the Software without restriction, including without limitation
  10 // the rights to use, copy, modify, merge, publish, distribute, sublicense,
  11 // and/or sell copies of the Software, and to permit persons to whom the
  12 // Software is furnished to do so, subject to the following conditions:
  13 //
  14 // The above copyright notice and this permission notice shall be included in
  15 // all copies or substantial portions of the Software.
  16 //
  17 // THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  18 // IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  19 // FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
  20 // AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
  21 // LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
  22 // FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
  23 // DEALINGS IN THE SOFTWARE.
  24 //
  25
  26 using System;
  27 using System.IO;
  28 using System.Collections;
  29 using System.Threading;
  30 using System.Text;
  31
  32 using System.Xml;
  33 using System.Xml.Serialization;
  34
  35 using Beagle.Daemon;
  36 using Beagle.Util;
  37
  38 namespace Beagle.Daemon.AkregatorQueryable {
  39
  40         [QueryableFlavor (Name="Akregator", Domain=QueryDomain.Local, RequireInotify=false)]
  41         public class AkregatorQueryable : LuceneFileQueryable, IIndexableGenerator  {
  42
  43                 private static Logger log = Logger.Get ("AkregatorQueryable");
  44
  45                 string akregator_dir;
  46
  47                 public AkregatorQueryable () : base ("AkregatorIndex")
  48                 {
  49                         akregator_dir = Path.Combine (PathFinder.HomeDir, ".kde");
  50                         akregator_dir = Path.Combine (akregator_dir, "share");
  51                         akregator_dir = Path.Combine (akregator_dir, "apps");
  52                         akregator_dir = Path.Combine (akregator_dir, "akregator");
  53                         akregator_dir = Path.Combine (akregator_dir, "Archive");
  54                 }
  55
  56                 /////////////////////////////////////////////////
  57
  58                 public override void Start ()
  59                 {
  60                         base.Start ();
  61
  62                         ExceptionHandlingThread.Start (new ThreadStart (StartWorker));
  63                 }
  64
  65                 private void StartWorker ()
  66                 {
  67                         if (!Directory.Exists (akregator_dir)) {
  68                                 GLib.Timeout.Add (60000, new GLib.TimeoutHandler (CheckForExistence));
  69                                 return;
  70                         }
  71
  72                         if (Inotify.Enabled) {
  73                                 Inotify.EventType mask = Inotify.EventType.CloseWrite;
  74
  75                                 Inotify.Subscribe (akregator_dir, OnInotifyEvent, mask);
  76                         } else {
  77                                 FileSystemWatcher fsw = new FileSystemWatcher ();
  78                                 fsw.Path = akregator_dir;
  79
  80                                 fsw.Changed += new FileSystemEventHandler (OnChanged);
  81                                 fsw.Created += new FileSystemEventHandler (OnChanged);
  82
  83                                 fsw.EnableRaisingEvents = true;
  84                         }
  85
  86                         log.Info ("Scanning Akregator feeds...");
  87
  88                         Stopwatch stopwatch = new Stopwatch ();
  89                         int feed_count = 0, item_count = 0;
  90
  91                         stopwatch.Start ();
  92
  93                         DirectoryInfo dir = new DirectoryInfo (akregator_dir);
  94                         this.files_to_parse = dir.GetFiles ();
  95                         Scheduler.Task task = NewAddTask (this);
  96                         task.Tag = "Akregator";
  97                         ThisScheduler.Add (task);
  98
  99                         stopwatch.Stop ();
 100                         log.Info ("{0} files will be parsed (scanned in {1})", this.files_to_parse.Count, stopwatch);
 101                 }
 102
 103                 private bool CheckForExistence ()
 104                 {
 105                         if (!Directory.Exists (akregator_dir))
 106                                 return true;
 107
 108                         this.Start ();
 109
 110                         return false;
 111                 }
 112
 113                 /////////////////////////////////////////////////
 114
 115                 // Modified/Created event using Inotify
 116
 117                 private void OnInotifyEvent (Inotify.Watch watch,
 118                                              string path,
 119                                              string subitem,
 120                                              string srcpath,
 121                                              Inotify.EventType type)
 122                 {
 123                         if (subitem == "")
 124                                 return;
 125
 126                         IndexSingleFeed (Path.Combine (path, subitem), Scheduler.Priority.Immediate);
 127                 }
 128
 129                 // Modified/Created event using FSW
 130
 131                 private void OnChanged (object o, FileSystemEventArgs args)
 132                 {
 133                         IndexSingleFeed (args.FullPath, Scheduler.Priority.Immediate);
 134                 }
 135
 136                 /////////////////////////////////////////////////
 137
 138                 private bool IsFeedDeleted (Channel channel, Item item)
 139                 {
 140                         for (int i=0; i<item.MetaList.Count; ++i) {
 141                             MetaInfo meta = (MetaInfo)item.MetaList[i];
 142                             if (meta.Type == "deleted" && meta.value == "true") {
 143                                     return true;
 144                             }
 145                         }
 146                         return false;
 147                 }
 148
 149                 private Indexable FeedItemToIndexable (Channel channel, Item item, FileInfo file)
 150                 {
 151                         Indexable indexable = new Indexable (new Uri (String.Format ("feed:{0};item={1}", channel.Link, item.Link)));
 152                         indexable.ParentUri = UriFu.PathToFileUri (file.FullName);
 153                         indexable.MimeType = "text/html";
 154                         indexable.Type = "FeedItem";
 155
 156                         int offset; //will be ignored - only store the time at current machine
 157                         DateTime date = GMime.Utils.HeaderDecodeDate (item.PubDate, out offset);
 158
 159                         indexable.Timestamp = date;
 160
 161                         indexable.AddProperty (Property.New ("dc:title", item.Title));
 162                         indexable.AddProperty (Property.NewDate ("fixme:published", date));
 163                         indexable.AddProperty (Property.NewKeyword ("fixme:itemuri", item.Link));
 164                         indexable.AddProperty (Property.NewKeyword ("fixme:webloguri", channel.Link));
 165
 166                         StringReader reader = new StringReader (item.Description);
 167                         indexable.SetTextReader (reader);
 168
 169                         return indexable;
 170                 }
 171                 // Parse and index a single feed
 172
 173                 private int IndexSingleFeed (string filename, Scheduler.Priority priority)
 174                 {
 175                         FileInfo file = new FileInfo(filename);
 176
 177                         RSS feed;
 178                         int item_count = 0;
 179
 180                         if (IsUpToDate (file.FullName))
 181                                 return 0;
 182
 183                         feed = RSS.LoadFromFile(file.FullName);
 184
 185                         if(feed == null || feed.channel == null || feed.channel.Items == null)
 186                                 return 0;
 187
 188                         foreach (Item item in feed.channel.Items) {
 189                                 if (IsFeedDeleted (feed.channel, item))
 190                                         continue;
 191
 192                                 item_count++;
 193
 194                                 Indexable indexable = FeedItemToIndexable (feed.channel, item, file);
 195
 196                                 Scheduler.Task task = NewAddTask (indexable);
 197                                 task.Priority = priority;
 198                                 task.SubPriority = 0;
 199                                 ThisScheduler.Add (task);
 200
 201                         }
 202
 203                         return item_count;
 204                 }
 205
 206                 ////////////////////////////////////////////////
 207
 208                 // IIndexableGenerator implementation
 209
 210                 private ICollection files_to_parse;
 211                 private IEnumerator file_enumerator = null;
 212                 private IEnumerator item_enumerator = null;
 213                 private RSS current_feed;
 214
 215                 public Indexable GetNextIndexable ()
 216                 {
 217                         Item item = (Item) this.item_enumerator.Current;
 218                         FileInfo file = (FileInfo) this.file_enumerator.Current;
 219                         // FIXME: We should find the next valid feed and return that
 220                         // that wont waste unnecessary function calls
 221                         // but that would need to handle HasNextIndexable as well
 222                         // Right now we return null as LuceneQueryable can handle null
 223                         if (IsFeedDeleted (this.current_feed.channel, item))
 224                                 return null;
 225
 226                         return FeedItemToIndexable (this.current_feed.channel, item, file);
 227                 }
 228
 229                 public bool HasNextIndexable ()
 230                 {
 231                         if (this.files_to_parse.Count == 0)
 232                                 return false;
 233
 234                         while (this.item_enumerator == null || !this.item_enumerator.MoveNext ()) {
 235                                 if (this.file_enumerator == null)
 236                                         this.file_enumerator = this.files_to_parse.GetEnumerator ();
 237
 238                                 do {
 239                                         if (!this.file_enumerator.MoveNext ())
 240                                                 return false;
 241
 242                                         FileInfo file = (FileInfo) this.file_enumerator.Current;
 243
 244                                         if (IsUpToDate (file.FullName))
 245                                                 continue;
 246
 247                                         RSS feed = RSS.LoadFromFile (file.FullName);
 248
 249                                         if (feed == null || feed.channel == null || feed.channel.Items == null)
 250                                                 continue;
 251
 252                                         this.current_feed = feed;
 253
 254                                 } while (this.current_feed == null);
 255
 256                                 this.item_enumerator = this.current_feed.channel.Items.GetEnumerator ();
 257                         }
 258
 259                         return true;
 260                 }
 261
 262                 public string StatusName {
 263                         get { return null; }
 264                 }
 265
 266         }
 267
 268         ////////////////////////////////////////////////
 269
 270         // De-serialization classes
 271         // Changing to standard stream parsing will increse performance no doubt
 272         // but not sure if it will be noticable
 273
 274         public class MetaInfo {
 275                 [XmlText]
 276                 public string value = "";
 277                 [XmlAttribute ("type")] public string Type = "";
 278         }
 279
 280         public class Item {
 281                 [XmlElement ("pubDate")] public string PubDate;
 282                 [XmlElement ("title")] public string Title = "";
 283                 [XmlElement ("description")] public string Description ="";
 284                 [XmlElement ("link")] public string Link="";
 285                 [XmlElement ("meta", typeof (MetaInfo), Namespace="http://foobar")]
 286                 public ArrayList MetaList {
 287                     get { return metaList; }
 288                     set { metaList = value; }
 289                 }
 290                 private ArrayList metaList = new ArrayList ();
 291         }
 292
 293         public class Channel{
 294                 [XmlElement ("title")] public string Title="";
 295                 [XmlElement ("link")] public string Link="";
 296                 [XmlElement ("description")] public string Description="";
 297
 298
 299                 [XmlElement ("item", typeof (Item))]
 300                 public ArrayList Items {
 301                         get { return mItems; }
 302                         set { mItems = value; }
 303                 }
 304                 private ArrayList mItems = new ArrayList ();
 305         }
 306
 307         public class RSS{
 308                 [XmlElement ("channel", typeof (Channel))]
 309                 public Channel channel;
 310
 311                 public static RSS LoadFromFile (string filename) {
 312                         RSS f;
 313                         XmlRootAttribute xRoot = new XmlRootAttribute();
 314                         xRoot.ElementName = "rss";
 315
 316                         XmlSerializer serializer = new XmlSerializer (typeof (RSS), xRoot);
 317                         Stream stream = new FileStream (filename,
 318                                                         FileMode.Open,
 319                                                         FileAccess.Read,
 320                                                         FileShare.Read);
 321                         XmlTextReader reader = new XmlTextReader (stream);
 322
 323                         if (!serializer.CanDeserialize(reader) )
 324                                 Console.WriteLine ("Muopp");
 325                         f = (RSS) serializer.Deserialize (reader);
 326
 327                         reader.Close ();
 328                         stream.Close ();
 329                         return f;
 330                 }
 331
 332         }
 333 }