beagled/DumpIndex.cs

   1 //
   2 // DumpIndex.cs
   3 //
   4 // Copyright (C) 2004 Novell, Inc.
   5 //
   6
   7 //
   8 // Permission is hereby granted, free of charge, to any person obtaining a copy
   9 // of this software and associated documentation files (the "Software"), to deal
  10 // in the Software without restriction, including without limitation the rights
  11 // to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
  12 // copies of the Software, and to permit persons to whom the Software is
  13 // furnished to do so, subject to the following conditions:
  14 //
  15 // The above copyright notice and this permission notice shall be included in all
  16 // copies or substantial portions of the Software.
  17 //
  18 // THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  19 // IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  20 // FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
  21 // AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
  22 // LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
  23 // OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
  24 // SOFTWARE.
  25 //
  26
  27
  28 using System;
  29 using System.Collections;
  30 using System.IO;
  31 using System.Net;
  32
  33 using Beagle;
  34 using Beagle.Util;
  35 using Beagle.Daemon;
  36 using Hit = Beagle.Hit;
  37
  38 using Lucene.Net.Index;
  39 using Lucene.Net.Search;
  40 using Lucene.Net.Documents;
  41
  42 class DumpIndexTool {
  43
  44         public class HitByUriComparer : IComparer {
  45
  46                 public int Compare (object a, object b)
  47                 {
  48                         // All of this mapping to and from strings is dreadful.
  49                         return String.Compare (((Hit) a).Uri.ToString (), ((Hit) b).Uri.ToString ());
  50                 }
  51         }
  52
  53         static string RemapUriToPath (Hashtable all_hits_by_uri, Hit hit)
  54         {
  55                 string exact_name = hit.GetFirstProperty ("beagle:ExactFilename");
  56                 string parent_uri_str = hit.GetFirstProperty ("_private:ParentDirUri");
  57
  58                 if (parent_uri_str == null)
  59                         return exact_name;
  60                 else
  61                         return Path.Combine (RemapUriToPath (all_hits_by_uri, (Hit) all_hits_by_uri [parent_uri_str]),
  62                                              exact_name);
  63         }
  64
  65         static int DumpOneIndex_Metadata (string index_name, bool only_dump_the_urls)
  66         {
  67                 Console.WriteLine (); // a visual cue that something has changed
  68                 LuceneQueryingDriver driver;
  69                 driver = new LuceneQueryingDriver (index_name, -1, true);
  70
  71                 Hashtable all_hits_by_uri;
  72                 all_hits_by_uri = driver.GetAllHitsByUri ();
  73
  74                 ArrayList all_hits;
  75                 all_hits = new ArrayList (all_hits_by_uri.Values);
  76
  77                 if (index_name == "FileSystemIndex") // A hard-wired hack
  78                         foreach (Hit hit in all_hits)
  79                                 hit.Uri = UriFu.PathToFileUri (RemapUriToPath (all_hits_by_uri, hit));
  80
  81                 all_hits.Sort (new HitByUriComparer ());
  82
  83                 foreach (Hit hit in all_hits) {
  84
  85                         if (only_dump_the_urls) {
  86                                 Console.WriteLine ("{0}: {1}", index_name, hit.Uri);
  87                                 continue;
  88                         }
  89
  90                         Console.WriteLine (" Index: {0}", index_name);
  91                         Console.WriteLine ("   Uri: {0}", hit.Uri);
  92                         if (hit.ParentUri != null)
  93                                 Console.WriteLine ("Parent: {0}", hit.ParentUri);
  94                         Console.WriteLine (" MimeT: {0}", hit.MimeType);
  95                         Console.WriteLine ("  Type: {0}", hit.Type);
  96
  97                         ArrayList props;
  98                         props = new ArrayList (hit.Properties);
  99                         props.Sort ();
 100                         foreach (Property prop in props)
 101                                 if (! prop.Key.StartsWith ("_private:"))
 102                                     Console.WriteLine ("  Prop: {0} = '{1}'", prop.Key, prop.Value);
 103
 104                         Console.WriteLine ();
 105                 }
 106
 107                 return all_hits.Count;
 108         }
 109
 110         static Term initial_enum_term;
 111         // Dump the term frequencies: we do this via direct Lucene access.
 112         static void DumpOneIndex_TermFrequencies (string index_name)
 113         {
 114                 LuceneQueryingDriver driver;
 115                 driver = new LuceneQueryingDriver (index_name, -1, true);
 116
 117                 IndexReader reader;
 118                 reader = IndexReader.Open (driver.PrimaryStore);
 119
 120                 TermEnum term_enum;
 121                 term_enum = reader.Terms (initial_enum_term);
 122
 123                 int distinct_term_count = 0;
 124                 int term_count = 0;
 125
 126                 // from LuceneFAQ
 127                 // Terms are sorted first by field, then by text
 128                 // so all terms with a given field are adjacent in enumerations.
 129                 if (term_enum.Term () != null) {
 130                         while (term_enum.Term().Field() == "Text") {
 131                                 int freq;
 132                                 freq = term_enum.DocFreq ();
 133
 134                                 Console.WriteLine ("{0} {1} {2}", index_name, term_enum.Term ().Text (), freq);
 135
 136                                 // FIXME: spew these as a count
 137                                 ++distinct_term_count;
 138                                 term_count += freq;
 139
 140                                 if (!term_enum.Next ())
 141                                         break;
 142                         }
 143                 }
 144
 145                 term_enum.Close ();
 146                 reader.Close ();
 147
 148                 Console.WriteLine ();
 149         }
 150
 151         public class IndexInfo : IComparable {
 152                 public string Name;
 153                 public int    Count;
 154
 155                 public IndexInfo (string name)
 156                 {
 157                         Name = name;
 158                 }
 159
 160                 public int CompareTo (object obj)
 161                 {
 162                         IndexInfo other = (IndexInfo) obj;
 163                         return String.Compare (this.Name, other.Name);
 164                 }
 165         }
 166
 167         static void DumpIndexInformation (Mode mode, bool show_counts)
 168         {
 169                 ArrayList index_info_list;
 170                 index_info_list = new ArrayList ();
 171
 172                 DirectoryInfo dir;
 173                 dir = new DirectoryInfo (PathFinder.IndexDir);
 174                 foreach (DirectoryInfo subdir in dir.GetDirectories ())
 175                         index_info_list.Add (new IndexInfo (subdir.Name));
 176
 177                 index_info_list.Sort ();
 178
 179                 bool set_counts = false;
 180
 181                 if (mode == Mode.TermFrequencies)
 182                         initial_enum_term = new Term ("Text", "");
 183
 184                 foreach (IndexInfo info in index_info_list) {
 185                         if (mode == Mode.Uris || mode == Mode.Properties) {
 186                                 info.Count = DumpOneIndex_Metadata (info.Name, mode == Mode.Uris);
 187                                 set_counts = true;
 188                         } else {
 189                                 DumpOneIndex_TermFrequencies (info.Name);
 190                         }
 191                 }
 192
 193                 if (show_counts && set_counts) {
 194                         Console.WriteLine ();
 195                         Console.WriteLine ("FINAL COUNTS");
 196
 197                         foreach (IndexInfo info in index_info_list)
 198                                 Console.WriteLine ("{0} {1}", info.Count.ToString ().PadLeft (7), info.Name);
 199                 }
 200         }
 201
 202         class DummyQueryResult : IQueryResult {
 203                 public void Add (ICollection hits)
 204                 {
 205                 }
 206
 207                 public void Subtract (ICollection hits)
 208                 {
 209                 }
 210         }
 211
 212         static void DumpFileIndexInformation (string path, string indexdir)
 213         {
 214                 //Uri uri = UriFu.PathToFileUri (path);
 215                 //Console.WriteLine ("Dumping information about:" + uri.AbsolutePath);
 216                 //path = uri.AbsolutePath;
 217                 if ((! File.Exists (path)) && (! Directory.Exists (path))) {
 218                         Console.WriteLine ("No such file or directory:" + path);
 219                         return;
 220                 }
 221
 222                 if (indexdir == null)
 223                         // default is ~/.beagle/Indexes/FileSystemIndex
 224                         indexdir = Path.Combine (PathFinder.IndexDir, "FileSystemIndex");
 225                 if (! Directory.Exists (indexdir)) {
 226                         Console.WriteLine ("Index:{0} doesnt exist.", indexdir);
 227                         return;
 228                 }
 229
 230                 // get fingerprint
 231                 TextReader reader;
 232                 reader = new StreamReader (Path.Combine (indexdir, "fingerprint"));
 233                 string fingerprint = reader.ReadLine ();
 234                 reader.Close ();
 235                 //Console.WriteLine ("Read fingerprint:" + fingerprint);
 236
 237                 // find out uid
 238                 FileAttributesStore fa_store = new FileAttributesStore (new FileAttributesStore_Mixed (indexdir, fingerprint));
 239                 Beagle.Daemon.FileAttributes attr = fa_store.Read (path);
 240                 if (attr == null) {
 241                         Console.WriteLine ("No information about this file in index. Ignoring.");
 242                         return;
 243                 }
 244                 string uri_string = "uid:" + GuidFu.ToShortString (attr.UniqueId);
 245                 Console.WriteLine ("Uri = " + uri_string);
 246                 //Console.WriteLine ("FilterName:" + attr.FilterName);
 247                 Console.WriteLine ("LastAttrTime:" + attr.LastAttrTime);
 248                 Console.WriteLine ("LastWriteTime:" + attr.LastWriteTime);
 249
 250                 LuceneQueryingDriver driver;
 251                 driver = new LuceneQueryingDriver (indexdir, -1, true);
 252
 253
 254                 // first try for the Uri:"uid:xxxxxxxxxxxxxxx"
 255                 Lucene.Net.Search.Query query = new TermQuery(new Term("Uri", uri_string));
 256                 if (DoQuery (driver, query))
 257                         return;
 258
 259                 // else query by path - this is for static indexes
 260                 path = UriFu.PathToFileUriString (path);
 261                 Console.WriteLine ("Querying by:[" + path + "]");
 262                 query = new TermQuery(new Term("Uri", path));
 263                 DoQuery (driver, query);
 264
 265         }
 266
 267         static bool DoQuery (LuceneQueryingDriver driver, Lucene.Net.Search.Query query)
 268         {
 269                 IndexSearcher primary_searcher = LuceneCommon.GetSearcher (driver.PrimaryStore);
 270                 IndexSearcher secondary_searcher = LuceneCommon.GetSearcher (driver.SecondaryStore);
 271
 272                 Hits primary_hits = primary_searcher.Search(query);
 273                 Hits secondary_hits = secondary_searcher.Search (query);
 274                 Console.WriteLine ("{0} hits from primary store; {1} hits from secondary store", primary_hits.Length (), secondary_hits.Length ());
 275
 276                 Document primary_doc, secondary_doc;
 277                 // there should be exactly one primary hit and 0/1 secondary hit
 278                 if (primary_hits.Length () == 1) {
 279                         primary_doc = primary_hits.Doc (0);
 280                         Console.WriteLine (
 281                         "-----------------------------------------[ Immutable data ]--------------------------------------");
 282                         foreach (Field f in primary_doc.Fields ()) {
 283
 284                                 String name = f.Name ();
 285                                 String val = f.StringValue ();
 286                                 bool stored = f.IsStored ();
 287                                 bool searchable = (val [0] == 's');
 288                                 bool tokenized = f.IsTokenized();
 289                                 if (name.Length >= 7 && name.StartsWith ("prop:"))
 290                                         tokenized = (name [5] != 't');
 291                                 float boost = f.GetBoost();
 292
 293                                 Console.WriteLine ("{0,-30} = [{1}] (stored? {2}, searchable? {3}, tokenized? {4}, boost={5})",
 294                                                     name, val, stored, searchable, tokenized, boost);
 295                         }
 296                 }
 297
 298                 if (secondary_hits.Length () == 1) {
 299                         secondary_doc = secondary_hits.Doc (0);
 300                         Console.WriteLine (
 301                         "------------------------------------------[ Mutable data ]---------------------------------------");
 302                         foreach (Field f in secondary_doc.Fields ()) {
 303
 304                                 String name = f.Name ();
 305                                 String val = f.StringValue ();
 306                                 bool stored = f.IsStored ();
 307                                 bool searchable = (val [0] == 's');
 308                                 bool tokenized = f.IsTokenized();
 309                                 if (name.Length >= 7 && name.StartsWith ("prop:"))
 310                                         tokenized = (name [5] != 't');
 311                                 float boost = f.GetBoost();
 312
 313                                 Console.WriteLine ("{0,-30} = [{1}] (stored? {2}, searchable? {3}, tokenized? {4}, boost={5})",
 314                                                     name, val, stored, searchable, tokenized, boost);
 315                         }
 316                 }
 317
 318                 LuceneCommon.ReleaseSearcher (primary_searcher);
 319                 LuceneCommon.ReleaseSearcher (secondary_searcher);
 320
 321                 if (primary_hits.Length () != 0 || secondary_hits.Length () != 0)
 322                         return true;
 323                 else
 324                         return false;
 325         }
 326
 327         enum Mode {
 328                 Uris,
 329                 Properties,
 330                 TermFrequencies
 331         }
 332
 333
 334         static void Main (string [] args)
 335         {
 336                 Mode mode = Mode.Uris;
 337                 bool show_counts = true;
 338                 string file = null;
 339                 string indexdir = null;
 340
 341                 foreach (string arg in args) {
 342
 343                         switch (arg) {
 344
 345                         case "--help":
 346                                 Console.WriteLine (@"
 347 beagle-dump-index [options] [ [--indexdir=dir] file]
 348
 349 --uris                   Dump all Uris (default)
 350 --properties             Dump all properties
 351 --term-frequencies       Dump term frequencies
 352
 353 --show-counts            Show index count totals (default)
 354 --hide-counts            Hide index count totals
 355
 356 --indexdir=<index directory>
 357                          Absolute path of the directory storing the index
 358                          e.g. /home/user/.beagle/Indexes/FileSystemIndex
 359 file                     Get information in index about this file or directory
 360
 361 --help                         What you just did");
 362                                 Environment.Exit (0);
 363                                 break;
 364
 365                         case "--uris":
 366                                 mode = Mode.Uris;
 367                                 break;
 368
 369                         case "--properties":
 370                                 mode = Mode.Properties;
 371                                 break;
 372
 373                         case "--term-frequencies":
 374                                 mode = Mode.TermFrequencies;
 375                                 break;
 376
 377                         case "--hide-counts":
 378                                 show_counts = false;
 379                                 break;
 380
 381                         case "--show-counts":
 382                                 show_counts = false;
 383                                 break;
 384
 385                         default:
 386                                 if (arg.StartsWith ("--indexdir="))
 387                                         indexdir = arg.Remove (0, 11);
 388                                 else
 389                                         file = arg;
 390                                 break;
 391                         }
 392                 }
 393
 394                 if (file == null)
 395                         DumpIndexInformation (mode, show_counts);
 396                 else
 397                         DumpFileIndexInformation (file, indexdir);
 398
 399         }
 400 }