beagled/DumpIndex.cs

   1 //
   2 // DumpIndex.cs
   3 //
   4 // Copyright (C) 2004 Novell, Inc.
   5 //
   6
   7 //
   8 // Permission is hereby granted, free of charge, to any person obtaining a copy
   9 // of this software and associated documentation files (the "Software"), to deal
  10 // in the Software without restriction, including without limitation the rights
  11 // to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
  12 // copies of the Software, and to permit persons to whom the Software is
  13 // furnished to do so, subject to the following conditions:
  14 //
  15 // The above copyright notice and this permission notice shall be included in all
  16 // copies or substantial portions of the Software.
  17 //
  18 // THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  19 // IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  20 // FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
  21 // AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
  22 // LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
  23 // OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
  24 // SOFTWARE.
  25 //
  26
  27
  28 using System;
  29 using System.Collections;
  30 using System.IO;
  31 using System.Net;
  32
  33 using Beagle;
  34 using Beagle.Util;
  35 using Beagle.Daemon;
  36
  37 using Lucene.Net.Index;
  38 using Lucene.Net.Search;
  39 using Lucene.Net.Documents;
  40
  41 class DumpIndexTool {
  42
  43         public class HitByUriComparer : IComparer {
  44
  45                 public int Compare (object a, object b)
  46                 {
  47                         // All of this mapping to and from strings is dreadful.
  48                         return String.Compare (((Hit) a).Uri.ToString (), ((Hit) b).Uri.ToString ());
  49                 }
  50         }
  51
  52         static string RemapUriToPath (Hashtable all_hits_by_uri, Hit hit)
  53         {
  54                 string exact_name = hit.GetFirstProperty ("beagle:ExactFilename");
  55                 string parent_uri_str = hit.GetFirstProperty ("_private:ParentDirUri");
  56
  57                 if (parent_uri_str == null)
  58                         return exact_name;
  59                 else
  60                         return Path.Combine (RemapUriToPath (all_hits_by_uri, (Hit) all_hits_by_uri [parent_uri_str]),
  61                                              exact_name);
  62         }
  63
  64         static int DumpOneIndex_Metadata (string index_name, bool only_dump_the_urls)
  65         {
  66                 Console.WriteLine (); // a visual cue that something has changed
  67                 LuceneQueryingDriver driver;
  68                 driver = new LuceneQueryingDriver (index_name, -1, true);
  69
  70                 Hashtable all_hits_by_uri;
  71                 all_hits_by_uri = driver.GetAllHitsByUri ();
  72
  73                 ArrayList all_hits;
  74                 all_hits = new ArrayList (all_hits_by_uri.Values);
  75
  76                 if (index_name == "FileSystemIndex") // A hard-wired hack
  77                         foreach (Hit hit in all_hits)
  78                                 hit.Uri = UriFu.PathToFileUri (RemapUriToPath (all_hits_by_uri, hit));
  79
  80                 all_hits.Sort (new HitByUriComparer ());
  81
  82                 foreach (Hit hit in all_hits) {
  83
  84                         if (only_dump_the_urls) {
  85                                 Console.WriteLine ("{0}: {1}", index_name, hit.Uri);
  86                                 continue;
  87                         }
  88
  89                         Console.WriteLine (" Index: {0}", index_name);
  90                         Console.WriteLine ("   Uri: {0}", hit.Uri);
  91                         if (hit.ParentUri != null)
  92                                 Console.WriteLine ("Parent: {0}", hit.ParentUri);
  93                         Console.WriteLine (" MimeT: {0}", hit.MimeType);
  94                         Console.WriteLine ("  Type: {0}", hit.Type);
  95
  96                         ArrayList props;
  97                         props = new ArrayList (hit.Properties);
  98                         props.Sort ();
  99                         foreach (Property prop in props)
 100                                 if (! prop.Key.StartsWith ("_private:"))
 101                                     Console.WriteLine ("  Prop: {0} = '{1}'", prop.Key, prop.Value);
 102
 103                         Console.WriteLine ();
 104                 }
 105
 106                 return all_hits.Count;
 107         }
 108
 109         static Term initial_enum_term;
 110         // Dump the term frequencies: we do this via direct Lucene access.
 111         static void DumpOneIndex_TermFrequencies (string index_name)
 112         {
 113                 LuceneQueryingDriver driver;
 114                 driver = new LuceneQueryingDriver (index_name, -1, true);
 115
 116                 IndexReader reader;
 117                 reader = IndexReader.Open (driver.PrimaryStore);
 118
 119                 TermEnum term_enum;
 120                 term_enum = reader.Terms (initial_enum_term);
 121
 122                 int distinct_term_count = 0;
 123                 int term_count = 0;
 124
 125                 // from LuceneFAQ
 126                 // Terms are sorted first by field, then by text
 127                 // so all terms with a given field are adjacent in enumerations.
 128                 if (term_enum.Term () != null) {
 129                         while (term_enum.Term().Field() == "Text") {
 130                                 int freq;
 131                                 freq = term_enum.DocFreq ();
 132
 133                                 Console.WriteLine ("{0} {1} {2}", index_name, term_enum.Term ().Text (), freq);
 134
 135                                 // FIXME: spew these as a count
 136                                 ++distinct_term_count;
 137                                 term_count += freq;
 138
 139                                 if (!term_enum.Next ())
 140                                         break;
 141                         }
 142                 }
 143
 144                 term_enum.Close ();
 145                 reader.Close ();
 146
 147                 Console.WriteLine ();
 148         }
 149
 150         public class IndexInfo : IComparable {
 151                 public string Name;
 152                 public int    Count;
 153
 154                 public IndexInfo (string name)
 155                 {
 156                         Name = name;
 157                 }
 158
 159                 public int CompareTo (object obj)
 160                 {
 161                         IndexInfo other = (IndexInfo) obj;
 162                         return String.Compare (this.Name, other.Name);
 163                 }
 164         }
 165
 166         static void DumpIndexInformation (Mode mode, bool show_counts)
 167         {
 168                 ArrayList index_info_list;
 169                 index_info_list = new ArrayList ();
 170
 171                 DirectoryInfo dir;
 172                 dir = new DirectoryInfo (PathFinder.IndexDir);
 173                 foreach (DirectoryInfo subdir in dir.GetDirectories ())
 174                         index_info_list.Add (new IndexInfo (subdir.Name));
 175
 176                 index_info_list.Sort ();
 177
 178                 bool set_counts = false;
 179
 180                 if (mode == Mode.TermFrequencies)
 181                         initial_enum_term = new Term ("Text", "");
 182
 183                 foreach (IndexInfo info in index_info_list) {
 184                         if (mode == Mode.Uris || mode == Mode.Properties) {
 185                                 info.Count = DumpOneIndex_Metadata (info.Name, mode == Mode.Uris);
 186                                 set_counts = true;
 187                         } else {
 188                                 DumpOneIndex_TermFrequencies (info.Name);
 189                         }
 190                 }
 191
 192                 if (show_counts && set_counts) {
 193                         Console.WriteLine ();
 194                         Console.WriteLine ("FINAL COUNTS");
 195
 196                         foreach (IndexInfo info in index_info_list)
 197                                 Console.WriteLine ("{0} {1}", info.Count.ToString ().PadLeft (7), info.Name);
 198                 }
 199         }
 200
 201         class DummyQueryResult : IQueryResult {
 202                 public void Add (ICollection hits)
 203                 {
 204                 }
 205
 206                 public void Subtract (ICollection hits)
 207                 {
 208                 }
 209         }
 210
 211         static void DumpFileIndexInformation (string path, string indexdir)
 212         {
 213                 //Uri uri = UriFu.PathToFileUri (path);
 214                 //Console.WriteLine ("Dumping information about:" + uri.AbsolutePath);
 215                 //path = uri.AbsolutePath;
 216                 if ((! File.Exists (path)) && (! Directory.Exists (path))) {
 217                         Console.WriteLine ("No such file or directory:" + path);
 218                         return;
 219                 }
 220
 221                 if (indexdir == null)
 222                         // default is ~/.beagle/Indexes/FileSystemIndex
 223                         indexdir = Path.Combine (PathFinder.IndexDir, "FileSystemIndex");
 224                 if (! Directory.Exists (indexdir)) {
 225                         Console.WriteLine ("Index:{0} doesnt exist.", indexdir);
 226                         return;
 227                 }
 228
 229                 // get fingerprint
 230                 TextReader reader;
 231                 reader = new StreamReader (Path.Combine (indexdir, "fingerprint"));
 232                 string fingerprint = reader.ReadLine ();
 233                 reader.Close ();
 234                 //Console.WriteLine ("Read fingerprint:" + fingerprint);
 235
 236                 // find out uid
 237                 FileAttributesStore fa_store = new FileAttributesStore (new FileAttributesStore_Mixed (indexdir, fingerprint));
 238                 Beagle.Daemon.FileAttributes attr = fa_store.Read (path);
 239                 if (attr == null) {
 240                         Console.WriteLine ("No information about this file in index. Ignoring.");
 241                         return;
 242                 }
 243                 string uri_string = "uid:" + GuidFu.ToShortString (attr.UniqueId);
 244                 Console.WriteLine ("Uri = " + uri_string);
 245                 //Console.WriteLine ("FilterName:" + attr.FilterName);
 246                 Console.WriteLine ("LastAttrTime:" + attr.LastAttrTime);
 247                 Console.WriteLine ("LastWriteTime:" + attr.LastWriteTime);
 248
 249                 LuceneQueryingDriver driver;
 250                 driver = new LuceneQueryingDriver (indexdir, -1, true);
 251
 252
 253                 // first try for the Uri:"uid:xxxxxxxxxxxxxxx"
 254                 Lucene.Net.Search.Query query = new TermQuery(new Term("Uri", uri_string));
 255                 if (DoQuery (driver, query))
 256                         return;
 257
 258                 // else query by path - this is for static indexes
 259                 path = StringFu.PathToQuotedFileUri (path);
 260                 Console.WriteLine ("Querying by:[" + path + "]");
 261                 query = new TermQuery(new Term("Uri", path));
 262                 DoQuery (driver, query);
 263
 264         }
 265
 266         static bool DoQuery (LuceneQueryingDriver driver, Lucene.Net.Search.Query query)
 267         {
 268                 IndexSearcher primary_searcher = LuceneCommon.GetSearcher (driver.PrimaryStore);
 269                 IndexSearcher secondary_searcher = LuceneCommon.GetSearcher (driver.SecondaryStore);
 270
 271                 Hits primary_hits = primary_searcher.Search(query);
 272                 Hits secondary_hits = secondary_searcher.Search (query);
 273                 Console.WriteLine ("{0} hits from primary store; {1} hits from secondary store", primary_hits.Length (), secondary_hits.Length ());
 274
 275                 Document primary_doc, secondary_doc;
 276                 // there should be exactly one primary hit and 0/1 secondary hit
 277                 if (primary_hits.Length () == 1) {
 278                         primary_doc = primary_hits.Doc (0);
 279                         Console.WriteLine (
 280                         "-----------------------------------------[ Immutable data ]--------------------------------------");
 281                         foreach (Field f in primary_doc.Fields ()) {
 282
 283                                 String name = f.Name ();
 284                                 String val = f.StringValue ();
 285                                 bool stored = f.IsStored ();
 286                                 bool searchable = (val [0] == 's');
 287                                 bool tokenized = f.IsTokenized();
 288                                 if (name.Length >= 7 && name.StartsWith ("prop:"))
 289                                         tokenized = (name [5] != 't');
 290                                 float boost = f.GetBoost();
 291
 292                                 Console.WriteLine ("{0,-30} = [{1}] (stored? {2}, searchable? {3}, tokenized? {4}, boost={5})",
 293                                                     name, val, stored, searchable, tokenized, boost);
 294                         }
 295                 }
 296
 297                 if (secondary_hits.Length () == 1) {
 298                         secondary_doc = secondary_hits.Doc (0);
 299                         Console.WriteLine (
 300                         "------------------------------------------[ Mutable data ]---------------------------------------");
 301                         foreach (Field f in secondary_doc.Fields ()) {
 302
 303                                 String name = f.Name ();
 304                                 String val = f.StringValue ();
 305                                 bool stored = f.IsStored ();
 306                                 bool searchable = (val [0] == 's');
 307                                 bool tokenized = f.IsTokenized();
 308                                 if (name.Length >= 7 && name.StartsWith ("prop:"))
 309                                         tokenized = (name [5] != 't');
 310                                 float boost = f.GetBoost();
 311
 312                                 Console.WriteLine ("{0,-30} = [{1}] (stored? {2}, searchable? {3}, tokenized? {4}, boost={5})",
 313                                                     name, val, stored, searchable, tokenized, boost);
 314                         }
 315                 }
 316
 317                 LuceneCommon.ReleaseSearcher (primary_searcher);
 318                 LuceneCommon.ReleaseSearcher (secondary_searcher);
 319
 320                 if (primary_hits.Length () != 0 || secondary_hits.Length () != 0)
 321                         return true;
 322                 else
 323                         return false;
 324         }
 325
 326         enum Mode {
 327                 Uris,
 328                 Properties,
 329                 TermFrequencies
 330         }
 331
 332
 333         static void Main (string [] args)
 334         {
 335                 Mode mode = Mode.Uris;
 336                 bool show_counts = true;
 337                 string file = null;
 338                 string indexdir = null;
 339
 340                 foreach (string arg in args) {
 341
 342                         switch (arg) {
 343
 344                         case "--help":
 345                                 Console.WriteLine (@"
 346 beagle-dump-index [options] [ [--indexdir=dir] file]
 347
 348 --uris                   Dump all Uris (default)
 349 --properties             Dump all properties
 350 --term-frequencies       Dump term frequencies
 351
 352 --show-counts            Show index count totals (default)
 353 --hide-counts            Hide index count totals
 354
 355 --indexdir=<index directory>
 356                          Absolute path of the directory storing the index
 357                          e.g. /home/user/.beagle/Indexes/FileSystemIndex
 358 file                     Get information in index about this file or directory
 359
 360 --help                         What you just did");
 361                                 Environment.Exit (0);
 362                                 break;
 363
 364                         case "--uris":
 365                                 mode = Mode.Uris;
 366                                 break;
 367
 368                         case "--properties":
 369                                 mode = Mode.Properties;
 370                                 break;
 371
 372                         case "--term-frequencies":
 373                                 mode = Mode.TermFrequencies;
 374                                 break;
 375
 376                         case "--hide-counts":
 377                                 show_counts = false;
 378                                 break;
 379
 380                         case "--show-counts":
 381                                 show_counts = false;
 382                                 break;
 383
 384                         default:
 385                                 if (arg.StartsWith ("--indexdir="))
 386                                         indexdir = arg.Remove (0, 11);
 387                                 else
 388                                         file = arg;
 389                                 break;
 390                         }
 391                 }
 392
 393                 if (file == null)
 394                         DumpIndexInformation (mode, show_counts);
 395                 else
 396                         DumpFileIndexInformation (file, indexdir);
 397
 398         }
 399 }