src/Indexer/Program.cs

   1 using System;
   2 using System.Collections.Generic;
   3 using System.Diagnostics;
   4 using System.IO;
   5 using System.Linq;
   6 using System.Runtime.Serialization;
   7 using System.Runtime.Serialization.Formatters.Binary;
   8 using System.ServiceModel.Channels;
   9 using System.Threading;
  10
  11 using Indexer.ShareDiscoveryReference;
  12
  13 using ShareDiscovery;
  14
  15 namespace Indexer
  16 {
  17     internal class Program
  18     {
  19         private static int _sleepTimeout = /*1 minute*/ 60000;
  20         private static int _numSharesIndexed = 0;
  21         private static int _nDenial = 0;
  22
  23         private static void Main( string[] args )
  24         {
  25             ShareDiscoveryServiceClient client = new ShareDiscoveryServiceClient();
  26             while ( true )
  27             {
  28                 string share = null;
  29                 Stopwatch startWatch = new Stopwatch();
  30                 startWatch.Start();
  31
  32                 string errorMessage = "Got no shares to index.";
  33                 if ( _numSharesIndexed == 0 )
  34                 {
  35                     errorMessage += " Share discovery service is probably initializing now.";
  36                 }
  37
  38                 try
  39                 {
  40                     share = (new StreamReader(client.GetShareToIndex())).ReadToEnd();
  41                 }
  42                 catch ( TimeoutException )
  43                 {
  44                     errorMessage = "Share discovery service timed out.";
  45                 }
  46
  47                 if ( String.IsNullOrEmpty( share ) )
  48                 {
  49                     ++_nDenial;
  50                     double timeout = _sleepTimeout * Math.Sqrt( _nDenial );
  51                     Console.WriteLine( "[{0}] Error: {1} Sleeping for {2:0.##} seconds.", DateTime.Now, errorMessage,
  52                                        timeout / 1000f );
  53                     Thread.Sleep( (int) timeout );
  54                     continue;
  55                 }
  56                 _nDenial = 0;
  57
  58                 Console.WriteLine( "[{0}] About to index share {1}.", DateTime.Now, share );
  59
  60                 try
  61                 {
  62                     Stopwatch indexWatch = new Stopwatch();
  63
  64                     indexWatch.Start();
  65                     TreeNode shareNode = IndexFolder( share );
  66                     indexWatch.Stop();
  67
  68                     if ( shareNode != null )
  69                     {
  70                         if ( startWatch.IsRunning )
  71                         {
  72                             startWatch.Stop();
  73                             Console.WriteLine( "[{0}] Share discovery service initialized in {1}", DateTime.Now,
  74                                                startWatch.Elapsed );
  75                         }
  76
  77                         string[] parts = share.Split( new[] { '\\' }, StringSplitOptions.RemoveEmptyEntries );
  78
  79                         FileInfo shareInfo = new FileInfo( share );
  80                         shareNode.CreationTime = shareInfo.CreationTime;
  81                         shareNode.AccessTime = shareInfo.LastAccessTime;
  82                         shareNode.ModificationTime = shareInfo.LastWriteTime;
  83
  84                         TreeNode compNode = new TreeNode();
  85                         compNode.Name = "//" + parts[ 0 ];
  86                         compNode.Children = new[] { shareNode };
  87
  88                         // Count indexed nodes.
  89                         uint numNodes = 0;
  90                         Queue<TreeNode> queue = new Queue<TreeNode>();
  91                         queue.Enqueue( compNode );
  92
  93                         while ( queue.Count > 0 )
  94                         {
  95                             TreeNode next = queue.Dequeue();
  96                             ++numNodes;
  97                             if ( next.Children != null )
  98                             {
  99                                 foreach ( TreeNode child in next.Children )
 100                                 {
 101                                     queue.Enqueue( child );
 102                                 }
 103                             }
 104                         }
 105
 106                         if ( numNodes > 3 )
 107                         {
 108                             ++_numSharesIndexed;
 109                             Console.WriteLine( "[{0}] Indexed {1}-th share {2} in {3}. {4} nodes in graph.",
 110                                                DateTime.Now,
 111                                                _numSharesIndexed,
 112                                                share, indexWatch.Elapsed, numNodes );
 113
 114                             Stream stream = new MemoryStream();
 115                             IFormatter fmt = new BinaryFormatter();
 116                             fmt.Serialize( stream, compNode );
 117                             stream.Seek( 0, SeekOrigin.Begin );
 118                             client.SaveIndex( stream );
 119                             stream.Close();
 120
 121                             Console.WriteLine( "[{0}] Saved share {1}.", DateTime.Now, share );
 122                         }
 123                     }
 124                     else
 125                     {
 126                         Console.WriteLine( "[{0}] Wasted {1} on indexing {2}.", DateTime.Now, indexWatch.Elapsed, share );
 127                     }
 128                 }
 129                 catch ( IOException ex )
 130                 {
 131                     Console.Error.WriteLine( "[{0}] Error: failed to save {1}: {2}", DateTime.Now, share, ex.Message );
 132                 }
 133             }
 134         }
 135
 136         public static TreeNode IndexFolder( string directoryName )
 137         {
 138             FileInfo dirinfo = null;
 139             TreeNode dir = null;
 140
 141             try
 142             {
 143                 dirinfo = new FileInfo( directoryName );
 144
 145                 dir = new TreeNode
 146                           {
 147                               Name = Path.GetFileName( directoryName ),
 148                               CreationTime = dirinfo.CreationTime,
 149                               AccessTime = dirinfo.LastAccessTime,
 150                               ModificationTime = dirinfo.LastWriteTime
 151                           };
 152
 153                 List<TreeNode> listing = new List<TreeNode>();
 154
 155                 try
 156                 {
 157                     string[] subdirs = Directory.GetDirectories( directoryName );
 158                     string[] subfiles = Directory.GetFiles( directoryName );
 159
 160                     if ( subdirs.Length == 0 && subfiles.Length == 0 )
 161                     {
 162                         return null;
 163                     }
 164
 165                     dir.Children
 166                         = (
 167                               (
 168                                   from directory in subdirs
 169                                   let child = IndexFolder( directory )
 170                                   where child != null && child.Name != ".svn"
 171                                   select child
 172                               ).Union(
 173                                   from fileName in subfiles
 174                                   let info = new FileInfo( fileName )
 175                                   select
 176                                       new TreeNode
 177                                           {
 178                                               Name = info.Name,
 179                                               Size = info.Length,
 180                                               CreationTime = info.CreationTime,
 181                                               AccessTime = info.LastAccessTime,
 182                                               ModificationTime = info.LastWriteTime
 183                                           }
 184                                   )
 185                           ).ToList();
 186                 }
 187                 catch ( UnauthorizedAccessException ex )
 188                 {
 189                     Console.Error.WriteLine( "Access denied to {0}", directoryName );
 190                 }
 191                 catch ( IOException ex )
 192                 {
 193                     Console.Error.WriteLine( "Cannot read {0}: {1}", directoryName, ex.Message );
 194                 }
 195             }
 196             catch ( IOException ex )
 197             {
 198                 Console.Error.WriteLine( "Path {0} not found.", directoryName );
 199             }
 200
 201             return dir;
 202         }
 203     }
 204 }