gov/nasa/worldwind/retrieve/URLRetriever.java

   1 /*
   2 Copyright (C) 2001, 2006 United States Government
   3 as represented by the Administrator of the
   4 National Aeronautics and Space Administration.
   5 All Rights Reserved.
   6 */
   7 package gov.nasa.worldwind.retrieve;
   8
   9 import gov.nasa.worldwind.*;
  10 import gov.nasa.worldwind.avlist.AVKey;
  11 import gov.nasa.worldwind.util.*;
  12
  13 import java.io.*;
  14 import java.net.*;
  15 import java.nio.ByteBuffer;
  16 import java.nio.channels.*;
  17 import java.util.concurrent.atomic.AtomicInteger;
  18 import java.util.logging.Level;
  19 import java.util.zip.*;
  20
  21 /**
  22  * @author Tom Gaskins
  23  * @version $Id: URLRetriever.java 3558 2007-11-17 08:36:45Z tgaskins $
  24  */
  25 public abstract class URLRetriever extends WWObjectImpl implements Retriever
  26 {
  27     private volatile String state = RETRIEVER_STATE_NOT_STARTED;
  28     private volatile int contentLength = 0;
  29     private AtomicInteger contentLengthRead = new AtomicInteger(0);
  30     private volatile String contentType;
  31     private volatile ByteBuffer byteBuffer;
  32     private volatile URLConnection connection;
  33     private final URL url;
  34     private final RetrievalPostProcessor postProcessor;
  35     private int connectTimeout = Configuration.getIntegerValue(AVKey.URL_CONNECT_TIMEOUT, 8000);
  36     private int readTimeout = Configuration.getIntegerValue(AVKey.URL_READ_TIMEOUT, 5000);
  37     private int staleRequestLimit = -1;
  38     private long submitTime;
  39     private long beginTime;
  40     private long endTime;
  41
  42     /**
  43      * @param url           the URL of the resource to retrieve.
  44      * @param postProcessor the retrieval post-processor to invoke when the resource is retrieved.
  45      * @throws IllegalArgumentException if <code>url</code> or <code>postProcessor</code> is null.
  46      */
  47     public URLRetriever(URL url, RetrievalPostProcessor postProcessor)
  48     {
  49         if (url == null)
  50         {
  51             String message = Logging.getMessage("nullValue.URLIsNull");
  52             Logging.logger().severe(message);
  53             throw new IllegalArgumentException(message);
  54         }
  55         if (postProcessor == null)
  56         {
  57             String message = Logging.getMessage("nullValue.PostProcessorIsNull");
  58             Logging.logger().severe(message);
  59             throw new IllegalArgumentException(message);
  60         }
  61
  62         this.url = url;
  63         this.postProcessor = postProcessor;
  64     }
  65
  66     public final URL getUrl()
  67     {
  68         return url;
  69     }
  70
  71     public final int getContentLength()
  72     {
  73         return this.contentLength;
  74     }
  75
  76     protected void setContentLengthRead(int length)
  77     {
  78         this.contentLengthRead.set(length);
  79     }
  80
  81     public final int getContentLengthRead()
  82     {
  83         return this.contentLengthRead.get();
  84     }
  85
  86     public final String getContentType()
  87     {
  88         return this.contentType;
  89     }
  90
  91     public final ByteBuffer getBuffer()
  92     {
  93         return this.byteBuffer;
  94     }
  95
  96     public final String getName()
  97     {
  98         return this.url.toString();
  99     }
 100
 101     public final String getState()
 102     {
 103         return this.state;
 104     }
 105
 106     protected final URLConnection getConnection()
 107     {
 108         return this.connection;
 109     }
 110
 111     public final RetrievalPostProcessor getPostProcessor()
 112     {
 113         return postProcessor;
 114     }
 115
 116     public final int getConnectTimeout()
 117     {
 118         return connectTimeout;
 119     }
 120
 121     public int getReadTimeout()
 122     {
 123         return readTimeout;
 124     }
 125
 126     public void setReadTimeout(int readTimeout)
 127     {
 128         this.readTimeout = readTimeout;
 129     }
 130
 131     public int getStaleRequestLimit()
 132     {
 133         return staleRequestLimit;
 134     }
 135
 136     public void setStaleRequestLimit(int staleRequestLimit)
 137     {
 138         this.staleRequestLimit = staleRequestLimit;
 139     }
 140
 141     public final void setConnectTimeout(int connectTimeout)
 142     {
 143         this.connectTimeout = connectTimeout;
 144     }
 145
 146     public long getSubmitTime()
 147     {
 148         return submitTime;
 149     }
 150
 151     public void setSubmitTime(long submitTime)
 152     {
 153         this.submitTime = submitTime;
 154     }
 155
 156     public long getBeginTime()
 157     {
 158         return beginTime;
 159     }
 160
 161     public void setBeginTime(long beginTime)
 162     {
 163         this.beginTime = beginTime;
 164     }
 165
 166     public long getEndTime()
 167     {
 168         return endTime;
 169     }
 170
 171     public void setEndTime(long endTime)
 172     {
 173         this.endTime = endTime;
 174     }
 175
 176     public final Retriever call() throws Exception
 177     {
 178         if (this.interrupted())
 179             return this;
 180
 181         try
 182         {
 183             this.setState(RETRIEVER_STATE_STARTED);
 184
 185             if (!this.interrupted())
 186             {
 187                 this.setState(RETRIEVER_STATE_CONNECTING);
 188                 this.connection = this.openConnection();
 189             }
 190
 191             if (!this.interrupted())
 192             {
 193                 this.setState(RETRIEVER_STATE_READING);
 194                 this.byteBuffer = this.read();
 195             }
 196
 197             if (!this.interrupted())
 198                 this.setState(RETRIEVER_STATE_SUCCESSFUL);
 199
 200             WorldWind.getNetworkStatus().logAvailableHost(this.url);
 201         }
 202         catch (UnknownHostException e)
 203         {
 204             this.setState(RETRIEVER_STATE_ERROR);
 205             WorldWind.getNetworkStatus().logUnavailableHost(this.url);
 206             throw e;
 207         }
 208         catch (SocketException e)
 209         {
 210             this.setState(RETRIEVER_STATE_ERROR);
 211             WorldWind.getNetworkStatus().logUnavailableHost(this.url);
 212             throw e;
 213         }
 214         catch (Exception e)
 215         {
 216             this.setState(RETRIEVER_STATE_ERROR);
 217             if (!(e instanceof java.net.SocketTimeoutException))
 218             {
 219                 Logging.logger().log(Level.SEVERE,
 220                     Logging.getMessage("URLRetriever.ErrorAttemptingToRetrieve", this.url.toString()), e);
 221             }
 222             throw e;
 223         }
 224         finally
 225         {
 226             this.end();
 227         }
 228
 229         return this;
 230     }
 231
 232     private void setState(String state)
 233     {
 234         String oldState = this.state;
 235         this.state = state;
 236         this.firePropertyChange(AVKey.RETRIEVER_STATE, oldState, this.state);
 237     }
 238
 239     private boolean interrupted()
 240     {
 241         if (Thread.currentThread().isInterrupted())
 242         {
 243             this.setState(RETRIEVER_STATE_INTERRUPTED);
 244             String message = Logging.getMessage("URLRetriever.RetrievalInterruptedFor", this.url.toString());
 245             Logging.logger().fine(message);
 246             return true;
 247         }
 248         return false;
 249     }
 250
 251     private URLConnection openConnection() throws IOException
 252     {
 253         try
 254         {
 255             Proxy proxy = WWIO.configureProxy();
 256             if (proxy != null)
 257                 this.connection = this.url.openConnection(proxy);
 258             else
 259                 this.connection = this.url.openConnection();
 260         }
 261         catch (java.io.IOException e)
 262         {
 263             Logging.logger().log(Level.SEVERE,
 264                 Logging.getMessage("URLRetriever.ErrorOpeningConnection", this.url.toString()), e);
 265             throw e;
 266         }
 267
 268         if (this.connection == null) // java.net.URL docs imply that this won't happen. We check anyway.
 269         {
 270             String message = Logging.getMessage("URLRetriever.NullReturnedFromOpenConnection", this.url);
 271             Logging.logger().severe(message);
 272             throw new IllegalStateException(message);
 273         }
 274
 275         this.connection.setConnectTimeout(this.connectTimeout);
 276         this.connection.setReadTimeout(this.readTimeout);
 277
 278         return connection;
 279     }
 280
 281     private void end() throws Exception
 282     {
 283         try
 284         {
 285             if (this.postProcessor != null)
 286             {
 287                 this.byteBuffer = this.postProcessor.run(this);
 288             }
 289         }
 290         catch (Exception e)
 291         {
 292             this.setState(RETRIEVER_STATE_ERROR);
 293             Logging.logger().log(Level.SEVERE,
 294                 Logging.getMessage("URLRetriever.ErrorPostProcessing", this.url.toString()), e);
 295             throw e;
 296         }
 297     }
 298
 299     private ByteBuffer read() throws Exception
 300     {
 301         try
 302         {
 303             ByteBuffer buffer = this.doRead(this.connection);
 304             if (buffer == null)
 305                 this.contentLength = 0;
 306             return buffer;
 307         }
 308         catch (Exception e)
 309         {
 310             if (!(e instanceof java.net.SocketTimeoutException || e instanceof UnknownHostException
 311                 || e instanceof SocketException))
 312             {
 313                 Logging.logger().log(Level.SEVERE,
 314                     Logging.getMessage("URLRetriever.ErrorReadingFromConnection", this.url.toString()), e);
 315             }
 316             throw e;
 317         }
 318     }
 319
 320     /**
 321      * @param connection the connection to read from.
 322      * @return a buffer containing the content read from the connection
 323      * @throws Exception                if <code>connection</code> is null or an exception occurs during reading.
 324      * @throws IllegalArgumentException if <code>connection</code> is null
 325      */
 326     protected ByteBuffer doRead(URLConnection connection) throws Exception
 327     {
 328         if (connection == null)
 329         {
 330             String msg = Logging.getMessage("nullValue.ConnectionIsNull");
 331             Logging.logger().severe(msg);
 332             throw new IllegalArgumentException(msg);
 333         }
 334
 335         this.contentLength = this.connection.getContentLength();
 336
 337         ByteBuffer buffer;
 338         InputStream inputStream = null;
 339         try
 340         {
 341             inputStream = this.connection.getInputStream();
 342             if (inputStream == null)
 343             {
 344                 Logging.logger().log(Level.SEVERE, "URLRetriever.InputStreamFromConnectionNull", connection.getURL());
 345                 return null;
 346             }
 347
 348             // TODO: Make decompression of zip file configurable
 349             // TODO: Make this more generally configurable based on content type
 350             // todo: make a zip reader that handles streams of unknown length
 351             // TODO: add a gzip reader
 352             // TODO: this code is checking content type for compression when it should be checking content encoding,
 353             // but the WW servers are sending application/zip as the content type, and null for the content encoding.
 354             this.contentType = connection.getContentType();
 355             if (this.contentType != null && this.contentType.equalsIgnoreCase("application/zip"))
 356                 buffer =
 357                     this.readZipStream(inputStream, connection.getURL()); // assume single file in zip and decompress it
 358             else
 359                 buffer = this.readNonSpecificStream(inputStream, connection);
 360
 361             this.contentType = connection.getContentType();
 362         }
 363         finally
 364         {
 365             if (inputStream != null)
 366                 try
 367                 {
 368                     inputStream.close();
 369                 }
 370                 catch (IOException e)
 371                 {
 372                     Logging.logger().log(Level.SEVERE, "URLRetriever.ExceptionClosingInputStreamToConnection",
 373                         connection.getURL());
 374                 }
 375         }
 376
 377         return buffer;
 378     }
 379
 380     private ByteBuffer readNonSpecificStream(InputStream inputStream, URLConnection connection) throws IOException
 381     {
 382         if (inputStream == null)
 383         {
 384             String message = Logging.getMessage("URLRetriever.InputStreamNullFor", connection.getURL());
 385             Logging.logger().severe(message);
 386             throw new IllegalArgumentException(message);
 387         }
 388
 389         if (this.contentLength < 1)
 390         {
 391             return readNonSpecificStreamUnknownLength(inputStream);
 392         }
 393
 394         ReadableByteChannel channel = Channels.newChannel(inputStream);
 395         ByteBuffer buffer = ByteBuffer.allocate(this.contentLength);
 396
 397         int numBytesRead = 0;
 398         while (!this.interrupted() && numBytesRead >= 0 && numBytesRead < buffer.limit())
 399         {
 400             int count = channel.read(buffer);
 401             if (count > 0)
 402                 this.contentLengthRead.getAndAdd(numBytesRead += count);
 403         }
 404
 405         if (buffer != null)
 406             buffer.flip();
 407
 408         return buffer;
 409     }
 410
 411     private ByteBuffer readNonSpecificStreamUnknownLength(InputStream inputStream) throws IOException
 412     {
 413         final int PAGE_SIZE = 4096;
 414
 415         ReadableByteChannel channel = Channels.newChannel(inputStream);
 416         ByteBuffer buffer = ByteBuffer.allocate(PAGE_SIZE);
 417
 418         int count = 0;
 419         int numBytesRead = 0;
 420         while (!this.interrupted() && count >= 0)
 421         {
 422             count = channel.read(buffer);
 423             if (count > 0)
 424                 this.contentLengthRead.getAndAdd(numBytesRead += count);
 425
 426             if (count > 0 && !buffer.hasRemaining())
 427             {
 428                 ByteBuffer biggerBuffer = ByteBuffer.allocate(buffer.limit() + PAGE_SIZE);
 429                 biggerBuffer.put((ByteBuffer) buffer.rewind());
 430                 buffer = biggerBuffer;
 431             }
 432         }
 433
 434         if (buffer != null)
 435             buffer.flip();
 436
 437         return buffer;
 438     }
 439
 440     /**
 441      * @param inputStream a stream to the zip connection.
 442      * @param url         the URL of the zip resource.
 443      * @return a buffer containing the content read from the zip stream.
 444      * @throws java.io.IOException if the stream does not refer to a zip resource or an exception occurs during reading.
 445      * @throws IllegalArgumentException if <code>inputStream</code> is null
 446      */
 447     private ByteBuffer readZipStream(InputStream inputStream, URL url) throws IOException
 448     {
 449         ZipInputStream zis = new ZipInputStream(inputStream);
 450         ZipEntry ze = zis.getNextEntry();
 451         if (ze == null)
 452         {
 453             Logging.logger().severe(Logging.getMessage("URLRetriever.NoZipEntryFor") + url);
 454             return null;
 455         }
 456
 457         ByteBuffer buffer = null;
 458         if (ze.getSize() > 0)
 459         {
 460             buffer = ByteBuffer.allocate((int) ze.getSize());
 461
 462             byte[] inputBuffer = new byte[8192];
 463             while (buffer.hasRemaining())
 464             {
 465                 int count = zis.read(inputBuffer);
 466                 if (count > 0)
 467                 {
 468                     buffer.put(inputBuffer, 0, count);
 469                     this.contentLengthRead.getAndAdd(buffer.position() + 1);
 470                 }
 471             }
 472         }
 473         if (buffer != null)
 474             buffer.flip();
 475
 476         return buffer;
 477     }
 478
 479     @Override
 480     public boolean equals(Object o)
 481     {
 482         if (this == o)
 483             return true;
 484         if (o == null || getClass() != o.getClass())
 485             return false;
 486
 487         final URLRetriever that = (URLRetriever) o;
 488
 489         // Retrievers are considered identical if they are for the same URL. This convention is used by the
 490         // retrieval service to filter out duplicate retreival requests.
 491         return !(url != null ? !url.toString().contentEquals(that.url.toString()) : that.url != null);
 492     }
 493
 494     @Override
 495     public int hashCode()
 496     {
 497         int result;
 498         result = (url != null ? url.hashCode() : 0);
 499         return result;
 500     }
 501
 502     @Override
 503     public String toString()
 504     {
 505         return this.getName() != null ? this.getName() : super.toString();
 506     }
 507 }