gov/nasa/worldwind/URLRetriever.java

   1 /*
   2 Copyright (C) 2001, 2006 United States Government
   3 as represented by the Administrator of the
   4 National Aeronautics and Space Administration.
   5 All Rights Reserved.
   6 */
   7 package gov.nasa.worldwind;
   8
   9 import java.util.concurrent.atomic.*;
  10 import java.util.zip.*;
  11 import java.nio.*;
  12 import java.nio.channels.*;
  13 import java.net.*;
  14 import java.io.*;
  15
  16 /**
  17  * @author Tom Gaskins
  18  * @version $Id: URLRetriever.java 2228 2007-07-06 22:18:04Z tgaskins $
  19  */
  20 public abstract class URLRetriever extends WWObjectImpl implements Retriever
  21 {
  22     private volatile String state = RETRIEVER_STATE_NOT_STARTED;
  23     private volatile int contentLength = 0;
  24     private AtomicInteger contentLengthRead = new AtomicInteger(0);
  25     private volatile String contentType;
  26     private volatile ByteBuffer byteBuffer;
  27     private volatile URLConnection connection;
  28     private final URL url;
  29     private final RetrievalPostProcessor postProcessor;
  30     private int connectTimeout = Configuration.getIntegerValue(AVKey.URL_CONNECT_TIMEOUT, 8000);
  31     private int readTimeout = Configuration.getIntegerValue(AVKey.URL_READ_TIMEOUT, 5000);
  32     private int staleRequestLimit = -1;
  33     private long submitTime;
  34     private long beginTime;
  35     private long endTime;
  36
  37     /**
  38      * @param url
  39      * @param postProcessor
  40      * @throws IllegalArgumentException if <code>url</code> or <code>postProcessor</code> is null
  41      */
  42     public URLRetriever(URL url, RetrievalPostProcessor postProcessor)
  43     {
  44         if (url == null)
  45         {
  46             String message = WorldWind.retrieveErrMsg("nullValue.URLIsNull");
  47             WorldWind.logger().log(java.util.logging.Level.FINE, message);
  48             throw new IllegalArgumentException(message);
  49         }
  50         if (postProcessor == null)
  51         {
  52             String message = WorldWind.retrieveErrMsg("nullValue.PostProcessorIsNull");
  53             WorldWind.logger().log(java.util.logging.Level.FINE, message);
  54             throw new IllegalArgumentException(message);
  55         }
  56
  57         this.url = url;
  58         this.postProcessor = postProcessor;
  59     }
  60
  61     public final URL getUrl()
  62     {
  63         return url;
  64     }
  65
  66     public final int getContentLength()
  67     {
  68         return this.contentLength;
  69     }
  70
  71     protected void setContentLengthRead(int length)
  72     {
  73         this.contentLengthRead.set(length);
  74     }
  75
  76     public final int getContentLengthRead()
  77     {
  78         return this.contentLengthRead.get();
  79     }
  80
  81     public final String getContentType()
  82     {
  83         return this.contentType;
  84     }
  85
  86     public final ByteBuffer getBuffer()
  87     {
  88         return this.byteBuffer;
  89     }
  90
  91     public final String getName()
  92     {
  93         return this.url.toString();
  94     }
  95
  96     public final String getState()
  97     {
  98         return this.state;
  99     }
 100
 101     protected final URLConnection getConnection()
 102     {
 103         return this.connection;
 104     }
 105
 106     public final RetrievalPostProcessor getPostProcessor()
 107     {
 108         return postProcessor;
 109     }
 110
 111     public final int getConnectTimeout()
 112     {
 113         return connectTimeout;
 114     }
 115
 116     public int getReadTimeout()
 117     {
 118         return readTimeout;
 119     }
 120
 121     public void setReadTimeout(int readTimeout)
 122     {
 123         this.readTimeout = readTimeout;
 124     }
 125
 126     public int getStaleRequestLimit()
 127     {
 128         return staleRequestLimit;
 129     }
 130
 131     public void setStaleRequestLimit(int staleRequestLimit)
 132     {
 133         this.staleRequestLimit = staleRequestLimit;
 134     }
 135
 136     public final void setConnectTimeout(int connectTimeout)
 137     {
 138         this.connectTimeout = connectTimeout;
 139     }
 140
 141     public long getSubmitTime()
 142     {
 143         return submitTime;
 144     }
 145
 146     public void setSubmitTime(long submitTime)
 147     {
 148         this.submitTime = submitTime;
 149     }
 150
 151     public long getBeginTime()
 152     {
 153         return beginTime;
 154     }
 155
 156     public void setBeginTime(long beginTime)
 157     {
 158         this.beginTime = beginTime;
 159     }
 160
 161     public long getEndTime()
 162     {
 163         return endTime;
 164     }
 165
 166     public void setEndTime(long endTime)
 167     {
 168         this.endTime = endTime;
 169     }
 170
 171     public final Retriever call() throws Exception
 172     {
 173         if (this.interrupted())
 174             return this;
 175
 176         try
 177         {
 178             this.setState(RETRIEVER_STATE_STARTED);
 179
 180             if (!this.interrupted())
 181             {
 182                 this.setState(RETRIEVER_STATE_CONNECTING);
 183                 this.connection = this.openConnection();
 184             }
 185
 186             if (!this.interrupted())
 187             {
 188                 this.setState(RETRIEVER_STATE_READING);
 189                 this.byteBuffer = this.read();
 190             }
 191
 192             if (!this.interrupted())
 193                 this.setState(RETRIEVER_STATE_SUCCESSFUL);
 194         }
 195         catch (Exception e)
 196         {
 197             this.setState(RETRIEVER_STATE_ERROR);
 198             if (!(e instanceof java.net.SocketTimeoutException))
 199             {
 200                 String message = WorldWind.retrieveErrMsg("URLRetriever.ErrorAttemptingToRetrieve")
 201                     + this.url.toString();
 202                 WorldWind.logger().log(java.util.logging.Level.FINE, message, e);
 203             }
 204             throw e;
 205         }
 206         finally
 207         {
 208             this.end();
 209         }
 210
 211         return this;
 212     }
 213
 214     private void setState(String state)
 215     {
 216         String oldState = this.state;
 217         this.state = state;
 218         this.firePropertyChange(AVKey.RETRIEVER_STATE, oldState, this.state);
 219     }
 220
 221     private boolean interrupted()
 222     {
 223         if (Thread.currentThread().isInterrupted())
 224         {
 225             this.setState(RETRIEVER_STATE_INTERRUPTED);
 226             String message = WorldWind.retrieveErrMsg("URLRetriever.RetrievalInterruptedFor") + this.url.toString();
 227             WorldWind.logger().log(java.util.logging.Level.FINER, message);
 228             return true;
 229         }
 230         return false;
 231     }
 232
 233     private URLConnection openConnection() throws IOException
 234     {
 235         try
 236         {
 237             this.connection = this.url.openConnection();
 238         }
 239         catch (java.io.IOException e)
 240         {
 241             String message = WorldWind.retrieveErrMsg("URLRetriever.ErrorOpeningConnection") + this.url.toString()
 242                 + " " + e.getLocalizedMessage();
 243             WorldWind.logger().log(java.util.logging.Level.FINE, message, e);
 244             throw e;
 245         }
 246
 247         if (this.connection == null) // java.net.URL docs imply that this won't happen. We check anyway.
 248         {
 249             String message = WorldWind.retrieveErrMsg("URLRetriever.NullReturnedFromOpenConnection") + this.url;
 250             WorldWind.logger().log(java.util.logging.Level.FINE, message);
 251             throw new IllegalStateException(message);
 252         }
 253
 254         this.connection.setConnectTimeout(this.connectTimeout);
 255         this.connection.setReadTimeout(this.readTimeout);
 256
 257         return connection;
 258     }
 259
 260     private void end() throws Exception
 261     {
 262         try
 263         {
 264             if (this.postProcessor != null)
 265             {
 266                 this.byteBuffer = this.postProcessor.run(this);
 267             }
 268         }
 269         catch (Exception e)
 270         {
 271             this.setState(RETRIEVER_STATE_ERROR);
 272             String message = WorldWind.retrieveErrMsg("URLRetriever.ErrorPostProcessing") + this.url.toString();
 273             WorldWind.logger().log(java.util.logging.Level.FINE, message + " " + e.getLocalizedMessage());
 274             throw e;
 275         }
 276     }
 277
 278     private ByteBuffer read() throws Exception
 279     {
 280         try
 281         {
 282             ByteBuffer buffer = this.doRead(this.connection);
 283             if (buffer == null)
 284                 this.contentLength = 0;
 285             return buffer;
 286         }
 287         catch (Exception e)
 288         {
 289             if (!(e instanceof java.net.SocketTimeoutException))
 290             {
 291                 String message = WorldWind.retrieveErrMsg("URLRetriever.ErrorReadingFromConnection")
 292                     + this.url.toString() + e.getLocalizedMessage();
 293                 WorldWind.logger().log(java.util.logging.Level.FINE, message, e);
 294             }
 295             throw e;
 296         }
 297     }
 298
 299     /**
 300      * @param connection
 301      * @return
 302      * @throws Exception
 303      * @throws IllegalArgumentException if <code>connection</code> is null
 304      */
 305     protected ByteBuffer doRead(URLConnection connection) throws Exception
 306     {
 307         if (connection == null)
 308         {
 309             String msg = WorldWind.retrieveErrMsg("nullValue.ConnectionIsNull");
 310             WorldWind.logger().log(java.util.logging.Level.FINE, msg);
 311             throw new IllegalArgumentException(msg);
 312         }
 313
 314         this.contentLength = this.connection.getContentLength();
 315
 316         ByteBuffer buffer;
 317         InputStream inputStream = null;
 318         try
 319         {
 320             inputStream = this.connection.getInputStream();
 321             if (inputStream == null)
 322             {
 323                 WorldWind.logger().log(java.util.logging.Level.FINE, WorldWind.retrieveErrMsg(
 324                     "URLRetriever.InputStreamFromConnectionNull") + connection.getURL());
 325                 return null;
 326             }
 327
 328             // TODO: Make decompression of zip file configurable
 329             // TODO: Make this more generally configurable based on content type
 330             // todo: make a zip reader that handles streams of unknown length
 331             // TODO: add a gzip reader
 332             // TODO: this code is checking content type for compression when it should be checking content encoding,
 333             // but the WW servers are sending application/zip as the content type, and null for the content encoding.
 334             this.contentType = connection.getContentType();
 335             if (this.contentType != null && this.contentType.equalsIgnoreCase("application/zip"))
 336                 buffer = this.readZipStream(inputStream, connection); // assume single file in zip and decompress it
 337             else
 338                 buffer = this.readNonSpecificStream(inputStream, connection);
 339
 340             this.contentType = connection.getContentType();
 341         }
 342         finally
 343         {
 344             if (inputStream != null)
 345                 try
 346                 {
 347                     inputStream.close();
 348                 }
 349                 catch (IOException e)
 350                 {
 351                     WorldWind.logger().log(java.util.logging.Level.FINE, WorldWind.retrieveErrMsg(
 352                         "URLRetriever.ExceptionClosingInputStreamToConnection") + connection.getURL());
 353                 }
 354         }
 355
 356         return buffer;
 357     }
 358
 359     private ByteBuffer readNonSpecificStream(InputStream inputStream, URLConnection connection) throws IOException
 360     {
 361         if (inputStream == null)
 362         {
 363             String message = WorldWind.retrieveErrMsg("URLRetriever.InputStreamNullFor") + connection.getURL();
 364             WorldWind.logger().log(java.util.logging.Level.FINE, message);
 365             throw new IllegalArgumentException(message);
 366         }
 367
 368         if (this.contentLength < 1)
 369         {
 370             return readNonSpecificStreamUnknownLength(inputStream);
 371         }
 372
 373         ReadableByteChannel channel = Channels.newChannel(inputStream);
 374         ByteBuffer buffer = ByteBuffer.allocate(this.contentLength);
 375
 376         int numBytesRead = 0;
 377         while (!this.interrupted() && numBytesRead >= 0 && numBytesRead < buffer.limit())
 378         {
 379             int count = channel.read(buffer);
 380             if (count > 0)
 381                 this.contentLengthRead.getAndAdd(numBytesRead += count);
 382         }
 383
 384         if (buffer != null)
 385             buffer.flip();
 386
 387         return buffer;
 388     }
 389
 390     private ByteBuffer readNonSpecificStreamUnknownLength(InputStream inputStream) throws IOException
 391     {
 392         final int PAGE_SIZE = 4096;
 393
 394         ReadableByteChannel channel = Channels.newChannel(inputStream);
 395         ByteBuffer buffer = ByteBuffer.allocate(PAGE_SIZE);
 396
 397         int count = 0;
 398         int numBytesRead = 0;
 399         while (!this.interrupted() && count >= 0)
 400         {
 401             count = channel.read(buffer);
 402             if (count > 0)
 403                 this.contentLengthRead.getAndAdd(numBytesRead += count);
 404
 405             if (count > 0 && !buffer.hasRemaining())
 406             {
 407                 ByteBuffer biggerBuffer = ByteBuffer.allocate(buffer.limit() + PAGE_SIZE);
 408                 biggerBuffer.put((ByteBuffer) buffer.rewind());
 409                 buffer = biggerBuffer;
 410             }
 411         }
 412
 413         if (buffer != null)
 414             buffer.flip();
 415
 416         return buffer;
 417     }
 418
 419     /**
 420      * @param inputStream
 421      * @param connection
 422      * @return
 423      * @throws java.io.IOException
 424      * @throws IllegalArgumentException if <code>inputStream</code> is null
 425      */
 426     private ByteBuffer readZipStream(InputStream inputStream, URLConnection connection) throws IOException
 427     {
 428         ZipInputStream zis = new ZipInputStream(inputStream);
 429         ZipEntry ze = zis.getNextEntry();
 430         if (ze == null)
 431         {
 432             WorldWind.logger().log(java.util.logging.Level.FINE, WorldWind.retrieveErrMsg("URLRetriever.NoZipEntryFor")
 433                 + connection.getURL());
 434             return null;
 435         }
 436
 437         ByteBuffer buffer = null;
 438         if (ze.getSize() > 0)
 439         {
 440             buffer = ByteBuffer.allocate((int) ze.getSize());
 441
 442             byte[] inputBuffer = new byte[8192];
 443             while (buffer.hasRemaining())
 444             {
 445                 int count = zis.read(inputBuffer);
 446                 if (count > 0)
 447                 {
 448                     buffer.put(inputBuffer, 0, count);
 449                     this.contentLengthRead.getAndAdd(buffer.position() + 1);
 450                 }
 451             }
 452         }
 453         if (buffer != null)
 454             buffer.flip();
 455
 456         return buffer;
 457     }
 458
 459     @Override
 460     public boolean equals(Object o)
 461     {
 462         if (this == o)
 463             return true;
 464         if (o == null || getClass() != o.getClass())
 465             return false;
 466
 467         final URLRetriever that = (URLRetriever) o;
 468
 469         // Retrievers are considered identical if they are for the same URL. This convention is used by the
 470         // retrieval service to filter out duplicate retreival requests.
 471         return !(url != null ? !url.toString().contentEquals(that.url.toString()) : that.url != null);
 472     }
 473
 474     @Override
 475     public int hashCode()
 476     {
 477         int result;
 478         result = (url != null ? url.hashCode() : 0);
 479         return result;
 480     }
 481
 482     @Override
 483     public String toString()
 484     {
 485         return this.getName() != null ? this.getName() : super.toString();
 486     }
 487 }