gov/nasa/worldwind/URLRetriever.java

   1 /*
   2 Copyright (C) 2001, 2006 United States Government
   3 as represented by the Administrator of the
   4 National Aeronautics and Space Administration.
   5 All Rights Reserved.
   6 */
   7 package gov.nasa.worldwind;
   8
   9 import java.util.concurrent.atomic.*;
  10 import java.util.zip.*;
  11 import java.nio.*;
  12 import java.nio.channels.*;
  13 import java.net.*;
  14 import java.io.*;
  15
  16 /**
  17  * @author Tom Gaskins
  18  * @version $Id: URLRetriever.java 1792 2007-05-08 21:28:37Z tgaskins $
  19  */
  20 public abstract class URLRetriever extends WWObjectImpl implements Retriever
  21 {
  22     private volatile String state = RETRIEVER_STATE_NOT_STARTED;
  23     private volatile int contentLength = 0;
  24     private AtomicInteger contentLengthRead = new AtomicInteger(0);
  25     private volatile String contentType;
  26     private volatile ByteBuffer byteBuffer;
  27     private volatile URLConnection connection;
  28     private final URL url;
  29     private final RetrievalPostProcessor postProcessor;
  30     private int connectTimeout = Configuration.getIntegerValue(AVKey.URL_CONNECT_TIMEOUT, 8000);
  31     private int readTimeout = Configuration.getIntegerValue(AVKey.URL_READ_TIMEOUT, 5000);
  32     private long submitTime;
  33     private long beginTime;
  34     private long endTime;
  35
  36     /**
  37      * @param url
  38      * @param postProcessor
  39      * @throws IllegalArgumentException if <code>url</code> or <code>postProcessor</code> is null
  40      */
  41     public URLRetriever(URL url, RetrievalPostProcessor postProcessor)
  42     {
  43         if (url == null)
  44         {
  45             String message = WorldWind.retrieveErrMsg("nullValue.URLIsNull");
  46             WorldWind.logger().log(java.util.logging.Level.FINE, message);
  47             throw new IllegalArgumentException(message);
  48         }
  49         if (postProcessor == null)
  50         {
  51             String message = WorldWind.retrieveErrMsg("nullValue.PostProcessorIsNull");
  52             WorldWind.logger().log(java.util.logging.Level.FINE, message);
  53             throw new IllegalArgumentException(message);
  54         }
  55
  56         this.url = url;
  57         this.postProcessor = postProcessor;
  58     }
  59
  60     public final URL getUrl()
  61     {
  62         return url;
  63     }
  64
  65     public final int getContentLength()
  66     {
  67         return this.contentLength;
  68     }
  69
  70     protected void setContentLengthRead(int length)
  71     {
  72         this.contentLengthRead.set(length);
  73     }
  74
  75     public final int getContentLengthRead()
  76     {
  77         return this.contentLengthRead.get();
  78     }
  79
  80     public final String getContentType()
  81     {
  82         return this.contentType;
  83     }
  84
  85     public final ByteBuffer getBuffer()
  86     {
  87         return this.byteBuffer;
  88     }
  89
  90     public final String getName()
  91     {
  92         return this.url.toString();
  93     }
  94
  95     public final String getState()
  96     {
  97         return this.state;
  98     }
  99
 100     protected final URLConnection getConnection()
 101     {
 102         return this.connection;
 103     }
 104
 105     public final RetrievalPostProcessor getPostProcessor()
 106     {
 107         return postProcessor;
 108     }
 109
 110     public final int getConnectTimeout()
 111     {
 112         return connectTimeout;
 113     }
 114
 115     /**
 116      * @param connectTimeout
 117      * @throws IllegalArgumentException if <code>connectTimeOut</code> is less than zero
 118      */
 119     public final void setConnectTimeout(int connectTimeout)
 120     {
 121         if (connectTimeout < 0)
 122         {
 123             String message = WorldWind.retrieveErrMsg("URLRetriever.ConnectTimeoutLessThanZero");
 124             WorldWind.logger().log(java.util.logging.Level.FINE, message);
 125             throw new IllegalArgumentException(message);
 126         }
 127
 128         this.connectTimeout = connectTimeout;
 129     }
 130
 131     public long getSubmitTime()
 132     {
 133         return submitTime;
 134     }
 135
 136     public void setSubmitTime(long submitTime)
 137     {
 138         this.submitTime = submitTime;
 139     }
 140
 141     public long getBeginTime()
 142     {
 143         return beginTime;
 144     }
 145
 146     public void setBeginTime(long beginTime)
 147     {
 148         this.beginTime = beginTime;
 149     }
 150
 151     public long getEndTime()
 152     {
 153         return endTime;
 154     }
 155
 156     public void setEndTime(long endTime)
 157     {
 158         this.endTime = endTime;
 159     }
 160
 161     public final Retriever call() throws Exception
 162     {
 163         if (this.interrupted())
 164             return this;
 165
 166         try
 167         {
 168             this.setState(RETRIEVER_STATE_STARTED);
 169
 170             if (!this.interrupted())
 171             {
 172                 this.setState(RETRIEVER_STATE_CONNECTING);
 173                 this.connection = this.openConnection();
 174             }
 175
 176             if (!this.interrupted())
 177             {
 178                 this.setState(RETRIEVER_STATE_READING);
 179                 this.byteBuffer = this.read();
 180             }
 181
 182             if (!this.interrupted())
 183                 this.setState(RETRIEVER_STATE_SUCCESSFUL);
 184         }
 185         catch (Exception e)
 186         {
 187             this.setState(RETRIEVER_STATE_ERROR);
 188             if (!(e instanceof java.net.SocketTimeoutException))
 189             {
 190                 String message = WorldWind.retrieveErrMsg("URLRetriever.ErrorAttemptingToRetrieve")
 191                     + this.url.toString();
 192                 WorldWind.logger().log(java.util.logging.Level.FINE, message, e);
 193             }
 194             throw e;
 195         }
 196         finally
 197         {
 198             this.end();
 199         }
 200
 201         return this;
 202     }
 203
 204     private void setState(String state)
 205     {
 206         String oldState = this.state;
 207         this.state = state;
 208         this.firePropertyChange(AVKey.RETRIEVER_STATE, oldState, this.state);
 209     }
 210
 211     private boolean interrupted()
 212     {
 213         if (Thread.currentThread().isInterrupted())
 214         {
 215             this.setState(RETRIEVER_STATE_INTERRUPTED);
 216             String message = WorldWind.retrieveErrMsg("URLRetriever.RetrievalInterruptedFor") + this.url.toString();
 217             WorldWind.logger().log(java.util.logging.Level.FINER, message);
 218             return true;
 219         }
 220         return false;
 221     }
 222
 223     private URLConnection openConnection() throws IOException
 224     {
 225         try
 226         {
 227             this.connection = this.url.openConnection();
 228         }
 229         catch (java.io.IOException e)
 230         {
 231             String message = WorldWind.retrieveErrMsg("URLRetriever.ErrorOpeningConnection") + this.url.toString()
 232                 + " " + e.getLocalizedMessage();
 233             WorldWind.logger().log(java.util.logging.Level.FINE, message, e);
 234             throw e;
 235         }
 236
 237         if (this.connection == null) // java.net.URL docs imply that this won't happen. We check anyway.
 238         {
 239             String message = WorldWind.retrieveErrMsg("URLRetriever.NullReturnedFromOpenConnection") + this.url;
 240             WorldWind.logger().log(java.util.logging.Level.FINE, message);
 241             throw new IllegalStateException(message);
 242         }
 243
 244         this.connection.setConnectTimeout(this.connectTimeout);
 245         this.connection.setReadTimeout(this.readTimeout);
 246
 247         return connection;
 248     }
 249
 250     private void end() throws Exception
 251     {
 252         try
 253         {
 254             if (this.postProcessor != null)
 255             {
 256                 this.byteBuffer = this.postProcessor.run(this);
 257             }
 258         }
 259         catch (Exception e)
 260         {
 261             this.setState(RETRIEVER_STATE_ERROR);
 262             String message = WorldWind.retrieveErrMsg("URLRetriever.ErrorPostProcessing") + this.url.toString();
 263             WorldWind.logger().log(java.util.logging.Level.FINE, message + " " + e.getLocalizedMessage());
 264             throw e;
 265         }
 266     }
 267
 268     private ByteBuffer read() throws Exception
 269     {
 270         try
 271         {
 272             ByteBuffer buffer = this.doRead(this.connection);
 273             if (buffer == null)
 274                 this.contentLength = 0;
 275             return buffer;
 276         }
 277         catch (Exception e)
 278         {
 279             if (!(e instanceof java.net.SocketTimeoutException))
 280             {
 281                 String message = WorldWind.retrieveErrMsg("URLRetriever.ErrorReadingFromConnection")
 282                     + this.url.toString() + e.getLocalizedMessage();
 283                 WorldWind.logger().log(java.util.logging.Level.FINE, message, e);
 284             }
 285             throw e;
 286         }
 287     }
 288
 289     /**
 290      * @param connection
 291      * @return
 292      * @throws Exception
 293      * @throws IllegalArgumentException if <code>connection</code> is null
 294      */
 295     protected ByteBuffer doRead(URLConnection connection) throws Exception
 296     {
 297         if (connection == null)
 298         {
 299             String msg = WorldWind.retrieveErrMsg("nullValue.ConnectionIsNull");
 300             WorldWind.logger().log(java.util.logging.Level.FINE, msg);
 301             throw new IllegalArgumentException(msg);
 302         }
 303
 304         this.contentLength = this.connection.getContentLength();
 305
 306         ByteBuffer buffer;
 307         InputStream inputStream = null;
 308         try
 309         {
 310             inputStream = this.connection.getInputStream();
 311             if (inputStream == null)
 312             {
 313                 WorldWind.logger().log(java.util.logging.Level.FINE, WorldWind.retrieveErrMsg(
 314                     "URLRetriever.InputStreamFromConnectionNull") + connection.getURL());
 315                 return null;
 316             }
 317             // TODO: Make decompression of zip file configurable
 318             // TODO: Make this more generally configurable based on content type
 319             // todo: make a zip reader that handles streams of unknown length
 320             // TODO: add a gzip reader
 321             // TODO: this code is checking content type for compression when it should be checking content encoding,
 322             // but the WW servers are sending application/zip as the content type, and null for the content encoding.
 323             if (connection.getContentType().equalsIgnoreCase("application/zip"))
 324                 buffer = this.readZipStream(inputStream, connection); // assume single file in zip and decompress it
 325             else
 326                 buffer = this.readNonSpecificStream(inputStream, connection);
 327
 328             this.contentType = connection.getContentType();
 329         }
 330         finally
 331         {
 332             if (inputStream != null)
 333                 try
 334                 {
 335                     inputStream.close();
 336                 }
 337                 catch (IOException e)
 338                 {
 339                     WorldWind.logger().log(java.util.logging.Level.FINE, WorldWind.retrieveErrMsg(
 340                         "URLRetriever.ExceptionClosingInputStreamToConnection") + connection.getURL());
 341                 }
 342         }
 343
 344         return buffer;
 345     }
 346
 347     private ByteBuffer readNonSpecificStream(InputStream inputStream, URLConnection connection) throws IOException
 348     {
 349         if (inputStream == null)
 350         {
 351             String message = WorldWind.retrieveErrMsg("URLRetriever.InputStreamNullFor") + connection.getURL();
 352             WorldWind.logger().log(java.util.logging.Level.FINE, message);
 353             throw new IllegalArgumentException(message);
 354         }
 355
 356         if (this.contentLength < 1)
 357         {
 358             return readNonSpecificStreamUnknownLength(inputStream);
 359         }
 360
 361         ReadableByteChannel channel = Channels.newChannel(inputStream);
 362         ByteBuffer buffer = ByteBuffer.allocate(this.contentLength);
 363
 364         int numBytesRead = 0;
 365         while (!this.interrupted() && numBytesRead >= 0 && numBytesRead < buffer.limit())
 366         {
 367             int count = channel.read(buffer);
 368             if (count > 0)
 369                 this.contentLengthRead.getAndAdd(numBytesRead += count);
 370         }
 371
 372         if (buffer != null)
 373             buffer.flip();
 374
 375         return buffer;
 376     }
 377
 378     private ByteBuffer readNonSpecificStreamUnknownLength(InputStream inputStream) throws IOException
 379     {
 380         final int PAGE_SIZE = 4096;
 381
 382         ReadableByteChannel channel = Channels.newChannel(inputStream);
 383         ByteBuffer buffer = ByteBuffer.allocate(PAGE_SIZE);
 384
 385         int count = 0;
 386         int numBytesRead = 0;
 387         while (!this.interrupted() && count >= 0)
 388         {
 389             count = channel.read(buffer);
 390             if (count > 0)
 391                 this.contentLengthRead.getAndAdd(numBytesRead += count);
 392
 393             if (count > 0 && !buffer.hasRemaining())
 394             {
 395                 ByteBuffer biggerBuffer = ByteBuffer.allocate(buffer.limit() + PAGE_SIZE);
 396                 biggerBuffer.put((ByteBuffer) buffer.rewind());
 397                 buffer = biggerBuffer;
 398             }
 399         }
 400
 401         if (buffer != null)
 402             buffer.flip();
 403
 404         return buffer;
 405     }
 406
 407     /**
 408      * @param inputStream
 409      * @param connection
 410      * @return
 411      * @throws java.io.IOException
 412      * @throws IllegalArgumentException if <code>inputStream</code> is null
 413      */
 414     private ByteBuffer readZipStream(InputStream inputStream, URLConnection connection) throws IOException
 415     {
 416         ZipInputStream zis = new ZipInputStream(inputStream);
 417         ZipEntry ze = zis.getNextEntry();
 418         if (ze == null)
 419         {
 420             WorldWind.logger().log(java.util.logging.Level.FINE, WorldWind.retrieveErrMsg("URLRetriever.NoZipEntryFor")
 421                 + connection.getURL());
 422             return null;
 423         }
 424
 425         ByteBuffer buffer = null;
 426         if (ze.getSize() > 0)
 427         {
 428             buffer = ByteBuffer.allocate((int) ze.getSize());
 429
 430             byte[] inputBuffer = new byte[8192];
 431             while (buffer.hasRemaining())
 432             {
 433                 int count = zis.read(inputBuffer);
 434                 if (count > 0)
 435                 {
 436                     buffer.put(inputBuffer, 0, count);
 437                     this.contentLengthRead.getAndAdd(buffer.position() + 1);
 438                 }
 439             }
 440         }
 441         if (buffer != null)
 442             buffer.flip();
 443
 444         return buffer;
 445     }
 446
 447     @Override
 448     public boolean equals(Object o)
 449     {
 450         if (this == o)
 451             return true;
 452         if (o == null || getClass() != o.getClass())
 453             return false;
 454
 455         final URLRetriever that = (URLRetriever) o;
 456
 457         // Retrievers are considered identical if they are for the same URL. This convention is used by the
 458         // retrieval service to filter out duplicate retreival requests.
 459         return !(url != null ? !url.equals(that.url) : that.url != null);
 460     }
 461
 462     @Override
 463     public int hashCode()
 464     {
 465         int result;
 466         result = (url != null ? url.hashCode() : 0);
 467         return result;
 468     }
 469
 470     @Override
 471     public String toString()
 472     {
 473         return this.getName() != null ? this.getName() : super.toString();
 474     }
 475 }