test/ch/cyberduck/core/transfer/UploadTransferTest.java

   1 package ch.cyberduck.core.transfer;
   2
   3 import ch.cyberduck.core.*;
   4 import ch.cyberduck.core.exception.AccessDeniedException;
   5 import ch.cyberduck.core.exception.BackgroundException;
   6 import ch.cyberduck.core.features.Delete;
   7 import ch.cyberduck.core.features.Find;
   8 import ch.cyberduck.core.features.Move;
   9 import ch.cyberduck.core.features.Write;
  10 import ch.cyberduck.core.filter.UploadRegexFilter;
  11 import ch.cyberduck.core.ftp.FTPSession;
  12 import ch.cyberduck.core.ftp.FTPTLSProtocol;
  13 import ch.cyberduck.core.io.DisabledStreamListener;
  14 import ch.cyberduck.core.io.StreamListener;
  15 import ch.cyberduck.core.local.LocalTouchFactory;
  16 import ch.cyberduck.core.openstack.SwiftSession;
  17 import ch.cyberduck.core.s3.S3Session;
  18 import ch.cyberduck.core.test.NullLocal;
  19 import ch.cyberduck.core.test.NullSession;
  20 import ch.cyberduck.core.transfer.upload.AbstractUploadFilter;
  21 import ch.cyberduck.core.transfer.upload.UploadFilterOptions;
  22 import ch.cyberduck.core.transfer.upload.UploadRegexPriorityComparator;
  23 import ch.cyberduck.core.worker.SingleTransferWorker;
  24
  25 import org.apache.commons.io.IOUtils;
  26 import org.apache.commons.lang3.RandomStringUtils;
  27 import org.junit.Test;
  28
  29 import java.io.OutputStream;
  30 import java.util.Collections;
  31 import java.util.EnumSet;
  32 import java.util.HashMap;
  33 import java.util.List;
  34 import java.util.Map;
  35 import java.util.UUID;
  36 import java.util.concurrent.atomic.AtomicBoolean;
  37 import java.util.concurrent.atomic.AtomicInteger;
  38
  39 import static org.junit.Assert.*;
  40
  41 /**
  42  * @version $Id$
  43  */
  44 public class UploadTransferTest extends AbstractTestCase {
  45
  46     @Test
  47     public void testChildrenEmpty() throws Exception {
  48         final Path root = new Path("/t", EnumSet.of(Path.Type.directory)) {
  49         };
  50         Transfer t = new UploadTransfer(new Host("t"), root,
  51                 new Local(System.getProperty("java.io.tmpdir"), UUID.randomUUID().toString()));
  52         assertTrue(t.list(new NullSession(new Host("t")), root, new NullLocal("t") {
  53             @Override
  54             public AttributedList<Local> list() {
  55                 return AttributedList.emptyList();
  56             }
  57         }, new DisabledListProgressListener()).isEmpty());
  58     }
  59
  60     @Test
  61     public void testList() throws Exception {
  62         final NullLocal local = new NullLocal("t") {
  63             @Override
  64             public AttributedList<Local> list() {
  65                 AttributedList<Local> l = new AttributedList<Local>();
  66                 l.add(new NullLocal(this.getAbsolute(), "c"));
  67                 return l;
  68             }
  69         };
  70         final Path root = new Path("/t", EnumSet.of(Path.Type.file));
  71         Transfer t = new UploadTransfer(new Host("t"), root, local);
  72         assertEquals(Collections.<TransferItem>singletonList(new TransferItem(new Path("/t/c", EnumSet.of(Path.Type.file)), new NullLocal("t/c"))),
  73                 t.list(new NullSession(new Host("t")), root, local, new DisabledListProgressListener()));
  74     }
  75
  76     @Test
  77     public void testListSorted() throws Exception {
  78         final NullLocal local = new NullLocal("t") {
  79             @Override
  80             public AttributedList<Local> list() {
  81                 AttributedList<Local> l = new AttributedList<Local>();
  82                 l.add(new NullLocal(this.getAbsolute(), "c"));
  83                 l.add(new NullLocal(this.getAbsolute(), "c.html"));
  84                 return l;
  85             }
  86         };
  87         final Path root = new Path("/t", EnumSet.of(Path.Type.file));
  88         {
  89             Transfer t = new UploadTransfer(new Host("t"), Collections.singletonList(new TransferItem(root, local)),
  90                     new UploadRegexFilter(), new UploadRegexPriorityComparator(".*\\.html"));
  91             final List<TransferItem> list = t.list(new NullSession(new Host("t")), root, local, new DisabledListProgressListener());
  92             assertEquals(new NullLocal(local.getAbsolute(), "c.html"), list.get(0).local);
  93             assertEquals(new NullLocal(local.getAbsolute(), "c"), list.get(1).local);
  94         }
  95         {
  96             Transfer t = new UploadTransfer(new Host("t"), root, local, new UploadRegexFilter());
  97             final List<TransferItem> list = t.list(new NullSession(new Host("t")), root, local, new DisabledListProgressListener());
  98             assertEquals(new NullLocal(local.getAbsolute(), "c.html"), list.get(1).local);
  99             assertEquals(new NullLocal(local.getAbsolute(), "c"), list.get(0).local);
 100         }
 101     }
 102
 103     @Test
 104     public void testCacheResume() throws Exception {
 105         final AtomicInteger c = new AtomicInteger();
 106         final NullLocal local = new NullLocal("t") {
 107             @Override
 108             public AttributedList<Local> list() {
 109                 AttributedList<Local> l = new AttributedList<Local>();
 110                 l.add(new NullLocal(this.getAbsolute(), "a") {
 111                     @Override
 112                     public boolean exists() {
 113                         return true;
 114                     }
 115                 });
 116                 l.add(new NullLocal(this.getAbsolute(), "b") {
 117                     @Override
 118                     public boolean exists() {
 119                         return true;
 120                     }
 121                 });
 122                 l.add(new NullLocal(this.getAbsolute(), "c") {
 123                     @Override
 124                     public boolean exists() {
 125                         return true;
 126                     }
 127                 });
 128                 return l;
 129             }
 130
 131             @Override
 132             public boolean exists() {
 133                 return true;
 134             }
 135         };
 136         final Path root = new Path("/t", EnumSet.of(Path.Type.directory));
 137         final NullSession session = new NullSession(new Host("t")) {
 138             @Override
 139             public AttributedList<Path> list(final Path file, final ListProgressListener listener) {
 140                 if(file.equals(root.getParent())) {
 141                     c.incrementAndGet();
 142                 }
 143                 return AttributedList.emptyList();
 144             }
 145         };
 146         Transfer t = new UploadTransfer(new Host("t"), root, local) {
 147             @Override
 148             public void transfer(final Session<?> session, final Path file, Local local,
 149                                  final TransferOptions options, final TransferStatus status,
 150                                  final ConnectionCallback callback,
 151                                  final ProgressListener listener, final StreamListener streamListener) throws BackgroundException {
 152                 assertEquals(true, options.resumeRequested);
 153             }
 154         };
 155         final TransferOptions options = new TransferOptions();
 156         options.resumeRequested = true;
 157         new SingleTransferWorker(session, t, options, new TransferSpeedometer(t), new DisabledTransferPrompt() {
 158             @Override
 159             public TransferAction prompt(final TransferItem file) {
 160                 fail();
 161                 return null;
 162             }
 163         }, new DisabledTransferErrorCallback(), new DisabledTransferItemCallback(),
 164                 new DisabledProgressListener(), new DisabledStreamListener(), new DisabledLoginCallback()).run(session);
 165         assertEquals(1, c.get());
 166     }
 167
 168     @Test
 169     public void testCacheRename() throws Exception {
 170         final AtomicInteger c = new AtomicInteger();
 171         final NullLocal local = new NullLocal("t") {
 172             @Override
 173             public AttributedList<Local> list() {
 174                 AttributedList<Local> l = new AttributedList<Local>();
 175                 l.add(new NullLocal(this.getAbsolute(), "a") {
 176                     @Override
 177                     public boolean exists() {
 178                         return true;
 179                     }
 180                 });
 181                 l.add(new NullLocal(this.getAbsolute(), "b") {
 182                     @Override
 183                     public boolean exists() {
 184                         return true;
 185                     }
 186                 });
 187                 l.add(new NullLocal(this.getAbsolute(), "c") {
 188                     @Override
 189                     public boolean exists() {
 190                         return true;
 191                     }
 192                 });
 193                 return l;
 194             }
 195
 196             @Override
 197             public boolean exists() {
 198                 return true;
 199             }
 200         };
 201         final Path root = new Path("/t", EnumSet.of(Path.Type.directory));
 202         final NullSession session = new NullSession(new Host("t")) {
 203             @Override
 204             public AttributedList<Path> list(final Path file, final ListProgressListener listener) {
 205                 c.incrementAndGet();
 206                 return AttributedList.emptyList();
 207             }
 208         };
 209         Transfer t = new UploadTransfer(new Host("t"), root, local) {
 210             @Override
 211             public void transfer(final Session<?> session, final Path file, Local local,
 212                                  final TransferOptions options, final TransferStatus status,
 213                                  final ConnectionCallback callback,
 214                                  final ProgressListener listener, final StreamListener streamListener) throws BackgroundException {
 215                 //
 216             }
 217         };
 218         new SingleTransferWorker(session, t, new TransferOptions(), new TransferSpeedometer(t), new DisabledTransferPrompt() {
 219             @Override
 220             public TransferAction prompt(final TransferItem file) {
 221                 return TransferAction.rename;
 222             }
 223         }, new DisabledTransferErrorCallback(), new DisabledTransferItemCallback(),
 224                 new DisabledProgressListener(), new DisabledStreamListener(), new DisabledLoginCallback()).run(session);
 225         assertEquals(1, c.get());
 226     }
 227
 228     @Test
 229     public void testPrepareUploadOverrideFilter() throws Exception {
 230         final Host host = new Host(new FTPTLSProtocol(), "test.cyberduck.ch", new Credentials(
 231                 properties.getProperty("ftp.user"), properties.getProperty("ftp.password")
 232         ));
 233         final FTPSession session = new FTPSession(host);
 234         session.open(new DisabledHostKeyCallback(), new DisabledTranscriptListener());
 235         session.login(new DisabledPasswordStore(), new DisabledLoginCallback(), new DisabledCancelCallback());
 236         final Path test = new Path("/transfer", EnumSet.of(Path.Type.directory));
 237         final String name = UUID.randomUUID().toString();
 238         final Local local = new Local(System.getProperty("java.io.tmpdir"), "transfer");
 239         LocalTouchFactory.get().touch(local);
 240         LocalTouchFactory.get().touch(new Local(local, name));
 241         final Transfer transfer = new UploadTransfer(host, test, local);
 242         Map<Path, TransferStatus> table
 243                 = new HashMap<Path, TransferStatus>();
 244         final SingleTransferWorker worker = new SingleTransferWorker(session, transfer, new TransferOptions(),
 245                 new TransferSpeedometer(transfer), new DisabledTransferPrompt() {
 246             @Override
 247             public TransferAction prompt(final TransferItem file) {
 248                 fail();
 249                 return null;
 250             }
 251         }, new DisabledTransferErrorCallback(), new DisabledTransferItemCallback(),
 252                 new DisabledProgressListener(), new DisabledStreamListener(), new DisabledLoginCallback(), table);
 253         worker.prepare(test, new Local(System.getProperty("java.io.tmpdir"), "transfer"), new TransferStatus().exists(true),
 254                 TransferAction.overwrite);
 255         assertEquals(new TransferStatus().exists(true), table.get(test));
 256         final TransferStatus expected = new TransferStatus();
 257         assertEquals(expected, table.get(new Path("/transfer/" + name, EnumSet.of(Path.Type.file))));
 258     }
 259
 260     @Test
 261     public void testPrepareUploadResumeFilter() throws Exception {
 262         final Host host = new Host(new FTPTLSProtocol(), "test.cyberduck.ch", new Credentials(
 263                 properties.getProperty("ftp.user"), properties.getProperty("ftp.password")
 264         ));
 265         final FTPSession session = new FTPSession(host);
 266         session.open(new DisabledHostKeyCallback(), new DisabledTranscriptListener());
 267         session.login(new DisabledPasswordStore(), new DisabledLoginCallback(), new DisabledCancelCallback());
 268         final Path test = new Path("/transfer", EnumSet.of(Path.Type.directory));
 269         final String name = "test";
 270         final Local local = new Local(System.getProperty("java.io.tmpdir") + "/transfer", name);
 271         LocalTouchFactory.get().touch(local);
 272         final OutputStream out = local.getOutputStream(false);
 273         final byte[] bytes = RandomStringUtils.random(1000).getBytes();
 274         IOUtils.write(bytes, out);
 275         IOUtils.closeQuietly(out);
 276         final NullLocal directory = new NullLocal(System.getProperty("java.io.tmpdir"), "transfer") {
 277             @Override
 278             public AttributedList<Local> list() throws AccessDeniedException {
 279                 return new AttributedList<Local>(Collections.<Local>singletonList(local));
 280             }
 281         };
 282         final Transfer transfer = new UploadTransfer(host, test, directory);
 283         final Map<Path, TransferStatus> table
 284                 = new HashMap<Path, TransferStatus>();
 285         final SingleTransferWorker worker = new SingleTransferWorker(session, transfer, new TransferOptions(),
 286                 new TransferSpeedometer(transfer), new DisabledTransferPrompt() {
 287             @Override
 288             public TransferAction prompt(final TransferItem file) {
 289                 fail();
 290                 return null;
 291             }
 292         }, new DisabledTransferErrorCallback(), new DisabledTransferItemCallback(),
 293                 new DisabledProgressListener(), new DisabledStreamListener(), new DisabledLoginCallback(), table);
 294         worker.prepare(test, directory, new TransferStatus().exists(true),
 295                 TransferAction.resume);
 296         assertEquals(new TransferStatus().exists(true), table.get(test));
 297         final TransferStatus expected = new TransferStatus().exists(true);
 298         expected.setAppend(true);
 299         // Remote size
 300         expected.setOffset(5L);
 301         // Local size
 302         expected.setLength(bytes.length - 5L);
 303         assertEquals(expected, table.get(new Path("/transfer/" + name, EnumSet.of(Path.Type.file))));
 304         local.delete();
 305     }
 306
 307     @Test
 308     public void testUploadTemporaryName() throws Exception {
 309         final Path test = new Path("/f", EnumSet.of(Path.Type.file));
 310         final AtomicBoolean moved = new AtomicBoolean();
 311         final Host host = new Host("t");
 312         final Session session = new NullSession(host) {
 313             @Override
 314             public <T> T getFeature(final Class<T> type) {
 315                 if(type.equals(Find.class)) {
 316                     return (T) new Find() {
 317                         @Override
 318                         public boolean find(final Path f) throws BackgroundException {
 319                             return true;
 320                         }
 321
 322                         @Override
 323                         public Find withCache(PathCache cache) {
 324                             return this;
 325                         }
 326                     };
 327                 }
 328                 if(type.equals(Move.class)) {
 329                     return (T) new Move() {
 330                         @Override
 331                         public void move(final Path file, final Path renamed, boolean exists, final Delete.Callback callback) throws BackgroundException {
 332                             assertEquals(test, renamed);
 333                             moved.set(true);
 334                         }
 335
 336                         @Override
 337                         public boolean isSupported(final Path file) {
 338                             return true;
 339                         }
 340                     };
 341                 }
 342                 if(type.equals(ch.cyberduck.core.features.Attributes.class)) {
 343                     return (T) new ch.cyberduck.core.features.Attributes() {
 344                         @Override
 345                         public PathAttributes find(final Path file) throws BackgroundException {
 346                             return new PathAttributes();
 347                         }
 348
 349                         @Override
 350                         public ch.cyberduck.core.features.Attributes withCache(PathCache cache) {
 351                             return this;
 352                         }
 353                     };
 354                 }
 355                 if(type.equals(Write.class)) {
 356                     return (T) new Write() {
 357                         @Override
 358                         public OutputStream write(final Path file, final TransferStatus status) throws BackgroundException {
 359                             fail();
 360                             return null;
 361                         }
 362
 363                         @Override
 364                         public Append append(final Path file, final Long length, final PathCache cache) throws BackgroundException {
 365                             fail();
 366                             return new Write.Append(0L);
 367                         }
 368
 369                         @Override
 370                         public boolean temporary() {
 371                             return true;
 372                         }
 373                     };
 374                 }
 375                 return null;
 376             }
 377         };
 378         final AtomicBoolean set = new AtomicBoolean();
 379         final Map<Path, TransferStatus> table
 380                 = new HashMap<Path, TransferStatus>();
 381         final Local local = new Local(System.getProperty("java.io.tmpdir"), UUID.randomUUID().toString());
 382         LocalTouchFactory.get().touch(local);
 383         final Transfer transfer = new UploadTransfer(host, test, local) {
 384             @Override
 385             public void transfer(final Session<?> session, final Path file, Local local,
 386                                  final TransferOptions options, final TransferStatus status,
 387                                  final ConnectionCallback callback, final ProgressListener listener, final StreamListener streamListener) throws BackgroundException {
 388                 assertEquals(table.get(test).getRename().remote, file);
 389                 status.setComplete();
 390                 set.set(true);
 391             }
 392
 393             @Override
 394             public AbstractUploadFilter filter(final Session<?> session, final TransferAction action, final ProgressListener listener) {
 395                 return super.filter(session, action, listener).withOptions(new UploadFilterOptions().withTemporary(true));
 396             }
 397         };
 398         final SingleTransferWorker worker = new SingleTransferWorker(session, transfer, new TransferOptions(),
 399                 new TransferSpeedometer(transfer), new DisabledTransferPrompt() {
 400             @Override
 401             public TransferAction prompt(final TransferItem file) {
 402                 fail();
 403                 return null;
 404             }
 405         }, new DisabledTransferErrorCallback(), new DisabledTransferItemCallback(),
 406                 new DisabledProgressListener(), new DisabledStreamListener(), new DisabledLoginCallback(), table);
 407         worker.prepare(test, local, new TransferStatus().exists(true), TransferAction.overwrite);
 408         assertNotNull(table.get(test));
 409         assertNotNull(table.get(test).getRename());
 410         worker.transfer(new TransferItem(test, local), TransferAction.overwrite);
 411         assertTrue(set.get());
 412         assertTrue(moved.get());
 413     }
 414
 415     @Test
 416     public void testTemporaryDisabledLargeUpload() throws Exception {
 417         final Host h = new Host("h");
 418         final AbstractUploadFilter f = new UploadTransfer(h, Collections.<TransferItem>emptyList())
 419                 .filter(new SwiftSession(h), TransferAction.overwrite, new DisabledProgressListener());
 420         final Path file = new Path("/t", EnumSet.of(Path.Type.file));
 421         final TransferStatus status = f.prepare(file, new NullLocal("t"), new TransferStatus());
 422         assertNull(status.getRename().local);
 423         assertNull(status.getRename().remote);
 424     }
 425
 426     @Test
 427     public void testTemporaryDisabledMultipartUpload() throws Exception {
 428         final Host h = new Host("h");
 429         final AbstractUploadFilter f = new UploadTransfer(h, Collections.<TransferItem>emptyList())
 430                 .filter(new S3Session(h), TransferAction.overwrite, new DisabledProgressListener());
 431         final Path file = new Path("/t", EnumSet.of(Path.Type.file));
 432         final TransferStatus status = f.prepare(file, new NullLocal("t"), new TransferStatus());
 433         assertNull(status.getRename().local);
 434         assertNull(status.getRename().remote);
 435     }
 436
 437 }