source/ch/cyberduck/core/openstack/SwiftSegmentService.java

   1 package ch.cyberduck.core.openstack;
   2
   3 /*
   4  * Copyright (c) 2002-2013 David Kocher. All rights reserved.
   5  * http://cyberduck.ch/
   6  *
   7  * This program is free software; you can redistribute it and/or modify
   8  * it under the terms of the GNU General Public License as published by
   9  * the Free Software Foundation; either version 2 of the License, or
  10  * (at your option) any later version.
  11  *
  12  * This program is distributed in the hope that it will be useful,
  13  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  14  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  15  * GNU General Public License for more details.
  16  *
  17  * Bug fixes, suggestions and comments should be sent to feedback@cyberduck.ch
  18  */
  19
  20 import ch.cyberduck.core.DefaultIOExceptionMappingService;
  21 import ch.cyberduck.core.Path;
  22 import ch.cyberduck.core.PathContainerService;
  23 import ch.cyberduck.core.date.ISO8601DateParser;
  24 import ch.cyberduck.core.date.InvalidDateException;
  25 import ch.cyberduck.core.exception.BackgroundException;
  26 import ch.cyberduck.core.exception.ChecksumException;
  27 import ch.cyberduck.core.io.Checksum;
  28 import ch.cyberduck.core.io.ChecksumCompute;
  29 import ch.cyberduck.core.preferences.PreferencesFactory;
  30
  31 import org.apache.commons.io.IOUtils;
  32 import org.apache.commons.lang3.StringUtils;
  33 import org.apache.log4j.Logger;
  34
  35 import java.io.IOException;
  36 import java.util.ArrayList;
  37 import java.util.Collections;
  38 import java.util.EnumSet;
  39 import java.util.List;
  40 import java.util.Map;
  41
  42 import ch.iterate.openstack.swift.exception.GenericException;
  43 import ch.iterate.openstack.swift.model.StorageObject;
  44 import com.google.gson.JsonArray;
  45 import com.google.gson.JsonObject;
  46
  47 /**
  48  * @version $Id$
  49  */
  50 public class SwiftSegmentService {
  51     private static final Logger log = Logger.getLogger(SwiftSegmentService.class);
  52
  53     private SwiftSession session;
  54
  55     private PathContainerService containerService
  56             = new SwiftPathContainerService();
  57
  58     private ISO8601DateParser dateParser
  59             = new ISO8601DateParser();
  60
  61     /**
  62      * Segement files prefix
  63      */
  64     private String prefix;
  65
  66     public SwiftSegmentService(final SwiftSession session) {
  67         this(session, PreferencesFactory.get().getProperty("openstack.upload.largeobject.segments.prefix"));
  68     }
  69
  70     public SwiftSegmentService(final SwiftSession session, final String prefix) {
  71         this.session = session;
  72         this.prefix = prefix;
  73     }
  74
  75     public List<Path> list(final Path file) throws BackgroundException {
  76         try {
  77             final Path container = containerService.getContainer(file);
  78             final Map<String, List<StorageObject>> segments
  79                     = session.getClient().listObjectSegments(new SwiftRegionService(session).lookup(container),
  80                     container.getName(), containerService.getKey(file));
  81             if(null == segments) {
  82                 // Not a large object
  83                 return Collections.emptyList();
  84             }
  85             final List<Path> objects = new ArrayList<Path>();
  86             if(segments.containsKey(container.getName())) {
  87                 for(StorageObject s : segments.get(container.getName())) {
  88                     final Path segment = new Path(container, s.getName(), EnumSet.of(Path.Type.file));
  89                     segment.attributes().setSize(s.getSize());
  90                     try {
  91                         segment.attributes().setModificationDate(dateParser.parse(s.getLastModified()).getTime());
  92                     }
  93                     catch(InvalidDateException e) {
  94                         log.warn(String.format("%s is not ISO 8601 format %s", s.getLastModified(), e.getMessage()));
  95                     }
  96                     if(StringUtils.isNotBlank(s.getMd5sum())) {
  97                         segment.attributes().setChecksum(Checksum.parse(s.getMd5sum()));
  98                     }
  99                     objects.add(segment);
 100                 }
 101             }
 102             return objects;
 103         }
 104         catch(GenericException e) {
 105             throw new SwiftExceptionMappingService().map("Failure to read attributes of {0}", e, file);
 106         }
 107         catch(IOException e) {
 108             throw new DefaultIOExceptionMappingService().map("Failure to read attributes of {0}", e, file);
 109         }
 110     }
 111
 112     public String basename(final Path file, final Long size) {
 113         return String.format("%s%s/%d", prefix, containerService.getKey(file), size);
 114     }
 115
 116     public String name(final Path file, final Long size, int segmentNumber) {
 117         return String.format("%s/%08d", this.basename(file, size), segmentNumber);
 118     }
 119
 120     /**
 121      * Create the appropriate manifest structure for a static large object (SLO).
 122      * The number of object segments is limited to a configurable amount, default 1000. Each segment,
 123      * except for the final one, must be at least 1 megabyte (configurable).
 124      *
 125      * @param objects Ordered list of segments
 126      * @return ETag returned by the simple upload total size of segment uploaded path of segment
 127      */
 128     public String manifest(final String container, final List<StorageObject> objects) {
 129         JsonArray manifestSLO = new JsonArray();
 130         for(StorageObject s : objects) {
 131             JsonObject segmentJSON = new JsonObject();
 132             // this is the container and object name in the format {container-name}/{object-name}
 133             segmentJSON.addProperty("path", String.format("/%s/%s", container, s.getName()));
 134             // MD5 checksum of the content of the segment object
 135             segmentJSON.addProperty("etag", s.getMd5sum());
 136             segmentJSON.addProperty("size_bytes", s.getSize());
 137             manifestSLO.add(segmentJSON);
 138         }
 139         return manifestSLO.toString();
 140     }
 141
 142     /**
 143      * The value of the ETag header is calculated by taking
 144      * the ETag value of each segment, concatenating them together, and then returning the MD5 checksum of the result.
 145      *
 146      * @param checksum Checksum compute service
 147      * @param objects  Files
 148      * @return Concatenated checksum
 149      */
 150     public Checksum checksum(final ChecksumCompute checksum, final List<StorageObject> objects) throws ChecksumException {
 151         final StringBuilder concatenated = new StringBuilder();
 152         for(StorageObject s : objects) {
 153             concatenated.append(s.getMd5sum());
 154         }
 155         return checksum.compute(IOUtils.toInputStream(concatenated.toString()));
 156     }
 157 }