source/ch/cyberduck/core/openstack/SwiftSegmentService.java

   1 package ch.cyberduck.core.openstack;
   2
   3 /*
   4  * Copyright (c) 2002-2013 David Kocher. All rights reserved.
   5  * http://cyberduck.ch/
   6  *
   7  * This program is free software; you can redistribute it and/or modify
   8  * it under the terms of the GNU General Public License as published by
   9  * the Free Software Foundation; either version 2 of the License, or
  10  * (at your option) any later version.
  11  *
  12  * This program is distributed in the hope that it will be useful,
  13  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  14  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  15  * GNU General Public License for more details.
  16  *
  17  * Bug fixes, suggestions and comments should be sent to feedback@cyberduck.ch
  18  */
  19
  20 import ch.cyberduck.core.DefaultIOExceptionMappingService;
  21 import ch.cyberduck.core.Path;
  22 import ch.cyberduck.core.PathContainerService;
  23 import ch.cyberduck.core.date.ISO8601DateParser;
  24 import ch.cyberduck.core.date.InvalidDateException;
  25 import ch.cyberduck.core.exception.BackgroundException;
  26 import ch.cyberduck.core.exception.ChecksumException;
  27 import ch.cyberduck.core.io.Checksum;
  28 import ch.cyberduck.core.io.ChecksumCompute;
  29 import ch.cyberduck.core.preferences.PreferencesFactory;
  30
  31 import org.apache.commons.io.IOUtils;
  32 import org.apache.commons.lang3.StringUtils;
  33 import org.apache.log4j.Logger;
  34
  35 import java.io.IOException;
  36 import java.util.ArrayList;
  37 import java.util.Collections;
  38 import java.util.EnumSet;
  39 import java.util.List;
  40 import java.util.Map;
  41
  42 import ch.iterate.openstack.swift.exception.GenericException;
  43 import ch.iterate.openstack.swift.model.StorageObject;
  44 import com.google.gson.JsonArray;
  45 import com.google.gson.JsonObject;
  46
  47 /**
  48  * @version $Id$
  49  */
  50 public class SwiftSegmentService {
  51     private static final Logger log = Logger.getLogger(SwiftSegmentService.class);
  52
  53     private SwiftSession session;
  54
  55     private PathContainerService containerService
  56             = new SwiftPathContainerService();
  57
  58     private ISO8601DateParser dateParser
  59             = new ISO8601DateParser();
  60
  61     /**
  62      * Segement files prefix
  63      */
  64     private String prefix;
  65
  66     private SwiftRegionService regionService;
  67
  68     public SwiftSegmentService(final SwiftSession session) {
  69         this(session, new SwiftRegionService(session));
  70     }
  71
  72     public SwiftSegmentService(final SwiftSession session, final String prefix) {
  73         this(session, new SwiftRegionService(session), prefix);
  74     }
  75
  76     public SwiftSegmentService(final SwiftSession session, final SwiftRegionService regionService) {
  77         this(session, regionService, PreferencesFactory.get().getProperty("openstack.upload.largeobject.segments.prefix"));
  78     }
  79
  80     public SwiftSegmentService(final SwiftSession session, final SwiftRegionService regionService, final String prefix) {
  81         this.session = session;
  82         this.prefix = prefix;
  83         this.regionService = regionService;
  84     }
  85
  86     public List<Path> list(final Path file) throws BackgroundException {
  87         try {
  88             final Path container = containerService.getContainer(file);
  89             final Map<String, List<StorageObject>> segments
  90                     = session.getClient().listObjectSegments(regionService.lookup(container),
  91                     container.getName(), containerService.getKey(file));
  92             if(null == segments) {
  93                 // Not a large object
  94                 return Collections.emptyList();
  95             }
  96             final List<Path> objects = new ArrayList<Path>();
  97             if(segments.containsKey(container.getName())) {
  98                 for(StorageObject s : segments.get(container.getName())) {
  99                     final Path segment = new Path(container, s.getName(), EnumSet.of(Path.Type.file));
 100                     segment.attributes().setSize(s.getSize());
 101                     try {
 102                         segment.attributes().setModificationDate(dateParser.parse(s.getLastModified()).getTime());
 103                     }
 104                     catch(InvalidDateException e) {
 105                         log.warn(String.format("%s is not ISO 8601 format %s", s.getLastModified(), e.getMessage()));
 106                     }
 107                     if(StringUtils.isNotBlank(s.getMd5sum())) {
 108                         segment.attributes().setChecksum(Checksum.parse(s.getMd5sum()));
 109                     }
 110                     objects.add(segment);
 111                 }
 112             }
 113             return objects;
 114         }
 115         catch(GenericException e) {
 116             throw new SwiftExceptionMappingService().map("Failure to read attributes of {0}", e, file);
 117         }
 118         catch(IOException e) {
 119             throw new DefaultIOExceptionMappingService().map("Failure to read attributes of {0}", e, file);
 120         }
 121     }
 122
 123     public String basename(final Path file, final Long size) {
 124         return String.format("%s%s/%d", prefix, containerService.getKey(file), size);
 125     }
 126
 127     public String name(final Path file, final Long size, int segmentNumber) {
 128         return String.format("%s/%08d", this.basename(file, size), segmentNumber);
 129     }
 130
 131     /**
 132      * Create the appropriate manifest structure for a static large object (SLO).
 133      * The number of object segments is limited to a configurable amount, default 1000. Each segment,
 134      * except for the final one, must be at least 1 megabyte (configurable).
 135      *
 136      * @param objects Ordered list of segments
 137      * @return ETag returned by the simple upload total size of segment uploaded path of segment
 138      */
 139     public String manifest(final String container, final List<StorageObject> objects) {
 140         JsonArray manifestSLO = new JsonArray();
 141         for(StorageObject s : objects) {
 142             JsonObject segmentJSON = new JsonObject();
 143             // this is the container and object name in the format {container-name}/{object-name}
 144             segmentJSON.addProperty("path", String.format("/%s/%s", container, s.getName()));
 145             // MD5 checksum of the content of the segment object
 146             segmentJSON.addProperty("etag", s.getMd5sum());
 147             segmentJSON.addProperty("size_bytes", s.getSize());
 148             manifestSLO.add(segmentJSON);
 149         }
 150         return manifestSLO.toString();
 151     }
 152
 153     /**
 154      * The value of the ETag header is calculated by taking
 155      * the ETag value of each segment, concatenating them together, and then returning the MD5 checksum of the result.
 156      *
 157      * @param checksum Checksum compute service
 158      * @param objects  Files
 159      * @return Concatenated checksum
 160      */
 161     public Checksum checksum(final ChecksumCompute checksum, final List<StorageObject> objects) throws ChecksumException {
 162         final StringBuilder concatenated = new StringBuilder();
 163         for(StorageObject s : objects) {
 164             concatenated.append(s.getMd5sum());
 165         }
 166         return checksum.compute(IOUtils.toInputStream(concatenated.toString()));
 167     }
 168 }