Fix concurrent read / write issue in LockFile on Windows

[jgit.git] / org.eclipse.jgit / src / org / eclipse / jgit / transport / IndexPack.java

/*
 * Copyright (C) 2008-2010, Google Inc.
 * Copyright (C) 2007-2008, Robin Rosenberg <robin.rosenberg@dewire.com>
 * Copyright (C) 2008, Shawn O. Pearce <spearce@spearce.org>
 * and other copyright owners as documented in the project's IP log.
 *
 * This program and the accompanying materials are made available
 * under the terms of the Eclipse Distribution License v1.0 which
 * accompanies this distribution, is reproduced below, and is
 * available at http://www.eclipse.org/org/documents/edl-v10.php
 *
 * All rights reserved.
 *
 * Redistribution and use in source and binary forms, with or
 * without modification, are permitted provided that the following
 * conditions are met:
 *
 * - Redistributions of source code must retain the above copyright
 *   notice, this list of conditions and the following disclaimer.
 *
 * - Redistributions in binary form must reproduce the above
 *   copyright notice, this list of conditions and the following
 *   disclaimer in the documentation and/or other materials provided
 *   with the distribution.
 *
 * - Neither the name of the Eclipse Foundation, Inc. nor the
 *   names of its contributors may be used to endorse or promote
 *   products derived from this software without specific prior
 *   written permission.
 *
 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND
 * CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES,
 * INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
 * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR
 * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
 * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
 * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
 * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
 * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF
 * ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 */

package org.eclipse.jgit.transport;

import java.io.EOFException;
import java.io.File;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.io.RandomAccessFile;
import java.security.MessageDigest;
import java.text.MessageFormat;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
import java.util.zip.CRC32;
import java.util.zip.DataFormatException;
import java.util.zip.Deflater;
import java.util.zip.Inflater;

import org.eclipse.jgit.JGitText;
import org.eclipse.jgit.errors.CorruptObjectException;
import org.eclipse.jgit.errors.MissingObjectException;
import org.eclipse.jgit.lib.AnyObjectId;
import org.eclipse.jgit.lib.Constants;
import org.eclipse.jgit.lib.CoreConfig;
import org.eclipse.jgit.lib.InflaterCache;
import org.eclipse.jgit.lib.MutableObjectId;
import org.eclipse.jgit.lib.ObjectChecker;
import org.eclipse.jgit.lib.ObjectDatabase;
import org.eclipse.jgit.lib.ObjectId;
import org.eclipse.jgit.lib.ObjectIdSubclassMap;
import org.eclipse.jgit.lib.ObjectLoader;
import org.eclipse.jgit.lib.ProgressMonitor;
import org.eclipse.jgit.lib.Repository;
import org.eclipse.jgit.lib.ObjectReader;
import org.eclipse.jgit.storage.file.PackIndexWriter;
import org.eclipse.jgit.storage.file.PackLock;
import org.eclipse.jgit.storage.pack.BinaryDelta;
import org.eclipse.jgit.util.NB;

/** Indexes Git pack files for local use. */
public class IndexPack {
        /** Progress message when reading raw data from the pack. */
        public static final String PROGRESS_DOWNLOAD = JGitText.get().receivingObjects;

        /** Progress message when computing names of delta compressed objects. */
        public static final String PROGRESS_RESOLVE_DELTA = JGitText.get().resolvingDeltas;

        /**
         * Size of the internal stream buffer.
         * <p>
         * If callers are going to be supplying IndexPack a BufferedInputStream they
         * should use this buffer size as the size of the buffer for that
         * BufferedInputStream, and any other its may be wrapping. This way the
         * buffers will cascade efficiently and only the IndexPack buffer will be
         * receiving the bulk of the data stream.
         */
        public static final int BUFFER_SIZE = 8192;

        /**
         * Create an index pack instance to load a new pack into a repository.
         * <p>
         * The received pack data and generated index will be saved to temporary
         * files within the repository's <code>objects</code> directory. To use the
         * data contained within them call {@link #renameAndOpenPack()} once the
         * indexing is complete.
         *
         * @param db
         *            the repository that will receive the new pack.
         * @param is
         *            stream to read the pack data from. If the stream is buffered
         *            use {@link #BUFFER_SIZE} as the buffer size for the stream.
         * @return a new index pack instance.
         * @throws IOException
         *             a temporary file could not be created.
         */
        public static IndexPack create(final Repository db, final InputStream is)
                        throws IOException {
                final String suffix = ".pack";
                final File objdir = db.getObjectsDirectory();
                final File tmp = File.createTempFile("incoming_", suffix, objdir);
                final String n = tmp.getName();
                final File base;

                base = new File(objdir, n.substring(0, n.length() - suffix.length()));
                final IndexPack ip = new IndexPack(db, is, base);
                ip.setIndexVersion(db.getConfig().get(CoreConfig.KEY)
                                .getPackIndexVersion());
                return ip;
        }

        private static enum Source {
                /** Data is read from the incoming stream. */
                INPUT,

                /**
                 * Data is read from the spooled pack file.
                 * <p>
                 * During streaming, some (or all) data might be saved into the spooled
                 * pack file so it can be randomly accessed later.
                 */
                FILE;
        }

        private final Repository repo;

        /**
         * Object database used for loading existing objects
         */
        private final ObjectDatabase objectDatabase;

        private Inflater inflater;

        private final MessageDigest objectDigest;

        private final MutableObjectId tempObjectId;

        private InputStream in;

        private byte[] buf;

        private long bBase;

        private int bOffset;

        private int bAvail;

        private ObjectChecker objCheck;

        private boolean fixThin;

        private boolean keepEmpty;

        private boolean needBaseObjectIds;

        private int outputVersion;

        private final File dstPack;

        private final File dstIdx;

        private long objectCount;

        private PackedObjectInfo[] entries;

        /**
         * Every object contained within the incoming pack.
         * <p>
         * This is a subset of {@link #entries}, as thin packs can add additional
         * objects to {@code entries} by copying already existing objects from the
         * repository onto the end of the thin pack to make it self-contained.
         */
        private ObjectIdSubclassMap<ObjectId> newObjectIds;

        private int deltaCount;

        private int entryCount;

        private final CRC32 crc = new CRC32();

        private ObjectIdSubclassMap<DeltaChain> baseById;

        /**
         * Objects referenced by their name from deltas, that aren't in this pack.
         * <p>
         * This is the set of objects that were copied onto the end of this pack to
         * make it complete. These objects were not transmitted by the remote peer,
         * but instead were assumed to already exist in the local repository.
         */
        private ObjectIdSubclassMap<ObjectId> baseObjectIds;

        private LongMap<UnresolvedDelta> baseByPos;

        private byte[] skipBuffer;

        private MessageDigest packDigest;

        private RandomAccessFile packOut;

        private byte[] packcsum;

        /** If {@link #fixThin} this is the last byte of the original checksum. */
        private long originalEOF;

        private ObjectReader readCurs;

        /**
         * Create a new pack indexer utility.
         *
         * @param db
         * @param src
         *            stream to read the pack data from. If the stream is buffered
         *            use {@link #BUFFER_SIZE} as the buffer size for the stream.
         * @param dstBase
         * @throws IOException
         *             the output packfile could not be created.
         */
        public IndexPack(final Repository db, final InputStream src,
                        final File dstBase) throws IOException {
                repo = db;
                objectDatabase = db.getObjectDatabase().newCachedDatabase();
                in = src;
                inflater = InflaterCache.get();
                readCurs = objectDatabase.newReader();
                buf = new byte[BUFFER_SIZE];
                skipBuffer = new byte[512];
                objectDigest = Constants.newMessageDigest();
                tempObjectId = new MutableObjectId();
                packDigest = Constants.newMessageDigest();

                if (dstBase != null) {
                        final File dir = dstBase.getParentFile();
                        final String nam = dstBase.getName();
                        dstPack = new File(dir, nam + ".pack");
                        dstIdx = new File(dir, nam + ".idx");
                        packOut = new RandomAccessFile(dstPack, "rw");
                        packOut.setLength(0);
                } else {
                        dstPack = null;
                        dstIdx = null;
                }
        }

        /**
         * Set the pack index file format version this instance will create.
         *
         * @param version
         *            the version to write. The special version 0 designates the
         *            oldest (most compatible) format available for the objects.
         * @see PackIndexWriter
         */
        public void setIndexVersion(final int version) {
                outputVersion = version;
        }

        /**
         * Configure this index pack instance to make a thin pack complete.
         * <p>
         * Thin packs are sometimes used during network transfers to allow a delta
         * to be sent without a base object. Such packs are not permitted on disk.
         * They can be fixed by copying the base object onto the end of the pack.
         *
         * @param fix
         *            true to enable fixing a thin pack.
         */
        public void setFixThin(final boolean fix) {
                fixThin = fix;
        }

        /**
         * Configure this index pack instance to keep an empty pack.
         * <p>
         * By default an empty pack (a pack with no objects) is not kept, as doing
         * so is completely pointless. With no objects in the pack there is no data
         * stored by it, so the pack is unnecessary.
         *
         * @param empty true to enable keeping an empty pack.
         */
        public void setKeepEmpty(final boolean empty) {
                keepEmpty = empty;
        }

        /**
         * Configure this index pack instance to keep track of new objects.
         * <p>
         * By default an index pack doesn't save the new objects that were created
         * when it was instantiated. Setting this flag to {@code true} allows the
         * caller to use {@link #getNewObjectIds()} to retrieve that list.
         *
         * @param b {@code true} to enable keeping track of new objects.
         */
        public void setNeedNewObjectIds(boolean b) {
                if (b)
                        newObjectIds = new ObjectIdSubclassMap<ObjectId>();
                else
                        newObjectIds = null;
        }

        private boolean needNewObjectIds() {
                return newObjectIds != null;
        }

        /**
         * Configure this index pack instance to keep track of the objects assumed
         * for delta bases.
         * <p>
         * By default an index pack doesn't save the objects that were used as delta
         * bases. Setting this flag to {@code true} will allow the caller to
         * use {@link #getBaseObjectIds()} to retrieve that list.
         *
         * @param b {@code true} to enable keeping track of delta bases.
         */
        public void setNeedBaseObjectIds(boolean b) {
                this.needBaseObjectIds = b;
        }

        /** @return the new objects that were sent by the user */
        public ObjectIdSubclassMap<ObjectId> getNewObjectIds() {
                if (newObjectIds != null)
                        return newObjectIds;
                return new ObjectIdSubclassMap<ObjectId>();
        }

        /** @return set of objects the incoming pack assumed for delta purposes */
        public ObjectIdSubclassMap<ObjectId> getBaseObjectIds() {
                if (baseObjectIds != null)
                        return baseObjectIds;
                return new ObjectIdSubclassMap<ObjectId>();
        }

        /**
         * Configure the checker used to validate received objects.
         * <p>
         * Usually object checking isn't necessary, as Git implementations only
         * create valid objects in pack files. However, additional checking may be
         * useful if processing data from an untrusted source.
         *
         * @param oc
         *            the checker instance; null to disable object checking.
         */
        public void setObjectChecker(final ObjectChecker oc) {
                objCheck = oc;
        }

        /**
         * Configure the checker used to validate received objects.
         * <p>
         * Usually object checking isn't necessary, as Git implementations only
         * create valid objects in pack files. However, additional checking may be
         * useful if processing data from an untrusted source.
         * <p>
         * This is shorthand for:
         *
         * <pre>
         * setObjectChecker(on ? new ObjectChecker() : null);
         * </pre>
         *
         * @param on
         *            true to enable the default checker; false to disable it.
         */
        public void setObjectChecking(final boolean on) {
                setObjectChecker(on ? new ObjectChecker() : null);
        }

        /**
         * Consume data from the input stream until the packfile is indexed.
         *
         * @param progress
         *            progress feedback
         *
         * @throws IOException
         */
        public void index(final ProgressMonitor progress) throws IOException {
                progress.start(2 /* tasks */);
                try {
                        try {
                                readPackHeader();

                                entries = new PackedObjectInfo[(int) objectCount];
                                baseById = new ObjectIdSubclassMap<DeltaChain>();
                                baseByPos = new LongMap<UnresolvedDelta>();

                                progress.beginTask(PROGRESS_DOWNLOAD, (int) objectCount);
                                for (int done = 0; done < objectCount; done++) {
                                        indexOneObject();
                                        progress.update(1);
                                        if (progress.isCancelled())
                                                throw new IOException(JGitText.get().downloadCancelled);
                                }
                                readPackFooter();
                                endInput();
                                progress.endTask();
                                if (deltaCount > 0) {
                                        if (packOut == null)
                                                throw new IOException(JGitText.get().needPackOut);
                                        resolveDeltas(progress);
                                        if (entryCount < objectCount) {
                                                if (!fixThin) {
                                                        throw new IOException(MessageFormat.format(
                                                                        JGitText.get().packHasUnresolvedDeltas, (objectCount - entryCount)));
                                                }
                                                fixThinPack(progress);
                                        }
                                }
                                if (packOut != null && (keepEmpty || entryCount > 0))
                                        packOut.getChannel().force(true);

                                packDigest = null;
                                baseById = null;
                                baseByPos = null;

                                if (dstIdx != null && (keepEmpty || entryCount > 0))
                                        writeIdx();

                        } finally {
                                try {
                                        if (readCurs != null)
                                                readCurs.release();
                                } finally {
                                        readCurs = null;
                                }

                                try {
                                        InflaterCache.release(inflater);
                                } finally {
                                        inflater = null;
                                        objectDatabase.close();
                                }

                                progress.endTask();
                                if (packOut != null)
                                        packOut.close();
                        }

                        if (keepEmpty || entryCount > 0) {
                                if (dstPack != null)
                                        dstPack.setReadOnly();
                                if (dstIdx != null)
                                        dstIdx.setReadOnly();
                        }
                } catch (IOException err) {
                        if (dstPack != null)
                                dstPack.delete();
                        if (dstIdx != null)
                                dstIdx.delete();
                        throw err;
                }
        }

        private void resolveDeltas(final ProgressMonitor progress)
                        throws IOException {
                progress.beginTask(PROGRESS_RESOLVE_DELTA, deltaCount);
                final int last = entryCount;
                for (int i = 0; i < last; i++) {
                        final int before = entryCount;
                        resolveDeltas(entries[i]);
                        progress.update(entryCount - before);
                        if (progress.isCancelled())
                                throw new IOException(JGitText.get().downloadCancelledDuringIndexing);
                }
                progress.endTask();
        }

        private void resolveDeltas(final PackedObjectInfo oe) throws IOException {
                final int oldCRC = oe.getCRC();
                if (baseById.get(oe) != null || baseByPos.containsKey(oe.getOffset()))
                        resolveDeltas(oe.getOffset(), oldCRC, Constants.OBJ_BAD, null, oe);
        }

        private void resolveDeltas(final long pos, final int oldCRC, int type,
                        byte[] data, PackedObjectInfo oe) throws IOException {
                crc.reset();
                position(pos);
                int c = readFrom(Source.FILE);
                final int typeCode = (c >> 4) & 7;
                long sz = c & 15;
                int shift = 4;
                while ((c & 0x80) != 0) {
                        c = readFrom(Source.FILE);
                        sz += (c & 0x7f) << shift;
                        shift += 7;
                }

                switch (typeCode) {
                case Constants.OBJ_COMMIT:
                case Constants.OBJ_TREE:
                case Constants.OBJ_BLOB:
                case Constants.OBJ_TAG:
                        type = typeCode;
                        data = inflateAndReturn(Source.FILE, sz);
                        break;
                case Constants.OBJ_OFS_DELTA: {
                        c = readFrom(Source.FILE) & 0xff;
                        while ((c & 128) != 0)
                                c = readFrom(Source.FILE) & 0xff;
                        data = BinaryDelta.apply(data, inflateAndReturn(Source.FILE, sz));
                        break;
                }
                case Constants.OBJ_REF_DELTA: {
                        crc.update(buf, fill(Source.FILE, 20), 20);
                        use(20);
                        data = BinaryDelta.apply(data, inflateAndReturn(Source.FILE, sz));
                        break;
                }
                default:
                        throw new IOException(MessageFormat.format(JGitText.get().unknownObjectType, typeCode));
                }

                final int crc32 = (int) crc.getValue();
                if (oldCRC != crc32)
                        throw new IOException(MessageFormat.format(JGitText.get().corruptionDetectedReReadingAt, pos));
                if (oe == null) {
                        objectDigest.update(Constants.encodedTypeString(type));
                        objectDigest.update((byte) ' ');
                        objectDigest.update(Constants.encodeASCII(data.length));
                        objectDigest.update((byte) 0);
                        objectDigest.update(data);
                        tempObjectId.fromRaw(objectDigest.digest(), 0);

                        verifySafeObject(tempObjectId, type, data);
                        oe = new PackedObjectInfo(pos, crc32, tempObjectId);
                        addObjectAndTrack(oe);
                }

                resolveChildDeltas(pos, type, data, oe);
        }

        private UnresolvedDelta removeBaseById(final AnyObjectId id){
                final DeltaChain d = baseById.get(id);
                return d != null ? d.remove() : null;
        }

        private static UnresolvedDelta reverse(UnresolvedDelta c) {
                UnresolvedDelta tail = null;
                while (c != null) {
                        final UnresolvedDelta n = c.next;
                        c.next = tail;
                        tail = c;
                        c = n;
                }
                return tail;
        }

        private void resolveChildDeltas(final long pos, int type, byte[] data,
                        PackedObjectInfo oe) throws IOException {
                UnresolvedDelta a = reverse(removeBaseById(oe));
                UnresolvedDelta b = reverse(baseByPos.remove(pos));
                while (a != null && b != null) {
                        if (a.position < b.position) {
                                resolveDeltas(a.position, a.crc, type, data, null);
                                a = a.next;
                        } else {
                                resolveDeltas(b.position, b.crc, type, data, null);
                                b = b.next;
                        }
                }
                resolveChildDeltaChain(type, data, a);
                resolveChildDeltaChain(type, data, b);
        }

        private void resolveChildDeltaChain(final int type, final byte[] data,
                        UnresolvedDelta a) throws IOException {
                while (a != null) {
                        resolveDeltas(a.position, a.crc, type, data, null);
                        a = a.next;
                }
        }

        private void fixThinPack(final ProgressMonitor progress) throws IOException {
                growEntries();

                if (needBaseObjectIds)
                        baseObjectIds = new ObjectIdSubclassMap<ObjectId>();

                packDigest.reset();
                originalEOF = packOut.length() - 20;
                final Deflater def = new Deflater(Deflater.DEFAULT_COMPRESSION, false);
                final List<DeltaChain> missing = new ArrayList<DeltaChain>(64);
                long end = originalEOF;
                for (final DeltaChain baseId : baseById) {
                        if (baseId.head == null)
                                continue;
                        if (needBaseObjectIds)
                                baseObjectIds.add(baseId);
                        final ObjectLoader ldr;
                        try {
                                ldr = readCurs.open(baseId);
                        } catch (MissingObjectException notFound) {
                                missing.add(baseId);
                                continue;
                        }
                        final byte[] data = ldr.getCachedBytes();
                        final int typeCode = ldr.getType();
                        final PackedObjectInfo oe;

                        crc.reset();
                        packOut.seek(end);
                        writeWhole(def, typeCode, data);
                        oe = new PackedObjectInfo(end, (int) crc.getValue(), baseId);
                        entries[entryCount++] = oe;
                        end = packOut.getFilePointer();

                        resolveChildDeltas(oe.getOffset(), typeCode, data, oe);
                        if (progress.isCancelled())
                                throw new IOException(JGitText.get().downloadCancelledDuringIndexing);
                }
                def.end();

                for (final DeltaChain base : missing) {
                        if (base.head != null)
                                throw new MissingObjectException(base, "delta base");
                }

                if (end - originalEOF < 20) {
                        // Ugly corner case; if what we appended on to complete deltas
                        // doesn't completely cover the SHA-1 we have to truncate off
                        // we need to shorten the file, otherwise we will include part
                        // of the old footer as object content.
                        packOut.setLength(end);
                }

                fixHeaderFooter(packcsum, packDigest.digest());
        }

        private void writeWhole(final Deflater def, final int typeCode,
                        final byte[] data) throws IOException {
                int sz = data.length;
                int hdrlen = 0;
                buf[hdrlen++] = (byte) ((typeCode << 4) | sz & 15);
                sz >>>= 4;
                while (sz > 0) {
                        buf[hdrlen - 1] |= 0x80;
                        buf[hdrlen++] = (byte) (sz & 0x7f);
                        sz >>>= 7;
                }
                packDigest.update(buf, 0, hdrlen);
                crc.update(buf, 0, hdrlen);
                packOut.write(buf, 0, hdrlen);
                def.reset();
                def.setInput(data);
                def.finish();
                while (!def.finished()) {
                        final int datlen = def.deflate(buf);
                        packDigest.update(buf, 0, datlen);
                        crc.update(buf, 0, datlen);
                        packOut.write(buf, 0, datlen);
                }
        }

        private void fixHeaderFooter(final byte[] origcsum, final byte[] tailcsum)
                        throws IOException {
                final MessageDigest origDigest = Constants.newMessageDigest();
                final MessageDigest tailDigest = Constants.newMessageDigest();
                long origRemaining = originalEOF;

                packOut.seek(0);
                bAvail = 0;
                bOffset = 0;
                fill(Source.FILE, 12);

                {
                        final int origCnt = (int) Math.min(bAvail, origRemaining);
                        origDigest.update(buf, 0, origCnt);
                        origRemaining -= origCnt;
                        if (origRemaining == 0)
                                tailDigest.update(buf, origCnt, bAvail - origCnt);
                }

                NB.encodeInt32(buf, 8, entryCount);
                packOut.seek(0);
                packOut.write(buf, 0, 12);
                packOut.seek(bAvail);

                packDigest.reset();
                packDigest.update(buf, 0, bAvail);
                for (;;) {
                        final int n = packOut.read(buf);
                        if (n < 0)
                                break;
                        if (origRemaining != 0) {
                                final int origCnt = (int) Math.min(n, origRemaining);
                                origDigest.update(buf, 0, origCnt);
                                origRemaining -= origCnt;
                                if (origRemaining == 0)
                                        tailDigest.update(buf, origCnt, n - origCnt);
                        } else
                                tailDigest.update(buf, 0, n);

                        packDigest.update(buf, 0, n);
                }

                if (!Arrays.equals(origDigest.digest(), origcsum)
                                || !Arrays.equals(tailDigest.digest(), tailcsum))
                        throw new IOException(JGitText.get().packCorruptedWhileWritingToFilesystem);

                packcsum = packDigest.digest();
                packOut.write(packcsum);
        }

        private void growEntries() {
                final PackedObjectInfo[] ne;

                ne = new PackedObjectInfo[(int) objectCount + baseById.size()];
                System.arraycopy(entries, 0, ne, 0, entryCount);
                entries = ne;
        }

        private void writeIdx() throws IOException {
                Arrays.sort(entries, 0, entryCount);
                List<PackedObjectInfo> list = Arrays.asList(entries);
                if (entryCount < entries.length)
                        list = list.subList(0, entryCount);

                final FileOutputStream os = new FileOutputStream(dstIdx);
                try {
                        final PackIndexWriter iw;
                        if (outputVersion <= 0)
                                iw = PackIndexWriter.createOldestPossible(os, list);
                        else
                                iw = PackIndexWriter.createVersion(os, outputVersion);
                        iw.write(list, packcsum);
                        os.getChannel().force(true);
                } finally {
                        os.close();
                }
        }

        private void readPackHeader() throws IOException {
                final int hdrln = Constants.PACK_SIGNATURE.length + 4 + 4;
                final int p = fill(Source.INPUT, hdrln);
                for (int k = 0; k < Constants.PACK_SIGNATURE.length; k++)
                        if (buf[p + k] != Constants.PACK_SIGNATURE[k])
                                throw new IOException(JGitText.get().notAPACKFile);

                final long vers = NB.decodeUInt32(buf, p + 4);
                if (vers != 2 && vers != 3)
                        throw new IOException(MessageFormat.format(JGitText.get().unsupportedPackVersion, vers));
                objectCount = NB.decodeUInt32(buf, p + 8);
                use(hdrln);
        }

        private void readPackFooter() throws IOException {
                sync();
                final byte[] cmpcsum = packDigest.digest();
                final int c = fill(Source.INPUT, 20);
                packcsum = new byte[20];
                System.arraycopy(buf, c, packcsum, 0, 20);
                use(20);
                if (packOut != null)
                        packOut.write(packcsum);

                if (!Arrays.equals(cmpcsum, packcsum))
                        throw new CorruptObjectException(JGitText.get().corruptObjectPackfileChecksumIncorrect);
        }

        // Cleanup all resources associated with our input parsing.
        private void endInput() {
                in = null;
                skipBuffer = null;
        }

        // Read one entire object or delta from the input.
        private void indexOneObject() throws IOException {
                final long pos = position();

                crc.reset();
                int c = readFrom(Source.INPUT);
                final int typeCode = (c >> 4) & 7;
                long sz = c & 15;
                int shift = 4;
                while ((c & 0x80) != 0) {
                        c = readFrom(Source.INPUT);
                        sz += (c & 0x7f) << shift;
                        shift += 7;
                }

                switch (typeCode) {
                case Constants.OBJ_COMMIT:
                case Constants.OBJ_TREE:
                case Constants.OBJ_BLOB:
                case Constants.OBJ_TAG:
                        whole(typeCode, pos, sz);
                        break;
                case Constants.OBJ_OFS_DELTA: {
                        c = readFrom(Source.INPUT);
                        long ofs = c & 127;
                        while ((c & 128) != 0) {
                                ofs += 1;
                                c = readFrom(Source.INPUT);
                                ofs <<= 7;
                                ofs += (c & 127);
                        }
                        final long base = pos - ofs;
                        final UnresolvedDelta n;
                        inflateAndSkip(Source.INPUT, sz);
                        n = new UnresolvedDelta(pos, (int) crc.getValue());
                        n.next = baseByPos.put(base, n);
                        deltaCount++;
                        break;
                }
                case Constants.OBJ_REF_DELTA: {
                        c = fill(Source.INPUT, 20);
                        crc.update(buf, c, 20);
                        final ObjectId base = ObjectId.fromRaw(buf, c);
                        use(20);
                        DeltaChain r = baseById.get(base);
                        if (r == null) {
                                r = new DeltaChain(base);
                                baseById.add(r);
                        }
                        inflateAndSkip(Source.INPUT, sz);
                        r.add(new UnresolvedDelta(pos, (int) crc.getValue()));
                        deltaCount++;
                        break;
                }
                default:
                        throw new IOException(MessageFormat.format(JGitText.get().unknownObjectType, typeCode));
                }
        }

        private void whole(final int type, final long pos, final long sz)
                        throws IOException {
                final byte[] data = inflateAndReturn(Source.INPUT, sz);
                objectDigest.update(Constants.encodedTypeString(type));
                objectDigest.update((byte) ' ');
                objectDigest.update(Constants.encodeASCII(sz));
                objectDigest.update((byte) 0);
                objectDigest.update(data);
                tempObjectId.fromRaw(objectDigest.digest(), 0);

                verifySafeObject(tempObjectId, type, data);
                final int crc32 = (int) crc.getValue();
                addObjectAndTrack(new PackedObjectInfo(pos, crc32, tempObjectId));
        }

        private void verifySafeObject(final AnyObjectId id, final int type,
                        final byte[] data) throws IOException {
                if (objCheck != null) {
                        try {
                                objCheck.check(type, data);
                        } catch (CorruptObjectException e) {
                                throw new IOException(MessageFormat.format(JGitText.get().invalidObject
                                                , Constants.typeString(type) , id.name() , e.getMessage()));
                        }
                }

                try {
                        final ObjectLoader ldr = readCurs.open(id, type);
                        final byte[] existingData = ldr.getCachedBytes();
                        if (!Arrays.equals(data, existingData)) {
                                throw new IOException(MessageFormat.format(JGitText.get().collisionOn, id.name()));
                        }
                } catch (MissingObjectException notLocal) {
                        // This is OK, we don't have a copy of the object locally
                        // but the API throws when we try to read it as usually its
                        // an error to read something that doesn't exist.
                }
        }

        // Current position of {@link #bOffset} within the entire file.
        private long position() {
                return bBase + bOffset;
        }

        private void position(final long pos) throws IOException {
                packOut.seek(pos);
                bBase = pos;
                bOffset = 0;
                bAvail = 0;
        }

        // Consume exactly one byte from the buffer and return it.
        private int readFrom(final Source src) throws IOException {
                if (bAvail == 0)
                        fill(src, 1);
                bAvail--;
                final int b = buf[bOffset++] & 0xff;
                crc.update(b);
                return b;
        }

        // Consume cnt bytes from the buffer.
        private void use(final int cnt) {
                bOffset += cnt;
                bAvail -= cnt;
        }

        // Ensure at least need bytes are available in in {@link #buf}.
        private int fill(final Source src, final int need) throws IOException {
                while (bAvail < need) {
                        int next = bOffset + bAvail;
                        int free = buf.length - next;
                        if (free + bAvail < need) {
                                switch(src){
                                case INPUT:
                                        sync();
                                        break;
                                case FILE:
                                        if (bAvail > 0)
                                                System.arraycopy(buf, bOffset, buf, 0, bAvail);
                                        bOffset = 0;
                                        break;
                                }
                                next = bAvail;
                                free = buf.length - next;
                        }
                        switch(src){
                        case INPUT:
                                next = in.read(buf, next, free);
                                break;
                        case FILE:
                                next = packOut.read(buf, next, free);
                                break;
                        }
                        if (next <= 0)
                                throw new EOFException(JGitText.get().packfileIsTruncated);
                        bAvail += next;
                }
                return bOffset;
        }

        // Store consumed bytes in {@link #buf} up to {@link #bOffset}.
        private void sync() throws IOException {
                packDigest.update(buf, 0, bOffset);
                if (packOut != null)
                        packOut.write(buf, 0, bOffset);
                if (bAvail > 0)
                        System.arraycopy(buf, bOffset, buf, 0, bAvail);
                bBase += bOffset;
                bOffset = 0;
        }

        private void inflateAndSkip(final Source src, final long inflatedSize)
                        throws IOException {
                inflate(src, inflatedSize, skipBuffer, false /* do not keep result */);
        }

        private byte[] inflateAndReturn(final Source src, final long inflatedSize)
                        throws IOException {
                final byte[] dst = new byte[(int) inflatedSize];
                inflate(src, inflatedSize, dst, true /* keep result in dst */);
                return dst;
        }

        private void inflate(final Source src, final long inflatedSize,
                        final byte[] dst, final boolean keep) throws IOException {
                final Inflater inf = inflater;
                try {
                        int off = 0;
                        long cnt = 0;
                        int p = fill(src, 24);
                        inf.setInput(buf, p, bAvail);

                        for (;;) {
                                int r = inf.inflate(dst, off, dst.length - off);
                                if (r == 0) {
                                        if (inf.finished())
                                                break;
                                        if (inf.needsInput()) {
                                                if (p >= 0) {
                                                        crc.update(buf, p, bAvail);
                                                        use(bAvail);
                                                }
                                                p = fill(src, 24);
                                                inf.setInput(buf, p, bAvail);
                                        } else {
                                                throw new CorruptObjectException(MessageFormat.format(
                                                                JGitText.get().packfileCorruptionDetected,
                                                                JGitText.get().unknownZlibError));
                                        }
                                }
                                cnt += r;
                                if (keep)
                                        off += r;
                        }

                        if (cnt != inflatedSize) {
                                throw new CorruptObjectException(MessageFormat.format(JGitText
                                                .get().packfileCorruptionDetected,
                                                JGitText.get().wrongDecompressedLength));
                        }

                        int left = bAvail - inf.getRemaining();
                        if (left > 0) {
                                crc.update(buf, p, left);
                                use(left);
                        }
                } catch (DataFormatException dfe) {
                        throw new CorruptObjectException(MessageFormat.format(JGitText
                                        .get().packfileCorruptionDetected, dfe.getMessage()));
                } finally {
                        inf.reset();
                }
        }

        private static class DeltaChain extends ObjectId {
                UnresolvedDelta head;

                DeltaChain(final AnyObjectId id) {
                        super(id);
                }

                UnresolvedDelta remove() {
                        final UnresolvedDelta r = head;
                        if (r != null)
                                head = null;
                        return r;
                }

                void add(final UnresolvedDelta d) {
                        d.next = head;
                        head = d;
                }
        }

        private static class UnresolvedDelta {
                final long position;

                final int crc;

                UnresolvedDelta next;

                UnresolvedDelta(final long headerOffset, final int crc32) {
                        position = headerOffset;
                        crc = crc32;
                }
        }

        /**
         * Rename the pack to it's final name and location and open it.
         * <p>
         * If the call completes successfully the repository this IndexPack instance
         * was created with will have the objects in the pack available for reading
         * and use, without needing to scan for packs.
         *
         * @throws IOException
         *             The pack could not be inserted into the repository's objects
         *             directory. The pack no longer exists on disk, as it was
         *             removed prior to throwing the exception to the caller.
         */
        public void renameAndOpenPack() throws IOException {
                renameAndOpenPack(null);
        }

        /**
         * Rename the pack to it's final name and location and open it.
         * <p>
         * If the call completes successfully the repository this IndexPack instance
         * was created with will have the objects in the pack available for reading
         * and use, without needing to scan for packs.
         *
         * @param lockMessage
         *            message to place in the pack-*.keep file. If null, no lock
         *            will be created, and this method returns null.
         * @return the pack lock object, if lockMessage is not null.
         * @throws IOException
         *             The pack could not be inserted into the repository's objects
         *             directory. The pack no longer exists on disk, as it was
         *             removed prior to throwing the exception to the caller.
         */
        public PackLock renameAndOpenPack(final String lockMessage)
                        throws IOException {
                if (!keepEmpty && entryCount == 0) {
                        cleanupTemporaryFiles();
                        return null;
                }

                final MessageDigest d = Constants.newMessageDigest();
                final byte[] oeBytes = new byte[Constants.OBJECT_ID_LENGTH];
                for (int i = 0; i < entryCount; i++) {
                        final PackedObjectInfo oe = entries[i];
                        oe.copyRawTo(oeBytes, 0);
                        d.update(oeBytes);
                }

                final String name = ObjectId.fromRaw(d.digest()).name();
                final File packDir = new File(repo.getObjectsDirectory(), "pack");
                final File finalPack = new File(packDir, "pack-" + name + ".pack");
                final File finalIdx = new File(packDir, "pack-" + name + ".idx");
                final PackLock keep = new PackLock(finalPack, repo.getFS());

                if (!packDir.exists() && !packDir.mkdir() && !packDir.exists()) {
                        // The objects/pack directory isn't present, and we are unable
                        // to create it. There is no way to move this pack in.
                        //
                        cleanupTemporaryFiles();
                        throw new IOException(MessageFormat.format(JGitText.get().cannotCreateDirectory, packDir.getAbsolutePath()));
                }

                if (finalPack.exists()) {
                        // If the pack is already present we should never replace it.
                        //
                        cleanupTemporaryFiles();
                        return null;
                }

                if (lockMessage != null) {
                        // If we have a reason to create a keep file for this pack, do
                        // so, or fail fast and don't put the pack in place.
                        //
                        try {
                                if (!keep.lock(lockMessage))
                                        throw new IOException(MessageFormat.format(JGitText.get().cannotLockPackIn, finalPack));
                        } catch (IOException e) {
                                cleanupTemporaryFiles();
                                throw e;
                        }
                }

                if (!dstPack.renameTo(finalPack)) {
                        cleanupTemporaryFiles();
                        keep.unlock();
                        throw new IOException(MessageFormat.format(JGitText.get().cannotMovePackTo, finalPack));
                }

                if (!dstIdx.renameTo(finalIdx)) {
                        cleanupTemporaryFiles();
                        keep.unlock();
                        if (!finalPack.delete())
                                finalPack.deleteOnExit();
                        throw new IOException(MessageFormat.format(JGitText.get().cannotMoveIndexTo, finalIdx));
                }

                try {
                        repo.openPack(finalPack, finalIdx);
                } catch (IOException err) {
                        keep.unlock();
                        finalPack.delete();
                        finalIdx.delete();
                        throw err;
                }

                return lockMessage != null ? keep : null;
        }

        private void cleanupTemporaryFiles() {
                if (!dstIdx.delete())
                        dstIdx.deleteOnExit();
                if (!dstPack.delete())
                        dstPack.deleteOnExit();
        }

        private void addObjectAndTrack(PackedObjectInfo oe) {
                entries[entryCount++] = oe;
                if (needNewObjectIds())
                        newObjectIds.add(oe);
        }
}