org.spearce.jgit.test/tst/org/spearce/jgit/lib/PackWriterTest.java

   1 /*
   2  * Copyright (C) 2008, Marek Zawirski <marek.zawirski@gmail.com>
   3  *
   4  * All rights reserved.
   5  *
   6  * Redistribution and use in source and binary forms, with or
   7  * without modification, are permitted provided that the following
   8  * conditions are met:
   9  *
  10  * - Redistributions of source code must retain the above copyright
  11  *   notice, this list of conditions and the following disclaimer.
  12  *
  13  * - Redistributions in binary form must reproduce the above
  14  *   copyright notice, this list of conditions and the following
  15  *   disclaimer in the documentation and/or other materials provided
  16  *   with the distribution.
  17  *
  18  * - Neither the name of the Git Development Community nor the
  19  *   names of its contributors may be used to endorse or promote
  20  *   products derived from this software without specific prior
  21  *   written permission.
  22  *
  23  * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND
  24  * CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES,
  25  * INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
  26  * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
  27  * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR
  28  * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
  29  * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
  30  * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
  31  * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
  32  * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
  33  * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
  34  * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF
  35  * ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  36  */
  37
  38 package org.spearce.jgit.lib;
  39
  40 import java.io.ByteArrayInputStream;
  41 import java.io.ByteArrayOutputStream;
  42 import java.io.File;
  43 import java.io.IOException;
  44 import java.io.InputStream;
  45 import java.util.ArrayList;
  46 import java.util.Arrays;
  47 import java.util.Collection;
  48 import java.util.Collections;
  49 import java.util.Comparator;
  50 import java.util.Iterator;
  51 import java.util.LinkedList;
  52 import java.util.List;
  53
  54 import org.spearce.jgit.errors.MissingObjectException;
  55 import org.spearce.jgit.lib.PackIndex.MutableEntry;
  56 import org.spearce.jgit.revwalk.RevObject;
  57 import org.spearce.jgit.revwalk.RevWalk;
  58 import org.spearce.jgit.transport.IndexPack;
  59 import org.spearce.jgit.util.CountingOutputStream;
  60
  61 public class PackWriterTest extends RepositoryTestCase {
  62
  63         private static final List<ObjectId> EMPTY_LIST_OBJECT = Collections
  64                         .<ObjectId> emptyList();
  65
  66         private static final List<RevObject> EMPTY_LIST_REVS = Collections
  67                         .<RevObject> emptyList();
  68
  69         private PackWriter writer;
  70
  71         private ByteArrayOutputStream os;
  72
  73         private CountingOutputStream cos;
  74
  75         private File packBase;
  76
  77         private File packFile;
  78
  79         private File indexFile;
  80
  81         private PackFile pack;
  82
  83         public void setUp() throws Exception {
  84                 super.setUp();
  85                 os = new ByteArrayOutputStream();
  86                 cos = new CountingOutputStream(os);
  87                 packBase = new File(trash, "tmp_pack");
  88                 packFile = new File(trash, "tmp_pack.pack");
  89                 indexFile = new File(trash, "tmp_pack.idx");
  90                 writer = new PackWriter(db, new TextProgressMonitor());
  91         }
  92
  93         /**
  94          * Test constructor for exceptions, default settings, initialization.
  95          */
  96         public void testContructor() {
  97                 assertEquals(false, writer.isDeltaBaseAsOffset());
  98                 assertEquals(true, writer.isReuseDeltas());
  99                 assertEquals(true, writer.isReuseObjects());
 100                 assertEquals(0, writer.getObjectsNumber());
 101         }
 102
 103         /**
 104          * Change default settings and verify them.
 105          */
 106         public void testModifySettings() {
 107                 writer.setDeltaBaseAsOffset(true);
 108                 writer.setReuseDeltas(false);
 109                 writer.setReuseObjects(false);
 110
 111                 assertEquals(true, writer.isDeltaBaseAsOffset());
 112                 assertEquals(false, writer.isReuseDeltas());
 113                 assertEquals(false, writer.isReuseObjects());
 114         }
 115
 116         /**
 117          * Write empty pack by providing empty sets of interesting/uninteresting
 118          * objects and check for correct format.
 119          *
 120          * @throws IOException
 121          */
 122         public void testWriteEmptyPack1() throws IOException {
 123                 createVerifyOpenPack(EMPTY_LIST_OBJECT, EMPTY_LIST_OBJECT, false, false);
 124
 125                 assertEquals(0, writer.getObjectsNumber());
 126                 assertEquals(0, pack.getObjectCount());
 127                 assertEquals("da39a3ee5e6b4b0d3255bfef95601890afd80709", writer
 128                                 .computeName().name());
 129         }
 130
 131         /**
 132          * Write empty pack by providing empty iterator of objects to write and
 133          * check for correct format.
 134          *
 135          * @throws IOException
 136          */
 137         public void testWriteEmptyPack2() throws IOException {
 138                 createVerifyOpenPack(EMPTY_LIST_REVS.iterator());
 139
 140                 assertEquals(0, writer.getObjectsNumber());
 141                 assertEquals(0, pack.getObjectCount());
 142         }
 143
 144         /**
 145          * Try to pass non-existing object as uninteresting, with non-ignoring
 146          * setting.
 147          *
 148          * @throws IOException
 149          */
 150         public void testNotIgnoreNonExistingObjects() throws IOException {
 151                 final ObjectId nonExisting = ObjectId
 152                                 .fromString("0000000000000000000000000000000000000001");
 153                 try {
 154                         createVerifyOpenPack(EMPTY_LIST_OBJECT, Collections.nCopies(1,
 155                                         nonExisting), false, false);
 156                         fail("Should have thrown MissingObjectException");
 157                 } catch (MissingObjectException x) {
 158                         // expected
 159                 }
 160         }
 161
 162         /**
 163          * Try to pass non-existing object as uninteresting, with ignoring setting.
 164          *
 165          * @throws IOException
 166          */
 167         public void testIgnoreNonExistingObjects() throws IOException {
 168                 final ObjectId nonExisting = ObjectId
 169                                 .fromString("0000000000000000000000000000000000000001");
 170                 createVerifyOpenPack(EMPTY_LIST_OBJECT, Collections.nCopies(1,
 171                                 nonExisting), false, true);
 172                 // shouldn't throw anything
 173         }
 174
 175         /**
 176          * Create pack basing on only interesting objects, then precisely verify
 177          * content. No delta reuse here.
 178          *
 179          * @throws IOException
 180          */
 181         public void testWritePack1() throws IOException {
 182                 writer.setReuseDeltas(false);
 183                 writeVerifyPack1();
 184         }
 185
 186         /**
 187          * Test writing pack without object reuse. Pack content/preparation as in
 188          * {@link #testWritePack1()}.
 189          *
 190          * @throws IOException
 191          */
 192         public void testWritePack1NoObjectReuse() throws IOException {
 193                 writer.setReuseDeltas(false);
 194                 writer.setReuseObjects(false);
 195                 writeVerifyPack1();
 196         }
 197
 198         /**
 199          * Create pack basing on both interesting and uninteresting objects, then
 200          * precisely verify content. No delta reuse here.
 201          *
 202          * @throws IOException
 203          */
 204         public void testWritePack2() throws IOException {
 205                 writeVerifyPack2(false);
 206         }
 207
 208         /**
 209          * Test pack writing with deltas reuse, delta-base first rule. Pack
 210          * content/preparation as in {@link #testWritePack2()}.
 211          *
 212          * @throws IOException
 213          */
 214         public void testWritePack2DeltasReuseRefs() throws IOException {
 215                 writeVerifyPack2(true);
 216         }
 217
 218         /**
 219          * Test pack writing with delta reuse. Delta bases referred as offsets. Pack
 220          * configuration as in {@link #testWritePack2DeltasReuseRefs()}.
 221          *
 222          * @throws IOException
 223          */
 224         public void testWritePack2DeltasReuseOffsets() throws IOException {
 225                 writer.setDeltaBaseAsOffset(true);
 226                 writeVerifyPack2(true);
 227         }
 228
 229         /**
 230          * Test pack writing with delta reuse. Raw-data copy (reuse) is made on a
 231          * pack with CRC32 index. Pack configuration as in
 232          * {@link #testWritePack2DeltasReuseRefs()}.
 233          *
 234          * @throws IOException
 235          */
 236         public void testWritePack2DeltasCRC32Copy() throws IOException {
 237                 final File packDir = new File(db.getObjectsDirectory(), "pack");
 238                 final File crc32Pack = new File(packDir,
 239                                 "pack-34be9032ac282b11fa9babdc2b2a93ca996c9c2f.pack");
 240                 final File crc32Idx = new File(packDir,
 241                                 "pack-34be9032ac282b11fa9babdc2b2a93ca996c9c2f.idx");
 242                 copyFile(new File(new File("tst"),
 243                                 "pack-34be9032ac282b11fa9babdc2b2a93ca996c9c2f.idxV2"),
 244                                 crc32Idx);
 245                 db.openPack(crc32Pack, crc32Idx);
 246
 247                 writeVerifyPack2(true);
 248         }
 249
 250         /**
 251          * Create pack basing on fixed objects list, then precisely verify content.
 252          * No delta reuse here.
 253          *
 254          * @throws IOException
 255          * @throws MissingObjectException
 256          *
 257          */
 258         public void testWritePack3() throws MissingObjectException, IOException {
 259                 writer.setReuseDeltas(false);
 260                 final ObjectId forcedOrder[] = new ObjectId[] {
 261                                 ObjectId.fromString("82c6b885ff600be425b4ea96dee75dca255b69e7"),
 262                                 ObjectId.fromString("c59759f143fb1fe21c197981df75a7ee00290799"),
 263                                 ObjectId.fromString("aabf2ffaec9b497f0950352b3e582d73035c2035"),
 264                                 ObjectId.fromString("902d5476fa249b7abc9d84c611577a81381f0327"),
 265                                 ObjectId.fromString("5b6e7c66c276e7610d4a73c70ec1a1f7c1003259"),
 266                                 ObjectId.fromString("6ff87c4664981e4397625791c8ea3bbb5f2279a3") };
 267                 final RevWalk parser = new RevWalk(db);
 268                 final RevObject forcedOrderRevs[] = new RevObject[forcedOrder.length];
 269                 for (int i = 0; i < forcedOrder.length; i++)
 270                         forcedOrderRevs[i] = parser.parseAny(forcedOrder[i]);
 271
 272                 createVerifyOpenPack(Arrays.asList(forcedOrderRevs).iterator());
 273
 274                 assertEquals(forcedOrder.length, writer.getObjectsNumber());
 275                 verifyObjectsOrder(forcedOrder);
 276                 assertEquals("ed3f96b8327c7c66b0f8f70056129f0769323d86", writer
 277                                 .computeName().name());
 278         }
 279
 280         /**
 281          * Another pack creation: basing on both interesting and uninteresting
 282          * objects. No delta reuse possible here, as this is a specific case when we
 283          * write only 1 commit, associated with 1 tree, 1 blob.
 284          *
 285          * @throws IOException
 286          */
 287         public void testWritePack4() throws IOException {
 288                 writeVerifyPack4(false);
 289         }
 290
 291         /**
 292          * Test thin pack writing: 1 blob delta base is on objects edge. Pack
 293          * configuration as in {@link #testWritePack4()}.
 294          *
 295          * @throws IOException
 296          */
 297         public void testWritePack4ThinPack() throws IOException {
 298                 writeVerifyPack4(true);
 299         }
 300
 301         /**
 302          * Compare sizes of packs created using {@link #testWritePack2()} and
 303          * {@link #testWritePack2DeltasReuseRefs()}. The pack using deltas should
 304          * be smaller.
 305          *
 306          * @throws Exception
 307          */
 308         public void testWritePack2SizeDeltasVsNoDeltas() throws Exception {
 309                 testWritePack2();
 310                 final int sizePack2NoDeltas = cos.getCount();
 311                 setUp();
 312                 testWritePack2DeltasReuseRefs();
 313                 final int sizePack2DeltasRefs = cos.getCount();
 314
 315                 assertTrue(sizePack2NoDeltas > sizePack2DeltasRefs);
 316         }
 317
 318         /**
 319          * Compare sizes of packs created using
 320          * {@link #testWritePack2DeltasReuseRefs()} and
 321          * {@link #testWritePack2DeltasReuseOffsets()}. The pack with delta bases
 322          * written as offsets should be smaller.
 323          *
 324          * @throws Exception
 325          */
 326         public void testWritePack2SizeOffsetsVsRefs() throws Exception {
 327                 testWritePack2DeltasReuseRefs();
 328                 final int sizePack2DeltasRefs = cos.getCount();
 329                 setUp();
 330                 testWritePack2DeltasReuseOffsets();
 331                 final int sizePack2DeltasOffsets = cos.getCount();
 332
 333                 assertTrue(sizePack2DeltasRefs > sizePack2DeltasOffsets);
 334         }
 335
 336         /**
 337          * Compare sizes of packs created using {@link #testWritePack4()} and
 338          * {@link #testWritePack4ThinPack()}. Obviously, the thin pack should be
 339          * smaller.
 340          *
 341          * @throws Exception
 342          */
 343         public void testWritePack4SizeThinVsNoThin() throws Exception {
 344                 testWritePack4();
 345                 final int sizePack4 = cos.getCount();
 346                 setUp();
 347                 testWritePack4ThinPack();
 348                 final int sizePack4Thin = cos.getCount();
 349
 350                 assertTrue(sizePack4 > sizePack4Thin);
 351         }
 352
 353         // TODO: testWritePackDeltasCycle()
 354         // TODO: testWritePackDeltasDepth()
 355
 356         private void writeVerifyPack1() throws IOException {
 357                 final LinkedList<ObjectId> interestings = new LinkedList<ObjectId>();
 358                 interestings.add(ObjectId
 359                                 .fromString("82c6b885ff600be425b4ea96dee75dca255b69e7"));
 360                 createVerifyOpenPack(interestings, EMPTY_LIST_OBJECT, false, false);
 361
 362                 final ObjectId expectedOrder[] = new ObjectId[] {
 363                                 ObjectId.fromString("82c6b885ff600be425b4ea96dee75dca255b69e7"),
 364                                 ObjectId.fromString("c59759f143fb1fe21c197981df75a7ee00290799"),
 365                                 ObjectId.fromString("540a36d136cf413e4b064c2b0e0a4db60f77feab"),
 366                                 ObjectId.fromString("aabf2ffaec9b497f0950352b3e582d73035c2035"),
 367                                 ObjectId.fromString("902d5476fa249b7abc9d84c611577a81381f0327"),
 368                                 ObjectId.fromString("4b825dc642cb6eb9a060e54bf8d69288fbee4904"),
 369                                 ObjectId.fromString("5b6e7c66c276e7610d4a73c70ec1a1f7c1003259"),
 370                                 ObjectId.fromString("6ff87c4664981e4397625791c8ea3bbb5f2279a3") };
 371
 372                 assertEquals(expectedOrder.length, writer.getObjectsNumber());
 373                 verifyObjectsOrder(expectedOrder);
 374                 assertEquals("34be9032ac282b11fa9babdc2b2a93ca996c9c2f", writer
 375                                 .computeName().name());
 376         }
 377
 378         private void writeVerifyPack2(boolean deltaReuse) throws IOException {
 379                 writer.setReuseDeltas(deltaReuse);
 380                 final LinkedList<ObjectId> interestings = new LinkedList<ObjectId>();
 381                 interestings.add(ObjectId
 382                                 .fromString("82c6b885ff600be425b4ea96dee75dca255b69e7"));
 383                 final LinkedList<ObjectId> uninterestings = new LinkedList<ObjectId>();
 384                 uninterestings.add(ObjectId
 385                                 .fromString("540a36d136cf413e4b064c2b0e0a4db60f77feab"));
 386                 createVerifyOpenPack(interestings, uninterestings, false, false);
 387
 388                 final ObjectId expectedOrder[] = new ObjectId[] {
 389                                 ObjectId.fromString("82c6b885ff600be425b4ea96dee75dca255b69e7"),
 390                                 ObjectId.fromString("c59759f143fb1fe21c197981df75a7ee00290799"),
 391                                 ObjectId.fromString("aabf2ffaec9b497f0950352b3e582d73035c2035"),
 392                                 ObjectId.fromString("902d5476fa249b7abc9d84c611577a81381f0327"),
 393                                 ObjectId.fromString("5b6e7c66c276e7610d4a73c70ec1a1f7c1003259"),
 394                                 ObjectId.fromString("6ff87c4664981e4397625791c8ea3bbb5f2279a3") };
 395                 if (deltaReuse) {
 396                         // objects order influenced (swapped) by delta-base first rule
 397                         ObjectId temp = expectedOrder[4];
 398                         expectedOrder[4] = expectedOrder[5];
 399                         expectedOrder[5] = temp;
 400                 }
 401                 assertEquals(expectedOrder.length, writer.getObjectsNumber());
 402                 verifyObjectsOrder(expectedOrder);
 403                 assertEquals("ed3f96b8327c7c66b0f8f70056129f0769323d86", writer
 404                                 .computeName().name());
 405         }
 406
 407         private void writeVerifyPack4(final boolean thin) throws IOException {
 408                 final LinkedList<ObjectId> interestings = new LinkedList<ObjectId>();
 409                 interestings.add(ObjectId
 410                                 .fromString("82c6b885ff600be425b4ea96dee75dca255b69e7"));
 411                 final LinkedList<ObjectId> uninterestings = new LinkedList<ObjectId>();
 412                 uninterestings.add(ObjectId
 413                                 .fromString("c59759f143fb1fe21c197981df75a7ee00290799"));
 414                 createVerifyOpenPack(interestings, uninterestings, thin, false);
 415
 416                 final ObjectId writtenObjects[] = new ObjectId[] {
 417                                 ObjectId.fromString("82c6b885ff600be425b4ea96dee75dca255b69e7"),
 418                                 ObjectId.fromString("aabf2ffaec9b497f0950352b3e582d73035c2035"),
 419                                 ObjectId.fromString("5b6e7c66c276e7610d4a73c70ec1a1f7c1003259") };
 420                 assertEquals(writtenObjects.length, writer.getObjectsNumber());
 421                 ObjectId expectedObjects[];
 422                 if (thin) {
 423                         expectedObjects = new ObjectId[4];
 424                         System.arraycopy(writtenObjects, 0, expectedObjects, 0,
 425                                         writtenObjects.length);
 426                         expectedObjects[3] = ObjectId
 427                                         .fromString("6ff87c4664981e4397625791c8ea3bbb5f2279a3");
 428
 429                 } else {
 430                         expectedObjects = writtenObjects;
 431                 }
 432                 verifyObjectsOrder(expectedObjects);
 433                 assertEquals("cded4b74176b4456afa456768b2b5aafb41c44fc", writer
 434                                 .computeName().name());
 435         }
 436
 437         private void createVerifyOpenPack(final Collection<ObjectId> interestings,
 438                         final Collection<ObjectId> uninterestings, final boolean thin,
 439                         final boolean ignoreMissingUninteresting)
 440                         throws MissingObjectException, IOException {
 441                 writer.preparePack(interestings, uninterestings, thin,
 442                                 ignoreMissingUninteresting);
 443                 writer.writePack(cos);
 444                 verifyOpenPack(thin);
 445         }
 446
 447         private void createVerifyOpenPack(final Iterator<RevObject> objectSource)
 448                         throws MissingObjectException, IOException {
 449                 writer.preparePack(objectSource);
 450                 writer.writePack(cos);
 451                 verifyOpenPack(false);
 452         }
 453
 454         private void verifyOpenPack(final boolean thin) throws IOException {
 455                 if (thin) {
 456                         final InputStream is = new ByteArrayInputStream(os.toByteArray());
 457                         final IndexPack indexer = new IndexPack(db, is, packBase);
 458                         try {
 459                                 indexer.index(new TextProgressMonitor());
 460                                 fail("indexer should grumble about missing object");
 461                         } catch (IOException x) {
 462                                 // expected
 463                         }
 464                 }
 465                 final InputStream is = new ByteArrayInputStream(os.toByteArray());
 466                 final IndexPack indexer = new IndexPack(db, is, packBase);
 467                 indexer.setFixThin(thin);
 468                 indexer.index(new TextProgressMonitor());
 469                 pack = new PackFile(db, indexFile, packFile);
 470         }
 471
 472         private void verifyObjectsOrder(final ObjectId objectsOrder[]) {
 473                 final List<PackIndex.MutableEntry> entries = new ArrayList<PackIndex.MutableEntry>();
 474
 475                 for (MutableEntry me : pack) {
 476                         entries.add(me.cloneEntry());
 477                 }
 478                 Collections.sort(entries, new Comparator<PackIndex.MutableEntry>() {
 479                         public int compare(MutableEntry o1, MutableEntry o2) {
 480                                 return Long.signum(o1.getOffset() - o2.getOffset());
 481                         }
 482                 });
 483
 484                 int i = 0;
 485                 for (MutableEntry me : entries) {
 486                         assertEquals(objectsOrder[i++], me.copy());
 487                 }
 488         }
 489 }