task-4499/GenerateSampleBridgeDescriptorTarballs.java

   1 import java.io.*;
   2 import java.util.*;
   3 import org.apache.commons.codec.*;
   4 import org.apache.commons.codec.digest.*;
   5 import org.apache.commons.codec.binary.*;
   6
   7 /* Generate sample bridge descriptor tarball contents for metrics-db and
   8  * BridgeDB load tests.  Accept an extracted, non-sanitized bridge
   9  * descriptor tarball as input and generate sample tarball contents with
  10  * multiples of bridges up to a given maximum multiplier as output.
  11  * Descriptors are multiplied by overwriting the first four hex characters
  12  * of bridge fingerprints with 0000, 0001, etc., keeping references
  13  * between descriptors intact.
  14  *
  15  * NOTE THAT THE OUTPUT TARBALL CONTENTS ARE NOT SANITIZED!
  16  *
  17  * The changes are only sufficient to trick metrics-db and BridgeDB that
  18  * bridges are distinct.  Descriptors may still contain original IP
  19  * addresses in exit policies and other contact information.  Sanitized
  20  * descriptors could not be used as input, because they may have skewed
  21  * results too much. */
  22 public class GenerateSampleBridgeDescriptorTarballs {
  23   public static void main(String[] args) throws Exception {
  24     if (args.length != 3) {
  25       System.err.println("Usage: java "
  26           + GenerateSampleBridgeDescriptorTarballs.class.getName()
  27           + " in-directory out-directory max-multiplier");
  28       System.exit(1);
  29     }
  30     File inDirectory = new File(args[0]);
  31     File outDirectory = new File(args[1]);
  32     int maxMultiplier = Integer.parseInt(args[2]);
  33     readDescriptors(inDirectory);
  34     for (int multiplier = 1; multiplier <= maxMultiplier;
  35         multiplier *= 2) {
  36       writeDescriptors(new File(outDirectory, String.format("%04d",
  37           multiplier)), multiplier);
  38     }
  39   }
  40
  41   private static void readDescriptors(File inDirectory) throws Exception {
  42     readNetworkstatusBridges(new File(inDirectory,
  43         "networkstatus-bridges"));
  44     readBridgeDescriptors(new File(inDirectory, "bridge-descriptors"));
  45     readCachedExtrainfos(new File(inDirectory, "cached-extrainfo"));
  46     readCachedExtrainfos(new File(inDirectory, "cached-extrainfo.new"));
  47   }
  48
  49   private static SortedMap<String, String> networkstatusEntries =
  50       new TreeMap<String, String>();
  51   private static void readNetworkstatusBridges(
  52       File networkstatusBridgesFile) throws Exception {
  53     BufferedReader br = new BufferedReader(new FileReader(
  54         networkstatusBridgesFile));
  55     String line, fingerprint = null, published = null;
  56     StringBuilder sb = null;
  57     while ((line = br.readLine()) != null) {
  58       if (line.startsWith("r ")) {
  59         if (sb != null) {
  60           networkstatusEntries.put(fingerprint + " " + published,
  61               sb.toString());
  62         }
  63         sb = new StringBuilder();
  64         String[] parts = line.split(" ");
  65         fingerprint = Hex.encodeHexString(Base64.decodeBase64(
  66             parts[2] + "=")).toUpperCase();
  67         published = parts[4] + " " + parts[5];
  68       }
  69       sb.append(line + "\n");
  70     }
  71     if (sb != null) {
  72       networkstatusEntries.put(fingerprint + " " + published,
  73           sb.toString());
  74     }
  75     br.close();
  76   }
  77
  78   private static SortedMap<String, String> bridgeDescriptors =
  79       new TreeMap<String, String>();
  80   private static void readBridgeDescriptors(File bridgeDescriptorsFile)
  81       throws Exception {
  82     BufferedReader br = new BufferedReader(new FileReader(
  83         bridgeDescriptorsFile));
  84     String line, fingerprint = null, published = null;
  85     StringBuilder sb = null;
  86     while ((line = br.readLine()) != null) {
  87       if (line.startsWith("@purpose ")) {
  88         if (sb != null) {
  89           bridgeDescriptors.put(fingerprint + " " + published,
  90               sb.toString());
  91         }
  92         sb = new StringBuilder();
  93       } else if (line.startsWith("published ")) {
  94         published = line.substring("published ".length());
  95       } else if (line.startsWith("opt fingerprint ")) {
  96         fingerprint = line.substring("opt fingerprint ".length()).
  97             replaceAll(" ", "");
  98       }
  99       sb.append(line + "\n");
 100     }
 101     if (sb != null) {
 102       bridgeDescriptors.put(fingerprint + " " + published, sb.toString());
 103     }
 104     br.close();
 105
 106   }
 107
 108   private static SortedMap<String, String> cachedExtrainfos =
 109       new TreeMap<String, String>();
 110   private static void readCachedExtrainfos(File cachedExtrainfoFile)
 111       throws Exception {
 112     BufferedReader br = new BufferedReader(new FileReader(
 113         cachedExtrainfoFile));
 114     String line, fingerprint = null, published = null;
 115     StringBuilder sb = null;
 116     while ((line = br.readLine()) != null) {
 117       if (line.startsWith("extra-info ")) {
 118         if (sb != null) {
 119           cachedExtrainfos.put(fingerprint + " " + published,
 120               sb.toString());
 121         }
 122         sb = new StringBuilder();
 123         fingerprint = line.split(" ")[2];
 124       } else if (line.startsWith("published ")) {
 125         published = line.substring("published ".length());
 126       }
 127       sb.append(line + "\n");
 128     }
 129     if (sb != null) {
 130       cachedExtrainfos.put(fingerprint + " " + published, sb.toString());
 131     }
 132     br.close();
 133   }
 134
 135   private static void writeDescriptors(File outDirectory, int multiplier)
 136       throws Exception {
 137     outDirectory.mkdirs();
 138     for (File file : outDirectory.listFiles()) {
 139       file.delete();
 140     }
 141     for (int i = 0; i < multiplier; i++) {
 142       String fingerprintPrefix = String.format("%04x", i);
 143       SortedMap<String, String> extraInfoDigests = writeCachedExtrainfos(
 144           outDirectory, fingerprintPrefix);
 145       SortedMap<String, String> descriptorDigests =
 146           writeBridgeDescriptors(outDirectory, extraInfoDigests,
 147           fingerprintPrefix);
 148       writeNetworkstatusBridges(outDirectory, descriptorDigests,
 149           fingerprintPrefix);
 150     }
 151   }
 152
 153   private static SortedMap<String, String> writeCachedExtrainfos(
 154       File outDirectory, String fingerprintPrefix) throws Exception {
 155     SortedMap<String, String> extraInfoDigests =
 156         new TreeMap<String, String>();
 157     BufferedWriter bw = new BufferedWriter(new FileWriter(new File(
 158         outDirectory, "cached-extrainfo"), true));
 159     for (Map.Entry<String, String> e : cachedExtrainfos.entrySet()) {
 160       String fingerprintPublished = e.getKey();
 161       String cachedExtrainfo = e.getValue();
 162       BufferedReader br = new BufferedReader(new StringReader(
 163           cachedExtrainfo));
 164       String line;
 165       StringBuilder sb = new StringBuilder();
 166       while ((line = br.readLine()) != null) {
 167         if (line.startsWith("extra-info ")) {
 168           String[] parts = line.split(" ");
 169           sb.append(parts[0] + " " + parts[1] + " " + fingerprintPrefix
 170               + parts[2].substring(4) + "\n");
 171         } else if (line.equals("router-signature")) {
 172           sb.append(line + "\n");
 173           String digest = DigestUtils.shaHex(sb.toString()).toUpperCase();
 174           extraInfoDigests.put(fingerprintPublished, digest);
 175         } else {
 176           sb.append(line + "\n");
 177         }
 178       }
 179       bw.write(sb.toString());
 180     }
 181     bw.close();
 182     return extraInfoDigests;
 183   }
 184
 185   private static SortedMap<String, String> writeBridgeDescriptors(
 186       File outDirectory, SortedMap<String, String> extraInfoDigests,
 187       String fingerprintPrefix) throws Exception {
 188     SortedMap<String, String> descriptorDigests =
 189         new TreeMap<String, String>();
 190     BufferedWriter bw = new BufferedWriter(new FileWriter(new File(
 191         outDirectory, "bridge-descriptors"), true));
 192     for (Map.Entry<String, String> e : bridgeDescriptors.entrySet()) {
 193       String fingerprintPublished = e.getKey();
 194       String bridgeDescriptor = e.getValue();
 195       BufferedReader br = new BufferedReader(new StringReader(
 196           bridgeDescriptor));
 197       String line;
 198       StringBuilder sb = new StringBuilder();
 199       while ((line = br.readLine()) != null) {
 200         if (line.startsWith("@purpose ")) {
 201         } else if (line.startsWith("opt fingerprint ")) {
 202           sb.append("opt fingerprint " + fingerprintPrefix
 203               + line.substring("opt fingerprint 0000".length()) + "\n");
 204         } else if (line.startsWith("opt extra-info-digest ")) {
 205           String extraInfoDigest = null;
 206           if (extraInfoDigests.containsKey(fingerprintPublished)) {
 207             extraInfoDigest = extraInfoDigests.get(fingerprintPublished);
 208           } else {
 209             extraInfoDigest = fingerprintPrefix
 210                 + line.split(" ")[2].substring(4);
 211           }
 212           sb.append("opt extra-info-digest " + extraInfoDigest + "\n");
 213         } else if (line.equals("router-signature")) {
 214           sb.append(line + "\n");
 215           String digest = DigestUtils.shaHex(sb.toString()).toUpperCase();
 216           descriptorDigests.put(fingerprintPublished, digest);
 217         } else {
 218           sb.append(line + "\n");
 219         }
 220       }
 221       bw.write("@purpose bridge\n" + sb.toString());
 222     }
 223     bw.close();
 224     return descriptorDigests;
 225   }
 226
 227   private static void writeNetworkstatusBridges(File outDirectory,
 228       SortedMap<String, String> descriptorDigests,
 229       String fingerprintPrefix) throws Exception {
 230     BufferedWriter bw = new BufferedWriter(new FileWriter(new File(
 231         outDirectory, "networkstatus-bridges"), true));
 232     for (Map.Entry<String, String> e : networkstatusEntries.entrySet()) {
 233       String fingerprintPublished = e.getKey();
 234       String networkstatusEntry = e.getValue();
 235       BufferedReader br = new BufferedReader(new StringReader(
 236           networkstatusEntry));
 237       String line;
 238       StringBuilder sb = new StringBuilder();
 239       while ((line = br.readLine()) != null) {
 240         if (line.startsWith("r ")) {
 241           String[] parts = line.split(" ");
 242           String fingerprint = parts[2], descriptorDigest = parts[3];
 243           String newFingerprint = Base64.encodeBase64String(Hex.decodeHex(
 244               (fingerprintPrefix + fingerprintPublished.split(" ")[0].
 245               substring(4)).toCharArray())).substring(0, 27);
 246           String newDescriptorDigest = null;
 247           if (descriptorDigests.containsKey(fingerprintPublished)) {
 248             newDescriptorDigest = Base64.encodeBase64String(Hex.decodeHex(
 249                 descriptorDigests.get(fingerprintPublished).
 250                 toCharArray())).substring(0, 27);
 251           } else {
 252             newDescriptorDigest = "AA" + descriptorDigest.substring(2);
 253           }
 254           sb.append("r " + parts[1] + " " + newFingerprint + " "
 255               + newDescriptorDigest + " " + parts[4] + " " + parts[5]
 256               + " " + parts[6] + " " + parts[7] + " " + parts[8] + "\n");
 257         } else {
 258           sb.append(line + "\n");
 259         }
 260       }
 261       bw.write(sb.toString());
 262     }
 263     bw.close();
 264   }
 265 }
 266