bin/oss-fuzz-setup.sh

   1 #!/usr/bin/env bash
   2
   3 set -e
   4
   5 echo start downloading dependencies at `date -u`
   6
   7 # convert FOO := BAR$(MICRO) to export FOO=BAR$MICRO
   8 source <(sed -e's#\([^ ]\{1,\}\) := #export \1=#g' download.lst | sed -e 's#[)(]##g')
   9
  10 mkdir $SRC/external-tar
  11 cd $SRC/external-tar
  12
  13 #cache build dependencies
  14 curl --no-progress-meter -S \
  15     -C - -O https://dev-www.libreoffice.org/src/$FONT_LIBERATION_TARBALL \
  16     -C - -O https://dev-www.libreoffice.org/extern/$OPENSYMBOL_TTF \
  17     -C - -O https://dev-www.libreoffice.org/src/$RASQAL_TARBALL \
  18     -C - -O https://dev-www.libreoffice.org/src/$CLUCENE_TARBALL \
  19     -C - -O https://dev-www.libreoffice.org/src/$HYPHEN_TARBALL \
  20     -C - -O https://dev-www.libreoffice.org/src/$RAPTOR_TARBALL \
  21     -C - -O https://dev-www.libreoffice.org/src/$MYTHES_TARBALL \
  22     -C - -O https://dev-www.libreoffice.org/src/$REDLAND_TARBALL \
  23     -C - -O https://dev-www.libreoffice.org/src/$BOOST_TARBALL \
  24     -C - -O https://dev-www.libreoffice.org/src/$BOX2D_TARBALL \
  25     -C - -O https://dev-www.libreoffice.org/src/$DTOA_TARBALL \
  26     -C - -O https://dev-www.libreoffice.org/src/$EXPAT_TARBALL \
  27     -C - -O https://dev-www.libreoffice.org/src/$LIBJPEG_TURBO_TARBALL \
  28     -C - -O https://dev-www.libreoffice.org/src/$LCMS2_TARBALL \
  29     -C - -O https://dev-www.libreoffice.org/src/$LIBEXTTEXTCAT_TARBALL \
  30     -C - -O https://dev-www.libreoffice.org/src/$CPPUNIT_TARBALL \
  31     -C - -O https://dev-www.libreoffice.org/src/$CAIRO_TARBALL \
  32     -C - -O https://dev-www.libreoffice.org/src/$CURL_TARBALL \
  33     -C - -O https://dev-www.libreoffice.org/src/$XMLSEC_TARBALL \
  34     -C - -O https://dev-www.libreoffice.org/src/$LIBLANGTAG_TARBALL \
  35     -C - -O https://dev-www.libreoffice.org/src/$ABW_TARBALL \
  36     -C - -O https://dev-www.libreoffice.org/src/$CDR_TARBALL \
  37     -C - -O https://dev-www.libreoffice.org/src/$LIBCMIS_TARBALL \
  38     -C - -O https://dev-www.libreoffice.org/src/$EBOOK_TARBALL \
  39     -C - -O https://dev-www.libreoffice.org/src/$ETONYEK_TARBALL \
  40     -C - -O https://dev-www.libreoffice.org/src/$FREEHAND_TARBALL \
  41     -C - -O https://dev-www.libreoffice.org/src/$MSPUB_TARBALL \
  42     -C - -O https://dev-www.libreoffice.org/src/$MWAW_TARBALL \
  43     -C - -O https://dev-www.libreoffice.org/src/$ODFGEN_TARBALL \
  44     -C - -O https://dev-www.libreoffice.org/src/$ORCUS_TARBALL \
  45     -C - -O https://dev-www.libreoffice.org/src/$PAGEMAKER_TARBALL \
  46     -C - -O https://dev-www.libreoffice.org/src/$LIBPNG_TARBALL \
  47     -C - -O https://dev-www.libreoffice.org/src/$REVENGE_TARBALL \
  48     -C - -O https://dev-www.libreoffice.org/src/$STAROFFICE_TARBALL \
  49     -C - -O https://dev-www.libreoffice.org/src/$VISIO_TARBALL \
  50     -C - -O https://dev-www.libreoffice.org/src/$WPD_TARBALL \
  51     -C - -O https://dev-www.libreoffice.org/src/$WPG_TARBALL \
  52     -C - -O https://dev-www.libreoffice.org/src/$WPS_TARBALL \
  53     -C - -O https://dev-www.libreoffice.org/src/$ZMF_TARBALL \
  54     -C - -O https://dev-www.libreoffice.org/src/$PIXMAN_TARBALL \
  55     -C - -O https://dev-www.libreoffice.org/src/$ZLIB_TARBALL \
  56     -C - -O https://dev-www.libreoffice.org/src/$MDDS_TARBALL \
  57     -C - -O https://dev-www.libreoffice.org/src/$OPENSSL_TARBALL \
  58     -C - -O https://dev-www.libreoffice.org/src/$LANGTAGREG_TARBALL \
  59     -C - -O https://dev-www.libreoffice.org/src/$GRAPHITE_TARBALL \
  60     -C - -O https://dev-www.libreoffice.org/src/$HARFBUZZ_TARBALL \
  61     -C - -O https://dev-www.libreoffice.org/src/$GLM_TARBALL \
  62     -C - -O https://dev-www.libreoffice.org/src/$ICU_TARBALL \
  63     -C - -O https://dev-www.libreoffice.org/src/$ICU_DATA_TARBALL \
  64     -C - -O https://dev-www.libreoffice.org/src/$LIBXML_TARBALL \
  65     -C - -O https://dev-www.libreoffice.org/src/$LIBXSLT_TARBALL \
  66     -C - -O https://dev-www.libreoffice.org/src/$HUNSPELL_TARBALL \
  67     -C - -O https://dev-www.libreoffice.org/src/$FREETYPE_TARBALL \
  68     -C - -O https://dev-www.libreoffice.org/src/$FONTCONFIG_TARBALL \
  69     -C - -O https://dev-www.libreoffice.org/src/$EPOXY_TARBALL \
  70     -C - -O https://dev-www.libreoffice.org/src/$EPUBGEN_TARBALL \
  71     -C - -O https://dev-www.libreoffice.org/src/$LIBNUMBERTEXT_TARBALL \
  72     -C - -O https://dev-www.libreoffice.org/src/$QXP_TARBALL \
  73     -C - -O https://dev-www.libreoffice.org/src/$LIBWEBP_TARBALL \
  74     -C - -O https://dev-www.libreoffice.org/src/$LIBTIFF_TARBALL \
  75     -C - -O https://dev-www.libreoffice.org/src/$DRAGONBOX_TARBALL
  76
  77 cd $SRC
  78
  79 #fuzzing dictionaries
  80 curl --no-progress-meter -S \
  81     -C - -O https://raw.githubusercontent.com/google/fuzzing/master/dictionaries/gif.dict \
  82     -C - -O https://raw.githubusercontent.com/google/fuzzing/master/dictionaries/jpeg.dict \
  83     -C - -O https://raw.githubusercontent.com/google/fuzzing/master/dictionaries/png.dict \
  84     -C - -O https://raw.githubusercontent.com/google/fuzzing/master/dictionaries/tiff.dict \
  85     -C - -O https://raw.githubusercontent.com/google/fuzzing/master/dictionaries/xml.dict \
  86     -C - -O https://raw.githubusercontent.com/google/fuzzing/master/dictionaries/html.dict \
  87     -C - -O https://raw.githubusercontent.com/google/fuzzing/master/dictionaries/svg.dict \
  88     -C - -O https://raw.githubusercontent.com/google/fuzzing/master/dictionaries/bmp.dict \
  89     -C - -O https://raw.githubusercontent.com/google/fuzzing/master/dictionaries/otf.dict \
  90     -C - -O https://raw.githubusercontent.com/google/fuzzing/master/dictionaries/psd.dict \
  91     -C - -O https://raw.githubusercontent.com/google/fuzzing/master/dictionaries/rtf.dict \
  92     -C - -O https://raw.githubusercontent.com/google/fuzzing/master/dictionaries/webp.dict \
  93     -C - -O https://raw.githubusercontent.com/google/fuzzing/master/dictionaries/zip.dict \
  94     -C - -O https://raw.githubusercontent.com/google/fuzzing/master/dictionaries/mathml.dict
  95 # build our own fuzz dict for odf, following the pattern of svg.dict
  96 echo "# Keywords taken from libreoffice/schema/odf1.3/OpenDocument-v1.3-schema.rng" > odf.dict
  97 echo "# tags" >> odf.dict
  98 grep "rng:element name=" libreoffice/schema/odf1.3/OpenDocument-v1.3-schema.rng | sed 's#<rng:element name="#"<#;s#^[[:blank:]]*##;s#[[:blank:]>]*$##' >> odf.dict
  99 echo "# attributes " >> odf.dict
 100 grep "rng:attribute name=" libreoffice/schema/odf1.3/OpenDocument-v1.3-schema.rng | sed 's#<rng:attribute name="#"#;s#^[[:blank:]]*##;s#[[:blank:]>]*$##' >> odf.dict
 101 echo "# attributes' values" >> odf.dict
 102 grep "rng:value" libreoffice/schema/odf1.3/OpenDocument-v1.3-schema.rng | sed 's#<rng:value>#"#;s#</rng:value>#"#;s#^[[:blank:]]*##;s#[[:blank:]>]*$##' | sort | uniq >> odf.dict
 103
 104 #fuzzing corpuses
 105 #afl jpeg, gif, bmp, png, webp
 106 curl --no-progress-meter -S -C - -O https://lcamtuf.coredump.cx/afl/demo/afl_testcases.tgz
 107 mkdir -p afl-testcases && cd afl-testcases/ && tar xf $SRC/afl_testcases.tgz && cd .. && \
 108     zip -q $SRC/jpgfuzzer_seed_corpus.zip afl-testcases/jpeg*/full/images/* && \
 109     zip -q $SRC/giffuzzer_seed_corpus.zip afl-testcases/gif*/full/images/* && \
 110     zip -q $SRC/bmpfuzzer_seed_corpus.zip afl-testcases/bmp*/full/images/* && \
 111     zip -q $SRC/pngfuzzer_seed_corpus.zip afl-testcases/png*/full/images/* && \
 112     zip -q $SRC/webpfuzzer_seed_corpus.zip afl-testcases/webp*/full/images/*
 113 # using github's svn view to use svn export as a hack to just export part of the git repo
 114 svn export --force -q https://github.com/khaledhosny/ots/trunk/tests/fonts $SRC/sample-sft-fonts/ots
 115 svn export --force -q https://github.com/unicode-org/text-rendering-tests/trunk/fonts/ $SRC/sample-sft-fonts/unicode-org
 116 svn export --force -q https://github.com/harfbuzz/harfbuzz/trunk/test/shape/data/in-house/fonts $SRC/sample-sft-fonts/harfbuzz
 117 mkdir -p $SRC/sample-sft-fonts/adobe
 118 curl --no-progress-meter -S \
 119     -C - -o $SRC/sample-sft-fonts/adobe/AdobeVFPrototype.otf https://github.com/adobe-fonts/adobe-variable-font-prototype/releases/download/1.001/AdobeVFPrototype.otf
 120 zip -qr $SRC/sftfuzzer_seed_corpus.zip $SRC/sample-sft-fonts
 121 curl --no-progress-meter -S -C - https://storage.googleapis.com/skia-fuzzer/oss-fuzz/svg_seed_corpus.zip -o svgfuzzer_seed_corpus.zip
 122 curl --no-progress-meter -S \
 123     -C - -O https://dev-www.libreoffice.org/corpus/wmffuzzer_seed_corpus.zip \
 124     -C - -O https://dev-www.libreoffice.org/corpus/xbmfuzzer_seed_corpus.zip \
 125     -C - -O https://dev-www.libreoffice.org/corpus/xpmfuzzer_seed_corpus.zip \
 126     -C - -O https://dev-www.libreoffice.org/corpus/svmfuzzer_seed_corpus.zip \
 127     -C - -O https://dev-www.libreoffice.org/corpus/pcdfuzzer_seed_corpus.zip \
 128     -C - -O https://dev-www.libreoffice.org/corpus/dxffuzzer_seed_corpus.zip \
 129     -C - -O https://dev-www.libreoffice.org/corpus/metfuzzer_seed_corpus.zip \
 130     -C - -O https://dev-www.libreoffice.org/corpus/ppmfuzzer_seed_corpus.zip \
 131     -C - -O https://dev-www.libreoffice.org/corpus/psdfuzzer_seed_corpus.zip \
 132     -C - -O https://dev-www.libreoffice.org/corpus/epsfuzzer_seed_corpus.zip \
 133     -C - -O https://dev-www.libreoffice.org/corpus/pctfuzzer_seed_corpus.zip \
 134     -C - -O https://dev-www.libreoffice.org/corpus/pcxfuzzer_seed_corpus.zip \
 135     -C - -O https://dev-www.libreoffice.org/corpus/rasfuzzer_seed_corpus.zip \
 136     -C - -O https://dev-www.libreoffice.org/corpus/tgafuzzer_seed_corpus.zip \
 137     -C - -O https://dev-www.libreoffice.org/corpus/tiffuzzer_seed_corpus.zip \
 138     -C - -O https://dev-www.libreoffice.org/corpus/hwpfuzzer_seed_corpus.zip \
 139     -C - -O https://dev-www.libreoffice.org/corpus/602fuzzer_seed_corpus.zip \
 140     -C - -O https://dev-www.libreoffice.org/corpus/lwpfuzzer_seed_corpus.zip \
 141     -C - -O https://dev-www.libreoffice.org/corpus/pptfuzzer_seed_corpus.zip \
 142     -C - -O https://dev-www.libreoffice.org/corpus/rtffuzzer_seed_corpus.zip \
 143     -C - -O https://dev-www.libreoffice.org/corpus/olefuzzer_seed_corpus.zip \
 144     -C - -O https://dev-www.libreoffice.org/corpus/cgmfuzzer_seed_corpus.zip \
 145     -C - -O https://dev-www.libreoffice.org/corpus/ww2fuzzer_seed_corpus.zip \
 146     -C - -O https://dev-www.libreoffice.org/corpus/ww6fuzzer_seed_corpus.zip \
 147     -C - -O https://dev-www.libreoffice.org/corpus/ww8fuzzer_seed_corpus.zip \
 148     -C - -O https://dev-www.libreoffice.org/corpus/qpwfuzzer_seed_corpus.zip \
 149     -C - -O https://dev-www.libreoffice.org/corpus/slkfuzzer_seed_corpus.zip \
 150     -C - -O https://dev-www.libreoffice.org/corpus/fodtfuzzer_seed_corpus.zip \
 151     -C - -O https://dev-www.libreoffice.org/corpus/fodsfuzzer_seed_corpus.zip \
 152     -C - -O https://dev-www.libreoffice.org/corpus/fodgfuzzer_seed_corpus.zip \
 153     -C - -O https://dev-www.libreoffice.org/corpus/fodpfuzzer_seed_corpus.zip \
 154     -C - -O https://dev-www.libreoffice.org/corpus/xlsfuzzer_seed_corpus.zip \
 155     -C - -O https://dev-www.libreoffice.org/corpus/scrtffuzzer_seed_corpus.zip \
 156     -C - -O https://dev-www.libreoffice.org/corpus/wksfuzzer_seed_corpus.zip \
 157     -C - -O https://dev-www.libreoffice.org/corpus/dbffuzzer_seed_corpus.zip \
 158     -C - -O https://dev-www.libreoffice.org/corpus/diffuzzer_seed_corpus.zip \
 159     -C - -O https://dev-www.libreoffice.org/corpus/docxfuzzer_seed_corpus.zip \
 160     -C - -O https://dev-www.libreoffice.org/corpus/xlsxfuzzer_seed_corpus.zip \
 161     -C - -O https://dev-www.libreoffice.org/corpus/pptxfuzzer_seed_corpus.zip \
 162     -C - -O https://dev-www.libreoffice.org/corpus/mmlfuzzer_seed_corpus.zip \
 163     -C - -O https://dev-www.libreoffice.org/corpus/mtpfuzzer_seed_corpus.zip \
 164     -C - -O https://dev-www.libreoffice.org/corpus/htmlfuzzer_seed_corpus.zip \
 165     -C - -O https://dev-www.libreoffice.org/corpus/zipfuzzer_seed_corpus.zip
 166 cp fodtfuzzer_seed_corpus.zip fodt2pdffuzzer_seed_corpus.zip
 167
 168 echo end downloading dependencies at `date -u`