media/ffmpeg/ffmpeg_regression_tests.cc

   1 // Copyright (c) 2012 The Chromium Authors. All rights reserved.
   2 // Use of this source code is governed by a BSD-style license that can be
   3 // found in the LICENSE file.
   4 //
   5 // Regression tests for FFmpeg.  Test files can be found in the internal media
   6 // test data directory:
   7 //
   8 //    svn://svn.chromium.org/chrome-internal/trunk/data/media/
   9 //
  10 // Simply add the custom_dep below to your gclient and sync:
  11 //
  12 //    "src/media/test/data/internal":
  13 //        "svn://svn.chromium.org/chrome-internal/trunk/data/media"
  14 //
  15 // Many of the files here do not cause issues outside of tooling, so you'll need
  16 // to run this test under ASAN, TSAN, and Valgrind to ensure that all issues are
  17 // caught.
  18 //
  19 // Test cases labeled FLAKY may not always pass, but they should never crash or
  20 // cause any kind of warnings or errors under tooling.
  21
  22 #include <string>
  23
  24 #include "base/bind.h"
  25 #include "media/test/pipeline_integration_test_base.h"
  26
  27 namespace media {
  28
  29 const char kRegressionTestDataPathPrefix[] = "internal/";
  30
  31 struct RegressionTestData {
  32   RegressionTestData(const char* filename,
  33                      PipelineStatus init_status,
  34                      PipelineStatus end_status)
  35       : filename(std::string(kRegressionTestDataPathPrefix) + filename),
  36         init_status(init_status),
  37         end_status(end_status) {}
  38
  39   std::string filename;
  40   PipelineStatus init_status;
  41   PipelineStatus end_status;
  42 };
  43
  44 // Used for tests which just need to run without crashing or tooling errors, but
  45 // which may have undefined PipelineStatus results.
  46 struct FlakyRegressionTestData {
  47   FlakyRegressionTestData(const char* filename)
  48       : filename(std::string(kRegressionTestDataPathPrefix) + filename) {
  49   }
  50
  51   std::string filename;
  52 };
  53
  54 class FFmpegRegressionTest
  55     : public testing::TestWithParam<RegressionTestData>,
  56       public PipelineIntegrationTestBase {
  57 };
  58
  59 class FlakyFFmpegRegressionTest
  60     : public testing::TestWithParam<FlakyRegressionTestData>,
  61       public PipelineIntegrationTestBase {
  62 };
  63
  64 #define FFMPEG_TEST_CASE(name, fn, init_status, end_status) \
  65   INSTANTIATE_TEST_CASE_P(                                  \
  66       name, FFmpegRegressionTest,                           \
  67       testing::Values(RegressionTestData(fn, init_status, end_status)));
  68
  69 #define FLAKY_FFMPEG_TEST_CASE(name, fn) \
  70     INSTANTIATE_TEST_CASE_P(FLAKY_##name, FlakyFFmpegRegressionTest, \
  71                             testing::Values(FlakyRegressionTestData(fn)));
  72
  73 // Test cases from issues.
  74 FFMPEG_TEST_CASE(Cr47325, "security/47325.mp4", PIPELINE_OK, PIPELINE_OK);
  75 FFMPEG_TEST_CASE(Cr47761, "crbug47761.ogg", PIPELINE_OK, PIPELINE_OK);
  76 FFMPEG_TEST_CASE(Cr50045, "crbug50045.mp4", PIPELINE_OK, PIPELINE_OK);
  77 FFMPEG_TEST_CASE(Cr62127, "crbug62127.webm", PIPELINE_OK, PIPELINE_OK);
  78 FFMPEG_TEST_CASE(Cr93620, "security/93620.ogg", PIPELINE_OK, PIPELINE_OK);
  79 FFMPEG_TEST_CASE(Cr100492,
  80                  "security/100492.webm",
  81                  DECODER_ERROR_NOT_SUPPORTED,
  82                  DECODER_ERROR_NOT_SUPPORTED);
  83 FFMPEG_TEST_CASE(Cr100543, "security/100543.webm", PIPELINE_OK, PIPELINE_OK);
  84 FFMPEG_TEST_CASE(Cr101458, "security/101458.webm", PIPELINE_OK, PIPELINE_OK);
  85 FFMPEG_TEST_CASE(Cr108416, "security/108416.webm", PIPELINE_OK, PIPELINE_OK);
  86 FFMPEG_TEST_CASE(Cr110849,
  87                  "security/110849.mkv",
  88                  DEMUXER_ERROR_COULD_NOT_OPEN,
  89                  DEMUXER_ERROR_NO_SUPPORTED_STREAMS);
  90 FFMPEG_TEST_CASE(Cr112384,
  91                  "security/112384.webm",
  92                  DEMUXER_ERROR_COULD_NOT_PARSE,
  93                  DEMUXER_ERROR_COULD_NOT_PARSE);
  94 FFMPEG_TEST_CASE(Cr112976, "security/112976.ogg", PIPELINE_OK, PIPELINE_OK);
  95 FFMPEG_TEST_CASE(Cr116927,
  96                  "security/116927.ogv",
  97                  DEMUXER_ERROR_NO_SUPPORTED_STREAMS,
  98                  DEMUXER_ERROR_NO_SUPPORTED_STREAMS);
  99 FFMPEG_TEST_CASE(Cr117912,
 100                  "security/117912.webm",
 101                  DEMUXER_ERROR_COULD_NOT_OPEN,
 102                  DEMUXER_ERROR_COULD_NOT_OPEN);
 103 FFMPEG_TEST_CASE(Cr123481, "security/123481.ogv", PIPELINE_OK, PIPELINE_OK);
 104 FFMPEG_TEST_CASE(Cr132779,
 105                  "security/132779.webm",
 106                  DEMUXER_ERROR_COULD_NOT_PARSE,
 107                  DEMUXER_ERROR_COULD_NOT_PARSE);
 108 FFMPEG_TEST_CASE(Cr140165, "security/140165.ogg", PIPELINE_OK, PIPELINE_OK);
 109 FFMPEG_TEST_CASE(Cr140647,
 110                  "security/140647.ogv",
 111                  DEMUXER_ERROR_COULD_NOT_OPEN,
 112                  DEMUXER_ERROR_COULD_NOT_OPEN);
 113 FFMPEG_TEST_CASE(Cr142738, "crbug142738.ogg", PIPELINE_OK, PIPELINE_OK);
 114 FFMPEG_TEST_CASE(Cr152691,
 115                  "security/152691.mp3",
 116                  PIPELINE_OK,
 117                  PIPELINE_ERROR_DECODE);
 118 FFMPEG_TEST_CASE(Cr161639,
 119                  "security/161639.m4a",
 120                  PIPELINE_OK,
 121                  PIPELINE_ERROR_DECODE);
 122 FFMPEG_TEST_CASE(Cr222754,
 123                  "security/222754.mp4",
 124                  PIPELINE_OK,
 125                  PIPELINE_ERROR_DECODE);
 126 FFMPEG_TEST_CASE(Cr234630a, "security/234630a.mov", PIPELINE_OK, PIPELINE_OK);
 127 FFMPEG_TEST_CASE(Cr234630b,
 128                  "security/234630b.mov",
 129                  PIPELINE_OK,
 130                  PIPELINE_ERROR_DECODE);
 131 FFMPEG_TEST_CASE(Cr242786, "security/242786.webm", PIPELINE_OK, PIPELINE_OK);
 132 // Test for out-of-bounds access with slightly corrupt file (detection logic
 133 // thinks it's a MONO file, but actually contains STEREO audio).
 134 FFMPEG_TEST_CASE(Cr275590,
 135                  "security/275590.m4a",
 136                  DECODER_ERROR_NOT_SUPPORTED,
 137                  DEMUXER_ERROR_COULD_NOT_OPEN);
 138 FFMPEG_TEST_CASE(Cr444522, "security/444522.mp4", PIPELINE_OK, PIPELINE_OK);
 139 FFMPEG_TEST_CASE(Cr444539,
 140                  "security/444539.m4a",
 141                  DEMUXER_ERROR_COULD_NOT_OPEN,
 142                  DEMUXER_ERROR_COULD_NOT_OPEN);
 143 FFMPEG_TEST_CASE(Cr444546,
 144                  "security/444546.mp4",
 145                  DEMUXER_ERROR_COULD_NOT_OPEN,
 146                  DEMUXER_ERROR_COULD_NOT_OPEN);
 147 FFMPEG_TEST_CASE(Cr449958,
 148                  "security/449958.webm",
 149                  PIPELINE_OK,
 150                  PIPELINE_ERROR_DECODE);
 151
 152 // General MP4 test cases.
 153 FFMPEG_TEST_CASE(MP4_0,
 154                  "security/aac.10419.mp4",
 155                  DEMUXER_ERROR_COULD_NOT_OPEN,
 156                  DEMUXER_ERROR_COULD_NOT_OPEN);
 157 FFMPEG_TEST_CASE(MP4_1,
 158                  "security/clockh264aac_200021889.mp4",
 159                  DEMUXER_ERROR_COULD_NOT_OPEN,
 160                  DEMUXER_ERROR_COULD_NOT_OPEN);
 161 FFMPEG_TEST_CASE(MP4_2,
 162                  "security/clockh264aac_200701257.mp4",
 163                  PIPELINE_OK,
 164                  PIPELINE_OK);
 165 FFMPEG_TEST_CASE(MP4_5,
 166                  "security/clockh264aac_3022500.mp4",
 167                  DEMUXER_ERROR_NO_SUPPORTED_STREAMS,
 168                  DEMUXER_ERROR_NO_SUPPORTED_STREAMS);
 169 FFMPEG_TEST_CASE(MP4_6,
 170                  "security/clockh264aac_344289.mp4",
 171                  PIPELINE_OK,
 172                  PIPELINE_OK);
 173 FFMPEG_TEST_CASE(MP4_7,
 174                  "security/clockh264mp3_187697.mp4",
 175                  PIPELINE_OK,
 176                  PIPELINE_OK);
 177 FFMPEG_TEST_CASE(MP4_8,
 178                  "security/h264.705767.mp4",
 179                  DEMUXER_ERROR_COULD_NOT_PARSE,
 180                  DEMUXER_ERROR_COULD_NOT_PARSE);
 181 FFMPEG_TEST_CASE(MP4_9,
 182                  "security/smclockmp4aac_1_0.mp4",
 183                  DEMUXER_ERROR_COULD_NOT_OPEN,
 184                  DEMUXER_ERROR_COULD_NOT_OPEN);
 185 FFMPEG_TEST_CASE(MP4_11, "security/null1.mp4", PIPELINE_OK, PIPELINE_OK);
 186 FFMPEG_TEST_CASE(MP4_16,
 187                  "security/looping2.mov",
 188                  DEMUXER_ERROR_COULD_NOT_OPEN,
 189                  DEMUXER_ERROR_COULD_NOT_OPEN);
 190 FFMPEG_TEST_CASE(MP4_17, "security/assert2.mov", PIPELINE_OK, PIPELINE_OK);
 191
 192 // General OGV test cases.
 193 FFMPEG_TEST_CASE(OGV_1,
 194                  "security/out.163.ogv",
 195                  DECODER_ERROR_NOT_SUPPORTED,
 196                  DECODER_ERROR_NOT_SUPPORTED);
 197 FFMPEG_TEST_CASE(OGV_2,
 198                  "security/out.391.ogv",
 199                  DECODER_ERROR_NOT_SUPPORTED,
 200                  DECODER_ERROR_NOT_SUPPORTED);
 201 FFMPEG_TEST_CASE(OGV_5,
 202                  "security/smclocktheora_1_0.ogv",
 203                  DECODER_ERROR_NOT_SUPPORTED,
 204                  DECODER_ERROR_NOT_SUPPORTED);
 205 FFMPEG_TEST_CASE(OGV_7,
 206                  "security/smclocktheora_1_102.ogv",
 207                  DECODER_ERROR_NOT_SUPPORTED,
 208                  DECODER_ERROR_NOT_SUPPORTED);
 209 FFMPEG_TEST_CASE(OGV_8,
 210                  "security/smclocktheora_1_104.ogv",
 211                  DECODER_ERROR_NOT_SUPPORTED,
 212                  DECODER_ERROR_NOT_SUPPORTED);
 213 FFMPEG_TEST_CASE(OGV_9,
 214                  "security/smclocktheora_1_110.ogv",
 215                  DECODER_ERROR_NOT_SUPPORTED,
 216                  DECODER_ERROR_NOT_SUPPORTED);
 217 FFMPEG_TEST_CASE(OGV_10,
 218                  "security/smclocktheora_1_179.ogv",
 219                  DECODER_ERROR_NOT_SUPPORTED,
 220                  DECODER_ERROR_NOT_SUPPORTED);
 221 FFMPEG_TEST_CASE(OGV_11,
 222                  "security/smclocktheora_1_20.ogv",
 223                  DECODER_ERROR_NOT_SUPPORTED,
 224                  DECODER_ERROR_NOT_SUPPORTED);
 225 FFMPEG_TEST_CASE(OGV_12,
 226                  "security/smclocktheora_1_723.ogv",
 227                  DECODER_ERROR_NOT_SUPPORTED,
 228                  DECODER_ERROR_NOT_SUPPORTED);
 229 FFMPEG_TEST_CASE(OGV_14,
 230                  "security/smclocktheora_2_10405.ogv",
 231                  PIPELINE_OK,
 232                  PIPELINE_OK);
 233 FFMPEG_TEST_CASE(OGV_15,
 234                  "security/smclocktheora_2_10619.ogv",
 235                  DECODER_ERROR_NOT_SUPPORTED,
 236                  DECODER_ERROR_NOT_SUPPORTED);
 237 FFMPEG_TEST_CASE(OGV_16,
 238                  "security/smclocktheora_2_1075.ogv",
 239                  DECODER_ERROR_NOT_SUPPORTED,
 240                  DECODER_ERROR_NOT_SUPPORTED);
 241 FFMPEG_TEST_CASE(OGV_17,
 242                  "security/vorbis.482086.ogv",
 243                  PIPELINE_OK,
 244                  PIPELINE_OK);
 245 FFMPEG_TEST_CASE(OGV_18,
 246                  "security/wav.711.ogv",
 247                  DECODER_ERROR_NOT_SUPPORTED,
 248                  DECODER_ERROR_NOT_SUPPORTED);
 249 FFMPEG_TEST_CASE(OGV_19,
 250                  "security/null1.ogv",
 251                  DECODER_ERROR_NOT_SUPPORTED,
 252                  DECODER_ERROR_NOT_SUPPORTED);
 253 FFMPEG_TEST_CASE(OGV_20,
 254                  "security/null2.ogv",
 255                  DECODER_ERROR_NOT_SUPPORTED,
 256                  DECODER_ERROR_NOT_SUPPORTED);
 257 FFMPEG_TEST_CASE(OGV_21,
 258                  "security/assert1.ogv",
 259                  DECODER_ERROR_NOT_SUPPORTED,
 260                  DECODER_ERROR_NOT_SUPPORTED);
 261 FFMPEG_TEST_CASE(OGV_22,
 262                  "security/assert2.ogv",
 263                  DECODER_ERROR_NOT_SUPPORTED,
 264                  DECODER_ERROR_NOT_SUPPORTED);
 265 FFMPEG_TEST_CASE(OGV_23,
 266                  "security/assert2.ogv",
 267                  DECODER_ERROR_NOT_SUPPORTED,
 268                  DECODER_ERROR_NOT_SUPPORTED);
 269
 270 // General WebM test cases.
 271 FFMPEG_TEST_CASE(WEBM_0, "security/memcpy.webm", PIPELINE_OK, PIPELINE_OK);
 272 FFMPEG_TEST_CASE(WEBM_1, "security/no-bug.webm", PIPELINE_OK, PIPELINE_OK);
 273 FFMPEG_TEST_CASE(WEBM_2,
 274                  "security/uninitialize.webm",
 275                  DEMUXER_ERROR_NO_SUPPORTED_STREAMS,
 276                  DEMUXER_ERROR_NO_SUPPORTED_STREAMS);
 277 FFMPEG_TEST_CASE(WEBM_4,
 278                  "security/out.webm.68798.1929",
 279                  DECODER_ERROR_NOT_SUPPORTED,
 280                  DECODER_ERROR_NOT_SUPPORTED);
 281 FFMPEG_TEST_CASE(WEBM_5, "frame_size_change.webm", PIPELINE_OK, PIPELINE_OK);
 282
 283 // General MKV test cases.
 284 FFMPEG_TEST_CASE(MKV_0,
 285                  "security/nested_tags_lang.mka.627.628",
 286                  PIPELINE_OK,
 287                  PIPELINE_ERROR_DECODE);
 288 FFMPEG_TEST_CASE(MKV_1,
 289                  "security/nested_tags_lang.mka.667.628",
 290                  PIPELINE_OK,
 291                  PIPELINE_ERROR_DECODE);
 292
 293 // Allocate gigabytes of memory, likely can't be run on 32bit machines.
 294 FFMPEG_TEST_CASE(BIG_MEM_1,
 295                  "security/bigmem1.mov",
 296                  DEMUXER_ERROR_COULD_NOT_OPEN,
 297                  DEMUXER_ERROR_COULD_NOT_OPEN);
 298 FFMPEG_TEST_CASE(BIG_MEM_2,
 299                  "security/looping1.mov",
 300                  DEMUXER_ERROR_COULD_NOT_OPEN,
 301                  DEMUXER_ERROR_COULD_NOT_OPEN);
 302 FFMPEG_TEST_CASE(BIG_MEM_5,
 303                  "security/looping5.mov",
 304                  DEMUXER_ERROR_COULD_NOT_OPEN,
 305                  DEMUXER_ERROR_COULD_NOT_OPEN);
 306 FLAKY_FFMPEG_TEST_CASE(BIG_MEM_3, "security/looping3.mov");
 307 FLAKY_FFMPEG_TEST_CASE(BIG_MEM_4, "security/looping4.mov");
 308
 309 // Flaky under threading or for other reasons.  Per rbultje, most of these will
 310 // never be reliable since FFmpeg does not guarantee consistency in error cases.
 311 // We only really care that these don't cause crashes or errors under tooling.
 312 FLAKY_FFMPEG_TEST_CASE(Cr99652, "security/99652.webm");
 313 FLAKY_FFMPEG_TEST_CASE(Cr100464, "security/100464.webm");
 314 FLAKY_FFMPEG_TEST_CASE(Cr111342, "security/111342.ogm");
 315 FLAKY_FFMPEG_TEST_CASE(Cr368980, "security/368980.mp4");
 316 FLAKY_FFMPEG_TEST_CASE(OGV_0, "security/big_dims.ogv");
 317 FLAKY_FFMPEG_TEST_CASE(OGV_3, "security/smclock_1_0.ogv");
 318 FLAKY_FFMPEG_TEST_CASE(OGV_4, "security/smclock.ogv.1.0.ogv");
 319 FLAKY_FFMPEG_TEST_CASE(OGV_6, "security/smclocktheora_1_10000.ogv");
 320 FLAKY_FFMPEG_TEST_CASE(OGV_13, "security/smclocktheora_1_790.ogv");
 321 FLAKY_FFMPEG_TEST_CASE(MP4_3, "security/clockh264aac_300413969.mp4");
 322 FLAKY_FFMPEG_TEST_CASE(MP4_4, "security/clockh264aac_301350139.mp4");
 323 FLAKY_FFMPEG_TEST_CASE(MP4_12, "security/assert1.mov");
 324 FLAKY_FFMPEG_TEST_CASE(WEBM_3, "security/out.webm.139771.2965");
 325
 326 // Not really flaky, but can't pass the seek test.
 327 FLAKY_FFMPEG_TEST_CASE(MP4_10, "security/null1.m4a");
 328 FLAKY_FFMPEG_TEST_CASE(Cr112670, "security/112670.mp4");
 329
 330 TEST_P(FFmpegRegressionTest, BasicPlayback) {
 331   if (GetParam().init_status == PIPELINE_OK) {
 332     ASSERT_EQ(PIPELINE_OK, Start(GetParam().filename, kClockless));
 333     Play();
 334     ASSERT_EQ(WaitUntilEndedOrError(), GetParam().end_status);
 335
 336     // Check for ended if the pipeline is expected to finish okay.
 337     if (GetParam().end_status == PIPELINE_OK) {
 338       ASSERT_TRUE(ended_);
 339
 340       // Tack a seek on the end to catch any seeking issues.
 341       Seek(base::TimeDelta::FromMilliseconds(0));
 342     }
 343   } else {
 344     // Don't bother checking the exact status as we only care that the
 345     // pipeline failed to start.
 346     EXPECT_NE(PIPELINE_OK, Start(GetParam().filename));
 347   }
 348 }
 349
 350 TEST_P(FlakyFFmpegRegressionTest, BasicPlayback) {
 351   if (Start(GetParam().filename, kClockless) == PIPELINE_OK) {
 352     Play();
 353     WaitUntilEndedOrError();
 354   }
 355 }
 356
 357 }  // namespace media