media/ffmpeg/ffmpeg_regression_tests.cc

   1 // Copyright (c) 2012 The Chromium Authors. All rights reserved.
   2 // Use of this source code is governed by a BSD-style license that can be
   3 // found in the LICENSE file.
   4 //
   5 // Regression tests for FFmpeg.  Test files can be found in the internal media
   6 // test data directory:
   7 //
   8 //    https://chrome-internal.googlesource.com/chrome/data/media
   9 //
  10 // Simply add the custom_dep below to your gclient and sync:
  11 //
  12 //    "src/media/test/data/internal":
  13 //        "https://chrome-internal.googlesource.com/chrome/data/media"
  14 //
  15 // Many of the files here do not cause issues outside of tooling, so you'll need
  16 // to run this test under ASAN, TSAN, and Valgrind to ensure that all issues are
  17 // caught.
  18 //
  19 // Test cases labeled FLAKY may not always pass, but they should never crash or
  20 // cause any kind of warnings or errors under tooling.
  21
  22 #include <string>
  23
  24 #include "base/bind.h"
  25 #include "media/test/pipeline_integration_test_base.h"
  26
  27 namespace media {
  28
  29 const char kRegressionTestDataPathPrefix[] = "internal/";
  30
  31 struct RegressionTestData {
  32   RegressionTestData(const char* filename,
  33                      PipelineStatus init_status,
  34                      PipelineStatus end_status)
  35       : filename(std::string(kRegressionTestDataPathPrefix) + filename),
  36         init_status(init_status),
  37         end_status(end_status) {}
  38
  39   std::string filename;
  40   PipelineStatus init_status;
  41   PipelineStatus end_status;
  42 };
  43
  44 // Used for tests which just need to run without crashing or tooling errors, but
  45 // which may have undefined PipelineStatus results.
  46 struct FlakyRegressionTestData {
  47   FlakyRegressionTestData(const char* filename)
  48       : filename(std::string(kRegressionTestDataPathPrefix) + filename) {
  49   }
  50
  51   std::string filename;
  52 };
  53
  54 class FFmpegRegressionTest
  55     : public testing::TestWithParam<RegressionTestData>,
  56       public PipelineIntegrationTestBase {
  57 };
  58
  59 class FlakyFFmpegRegressionTest
  60     : public testing::TestWithParam<FlakyRegressionTestData>,
  61       public PipelineIntegrationTestBase {
  62 };
  63
  64 #define FFMPEG_TEST_CASE(name, fn, init_status, end_status) \
  65   INSTANTIATE_TEST_CASE_P(                                  \
  66       name, FFmpegRegressionTest,                           \
  67       testing::Values(RegressionTestData(fn, init_status, end_status)));
  68
  69 #define FLAKY_FFMPEG_TEST_CASE(name, fn) \
  70     INSTANTIATE_TEST_CASE_P(FLAKY_##name, FlakyFFmpegRegressionTest, \
  71                             testing::Values(FlakyRegressionTestData(fn)));
  72
  73 // Test cases from issues.
  74 FFMPEG_TEST_CASE(Cr47325, "security/47325.mp4", PIPELINE_OK, PIPELINE_OK);
  75 FFMPEG_TEST_CASE(Cr47761, "crbug47761.ogg", PIPELINE_OK, PIPELINE_OK);
  76 FFMPEG_TEST_CASE(Cr50045, "crbug50045.mp4", PIPELINE_OK, PIPELINE_OK);
  77 FFMPEG_TEST_CASE(Cr62127, "crbug62127.webm", PIPELINE_OK, PIPELINE_OK);
  78 FFMPEG_TEST_CASE(Cr93620, "security/93620.ogg", PIPELINE_OK, PIPELINE_OK);
  79 FFMPEG_TEST_CASE(Cr100492,
  80                  "security/100492.webm",
  81                  DECODER_ERROR_NOT_SUPPORTED,
  82                  DECODER_ERROR_NOT_SUPPORTED);
  83 FFMPEG_TEST_CASE(Cr100543, "security/100543.webm", PIPELINE_OK, PIPELINE_OK);
  84 FFMPEG_TEST_CASE(Cr101458, "security/101458.webm", PIPELINE_OK, PIPELINE_OK);
  85 FFMPEG_TEST_CASE(Cr108416, "security/108416.webm", PIPELINE_OK, PIPELINE_OK);
  86 FFMPEG_TEST_CASE(Cr110849,
  87                  "security/110849.mkv",
  88                  DEMUXER_ERROR_COULD_NOT_OPEN,
  89                  DEMUXER_ERROR_NO_SUPPORTED_STREAMS);
  90 FFMPEG_TEST_CASE(Cr112384, "security/112384.webm", PIPELINE_OK, PIPELINE_OK);
  91 FFMPEG_TEST_CASE(Cr112976, "security/112976.ogg", PIPELINE_OK, PIPELINE_OK);
  92 FFMPEG_TEST_CASE(Cr116927,
  93                  "security/116927.ogv",
  94                  DEMUXER_ERROR_NO_SUPPORTED_STREAMS,
  95                  DEMUXER_ERROR_NO_SUPPORTED_STREAMS);
  96 FFMPEG_TEST_CASE(Cr117912,
  97                  "security/117912.webm",
  98                  DEMUXER_ERROR_COULD_NOT_OPEN,
  99                  DEMUXER_ERROR_COULD_NOT_OPEN);
 100 FFMPEG_TEST_CASE(Cr123481, "security/123481.ogv", PIPELINE_OK, PIPELINE_OK);
 101 FFMPEG_TEST_CASE(Cr132779,
 102                  "security/132779.webm",
 103                  DEMUXER_ERROR_COULD_NOT_PARSE,
 104                  DEMUXER_ERROR_COULD_NOT_PARSE);
 105 FFMPEG_TEST_CASE(Cr140165, "security/140165.ogg", PIPELINE_OK, PIPELINE_OK);
 106 FFMPEG_TEST_CASE(Cr140647,
 107                  "security/140647.ogv",
 108                  DEMUXER_ERROR_COULD_NOT_OPEN,
 109                  DEMUXER_ERROR_COULD_NOT_OPEN);
 110 FFMPEG_TEST_CASE(Cr142738, "crbug142738.ogg", PIPELINE_OK, PIPELINE_OK);
 111 FFMPEG_TEST_CASE(Cr152691,
 112                  "security/152691.mp3",
 113                  PIPELINE_OK,
 114                  PIPELINE_ERROR_DECODE);
 115 FFMPEG_TEST_CASE(Cr161639, "security/161639.m4a", PIPELINE_OK, PIPELINE_OK);
 116 FFMPEG_TEST_CASE(Cr222754,
 117                  "security/222754.mp4",
 118                  PIPELINE_OK,
 119                  PIPELINE_ERROR_DECODE);
 120 FFMPEG_TEST_CASE(Cr234630a, "security/234630a.mov", PIPELINE_OK, PIPELINE_OK);
 121 FFMPEG_TEST_CASE(Cr234630b,
 122                  "security/234630b.mov",
 123                  PIPELINE_OK,
 124                  PIPELINE_ERROR_DECODE);
 125 FFMPEG_TEST_CASE(Cr242786, "security/242786.webm", PIPELINE_OK, PIPELINE_OK);
 126 // Test for out-of-bounds access with slightly corrupt file (detection logic
 127 // thinks it's a MONO file, but actually contains STEREO audio).
 128 FFMPEG_TEST_CASE(Cr275590, "security/275590.m4a", PIPELINE_OK, PIPELINE_OK);
 129 FFMPEG_TEST_CASE(Cr444522,
 130                  "security/444522.mp4",
 131                  DEMUXER_ERROR_COULD_NOT_OPEN,
 132                  DEMUXER_ERROR_COULD_NOT_OPEN);
 133 FFMPEG_TEST_CASE(Cr444539,
 134                  "security/444539.m4a",
 135                  DEMUXER_ERROR_COULD_NOT_OPEN,
 136                  DEMUXER_ERROR_COULD_NOT_OPEN);
 137 FFMPEG_TEST_CASE(Cr444546,
 138                  "security/444546.mp4",
 139                  DEMUXER_ERROR_COULD_NOT_OPEN,
 140                  DEMUXER_ERROR_COULD_NOT_OPEN);
 141 FFMPEG_TEST_CASE(Cr449958,
 142                  "security/449958.webm",
 143                  PIPELINE_OK,
 144                  PIPELINE_ERROR_DECODE);
 145
 146 // General MP4 test cases.
 147 FFMPEG_TEST_CASE(MP4_0,
 148                  "security/aac.10419.mp4",
 149                  DEMUXER_ERROR_COULD_NOT_OPEN,
 150                  DEMUXER_ERROR_COULD_NOT_OPEN);
 151 FFMPEG_TEST_CASE(MP4_1,
 152                  "security/clockh264aac_200021889.mp4",
 153                  DEMUXER_ERROR_COULD_NOT_OPEN,
 154                  DEMUXER_ERROR_COULD_NOT_OPEN);
 155 FFMPEG_TEST_CASE(MP4_2,
 156                  "security/clockh264aac_200701257.mp4",
 157                  PIPELINE_OK,
 158                  PIPELINE_OK);
 159 FFMPEG_TEST_CASE(MP4_5,
 160                  "security/clockh264aac_3022500.mp4",
 161                  DEMUXER_ERROR_NO_SUPPORTED_STREAMS,
 162                  DEMUXER_ERROR_NO_SUPPORTED_STREAMS);
 163 FFMPEG_TEST_CASE(MP4_6,
 164                  "security/clockh264aac_344289.mp4",
 165                  PIPELINE_OK,
 166                  PIPELINE_OK);
 167 FFMPEG_TEST_CASE(MP4_7,
 168                  "security/clockh264mp3_187697.mp4",
 169                  PIPELINE_OK,
 170                  PIPELINE_OK);
 171 FFMPEG_TEST_CASE(MP4_8,
 172                  "security/h264.705767.mp4",
 173                  DEMUXER_ERROR_COULD_NOT_PARSE,
 174                  DEMUXER_ERROR_COULD_NOT_PARSE);
 175 FFMPEG_TEST_CASE(MP4_9,
 176                  "security/smclockmp4aac_1_0.mp4",
 177                  DEMUXER_ERROR_COULD_NOT_OPEN,
 178                  DEMUXER_ERROR_COULD_NOT_OPEN);
 179 FFMPEG_TEST_CASE(MP4_11, "security/null1.mp4", PIPELINE_OK, PIPELINE_OK);
 180 FFMPEG_TEST_CASE(MP4_16,
 181                  "security/looping2.mov",
 182                  DEMUXER_ERROR_COULD_NOT_OPEN,
 183                  DEMUXER_ERROR_COULD_NOT_OPEN);
 184 FFMPEG_TEST_CASE(MP4_17, "security/assert2.mov", PIPELINE_OK, PIPELINE_OK);
 185
 186 // General OGV test cases.
 187 FFMPEG_TEST_CASE(OGV_1,
 188                  "security/out.163.ogv",
 189                  DECODER_ERROR_NOT_SUPPORTED,
 190                  DECODER_ERROR_NOT_SUPPORTED);
 191 FFMPEG_TEST_CASE(OGV_2,
 192                  "security/out.391.ogv",
 193                  DECODER_ERROR_NOT_SUPPORTED,
 194                  DECODER_ERROR_NOT_SUPPORTED);
 195 FFMPEG_TEST_CASE(OGV_5,
 196                  "security/smclocktheora_1_0.ogv",
 197                  DECODER_ERROR_NOT_SUPPORTED,
 198                  DECODER_ERROR_NOT_SUPPORTED);
 199 FFMPEG_TEST_CASE(OGV_7,
 200                  "security/smclocktheora_1_102.ogv",
 201                  DECODER_ERROR_NOT_SUPPORTED,
 202                  DECODER_ERROR_NOT_SUPPORTED);
 203 FFMPEG_TEST_CASE(OGV_8,
 204                  "security/smclocktheora_1_104.ogv",
 205                  DECODER_ERROR_NOT_SUPPORTED,
 206                  DECODER_ERROR_NOT_SUPPORTED);
 207 FFMPEG_TEST_CASE(OGV_9,
 208                  "security/smclocktheora_1_110.ogv",
 209                  DECODER_ERROR_NOT_SUPPORTED,
 210                  DECODER_ERROR_NOT_SUPPORTED);
 211 FFMPEG_TEST_CASE(OGV_10,
 212                  "security/smclocktheora_1_179.ogv",
 213                  DECODER_ERROR_NOT_SUPPORTED,
 214                  DECODER_ERROR_NOT_SUPPORTED);
 215 FFMPEG_TEST_CASE(OGV_11,
 216                  "security/smclocktheora_1_20.ogv",
 217                  DECODER_ERROR_NOT_SUPPORTED,
 218                  DECODER_ERROR_NOT_SUPPORTED);
 219 FFMPEG_TEST_CASE(OGV_12,
 220                  "security/smclocktheora_1_723.ogv",
 221                  DECODER_ERROR_NOT_SUPPORTED,
 222                  DECODER_ERROR_NOT_SUPPORTED);
 223 FFMPEG_TEST_CASE(OGV_14,
 224                  "security/smclocktheora_2_10405.ogv",
 225                  PIPELINE_OK,
 226                  PIPELINE_OK);
 227 FFMPEG_TEST_CASE(OGV_15,
 228                  "security/smclocktheora_2_10619.ogv",
 229                  DECODER_ERROR_NOT_SUPPORTED,
 230                  DECODER_ERROR_NOT_SUPPORTED);
 231 FFMPEG_TEST_CASE(OGV_16,
 232                  "security/smclocktheora_2_1075.ogv",
 233                  DECODER_ERROR_NOT_SUPPORTED,
 234                  DECODER_ERROR_NOT_SUPPORTED);
 235 FFMPEG_TEST_CASE(OGV_17,
 236                  "security/vorbis.482086.ogv",
 237                  PIPELINE_OK,
 238                  PIPELINE_OK);
 239 FFMPEG_TEST_CASE(OGV_18,
 240                  "security/wav.711.ogv",
 241                  DECODER_ERROR_NOT_SUPPORTED,
 242                  DECODER_ERROR_NOT_SUPPORTED);
 243 FFMPEG_TEST_CASE(OGV_19,
 244                  "security/null1.ogv",
 245                  DECODER_ERROR_NOT_SUPPORTED,
 246                  DECODER_ERROR_NOT_SUPPORTED);
 247 FFMPEG_TEST_CASE(OGV_20,
 248                  "security/null2.ogv",
 249                  DECODER_ERROR_NOT_SUPPORTED,
 250                  DECODER_ERROR_NOT_SUPPORTED);
 251 FFMPEG_TEST_CASE(OGV_21,
 252                  "security/assert1.ogv",
 253                  DECODER_ERROR_NOT_SUPPORTED,
 254                  DECODER_ERROR_NOT_SUPPORTED);
 255 FFMPEG_TEST_CASE(OGV_22,
 256                  "security/assert2.ogv",
 257                  DECODER_ERROR_NOT_SUPPORTED,
 258                  DECODER_ERROR_NOT_SUPPORTED);
 259 FFMPEG_TEST_CASE(OGV_23,
 260                  "security/assert2.ogv",
 261                  DECODER_ERROR_NOT_SUPPORTED,
 262                  DECODER_ERROR_NOT_SUPPORTED);
 263
 264 // General WebM test cases.
 265 FFMPEG_TEST_CASE(WEBM_0, "security/memcpy.webm", PIPELINE_OK, PIPELINE_OK);
 266 FFMPEG_TEST_CASE(WEBM_1, "security/no-bug.webm", PIPELINE_OK, PIPELINE_OK);
 267 FFMPEG_TEST_CASE(WEBM_2,
 268                  "security/uninitialize.webm",
 269                  DEMUXER_ERROR_NO_SUPPORTED_STREAMS,
 270                  DEMUXER_ERROR_NO_SUPPORTED_STREAMS);
 271 FFMPEG_TEST_CASE(WEBM_4,
 272                  "security/out.webm.68798.1929",
 273                  DECODER_ERROR_NOT_SUPPORTED,
 274                  DECODER_ERROR_NOT_SUPPORTED);
 275 FFMPEG_TEST_CASE(WEBM_5, "frame_size_change.webm", PIPELINE_OK, PIPELINE_OK);
 276
 277 // General MKV test cases.
 278 FFMPEG_TEST_CASE(MKV_0,
 279                  "security/nested_tags_lang.mka.627.628",
 280                  PIPELINE_OK,
 281                  PIPELINE_ERROR_DECODE);
 282 FFMPEG_TEST_CASE(MKV_1,
 283                  "security/nested_tags_lang.mka.667.628",
 284                  PIPELINE_OK,
 285                  PIPELINE_ERROR_DECODE);
 286
 287 // Allocate gigabytes of memory, likely can't be run on 32bit machines.
 288 FFMPEG_TEST_CASE(BIG_MEM_1,
 289                  "security/bigmem1.mov",
 290                  DEMUXER_ERROR_COULD_NOT_OPEN,
 291                  DEMUXER_ERROR_COULD_NOT_OPEN);
 292 FFMPEG_TEST_CASE(BIG_MEM_2,
 293                  "security/looping1.mov",
 294                  DEMUXER_ERROR_COULD_NOT_OPEN,
 295                  DEMUXER_ERROR_COULD_NOT_OPEN);
 296 FFMPEG_TEST_CASE(BIG_MEM_5,
 297                  "security/looping5.mov",
 298                  DEMUXER_ERROR_COULD_NOT_OPEN,
 299                  DEMUXER_ERROR_COULD_NOT_OPEN);
 300 FLAKY_FFMPEG_TEST_CASE(BIG_MEM_3, "security/looping3.mov");
 301 FLAKY_FFMPEG_TEST_CASE(BIG_MEM_4, "security/looping4.mov");
 302
 303 // Flaky under threading or for other reasons.  Per rbultje, most of these will
 304 // never be reliable since FFmpeg does not guarantee consistency in error cases.
 305 // We only really care that these don't cause crashes or errors under tooling.
 306 FLAKY_FFMPEG_TEST_CASE(Cr99652, "security/99652.webm");
 307 FLAKY_FFMPEG_TEST_CASE(Cr100464, "security/100464.webm");
 308 FLAKY_FFMPEG_TEST_CASE(Cr111342, "security/111342.ogm");
 309 FLAKY_FFMPEG_TEST_CASE(Cr368980, "security/368980.mp4");
 310 FLAKY_FFMPEG_TEST_CASE(OGV_0, "security/big_dims.ogv");
 311 FLAKY_FFMPEG_TEST_CASE(OGV_3, "security/smclock_1_0.ogv");
 312 FLAKY_FFMPEG_TEST_CASE(OGV_4, "security/smclock.ogv.1.0.ogv");
 313 FLAKY_FFMPEG_TEST_CASE(OGV_6, "security/smclocktheora_1_10000.ogv");
 314 FLAKY_FFMPEG_TEST_CASE(OGV_13, "security/smclocktheora_1_790.ogv");
 315 FLAKY_FFMPEG_TEST_CASE(MP4_3, "security/clockh264aac_300413969.mp4");
 316 FLAKY_FFMPEG_TEST_CASE(MP4_4, "security/clockh264aac_301350139.mp4");
 317 FLAKY_FFMPEG_TEST_CASE(MP4_12, "security/assert1.mov");
 318 FLAKY_FFMPEG_TEST_CASE(WEBM_3, "security/out.webm.139771.2965");
 319
 320 // Not really flaky, but can't pass the seek test.
 321 FLAKY_FFMPEG_TEST_CASE(MP4_10, "security/null1.m4a");
 322 FLAKY_FFMPEG_TEST_CASE(Cr112670, "security/112670.mp4");
 323
 324 TEST_P(FFmpegRegressionTest, BasicPlayback) {
 325   if (GetParam().init_status == PIPELINE_OK) {
 326     ASSERT_EQ(PIPELINE_OK, Start(GetParam().filename, kClockless));
 327     Play();
 328     ASSERT_EQ(GetParam().end_status, WaitUntilEndedOrError());
 329
 330     // Check for ended if the pipeline is expected to finish okay.
 331     if (GetParam().end_status == PIPELINE_OK) {
 332       ASSERT_TRUE(ended_);
 333
 334       // Tack a seek on the end to catch any seeking issues.
 335       Seek(base::TimeDelta::FromMilliseconds(0));
 336     }
 337   } else {
 338     // Don't bother checking the exact status as we only care that the
 339     // pipeline failed to start.
 340     EXPECT_NE(PIPELINE_OK, Start(GetParam().filename));
 341   }
 342 }
 343
 344 TEST_P(FlakyFFmpegRegressionTest, BasicPlayback) {
 345   if (Start(GetParam().filename, kClockless) == PIPELINE_OK) {
 346     Play();
 347     WaitUntilEndedOrError();
 348   }
 349 }
 350
 351 }  // namespace media