src/ffmpeg.js

   1 var Logger = require("./logger");
   2 var Config = require("./config");
   3 var spawn = require("child_process").spawn;
   4 var https = require("https");
   5 var http = require("http");
   6 var urlparse = require("url");
   7 var path = require("path");
   8
   9 import { callOnce } from './util/call-once';
  10
  11 const CYTUBE_VERSION = require('../package.json').version;
  12
  13 const LOGGER = require('@calzoneman/jsli')('ffmpeg');
  14 const ECODE_MESSAGES = {
  15     ENOTFOUND: e => (
  16         `Unknown host "${e.hostname}".  ` +
  17         'Please check that the link is correct.'
  18     ),
  19     EPROTO: _e => 'The remote server does not support HTTPS.',
  20     ECONNRESET: _e => 'The remote server unexpectedly closed the connection.',
  21     ECONNREFUSED: _e => (
  22         'The remote server refused the connection.  ' +
  23         'Please check that the link is correct and the server is running.'
  24     ),
  25     ETIMEDOUT: _e => (
  26         'The connection to the remote server timed out.  ' +
  27         'Please check that the link is correct.'
  28     ),
  29     ENETUNREACH: _e => (
  30         "The remote server's network is unreachable from this server.  " +
  31         "Please contact an administrator for assistance."
  32     ),
  33     EHOSTUNREACH: _e => (
  34         "The remote server is unreachable from this server.  " +
  35         "Please contact the video server's administrator for assistance."
  36     ),
  37     ENOMEM: _e => (
  38         "An out of memory error caused the request to fail.  Please contact an " +
  39         "administrator for assistance."
  40     ),
  41
  42     DEPTH_ZERO_SELF_SIGNED_CERT: _e => (
  43         'The remote server provided an invalid ' +
  44         '(self-signed) SSL certificate.  Raw file support requires a ' +
  45         'trusted certificate.  See https://letsencrypt.org/ to get ' +
  46         'a free, trusted certificate.'
  47     ),
  48     SELF_SIGNED_CERT_IN_CHAIN: _e => (
  49         'The remote server provided an invalid ' +
  50         '(self-signed) SSL certificate.  Raw file support requires a ' +
  51         'trusted certificate.  See https://letsencrypt.org/ to get ' +
  52         'a free, trusted certificate.'
  53     ),
  54     UNABLE_TO_VERIFY_LEAF_SIGNATURE: _e => (
  55         "The remote server's SSL certificate chain could not be validated.  " +
  56         "Please contact the administrator of the server to correct their " +
  57         "SSL certificate configuration."
  58     ),
  59     CERT_HAS_EXPIRED: _e => (
  60         "The remote server's SSL certificate has expired.  Please contact " +
  61         "the administrator of the server to renew the certificate."
  62     ),
  63     ERR_TLS_CERT_ALTNAME_INVALID: _e => (
  64         "The remote server's SSL connection is misconfigured and has served " +
  65         "a certificate invalid for the given link."
  66     ),
  67
  68     // node's http parser barfs when careless servers ignore RFC 2616 and send a
  69     // response body in reply to a HEAD request
  70     HPE_INVALID_CONSTANT: _e => (
  71         "The remote server for this link is misconfigured."
  72     )
  73 };
  74
  75 var USE_JSON = true;
  76 var TIMEOUT = 30000;
  77
  78 var acceptedCodecs = {
  79     "mov/h264": true,
  80     "flv/h264": true,
  81     "matroska/vp8": true,
  82     "matroska/vp9": true,
  83     "ogg/theora": true,
  84     "mov/av1": true,
  85     "matroska/av1": true
  86 };
  87
  88 var acceptedAudioCodecs = {
  89     "mp3": true,
  90     "vorbis": true,
  91     "aac": true,
  92     "opus": true
  93 };
  94
  95 var audioOnlyContainers = {
  96     "mp3": true
  97 };
  98
  99 function fflog() { }
 100
 101 /* eslint no-func-assign: off */
 102 function initFFLog() {
 103     if (fflog.initialized) return;
 104     var logger = new Logger.Logger(path.resolve(__dirname, "..", "ffmpeg.log"));
 105     fflog = function () {
 106         logger.log.apply(logger, arguments);
 107     };
 108     fflog.initialized = true;
 109 }
 110
 111 function fixRedirectIfNeeded(urldata, redirect) {
 112     let parsedRedirect = urlparse.parse(redirect);
 113     if (parsedRedirect.host === null) {
 114         // Relative path, munge it to absolute
 115         redirect = urldata.protocol + "//" + urldata.host + redirect;
 116     }
 117
 118     return redirect;
 119 }
 120
 121 function translateStatusCode(statusCode) {
 122     switch (statusCode) {
 123         case 400:
 124             return "The request for the audio/video link was rejected as invalid.  " +
 125                 "Contact support for troubleshooting assistance.";
 126         case 401:
 127         case 403:
 128             return "Access to the link was denied.  Contact the owner of the " +
 129                 "website hosting the audio/video file to grant permission for " +
 130                 "the file to be downloaded.";
 131         case 404:
 132             return "The requested link could not be found (404).";
 133         case 405:
 134             return "The website hosting the link does not support HEAD requests, " +
 135                    "so the link could not be retrieved.";
 136         case 410:
 137             return "The requested link does not exist (410 Gone).";
 138         case 501:
 139             return "The requested link could not be retrieved because the server " +
 140                    "hosting it does not support CyTube's request.";
 141         case 500:
 142         case 503:
 143             return "The website hosting the audio/video link encountered an error " +
 144                 "and was unable to process the request.  Try again in a few minutes, " +
 145                 "and if the issue persists, contact the owner of the website hosting " +
 146                 "the link.";
 147         default:
 148             return "An unknown issue occurred when requesting the audio/video link.  " +
 149                 "Contact support for troubleshooting assistance.";
 150     }
 151 }
 152
 153 function getCookie(res) {
 154     if (!res.headers['set-cookie']) {
 155         return '';
 156     }
 157
 158     return res.headers['set-cookie'].map(c => c.split(';')[0]).join(';') + ';';
 159 }
 160
 161 function testUrl(url, cb, params = { redirCount: 0, cookie: '' }) {
 162     const { redirCount, cookie } = params;
 163     var data = urlparse.parse(url);
 164     if (!/https:/.test(data.protocol)) {
 165         if (redirCount > 0) {
 166             // If the original URL redirected, the user is probably not aware
 167             // that the link they entered (which was HTTPS) is redirecting to a
 168             // non-HTTPS endpoint
 169             return cb(`Unexpected redirect to a non-HTTPS link: ${url}`);
 170         }
 171
 172         return cb("Only links starting with 'https://' are supported " +
 173                   "for raw audio/video support");
 174     }
 175
 176     if (!data.hostname) {
 177         return cb("The link to the file is missing the website address and can't " +
 178                   "be processed.");
 179     }
 180
 181     var transport = (data.protocol === "https:") ? https : http;
 182     data.method = "HEAD";
 183     data.headers = {
 184         'User-Agent': `CyTube/${CYTUBE_VERSION}`
 185     };
 186     if (cookie) {
 187         data.headers['Cookie'] = cookie;
 188     }
 189
 190     try {
 191         var req = transport.request(data, function (res) {
 192             req.abort();
 193
 194             if (res.statusCode === 301 || res.statusCode === 302) {
 195                 if (redirCount > 2) {
 196                     return cb("The request for the audio/video file has been redirected " +
 197                               "more than twice.  This could indicate a misconfiguration " +
 198                               "on the website hosting the link.  For best results, use " +
 199                               "a direct link.  See https://git.io/vrE75 for details.");
 200                 }
 201
 202                 const nextParams = {
 203                     redirCount: redirCount + 1,
 204                     cookie: cookie + getCookie(res)
 205                 };
 206                 return testUrl(fixRedirectIfNeeded(data, res.headers["location"]), cb,
 207                         nextParams);
 208             }
 209
 210             if (res.statusCode !== 200) {
 211                 return cb(translateStatusCode(res.statusCode));
 212             }
 213
 214             if (!/^audio|^video/.test(res.headers["content-type"])) {
 215                 cb("Could not detect a supported audio/video type.  See " +
 216                    "https://git.io/fjtOK for a list of supported providers.  " +
 217                    "(Content-Type was: '" + res.headers["content-type"] + "')");
 218                 return;
 219             }
 220
 221             cb();
 222         });
 223
 224         req.on("error", function (err) {
 225             if (/hostname\/ip doesn't match/i.test(err.message)) {
 226                 cb("The remote server provided an invalid SSL certificate.  Details: "
 227                         + err.reason);
 228                 return;
 229             } else if (ECODE_MESSAGES.hasOwnProperty(err.code)) {
 230                 cb(`${ECODE_MESSAGES[err.code](err)} (error code: ${err.code})`);
 231                 return;
 232             }
 233
 234             LOGGER.error(
 235                 "Error sending preflight request: %s (code=%s) (link: %s)",
 236                 err.message,
 237                 err.code,
 238                 url
 239             );
 240
 241             cb("An unexpected error occurred while trying to process the link.  " +
 242                "If this link is hosted on a server you own, it is likely " +
 243                "misconfigured and you can join community support for assistance.  " +
 244                "If you are attempting to add links from third party websites, the " +
 245                "developers do not provide support for this." +
 246                (err.code ? (" Error code: " + err.code) : ""));
 247         });
 248
 249         req.end();
 250     } catch (error) {
 251         LOGGER.error('Unable to make raw file probe request: %s', error.stack);
 252         cb("An unexpected error occurred while trying to process the link.  " +
 253            "Try again, and contact support for further troubleshooting if the " +
 254            "problem continues.");
 255     }
 256 }
 257
 258 function readOldFormat(buf) {
 259     var lines = buf.split("\n");
 260     var tmp = { tags: {} };
 261     var data = {
 262         streams: []
 263     };
 264
 265     lines.forEach(function (line) {
 266         if (line.match(/\[stream\]|\[format\]/i)) {
 267             return;
 268         } else if (line.match(/\[\/stream\]/i)) {
 269             data.streams.push(tmp);
 270             tmp = { tags: {} };
 271         } else if (line.match(/\[\/format\]/i)) {
 272             data.format = tmp;
 273             tmp = { tags: {} };
 274         } else {
 275             var kv = line.split("=");
 276             var key = kv[0].toLowerCase();
 277             if (key.indexOf("tag:") === 0) {
 278                 tmp.tags[key.split(":")[1]] = kv[1];
 279             } else {
 280                 tmp[key] = kv[1];
 281             }
 282         }
 283     });
 284
 285     return data;
 286 }
 287
 288 function isAlternateDisposition(stream) {
 289     if (!stream.disposition) {
 290         return false;
 291     }
 292
 293     for (var key in stream) {
 294         if (key !== "default" && stream.disposition[key]) {
 295             return true;
 296         }
 297     }
 298
 299     return false;
 300 }
 301
 302 function reformatData(data) {
 303     var reformatted = {};
 304
 305     var duration = parseInt(data.format.duration, 10);
 306     if (isNaN(duration)) duration = "--:--";
 307     reformatted.duration = Math.ceil(duration);
 308
 309     var bitrate = parseInt(data.format.bit_rate, 10) / 1000;
 310     if (isNaN(bitrate)) bitrate = 0;
 311     reformatted.bitrate = bitrate;
 312
 313     reformatted.title = data.format.tags ? data.format.tags.title : null;
 314     var container = data.format.format_name.split(",")[0];
 315
 316     var isVideo = false;
 317     var audio = null;
 318     for (var i = 0; i < data.streams.length; i++) {
 319         const stream = data.streams[i];
 320
 321         // Trash streams with alternate dispositions, e.g. `attached_pic` for
 322         // embedded album art on MP3s (not a real video stream)
 323         if (isAlternateDisposition(stream)) {
 324             continue;
 325         }
 326
 327         if (stream.codec_type === "video" &&
 328                 !audioOnlyContainers.hasOwnProperty(container)) {
 329             isVideo = true;
 330             if (acceptedCodecs.hasOwnProperty(container + "/" + stream.codec_name)) {
 331                 reformatted.vcodec = stream.codec_name;
 332                 reformatted.medium = "video";
 333                 reformatted.type = [container, reformatted.vcodec].join("/");
 334
 335                 if (stream.tags && stream.tags.title) {
 336                     reformatted.title = stream.tags.title;
 337                 }
 338
 339                 return reformatted;
 340             }
 341         } else if (stream.codec_type === "audio" && !audio &&
 342                 acceptedAudioCodecs.hasOwnProperty(stream.codec_name)) {
 343             audio = {
 344                 acodec: stream.codec_name,
 345                 medium: "audio"
 346             };
 347
 348             if (stream.tags && stream.tags.title) {
 349                 audio.title = stream.tags.title;
 350             }
 351         }
 352     }
 353
 354     // Override to make sure video files with no valid video streams but some
 355     // acceptable audio stream are rejected.
 356     if (isVideo) {
 357         return reformatted;
 358     }
 359
 360     if (audio) {
 361         for (var key in audio) {
 362             reformatted[key] = audio[key];
 363         }
 364     }
 365
 366     return reformatted;
 367 }
 368
 369 exports.ffprobe = function ffprobe(filename, cb) {
 370     fflog("Spawning ffprobe for " + filename);
 371     var childErr;
 372     var args = ["-show_streams", "-show_format", filename];
 373     if (USE_JSON) args = ["-of", "json"].concat(args);
 374     let child;
 375     try {
 376         child = spawn(Config.get("ffmpeg.ffprobe-exec"), args);
 377     } catch (error) {
 378         LOGGER.error("Unable to spawn() ffprobe process: %s", error.stack);
 379         cb(error);
 380         return;
 381     }
 382     var stdout = "";
 383     var stderr = "";
 384     var timer = setTimeout(function () {
 385         LOGGER.warn("Timed out when probing " + filename);
 386         fflog("Killing ffprobe for " + filename + " after " + (TIMEOUT/1000) + " seconds");
 387         childErr = new Error(
 388                 "File query exceeded time limit of " + (TIMEOUT/1000) +
 389                 " seconds.  This can be caused if the remote server is far " +
 390                 "away or if you did not encode the video " +
 391                 "using the 'faststart' option: " +
 392                 "https://trac.ffmpeg.org/wiki/Encode/H.264#faststartforwebvideo"
 393         );
 394         child.kill("SIGKILL");
 395     }, TIMEOUT);
 396
 397     child.on("error", function (err) {
 398         childErr = err;
 399     });
 400
 401     child.stdout.on("data", function (data) {
 402         stdout += data;
 403     });
 404
 405     child.stderr.on("data", function (data) {
 406         stderr += data;
 407         if (stderr.match(/the tls connection was non-properly terminated/i)) {
 408             fflog("Killing ffprobe for " + filename + " due to TLS error");
 409             childErr = new Error("The connection was closed unexpectedly.  " +
 410                                  "If the problem continues, contact support " +
 411                                  "for troubleshooting assistance.");
 412             child.kill("SIGKILL");
 413         }
 414     });
 415
 416     child.on("close", function (code) {
 417         clearTimeout(timer);
 418         fflog("ffprobe exited with code " + code + " for file " + filename);
 419         if (code !== 0) {
 420             if (stderr.match(/unrecognized option|json/i) && USE_JSON) {
 421                 LOGGER.warn("ffprobe does not support -of json.  " +
 422                                   "Assuming it will have old output format.");
 423                 USE_JSON = false;
 424                 return ffprobe(filename, cb);
 425             }
 426
 427             if (!childErr) childErr = new Error(stderr);
 428             return cb(childErr);
 429         }
 430
 431         var result;
 432         if (USE_JSON) {
 433             try {
 434                 result = JSON.parse(stdout);
 435             } catch (e) {
 436                 return cb(new Error("Unable to parse ffprobe output: " + e.message));
 437             }
 438         } else {
 439             try {
 440                 result = readOldFormat(stdout);
 441             } catch (e) {
 442                 return cb(new Error("Unable to parse ffprobe output: " + e.message));
 443             }
 444         }
 445
 446         return cb(null, result);
 447     });
 448 };
 449
 450 exports.query = function (filename, cb) {
 451     if (Config.get("ffmpeg.log") && !fflog.initialized) {
 452         initFFLog();
 453     }
 454
 455     if (!Config.get("ffmpeg.enabled")) {
 456         return cb("Raw file playback is not enabled on this server");
 457     }
 458
 459     if (!filename.match(/^https:\/\//)) {
 460         return cb("Raw file playback is only supported for links accessible via HTTPS. " +
 461                   "Ensure that the link begins with 'https://'.");
 462     }
 463
 464     testUrl(filename, callOnce(function (err) {
 465         if (err) {
 466             return cb(err);
 467         }
 468
 469         exports.ffprobe(filename, function (err, data) {
 470             if (err) {
 471                 if (err.code && err.code === "ENOENT") {
 472                     return cb("Failed to execute `ffprobe`.  Set ffmpeg.ffprobe-exec " +
 473                               "to the correct name of the executable in config.yaml.  " +
 474                               "If you are using Debian or Ubuntu, it is probably " +
 475                               "avprobe.");
 476                 } else if (err.message) {
 477                     if (err.message.match(/protocol not found/i))
 478                         return cb("Link uses a protocol unsupported by this server's " +
 479                                   "version of ffmpeg.  Some older versions of " +
 480                                   "ffprobe/avprobe do not support HTTPS.");
 481
 482                     if (err.message.match(/exceeded time limit/) ||
 483                         err.message.match(/closed unexpectedly/i)) {
 484                         return cb(err.message);
 485                     }
 486
 487                     // Ignore ffprobe error messages, they are common and most often
 488                     // indicate a problem with the remote file, not with this code.
 489                     if (!/(av|ff)probe/.test(String(err)))
 490                         LOGGER.error(err.stack || err);
 491                     return cb("An unexpected error occurred while trying to process " +
 492                               "the link.  Contact support for troubleshooting " +
 493                               "assistance.");
 494                 } else {
 495                     if (!/(av|ff)probe/.test(String(err)))
 496                         LOGGER.error(err.stack || err);
 497                     return cb("An unexpected error occurred while trying to process " +
 498                               "the link.  Contact support for troubleshooting " +
 499                               "assistance.");
 500                 }
 501             }
 502
 503             try {
 504                 data = reformatData(data);
 505             } catch (e) {
 506                 LOGGER.error(e.stack || e);
 507                 return cb("An unexpected error occurred while trying to process " +
 508                           "the link.  Contact support for troubleshooting " +
 509                           "assistance.");
 510             }
 511
 512             if (data.medium === "video") {
 513                 data = {
 514                     title: data.title || "Raw Video",
 515                     duration: data.duration,
 516                     bitrate: data.bitrate,
 517                     codec: data.type
 518                 };
 519
 520                 cb(null, data);
 521             } else if (data.medium === "audio") {
 522                 data = {
 523                     title: data.title || "Raw Audio",
 524                     duration: data.duration,
 525                     bitrate: data.bitrate,
 526                     codec: data.acodec
 527                 };
 528
 529                 cb(null, data);
 530             } else {
 531                 return cb("File did not contain an acceptable codec.  See " +
 532                           "https://git.io/vrE75 for details.");
 533             }
 534         });
 535     }));
 536 };