src/ffmpeg.js

   1 var Logger = require("./logger");
   2 var Config = require("./config");
   3 var spawn = require("child_process").spawn;
   4 var https = require("https");
   5 var http = require("http");
   6 var urlparse = require("url");
   7 var path = require("path");
   8
   9 import { callOnce } from './util/call-once';
  10
  11 const CYTUBE_VERSION = require('../package.json').version;
  12
  13 const LOGGER = require('@calzoneman/jsli')('ffmpeg');
  14 const ECODE_MESSAGES = {
  15     ENOTFOUND: e => (
  16         `Unknown host "${e.hostname}".  ` +
  17         'Please check that the link is correct.'
  18     ),
  19     EPROTO: _e => 'The remote server does not support HTTPS.',
  20     ECONNRESET: _e => 'The remote server unexpectedly closed the connection.',
  21     ECONNREFUSED: _e => (
  22         'The remote server refused the connection.  ' +
  23         'Please check that the link is correct and the server is running.'
  24     ),
  25     ETIMEDOUT: _e => (
  26         'The connection to the remote server timed out.  ' +
  27         'Please check that the link is correct.'
  28     ),
  29     ENETUNREACH: _e => (
  30         "The remote server's network is unreachable from this server.  " +
  31         "Please contact an administrator for assistance."
  32     ),
  33     EHOSTUNREACH: _e => (
  34         "The remote server is unreachable from this server.  " +
  35         "Please contact the video server's administrator for assistance."
  36     ),
  37     ENOMEM: _e => (
  38         "An out of memory error caused the request to fail.  Please contact an " +
  39         "administrator for assistance."
  40     ),
  41
  42     DEPTH_ZERO_SELF_SIGNED_CERT: _e => (
  43         'The remote server provided an invalid ' +
  44         '(self-signed) SSL certificate.  Raw file support requires a ' +
  45         'trusted certificate.  See https://letsencrypt.org/ to get ' +
  46         'a free, trusted certificate.'
  47     ),
  48     SELF_SIGNED_CERT_IN_CHAIN: _e => (
  49         'The remote server provided an invalid ' +
  50         '(self-signed) SSL certificate.  Raw file support requires a ' +
  51         'trusted certificate.  See https://letsencrypt.org/ to get ' +
  52         'a free, trusted certificate.'
  53     ),
  54     UNABLE_TO_VERIFY_LEAF_SIGNATURE: _e => (
  55         "The remote server's SSL certificate chain could not be validated.  " +
  56         "Please contact the administrator of the server to correct their " +
  57         "SSL certificate configuration."
  58     ),
  59     CERT_HAS_EXPIRED: _e => (
  60         "The remote server's SSL certificate has expired.  Please contact " +
  61         "the administrator of the server to renew the certificate."
  62     ),
  63     ERR_TLS_CERT_ALTNAME_INVALID: _e => (
  64         "The remote server's SSL connection is misconfigured and has served " +
  65         "a certificate invalid for the given link."
  66     ),
  67
  68     // node's http parser barfs when careless servers ignore RFC 2616 and send a
  69     // response body in reply to a HEAD request
  70     HPE_INVALID_CONSTANT: _e => (
  71         "The remote server for this link is misconfigured."
  72     )
  73 };
  74
  75 var USE_JSON = true;
  76 var TIMEOUT = 30000;
  77
  78 var acceptedCodecs = {
  79     "mov/h264": true,
  80     "flv/h264": true,
  81     "matroska/vp8": true,
  82     "matroska/vp9": true,
  83     "ogg/theora": true
  84 };
  85
  86 var acceptedAudioCodecs = {
  87     "mp3": true,
  88     "vorbis": true,
  89     "aac": true
  90 };
  91
  92 var audioOnlyContainers = {
  93     "mp3": true
  94 };
  95
  96 function fflog() { }
  97
  98 /* eslint no-func-assign: off */
  99 function initFFLog() {
 100     if (fflog.initialized) return;
 101     var logger = new Logger.Logger(path.resolve(__dirname, "..", "ffmpeg.log"));
 102     fflog = function () {
 103         logger.log.apply(logger, arguments);
 104     };
 105     fflog.initialized = true;
 106 }
 107
 108 function fixRedirectIfNeeded(urldata, redirect) {
 109     let parsedRedirect = urlparse.parse(redirect);
 110     if (parsedRedirect.host === null) {
 111         // Relative path, munge it to absolute
 112         redirect = urldata.protocol + "//" + urldata.host + redirect;
 113     }
 114
 115     return redirect;
 116 }
 117
 118 function translateStatusCode(statusCode) {
 119     switch (statusCode) {
 120         case 400:
 121             return "The request for the audio/video link was rejected as invalid.  " +
 122                 "Contact support for troubleshooting assistance.";
 123         case 401:
 124         case 403:
 125             return "Access to the link was denied.  Contact the owner of the " +
 126                 "website hosting the audio/video file to grant permission for " +
 127                 "the file to be downloaded.";
 128         case 404:
 129             return "The requested link could not be found (404).";
 130         case 405:
 131             return "The website hosting the link does not support HEAD requests, " +
 132                    "so the link could not be retrieved.";
 133         case 410:
 134             return "The requested link does not exist (410 Gone).";
 135         case 501:
 136             return "The requested link could not be retrieved because the server " +
 137                    "hosting it does not support CyTube's request.";
 138         case 500:
 139         case 503:
 140             return "The website hosting the audio/video link encountered an error " +
 141                 "and was unable to process the request.  Try again in a few minutes, " +
 142                 "and if the issue persists, contact the owner of the website hosting " +
 143                 "the link.";
 144         default:
 145             return "An unknown issue occurred when requesting the audio/video link.  " +
 146                 "Contact support for troubleshooting assistance.";
 147     }
 148 }
 149
 150 function getCookie(res) {
 151     if (!res.headers['set-cookie']) {
 152         return '';
 153     }
 154
 155     return res.headers['set-cookie'].map(c => c.split(';')[0]).join(';') + ';';
 156 }
 157
 158 function testUrl(url, cb, params = { redirCount: 0, cookie: '' }) {
 159     const { redirCount, cookie } = params;
 160     var data = urlparse.parse(url);
 161     if (!/https:/.test(data.protocol)) {
 162         if (redirCount > 0) {
 163             // If the original URL redirected, the user is probably not aware
 164             // that the link they entered (which was HTTPS) is redirecting to a
 165             // non-HTTPS endpoint
 166             return cb(`Unexpected redirect to a non-HTTPS link: ${url}`);
 167         }
 168
 169         return cb("Only links starting with 'https://' are supported " +
 170                   "for raw audio/video support");
 171     }
 172
 173     if (!data.hostname) {
 174         return cb("The link to the file is missing the website address and can't " +
 175                   "be processed.");
 176     }
 177
 178     var transport = (data.protocol === "https:") ? https : http;
 179     data.method = "HEAD";
 180     data.headers = {
 181         'User-Agent': `CyTube/${CYTUBE_VERSION}`
 182     };
 183     if (cookie) {
 184         data.headers['Cookie'] = cookie;
 185     }
 186
 187     try {
 188         var req = transport.request(data, function (res) {
 189             req.abort();
 190
 191             if (res.statusCode === 301 || res.statusCode === 302) {
 192                 if (redirCount > 2) {
 193                     return cb("The request for the audio/video file has been redirected " +
 194                               "more than twice.  This could indicate a misconfiguration " +
 195                               "on the website hosting the link.  For best results, use " +
 196                               "a direct link.  See https://git.io/vrE75 for details.");
 197                 }
 198
 199                 const nextParams = {
 200                     redirCount: redirCount + 1,
 201                     cookie: cookie + getCookie(res)
 202                 };
 203                 return testUrl(fixRedirectIfNeeded(data, res.headers["location"]), cb,
 204                         nextParams);
 205             }
 206
 207             if (res.statusCode !== 200) {
 208                 return cb(translateStatusCode(res.statusCode));
 209             }
 210
 211             if (!/^audio|^video/.test(res.headers["content-type"])) {
 212                 cb("Could not detect a supported audio/video type.  See " +
 213                    "https://git.io/fjtOK for a list of supported providers.  " +
 214                    "(Content-Type was: '" + res.headers["content-type"] + "')");
 215                 return;
 216             }
 217
 218             cb();
 219         });
 220
 221         req.on("error", function (err) {
 222             if (/hostname\/ip doesn't match/i.test(err.message)) {
 223                 cb("The remote server provided an invalid SSL certificate.  Details: "
 224                         + err.reason);
 225                 return;
 226             } else if (ECODE_MESSAGES.hasOwnProperty(err.code)) {
 227                 cb(`${ECODE_MESSAGES[err.code](err)} (error code: ${err.code})`);
 228                 return;
 229             }
 230
 231             LOGGER.error(
 232                 "Error sending preflight request: %s (code=%s) (link: %s)",
 233                 err.message,
 234                 err.code,
 235                 url
 236             );
 237
 238             cb("An unexpected error occurred while trying to process the link.  " +
 239                "If this link is hosted on a server you own, it is likely " +
 240                "misconfigured and you can join community support for assistance.  " +
 241                "If you are attempting to add links from third party websites, the " +
 242                "developers do not provide support for this." +
 243                (err.code ? (" Error code: " + err.code) : ""));
 244         });
 245
 246         req.end();
 247     } catch (error) {
 248         LOGGER.error('Unable to make raw file probe request: %s', error.stack);
 249         cb("An unexpected error occurred while trying to process the link.  " +
 250            "Try again, and contact support for further troubleshooting if the " +
 251            "problem continues.");
 252     }
 253 }
 254
 255 function readOldFormat(buf) {
 256     var lines = buf.split("\n");
 257     var tmp = { tags: {} };
 258     var data = {
 259         streams: []
 260     };
 261
 262     lines.forEach(function (line) {
 263         if (line.match(/\[stream\]|\[format\]/i)) {
 264             return;
 265         } else if (line.match(/\[\/stream\]/i)) {
 266             data.streams.push(tmp);
 267             tmp = { tags: {} };
 268         } else if (line.match(/\[\/format\]/i)) {
 269             data.format = tmp;
 270             tmp = { tags: {} };
 271         } else {
 272             var kv = line.split("=");
 273             var key = kv[0].toLowerCase();
 274             if (key.indexOf("tag:") === 0) {
 275                 tmp.tags[key.split(":")[1]] = kv[1];
 276             } else {
 277                 tmp[key] = kv[1];
 278             }
 279         }
 280     });
 281
 282     return data;
 283 }
 284
 285 function isAlternateDisposition(stream) {
 286     if (!stream.disposition) {
 287         return false;
 288     }
 289
 290     for (var key in stream) {
 291         if (key !== "default" && stream.disposition[key]) {
 292             return true;
 293         }
 294     }
 295
 296     return false;
 297 }
 298
 299 function reformatData(data) {
 300     var reformatted = {};
 301
 302     var duration = parseInt(data.format.duration, 10);
 303     if (isNaN(duration)) duration = "--:--";
 304     reformatted.duration = Math.ceil(duration);
 305
 306     var bitrate = parseInt(data.format.bit_rate, 10) / 1000;
 307     if (isNaN(bitrate)) bitrate = 0;
 308     reformatted.bitrate = bitrate;
 309
 310     reformatted.title = data.format.tags ? data.format.tags.title : null;
 311     var container = data.format.format_name.split(",")[0];
 312
 313     var isVideo = false;
 314     var audio = null;
 315     for (var i = 0; i < data.streams.length; i++) {
 316         const stream = data.streams[i];
 317
 318         // Trash streams with alternate dispositions, e.g. `attached_pic` for
 319         // embedded album art on MP3s (not a real video stream)
 320         if (isAlternateDisposition(stream)) {
 321             continue;
 322         }
 323
 324         if (stream.codec_type === "video" &&
 325                 !audioOnlyContainers.hasOwnProperty(container)) {
 326             isVideo = true;
 327             if (acceptedCodecs.hasOwnProperty(container + "/" + stream.codec_name)) {
 328                 reformatted.vcodec = stream.codec_name;
 329                 reformatted.medium = "video";
 330                 reformatted.type = [container, reformatted.vcodec].join("/");
 331
 332                 if (stream.tags && stream.tags.title) {
 333                     reformatted.title = stream.tags.title;
 334                 }
 335
 336                 return reformatted;
 337             }
 338         } else if (stream.codec_type === "audio" && !audio &&
 339                 acceptedAudioCodecs.hasOwnProperty(stream.codec_name)) {
 340             audio = {
 341                 acodec: stream.codec_name,
 342                 medium: "audio"
 343             };
 344
 345             if (stream.tags && stream.tags.title) {
 346                 audio.title = stream.tags.title;
 347             }
 348         }
 349     }
 350
 351     // Override to make sure video files with no valid video streams but some
 352     // acceptable audio stream are rejected.
 353     if (isVideo) {
 354         return reformatted;
 355     }
 356
 357     if (audio) {
 358         for (var key in audio) {
 359             reformatted[key] = audio[key];
 360         }
 361     }
 362
 363     return reformatted;
 364 }
 365
 366 exports.ffprobe = function ffprobe(filename, cb) {
 367     fflog("Spawning ffprobe for " + filename);
 368     var childErr;
 369     var args = ["-show_streams", "-show_format", filename];
 370     if (USE_JSON) args = ["-of", "json"].concat(args);
 371     let child;
 372     try {
 373         child = spawn(Config.get("ffmpeg.ffprobe-exec"), args);
 374     } catch (error) {
 375         LOGGER.error("Unable to spawn() ffprobe process: %s", error.stack);
 376         cb(error);
 377         return;
 378     }
 379     var stdout = "";
 380     var stderr = "";
 381     var timer = setTimeout(function () {
 382         LOGGER.warn("Timed out when probing " + filename);
 383         fflog("Killing ffprobe for " + filename + " after " + (TIMEOUT/1000) + " seconds");
 384         childErr = new Error(
 385                 "File query exceeded time limit of " + (TIMEOUT/1000) +
 386                 " seconds.  This can be caused if the remote server is far " +
 387                 "away or if you did not encode the video " +
 388                 "using the 'faststart' option: " +
 389                 "https://trac.ffmpeg.org/wiki/Encode/H.264#faststartforwebvideo"
 390         );
 391         child.kill("SIGKILL");
 392     }, TIMEOUT);
 393
 394     child.on("error", function (err) {
 395         childErr = err;
 396     });
 397
 398     child.stdout.on("data", function (data) {
 399         stdout += data;
 400     });
 401
 402     child.stderr.on("data", function (data) {
 403         stderr += data;
 404         if (stderr.match(/the tls connection was non-properly terminated/i)) {
 405             fflog("Killing ffprobe for " + filename + " due to TLS error");
 406             childErr = new Error("The connection was closed unexpectedly.  " +
 407                                  "If the problem continues, contact support " +
 408                                  "for troubleshooting assistance.");
 409             child.kill("SIGKILL");
 410         }
 411     });
 412
 413     child.on("close", function (code) {
 414         clearTimeout(timer);
 415         fflog("ffprobe exited with code " + code + " for file " + filename);
 416         if (code !== 0) {
 417             if (stderr.match(/unrecognized option|json/i) && USE_JSON) {
 418                 LOGGER.warn("ffprobe does not support -of json.  " +
 419                                   "Assuming it will have old output format.");
 420                 USE_JSON = false;
 421                 return ffprobe(filename, cb);
 422             }
 423
 424             if (!childErr) childErr = new Error(stderr);
 425             return cb(childErr);
 426         }
 427
 428         var result;
 429         if (USE_JSON) {
 430             try {
 431                 result = JSON.parse(stdout);
 432             } catch (e) {
 433                 return cb(new Error("Unable to parse ffprobe output: " + e.message));
 434             }
 435         } else {
 436             try {
 437                 result = readOldFormat(stdout);
 438             } catch (e) {
 439                 return cb(new Error("Unable to parse ffprobe output: " + e.message));
 440             }
 441         }
 442
 443         return cb(null, result);
 444     });
 445 };
 446
 447 exports.query = function (filename, cb) {
 448     if (Config.get("ffmpeg.log") && !fflog.initialized) {
 449         initFFLog();
 450     }
 451
 452     if (!Config.get("ffmpeg.enabled")) {
 453         return cb("Raw file playback is not enabled on this server");
 454     }
 455
 456     if (!filename.match(/^https:\/\//)) {
 457         return cb("Raw file playback is only supported for links accessible via HTTPS. " +
 458                   "Ensure that the link begins with 'https://'.");
 459     }
 460
 461     testUrl(filename, callOnce(function (err) {
 462         if (err) {
 463             return cb(err);
 464         }
 465
 466         exports.ffprobe(filename, function (err, data) {
 467             if (err) {
 468                 if (err.code && err.code === "ENOENT") {
 469                     return cb("Failed to execute `ffprobe`.  Set ffmpeg.ffprobe-exec " +
 470                               "to the correct name of the executable in config.yaml.  " +
 471                               "If you are using Debian or Ubuntu, it is probably " +
 472                               "avprobe.");
 473                 } else if (err.message) {
 474                     if (err.message.match(/protocol not found/i))
 475                         return cb("Link uses a protocol unsupported by this server's " +
 476                                   "version of ffmpeg.  Some older versions of " +
 477                                   "ffprobe/avprobe do not support HTTPS.");
 478
 479                     if (err.message.match(/exceeded time limit/) ||
 480                         err.message.match(/closed unexpectedly/i)) {
 481                         return cb(err.message);
 482                     }
 483
 484                     // Ignore ffprobe error messages, they are common and most often
 485                     // indicate a problem with the remote file, not with this code.
 486                     if (!/(av|ff)probe/.test(String(err)))
 487                         LOGGER.error(err.stack || err);
 488                     return cb("An unexpected error occurred while trying to process " +
 489                               "the link.  Contact support for troubleshooting " +
 490                               "assistance.");
 491                 } else {
 492                     if (!/(av|ff)probe/.test(String(err)))
 493                         LOGGER.error(err.stack || err);
 494                     return cb("An unexpected error occurred while trying to process " +
 495                               "the link.  Contact support for troubleshooting " +
 496                               "assistance.");
 497                 }
 498             }
 499
 500             try {
 501                 data = reformatData(data);
 502             } catch (e) {
 503                 LOGGER.error(e.stack || e);
 504                 return cb("An unexpected error occurred while trying to process " +
 505                           "the link.  Contact support for troubleshooting " +
 506                           "assistance.");
 507             }
 508
 509             if (data.medium === "video") {
 510                 data = {
 511                     title: data.title || "Raw Video",
 512                     duration: data.duration,
 513                     bitrate: data.bitrate,
 514                     codec: data.type
 515                 };
 516
 517                 cb(null, data);
 518             } else if (data.medium === "audio") {
 519                 data = {
 520                     title: data.title || "Raw Audio",
 521                     duration: data.duration,
 522                     bitrate: data.bitrate,
 523                     codec: data.acodec
 524                 };
 525
 526                 cb(null, data);
 527             } else {
 528                 return cb("File did not contain an acceptable codec.  See " +
 529                           "https://git.io/vrE75 for details.");
 530             }
 531         });
 532     }));
 533 };