Add testing code
[youtube-dl.el.git] / youtube-dl.el
blobfdeda91678287b3e2bdee0f6eab52a3374cf25a7
1 ;;; youtube-dl.el --- manages a youtube-dl queue -*- lexical-binding: t; -*-
3 ;; This is free and unencumbered software released into the public domain.
5 ;; Author: Christopher Wellons <wellons@nullprogram.com>
6 ;; URL: https://github.com/skeeto/youtube-dl-emacs
7 ;; Version: 1.0
8 ;; Package-Requires: ((emacs "27.1"))
10 ;;; Commentary:
12 ;; This package manages a video download queue for the youtube-dl
13 ;; command line program, which serves as its back end. It manages a
14 ;; single youtube-dl subprocess to download one video at a time. New
15 ;; videos can be queued at any time.
17 ;; The `youtube-dl' command queues a URL for download. Failures are
18 ;; retried up to `youtube-dl-max-failures'. Items can be paused or set
19 ;; to be downloaded at a slower download rate (`youtube-dl-slow-rate-limit').
21 ;; The `youtube-dl-download-playlist' command queues an entire playlist, just
22 ;; as if you had individually queued each video on the playlist.
24 ;; The `youtube-dl-list' command displays a list of all active video
25 ;; downloads. From this list, items under point can be canceled (d),
26 ;; paused (p), slowed (s), and have its priority adjusted ([ and ]).
28 ;;; Code:
30 (require 'json)
31 (require 'cl-lib)
32 (require 'hl-line)
33 (require 'notifications)
35 (defgroup youtube-dl ()
36 "Download queue for the youtube-dl command line program."
37 :group 'external)
39 (defcustom youtube-dl-directory "~/Downloads/"
40 "Directory in which to run youtube-dl."
41 :group 'youtube-dl
42 :type 'directory)
44 (defcustom youtube-dl-program
45 ;; "yt-dlp" is a fork of "youtube-dl".
46 (cond
47 ((executable-find "yt-dlp") "yt-dlp")
48 ((executable-find "youtube-dl") "youtube-dl"))
49 "The name of the program invoked for downloading YouTube videos.
50 NOTE: It's specified by program name instead of program path."
51 :type 'string
52 :safe #'stringp
53 :group 'youtube-dl)
55 (defcustom youtube-dl-arguments
56 '("--newline" "--no-mtime" "--restrict-filenames" "--no-warnings")
57 "Arguments to be send to youtube-dl.
58 Instead of --limit-rate use custom option `youtube-dl-slow-rate-limit'."
59 :type '(repeat string)
60 :safe #'listp
61 :group 'youtube-dl)
63 (defcustom youtube-dl-proxy ""
64 "Specify the proxy for youtube-dl command.
65 For example:
67 127.0.0.1:8118
68 socks5://127.0.0.1:1086"
69 :type 'string
70 :safe #'stringp
71 :group 'youtube-dl)
73 (defcustom youtube-dl-proxy-url-list '()
74 "A list of URL domains which should use proxy for youtube-dl."
75 :type 'list
76 :safe #'listp
77 :group 'youtube-dl)
79 (defcustom youtube-dl-auto-show-list t
80 "Auto show youtube-dl-list buffer."
81 :type 'boolean
82 :safe #'booleanp
83 :group 'youtube-dl)
85 (defcustom youtube-dl-process-model 'multiple-processes
86 "Determine youtube-dl.el downloading process model."
87 :safe #'symbolp
88 :type '(choice (symbol :tag "single process only" 'single-process)
89 (symbol :tag "multiple processes" 'multiple-processes)))
91 (defcustom youtube-dl-max-failures 8
92 "Maximum number of retries for a single video."
93 :type 'integer
94 :safe #'numberp
95 :group 'youtube-dl)
97 (defcustom youtube-dl-slow-rate-limit "2M"
98 "Download rate for slow item (corresponding to option --limit-rate)."
99 :type 'string
100 :safe #'stringp
101 :group 'youtube-dl)
103 (defcustom youtube-dl-notify (cl-case system-type
104 (gnu/linux t)
105 (darwin nil)
106 (windows-nt nil))
107 "Whether raise notifications."
108 :type 'boolean
109 :safe #'booleanp
110 :group 'youtube-dl)
112 (defcustom youtube-dl-destination-illegal-filename-chars-regex "[#$%+!<>&*|{}?=@:/\\\"'`]"
113 "Specify the regex of invalid destination filename.
114 This will avoid youtube-dl can't save invalid filename caused error.
115 https://www.mtu.edu/umc/services/websites/writing/characters-avoid/"
116 :type 'string
117 :safe #'stringp
118 :group 'youtube-dl)
120 ;;; TEST:
121 ;; (replace-regexp-in-string
122 ;; youtube-dl-destination-illegal-filename-chars-regex "-"
123 ;; "[#$%+!<>&*|{}?=@:/\\\"'`]")
125 (defface youtube-dl-active
126 '((t :inherit font-lock-function-name-face :foreground "forest green"))
127 "Face for highlighting the active download item."
128 :group 'youtube-dl)
130 (defface youtube-dl-slow
131 '((t :inherit font-lock-variable-name-face :foreground "orange"))
132 "Face for highlighting the slow (S) tag."
133 :group 'youtube-dl)
135 (defface youtube-dl-pause
136 '((t :inherit font-lock-type-face :foreground "gray"))
137 "Face for highlighting the pause (P) tag."
138 :group 'youtube-dl)
140 (defface youtube-dl-priority
141 '((t :inherit font-lock-keyword-face :foreground "light blue"))
142 "Face for highlighting the priority marker."
143 :group 'youtube-dl)
145 (defface youtube-dl-failure
146 '((t :inherit font-lock-warning-face :foreground "red"))
147 "Face for highlighting the failure marker."
148 :group 'youtube-dl)
150 (defvar-local youtube-dl--log-item nil
151 "Item currently being displayed in the log buffer.")
153 (cl-defstruct (youtube-dl-item (:constructor youtube-dl-item--create)
154 (:copier nil))
155 "Represents a single video to be downloaded with youtube-dl."
156 url ; Video URL (string)
157 vid ; Video ID (integer)
158 directory ; Working directory for youtube-dl (string or nil)
159 destination ; Preferred destination file (string or nil)
160 running ; Status indicator of process (boolean)
161 failures ; Number of video download failures (integer)
162 priority ; Download priority (integer)
163 title ; Listing display title (string or nil)
164 percentage ; Current download percentage (string or nil)
165 total-size ; Total download size (string or nil)
166 log ; All program output (list of strings)
167 log-end ; Last log item (list of strings)
168 paused-p ; Non-nil if download is paused (boolean)
169 slow-p ; Non-nil if download should be rate limited (boolean)
170 rate-limit ; Download rate limit (e.g. 50K or 2M)
171 buffer ; the process buffer name
172 process ; the process object
173 download-rate ; Download rate in bytes per second (e.g. 50K or 2M)
174 eta ; ETA time (e.g. 01:01:57)
177 (defvar youtube-dl-items ()
178 "List of all items still to be downloaded.")
180 (defvar youtube-dl-process nil
181 "The currently active youtube-dl process.")
183 (defun youtube-dl--process-buffer-name (vid)
184 "Return the process buffer name which constructed with VID."
185 (format " *youtube-dl vid:%s*" vid))
187 (defun youtube-dl--proxy-append (url &optional option value)
188 "Decide whether append proxy option in youtube-dl command based on URL."
189 (let ((domain (url-domain (url-generic-parse-url url))))
190 (if (and (member domain youtube-dl-proxy-url-list) ; <-- whether toggle proxy?
191 (not (string-empty-p youtube-dl-proxy)))
192 (if option ; <-- whether has command-line option?
193 (list "--proxy" youtube-dl-proxy option value)
194 (list "--proxy" youtube-dl-proxy))
195 (if option ; <-- return original arguments for no proxy
196 (list option value)
197 nil ; <-- return nothing for url no need proxy
198 ))))
200 (defun youtube-dl--next ()
201 "Returns the next item to be downloaded."
202 (let (best best-score)
203 (dolist (item youtube-dl-items best)
204 (let* ((failures (youtube-dl-item-failures item))
205 (priority (youtube-dl-item-priority item))
206 (paused-p (youtube-dl-item-paused-p item))
207 (score (- priority failures)))
208 (when (and (not paused-p)
209 (< failures youtube-dl-max-failures))
210 (cond ((null best)
211 (setf best item
212 best-score score))
213 ((> score best-score)
214 (setf best item
215 best-score score))))))))
217 (defun youtube-dl--current ()
218 "Return the item currently being downloaded."
219 (when youtube-dl-process
220 (plist-get (process-plist youtube-dl-process) :item)))
222 (defun youtube-dl--remove (item)
223 "Remove ITEM from the queue and kill process."
224 (let ((proc (youtube-dl-item-process item)))
225 (when (process-live-p proc) ; `kill-process' only when `proc' is still alive.
226 (kill-process proc)))
227 (setf youtube-dl-items (cl-delete item youtube-dl-items)))
229 (defun youtube-dl--add (item)
230 "Add ITEM to the queue."
231 (setf youtube-dl-items (nconc youtube-dl-items (list item))))
233 (defvar youtube-dl--timer-item nil
234 "A global variable for passing ITEM into `youtube-dl--run' in `run-with-timer'.")
236 (defun youtube-dl--sentinel (proc status)
237 (when-let ((item (plist-get (process-plist proc) :item)))
238 ;; (setf youtube-dl-process proc) ; dont' need to set current process.
239 (cond
240 ;; process status is finished.
241 ((equal status "finished\n")
242 ;; mark item structure property `:running' to `nil'.
243 (setf (youtube-dl-item-running item) nil)
244 (youtube-dl--remove item)
245 (when youtube-dl-notify
246 (notifications-notify :title "youtube-dl.el" :body "Download finished.")))
247 ;; detect whether process is in "pause" process status?
248 ((youtube-dl-item-paused-p item)
249 ;; mark item structure property `:running' to `nil'.
250 (setf (youtube-dl-item-running item) nil)
251 (message "[youtube-dl] process %s is paused." proc))
252 ;; when process downloading failed, then retry downloading.
253 ((equal status "killed: 9\n")
254 ;; mark item structure property `:running' to `nil'.
255 (setf (youtube-dl-item-running item) nil)
256 (message "[youtube-dl] process %s is killed." proc))
258 ;; mark item structure property `:running' to `nil'.
259 (setf (youtube-dl-item-running item) nil)
260 (cl-incf (youtube-dl-item-failures item))
261 ;; record log output to process buffer
262 (let ((buf (youtube-dl--log-buffer item))
263 (inhibit-read-only t))
264 (with-current-buffer buf
265 (print status buf)
266 (print (process-plist proc) buf)))
267 ;; re-run process
268 (if (<= (youtube-dl-item-failures item) 10)
269 (youtube-dl--run)
270 (setq youtube-dl--timer-item item)
271 (run-with-timer (* 60 5) nil #'youtube-dl--run)
272 (message "[youtube-dl] delay process %s" proc))))))
274 (defun youtube-dl--progress (output)
275 "Return the download progress for the given output.
276 Progress lines that straddle output chunks are lost. That's fine
277 since this is just used for display purposes.
279 Return list: (\"34.6%\" \"~2.61GiB\" \"929.50KiB/s\") "
280 (let ((start 0)
281 (progress nil))
282 (cond
283 ;; [download] 34.6% of ~2.61GiB at 929.50KiB/s ETA 01:01:57
284 ;; +---+ +------+ +---------+ +------+
285 ((string-equal youtube-dl-program "youtube-dl")
286 (while (string-match "\\[download\\] +\\([^ ]+%\\) +of *~? *\\([^ ]+\\) +at +\\([^ ]+\\) +ETA +\\([^ \n]+\\)" output start)
287 (let ((percent (match-string 1 output))
288 (total-size (match-string 2 output))
289 (download-rate (match-string 3 output))
290 (eta (match-string 4 output)))
291 (setf progress (list percent total-size download-rate eta)
292 start (match-end 0)))))
293 ((string-equal youtube-dl-program "yt-dlp")
294 ;; The output has non-ASCII shell color codes. Disable it with command-line option "--no-colors".
295 ;; [download] 0.4% of ~ 314.23MiB at 7.54KiB/s ETA 11:48:26 (frag 0/216)
296 ;; +--+ +-------+ +-------+ +------+
297 (while (string-match "\\[download\\] +\\([^ ]+%\\) +of *~? *\\([^ ]+\\) +at +\\([^ ]+\\) +ETA +\\([^ \n]+\\)" output start)
298 (let ((percent (match-string 1 output))
299 (total-size (match-string 2 output))
300 (download-rate (match-string 3 output))
301 (eta (match-string 4 output)))
302 ;; filter out not correct matched data.
303 ;; DEBUG:
304 ;; (message "[DEBUG] [youtube-dl] download-rate regexp matched: >>> %s <<<" download-rate)
305 ;; (message "[DEBUG] [youtube-dl] eta regexp matched: >>> %s <<<" eta)
306 (unless (or (string-equal-ignore-case total-size "Unknown.*")
307 (string-equal-ignore-case eta "Unknown.*") ; [download] 0.0% of 2.14MiB at Unknown B/s ETA Unknown
309 (setf progress (list percent total-size download-rate eta)
310 start (match-end 0)))))))
311 ;; DEBUG:
312 ;; (when progress
313 ;; (cl-destructuring-bind (percentage total-size download-rate eta) progress
314 ;; (message "[DEBUG] [youtube-dl] progress: percent: %s, total-size: %s, speed: %s, eta: %s."
315 ;; percentage total-size download-rate eta)))
316 progress))
318 (defun youtube-dl--get-destination (url)
319 "Return the destination filename for the given `URL' (if any).
320 The destination filename may potentially straddle two output
321 chunks, but this is incredibly unlikely. It's only used for
322 display purposes anyway.
324 $ youtube-dl --get-filename <URL>"
325 (message "[youtube-dl] getting video destination.")
326 (let* ((get-destination-buffer " *youtube-dl-get-destination*")
327 (get-destination-error-buffer " *youtube-dl-get-destination error*")
328 (destination))
329 ;; clear destination buffer content to clear destination history.
330 (progn
331 (when (buffer-live-p (get-buffer get-destination-buffer))
332 (with-current-buffer get-destination-buffer (erase-buffer)))
333 (when (buffer-live-p (get-buffer get-destination-error-buffer))
334 (with-current-buffer get-destination-error-buffer (erase-buffer))))
335 ;; retrieve destination
336 (make-process
337 :command (list youtube-dl-program "--get-filename" url)
338 :sentinel (lambda (proc event)
339 (if (string= event "finished\n")
340 (setq destination
341 (replace-regexp-in-string
342 "\\ +\\[.*\\]" "" ; delete trailing [...] in filename.
343 (replace-regexp-in-string
344 "\n" "" ; delete trailing "\n" character.
345 (with-current-buffer get-destination-buffer (buffer-string)))))
346 (let* ((buffer-str (with-current-buffer get-destination-error-buffer (buffer-string)))
347 (match-regex "\\(ERROR\\|Error\\): \\(.*\\)")
348 (error-p (string-match-p match-regex buffer-str))
349 (matched-str (when (string-match match-regex buffer-str) (match-string 2 buffer-str))))
350 (if error-p
351 (progn
352 (message "[youtube-dl] `youtube-dl--get-destination' error!\n%s" matched-str)
353 ;; TODO:
354 ;; (funcall 'youtube-dl--get-destination url)
356 (progn
357 (message "[youtube-dl] get output filename success.")
358 ;; (kill-process proc)
359 ;; (kill-buffer (process-buffer proc))
360 )))))
361 :name "youtube-dl-get-destination"
362 :buffer get-destination-buffer
363 :stderr get-destination-error-buffer)
365 (setq destination-not-returned t)
366 (dotimes (i 10)
367 (when destination-not-returned
368 (sleep-for 1)
369 (if destination (setq destination-not-returned nil))))
371 ;; Prompt for user the destination interactively.
372 (unless destination
373 (setq destination (substring-no-properties (read-string "[youtube-dl] input destination: "))))
375 (when destination
376 (kill-new destination) ; copy to Emacs kill-ring for later operation.
377 (message "[youtube-dl] have got destination : %s" destination)
378 destination)))
380 ;;; TEST:
381 ;; (youtube-dl--get-destination "https://www.pornhub.com/view_video.php?viewkey=ph603b575e00170")
383 (defun youtube-dl--filter (proc output)
384 (if (plist-get (process-plist proc) :item)
385 (let* ((item (plist-get (process-plist proc) :item))
386 (progress (youtube-dl--progress output))
387 (destination (youtube-dl-item-destination item)))
388 ;; Append to program log.
389 (let ((logged (list output)))
390 (if (and (youtube-dl-item-log item) (youtube-dl-item-log-end item)) ; make sure not nil.
391 (setf (cdr (youtube-dl-item-log-end item)) logged
392 (youtube-dl-item-log-end item) logged)
393 (setf (youtube-dl-item-log item) logged
394 (youtube-dl-item-log-end item) logged)))
395 ;; Update progress information.
396 (when progress
397 (cl-destructuring-bind (percentage total-size download-rate eta) progress
398 (setf (youtube-dl-item-percentage item) percentage
399 (youtube-dl-item-total-size item) total-size
400 (youtube-dl-item-download-rate item) download-rate
401 (youtube-dl-item-eta item) eta)))
402 ;; monitor process output whether error occurred?
403 (let* ((item-log-end (youtube-dl-item-log-end item))
404 (log-end-str (cond
405 ((stringp item-log-end) item-log-end)
406 ((listp item-log-end) (car (last item-log-end)))))
407 (error-hander (lambda () (setf (youtube-dl-item-running item) nil)
408 (message "[youtube-dl] ✖ <proc: %s>, destination: %s"
409 (youtube-dl-item-process item) (youtube-dl-item-destination item))))
410 (running-handler (lambda () (setf (youtube-dl-item-running item) t)))
411 (finished-handler (lambda () (setf (youtube-dl-item-running item) nil)
412 (message "[youtube-dl] ✔ %s downloaded." (youtube-dl-item-destination item)))))
413 (when (stringp log-end-str)
414 (cond
415 ;; progress finished when percentage is 100%.
416 ((or (string-equal (youtube-dl-item-percentage item) "100%")
417 (string-match "\\[download\\]\\ *100%.*" log-end-str)) ; ".*100%.*"
418 (funcall finished-handler))
419 ;; process exited abnormally with code 1
420 ((string-match "exited abnormally with code 1" log-end-str)
421 (funcall error-hander))
422 ;; file already downloaded
423 ((string-match "ERROR: Fixed output name but more than one file to download:.*" log-end-str)
424 ;; (funcall error-hander)
425 (message "[youtube-dl] ERROR: Already has existing downloading process or output file! (%s)" (youtube-dl-item-destination item)))
426 ;; any other errors
427 ((string-match "ERROR:.*" log-end-str)
428 (funcall error-hander))
429 (t (funcall running-handler)))))
430 ;; DEBUG:
431 ;; (message "[DEBUG] [youtube-dl] destination/title: %s" destination)
432 ;; Set item title to destination if it's empty.
433 (unless (youtube-dl-item-title item)
434 (setf (youtube-dl-item-title item) destination))
435 ;; write output to process associated buffer.
436 (with-current-buffer (process-buffer proc)
437 (let ((moving (= (point) (process-mark proc)))
438 ;; avoid process buffer read-only issue for `insert'.
439 (inhibit-read-only t))
440 (save-excursion
441 ;; Insert the text, advancing the process marker.
442 (goto-char (process-mark proc))
443 (insert output)
444 (set-marker (process-mark proc) (point)))
445 (if moving (goto-char (process-mark proc)))))
446 (youtube-dl--redisplay))))
448 (defun youtube-dl--construct-command (item)
449 "Construct full command of youtube-dl with arguments from ITEM."
450 (let ((url (youtube-dl-item-url item))
451 (slow-p (youtube-dl-item-slow-p item))
452 (rate-limit (youtube-dl-item-rate-limit item))
453 (destination (youtube-dl-item-destination item)))
454 (append
455 (list youtube-dl-program "--newline")
456 youtube-dl-arguments
457 ;; Disable non-ASCII shell color codes in output.
458 (cond
459 ((string-equal youtube-dl-program "youtube-dl") '("--no-color"))
460 ((string-equal youtube-dl-program "yt-dlp") '("--no-colors")))
461 (youtube-dl--proxy-append url)
462 (when slow-p `("--rate-limit" ,rate-limit))
463 (when destination `("--output" ,destination))
464 `("--" ,url))))
466 (defun youtube-dl--run-single-process ()
467 "Start youtube-dl downloading in single process."
468 (let ((item (youtube-dl--next))
469 (current-item (youtube-dl--current)))
470 (if (eq item current-item)
471 (youtube-dl--redisplay) ; do nothing, just display the youtube-dl-list buffer.
472 (if youtube-dl-process
473 (progn
474 ;; Switch to higher priority job, but offset error count first.
475 (cl-decf (youtube-dl-item-failures current-item))
476 (kill-process youtube-dl-process)) ; sentinel will clean up
477 ;; No subprocess running, start a one.
478 (let* ((url (substring-no-properties (youtube-dl-item-url item)))
479 (directory (youtube-dl-item-directory item))
480 (destination (youtube-dl-item-destination item))
481 (vid (youtube-dl-item-vid item))
482 (proc-buffer-name (youtube-dl--process-buffer-name vid))
483 (default-directory
484 (if directory
485 (concat (directory-file-name directory) "/")
486 (concat (directory-file-name youtube-dl-directory) "/")))
487 (_ (mkdir default-directory t))
488 (slow-p (youtube-dl-item-slow-p item))
489 (rate-limit (or (youtube-dl-item-rate-limit item) youtube-dl-slow-rate-limit))
490 (proc (make-process
491 :name proc-buffer-name
492 :command (let ((command (youtube-dl--construct-command item)))
493 ;; Insert complete command into process buffer for debugging.
494 (with-current-buffer (get-buffer-create proc-buffer-name)
495 (insert (format "Command: %s" command)))
496 command)
497 :sentinel #'youtube-dl--sentinel
498 :filter #'youtube-dl--filter
499 :buffer proc-buffer-name)))
500 (set-process-plist proc (list :item item))
501 (setf youtube-dl-process proc)
502 ;; mark item structure property `:running' to `t'.
503 (setf (youtube-dl-item-running item) t))))
504 (youtube-dl--redisplay)))
506 (defun youtube-dl--run-multiple-processes (&optional item)
507 "Start youtube-dl downloading in multiple processes."
508 (let* ((item (or item
509 youtube-dl--timer-item ; use item from `run-with-timer'.
510 (with-current-buffer (youtube-dl--list-buffer)
511 (nth (1- (line-number-at-pos)) youtube-dl-items))))
512 (current-item (youtube-dl--current)) ; `youtube-dl-process' currently actived process.
513 (proc (youtube-dl-item-process item)))
514 (unless (and item (youtube-dl-item-running item)) ; whether item structure property `:running' is `t'?
515 (let* ((url (substring-no-properties (youtube-dl-item-url item)))
516 (directory (youtube-dl-item-directory item))
517 (destination (youtube-dl-item-destination item))
518 (vid (youtube-dl-item-vid item))
519 (proc-buffer-name (youtube-dl--process-buffer-name vid))
520 (default-directory
521 (if directory
522 (concat (directory-file-name directory) "/")
523 (concat (directory-file-name youtube-dl-directory) "/")))
524 (_ (mkdir default-directory t))
525 (slow-p (youtube-dl-item-slow-p item))
526 (rate-limit (or (youtube-dl-item-rate-limit item) youtube-dl-slow-rate-limit))
527 (failures (youtube-dl-item-failures item))
528 (proc (when (<= failures 10) ; re-run process only when failures <= 10.
529 (make-process
530 :name proc-buffer-name
531 :command (let ((command (youtube-dl--construct-command item)))
532 ;; Insert complete command into process buffer for debugging.
533 (let ((inhibit-read-only t))
534 (with-current-buffer (get-buffer-create proc-buffer-name)
535 (insert (format "Command: %s" command))))
536 command)
537 :sentinel #'youtube-dl--sentinel
538 :filter #'youtube-dl--filter
539 :buffer proc-buffer-name))))
540 ;; clear temporary item variable `youtube-dl--timer-item'.
541 (setq youtube-dl--timer-item nil)
542 (when (processp proc)
543 ;; set process property list.
544 (set-process-plist proc (list :item item))
545 ;; assign `proc' object to item slot `:process'.
546 (setf (youtube-dl-item-process item) proc)
547 ;; set current youtube-dl process variable.
548 (setf youtube-dl-process proc)
549 ;; mark item structure property `:running' to `t'.
550 (setf (youtube-dl-item-running item) t))))
551 (youtube-dl--redisplay)))
553 (defun youtube-dl--run (&optional item)
554 "Start youtube-dl downloading."
555 ;; if single process model, then start download next item.
556 ;; if multiple processes model, then don't start next item `youtube-dl--run'.
557 (cl-case youtube-dl-process-model
558 (single-process (youtube-dl--run-single-process item))
559 (multiple-processes (youtube-dl--run-multiple-processes item))))
561 (defun youtube-dl--get-vid (url)
562 "Get video `URL' video vid with youtube-dl option `--get-id'."
563 (message "[youtube-dl] getting video vid.")
564 (let* ((parsed-url (url-generic-parse-url url))
565 (domain (url-domain parsed-url))
566 (parameters (url-filename parsed-url)))
567 ;; URL domain matching with regexp to extract vid.
568 (pcase domain
569 ("bilibili.com" ; "/video/BV1B8411L7te/", "/video/BV1uj411N7cp?spm_id_from=..0.0"
570 (when (string-match "/video/\\([^/?&]*\\)" parameters)
571 (match-string 1 parameters)))
572 ("pornhub.com" ; "/view_video.php?viewkey=ph6238f9c7cc9e2"
573 (when (string-match "/view_video\\.php\\?viewkey=\\([^/?&]*\\)" parameters)
574 (match-string 1 parameters)))
575 ("youtube.com" ; "/watch?v=q-iLEteyeUQ", "/watch?v=48JlgiBpw_I&t=311s"
576 (when (string-match "/watch\\?v=\\([^/?&]*\\)" parameters)
577 (match-string 1 parameters)))
578 (_ (progn
579 (let* ((output (with-temp-buffer
580 (apply #'call-process
581 youtube-dl-program
582 nil t nil
583 (youtube-dl--proxy-append url "--get-id" url))
584 (buffer-string)))
585 (output-lines-list (string-lines output))
586 (error-p (string-match-p "ERROR" output)))
587 (cond
588 ;; ("VrAfJvZGGXE" "")
589 ;; TEST: (youtube-dl--get-vid "https://www.youtube.com/watch?v=VROjLiq9LeQ")
590 ((length= output-lines-list 2) (car output-lines-list))
591 ;; TEST: (youtube-dl--get-vid "https://hanime1.me/watch?v=22454")
592 ;; ("WARNING: Falling back on generic information extractor." "watch?v=22454" "")
593 ((length= output-lines-list 3) (car (last output-lines-list 1)))
594 ;; TEST: (youtube-dl--get-vid "https://www.pornhub.com/view_video.php?viewkey=ph637ed6daf1795")
595 ;; WARNING: unable to extract view count; please report this issue on https://yt-dl.org/bug . Make sure you are using the latest version; see https://yt-dl.org/update on how to update. Be sure to call youtube-dl with the --verbose flag and include its complete output.
596 ((string-match-p "WARNING" (car output-lines-list))
597 (car (last output-lines-list 1)))
598 ;; get vid failed, use URL string regex matching instead.
599 (t (progn
600 (when error-p
601 ;; WARNING: Could not send HEAD request to https://hanime1.me/watch?v=22709:
602 ;; HTTP Error 503: Service Temporarily Unavailable
603 ;; ERROR: Unable to download webpage: HTTP Error 503: Service Temporarily
604 ;; Unavailable (caused by <HTTPError 503: 'Service Temporarily
605 ;; Unavailable'>); please report this issue on https://yt-dl.org/bug . Make
606 ;; sure you are using the latest version; see https://yt-dl.org/update on
607 ;; how to update. Be sure to call youtube-dl with the --verbose flag and
608 ;; include its complete output.
610 ;; WARNING: unable to extract view count; please report this
611 ;; issue on https://yt-dl.org/bug . Make sure you are using
612 ;; the latest version; see https://yt-dl.org/update on how
613 ;; to update. Be sure to call youtube-dl with the --verbose
614 ;; flag and include its complete output.
615 (error (format "[youtube-dl] `youtube-dl--get-vid' retrive video vid error!\n%s" output)))
616 parameters)))))))))
618 ;;; TEST:
619 ;; (let ((parameters "/video/BV1B8411L7te/"))
620 ;; (when (string-match "/video/\\([^/]*\\)" parameters)
621 ;; (match-string 1 parameters)))
623 ;; (youtube-dl--get-vid "https://www.youtube.com/watch?v=VROjLiq9LeQ")
624 ;; (youtube-dl--get-vid "https://www.pornhub.com/view_video.php?viewkey=ph637ed6daf1795")
626 ;;;###autoload
627 (cl-defun youtube-dl
628 (url &key title (priority 0) directory destination paused slow)
629 "Queues URL for download using youtube-dl, returning the new item.
630 By default, it downloads to ~/Downloads/."
631 (interactive
632 (list (substring-no-properties
633 (read-from-minibuffer
634 "URL: " (or (thing-at-point 'url)
635 (when interprogram-paste-function
636 (funcall interprogram-paste-function)))))))
637 ;; remove this ID failure only on youtube.com, use URL as ID. or use youtube-dl extracted title, or hash on URL.
638 (let* ((vid (youtube-dl--get-vid url))
639 (destination (replace-regexp-in-string ; replace illegal filename characters in `destination'.
640 youtube-dl-destination-illegal-filename-chars-regex "-"
641 (or (youtube-dl--get-destination url)
642 destination)))
643 (title (or (replace-regexp-in-string (format "-%s.*" vid) "" destination)
644 (car (split-string destination "-"))))
645 (proc-buffer-name (youtube-dl--process-buffer-name vid))
646 (full-dir (expand-file-name (or directory "") youtube-dl-directory))
647 (item (youtube-dl-item--create :url url
648 :vid vid
649 :failures 0
650 :priority priority
651 :paused-p paused
652 :slow-p slow
653 :rate-limit nil
654 :directory full-dir
655 :destination destination
656 :title title
657 :buffer proc-buffer-name
658 :process nil)))
659 (prog1 item
660 (unless (youtube-dl-item-running item)
661 (youtube-dl--add item) ; Add ITEM to the queue.
662 (youtube-dl--run item) ; Start ITEM in the queue.
663 (when youtube-dl-auto-show-list
664 (youtube-dl-list))))))
666 (defalias 'youtube-dl-download-video 'youtube-dl)
668 (defun youtube-dl--playlist-list (playlist)
669 "For each video, return one plist with :index, :vid, and :title."
670 (with-temp-buffer
671 (when (zerop (call-process youtube-dl-program nil t nil
672 "--ignore-config"
673 "--dump-json"
674 "--flat-playlist"
675 playlist))
676 (goto-char (point-min))
677 (cl-loop with json-object-type = 'plist
678 for index upfrom 1
679 for video = (ignore-errors (json-read))
680 while video
681 collect (list :index index
682 :vid (plist-get video :vid)
683 :title (plist-get video :title))))))
685 (defun youtube-dl--playlist-reverse (list)
686 "Return a copy of LIST with the indexes reversed."
687 (let ((max (cl-loop for entry in list
688 maximize (plist-get entry :index))))
689 (cl-loop for entry in list
690 for index = (plist-get entry :index)
691 for copy = (copy-sequence entry)
692 collect (plist-put copy :index (- (1+ max) index)))))
694 (defun youtube-dl--playlist-cutoff (list n)
695 "Return a sorted copy of LIST with all items except where :index < N."
696 (let ((key (lambda (v) (plist-get v :index)))
697 (filter (lambda (v) (< (plist-get v :index) n)))
698 (copy (copy-sequence list)))
699 (cl-delete-if filter (cl-stable-sort copy #'< :key key))))
701 ;;;###autoload
702 (cl-defun youtube-dl-download-playlist
703 (url &key directory (first 1) paused (priority 0) reverse slow)
704 "Add entire playlist to download queue, with index prefixes.
706 :directory PATH -- Destination directory for all videos.
708 :first INDEX -- Start downloading from a given one-based index.
710 :paused BOOL -- Start all download entries as paused.
712 :priority PRIORITY -- Use this priority for all download entries.
714 :reverse BOOL -- Reverse the video numbering, solving the problem
715 of reversed playlists.
717 :slow BOOL -- Start all download entries in slow mode."
718 (interactive
719 (list (read-from-minibuffer
720 "URL: "
721 (when interprogram-paste-function
722 (funcall interprogram-paste-function)))))
723 (message "[youtube-dl] fetching playlist ...")
724 (let ((videos (youtube-dl--playlist-list url)))
725 (if (null videos)
726 (error "Failed to fetch playlist (%s)." url)
727 (let* ((max (cl-loop for entry in videos
728 maximize (plist-get entry :index)))
729 (width (1+ (floor (log max 10))))
730 (prefix-format (format "%%0%dd" width)))
731 (when reverse
732 (setf videos (youtube-dl--playlist-reverse videos)))
733 (dolist (video (youtube-dl--playlist-cutoff videos first))
734 (let* ((index (plist-get video :index))
735 (prefix (format prefix-format index))
736 (title (format "%s-%s" prefix (plist-get video :title)))
737 (dest (format "%s-%s" prefix "%(title)s-%(id)s.%(ext)s")))
738 (youtube-dl (plist-get video :url)
739 :title title
740 :priority priority
741 :directory directory
742 :destination dest
743 :paused paused
744 :slow slow)))))))
746 ;; List user interface:
748 ;;; refresh youtube-dl-list buffer (2).
749 (defun youtube-dl-list-redisplay ()
750 "Immediately redraw the queue list buffer."
751 (interactive)
752 (with-current-buffer (youtube-dl--list-buffer)
753 (let ((save-point (point))
754 (window (get-buffer-window (current-buffer))))
755 (youtube-dl--fill-listing)
756 (goto-char save-point)
757 (when window
758 (set-window-point window save-point))
759 (when hl-line-mode
760 (hl-line-highlight)))))
762 ;;; refresh youtube-dl-list buffer (1).
763 (defun youtube-dl--redisplay ()
764 "Redraw the queue list buffer only if visible."
765 (let ((log-buffer (youtube-dl--log-buffer)))
766 (when log-buffer
767 (with-current-buffer log-buffer
768 (let ((inhibit-read-only t)
769 (window (get-buffer-window log-buffer)))
770 (erase-buffer)
771 (mapc #'insert (youtube-dl-item-log youtube-dl--log-item))
772 (when window
773 (set-window-point window (point-max)))))))
774 (when (get-buffer-window (youtube-dl--list-buffer))
775 (youtube-dl-list-redisplay)))
777 (defun youtube-dl-list-log ()
778 "Display the log of the video under point."
779 (interactive)
780 (let* ((n (1- (line-number-at-pos)))
781 (item (nth n youtube-dl-items))
782 (buffer (youtube-dl--log-buffer item)))
783 (when item
784 (display-buffer buffer)
785 (select-window (get-buffer-window buffer))
786 (youtube-dl--redisplay))))
788 (defun youtube-dl-list-kill-log ()
789 "Kill the youtube-dl log buffer."
790 (interactive)
791 (let ((buffer (youtube-dl--log-buffer)))
792 (when buffer
793 (kill-buffer buffer))))
795 (defun youtube-dl-list-yank ()
796 "Copy the URL of the video under point to the clipboard."
797 (interactive)
798 (let* ((n (1- (line-number-at-pos)))
799 (item (nth n youtube-dl-items)))
800 (when item
801 (let ((url (concat "https://youtu.be/" (youtube-dl-item-vid item))))
802 (if (fboundp 'gui-set-selection)
803 (gui-set-selection nil url) ; >= Emacs 25
804 (with-no-warnings
805 (x-set-selection 'PRIMARY url))) ; <= Emacs 24
806 (message "[youtube-dl] yanked %s" url)))))
808 (defun youtube-dl-list-kill ()
809 "Remove the selected item from the queue."
810 (interactive)
811 (when youtube-dl-items ; avoid `youtube-dl-items' is `nil'.
812 (let* ((n (1- (line-number-at-pos)))
813 (item (nth n youtube-dl-items)))
814 (when item
815 (when (= n (1- (length youtube-dl-items)))
816 (forward-line -1))
817 (youtube-dl--remove item))))
818 (youtube-dl-list-redisplay))
820 (defun youtube-dl-list-pause ()
821 "Pause downloading of item under point."
822 (interactive)
823 (let* ((n (1- (line-number-at-pos)))
824 (item (nth n youtube-dl-items))
825 (proc (youtube-dl-item-process item))
826 (paused-p (youtube-dl-item-paused-p item)))
827 (if (and item paused-p)
828 ;; if paused, resume process.
829 (progn
830 (setf (youtube-dl-item-paused-p item) nil)
831 (youtube-dl--run))
832 ;; kill the process, but keep item on list buffer for pause status.
833 (when (process-live-p proc) (kill-process proc))
834 ;; after killed process, also setting item structure status properties.
835 (setf (youtube-dl-item-paused-p item) t)
836 (setf (youtube-dl-item-running item) nil))
837 (youtube-dl-list-redisplay)
838 (message "[youtube-dl] process %s is paused." proc)))
840 (defun youtube-dl-list-resume ()
841 "Resume failure/paused downloading item under point."
842 (interactive)
843 (when-let* ((n (1- (line-number-at-pos)))
844 (item (nth n youtube-dl-items))
845 (proc (youtube-dl-item-process item)))
846 (when item
847 (setf (youtube-dl-item-failures item) 0)
848 (cond
849 ((youtube-dl-item-paused-p item)
850 (setf (youtube-dl-item-paused-p item) nil)
851 (youtube-dl--run))
852 ((not (youtube-dl-item-running item))
853 (youtube-dl--run))
855 (setf (youtube-dl-item-running item) nil)
856 (youtube-dl--run)
857 ;; (user-error (format "[youtube-dl] Can't resume process %s correctly. Try resume again." proc))
858 ))))
859 (youtube-dl-list-redisplay))
861 (defun youtube-dl-list-toggle-slow (item &optional rate-limit)
862 "Set slow rate limit on item under point."
863 (interactive
864 (let* ((n (1- (line-number-at-pos))))
865 (list (nth n youtube-dl-items))))
866 (when item
867 (let ((proc (youtube-dl-item-process item))
868 (slow-p (youtube-dl-item-slow-p item))
869 (rate-limit (substring-no-properties
870 (or rate-limit
871 (completing-read "[youtube-dl] limit download rate (e.g. 100K): "
872 '("20K" "50K" "100K" "200K" "500K" "1M" "2M"))))))
873 ;; restart with a slower download rate.
874 (when (process-live-p proc) (kill-process proc))
875 (setf (youtube-dl-item-slow-p item) (not slow-p))
876 (setf (youtube-dl-item-rate-limit item) rate-limit)
877 (youtube-dl--run)))
878 (youtube-dl-list-redisplay))
880 (defun youtube-dl-list-toggle-slow-all ()
881 "Set slow rate on all items."
882 (interactive)
883 (let* ((count (length youtube-dl-items))
884 (slow-count (cl-count-if #'youtube-dl-item-slow-p youtube-dl-items))
885 (target (< slow-count (- count slow-count)))
886 (rate-limit (completing-read "[youtube-dl] limit download rate (e.g. 100K): "
887 '("20K" "50K" "100K" "200K" "500K" "1M" "2M"))))
888 (dolist (item youtube-dl-items)
889 (youtube-dl-list-toggle-slow item rate-limit)))
890 (youtube-dl--redisplay))
892 (defun youtube-dl-list-priority-modify (delta)
893 "Change priority of item under point by DELTA."
894 (let* ((n (1- (line-number-at-pos)))
895 (item (nth n youtube-dl-items)))
896 (when item
897 (cl-incf (youtube-dl-item-priority item) delta)
898 (youtube-dl--run))))
900 (defun youtube-dl-list-priority-up ()
901 "Decrease priority of item under point."
902 (interactive)
903 (youtube-dl-list-priority-modify 1))
905 (defun youtube-dl-list-priority-down ()
906 "Increase priority of item under point."
907 (interactive)
908 (youtube-dl-list-priority-modify -1))
910 (defvar youtube-dl-list-mode-map
911 (let ((map (make-sparse-keymap)))
912 (prog1 map
913 (define-key map "a" #'youtube-dl)
914 (define-key map "g" #'youtube-dl-list-redisplay)
915 (define-key map "l" #'youtube-dl-list-log)
916 (define-key map "L" #'youtube-dl-list-kill-log)
917 (define-key map "y" #'youtube-dl-list-yank)
918 (define-key map "k" #'youtube-dl-list-kill)
919 (define-key map "p" #'youtube-dl-list-pause)
920 (define-key map "r" #'youtube-dl-list-resume)
921 (define-key map "s" #'youtube-dl-list-toggle-slow)
922 (define-key map "S" #'youtube-dl-list-toggle-slow-all)
923 (define-key map "]" #'youtube-dl-list-priority-up)
924 (define-key map "[" #'youtube-dl-list-priority-down)))
925 "Keymap for `youtube-dl-list-mode'")
927 (defvar-local youtube-dl-list--auto-close-window-timer nil
928 "A timer to auto close youtube-dl list window.")
930 (defun youtube-dl-list--auto-close-window ()
931 "Auto close '*youtube-dl list*' buffer after finished all downloading."
932 (when (equal (length youtube-dl-items) 0)
933 (when-let ((buf (get-buffer-window (youtube-dl--list-buffer))))
934 (delete-window buf)
935 (when (timerp youtube-dl-list--auto-close-window-timer)
936 (cancel-timer youtube-dl-list--auto-close-window-timer)))))
938 (defvar youtube-dl-list--format
939 ;; (percentage download-rate)
940 ;; v
941 ;;space,vid,progress size eta fails
942 ;;| | | | | | status
943 ;;| | | | | | | title
944 ;;v v v v v v v v
945 "%s%-16s %-22.22s %-10.10s %-6.6s %-7.7s %-8.8s %s"
946 "Define the `youtube-dl-list-mode' `header-line-format' and list item format.")
948 (define-derived-mode youtube-dl-list-mode special-mode "youtube-dl"
949 "Major mode for listing the youtube-dl download queue."
950 :group 'youtube-dl
951 (use-local-map youtube-dl-list-mode-map)
952 (hl-line-mode)
953 (setq-local truncate-lines t) ; truncate long line text.
954 (setf header-line-format
955 (propertize
956 (format youtube-dl-list--format
957 (propertize " " 'display '((space :align-to 0))) ; space
958 " vid " "| progress" "| size" "| ETA" "| fails" "| status"
959 (concat "| title " (make-string 100 (string-to-char " "))))
960 'face '(:inverse-video t :extend t))))
962 (defvar youtube-dl--list-buffer-name " *youtube-dl list*"
963 "The buffer name of `youtube-dl-list-mode'.")
965 (defun youtube-dl--list-buffer ()
966 "Returns the queue listing buffer."
967 (if-let ((buf (get-buffer-create youtube-dl--list-buffer-name)))
968 (with-current-buffer buf
969 ;; TODO: use `tabulated-list-mode'.
970 ;; (tabulated-list-mode)
971 (youtube-dl-list-mode)
972 (current-buffer))))
974 (defun youtube-dl--log-buffer (&optional item)
975 "Returns a youtube-dl log buffer for ITEM."
976 (when item
977 (let* ((name (youtube-dl-item-buffer item))
978 (buffer (get-buffer-create name)))
979 (with-current-buffer buffer
980 (unless (eq major-mode 'special-mode)
981 (special-mode))
982 (setf youtube-dl--log-item item)
983 (setf (youtube-dl-item-log item) item)
984 (current-buffer)))))
986 ;;; refresh youtube-dl-list buffer (3).
987 (defun youtube-dl--fill-listing ()
988 "Erase and redraw the queue in the queue listing buffer."
989 (with-current-buffer (youtube-dl--list-buffer)
990 (let* ((inhibit-read-only t)
991 (active (youtube-dl--current)))
992 (erase-buffer)
993 (dolist (item youtube-dl-items)
994 (let ((vid (youtube-dl-item-vid item))
995 (running (youtube-dl-item-running item))
996 (failures (youtube-dl-item-failures item))
997 (priority (youtube-dl-item-priority item))
998 (percentage (youtube-dl-item-percentage item))
999 (download-rate (youtube-dl-item-download-rate item))
1000 (paused-p (youtube-dl-item-paused-p item))
1001 (slow-p (youtube-dl-item-slow-p item))
1002 (rate-limit (youtube-dl-item-rate-limit item))
1003 (total-size (youtube-dl-item-total-size item))
1004 (eta (youtube-dl-item-eta item))
1005 (title (youtube-dl-item-title item))
1006 (url (youtube-dl-item-url item)))
1007 (insert
1008 (propertize
1009 (format (concat youtube-dl-list--format "\n")
1010 ;; space
1012 ;; (propertize " " 'display '((space :align-to 0)))
1013 ;; vid
1014 (if running ; update `:running' property every time process update.
1015 (propertize vid 'face 'default)
1016 (propertize vid 'face 'youtube-dl-pause))
1017 ;; progress (percentage download-rate)
1018 (if (and percentage download-rate)
1019 (propertize (format "⦼%-5.5s ⇩%-17.17s " percentage download-rate) 'face 'youtube-dl-active)
1020 (propertize (format "⦼%-5.5s ⇩%-17.17s " percentage download-rate) 'face 'youtube-dl-pause))
1021 ;; size
1022 (or (propertize (format "%s" total-size) 'face 'youtube-dl-pause) "-")
1023 ;; eta
1024 (or (propertize (format "%s" eta) 'face 'youtube-dl-pause) "?")
1025 ;; failure
1026 (if (= failures 0)
1028 (propertize (format " [%d] " failures) 'face 'youtube-dl-failure))
1029 ;; priority
1030 ;; (if (= priority 0)
1031 ;; ""
1032 ;; (propertize (format "%+d " priority) 'face 'youtube-dl-priority))
1033 ;; status
1034 (concat
1035 (if slow-p (propertize "SLOW" 'face 'youtube-dl-slow))
1036 (if rate-limit (propertize (format "≤ %s" rate-limit) 'face 'youtube-dl-slow))
1037 (if paused-p (propertize "PAUSE" 'face 'youtube-dl-pause)))
1038 ;; title
1039 (or title ""))
1040 'url url)))))))
1042 ;;;###autoload
1043 (defun youtube-dl-list ()
1044 "Display a list of all videos queued for download."
1045 (interactive)
1046 (youtube-dl--fill-listing)
1047 ;; set timer to auto close `youtube-dl--list-buffer' "*youtube-dl list*" buffer after finished all downloading.
1048 (with-current-buffer (youtube-dl--list-buffer)
1049 (unless (timerp youtube-dl-list--auto-close-window-timer)
1050 (setq-local youtube-dl-list--auto-close-window-timer
1051 (run-with-timer 0 (* 60 2) #'youtube-dl-list--auto-close-window)))
1052 ;; jump to beginning of buffer.
1053 (goto-char (point-min)))
1054 (display-buffer (youtube-dl--list-buffer)))
1056 ;;;###autoload
1057 (defun youtube-dl-download-audio (url)
1058 "Download audio format of URL."
1059 (interactive
1060 (list (read-from-minibuffer
1061 "URL: " (or (thing-at-point 'url)
1062 (when interprogram-paste-function
1063 (funcall interprogram-paste-function))))))
1064 (let ((youtube-dl-arguments (append youtube-dl-arguments '("-x" "--audio-format" "best"))))
1065 (youtube-dl url)))
1067 ;;;###autoload
1068 (defun youtube-dl-download-subtitle (url)
1069 "Download video subtitle of URL."
1070 (interactive
1071 (list (read-from-minibuffer
1072 "URL: " (or (thing-at-point 'url)
1073 (when interprogram-paste-function
1074 (funcall interprogram-paste-function))))))
1075 (let ((youtube-dl-arguments (append youtube-dl-arguments
1076 '("--skip-download" "--no-warnings"
1077 ;; "--write-auto-sub" ; YouTube only, for auto-generated subtitle.
1078 "--write-sub"
1079 "--sub-lang" "en,zh-Hans"
1080 "--sub-format" "ass/srt/best" ; ass/srt/best, srt, vtt, ttml, srv3, srv2, srv1
1081 ))))
1082 (youtube-dl url)))
1084 ;;;###autoload
1085 (defun youtube-dl-download-thumbnail (url)
1086 "Download video thumbnail of URL."
1087 (interactive
1088 (list (read-from-minibuffer
1089 "URL: " (or (thing-at-point 'url)
1090 (when interprogram-paste-function
1091 (funcall interprogram-paste-function))))))
1092 (let ((youtube-dl-arguments (append youtube-dl-arguments
1093 '("--skip-download" "--no-warnings"
1094 "--write-thumbnail"))))
1095 (youtube-dl url)))
1097 (provide 'youtube-dl)
1099 ;;; youtube-dl.el ends here