2 Syren -- a lightweight downloader for Linux/BSD/Win/MacOSX
3 inspired by Axel Copyright 2001-2002 Wilmer van der Gaast
4 version 0.0.6 (atomic alien)
5 coded by Ketmar // Avalon Group
7 This program is free software; you can redistribute it and/or modify
8 it under the terms of the GNU General Public License as published by
9 the Free Software Foundation; either version 2 of the License, or
10 (at your option) any later version.
12 This program is distributed in the hope that it will be useful,
13 but WITHOUT ANY WARRANTY; without even the implied warranty of
14 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
15 GNU General Public License for more details.
17 You should have received a copy of the GNU General Public License with
18 the Debian GNU/Linux distribution in file /usr/doc/copyright/GPL;
19 if not, write to the Free Software Foundation, Inc., 59 Temple Place,
20 Suite 330, Boston, MA 02111-1307 USA
25 #include "syren_common.h"
27 #include "syren_str.h"
28 #include "syren_msg.h"
29 #include "syren_cfg.h"
30 #include "syren_hdrs.h"
31 #include "syren_tcp.h"
32 #include "syren_http.h"
33 #include "syren_ftp.h"
34 #include "syren_dloader.h"
36 #include "syren_scheme.h"
43 /* uncomment this to use Linux TTY ioctls */
46 /* uncomment this to use ANSI/VT100 TTY codes */
49 /* uncomment this to use "DEC private" TTY codes (ESC [ ? n)*/
57 cfgNoOutput
= SY_FALSE
,
59 cfgNoIndicator
= SY_FALSE
,
60 cfgUseProxyHTTP
= SY_TRUE
,
61 cfgUseProxyFTP
= SY_TRUE
,
62 cfgDecodeFTPURL
= SY_TRUE
;
64 cfgBufferSize
= 16384,
66 cfgReconnectDelay
= 10,
67 cfgReconnectAttempts
= 666,
72 *cfgDefaultName
= NULL
,
76 *cfgRefererStr
= NULL
,
80 *oOutFileName
= NULL
, /* -o/-O */
81 *oStateFileName
= NULL
, /* -s */
86 static TSyBool oResume
= SY_FALSE
;
87 static TSyBool oCanRename
= SY_TRUE
;
88 static TSyBool oCanRemoveState
= SY_TRUE
;
89 static TSyBool oIgnoreStatePos
= SY_FALSE
;
90 static TSyBool oNoStateFile
= SY_FALSE
;
91 static TSyBool oForceRetry
= SY_FALSE
;
92 static TSyBool writeCfg
= SY_FALSE
;
93 static TSyBool dumpState
= SY_FALSE
;
94 static TSyBool replyOnly
= SY_FALSE
;
95 static int uEncodeDecode
= 0;
97 static TSyBool doRXVT
= SY_FALSE
;
100 double lastSSave
= 0;
104 static int GetTTYWidth (void) {
106 if (!isatty(fileno(stderr
))) return -1;
107 if (ioctl(fileno(stderr
), TIOCGWINSZ
, &ws
)) return -1;
130 static void SyInitStats (TSyState
*state
, TSyStats
*si
) {
134 si
->interval
= 30; /* minute */
140 static void SyUpdateStats (TSyState
*state
, TSyStats
*si
, int64_t bytesDone
, int64_t bytesTotal
) {
141 TSyStatInfo
*i0
, *i1
;
142 double ctime
, ptime
, t
;
143 double w0
, w1
; /* weights */
149 ctime
= SyGetTimeD();
152 /* nothing started */
156 i1
->bytesDone
= bytesDone
;
164 /* update "bytes done" */
165 i1
->bytesDone
= bytesDone
-(i1
->bytesFrom
);
167 ptime
= ctime
-(i1
->time
);
168 if (ptime
>= si
->interval
) {
169 /* interval passed, move state */
171 i1
->bytesFrom
= bytesDone
;
174 i0
->time
= ptime
; /* i0->time: time, taken by this stat */
177 /* calculate weights */
178 t
= ctime
-(i1
->time
);
179 w1
= i0
->active
?t
/si
->interval
:1.0;
180 if (w1
> 1.0) w1
= 1.0;
184 bps0
= i0
->active
?(double)(i0
->bytesDone
)/i0
->time
:0.0;
185 bps1
= t
>=0.5?(double)(i1
->bytesDone
)/t
:0.0;
186 si
->bps
= (bps0
*w0
)+(bps1
*w1
);
187 /*fprintf(stderr, " %i:%i\n", (int)(w0*100), (int)(w1*100));
188 fprintf(stderr, " %i:%i:%i\n", (int)bps0, (int)bps1, (int)si->bps);
189 fprintf(stderr, " %i:%i:%i\n", (int)(bps0*w0), (int)(bps1*w1), (int)si->bps);*/
192 if (bytesTotal
> 0 && si
->bps
>= 1) {
193 si
->eta
= (double)(bytesTotal
-bytesDone
)/(si
->bps
);
194 } else si
->eta
= 0.0;
198 static TSyStats stats
;
201 static void PrintProgress (TSyState
*state
, double timePassed
, int64_t bytesDone
, int64_t bytesTotal
,
203 char tmp0
[128], tmp1
[128], tmp2
[128], tmp3
[128], tmp4
[128], fmt
[128], str
[1024];
204 int len
; int64_t fb
= state
->firstByte
;
205 if (cfgNoIndicator
) return;
206 SyUpdateStats(state
, &stats
, bytesDone
, bytesTotal
);
207 fprintf(stderr
, "\r");
209 fputs("\x1b[?7l", stderr
);
211 if (bytesTotal
> 0) {
212 len
= SyLong2StrComma(tmp1
, (bytesTotal
+fb
));
213 SyLong2StrComma(tmp0
, (bytesDone
+fb
));
214 sprintf(fmt
, "[%%%is/%%s] ", len
);
215 sprintf(tmp2
, fmt
, tmp0
, tmp1
); /*!*/
216 sprintf(tmp3
, "[%3i%%] ", (int)((double)100*(bytesDone
+fb
)/(bytesTotal
+fb
))); /*!*/
218 SySize2Str(tmp0
, (int64_t)stats
.bps
);
219 sprintf(tmp4
, "[SPD:%s] ", tmp0
); /*!*/
220 SyTime2Str(tmp0
, (int)(SyGetTimeD()-stats
.sttime
));
221 SyTime2Str(tmp1
, (int)stats
.eta
);
222 sprintf(fmt
, "TIME:%s ETA:%s", tmp0
, tmp1
); /*!*/
223 fprintf(stderr
, "%s%s%s%s", tmp2
, tmp3
, tmp4
, fmt
);
224 sprintf(str
, "%s%s%s/%s", tmp2
, tmp3
, tmp0
, tmp1
);
226 SyLong2StrComma(tmp0
, bytesDone
+fb
);
227 SyTime2Str(tmp1
, (int)(SyGetTimeD()-stats
.sttime
));
228 SySize2Str(fmt
, (int64_t)stats
.bps
);
229 sprintf(str
, "[%s] [SPD:%s] TIME:%s", tmp0
, fmt
, tmp1
);
230 fprintf(stderr
, "%s", str
);
233 if (isatty(fileno(stdout
)) && doRXVT
== SY_TRUE
) {
234 fprintf(stdout
, "\x1b]2;%s\7", str
);
239 fprintf(stderr
, "\x1b[K");
242 fputs("\x1b[?7h", stderr
);
244 if (done
== SY_TRUE
) fputs("\n", stderr
);
249 static void SyPrintStr (void *udata
, TSyMsgType msgtype
, const char *msg
) {
253 static const char *ltr
= "NMWE?";
254 if (cfgNoOutput
) return;
255 if (cfgQuiet
&& msgtype
< SY_MSG_MSG
) return;
259 fputs("\x1b[?7l", stderr
);
263 if (wdt
< 0) wdt
= strlen(msg
)+16;
266 if (msgtype
< SY_MSG_NOTICE
|| msgtype
> SY_MSG_ERROR
) msgtype
= SY_MSG_ERROR
+1;
267 fputc(ltr
[msgtype
], stderr
);
272 if (wdt
>= strlen(msg
)) fputs(msg
, stderr
);
273 else while (wdt
--) fputc(*(msg
++), stderr
);
280 fputs("\x1b[K", stderr
);
284 fputs("\x1b[?7h", stderr
);
290 if (isatty(fileno(stdout
)) && doRXVT
== SY_TRUE
&& msgtype
> SY_MSG_NOTICE
) {
291 fprintf(stdout
, "\x1b]2;%s\7", msg
);
299 static void SyDefaultCfg (TSyCfg
*cfg
) {
300 SyCfgAddIntKey(cfg
, "reconnect_delay", "number of seconds to wait before reconnecting to server", &cfgReconnectDelay
);
301 SyCfgAddIntKey(cfg
, "reconnect_attempts", "number of reconnection attempts (0: do not reconnect)", &cfgReconnectAttempts
);
302 SyCfgAddIntKey(cfg
, "connection_timeout", "i/o timeout in seconds", &cfgTimeout
);
303 SyCfgAddIntKey(cfg
, "save_state_interval", "save state every x seconds (0: disable)", &cfgSaveInterval
);
304 cfgDefaultName
= SyStrNew("index.html", -1);
305 SyCfgAddStrKey(cfg
, "default_file_name", "default file name for http", &cfgDefaultName
);
306 SyCfgAddIntKey(cfg
, "max_redirects", "maximum number of redirects/symlinks", &cfgMaxRedirects
);
307 SyCfgAddIntKey(cfg
, "buffer_size", "maximum amount of bytes to read at once", &cfgBufferSize
);
308 SyCfgAddBoolKey(cfg
, "quiet", "tan: do not show notices", &cfgQuiet
);
309 SyCfgAddBoolKey(cfg
, "no_output", "tan: disable any output", &cfgNoOutput
);
310 SyCfgAddBoolKey(cfg
, "no_indicator", "tan: hide download indicator", &cfgNoIndicator
);
312 cfgIFace
= SyStrNew("", -1);
313 SyCfgAddStrKey(cfg
, "interface", "net interface to use for connection (name or IP)", &cfgIFace
);
315 cfgProxyHTTP
= SyStrNew(getenv("http_proxy"), -1);
316 SyCfgAddStrKey(cfg
, "http_proxy", "proxy for http connections", &cfgProxyHTTP
);
317 cfgProxyFTP
= SyStrNew(getenv("ftp_proxy"), -1);
318 SyCfgAddStrKey(cfg
, "ftp_proxy", "proxy for ftp connections", &cfgProxyFTP
);
319 SyCfgAddBoolKey(cfg
, "use_http_proxy", "use http proxy (if specified)?", &cfgUseProxyHTTP
);
320 SyCfgAddBoolKey(cfg
, "use_ftp_proxy", "use ftp proxy (if specified)?", &cfgUseProxyFTP
);
322 SyCfgAddIntKey(cfg
, "referer_type", "set referer type (0:none; 1:orig URL; 2:follow URL; 3:user)", &cfgRefererType
);
323 cfgRefererStr
= SyStrNew(NULL
, -1);
324 SyCfgAddStrKey(cfg
, "referer_str", "referer for mode 3", &cfgRefererStr
);
325 SyCfgAddBoolKey(cfg
, "decode_ftp_url", "decode URLs for FTP", &cfgDecodeFTPURL
);
327 cfgUAStr
= SyStrNew(SYREN_DEFAULT_USER_AGENT
, -1);
328 SyCfgAddStrKey(cfg
, "ua_str", "user agent (set to empty string if no UA should be sent)", &cfgUAStr
);
333 TSyResult
SySaveInt (int fd
, int64_t v
, int size
) {
336 if (fd
< 0 || size
< 1 || size
> 8) return SY_ERROR
;
338 b
= (v
>>(size
*8))&0xff;
339 if (SyWriteFile(fd
, &b
, 1) != SY_OK
) return SY_ERROR
;
344 TSyResult
SyLoadInt (int fd
, int64_t *v
, int size
) {
345 uint8_t b
; int64_t res
= 0;
348 if (!fd
|| size
< 1 || size
> 8) return SY_ERROR
;
350 if (SyReadFile(fd
, &b
, 1) != SY_OK
) return SY_ERROR
;
358 TSyResult
SySaveStr (int fd
, const char *v
) {
361 if (!fd
) return SY_ERROR
;
362 if (v
) len
= strlen(v
);
363 if (len
> 65535) return SY_ERROR
;
364 if (SySaveInt(fd
, len
, 2) != SY_OK
) return SY_ERROR
;
365 if (len
) { if (SyWriteFile(fd
, v
, len
) != SY_OK
) return SY_ERROR
; }
370 char *SyLoadStr (int fd
) {
374 if (!fd
) return NULL
;
375 if (SyLoadInt(fd
, &len
, 2) != SY_OK
) return NULL
;
376 if (!len
) return SyStrNewEmpty();
377 v
= calloc(1, len
+8); if (!v
) return NULL
;
378 if (SyReadFile(fd
, v
, len
) != SY_OK
) { free(v
); return NULL
; }
383 TSyResult
SySaveState (const char *destfile
, TSyState
*state
) {
384 TSyResult res
= SY_ERROR
;
389 if (!state
) return SY_ERROR
;
391 s
= SySPrintf("%s://%s:%s@%s:%i%s%s%s%s",
392 url
->protostr
, url
->user
, url
->pass
, url
->host
, url
->port
,
393 url
->dir
, url
->file
, url
->query
, url
->anchor
);
394 if (!s
) return SY_ERROR
;
395 fd
= SyOpenFile(destfile
, SY_FMODE_WRITE
, SY_TRUE
);
398 if (SySaveStr(fd
, "SYREN STATE FILE v5") != SY_OK
) break;
399 if (SySaveInt(fd
, state
->fileSize
, 8) != SY_OK
) break;
400 if (SySaveInt(fd
, state
->currentByte
+state
->firstByte
, 8) != SY_OK
) break;
401 if (SySaveInt(fd
, state
->lastByte
, 8) != SY_OK
) break;
402 if (SySaveStr(fd
, s
) != SY_OK
) break;
403 t
= strrchr(oOutFileName
, '/'); if (!t
) t
= oOutFileName
; else t
++;
404 if (SySaveStr(fd
, t
) != SY_OK
) break;
405 if (SySaveInt(fd
, cfgRefererType
, 1) != SY_OK
) break;
406 if (SySaveStr(fd
, cfgRefererStr
) != SY_OK
) break;
407 if (SySaveStr(fd
, oPostData
) != SY_OK
) break;
412 if (res
!= SY_OK
) SyDeleteFile(destfile
);
418 /* return destfile */
419 char *SyLoadState (const char *statefile
, TSyState
*state
) {
420 char *destfile
= NULL
, *s
;
423 fd
= SyOpenFile(statefile
, SY_FMODE_READ
, SY_FALSE
);
424 if (fd
< 0) return NULL
;
426 s
= SyLoadStr(fd
); if (!s
) break;
427 if (strcmp(s
, "SYREN STATE FILE v5")) break;
428 if (SyLoadInt(fd
, &state
->fileSize
, 8) != SY_OK
) break;
429 if (SyLoadInt(fd
, &state
->firstByte
, 8) != SY_OK
) break;
430 if (oIgnoreStatePos
!= SY_FALSE
) state
->firstByte
= 0;
431 if (state
->firstByte
< 0 || (state
->fileSize
>= 0 && state
->firstByte
> state
->fileSize
)) break;
432 if (SyLoadInt(fd
, &state
->lastByte
, 8) != SY_OK
) break;
433 if (state
->lastByte
< -1 || (state
->lastByte
>= 0 && state
->firstByte
> state
->lastByte
)) break;
434 free(s
); s
= SyLoadStr(fd
); if (!s
) break;
435 if (oURL
) free(oURL
); oURL
= s
;
436 s
= SyLoadStr(fd
); if (!s
) break;
437 destfile
= s
; s
= NULL
;
438 if (SyLoadInt(fd
, &i
, 1) != SY_OK
) break;
439 cfgRefererType
= (int)i
;
440 s
= SyLoadStr(fd
); if (!s
) break;
441 if (cfgRefererStr
) free(cfgRefererStr
);
443 s
= SyLoadStr(fd
); if (!s
) { free(destfile
); destfile
= NULL
; break; }
444 if (state
->postData
) free(state
->postData
);
445 state
->postData
= s
; s
= NULL
;
454 void DumpState (TSyState
*state
) {
455 char t0
[32], t1
[32], t2
[32];
456 SyLong2StrComma(t0
, state
->fileSize
);
457 SyLong2StrComma(t1
, state
->firstByte
);
458 SyLong2StrComma(t2
, state
->lastByte
);
459 printf("state dump\n==========\nURL: %s\nfile name: %s\n"
460 "file size: %s\nfirst byte: %s\nlast byte: %s\n",
461 oURL
, oOutFileName
, t0
, t1
, t2
466 static void ShowHelp (void) {
467 fprintf(stderr
, "%s",
468 "usage: syren [options] url\n"
471 " -h all show 'long' configuration options\n"
472 " -h <longopt> show 'long' configuration option description\n"
473 " -o filename output to specified file\n" \
474 " -O filename output to specified file (same as '-o')\n" \
475 " -r filename resume download (do not use saved state, use file)\n" \
476 " -s filename restore state\n"
477 " -S filename redownload file using state (ignore starting position, recreate)\n"
478 " -D filename dump state file and exit\n"
479 " -k keep state file\n"
480 " -P <str|@fn> make POST request\n"
481 " -c <str|@fn> send cookies\n"
482 " -C filename dump cookies\n"
483 " -i download server reply and stop\n"
484 " -I name_or_ip use specified net interface (\"-\" any)\n"
485 " -X do not use proxies\n"
487 " -T turn off terminal window title changing\n"
489 " -w generate .syrenrc file with the current config\n"
490 " -N don't write state file\n"
491 " -F force retry on any download error\n"
495 " -Q be VERY quiet\n"
501 static char *GetStrOpt (TSyPrintStr
*prs
, int argc
, char *argv
[], int *f
, const char *optName
,
502 char **value
, TSyBool allowEmpty
, TSyBool exclusive
) {
504 if (*f
>= argc
|| !argv
[*f
]) { SyMessage(prs
, SY_MSG_ERROR
, "no value for '%s'", optName
); return NULL
; }
505 if (exclusive
== SY_TRUE
&& *value
) { SyMessage(prs
, SY_MSG_ERROR
, "duplicate '%s'", optName
); return NULL
; }
506 /*printf("parsing '%s'\n", optName);*/
507 res
= SyStrNew(argv
[*f
], -1); (*f
)++;
508 if (!res
) { SyMessage(prs
, SY_MSG_ERROR
, "memory error"); return NULL
; }
509 /*printf("parsing '%s'; value='%s'\n", optName, res);*/
510 if (allowEmpty
!= SY_TRUE
&& !res
[0]) {
512 SyMessage(prs
, SY_MSG_ERROR
, "empty value for '%s'", optName
);
515 s
= res
; while (*s
) { if ((*s
) == '\\') *s
= '/'; s
++; }
516 if (*value
) free(*value
);
518 /*printf("parsed '%s'; value='%s'\n", optName, *value);*/
523 static TSyResult
GetStrOptWithAt (TSyPrintStr
*prs
, int argc
, char *argv
[], int *f
, const char *optName
,
524 char **value
, TSyBool allowEmpty
, TSyBool exclusive
) {
525 char *res
= GetStrOpt(prs
, argc
, argv
, f
, optName
, value
, allowEmpty
, exclusive
);
526 if (!res
) return SY_ERROR
;
528 res
= Sy_LoadWholeFile(res
+1);
529 if (!res
) { SyMessage(prs
, SY_MSG_ERROR
, "can't load data from '%s'", (*value
)+1); return SY_ERROR
; }
530 free(*value
); *value
= res
;
536 static TSyResult
ParseCmdLine (TSyPrintStr
*prs
, TSyCfg
*cfg
, int argc
, char *argv
[]) {
537 int f
, nomoreopt
= 0;
538 char *s
, *sopt
, soptN
[2];
541 f
= 1; while (f
< argc
) {
543 if (!s
) break; if (!(*s
)) continue;
544 if (!nomoreopt
&& *s
== '-') {
546 /* long option (config) */
547 if (!s
[3]) nomoreopt
= 1;
550 if (SyCfgSet(cfg
, s
) != SY_OK
) { SyMessage(prs
, SY_MSG_ERROR
, "invalid option: %s", (s
-2)); return SY_ERROR
; }
555 soptN
[0] = *sopt
; soptN
[1] = '\0';
558 printf("%s", SYREN_VDHEADER_STRING
);
560 case 'e': uEncodeDecode
= 1; break;
561 case 'E': uEncodeDecode
= -1; break;
562 case 'w': writeCfg
= SY_TRUE
; break;
563 case 'X': cfgUseProxyHTTP
= cfgUseProxyFTP
= SY_FALSE
; break;
565 case 'T': doRXVT
= SY_FALSE
; break;
567 case 'q': cfgQuiet
= SY_TRUE
; break;
568 case 'Q': cfgQuiet
= cfgNoOutput
= SY_TRUE
; break;
569 case 'i': replyOnly
= SY_TRUE
; break;
570 case 'N': oNoStateFile
= SY_TRUE
; break;
571 case 'F': oForceRetry
= SY_TRUE
; break;
572 case 'k': oCanRemoveState
= SY_FALSE
; break;
575 SyMessage(prs
, SY_MSG_ERROR
, "no argument for -I");
579 cfgIFace
= SyStrDup(argv
[f
++]);
580 /*SyStrTrim(cfgIFace);*/
581 /*if (!cfgIFace[0] || !strcmp(cfgIFace, "-")) { SyStrFree(cfgIFace); cfgIFace = NULL; }*/
583 case 'o': case 'O': case 'r':
584 if (*sopt
== 'r') oResume
= SY_TRUE
;
585 s
= GetStrOpt(prs
, argc
, argv
, &f
, soptN
, &oOutFileName
, SY_FALSE
, SY_TRUE
);
586 if (!s
) return SY_ERROR
;
587 oCanRename
= SY_FALSE
;
589 case 's': case 'S': case 'D':
590 if (*sopt
== 'S') oIgnoreStatePos
= SY_TRUE
;
591 else if (*sopt
== 'D') dumpState
= SY_TRUE
;
592 s
= GetStrOpt(prs
, argc
, argv
, &f
, soptN
, &oStateFileName
, SY_FALSE
, SY_TRUE
);
593 if (!s
) return SY_ERROR
;
594 oCanRename
= SY_FALSE
;
597 if (GetStrOptWithAt(prs
, argc
, argv
, &f
, soptN
, &oPostData
, SY_FALSE
, SY_TRUE
) != SY_OK
) return SY_ERROR
;
600 if (GetStrOptWithAt(prs
, argc
, argv
, &f
, soptN
, &oSendCookies
, SY_FALSE
, SY_TRUE
) != SY_OK
) return SY_ERROR
;
603 if (!GetStrOpt(prs
, argc
, argv
, &f
, s
, &oCookieDump
, SY_FALSE
, SY_TRUE
)) return SY_ERROR
;
606 fprintf(stderr
, "%s\n", SYREN_VERSION_DATETIME_STRING
);
607 key
= cfg
->opts
->first
;
608 if (f
< argc
&& !strcmp(argv
[f
], "all")) {
609 fprintf(stderr
, "long options:\n");
610 while (key
) { fprintf(stderr
, " --%s\n", key
->key
); key
= key
->next
; }
612 if (f
< argc
&& argv
[f
]) {
614 if (!strcasecmp(key
->key
, argv
[f
])) {
615 fprintf(stderr
, "--%s %s\n %s\n", key
->key
,
616 key
->uidata
==SY_CI_STRING
?"string":(key
->uidata
==SY_CI_INT
?"integer":(key
->uidata
==SY_CI_BOOL
?"boolean":"?")),
623 if (!key
) ShowHelp();
627 SyMessage(prs
, SY_MSG_ERROR
, "unknown option: %s", soptN
);
633 if (oURL
) { SyMessage(prs
, SY_MSG_ERROR
, "too many URLs"); return SY_ERROR
; }
634 oURL
= SyStrNew(s
, -1);
635 if (!oURL
) { SyMessage(prs
, SY_MSG_ERROR
, "memory error"); return SY_ERROR
; }
639 if (!writeCfg
&& !oStateFileName
) {
640 if (!oURL
|| !(*oURL
)) { SyMessage(prs
, SY_MSG_ERROR
, "URL?"); return SY_ERROR
; }
642 if (oResume
&& oStateFileName
) { SyMessage(prs
, SY_MSG_ERROR
, "'-r'/'-s' conflict"); return SY_ERROR
; }
647 TSyResult
SyDrvAddHeaders (TSyState
*state
, TSyHdrs
*hdrs
) {
648 char *s
= NULL
, *t
, *cs
;
649 TSyURL
*url
= state
->url
;
652 if (cfgRefererType
) {
653 switch (cfgRefererType
) {
654 case 1: s
= SyStrNew(oURL
, -1); break;
656 if (url
->proto
== SY_PROTO_FTP
&& url
->port
!= 80) sprintf(port
, ":%i", url
->port
); else port
[0] = '\0';
657 s
= SySPrintf("%s://%s%s%s%s%s%s", url
->protostr
, url
->host
, port
,
658 url
->dir
, url
->file
, url
->query
, url
->anchor
);
660 default: s
= SyStrNew(cfgRefererStr
, -1); break;
662 if (!s
) { SyMessage(state
->pfn
, SY_MSG_ERROR
, "memory error"); return SY_ERROR
; }
663 t
= SySPrintf("Referer: %s", s
); free(s
);
664 if (!t
) { SyMessage(state
->pfn
, SY_MSG_ERROR
, "memory error"); return SY_ERROR
; }
665 if (SyHdrAddLine(hdrs
, t
) != SY_OK
) { free(t
); SyMessage(state
->pfn
, SY_MSG_ERROR
, "hdr error"); return SY_ERROR
; }
668 if (cfgUAStr
&& *cfgUAStr
) {
669 s
= SySPrintf("User-Agent: %s", cfgUAStr
);
670 if (!s
) { SyMessage(state
->pfn
, SY_MSG_ERROR
, "memory error"); return SY_ERROR
; }
671 if (SyHdrAddLine(hdrs
, s
) != SY_OK
) { free(s
); SyMessage(state
->pfn
, SY_MSG_ERROR
, "hdr error"); return SY_ERROR
; }
674 if (oSendCookies
&& *oSendCookies
) {
677 t
= s
; while (*t
&& *t
!= '\n') t
++;
678 cs
= SyStrNew(s
, t
-s
); if (!cs
) return SY_ERROR
;
680 /*fprintf(stderr, "%s\n", cs);*/
682 if (SyHdrSetCookie(hdrs
, cs
) != SY_OK
) { free(cs
); return SY_ERROR
; }
684 free(cs
); if (!s
[0]) break;
685 s
= t
; if (*s
) s
++; /* don't skip latest 0 */
692 TSyResult
SyDrvGotHeaders (TSyState
*state
, TSyHdrs
*hdrs
) {
693 TSyResult res
= SY_OK
;
694 TSyKVList
*cc
; TSyKVListItem
*item
;
696 static char *crlf
= "\r\n";
698 if (oCookieDump
&& *oCookieDump
) {
699 cc
= SyHdrFindCookieList(hdrs
);
700 if (cc
&& cc
->count
) {
701 if (strcmp(oCookieDump
, "-")) {
702 fd
= SyOpenFile(oCookieDump
, SY_FMODE_WRITE
, SY_TRUE
);
703 if (fd
< 0) { SyMessage(state
->pfn
, SY_MSG_ERROR
, "can't dump cookies to '%s'", oCookieDump
); return SY_ERROR
; }
707 if (SyWriteFile(fd
, item
->value
, strlen(item
->value
)) != SY_OK
) { res
= SY_ERROR
; break; }
708 if (SyWriteFile(fd
, crlf
, strlen(crlf
)) != SY_OK
) { res
= SY_ERROR
; break; }
711 if (fd
) SyCloseFile(fd
);
718 TSyResult
SyDrvOpenFile (TSyState
*state
) {
722 if (replyOnly
!= SY_FALSE
) return SY_OK
;
725 if (!oOutFileName
|| !oOutFileName
[0]) {
726 if (oOutFileName
) free(oOutFileName
);
727 if (state
->httpFName
&& *state
->httpFName
) oOutFileName
= SyStrDup(state
->httpFName
);
728 else oOutFileName
= SyStrDup(state
->url
->file
);
729 if (!oOutFileName
|| !oOutFileName
[0]) {
730 if (oOutFileName
) free(oOutFileName
);
731 oOutFileName
= SyStrNew((cfgDefaultName
&&*cfgDefaultName
)?cfgDefaultName
:"index.html", -1);
733 if (!oOutFileName
) { SyMessage(state
->pfn
, SY_MSG_ERROR
, "memory error"); return SY_ERROR
; }
735 /* check for duplicates */
736 fd
= SyOpenFile(oOutFileName
, SY_FMODE_READ
, SY_FALSE
);
738 int f
= -1; s
= NULL
;
740 SyCloseFile(fd
); if (s
) free(s
);
741 s
= SySPrintf("%s.%i", oOutFileName
, f
);
742 if (!s
) { SyMessage(state
->pfn
, SY_MSG_ERROR
, "memory error"); return SY_ERROR
; }
743 fd
= SyOpenFile(s
, SY_FMODE_READ
, SY_FALSE
);
745 free(oOutFileName
); oOutFileName
= s
;
748 /* open/create file */
749 if (!oOutFileName
|| !oOutFileName
[0]) { SyMessage(state
->pfn
, SY_MSG_ERROR
, "invalid file name"); return SY_ERROR
; }
750 if (state
->firstByte
> 0) {
751 if (!strcmp(oOutFileName
, "-")) { SyMessage(state
->pfn
, SY_MSG_ERROR
, "can't resume stdout"); return SY_ERROR
; }
752 fd
= SyOpenFile(oOutFileName
, SY_FMODE_WRITE
, SY_FALSE
);
754 len
= SyFileSize(fd
);
755 if (len
< 0) { SyCloseFile(fd
); SyMessage(state
->pfn
, SY_MSG_ERROR
, "can't resume: file error"); return SY_ERROR
; }
756 if (len
< state
->firstByte
) { SyCloseFile(fd
); SyMessage(state
->pfn
, SY_MSG_ERROR
, "can't resume: corrupted file"); return SY_ERROR
; }
757 if (SySeekFile(fd
, state
->firstByte
) != SY_OK
) {
759 SyMessage(state
->pfn
, SY_MSG_ERROR
, "can't resume: file error");
764 if (!strcmp(oOutFileName
, "-")) fd
= 0;
766 SyDeleteFile(oOutFileName
);
767 fd
= SyOpenFile(oOutFileName
, SY_FMODE_WRITE
, SY_TRUE
);
770 if (fd
< 0) { SyMessage(state
->pfn
, SY_MSG_ERROR
, "can't open file %s", oOutFileName
); return SY_ERROR
; }
771 if (state
->firstByte
> 0) {
772 len
= SyFileSize(fd
);
773 if (len
< 0) { SyCloseFile(fd
); SyMessage(state
->pfn
, SY_MSG_ERROR
, "can't determine file size"); return SY_ERROR
; }
774 if (len
< state
->firstByte
) { SyCloseFile(fd
); SyMessage(state
->pfn
, SY_MSG_ERROR
, "can't resume: invalid file"); return SY_ERROR
; }
775 SyLong2StrComma(tmp
, state
->firstByte
);
776 SyMessage(state
->pfn
, SY_MSG_MSG
, "resuming %s at %s", oOutFileName
, tmp
);
777 } else SyMessage(state
->pfn
, SY_MSG_MSG
, "writing to %s", oOutFileName
);
779 if (!oStateFileName
|| !oStateFileName
[0]) {
780 if (oStateFileName
) free(oStateFileName
);
781 oStateFileName
= SySPrintf("%s.syren", oOutFileName
);
782 if (!oStateFileName
) SyMessage(state
->pfn
, SY_MSG_WARNING
, "can't create state file name");
783 else if (oNoStateFile
!= SY_TRUE
) SySaveState(oStateFileName
, state
);
789 TSyResult
SyDrvCloseFile (TSyState
*state
) {
790 if (replyOnly
!= SY_FALSE
) return SY_OK
;
791 if (state
->udatai
>= 0) {
792 SyMessage(state
->pfn
, SY_MSG_MSG
, "closing %s%s", oOutFileName
,
793 (state
->error
==SY_FALSE
?"":" (incomplete file)"));
794 if (state
->udatai
> 0) SyCloseFile(state
->udatai
);
796 if (oStateFileName
&& *oStateFileName
&& oNoStateFile
!= SY_TRUE
) SySaveState(oStateFileName
, state
);
802 TSyResult
SyDrvWriteFile (TSyState
*state
, void *buf
, int bufSize
) {
803 if (replyOnly
!= SY_FALSE
) return SY_OK
;
804 if (state
->udatai
>= 0) {
805 if (SyWriteFile(state
->udatai
, buf
, bufSize
) != SY_OK
) return SY_ERROR
;
807 if (oStateFileName
&& *oStateFileName
&& oNoStateFile
!= SY_TRUE
) {
808 if (cfgSaveInterval
>= 1.0 && SyGetTimeD()-lastSSave
>= cfgSaveInterval
) {
809 SySaveState(oStateFileName
, state
);
810 lastSSave
= SyGetTimeD();
812 if (state
->udatai
>= 0) fsync(state
->udatai
);
820 TSyResult
SyDrvNoResume (TSyState
*state
) {
821 if (replyOnly
!= SY_FALSE
) {
822 SyMessage(state
->pfn
, SY_MSG_WARNING
, "can't resume");
825 if (state
->udatai
>= 0) {
826 if (state
->udatai
) SyCloseFile(state
->udatai
);
827 SyDeleteFile(oOutFileName
);
828 state
->udatai
= SyOpenFile(oOutFileName
, SY_FMODE_WRITE
, SY_TRUE
);
829 if (state
->udatai
< 0) {
830 SyMessage(state
->pfn
, SY_MSG_WARNING
, "can't create file %s", oOutFileName
);
834 SyMessage(state
->pfn
, SY_MSG_WARNING
, "can't resume file %s, file truncated", oOutFileName
);
839 TSyResult
CheckResume (TSyState
*state
) {
841 if (!strcmp(oOutFileName
, "-")) { SyMessage(state
->pfn
, SY_MSG_ERROR
, "can't resume stdout"); return SY_ERROR
; }
842 fd
= SyOpenFile(oOutFileName
, SY_FMODE_WRITE
, SY_FALSE
);
843 if (fd
< 0) { SyMessage(state
->pfn
, SY_MSG_ERROR
, "resume: can't open file %s", oOutFileName
); return SY_ERROR
; }
844 len
= SyFileSize(fd
);
846 if (len
< 0) { SyMessage(state
->pfn
, SY_MSG_ERROR
, "can't determine file size for %s", oOutFileName
); return SY_ERROR
; }
847 state
->firstByte
= len
;
852 TSyState
*curState
= NULL
;
856 BOOL __stdcall
WinCtrlSignal (DWORD type
) {
858 if (curState
) curState
->breakNow
= SY_TRUE
;
864 void BreakSignal (int signo
) {
866 if (curState
) curState
->breakNow
= SY_TRUE
;
870 void SaveStateSignal (int signo
) {
871 /*fprintf(stderr, "\nsave state\n");*/
878 int main (int argc
, char *argv
[]) {
887 prs
.print
= &SyPrintStr
; prs
.udata
= NULL
;
891 if (t
&& (!strcmp(t
, "rxvt") || !strcmp(t
, "mrxvt") || !strcmp(t
, "xterm") || !strcmp(t
, "eterm"))) doRXVT
= SY_TRUE
;
895 if (!state
) { SyMessage(&prs
, SY_MSG_ERROR
, "memory error"); }
896 prs
.udata
= (void *)state
;
899 if (!cfg
) { SyMessage(&prs
, SY_MSG_ERROR
, "memory error"); }
901 SyCfgLoad(cfg
, "/etc/syren/.syrenrc", &prs
);
903 t
= calloc(1, 65536);
905 GetModuleFileName(NULL
, t
, 65535);
906 s
= t
; while (*s
) { if (*s
== '\\') *s
= '/'; s
++; }
907 s
= strrchr(t
, '/'); if (s
) *s
= '\0';
913 s
= SySPrintf("%s/.syren/.syrenrc", t
);
914 SyCfgLoad(cfg
, s
, &prs
);
916 s
= SySPrintf("%s/.syrenrc", t
);
917 SyCfgLoad(cfg
, s
, &prs
);
923 SyCfgLoad(cfg
, ".syrenrc", &prs
);
925 if (ParseCmdLine(&prs
, cfg
, argc
, argv
) != SY_OK
) goto done
;
927 SyMessage(&prs
, SY_MSG_MSG
, "%s", SYREN_VERSION_DATETIME_STRING
);
934 if (cfgBufferSize
< 1) { SyMessage(&prs
, SY_MSG_ERROR
, "buffer size too small"); goto done
; }
935 if (cfgTimeout
< 1) { SyMessage(&prs
, SY_MSG_ERROR
, "timeout too small"); goto done
; }
937 if (writeCfg
== SY_TRUE
) {
938 SyMessage(&prs
, SY_MSG_MSG
, "generating sample config");
939 SyCfgSave(cfg
, ".syrenrc", &prs
);
944 if (SySocketInit() != SY_OK
) goto done
;
949 state
->fnprog
= PrintProgress
;
950 state
->fnprephdrs
= SyDrvAddHeaders
;
951 state
->fngothdrs
= SyDrvGotHeaders
;
953 state
->fnopen
= SyDrvOpenFile
;
954 state
->fnclose
= SyDrvCloseFile
;
955 state
->fnwrite
= SyDrvWriteFile
;
956 state
->fnnoresume
= SyDrvNoResume
;
958 state
->postData
= oPostData
; oPostData
= NULL
;
960 if (!oStateFileName
) {
961 if (!oURL
) { SyMessage(&prs
, SY_MSG_ERROR
, "URL?"); goto done
; }
963 SyMessage(&prs
, SY_MSG_MSG
, "restoring state from file %s", oStateFileName
);
964 if (oStateFileName
&& *oStateFileName
) {
965 s
= SyLoadState(oStateFileName
, state
);
968 SyMessage(&prs
, SY_MSG_ERROR
, "can't restore state");
971 if (oOutFileName
) free(oOutFileName
);
974 if (dumpState
== SY_TRUE
) { DumpState(state
); goto done
; }
977 if (replyOnly
!= SY_FALSE
&& oResume
== SY_TRUE
&& CheckResume(state
) != SY_OK
) goto done
;
979 if (SyPrepare(state
, oURL
,
980 cfgUseProxyHTTP
?cfgProxyHTTP
:NULL
,
981 cfgUseProxyFTP
?cfgProxyFTP
:NULL
, cfgIFace
) != SY_OK
) { SyMessage(&prs
, SY_MSG_ERROR
, "SyPrepare() failed"); goto done
; }
983 if (uEncodeDecode
< 0 ||
984 (cfgDecodeFTPURL
== SY_TRUE
&& state
->url
->proto
== SY_PROTO_FTP
)) {
985 s
= SyURLDecode(state
->url
->dir
);
986 if (!s
) { SyMessage(&prs
, SY_MSG_ERROR
, "SyURLDecode() failed"); goto done
; }
987 SyStrFree(state
->url
->dir
); state
->url
->dir
= s
;
988 s
= SyURLDecode(state
->url
->file
);
989 if (!s
) { SyMessage(&prs
, SY_MSG_ERROR
, "SyURLDecode() failed"); goto done
; }
990 SyStrFree(state
->url
->file
); state
->url
->file
= s
;
991 } else if (uEncodeDecode
> 0) {
992 s
= SyURLEncode(state
->url
->dir
);
993 if (!s
) { SyMessage(&prs
, SY_MSG_ERROR
, "SyURLEncode() failed"); goto done
; }
994 SyStrFree(state
->url
->dir
); state
->url
->dir
= s
;
995 s
= SyURLEncode(state
->url
->file
);
996 if (!s
) { SyMessage(&prs
, SY_MSG_ERROR
, "SyURLEncode() failed"); goto done
; }
997 SyStrFree(state
->url
->file
); state
->url
->file
= s
;
1001 SetConsoleCtrlHandler(WinCtrlSignal
, TRUE
);
1003 signal(SIGHUP
, SaveStateSignal
);
1004 signal(SIGUSR1
, SaveStateSignal
);
1006 signal(SIGINT
, BreakSignal
);
1007 signal(SIGQUIT
, BreakSignal
);
1008 signal(SIGTERM
, BreakSignal
);
1011 retCnt
= cfgReconnectAttempts
;
1013 state
->bufferSize
= cfgBufferSize
;
1014 state
->ioTimeout
= cfgTimeout
;
1015 state
->maxRedirects
= cfgMaxRedirects
;
1019 res
= SyBegin(state
);
1020 if (replyOnly
!= SY_FALSE
) break;
1021 SyInitStats(state
, &stats
);
1022 if (res
== SY_OK
) res
= SyRun(state
);
1024 if (res
== SY_OK
) break;
1025 if (state
->interrupted
== SY_TRUE
) {
1026 if (oStateFileName
&& *oStateFileName
&& state
->status
== SY_STATUS_DOWNLOADING
&& oNoStateFile
!= SY_TRUE
) {
1027 SyMessage(&prs
, SY_MSG_WARNING
, "user break, saving state");
1028 SySaveState(oStateFileName
, state
);
1029 } else SyMessage(&prs
, SY_MSG_WARNING
, "user break");
1032 if (retCnt
< 1) break;
1033 if (state
->status
!= SY_STATUS_DOWNLOADING
&& oForceRetry
!= SY_TRUE
) break;
1034 SyMessage(&prs
, SY_MSG_ERROR
, "download failed");
1035 if (cfgReconnectDelay
> 0) {
1036 SyMessage(&prs
, SY_MSG_MSG
, "waiting %i seconds", cfgReconnectDelay
);
1037 SySleep(cfgReconnectDelay
);
1039 SyMessage(&prs
, SY_MSG_MSG
, "restarting");
1040 state
->firstByte
+= state
->currentByte
; /* start from here */
1041 oCanRename
= SY_FALSE
;
1046 SyMessage(&prs
, SY_MSG_MSG
, "download complete");
1047 if (oCanRemoveState
&& oStateFileName
&& *oStateFileName
) {
1048 if (SyDeleteFile(oStateFileName
) == SY_OK
) SyMessage(&prs
, SY_MSG_NOTICE
, "state file removed");
1050 } else SyMessage(&prs
, SY_MSG_ERROR
, "download failed");
1052 mainres
= (res
==SY_OK
)?0:1;
1055 SyFree(state
); SyCfgFree(cfg
);
1056 if (cfgDefaultName
) free(cfgDefaultName
);
1057 if (cfgIFace
) free(cfgIFace
);
1058 if (cfgProxyHTTP
) free(cfgProxyHTTP
);
1059 if (cfgProxyFTP
) free(cfgProxyFTP
);
1060 if (cfgRefererStr
) free(cfgRefererStr
);
1061 if (cfgUAStr
) free(cfgUAStr
);
1062 if (oOutFileName
) free(oOutFileName
);
1063 if (oStateFileName
) free(oStateFileName
);
1064 if (oURL
) free(oURL
);
1065 if (oPostData
) free(oPostData
);
1066 if (oSendCookies
) free(oSendCookies
);
1067 if (oCookieDump
) free(oCookieDump
);