SPD/ETA corrections
[syren.git] / src / syren.c
blob4028fc28580a30d7cbddf5edd8a25fb3dcf1bff4
1 /*
2 Syren -- a lightweight downloader for Linux/BSD/Win/MacOSX
3 inspired by Axel Copyright 2001-2002 Wilmer van der Gaast
4 version 0.0.6 (atomic alien)
5 coded by Ketmar // Avalon Group
7 This program is free software; you can redistribute it and/or modify
8 it under the terms of the GNU General Public License as published by
9 the Free Software Foundation; either version 2 of the License, or
10 (at your option) any later version.
12 This program is distributed in the hope that it will be useful,
13 but WITHOUT ANY WARRANTY; without even the implied warranty of
14 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
15 GNU General Public License for more details.
17 You should have received a copy of the GNU General Public License with
18 the Debian GNU/Linux distribution in file /usr/doc/copyright/GPL;
19 if not, write to the Free Software Foundation, Inc., 59 Temple Place,
20 Suite 330, Boston, MA 02111-1307 USA
23 Syren main program
25 #include "syren_common.h"
26 #include "syren_os.h"
27 #include "syren_str.h"
28 #include "syren_msg.h"
29 #include "syren_cfg.h"
30 #include "syren_hdrs.h"
31 #include "syren_tcp.h"
32 #include "syren_http.h"
33 #include "syren_ftp.h"
34 #include "syren_dloader.h"
36 #include "syren_scheme.h"
39 #ifndef WINDOZE
41 #include "signal.h"
43 /* uncomment this to use Linux TTY ioctls */
44 /*#define LINUX_TTY*/
46 /* uncomment this to use ANSI/VT100 TTY codes */
47 #define ANSI_TTY
49 /* uncomment this to use "DEC private" TTY codes (ESC [ ? n)*/
50 #define DEC_TTY
52 #endif
56 static TSyBool
57 cfgNoOutput = SY_FALSE,
58 cfgQuiet = SY_FALSE,
59 cfgNoIndicator = SY_FALSE,
60 cfgUseProxyHTTP = SY_TRUE,
61 cfgUseProxyFTP = SY_TRUE,
62 cfgDecodeFTPURL = SY_TRUE;
63 static int
64 cfgBufferSize = 16384,
65 cfgSaveInterval = 10,
66 cfgReconnectDelay = 10,
67 cfgReconnectAttempts = 666,
68 cfgTimeout = 60,
69 cfgMaxRedirects = 6,
70 cfgRefererType = 0;
71 static char
72 *cfgDefaultName = NULL,
73 *cfgIFace = NULL,
74 *cfgProxyHTTP = NULL,
75 *cfgProxyFTP = NULL,
76 *cfgRefererStr = NULL,
77 *cfgUAStr = NULL;
79 static char
80 *oOutFileName = NULL, /* -o/-O */
81 *oStateFileName = NULL, /* -s */
82 *oURL = NULL,
83 *oPostData = NULL,
84 *oSendCookies = NULL,
85 *oCookieDump = NULL;
86 static TSyBool oResume = SY_FALSE;
87 static TSyBool oCanRename = SY_TRUE;
88 static TSyBool oCanRemoveState = SY_TRUE;
89 static TSyBool oIgnoreStatePos = SY_FALSE;
90 static TSyBool oNoStateFile = SY_FALSE;
91 static TSyBool oForceRetry = SY_FALSE;
92 static TSyBool writeCfg = SY_FALSE;
93 static TSyBool dumpState = SY_FALSE;
94 static TSyBool replyOnly = SY_FALSE;
95 static int uEncodeDecode = 0;
96 #ifndef WINDOZE
97 static TSyBool doRXVT = SY_FALSE;
98 #endif
100 double lastSSave = 0;
103 #ifdef LINUX_TTY
104 static int GetTTYWidth (void) {
105 struct winsize ws;
106 if (!isatty(fileno(stderr))) return -1;
107 if (ioctl(fileno(stderr), TIOCGWINSZ, &ws)) return -1;
108 return ws.ws_col;
110 #endif
113 typedef struct {
114 int active;
115 double time;
116 int64_t bytesFrom;
117 int64_t bytesDone;
118 } TSyStatInfo;
121 typedef struct {
122 TSyStatInfo i[2];
123 double interval;
124 double sttime;
125 double bps;
126 double eta;
127 } TSyStats;
130 static void SyInitStats (TSyState *state, TSyStats *si) {
131 TSyStatInfo *i;
132 si->i[0].active = 0;
133 si->i[1].active = 0;
134 si->interval = 30; /* minute */
135 si->bps = 0.0;
136 si->eta = -1;
140 static void SyUpdateStats (TSyState *state, TSyStats *si, int64_t bytesDone, int64_t bytesTotal) {
141 TSyStatInfo *i0, *i1;
142 double ctime, ptime, t;
143 double w0, w1; /* weights */
144 double bps0, bps1;
146 i0 = &(si->i[0]);
147 i1 = &(si->i[1]);
149 ctime = SyGetTimeD();
151 if (!i1->active) {
152 /* nothing started */
153 i0->active = 0;
154 i1->active = 1;
155 i1->bytesFrom = 0;
156 i1->bytesDone = bytesDone;
157 i1->time = ctime;
158 si->sttime = ctime;
159 si->bps = 0.0;
160 si->eta = 0.0;
161 return;
164 /* update "bytes done" */
165 i1->bytesDone = bytesDone-(i1->bytesFrom);
167 ptime = ctime-(i1->time);
168 if (ptime >= si->interval) {
169 /* interval passed, move state */
170 *i0 = *i1;
171 i1->bytesFrom = bytesDone;
172 i1->bytesDone = 0;
173 i1->time = ctime;
174 i0->time = ptime; /* i0->time: time, taken by this stat */
177 /* calculate weights */
178 t = ctime-(i1->time);
179 w1 = i0->active?t/si->interval:1.0;
180 if (w1 > 1.0) w1 = 1.0;
181 w0 = 1.0-w1;
183 /* calculate bps */
184 bps0 = i0->active?(double)(i0->bytesDone)/i0->time:0.0;
185 bps1 = t>=0.5?(double)(i1->bytesDone)/t:0.0;
186 si->bps = (bps0*w0)+(bps1*w1);
187 /*fprintf(stderr, " %i:%i\n", (int)(w0*100), (int)(w1*100));
188 fprintf(stderr, " %i:%i:%i\n", (int)bps0, (int)bps1, (int)si->bps);
189 fprintf(stderr, " %i:%i:%i\n", (int)(bps0*w0), (int)(bps1*w1), (int)si->bps);*/
191 /* calculate eta */
192 if (bytesTotal > 0 && si->bps >= 1) {
193 si->eta = (double)(bytesTotal-bytesDone)/(si->bps);
194 } else si->eta = 0.0;
198 static TSyStats stats;
201 static void PrintProgress (TSyState *state, double timePassed, int64_t bytesDone, int64_t bytesTotal,
202 TSyBool done) {
203 char tmp0[128], tmp1[128], tmp2[128], tmp3[128], tmp4[128], fmt[128], str[1024];
204 int len; int64_t fb = state->firstByte;
205 if (cfgNoIndicator) return;
206 SyUpdateStats(state, &stats, bytesDone, bytesTotal);
207 fprintf(stderr, "\r");
208 #ifdef DEC_TTY
209 fputs("\x1b[?7l", stderr);
210 #endif
211 if (bytesTotal > 0) {
212 len = SyLong2StrComma(tmp1, (bytesTotal+fb));
213 SyLong2StrComma(tmp0, (bytesDone+fb));
214 sprintf(fmt, "[%%%is/%%s] ", len);
215 sprintf(tmp2, fmt, tmp0, tmp1); /*!*/
216 sprintf(tmp3, "[%3i%%] ", (int)((double)100*(bytesDone+fb)/(bytesTotal+fb))); /*!*/
217 /* tmp3: res */
218 SySize2Str(tmp0, (int64_t)stats.bps);
219 sprintf(tmp4, "[SPD:%s] ", tmp0); /*!*/
220 SyTime2Str(tmp0, (int)(SyGetTimeD()-stats.sttime));
221 SyTime2Str(tmp1, (int)stats.eta);
222 sprintf(fmt, "TIME:%s ETA:%s", tmp0, tmp1); /*!*/
223 fprintf(stderr, "%s%s%s%s", tmp2, tmp3, tmp4, fmt);
224 sprintf(str, "%s%s%s/%s", tmp2, tmp3, tmp0, tmp1);
225 } else {
226 SyLong2StrComma(tmp0, bytesDone+fb);
227 SyTime2Str(tmp1, (int)(SyGetTimeD()-stats.sttime));
228 SySize2Str(fmt, (int64_t)stats.bps);
229 sprintf(str, "[%s] [SPD:%s] TIME:%s", tmp0, fmt, tmp1);
230 fprintf(stderr, "%s", str);
232 #ifndef WINDOZE
233 if (isatty(fileno(stdout)) && doRXVT == SY_TRUE) {
234 fprintf(stdout, "\x1b]2;%s\7", str);
235 fflush(stdout);
237 #endif
238 #ifdef ANSI_TTY
239 fprintf(stderr, "\x1b[K");
240 #endif
241 #ifdef DEC_TTY
242 fputs("\x1b[?7h", stderr);
243 #endif
244 if (done == SY_TRUE) fputs("\n", stderr);
245 fflush(stderr);
249 static void SyPrintStr (void *udata, TSyMsgType msgtype, const char *msg) {
250 #ifdef LINUX_TTY
251 int wdt;
252 #endif
253 static const char *ltr = "NMWE?";
254 if (cfgNoOutput) return;
255 if (cfgQuiet && msgtype < SY_MSG_MSG) return;
256 fflush(stderr);
257 #ifndef LINUX_TTY
258 #ifdef DEC_TTY
259 fputs("\x1b[?7l", stderr);
260 #endif
261 #else
262 wdt = GetTTYWidth();
263 if (wdt < 0) wdt = strlen(msg)+16;
264 if (wdt < 2) return;
265 #endif
266 if (msgtype < SY_MSG_NOTICE || msgtype > SY_MSG_ERROR) msgtype = SY_MSG_ERROR+1;
267 fputc(ltr[msgtype], stderr);
268 #ifdef LINUX_TTY
269 if (wdt > 4) {
270 fputs(": ", stderr);
271 wdt -= 4;
272 if (wdt >= strlen(msg)) fputs(msg, stderr);
273 else while (wdt--) fputc(*(msg++), stderr);
275 #else
276 fputs(": ", stderr);
277 fputs(msg, stderr);
278 #endif
279 #ifdef ANSI_TTY
280 fputs("\x1b[K", stderr);
281 #endif
282 #ifndef LINUX_TTY
283 #ifdef DEC_TTY
284 fputs("\x1b[?7h", stderr);
285 #endif
286 #endif
287 fputs("\n", stderr);
288 fflush(stderr);
289 #ifndef WINDOZE
290 if (isatty(fileno(stdout)) && doRXVT == SY_TRUE && msgtype > SY_MSG_NOTICE) {
291 fprintf(stdout, "\x1b]2;%s\7", msg);
292 fflush(stdout);
294 #endif
299 static void SyDefaultCfg (TSyCfg *cfg) {
300 SyCfgAddIntKey(cfg, "reconnect_delay", "number of seconds to wait before reconnecting to server", &cfgReconnectDelay);
301 SyCfgAddIntKey(cfg, "reconnect_attempts", "number of reconnection attempts (0: do not reconnect)", &cfgReconnectAttempts);
302 SyCfgAddIntKey(cfg, "connection_timeout", "i/o timeout in seconds", &cfgTimeout);
303 SyCfgAddIntKey(cfg, "save_state_interval", "save state every x seconds (0: disable)", &cfgSaveInterval);
304 cfgDefaultName = SyStrNew("index.html", -1);
305 SyCfgAddStrKey(cfg, "default_file_name", "default file name for http", &cfgDefaultName);
306 SyCfgAddIntKey(cfg, "max_redirects", "maximum number of redirects/symlinks", &cfgMaxRedirects);
307 SyCfgAddIntKey(cfg, "buffer_size", "maximum amount of bytes to read at once", &cfgBufferSize);
308 SyCfgAddBoolKey(cfg, "quiet", "tan: do not show notices", &cfgQuiet);
309 SyCfgAddBoolKey(cfg, "no_output", "tan: disable any output", &cfgNoOutput);
310 SyCfgAddBoolKey(cfg, "no_indicator", "tan: hide download indicator", &cfgNoIndicator);
312 cfgIFace = SyStrNew("", -1);
313 SyCfgAddStrKey(cfg, "interface", "net interface to use for connection (name or IP)", &cfgIFace);
315 cfgProxyHTTP = SyStrNew(getenv("http_proxy"), -1);
316 SyCfgAddStrKey(cfg, "http_proxy", "proxy for http connections", &cfgProxyHTTP);
317 cfgProxyFTP = SyStrNew(getenv("ftp_proxy"), -1);
318 SyCfgAddStrKey(cfg, "ftp_proxy", "proxy for ftp connections", &cfgProxyFTP);
319 SyCfgAddBoolKey(cfg, "use_http_proxy", "use http proxy (if specified)?", &cfgUseProxyHTTP);
320 SyCfgAddBoolKey(cfg, "use_ftp_proxy", "use ftp proxy (if specified)?", &cfgUseProxyFTP);
322 SyCfgAddIntKey(cfg, "referer_type", "set referer type (0:none; 1:orig URL; 2:follow URL; 3:user)", &cfgRefererType);
323 cfgRefererStr = SyStrNew(NULL, -1);
324 SyCfgAddStrKey(cfg, "referer_str", "referer for mode 3", &cfgRefererStr);
325 SyCfgAddBoolKey(cfg, "decode_ftp_url", "decode URLs for FTP", &cfgDecodeFTPURL);
327 cfgUAStr = SyStrNew(SYREN_DEFAULT_USER_AGENT, -1);
328 SyCfgAddStrKey(cfg, "ua_str", "user agent (set to empty string if no UA should be sent)", &cfgUAStr);
333 TSyResult SySaveInt (int fd, int64_t v, int size) {
334 uint8_t b;
336 if (fd < 0 || size < 1 || size > 8) return SY_ERROR;
337 while (size--) {
338 b = (v>>(size*8))&0xff;
339 if (SyWriteFile(fd, &b, 1) != SY_OK) return SY_ERROR;
341 return SY_OK;
344 TSyResult SyLoadInt (int fd, int64_t *v, int size) {
345 uint8_t b; int64_t res = 0;
347 if (v) *v = 0;
348 if (!fd || size < 1 || size > 8) return SY_ERROR;
349 while (size--) {
350 if (SyReadFile(fd, &b, 1) != SY_OK) return SY_ERROR;
351 res = (res<<8)+b;
353 if (v) *v = res;
354 return SY_OK;
358 TSyResult SySaveStr (int fd, const char *v) {
359 int len = 0;
361 if (!fd) return SY_ERROR;
362 if (v) len = strlen(v);
363 if (len > 65535) return SY_ERROR;
364 if (SySaveInt(fd, len, 2) != SY_OK) return SY_ERROR;
365 if (len) { if (SyWriteFile(fd, v, len) != SY_OK) return SY_ERROR; }
366 return SY_OK;
370 char *SyLoadStr (int fd) {
371 char *v;
372 int64_t len = 0;
374 if (!fd) return NULL;
375 if (SyLoadInt(fd, &len, 2) != SY_OK) return NULL;
376 if (!len) return SyStrNewEmpty();
377 v = calloc(1, len+8); if (!v) return NULL;
378 if (SyReadFile(fd, v, len) != SY_OK) { free(v); return NULL; }
379 return v;
383 TSyResult SySaveState (const char *destfile, TSyState *state) {
384 TSyResult res = SY_ERROR;
385 TSyURL *url;
386 char *s, *t;
387 int fd;
389 if (!state) return SY_ERROR;
390 url = state->url;
391 s = SySPrintf("%s://%s:%s@%s:%i%s%s%s%s",
392 url->protostr, url->user, url->pass, url->host, url->port,
393 url->dir, url->file, url->query, url->anchor);
394 if (!s) return SY_ERROR;
395 fd = SyOpenFile(destfile, SY_FMODE_WRITE, SY_TRUE);
396 if (fd >= 0) {
397 while (1) {
398 if (SySaveStr(fd, "SYREN STATE FILE v5") != SY_OK) break;
399 if (SySaveInt(fd, state->fileSize, 8) != SY_OK) break;
400 if (SySaveInt(fd, state->currentByte+state->firstByte, 8) != SY_OK) break;
401 if (SySaveInt(fd, state->lastByte, 8) != SY_OK) break;
402 if (SySaveStr(fd, s) != SY_OK) break;
403 t = strrchr(oOutFileName, '/'); if (!t) t = oOutFileName; else t++;
404 if (SySaveStr(fd, t) != SY_OK) break;
405 if (SySaveInt(fd, cfgRefererType, 1) != SY_OK) break;
406 if (SySaveStr(fd, cfgRefererStr) != SY_OK) break;
407 if (SySaveStr(fd, oPostData) != SY_OK) break;
408 res = SY_OK; break;
410 SyCloseFile(fd);
412 if (res != SY_OK) SyDeleteFile(destfile);
413 free(s);
414 return res;
418 /* return destfile */
419 char *SyLoadState (const char *statefile, TSyState *state) {
420 char *destfile = NULL, *s;
421 int fd; int64_t i;
423 fd = SyOpenFile(statefile, SY_FMODE_READ, SY_FALSE);
424 if (fd < 0) return NULL;
425 while (1) {
426 s = SyLoadStr(fd); if (!s) break;
427 if (strcmp(s, "SYREN STATE FILE v5")) break;
428 if (SyLoadInt(fd, &state->fileSize, 8) != SY_OK) break;
429 if (SyLoadInt(fd, &state->firstByte, 8) != SY_OK) break;
430 if (oIgnoreStatePos != SY_FALSE) state->firstByte = 0;
431 if (state->firstByte < 0 || (state->fileSize >= 0 && state->firstByte > state->fileSize)) break;
432 if (SyLoadInt(fd, &state->lastByte, 8) != SY_OK) break;
433 if (state->lastByte < -1 || (state->lastByte >= 0 && state->firstByte > state->lastByte)) break;
434 free(s); s = SyLoadStr(fd); if (!s) break;
435 if (oURL) free(oURL); oURL = s;
436 s = SyLoadStr(fd); if (!s) break;
437 destfile = s; s = NULL;
438 if (SyLoadInt(fd, &i, 1) != SY_OK) break;
439 cfgRefererType = (int)i;
440 s = SyLoadStr(fd); if (!s) break;
441 if (cfgRefererStr) free(cfgRefererStr);
442 cfgRefererStr = s;
443 s = SyLoadStr(fd); if (!s) { free(destfile); destfile = NULL; break; }
444 if (state->postData) free(state->postData);
445 state->postData = s; s = NULL;
446 break;
448 SyCloseFile(fd);
449 if (s) free(s);
450 return destfile;
454 void DumpState (TSyState *state) {
455 char t0[32], t1[32], t2[32];
456 SyLong2StrComma(t0, state->fileSize);
457 SyLong2StrComma(t1, state->firstByte);
458 SyLong2StrComma(t2, state->lastByte);
459 printf("state dump\n==========\nURL: %s\nfile name: %s\n"
460 "file size: %s\nfirst byte: %s\nlast byte: %s\n",
461 oURL, oOutFileName, t0, t1, t2
466 static void ShowHelp (void) {
467 fprintf(stderr, "%s",
468 "usage: syren [options] url\n"
469 "options:\n" \
470 " -h this help\n"
471 " -h all show 'long' configuration options\n"
472 " -h <longopt> show 'long' configuration option description\n"
473 " -o filename output to specified file\n" \
474 " -O filename output to specified file (same as '-o')\n" \
475 " -r filename resume download (do not use saved state, use file)\n" \
476 " -s filename restore state\n"
477 " -S filename redownload file using state (ignore starting position, recreate)\n"
478 " -D filename dump state file and exit\n"
479 " -k keep state file\n"
480 " -P <str|@fn> make POST request\n"
481 " -c <str|@fn> send cookies\n"
482 " -C filename dump cookies\n"
483 " -i download server reply and stop\n"
484 " -I name_or_ip use specified net interface (\"-\" any)\n"
485 " -X do not use proxies\n"
486 #ifndef WINDOZE
487 " -T turn off terminal window title changing\n"
488 #endif
489 " -w generate .syrenrc file with the current config\n"
490 " -N don't write state file\n"
491 " -F force retry on any download error\n"
492 " -e encode URL\n"
493 " -E decode URL\n"
494 " -q be quiet\n"
495 " -Q be VERY quiet\n"
496 " -V version\n"
501 static char *GetStrOpt (TSyPrintStr *prs, int argc, char *argv[], int *f, const char *optName,
502 char **value, TSyBool allowEmpty, TSyBool exclusive) {
503 char *res, *s;
504 if (*f >= argc || !argv[*f]) { SyMessage(prs, SY_MSG_ERROR, "no value for '%s'", optName); return NULL; }
505 if (exclusive == SY_TRUE && *value) { SyMessage(prs, SY_MSG_ERROR, "duplicate '%s'", optName); return NULL; }
506 /*printf("parsing '%s'\n", optName);*/
507 res = SyStrNew(argv[*f], -1); (*f)++;
508 if (!res) { SyMessage(prs, SY_MSG_ERROR, "memory error"); return NULL; }
509 /*printf("parsing '%s'; value='%s'\n", optName, res);*/
510 if (allowEmpty != SY_TRUE && !res[0]) {
511 free(res);
512 SyMessage(prs, SY_MSG_ERROR, "empty value for '%s'", optName);
513 return NULL;
515 s = res; while (*s) { if ((*s) == '\\') *s = '/'; s++; }
516 if (*value) free(*value);
517 *value = res;
518 /*printf("parsed '%s'; value='%s'\n", optName, *value);*/
519 return res;
523 static TSyResult GetStrOptWithAt (TSyPrintStr *prs, int argc, char *argv[], int *f, const char *optName,
524 char **value, TSyBool allowEmpty, TSyBool exclusive) {
525 char *res = GetStrOpt(prs, argc, argv, f, optName, value, allowEmpty, exclusive);
526 if (!res) return SY_ERROR;
527 if (*res == '@') {
528 res = Sy_LoadWholeFile(res+1);
529 if (!res) { SyMessage(prs, SY_MSG_ERROR, "can't load data from '%s'", (*value)+1); return SY_ERROR; }
530 free(*value); *value = res;
532 return SY_OK;
536 static TSyResult ParseCmdLine (TSyPrintStr *prs, TSyCfg *cfg, int argc, char *argv[]) {
537 int f, nomoreopt = 0;
538 char *s, *sopt, soptN[2];
539 TSyKVListItem *key;
541 f = 1; while (f < argc) {
542 s = argv[f++];
543 if (!s) break; if (!(*s)) continue;
544 if (!nomoreopt && *s == '-') {
545 if (s[1] == '-') {
546 /* long option (config) */
547 if (!s[3]) nomoreopt = 1;
548 else {
549 s += 2;
550 if (SyCfgSet(cfg, s) != SY_OK) { SyMessage(prs, SY_MSG_ERROR, "invalid option: %s", (s-2)); return SY_ERROR; }
552 } else {
553 sopt = s;
554 while (*(++sopt)) {
555 soptN[0] = *sopt; soptN[1] = '\0';
556 switch (*sopt) {
557 case 'V':
558 printf("%s", SYREN_VDHEADER_STRING);
559 return SY_ERROR;
560 case 'e': uEncodeDecode = 1; break;
561 case 'E': uEncodeDecode = -1; break;
562 case 'w': writeCfg = SY_TRUE; break;
563 case 'X': cfgUseProxyHTTP = cfgUseProxyFTP = SY_FALSE; break;
564 #ifndef WINDOZE
565 case 'T': doRXVT = SY_FALSE; break;
566 #endif
567 case 'q': cfgQuiet = SY_TRUE; break;
568 case 'Q': cfgQuiet = cfgNoOutput = SY_TRUE; break;
569 case 'i': replyOnly = SY_TRUE; break;
570 case 'N': oNoStateFile = SY_TRUE; break;
571 case 'F': oForceRetry = SY_TRUE; break;
572 case 'k': oCanRemoveState = SY_FALSE; break;
573 case 'I':
574 if (!argv[f]) {
575 SyMessage(prs, SY_MSG_ERROR, "no argument for -I");
576 return SY_ERROR;
578 SyStrFree(cfgIFace);
579 cfgIFace = SyStrDup(argv[f++]);
580 /*SyStrTrim(cfgIFace);*/
581 /*if (!cfgIFace[0] || !strcmp(cfgIFace, "-")) { SyStrFree(cfgIFace); cfgIFace = NULL; }*/
582 break;
583 case 'o': case 'O': case 'r':
584 if (*sopt == 'r') oResume = SY_TRUE;
585 s = GetStrOpt(prs, argc, argv, &f, soptN, &oOutFileName, SY_FALSE, SY_TRUE);
586 if (!s) return SY_ERROR;
587 oCanRename = SY_FALSE;
588 break;
589 case 's': case 'S': case 'D':
590 if (*sopt == 'S') oIgnoreStatePos = SY_TRUE;
591 else if (*sopt == 'D') dumpState = SY_TRUE;
592 s = GetStrOpt(prs, argc, argv, &f, soptN, &oStateFileName, SY_FALSE, SY_TRUE);
593 if (!s) return SY_ERROR;
594 oCanRename = SY_FALSE;
595 break;
596 case 'P':
597 if (GetStrOptWithAt(prs, argc, argv, &f, soptN, &oPostData, SY_FALSE, SY_TRUE) != SY_OK) return SY_ERROR;
598 break;
599 case 'c':
600 if (GetStrOptWithAt(prs, argc, argv, &f, soptN, &oSendCookies, SY_FALSE, SY_TRUE) != SY_OK) return SY_ERROR;
601 break;
602 case 'C':
603 if (!GetStrOpt(prs, argc, argv, &f, s, &oCookieDump, SY_FALSE, SY_TRUE)) return SY_ERROR;
604 break;
605 case 'h':
606 fprintf(stderr, "%s\n", SYREN_VERSION_DATETIME_STRING);
607 key = cfg->opts->first;
608 if (f < argc && !strcmp(argv[f], "all")) {
609 fprintf(stderr, "long options:\n");
610 while (key) { fprintf(stderr, " --%s\n", key->key); key = key->next; }
611 } else {
612 if (f < argc && argv[f]) {
613 while (key) {
614 if (!strcasecmp(key->key, argv[f])) {
615 fprintf(stderr, "--%s %s\n %s\n", key->key,
616 key->uidata==SY_CI_STRING?"string":(key->uidata==SY_CI_INT?"integer":(key->uidata==SY_CI_BOOL?"boolean":"?")),
617 key->ustr);
618 break;
620 key = key->next;
622 } else key = NULL;
623 if (!key) ShowHelp();
625 return SY_ERROR;
626 default:
627 SyMessage(prs, SY_MSG_ERROR, "unknown option: %s", soptN);
628 return SY_ERROR;
629 } /* switch */
630 } /* while */
631 } /* if */
632 } else {
633 if (oURL) { SyMessage(prs, SY_MSG_ERROR, "too many URLs"); return SY_ERROR; }
634 oURL = SyStrNew(s, -1);
635 if (!oURL) { SyMessage(prs, SY_MSG_ERROR, "memory error"); return SY_ERROR; }
636 SyStrTrim(oURL);
639 if (!writeCfg && !oStateFileName) {
640 if (!oURL || !(*oURL)) { SyMessage(prs, SY_MSG_ERROR, "URL?"); return SY_ERROR; }
642 if (oResume && oStateFileName) { SyMessage(prs, SY_MSG_ERROR, "'-r'/'-s' conflict"); return SY_ERROR; }
643 return SY_OK;
647 TSyResult SyDrvAddHeaders (TSyState *state, TSyHdrs *hdrs) {
648 char *s = NULL, *t, *cs;
649 TSyURL *url = state->url;
650 char port[32];
652 if (cfgRefererType) {
653 switch (cfgRefererType) {
654 case 1: s = SyStrNew(oURL, -1); break;
655 case 2:
656 if (url->proto == SY_PROTO_FTP && url->port != 80) sprintf(port, ":%i", url->port); else port[0] = '\0';
657 s = SySPrintf("%s://%s%s%s%s%s%s", url->protostr, url->host, port,
658 url->dir, url->file, url->query, url->anchor);
659 break;
660 default: s = SyStrNew(cfgRefererStr, -1); break;
662 if (!s) { SyMessage(state->pfn, SY_MSG_ERROR, "memory error"); return SY_ERROR; }
663 t = SySPrintf("Referer: %s", s); free(s);
664 if (!t) { SyMessage(state->pfn, SY_MSG_ERROR, "memory error"); return SY_ERROR; }
665 if (SyHdrAddLine(hdrs, t) != SY_OK) { free(t); SyMessage(state->pfn, SY_MSG_ERROR, "hdr error"); return SY_ERROR; }
666 free(t);
668 if (cfgUAStr && *cfgUAStr) {
669 s = SySPrintf("User-Agent: %s", cfgUAStr);
670 if (!s) { SyMessage(state->pfn, SY_MSG_ERROR, "memory error"); return SY_ERROR; }
671 if (SyHdrAddLine(hdrs, s) != SY_OK) { free(s); SyMessage(state->pfn, SY_MSG_ERROR, "hdr error"); return SY_ERROR; }
672 free(s);
674 if (oSendCookies && *oSendCookies) {
675 s = oSendCookies;
676 while (*s) {
677 t = s; while (*t && *t != '\n') t++;
678 cs = SyStrNew(s, t-s); if (!cs) return SY_ERROR;
679 SyStrTrim(cs);
680 /*fprintf(stderr, "%s\n", cs);*/
681 if (*cs) {
682 if (SyHdrSetCookie(hdrs, cs) != SY_OK) { free(cs); return SY_ERROR; }
684 free(cs); if (!s[0]) break;
685 s = t; if (*s) s++; /* don't skip latest 0 */
688 return SY_OK;
692 TSyResult SyDrvGotHeaders (TSyState *state, TSyHdrs *hdrs) {
693 TSyResult res = SY_OK;
694 TSyKVList *cc; TSyKVListItem *item;
695 int fd;
696 static char *crlf = "\r\n";
698 if (oCookieDump && *oCookieDump) {
699 cc = SyHdrFindCookieList(hdrs);
700 if (cc && cc->count) {
701 if (strcmp(oCookieDump, "-")) {
702 fd = SyOpenFile(oCookieDump, SY_FMODE_WRITE, SY_TRUE);
703 if (fd < 0) { SyMessage(state->pfn, SY_MSG_ERROR, "can't dump cookies to '%s'", oCookieDump); return SY_ERROR; }
704 } else fd = 0;
705 item = cc->first;
706 while (item) {
707 if (SyWriteFile(fd, item->value, strlen(item->value)) != SY_OK) { res = SY_ERROR; break; }
708 if (SyWriteFile(fd, crlf, strlen(crlf)) != SY_OK) { res = SY_ERROR; break; }
709 item = item->next;
711 if (fd) SyCloseFile(fd);
714 return res;
718 TSyResult SyDrvOpenFile (TSyState *state) {
719 int fd; int64_t len;
720 char *s, tmp[32];
722 if (replyOnly != SY_FALSE) return SY_OK;
724 if (oCanRename) {
725 if (!oOutFileName || !oOutFileName[0]) {
726 if (oOutFileName) free(oOutFileName);
727 if (state->httpFName && *state->httpFName) oOutFileName = SyStrDup(state->httpFName);
728 else oOutFileName = SyStrDup(state->url->file);
729 if (!oOutFileName || !oOutFileName[0]) {
730 if (oOutFileName) free(oOutFileName);
731 oOutFileName = SyStrNew((cfgDefaultName&&*cfgDefaultName)?cfgDefaultName:"index.html", -1);
733 if (!oOutFileName) { SyMessage(state->pfn, SY_MSG_ERROR, "memory error"); return SY_ERROR; }
735 /* check for duplicates */
736 fd = SyOpenFile(oOutFileName, SY_FMODE_READ, SY_FALSE);
737 if (fd >= 0) {
738 int f = -1; s = NULL;
739 do { f++;
740 SyCloseFile(fd); if (s) free(s);
741 s = SySPrintf("%s.%i", oOutFileName, f);
742 if (!s) { SyMessage(state->pfn, SY_MSG_ERROR, "memory error"); return SY_ERROR; }
743 fd = SyOpenFile(s, SY_FMODE_READ, SY_FALSE);
744 } while (fd >= 0);
745 free(oOutFileName); oOutFileName = s;
748 /* open/create file */
749 if (!oOutFileName || !oOutFileName[0]) { SyMessage(state->pfn, SY_MSG_ERROR, "invalid file name"); return SY_ERROR; }
750 if (state->firstByte > 0) {
751 if (!strcmp(oOutFileName, "-")) { SyMessage(state->pfn, SY_MSG_ERROR, "can't resume stdout"); return SY_ERROR; }
752 fd = SyOpenFile(oOutFileName, SY_FMODE_WRITE, SY_FALSE);
753 if (fd >= 0) {
754 len = SyFileSize(fd);
755 if (len < 0) { SyCloseFile(fd); SyMessage(state->pfn, SY_MSG_ERROR, "can't resume: file error"); return SY_ERROR; }
756 if (len < state->firstByte) { SyCloseFile(fd); SyMessage(state->pfn, SY_MSG_ERROR, "can't resume: corrupted file"); return SY_ERROR; }
757 if (SySeekFile(fd, state->firstByte) != SY_OK) {
758 SyCloseFile(fd);
759 SyMessage(state->pfn, SY_MSG_ERROR, "can't resume: file error");
760 return SY_ERROR;
763 } else {
764 if (!strcmp(oOutFileName, "-")) fd = 0;
765 else {
766 SyDeleteFile(oOutFileName);
767 fd = SyOpenFile(oOutFileName, SY_FMODE_WRITE, SY_TRUE);
770 if (fd < 0) { SyMessage(state->pfn, SY_MSG_ERROR, "can't open file %s", oOutFileName); return SY_ERROR; }
771 if (state->firstByte > 0) {
772 len = SyFileSize(fd);
773 if (len < 0) { SyCloseFile(fd); SyMessage(state->pfn, SY_MSG_ERROR, "can't determine file size"); return SY_ERROR; }
774 if (len < state->firstByte) { SyCloseFile(fd); SyMessage(state->pfn, SY_MSG_ERROR, "can't resume: invalid file"); return SY_ERROR; }
775 SyLong2StrComma(tmp, state->firstByte);
776 SyMessage(state->pfn, SY_MSG_MSG, "resuming %s at %s", oOutFileName, tmp);
777 } else SyMessage(state->pfn, SY_MSG_MSG, "writing to %s", oOutFileName);
778 state->udatai = fd;
779 if (!oStateFileName || !oStateFileName[0]) {
780 if (oStateFileName) free(oStateFileName);
781 oStateFileName = SySPrintf("%s.syren", oOutFileName);
782 if (!oStateFileName) SyMessage(state->pfn, SY_MSG_WARNING, "can't create state file name");
783 else if (oNoStateFile != SY_TRUE) SySaveState(oStateFileName, state);
785 return SY_OK;
789 TSyResult SyDrvCloseFile (TSyState *state) {
790 if (replyOnly != SY_FALSE) return SY_OK;
791 if (state->udatai >= 0) {
792 SyMessage(state->pfn, SY_MSG_MSG, "closing %s%s", oOutFileName,
793 (state->error==SY_FALSE?"":" (incomplete file)"));
794 if (state->udatai > 0) SyCloseFile(state->udatai);
795 state->udatai = -1;
796 if (oStateFileName && *oStateFileName && oNoStateFile != SY_TRUE) SySaveState(oStateFileName, state);
798 return SY_OK;
802 TSyResult SyDrvWriteFile (TSyState *state, void *buf, int bufSize) {
803 if (replyOnly != SY_FALSE) return SY_OK;
804 if (state->udatai >= 0) {
805 if (SyWriteFile(state->udatai, buf, bufSize) != SY_OK) return SY_ERROR;
807 if (oStateFileName && *oStateFileName && oNoStateFile != SY_TRUE) {
808 if (cfgSaveInterval >= 1.0 && SyGetTimeD()-lastSSave >= cfgSaveInterval) {
809 SySaveState(oStateFileName, state);
810 lastSSave = SyGetTimeD();
811 #ifndef WINDOZE
812 if (state->udatai >= 0) fsync(state->udatai);
813 #endif
816 return SY_OK;
820 TSyResult SyDrvNoResume (TSyState *state) {
821 if (replyOnly != SY_FALSE) {
822 SyMessage(state->pfn, SY_MSG_WARNING, "can't resume");
823 return SY_OK;
825 if (state->udatai >= 0) {
826 if (state->udatai) SyCloseFile(state->udatai);
827 SyDeleteFile(oOutFileName);
828 state->udatai = SyOpenFile(oOutFileName, SY_FMODE_WRITE, SY_TRUE);
829 if (state->udatai < 0) {
830 SyMessage(state->pfn, SY_MSG_WARNING, "can't create file %s", oOutFileName);
831 return SY_ERROR;
834 SyMessage(state->pfn, SY_MSG_WARNING, "can't resume file %s, file truncated", oOutFileName);
835 return SY_OK;
839 TSyResult CheckResume (TSyState *state) {
840 int fd; int64_t len;
841 if (!strcmp(oOutFileName, "-")) { SyMessage(state->pfn, SY_MSG_ERROR, "can't resume stdout"); return SY_ERROR; }
842 fd = SyOpenFile(oOutFileName, SY_FMODE_WRITE, SY_FALSE);
843 if (fd < 0) { SyMessage(state->pfn, SY_MSG_ERROR, "resume: can't open file %s", oOutFileName); return SY_ERROR; }
844 len = SyFileSize(fd);
845 SyCloseFile(fd);
846 if (len < 0) { SyMessage(state->pfn, SY_MSG_ERROR, "can't determine file size for %s", oOutFileName); return SY_ERROR; }
847 state->firstByte = len;
848 return SY_OK;
852 TSyState *curState = NULL;
854 #ifdef WINDOZE
856 BOOL __stdcall WinCtrlSignal (DWORD type) {
857 lastSSave = 0;
858 if (curState) curState->breakNow = SY_TRUE;
859 return TRUE;
862 #else
864 void BreakSignal (int signo) {
865 lastSSave = 0;
866 if (curState) curState->breakNow = SY_TRUE;
870 void SaveStateSignal (int signo) {
871 /*fprintf(stderr, "\nsave state\n");*/
872 lastSSave = 0;
875 #endif
878 int main (int argc, char *argv[]) {
879 int mainres = 1;
880 TSyPrintStr prs;
881 TSyCfg *cfg;
882 char *s, *t;
883 TSyState *state;
884 TSyResult res;
885 int retCnt;
887 prs.print = &SyPrintStr; prs.udata = NULL;
889 #ifndef WINDOZE
890 t = getenv("TERM");
891 if (t && (!strcmp(t, "rxvt") || !strcmp(t, "mrxvt") || !strcmp(t, "xterm") || !strcmp(t, "eterm"))) doRXVT = SY_TRUE;
892 #endif
894 state = SyNew();
895 if (!state) { SyMessage(&prs, SY_MSG_ERROR, "memory error"); }
896 prs.udata = (void *)state;
898 cfg = SyCfgNew();
899 if (!cfg) { SyMessage(&prs, SY_MSG_ERROR, "memory error"); }
900 SyDefaultCfg(cfg);
901 SyCfgLoad(cfg, "/etc/syren/.syrenrc", &prs);
902 #ifdef WINDOZE
903 t = calloc(1, 65536);
904 if (t) {
905 GetModuleFileName(NULL, t, 65535);
906 s = t; while (*s) { if (*s == '\\') *s = '/'; s++; }
907 s = strrchr(t, '/'); if (s) *s = '\0';
909 #else
910 t = getenv("HOME");
911 #endif
912 if (t != NULL) {
913 s = SySPrintf("%s/.syren/.syrenrc", t);
914 SyCfgLoad(cfg, s, &prs);
915 free(s);
916 s = SySPrintf("%s/.syrenrc", t);
917 SyCfgLoad(cfg, s, &prs);
918 free(s);
920 #ifdef WINDOZE
921 free(t);
922 #endif
923 SyCfgLoad(cfg, ".syrenrc", &prs);
925 if (ParseCmdLine(&prs, cfg, argc, argv) != SY_OK) goto done;
927 SyMessage(&prs, SY_MSG_MSG, "%s", SYREN_VERSION_DATETIME_STRING);
930 Sy_SchemeInit(&prs);
931 Sy_SchemeLoadInit();
934 if (cfgBufferSize < 1) { SyMessage(&prs, SY_MSG_ERROR, "buffer size too small"); goto done; }
935 if (cfgTimeout < 1) { SyMessage(&prs, SY_MSG_ERROR, "timeout too small"); goto done; }
937 if (writeCfg == SY_TRUE) {
938 SyMessage(&prs, SY_MSG_MSG, "generating sample config");
939 SyCfgSave(cfg, ".syrenrc", &prs);
940 mainres = 0;
941 goto done;
944 if (SySocketInit() != SY_OK) goto done;
946 state->udatai = -1;
948 state->pfn = &prs;
949 state->fnprog = PrintProgress;
950 state->fnprephdrs = SyDrvAddHeaders;
951 state->fngothdrs = SyDrvGotHeaders;
953 state->fnopen = SyDrvOpenFile;
954 state->fnclose = SyDrvCloseFile;
955 state->fnwrite = SyDrvWriteFile;
956 state->fnnoresume = SyDrvNoResume;
958 state->postData = oPostData; oPostData = NULL;
960 if (!oStateFileName) {
961 if (!oURL) { SyMessage(&prs, SY_MSG_ERROR, "URL?"); goto done; }
962 } else {
963 SyMessage(&prs, SY_MSG_MSG, "restoring state from file %s", oStateFileName);
964 if (oStateFileName && *oStateFileName) {
965 s = SyLoadState(oStateFileName, state);
966 if (!s || !s[0]) {
967 if (s) free(s);
968 SyMessage(&prs, SY_MSG_ERROR, "can't restore state");
969 goto done;
971 if (oOutFileName) free(oOutFileName);
972 oOutFileName = s;
974 if (dumpState == SY_TRUE) { DumpState(state); goto done; }
977 if (replyOnly != SY_FALSE && oResume == SY_TRUE && CheckResume(state) != SY_OK) goto done;
979 if (SyPrepare(state, oURL,
980 cfgUseProxyHTTP?cfgProxyHTTP:NULL,
981 cfgUseProxyFTP?cfgProxyFTP:NULL, cfgIFace) != SY_OK) { SyMessage(&prs, SY_MSG_ERROR, "SyPrepare() failed"); goto done; }
983 if (uEncodeDecode < 0 ||
984 (cfgDecodeFTPURL == SY_TRUE && state->url->proto == SY_PROTO_FTP)) {
985 s = SyURLDecode(state->url->dir);
986 if (!s) { SyMessage(&prs, SY_MSG_ERROR, "SyURLDecode() failed"); goto done; }
987 SyStrFree(state->url->dir); state->url->dir = s;
988 s = SyURLDecode(state->url->file);
989 if (!s) { SyMessage(&prs, SY_MSG_ERROR, "SyURLDecode() failed"); goto done; }
990 SyStrFree(state->url->file); state->url->file = s;
991 } else if (uEncodeDecode > 0) {
992 s = SyURLEncode(state->url->dir);
993 if (!s) { SyMessage(&prs, SY_MSG_ERROR, "SyURLEncode() failed"); goto done; }
994 SyStrFree(state->url->dir); state->url->dir = s;
995 s = SyURLEncode(state->url->file);
996 if (!s) { SyMessage(&prs, SY_MSG_ERROR, "SyURLEncode() failed"); goto done; }
997 SyStrFree(state->url->file); state->url->file = s;
1000 #ifdef WINDOZE
1001 SetConsoleCtrlHandler(WinCtrlSignal, TRUE);
1002 #else
1003 signal(SIGHUP, SaveStateSignal);
1004 signal(SIGUSR1, SaveStateSignal);
1006 signal(SIGINT, BreakSignal);
1007 signal(SIGQUIT, BreakSignal);
1008 signal(SIGTERM, BreakSignal);
1009 #endif
1011 retCnt = cfgReconnectAttempts;
1013 state->bufferSize = cfgBufferSize;
1014 state->ioTimeout = cfgTimeout;
1015 state->maxRedirects = cfgMaxRedirects;
1017 while (1) {
1018 curState = state;
1019 res = SyBegin(state);
1020 if (replyOnly != SY_FALSE) break;
1021 SyInitStats(state, &stats);
1022 if (res == SY_OK) res = SyRun(state);
1023 curState = NULL;
1024 if (res == SY_OK) break;
1025 if (state->interrupted == SY_TRUE) {
1026 if (oStateFileName && *oStateFileName && state->status == SY_STATUS_DOWNLOADING && oNoStateFile != SY_TRUE) {
1027 SyMessage(&prs, SY_MSG_WARNING, "user break, saving state");
1028 SySaveState(oStateFileName, state);
1029 } else SyMessage(&prs, SY_MSG_WARNING, "user break");
1030 break;
1032 if (retCnt < 1) break;
1033 if (state->status != SY_STATUS_DOWNLOADING && oForceRetry != SY_TRUE) break;
1034 SyMessage(&prs, SY_MSG_ERROR, "download failed");
1035 if (cfgReconnectDelay > 0) {
1036 SyMessage(&prs, SY_MSG_MSG, "waiting %i seconds", cfgReconnectDelay);
1037 SySleep(cfgReconnectDelay);
1039 SyMessage(&prs, SY_MSG_MSG, "restarting");
1040 state->firstByte += state->currentByte; /* start from here */
1041 oCanRename = SY_FALSE;
1042 retCnt--;
1044 SyEnd(state);
1045 if (res == SY_OK) {
1046 SyMessage(&prs, SY_MSG_MSG, "download complete");
1047 if (oCanRemoveState && oStateFileName && *oStateFileName) {
1048 if (SyDeleteFile(oStateFileName) == SY_OK) SyMessage(&prs, SY_MSG_NOTICE, "state file removed");
1050 } else SyMessage(&prs, SY_MSG_ERROR, "download failed");
1052 mainres = (res==SY_OK)?0:1;
1054 done:
1055 SyFree(state); SyCfgFree(cfg);
1056 if (cfgDefaultName) free(cfgDefaultName);
1057 if (cfgIFace) free(cfgIFace);
1058 if (cfgProxyHTTP) free(cfgProxyHTTP);
1059 if (cfgProxyFTP) free(cfgProxyFTP);
1060 if (cfgRefererStr) free(cfgRefererStr);
1061 if (cfgUAStr) free(cfgUAStr);
1062 if (oOutFileName) free(oOutFileName);
1063 if (oStateFileName) free(oStateFileName);
1064 if (oURL) free(oURL);
1065 if (oPostData) free(oPostData);
1066 if (oSendCookies) free(oSendCookies);
1067 if (oCookieDump) free(oCookieDump);
1069 Sy_SchemeDeinit();
1071 SySocketShutdown();
1072 return mainres;