?  search:
summary | log | graphiclog1 | graphiclog2 | commit | commitdiff | tree | refs | edit | fork
blame | history | raw | HEAD
insert nonsensical useragent for sourceshit, so it won't try to show its idiotic...
[syren.git] / src / syren / syren.c
blob89416b4f93ab3fdb5cf721da9c659d5fd34dfa4d
1 /*
2 Syren -- a lightweight downloader for Linux/BSD/MacOSX
3 inspired by Axel Copyright 2001-2002 Wilmer van der Gaast
4 version 0.0.6 (atomic alien)
5 coded by Ketmar // Avalon Group
6
7 This program is free software; you can redistribute it and/or modify
8 it under the terms of the GNU General Public License as published by
9 the Free Software Foundation; either version 3 of the License, or
10 (at your option) any later version.
11
12 This program is distributed in the hope that it will be useful,
13 but WITHOUT ANY WARRANTY; without even the implied warranty of
14 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
15 GNU General Public License for more details.
16
17 You should have received a copy of the GNU General Public License with
18 the Debian GNU/Linux distribution in file /usr/doc/copyright/GPL;
19 if not, write to the Free Software Foundation, Inc., 59 Temple Place,
20 Suite 330, Boston, MA 02111-1307 USA
21 */
22 /*
23 * Syren main program
24 */
25 #ifdef __linux
26 # ifndef _GNU_SOURCE
27 # define _GNU_SOURCE
28 # endif
29 #endif
30
31
32 #define USE_NEW_PREALLOC 0
33
34
35 #include "syren_common.h"
36 #include "syren_os.h"
37 #include "syren_str.h"
38 #include "syren_msg.h"
39 #include "syren_cfg.h"
40 #include "syren_hdrs.h"
41 #include "syren_tcp.h"
42 #include "syren_http.h"
43 #include "syren_proxy.h"
44 #include "syren_ftp.h"
45 #include "syren_dloader.h"
46
47 #include "syren_script.h"
48
49
50 #include <signal.h>
51
52 /* posix_fallocate */
53 #include <fcntl.h>
54 /*
55 #ifdef __linux
56 # include <linux/fallocate.h>
57 #endif
58 */
59
60 /* uncomment this to use Linux TTY ioctls */
61 /*
62 #ifdef __linux
63 #define LINUX_TTY
64 #endif
65 */
66
67 /* uncomment this to use ANSI/VT100 TTY codes */
68 #define ANSI_TTY
69
70 /* uncomment this to use "DEC private" TTY codes (ESC [ ? n)*/
71 #define DEC_TTY
72
73
74
75 TSyBool
76 cfgNoOutput = SY_FALSE,
77 cfgQuiet = SY_FALSE,
78 cfgNoIndicator = SY_FALSE,
79 cfgUseProxyHTTP = SY_TRUE,
80 cfgUseProxyFTP = SY_TRUE,
81 cfgDecodeFTPURL = SY_TRUE,
82 cfgFTPUseConnect = SY_FALSE,
83 cfgProgressWriteCR = SY_FALSE,
84 optAnyCode = SY_FALSE;
85 int
86 cfgBufferSize = 8192,
87 cfgMaxBufferSize = 1024*1024,
88 cfgSaveInterval = 10,
89 cfgReconnectDelay = 10,
90 cfgReconnectAttempts = 666,
91 cfgTimeout = 60,
92 cfgMaxRedirects = 10,
93 cfgRefererType = 0;
94 char
95 *cfgDefaultName = NULL,
96 *cfgIFace = NULL,
97 *cfgProxyHTTP = NULL,
98 *cfgProxyFTP = NULL,
99 *cfgRefererStr = NULL,
100 *cfgUAStr = NULL;
101
102 char
103 *oOutFileName = NULL, /* -o/-O */
104 *oStateFileName = NULL, /* -s */
105 *oURL = NULL,
106 *oPostData = NULL,
107 *oSendCookies = NULL,
108 *oCookieDump = NULL,
109 *oAddonHeaders = NULL;
110 TSyBool
111 oResume = SY_FALSE,
112 oCanRename = SY_TRUE,
113 oCanRemoveState = SY_TRUE,
114 oIgnoreStatePos = SY_FALSE,
115 oNoStateFile = SY_FALSE,
116 oForceRetry = SY_FALSE,
117 writeCfg = SY_FALSE,
118 dumpState = SY_FALSE,
119 replyOnly = SY_FALSE,
120 oPreallocSpace = SY_TRUE;
121 int uEncodeDecode = 0;
122 TSyBool doRXVT = SY_FALSE;
123 int optWOS = 0;
124
125
126 TSyKVList *oURLList = NULL;
127
128 double lastSSave = 0;
129
130 TSyState *state = NULL; /* dl state */
131 #ifdef SYREN_USE_SCRIPT
132 TSyBool scAbort = SY_FALSE;
133 #endif
134
135 #ifdef LINUX_TTY
136 static int GetTTYWidth (void) {
137 struct winsize ws;
138 if (!isatty(fileno(stderr))) return -1;
139 if (ioctl(fileno(stderr), TIOCGWINSZ, &ws)) return -1;
140 return ws.ws_col;
141 }
142 #endif
143
144
145 static void SetTermTitle (const char *tit) {
146 if (!tit || !isatty(fileno(stdout)) || doRXVT != SY_TRUE) return;
147 fprintf(stdout, "\x1b]2;%s\7", tit);
148 fflush(stdout);
149 }
150
151
152 static TSyBool ctrlCPressed = SY_FALSE;
153
154 static void DoWait (int seconds) {
155 while (seconds > 0) {
156 char *tit = SySPrintf("waiting: %i seconds left", seconds);
157 SetTermTitle(tit);
158 SyStrFree(tit);
159 SySleep(1);
160 seconds--;
161 if (ctrlCPressed != SY_FALSE) return;
162 }
163 }
164
165
166 static void PrintProgress (TSyState *state, int64_t bytesDone, int64_t bytesTotal, TSyBool done) {
167 char tmp0[128], tmp1[128], tmp2[128], tmp3[128], tmp4[128], fmt[128], str[1024];
168 int len; int64_t fb = state->firstByte;
169 TSyStats *stats = &state->stats;
170
171 if (cfgNoIndicator) return;
172 fprintf(stderr, "\r");
173 if (isatty(fileno(stderr))) {
174 #ifdef DEC_TTY
175 fputs("\x1b[?7l", stderr);
176 #endif
177 }
178 if (bytesTotal > 0) {
179 len = SyLong2StrComma(tmp1, (bytesTotal+fb));
180 SyLong2StrComma(tmp0, (bytesDone+fb));
181 sprintf(fmt, "[%%%is/%%s] ", len);
182 sprintf(tmp2, fmt, tmp0, tmp1); /*!*/
183 sprintf(tmp3, "[%3i%%] ", (int)((double)100*(bytesDone+fb)/(bytesTotal+fb))); /*!*/
184 /* tmp3: res */
185 SySize2Str(tmp0, (int64_t)stats->bps);
186 sprintf(tmp4, "[SPD:%s] ", tmp0); /*!*/
187 SyTime2Str(tmp0, (int)(SyGetTimeD()-stats->sttime));
188 SyTime2Str(tmp1, (int)stats->eta);
189 sprintf(fmt, "TIME:%s ETA:%s", tmp0, tmp1); /*!*/
190 fprintf(stderr, "%s%s%s%s", tmp2, tmp3, tmp4, fmt);
191 sprintf(str, "%s%s%s/%s", tmp2, tmp3, tmp0, tmp1);
192 } else {
193 SyLong2StrComma(tmp0, bytesDone+fb);
194 SyTime2Str(tmp1, (int)(SyGetTimeD()-stats->sttime));
195 SySize2Str(fmt, (int64_t)stats->bps);
196 sprintf(str, "[%s] [SPD:%s] TIME:%s", tmp0, fmt, tmp1);
197 fprintf(stderr, "%s", str);
198 }
199 if (isatty(fileno(stdout)) && doRXVT == SY_TRUE) {
200 fprintf(stdout, "\x1b]2;%s\7", str);
201 fflush(stdout);
202 }
203 if (isatty(fileno(stderr))) {
204 #ifdef ANSI_TTY
205 fprintf(stderr, "\x1b[K");
206 #endif
207 #ifdef DEC_TTY
208 fputs("\x1b[?7h", stderr);
209 #endif
210 }
211 if (done == SY_TRUE) fputs("\n", stderr);
212 else if (cfgProgressWriteCR == SY_TRUE) fprintf(stderr, "\n");
213 fflush(stderr);
214 }
215
216
217 static void SyPrintStr (void *udata, TSyMsgType msgtype, const char *msg) {
218 #ifdef LINUX_TTY
219 int wdt;
220 #endif
221 static const char *ltr = "NMWE?";
222 if (cfgNoOutput) return;
223 if (cfgQuiet && msgtype < SY_MSG_MSG) return;
224 fflush(stderr);
225 if (isatty(fileno(stderr))) {
226 #ifndef LINUX_TTY
227 # ifdef DEC_TTY
228 fputs("\x1b[?7l", stderr);
229 # endif
230 #else
231 wdt = GetTTYWidth();
232 if (wdt < 0) wdt = strlen(msg)+16;
233 if (wdt < 2) return;
234 #endif
235 }
236 if (msgtype < SY_MSG_NOTICE || msgtype > SY_MSG_ERROR) msgtype = SY_MSG_ERROR+1;
237 fputc(ltr[msgtype], stderr);
238 #ifdef LINUX_TTY
239 if (wdt > 4) {
240 fputs(": ", stderr);
241 wdt -= 4;
242 if (wdt >= strlen(msg)) fputs(msg, stderr);
243 else while (wdt--) fputc(*(msg++), stderr);
244 }
245 #else
246 fputs(": ", stderr);
247 fputs(msg, stderr);
248 #endif
249 if (isatty(fileno(stderr))) {
250 #ifdef ANSI_TTY
251 fputs("\x1b[K", stderr);
252 #endif
253 #ifndef LINUX_TTY
254 # ifdef DEC_TTY
255 fputs("\x1b[?7h", stderr);
256 # endif
257 #endif
258 }
259 fputs("\n", stderr);
260 fflush(stderr);
261 if (isatty(fileno(stdout)) && doRXVT == SY_TRUE && msgtype > SY_MSG_NOTICE) {
262 fprintf(stdout, "\x1b]2;%s\7", msg);
263 fflush(stdout);
264 }
265 }
266
267
268 static void SyDefaultCfg (TSyCfg *cfg) {
269 SyCfgAddIntKey(cfg, "reconnect_delay", "number of seconds to wait before reconnecting to server", &cfgReconnectDelay);
270 SyCfgAddIntKey(cfg, "reconnect_attempts", "number of reconnection attempts (0: do not reconnect)", &cfgReconnectAttempts);
271 SyCfgAddIntKey(cfg, "io_timeout", "i/o timeout in seconds", &cfgTimeout);
272 SyCfgAddIntKey(cfg, "save_state_interval", "save state every x seconds (0: disable)", &cfgSaveInterval);
273 cfgDefaultName = SyStrNew("index.html", -1);
274 SyCfgAddStrKey(cfg, "default_file_name", "default file name for http", &cfgDefaultName);
275 SyCfgAddIntKey(cfg, "max_redirects", "maximum number of redirects/symlinks", &cfgMaxRedirects);
276 SyCfgAddIntKey(cfg, "init_buffer_size", "initial buffer size", &cfgBufferSize);
277 SyCfgAddIntKey(cfg, "max_buffer_size", "maximum amount of bytes read buffer can grow to", &cfgMaxBufferSize);
278 SyCfgAddBoolKey(cfg, "quiet", "tan: do not show notices", &cfgQuiet);
279 SyCfgAddBoolKey(cfg, "no_output", "tan: disable any output", &cfgNoOutput);
280 SyCfgAddBoolKey(cfg, "no_indicator", "tan: hide download indicator", &cfgNoIndicator);
281
282 cfgIFace = SyStrNew("", -1);
283 SyCfgAddStrKey(cfg, "interface", "net interface to use for connection (name or IP)", &cfgIFace);
284
285 cfgProxyHTTP = SyStrNew(getenv("http_proxy"), -1);
286 SyCfgAddStrKey(cfg, "http_proxy", "proxy for http connections", &cfgProxyHTTP);
287 cfgProxyFTP = SyStrNew(getenv("ftp_proxy"), -1);
288 SyCfgAddStrKey(cfg, "ftp_proxy", "proxy for ftp connections", &cfgProxyFTP);
289 SyCfgAddBoolKey(cfg, "use_http_proxy", "use http proxy (if specified)?", &cfgUseProxyHTTP);
290 SyCfgAddBoolKey(cfg, "use_ftp_proxy", "use ftp proxy (if specified)?", &cfgUseProxyFTP);
291 SyCfgAddBoolKey(cfg, "ftp_proxy_use_connect", "use CONNECT method for proxied ftp connections", &cfgFTPUseConnect);
292
293 SyCfgAddIntKey(cfg, "referer_type", "set referer type (0:none; 1:orig URL; 2:follow URL; 3:user)", &cfgRefererType);
294 cfgRefererStr = SyStrNew(NULL, -1);
295 SyCfgAddStrKey(cfg, "referer_str", "referer for mode 3", &cfgRefererStr);
296 SyCfgAddBoolKey(cfg, "decode_ftp_url", "decode URLs for FTP", &cfgDecodeFTPURL);
297
298 cfgUAStr = SyStrNew(SYREN_DEFAULT_USER_AGENT, -1);
299 SyCfgAddStrKey(cfg, "ua_str", "user agent (set to empty string if no UA should be sent)", &cfgUAStr);
300 }
301
302
303
304 TSyResult SySaveInt (int fd, int64_t v, int size) {
305 uint8_t b;
306
307 if (fd < 0 || size < 1 || size > 8) return SY_ERROR;
308 while (size--) {
309 b = (v>>(size*8))&0xff;
310 if (SyWriteFile(fd, &b, 1) != SY_OK) return SY_ERROR;
311 }
312
313 return SY_OK;
314 }
315
316
317 TSyResult SyLoadInt (int fd, int64_t *v, int size) {
318 uint8_t b; int64_t res = 0;
319
320 if (v) *v = 0;
321 if (!fd || size < 1 || size > 8) return SY_ERROR;
322 while (size--) {
323 if (SyReadFile(fd, &b, 1) != SY_OK) return SY_ERROR;
324 res = (res<<8)+b;
325 }
326 if (v) *v = res;
327
328 return SY_OK;
329 }
330
331
332 TSyResult SySaveStr (int fd, const char *v) {
333 int len = 0;
334
335 if (!fd) return SY_ERROR;
336 if (v) len = strlen(v);
337 if (len > 65535) return SY_ERROR;
338 if (SySaveInt(fd, len, 2) != SY_OK) return SY_ERROR;
339 if (len) { if (SyWriteFile(fd, v, len) != SY_OK) return SY_ERROR; }
340
341 return SY_OK;
342 }
343
344
345 char *SyLoadStr (int fd) {
346 char *v;
347 int64_t len = 0;
348
349 if (!fd) return NULL;
350 if (SyLoadInt(fd, &len, 2) != SY_OK) return NULL;
351 if (!len) return SyStrNewEmpty();
352 v = calloc(1, len+8); if (!v) return NULL;
353 if (SyReadFile(fd, v, len) != SY_OK) { free(v); return NULL; }
354
355 return v;
356 }
357
358
359 static const char *SF_STR = "SYREN STATE FILE v6\r\n\x1a";
360 TSyResult SySaveState (const char *destfile, TSyState *state) {
361 TSyResult res = SY_ERROR;
362 TSyURL *url;
363 char *s, *t;
364 int fd;
365
366 if (!state) return SY_ERROR;
367 url = state->url;
368 s = SySPrintf("%s://%s:%s@%s:%i%s%s%s%s",
369 url->protostr, url->user, url->pass, url->host, url->port,
370 url->dir, url->file, url->query, url->anchor);
371 if (!s) return SY_ERROR;
372 fd = SyOpenFile(destfile, SY_FMODE_WRITE, SY_TRUE);
373 if (fd >= 0) {
374 while (1) {
375 /*if (SySaveStr(fd, "SYREN STATE FILE v5") != SY_OK) break;*/
376 if (SyWriteFile(fd, SF_STR, strlen(SF_STR)) != SY_OK) break;
377 if (SySaveInt(fd, state->fileSize, 8) != SY_OK) break;
378 if (SySaveInt(fd, state->currentByte+state->firstByte, 8) != SY_OK) break;
379 if (SySaveInt(fd, state->lastByte, 8) != SY_OK) break;
380 if (SySaveStr(fd, s) != SY_OK) break;
381 t = strrchr(oOutFileName, '/'); if (!t) t = oOutFileName; else t++;
382 if (SySaveStr(fd, t) != SY_OK) break;
383 if (SySaveInt(fd, cfgRefererType, 1) != SY_OK) break;
384 if (SySaveStr(fd, cfgRefererStr) != SY_OK) break;
385 if (SySaveStr(fd, oPostData) != SY_OK) break;
386 res = SY_OK; break;
387 }
388 SyCloseFile(fd);
389 }
390 if (res != SY_OK) SyDeleteFile(destfile);
391 free(s);
392
393 return res;
394 }
395
396
397 char *SyLoadStateV5 (int fd, TSyState *state) {
398 char *destfile = NULL, *s = NULL;
399 int64_t i;
400
401 while (1) {
402 s = SyLoadStr(fd); if (!s) break;
403 if (strcmp(s, "SYREN STATE FILE v5")) break;
404 if (SyLoadInt(fd, &state->fileSize, 8) != SY_OK) break;
405 if (SyLoadInt(fd, &state->firstByte, 8) != SY_OK) break;
406 if (oIgnoreStatePos != SY_FALSE) state->firstByte = 0;
407 if (state->firstByte < 0 || (state->fileSize >= 0 && state->firstByte > state->fileSize)) break;
408 if (SyLoadInt(fd, &state->lastByte, 8) != SY_OK) break;
409 if (state->lastByte < -1 || (state->lastByte >= 0 && state->firstByte > state->lastByte)) break;
410 free(s); s = SyLoadStr(fd); if (!s) break;
411 if (oURL) free(oURL); oURL = s;
412 s = SyLoadStr(fd); if (!s) break;
413 destfile = s; s = NULL;
414 if (SyLoadInt(fd, &i, 1) != SY_OK) break;
415 cfgRefererType = (int)i;
416 s = SyLoadStr(fd); if (!s) break;
417 if (cfgRefererStr) free(cfgRefererStr);
418 cfgRefererStr = s;
419 s = SyLoadStr(fd); if (!s) { free(destfile); destfile = NULL; break; }
420 if (state->postData) free(state->postData);
421 state->postData = s; s = NULL;
422 break;
423 }
424 if (s) free(s);
425
426 return destfile;
427 }
428
429
430 char *SyLoadStateV6 (int fd, TSyState *state) {
431 char *destfile = NULL, *s = NULL, buf[128];
432 int64_t i;
433 int l = strlen(SF_STR);
434
435 while (1) {
436 memset(buf, 0, sizeof(buf));
437 if (SyReadFile(fd, buf, l) != SY_OK) break;
438 if (strcmp(buf, SF_STR)) break;
439 if (SyLoadInt(fd, &state->fileSize, 8) != SY_OK) break;
440 if (SyLoadInt(fd, &state->firstByte, 8) != SY_OK) break;
441 if (oIgnoreStatePos != SY_FALSE) state->firstByte = 0;
442 if (state->firstByte < 0 || (state->fileSize >= 0 && state->firstByte > state->fileSize)) break;
443 if (SyLoadInt(fd, &state->lastByte, 8) != SY_OK) break;
444 if (state->lastByte < -1 || (state->lastByte >= 0 && state->firstByte > state->lastByte)) break;
445 free(s); s = SyLoadStr(fd); if (!s) break;
446 if (oURL) free(oURL); oURL = s;
447 s = SyLoadStr(fd); if (!s) break;
448 destfile = s; s = NULL;
449 if (SyLoadInt(fd, &i, 1) != SY_OK) break;
450 cfgRefererType = (int)i;
451 s = SyLoadStr(fd); if (!s) break;
452 if (cfgRefererStr) free(cfgRefererStr);
453 cfgRefererStr = s;
454 s = SyLoadStr(fd); if (!s) { free(destfile); destfile = NULL; break; }
455 if (state->postData) free(state->postData);
456 state->postData = s; s = NULL;
457 break;
458 }
459 if (s) free(s);
460
461 return destfile;
462 }
463
464
465 /* return destfile */
466 char *SyLoadState (const char *statefile, TSyState *state) {
467 char *destfile = NULL;
468 int fd;
469 uint8_t b;
470
471 fd = SyOpenFile(statefile, SY_FMODE_READ, SY_FALSE);
472 if (fd < 0) return NULL;
473 if (SyReadFile(fd, &b, 1) == SY_OK) {
474 if (SySeekFile(fd, 0) == SY_OK) {
475 /*fprintf(stderr, "state v%i\n", b?6:5);*/
476 if (!b) destfile = SyLoadStateV5(fd, state);
477 else destfile = SyLoadStateV6(fd, state);
478 }
479 }
480 SyCloseFile(fd);
481
482 return destfile;
483 }
484
485
486 void DumpState (TSyState *state) {
487 char t0[32], t1[32], t2[32];
488 SyLong2StrComma(t0, state->fileSize);
489 SyLong2StrComma(t1, state->firstByte);
490 SyLong2StrComma(t2, state->lastByte);
491 printf("state dump\n==========\nURL: %s\nfile name: %s\n"
492 "file size: %s\nfirst byte: %s\nlast byte: %s\n",
493 oURL, oOutFileName, t0, t1, t2
494 );
495 }
496
497
498 static void ShowHelp (void) {
499 fprintf(stderr, "%s",
500 "usage: syren [options] url [+url] [+url] [@urllistfile]\n"
501 "options:\n"
502 " -h this help\n"
503 " -h all show 'long' configuration options\n"
504 " -h <longopt> show 'long' configuration option description\n"
505 " -l filename same as @urllistfile\n"
506 " -o filename output to specified file\n"
507 " -O filename output to specified file (same as '-o')\n"
508 " -r filename resume download (do not use saved state, use file)\n"
509 " -s filename restore state\n"
510 " -S filename redownload file using state (ignore starting position, recreate)\n"
511 " -D filename dump state file and exit\n"
512 " -k keep state file\n"
513 " -P <str|@fn> make POST request\n"
514 " -c <str|@fn> send cookies\n"
515 " -C filename dump cookies\n"
516 " -H <str|@fn> additional headers\n"
517 " -i download server reply and stop\n"
518 " -I name_or_ip use specified net interface (\"-\" any)\n"
519 " -X do not use proxies\n"
520 " -A don't preallocate disk space\n"
521 " -T turn off terminal window title changing\n"
522 " -Z write CR after progress string\n"
523 " -w generate .syrenrc file with the current config\n"
524 " -N don't write state file\n"
525 " -F force retry on any download error\n"
526 " -e encode URL\n"
527 " -E decode URL\n"
528 " -q be quiet\n"
529 " -Q be VERY quiet\n"
530 #ifdef SYREN_USE_SCRIPT
531 " -L disable scripting\n"
532 #endif
533 " -V version\n"
534 " -W accept any HTTP reply code\n"
535 " -9 WOS delay\n"
536 );
537 }
538
539
540 static char *GetStrOpt (TSyPrintStr *prs, int argc, char *argv[], int *f, const char *optName,
541 char **value, TSyBool allowEmpty, TSyBool exclusive) {
542 //
543 char *res/*, *s*/;
544 //
545 if (*f >= argc || !argv[*f]) { SyMessage(prs, SY_MSG_ERROR, "no value for '%s'", optName); return NULL; }
546 if (exclusive == SY_TRUE && (value && *value)) { SyMessage(prs, SY_MSG_ERROR, "duplicate '%s'", optName); return NULL; }
547 res = SyStrNew(argv[*f], -1); (*f)++;
548 if (!res) { SyMessage(prs, SY_MSG_ERROR, "memory error"); return NULL; }
549 if (allowEmpty != SY_TRUE && !res[0]) {
550 free(res);
551 SyMessage(prs, SY_MSG_ERROR, "empty value for '%s'", optName);
552 return NULL;
553 }
554 if (value) {
555 if (*value) free(*value);
556 *value = res;
557 }
558
559 return res;
560 }
561
562
563 static TSyResult GetStrOptWithAt (TSyPrintStr *prs, int argc, char *argv[], int *f, const char *optName,
564 char **value, TSyBool allowEmpty, TSyBool exclusive, TSyBool ishdr) {
565 //
566 //printf("[%s]: %p\n", optName, ishdr?NULL:value);
567 char *res = GetStrOpt(prs, argc, argv, f, optName, ishdr?NULL:value, allowEmpty, exclusive);
568 if (!res) return SY_ERROR;
569 if (*res == '@') {
570 res = SyLoadWholeFile(res+1);
571 if (!res) { SyMessage(prs, SY_MSG_ERROR, "can't load data from '%s'", (*value)+1); return SY_ERROR; }
572 //free(*value); *value = res;
573 }
574
575 if (ishdr) {
576 if (*value) {
577 if (*res) {
578 char *t = SySPrintf("%s\n%s", *value, res);
579 free(res);
580 free(*value);
581 *value = t;
582 } else {
583 free(res);
584 }
585 } else {
586 *value = res;
587 }
588 } else {
589 if (value != NULL) *value = res;
590 }
591
592 return SY_OK;
593 }
594
595
596 static void SyAddURLFromFile (const char *fname) {
597 char *fl, *s, *e;
598
599 fl = SyLoadWholeFile(fname);
600 if (!fl) return;
601 s = fl; while (*s) {
602 e = s; while (*e && *e != '\n') e++;
603 if (*e) *e++ = '\0';
604 SyStrTrim(s);
605 if (*s && s[0] != '#' && s[0] != ';') {
606 SyKVListDelete(oURLList, s);
607 SyKVListSet(oURLList, s, "", NULL);
608 }
609 s = e;
610 }
611 free(fl);
612 }
613
614
615 static TSyResult ParseCmdLine (TSyPrintStr *prs, TSyCfg *cfg, int argc, char *argv[]) {
616 int f, nomoreopt = 0;
617 char *s, *t, *sopt, soptN[2];
618 TSyKVListItem *key;
619
620 f = 1; while (f < argc) {
621 s = argv[f++];
622 if (!s) break; if (!(*s)) continue;
623 if (!nomoreopt && *s == '-') {
624 if (s[1] == '-') {
625 /* long option (config) */
626 if (!s[3]) nomoreopt = 1;
627 else {
628 s += 2;
629 if (SyCfgSet(cfg, s) != SY_OK) { SyMessage(prs, SY_MSG_ERROR, "invalid option: %s", (s-2)); return SY_ERROR; }
630 }
631 } else {
632 sopt = s;
633 while (*(++sopt)) {
634 soptN[0] = *sopt; soptN[1] = '\0';
635 switch (*sopt) {
636 case 'V':
637 printf("%s", SYREN_VDHEADER_STRING);
638 return SY_ERROR;
639 case '9': optWOS = 15; break;
640 case 'W': optAnyCode = SY_TRUE; break;
641 case 'e': uEncodeDecode = 1; break;
642 case 'E': uEncodeDecode = -1; break;
643 case 'w': writeCfg = SY_TRUE; break;
644 case 'X': cfgUseProxyHTTP = cfgUseProxyFTP = SY_FALSE; break;
645 case 'T': doRXVT = SY_FALSE; break;
646 case 'q': cfgQuiet = SY_TRUE; break;
647 case 'Q': cfgQuiet = cfgNoOutput = SY_TRUE; break;
648 case 'i': replyOnly = SY_TRUE; break;
649 case 'N': oNoStateFile = SY_TRUE; break;
650 case 'F': oForceRetry = SY_TRUE; break;
651 case 'k': oCanRemoveState = SY_FALSE; break;
652 case 'Z': cfgProgressWriteCR = SY_TRUE; break;
653 case 'A': oPreallocSpace = SY_FALSE; break;
654 case 'L':
655 #ifdef SYREN_USE_SCRIPT
656 syScriptEnabled = SY_FALSE;
657 #endif
658 break;
659 case 'I':
660 if (!argv[f]) {
661 SyMessage(prs, SY_MSG_ERROR, "no argument for -I");
662 return SY_ERROR;
663 }
664 SyStrFree(cfgIFace);
665 cfgIFace = SyStrDup(argv[f++]);
666 /*SyStrTrim(cfgIFace);*/
667 /*if (!cfgIFace[0] || !strcmp(cfgIFace, "-")) { SyStrFree(cfgIFace); cfgIFace = NULL; }*/
668 break;
669 case 'o': case 'O': case 'r':
670 if (*sopt == 'r') oResume = SY_TRUE;
671 s = GetStrOpt(prs, argc, argv, &f, soptN, &oOutFileName, SY_FALSE, SY_TRUE);
672 if (!s) return SY_ERROR;
673 oCanRename = SY_FALSE;
674 break;
675 case 's': case 'S': case 'D':
676 if (*sopt == 'S') oIgnoreStatePos = SY_TRUE;
677 else if (*sopt == 'D') dumpState = SY_TRUE;
678 s = GetStrOpt(prs, argc, argv, &f, soptN, &oStateFileName, SY_FALSE, SY_TRUE);
679 if (!s) return SY_ERROR;
680 oCanRename = SY_FALSE;
681 break;
682 case 'P':
683 if (GetStrOptWithAt(prs, argc, argv, &f, soptN, &oPostData, SY_FALSE, SY_TRUE, SY_FALSE) != SY_OK) return SY_ERROR;
684 break;
685 case 'c':
686 if (GetStrOptWithAt(prs, argc, argv, &f, soptN, &oSendCookies, SY_FALSE, SY_TRUE, SY_FALSE) != SY_OK) return SY_ERROR;
687 break;
688 case 'C':
689 if (!GetStrOpt(prs, argc, argv, &f, soptN, &oCookieDump, SY_FALSE, SY_TRUE)) return SY_ERROR;
690 break;
691 case 'H':
692 if (GetStrOptWithAt(prs, argc, argv, &f, soptN, &oAddonHeaders, SY_FALSE, SY_TRUE, SY_TRUE)) return SY_ERROR;
693 break;
694 case 'l':
695 if (f >= argc) {
696 fprintf(stderr, "E: \"-l\" without file name!\n");
697 return SY_ERROR;
698 }
699 SyAddURLFromFile(argv[f++]);
700 break;
701 case 'h':
702 fprintf(stderr, "%s\n", SYREN_VERSION_DATETIME_STRING);
703 key = cfg->opts->first;
704 if (f < argc && !strcmp(argv[f], "all")) {
705 fprintf(stderr, "long options:\n");
706 while (key) { fprintf(stderr, " --%s\n", key->key); key = key->next; }
707 } else {
708 if (f < argc && argv[f]) {
709 while (key) {
710 if (!strcasecmp(key->key, argv[f])) {
711 fprintf(stderr, "--%s %s\n %s\n", key->key,
712 key->uidata==SY_CI_STRING?"string":(key->uidata==SY_CI_INT?"integer":(key->uidata==SY_CI_BOOL?"boolean":"?")),
713 key->ustr);
714 break;
715 }
716 key = key->next;
717 }
718 } else key = NULL;
719 if (!key) ShowHelp();
720 }
721 return SY_ERROR;
722 default:
723 SyMessage(prs, SY_MSG_ERROR, "unknown option: %s", soptN);
724 return SY_ERROR;
725 } /* switch */
726 } /* while */
727 } /* if */
728 } else {
729 /* new URL */
730 t = SyStrNew(s, -1);
731 if (!t) { SyMessage(prs, SY_MSG_ERROR, "memory error"); return SY_ERROR; }
732 SyStrTrim(t);
733 if (*t == '@') SyAddURLFromFile(t+1);
734 else if (*t) {
735 if (t[0] == '+') { strcpy(t, &(t[1])); SyStrTrim(t); }
736 if (*t) {
737 SyKVListDelete(oURLList, t);
738 SyKVListSet(oURLList, t, "", NULL);
739 }
740 }
741 SyStrFree(t);
742 }
743 }
744 if (oResume && oStateFileName) { SyMessage(prs, SY_MSG_ERROR, "'-r'/'-s' conflict"); return SY_ERROR; }
745
746 return SY_OK;
747 }
748
749
750 #ifdef SYREN_USE_SCRIPT
751 TSyHdrs *scHdrs = NULL;
752 #endif
753
754 TSyResult SyDrvAddHeaders (TSyState *state, TSyHdrs *hdrs) {
755 char *s = NULL, *t, *cs;
756 TSyURL *url = state->url;
757 char port[32];
758
759 // fuck you, sourceshit
760 if (strstr(url->host, "sourceforge.net")) {
761 if (SyHdrAddLine(hdrs, "User-Agent: HereToPissOffSourceCrapSniffer") != SY_OK) { SyMessage(state->pfn, SY_MSG_ERROR, "hdr error"); return SY_ERROR; }
762 }
763
764 if (cfgRefererType) {
765 switch (cfgRefererType) {
766 case 1: s = SyStrNew(oURL, -1); break;
767 case 2:
768 if (url->proto == SY_PROTO_FTP && url->port != 80) sprintf(port, ":%i", url->port); else port[0] = '\0';
769 s = SySPrintf("%s://%s%s%s%s%s%s", url->protostr, url->host, port,
770 url->dir, url->file, url->query, url->anchor);
771 break;
772 default: s = SyStrNew(cfgRefererStr, -1); break;
773 }
774 if (!s) { SyMessage(state->pfn, SY_MSG_ERROR, "memory error"); return SY_ERROR; }
775 t = SySPrintf("Referer: %s", s); free(s);
776 if (!t) { SyMessage(state->pfn, SY_MSG_ERROR, "memory error"); return SY_ERROR; }
777 if (SyHdrAddLine(hdrs, t) != SY_OK) { free(t); SyMessage(state->pfn, SY_MSG_ERROR, "hdr error"); return SY_ERROR; }
778 free(t);
779 }
780 if (oAddonHeaders && *oAddonHeaders) {
781 s = oAddonHeaders;
782 while (*s) {
783 t = s; while (*t && *t != '\n') t++;
784 cs = SyStrNew(s, t-s); if (!cs) return SY_ERROR;
785 SyStrTrim(cs);
786 if (*cs) {
787 if (SyHdrAddLine(hdrs, cs) != SY_OK) { free(cs); SyMessage(state->pfn, SY_MSG_ERROR, "hdr error"); return SY_ERROR; }
788 }
789 free(cs); if (!s[0]) break;
790 s = t; if (*s) s++; /* don't skip latest 0 */
791 }
792 }
793 if (oSendCookies && *oSendCookies) {
794 char *newstr = calloc(strlen(oSendCookies)*3, 1); /* this should be enough */
795 s = oSendCookies;
796 while (*s) {
797 t = s; while (*t && *t != '\n') t++;
798 cs = SyStrNew(s, t-s); if (!cs) return SY_ERROR;
799 SyStrTrim(cs);
800 if (*cs) {
801 if (newstr[0]) strcat(newstr, "; ");
802 //if (SyHdrSetCookie(hdrs, cs) != SY_OK) { free(cs); return SY_ERROR; }
803 strcat(newstr, cs);
804 }
805 free(cs); if (!s[0]) break;
806 s = t; if (*s) s++; /* don't skip latest 0 */
807 }
808 if (newstr[0]) {
809 SyHdrSetCookie(hdrs, newstr);
810 SyHdrAddLine(hdrs, "Cookie2: $Version=1");
811 }
812 free(newstr);
813 }
814 #ifdef SYREN_USE_SCRIPT
815 scHdrs = hdrs;
816 SyScriptCallback("Event.AddHeaders", state->pfn);
817 scHdrs = NULL;
818 if (scAbort == SY_TRUE) state->breakNow = SY_TRUE;
819 #endif
820
821 return SY_OK;
822 }
823
824
825 TSyResult SyDrvGotHeaders (TSyState *state, TSyHdrs *hdrs) {
826 TSyResult res = SY_OK;
827 TSyKVList *cc; TSyKVListItem *item;
828 int fd;
829 static char *crlf = "\r\n";
830
831 #ifdef SYREN_USE_SCRIPT
832 scHdrs = hdrs;
833 SyScriptCallback("Event.GotHeaders", state->pfn);
834 /*fprintf(stderr, "\rGotHeaders: return\n");*/
835 scHdrs = NULL;
836 if (scAbort == SY_TRUE) state->breakNow = SY_TRUE;
837 #endif
838 if (oCookieDump && *oCookieDump) {
839 cc = SyHdrFindCookieList(hdrs);
840 if (cc && cc->count) {
841 if (strcmp(oCookieDump, "-")) {
842 fd = SyOpenFile(oCookieDump, SY_FMODE_WRITE, SY_TRUE);
843 if (fd < 0) { SyMessage(state->pfn, SY_MSG_ERROR, "can't dump cookies to '%s'", oCookieDump); return SY_ERROR; }
844 } else fd = 0;
845 item = cc->first;
846 while (item) {
847 if (SyWriteFile(fd, item->value, strlen(item->value)) != SY_OK) { res = SY_ERROR; break; }
848 if (SyWriteFile(fd, crlf, strlen(crlf)) != SY_OK) { res = SY_ERROR; break; }
849 item = item->next;
850 }
851 if (fd) SyCloseFile(fd);
852 }
853 }
854
855 return res;
856 }
857
858
859 static TSyState *curState = NULL;
860
861 static void BreakSignal (int signo) {
862 lastSSave = 0;
863 if (curState) curState->breakNow = SY_TRUE;
864 ctrlCPressed = SY_TRUE;
865 }
866
867
868 static void SaveStateSignal (int signo) {
869 /*fprintf(stderr, "\nsave state\n");*/
870 lastSSave = 0;
871 }
872
873
874 TSyResult SyXPrealloc (int fd, int64_t fileSize) {
875 #if !USE_NEW_PREALLOC
876 char tmp0[256];
877 char *buf;
878 int bufSz, wr;
879 double sttime = SyGetTimeD(), ctime;
880 int wasoutput = 0;
881
882 if (fileSize < 1) return SY_ERROR;
883
884 bufSz = 256*1024;
885 if (bufSz > fileSize) bufSz = fileSize;
886 buf = calloc(1, bufSz);
887 if (!buf) return SY_ERROR;
888
889 while (fileSize > 0) {
890 if (cfgQuiet != SY_TRUE || cfgNoOutput != SY_TRUE) {
891 ctime = SyGetTimeD()-sttime;
892 if (ctime >= 0.666) {
893 SyLong2StrComma(tmp0, fileSize);
894 fprintf(stderr, "\rN: preallocating: [%s] bytes left\x1b[K", tmp0);
895 fflush(stderr);
896 wasoutput = 1;
897 sttime = SyGetTimeD();
898 }
899 }
900 wr = fileSize>(int64_t)bufSz?bufSz:fileSize;
901 if (SyWriteFile(fd, buf, wr) != SY_OK) {
902 free(buf);
903 if (isatty(fileno(stderr))) {
904 if (wasoutput) { fprintf(stderr, "\r\x1b[K"); fflush(stderr); }
905 }
906 return SY_ERROR;
907 }
908 fileSize -= (int64_t)wr;
909 if (ctrlCPressed == SY_TRUE) {
910 free(buf);
911 if (isatty(fileno(stderr))) {
912 if (wasoutput) { fprintf(stderr, "\r\x1b[Kaborted!\n"); fflush(stderr); }
913 }
914 return SY_ERROR;
915 }
916 }
917 free(buf);
918 if (isatty(fileno(stderr))) {
919 if (wasoutput) { fprintf(stderr, "\r\x1b[K"); fflush(stderr); }
920 }
921 #else
922 if (fileSize > 0) {
923 # ifdef __linux
924 /*if (fallocate(fd, FALLOC_FL_KEEP_SIZE, 0, fileSize)) return SY_ERROR;*/
925 /*if (fallocate(fd, 0, 0, fileSize)) return SY_ERROR;*/
926 if (posix_fallocate(fd, 0, fileSize)) return SY_ERROR;
927 # else
928 if (posix_fallocate(fd, 0, fileSize)) return SY_ERROR;
929 # endif
930 }
931 #endif
932
933 return SySeekFile(fd, 0);
934 }
935
936
937 TSyResult SyDrvOpenFile (TSyState *state) {
938 int fd, sto = 0; int64_t len = -1;
939 char *s, tmp[32];
940
941 if (replyOnly != SY_FALSE) return SY_OK;
942
943 if (oCanRename) {
944 if (!oOutFileName || !oOutFileName[0]) {
945 if (oOutFileName) free(oOutFileName);
946 if (state->httpFName && *state->httpFName) {
947 char *t;
948 //
949 if ((t = strrchr(state->httpFName, '/')) != NULL) {
950 oOutFileName = SyStrDup(t[1]?t+1:"unnamed_file.bin");
951 } else {
952 oOutFileName = SyStrDup(state->httpFName);
953 }
954 } else {
955 oOutFileName = SyStrDup(state->url->file);
956 }
957 #ifdef SYREN_USE_SCRIPT
958 SyScriptCallback("Event.CheckFileName", state->pfn);
959 #endif
960 /*if (scAbort == SY_TRUE) goto done;*/
961 if (!oOutFileName || !oOutFileName[0]) {
962 if (oOutFileName) free(oOutFileName);
963 oOutFileName = SyStrNew((cfgDefaultName&&*cfgDefaultName)?cfgDefaultName:"index.html", -1);
964 }
965 if (!oOutFileName) { SyMessage(state->pfn, SY_MSG_ERROR, "memory error"); return SY_ERROR; }
966 }
967 /* check for duplicates */
968 fd = SyOpenFile(oOutFileName, SY_FMODE_READ, SY_FALSE);
969 if (fd >= 0) {
970 int f = -1; s = NULL;
971 do { f++;
972 SyCloseFile(fd); if (s) free(s);
973 s = SySPrintf("%s.%i", oOutFileName, f);
974 if (!s) { SyMessage(state->pfn, SY_MSG_ERROR, "memory error"); return SY_ERROR; }
975 fd = SyOpenFile(s, SY_FMODE_READ, SY_FALSE);
976 } while (fd >= 0);
977 free(oOutFileName); oOutFileName = s;
978 }
979 }
980 /* open/create file */
981 if (!oOutFileName || !oOutFileName[0]) { SyMessage(state->pfn, SY_MSG_ERROR, "invalid file name"); return SY_ERROR; }
982 if (state->firstByte > 0) {
983 if (!strcmp(oOutFileName, "-")) { SyMessage(state->pfn, SY_MSG_ERROR, "can't resume stdout"); return SY_ERROR; }
984 fd = SyOpenFile(oOutFileName, SY_FMODE_WRITE, SY_FALSE);
985 if (fd >= 0) {
986 len = SyFileSize(fd);
987 if (len < 0) { SyCloseFile(fd); SyMessage(state->pfn, SY_MSG_ERROR, "can't resume: file error"); return SY_ERROR; }
988 if (len < state->firstByte) { SyCloseFile(fd); SyMessage(state->pfn, SY_MSG_ERROR, "can't resume: corrupted file"); return SY_ERROR; }
989 if (SySeekFile(fd, state->firstByte) != SY_OK) {
990 SyCloseFile(fd);
991 SyMessage(state->pfn, SY_MSG_ERROR, "can't resume: file error");
992 return SY_ERROR;
993 }
994 SyLong2StrComma(tmp, state->firstByte);
995 SyMessage(state->pfn, SY_MSG_MSG, "resuming %s at %s", oOutFileName, tmp);
996 }
997 } else {
998 if (!strcmp(oOutFileName, "-")) { fd = 0; sto = 1; oNoStateFile = SY_TRUE; }
999 else {
1000 SyDeleteFile(oOutFileName);
1001 fd = SyOpenFile(oOutFileName, SY_FMODE_WRITE, SY_TRUE);
1002 /* preallocating w/o state file means no resumes, so disable it */
1003 if (fd > 0 && oPreallocSpace != SY_FALSE && state->fileSize > 0 && oNoStateFile != SY_TRUE) {
1004 SyLong2StrComma(tmp, state->fileSize);
1005 SyMessage(state->pfn, SY_MSG_MSG, "preallocating %s bytes", tmp);
1006 if (SyXPrealloc(fd, state->fileSize) != SY_OK) {
1007 SyMessage(state->pfn, SY_MSG_ERROR, "preallocating failed");
1008 SyCloseFile(fd);
1009 SyDeleteFile(oOutFileName);
1010 return SY_ERROR;
1011 }
1012 }
1013 }
1014 }
1015 if (fd < 0) { SyMessage(state->pfn, SY_MSG_ERROR, "can't open file %s", oOutFileName); return SY_ERROR; }
1016 if (state->firstByte <= 0 || sto) SyMessage(state->pfn, SY_MSG_MSG, "writing to %s", sto?"stdout":oOutFileName);
1017 state->udatai = sto?-2:fd;
1018 if (!oStateFileName || !oStateFileName[0]) {
1019 if (oStateFileName) free(oStateFileName);
1020 oStateFileName = SySPrintf("%s.syren", oOutFileName);
1021 if (!oStateFileName) SyMessage(state->pfn, SY_MSG_WARNING, "can't create state file name");
1022 else if (oNoStateFile != SY_TRUE) SySaveState(oStateFileName, state);
1023 }
1024
1025 return SY_OK;
1026 }
1027
1028
1029 TSyResult SyDrvCloseFile (TSyState *state) {
1030 if (replyOnly != SY_FALSE) return SY_OK;
1031 if (state->udatai >= 0) {
1032 SyMessage(state->pfn, SY_MSG_MSG, "closing %s%s", oOutFileName,
1033 (state->error==SY_FALSE?"":" (incomplete file)"));
1034 SyCloseFile(state->udatai);
1035 state->udatai = -1;
1036 if (oStateFileName && *oStateFileName && oNoStateFile != SY_TRUE) SySaveState(oStateFileName, state);
1037 }
1038
1039 return SY_OK;
1040 }
1041
1042
1043 TSyResult SyDrvWriteFile (TSyState *state, void *buf, int bufSize) {
1044 int fd;
1045
1046 if (replyOnly != SY_FALSE) return SY_OK;
1047 fd = state->udatai; if (fd == -2) fd = 1; /* stdout */
1048 if (fd >= 0) {
1049 if (SyWriteFile(fd, buf, bufSize) != SY_OK) return SY_ERROR;
1050 }
1051 if (oStateFileName && *oStateFileName && oNoStateFile != SY_TRUE && state->udatai >= 0) {
1052 if (cfgSaveInterval >= 1.0 && SyGetTimeD()-lastSSave >= cfgSaveInterval) {
1053 SySaveState(oStateFileName, state);
1054 lastSSave = SyGetTimeD();
1055 fsync(state->udatai);
1056 }
1057 }
1058
1059 return SY_OK;
1060 }
1061
1062
1063 TSyResult SyDrvNoResume (TSyState *state) {
1064 if (replyOnly != SY_FALSE) {
1065 SyMessage(state->pfn, SY_MSG_WARNING, "can't resume");
1066 return SY_OK;
1067 }
1068 if (state->udatai >= 0) {
1069 if (state->udatai) SyCloseFile(state->udatai);
1070 SyDeleteFile(oOutFileName);
1071 state->udatai = SyOpenFile(oOutFileName, SY_FMODE_WRITE, SY_TRUE);
1072 if (state->udatai < 0) {
1073 SyMessage(state->pfn, SY_MSG_WARNING, "can't create file %s", oOutFileName);
1074 return SY_ERROR;
1075 }
1076 }
1077 SyMessage(state->pfn, SY_MSG_WARNING, "can't resume file %s, file truncated", oOutFileName);
1078
1079 return SY_OK;
1080 }
1081
1082
1083 TSyResult CheckResume (TSyState *state) {
1084 int fd; int64_t len;
1085
1086 SyMessage(state->pfn, SY_MSG_NOTICE, "checking file for resuming");
1087 if (!strcmp(oOutFileName, "-")) { SyMessage(state->pfn, SY_MSG_ERROR, "can't resume stdout"); return SY_ERROR; }
1088 fd = SyOpenFile(oOutFileName, SY_FMODE_WRITE, SY_FALSE);
1089 if (fd < 0) { SyMessage(state->pfn, SY_MSG_ERROR, "resume: can't open file %s", oOutFileName); return SY_ERROR; }
1090 len = SyFileSize(fd);
1091 SyCloseFile(fd);
1092 if (len < 0) { SyMessage(state->pfn, SY_MSG_ERROR, "can't determine file size for %s", oOutFileName); return SY_ERROR; }
1093 state->firstByte = len;
1094 SyMessage(state->pfn, SY_MSG_NOTICE, "resume check passed");
1095
1096 return SY_OK;
1097 }
1098
1099
1100 static TSyCfg *curcfg = NULL;
1101 static char *cfgSaveBuf = NULL;
1102 static size_t cfgSaveBufSize = 0;
1103 static TSyPrintStr prs;
1104
1105
1106 static void loadDefaultConfig (void) {
1107 const char *home;
1108 char *s = NULL;
1109 //
1110 SyCfgClear(curcfg);
1111 SyDefaultCfg(curcfg);
1112 if ((home = getenv("HOME")) != NULL) {
1113 s = SySPrintf("%s/.syren/.syrenrc", home);
1114 if (access(s, R_OK) == 0) {
1115 SyCfgLoad(curcfg, s, &prs);
1116 goto quit;
1117 }
1118 free(s);
1119 s = SySPrintf("%s/.syrenrc", home);
1120 if (access(s, R_OK) == 0) {
1121 SyCfgLoad(curcfg, s, &prs);
1122 goto quit;
1123 }
1124 }
1125 if (access(".syrenrc", R_OK) == 0) SyCfgLoad(curcfg, ".syrenrc", &prs);
1126 else if (access("/etc/syren/.syrenrc", R_OK) == 0) SyCfgLoad(curcfg, "/etc/syren/.syrenrc", &prs);
1127 quit:
1128 if (s != NULL) free(s);
1129 }
1130
1131
1132 static void storeConfig (void) {
1133 FILE *fl = open_memstream(&cfgSaveBuf, &cfgSaveBufSize);
1134 SyCfgSaveFile(curcfg, fl, &prs);
1135 fclose(fl);
1136 }
1137
1138
1139 static void restoreConfig (void) {
1140 if (cfgSaveBuf != NULL) {
1141 FILE *fl;
1142 //
1143 fl = fmemopen(cfgSaveBuf, cfgSaveBufSize, "r");
1144 SyCfgLoadFile(curcfg, fl, &prs);
1145 fclose(fl);
1146 }
1147 }
1148
1149
1150 static char *getPerSiteConfigFileName (const char *host) {
1151 const char *t = getenv("HOME");
1152 //
1153 while (*host) {
1154 char *cfgname = SySPrintf("%s/%s/%s.rc", (t != NULL ? t : "/etc"), (t != NULL ? ".syren" : "syren"), host), *dot;
1155 //
1156 if (access(cfgname, R_OK) == 0) return cfgname;
1157 free(cfgname);
1158 if ((dot = strchr(host, '.')) == NULL) break;
1159 host = dot+1;
1160 }
1161 //
1162 return NULL;
1163 }
1164
1165
1166 static void loadPerSiteConfig (void) {
1167 TSyURL *url = SyURLNew();
1168 //
1169 if (SyURLParse(url, oURL) == SY_OK) {
1170 char *cfgname = getPerSiteConfigFileName(url->host);
1171 //
1172 if (cfgname != NULL) {
1173 SyCfgLoad(curcfg, cfgname, &prs);
1174 free(cfgname);
1175 }
1176 }
1177 //
1178 SyURLFree(url);
1179 }
1180
1181
1182 int main (int argc, char *argv[]) {
1183 int mainres = 1;
1184 char *s, *t;
1185 TSyResult res;
1186 int retCnt;
1187 int userBreak = 0;
1188
1189 prs.print = &SyPrintStr; prs.udata = NULL;
1190
1191 t = getenv("TERM");
1192 if (t && (!strcmp(t, "rxvt") || !strcmp(t, "mrxvt") || !strcmp(t, "xterm") || !strcmp(t, "eterm"))) doRXVT = SY_TRUE;
1193
1194 curcfg = SyCfgNew();
1195 if (!curcfg) { SyMessage(&prs, SY_MSG_ERROR, "memory error"); }
1196
1197 state = SyNew();
1198 if (!state) { SyMessage(&prs, SY_MSG_ERROR, "memory error"); }
1199 prs.udata = (void *)state;
1200
1201 oURLList = SyKVListNew();
1202 if (!oURLList) goto done;
1203 oURLList->casesens = 1;
1204
1205 loadDefaultConfig();
1206
1207 if (ParseCmdLine(&prs, curcfg, argc, argv) != SY_OK) goto done;
1208
1209 SyMessage(&prs, SY_MSG_MSG, "%s", SYREN_VERSION_DATETIME_STRING);
1210
1211 #ifdef SYREN_USE_SCRIPT
1212 SyScriptInit(&prs);
1213 SyScriptLoadInit(argv[0], &prs);
1214 #endif
1215
1216 if (!oStateFileName) {
1217 if (!oURLList->first) { SyMessage(&prs, SY_MSG_ERROR, "URL?"); goto done; }
1218 }
1219
1220 if (cfgMaxBufferSize < 1) { SyMessage(&prs, SY_MSG_ERROR, "buffer size too small"); goto done; }
1221 if (cfgTimeout < 1) { SyMessage(&prs, SY_MSG_ERROR, "timeout too small"); goto done; }
1222
1223 if (writeCfg == SY_TRUE) {
1224 SyMessage(&prs, SY_MSG_MSG, "generating sample config");
1225 SyCfgSave(curcfg, ".syrenrc", &prs);
1226 mainres = 0;
1227 goto done;
1228 }
1229
1230 if (SySocketInit() != SY_OK) goto done;
1231
1232 storeConfig();
1233
1234 state->udatai = -1;
1235
1236 state->ftpUseConnect = cfgFTPUseConnect;
1237
1238 state->pfn = &prs;
1239 state->fnprog = PrintProgress;
1240 state->fnprephdrs = SyDrvAddHeaders;
1241 state->fngothdrs = SyDrvGotHeaders;
1242
1243 state->fnopen = SyDrvOpenFile;
1244 state->fnclose = SyDrvCloseFile;
1245 state->fnwrite = SyDrvWriteFile;
1246 state->fnnoresume = SyDrvNoResume;
1247
1248 state->postData = oPostData; oPostData = NULL;
1249
1250 do {
1251 static int wosDo = 0;
1252 res = SY_ERROR;
1253 userBreak = 0;
1254
1255 restoreConfig();
1256
1257 if (!oStateFileName) {
1258 if (!oURLList->first) break;
1259 oURL = SyStrDup(oURLList->first->key);
1260 SyKVListDelete(oURLList, oURLList->first->key);
1261 if (!oURL) {
1262 SyMessage(&prs, SY_MSG_ERROR, "memory error");
1263 goto done;
1264 }
1265 /*if (!oURL) { SyMessage(&prs, SY_MSG_ERROR, "URL?"); goto done; }*/
1266 } else {
1267 SyKVListClear(oURLList);
1268 SyMessage(&prs, SY_MSG_MSG, "restoring state from file %s", oStateFileName);
1269 if (oStateFileName && *oStateFileName) {
1270 s = SyLoadState(oStateFileName, state);
1271 if (!s || !s[0]) {
1272 if (s) free(s);
1273 SyMessage(&prs, SY_MSG_ERROR, "can't restore state");
1274 goto done;
1275 }
1276 if (oOutFileName) free(oOutFileName);
1277 oOutFileName = s;
1278 }
1279 if (dumpState == SY_TRUE) { DumpState(state); goto done; }
1280 }
1281
1282 loadPerSiteConfig();
1283
1284 if (replyOnly == SY_FALSE && oResume == SY_TRUE && CheckResume(state) != SY_OK) goto done;
1285
1286 //if (strcasestr(oURL, "worldofspectrum.org")) optWOS = 15; // hehe
1287 if (wosDo && optWOS > 0) { SyMessage(&prs, SY_MSG_MSG, "WOS waiting: %d", optWOS); sleep(optWOS); }
1288 wosDo = 1;
1289
1290 #ifdef SYREN_USE_SCRIPT
1291 /*
1292 if (!(state->url = SyURLNew())) {
1293 SyMessage(&prs, SY_MSG_ERROR, "memory error!");
1294 goto done;
1295 }
1296 if (SyURLParse(state->url, oURL) != SY_OK) {
1297 SyMessage(state->pfn, SY_MSG_ERROR, "invalid URL");
1298 return SY_ERROR;
1299 }
1300 SyScriptCallback("Event.BeforePrepare", &prs);
1301 s = SyURL2StrEx(state->url, SY_TRUE, SY_FALSE);
1302 if (!s) {
1303 SyMessage(&prs, SY_MSG_ERROR, "memory error!");
1304 goto done;
1305 }
1306 SyStrFree(oURL); oURL = s; s = NULL;
1307 SyURLFree(state->url); state->url = NULL;
1308 fprintf(stderr, "URL: <%s>\n", oURL);
1309 */
1310 #endif
1311 if (SyPrepare(state, oURL,
1312 cfgUseProxyHTTP?cfgProxyHTTP:NULL,
1313 cfgUseProxyFTP?cfgProxyFTP:NULL, cfgIFace) != SY_OK) { SyMessage(&prs, SY_MSG_ERROR, "SyPrepare() failed"); goto done; }
1314 #ifdef SYREN_USE_SCRIPT
1315 SyScriptCallback("Event.AfterPrepare", &prs);
1316 if (scAbort == SY_TRUE) goto done;
1317 #endif
1318
1319 if (uEncodeDecode < 0 ||
1320 (cfgDecodeFTPURL == SY_TRUE && state->url->proto == SY_PROTO_FTP)) {
1321 s = SyURLDecode(state->url->dir);
1322 if (!s) { SyMessage(&prs, SY_MSG_ERROR, "SyURLDecode() failed"); goto done; }
1323 SyStrFree(state->url->dir); state->url->dir = s;
1324 s = SyURLDecode(state->url->file);
1325 if (!s) { SyMessage(&prs, SY_MSG_ERROR, "SyURLDecode() failed"); goto done; }
1326 SyStrFree(state->url->file); state->url->file = s;
1327 } else if (uEncodeDecode > 0) {
1328 s = SyURLEncode(state->url->dir);
1329 if (!s) { SyMessage(&prs, SY_MSG_ERROR, "SyURLEncode() failed"); goto done; }
1330 SyStrFree(state->url->dir); state->url->dir = s;
1331 s = SyURLEncode(state->url->file);
1332 if (!s) { SyMessage(&prs, SY_MSG_ERROR, "SyURLEncode() failed"); goto done; }
1333 SyStrFree(state->url->file); state->url->file = s;
1334 }
1335
1336 signal(SIGHUP, SaveStateSignal);
1337 signal(SIGUSR1, SaveStateSignal);
1338
1339 signal(SIGINT, BreakSignal);
1340 signal(SIGQUIT, BreakSignal);
1341 signal(SIGTERM, BreakSignal);
1342
1343 retCnt = cfgReconnectAttempts;
1344
1345 state->initBufferSize = cfgBufferSize;
1346 state->maxBufferSize = cfgMaxBufferSize;
1347 state->ioTimeout = cfgTimeout;
1348 state->maxRedirects = cfgMaxRedirects;
1349 if (cfgUAStr && *cfgUAStr) {
1350 SyStrFree(state->userAgent);
1351 state->userAgent = SyStrNew(cfgUAStr, -1);
1352 }
1353 while (1) {
1354 ctrlCPressed = SY_FALSE;
1355 curState = state;
1356 state->allowOnly2XX = optAnyCode=SY_TRUE ? SY_FALSE : SY_TRUE;
1357 res = SyBegin(state);
1358 if (replyOnly != SY_FALSE) break;
1359 if (res == SY_OK) res = SyRun(state);
1360 curState = NULL;
1361 if (res == SY_OK) break;
1362 if (state->interrupted == SY_TRUE) {
1363 userBreak = 1;
1364 if (oStateFileName && *oStateFileName && state->status == SY_STATUS_DOWNLOADING && oNoStateFile != SY_TRUE) {
1365 SyMessage(&prs, SY_MSG_WARNING, "user break, saving state");
1366 SySaveState(oStateFileName, state);
1367 } else SyMessage(&prs, SY_MSG_WARNING, "user break");
1368 break;
1369 }
1370 if (retCnt < 1) break;
1371 if (state->status != SY_STATUS_DOWNLOADING && oForceRetry != SY_TRUE) break;
1372 SyMessage(&prs, SY_MSG_ERROR, "download failed");
1373 if (cfgReconnectDelay > 0) {
1374 SyMessage(&prs, SY_MSG_MSG, "waiting %i seconds", cfgReconnectDelay);
1375 //SySleep(cfgReconnectDelay);
1376 DoWait(cfgReconnectDelay);
1377 if (ctrlCPressed != SY_FALSE) {
1378 SyMessage(&prs, SY_MSG_WARNING, "user break");
1379 res = SY_ERROR;
1380 break;
1381 }
1382 }
1383 SyReset(state);
1384 SyMessage(&prs, SY_MSG_MSG, "restarting");
1385 state->firstByte += state->currentByte; /* start from here */
1386 oCanRename = SY_FALSE;
1387 retCnt--;
1388 }
1389 #ifdef SYREN_USE_SCRIPT
1390 if (res == SY_OK) SyScriptCallback("Event.DownloadComplete", &prs);
1391 else SyScriptCallback("Event.DownloadFailed", &prs);
1392 if (scAbort == SY_TRUE) goto done;
1393 #endif
1394 SyEnd(state);
1395 if (res == SY_OK) {
1396 SyMessage(&prs, SY_MSG_MSG, "download complete");
1397 if (oCanRemoveState && oStateFileName && *oStateFileName) {
1398 if (SyDeleteFile(oStateFileName) == SY_OK) SyMessage(&prs, SY_MSG_NOTICE, "state file removed");
1399 }
1400 } else SyMessage(&prs, SY_MSG_ERROR, "download failed");
1401
1402 mainres = (res==SY_OK)?0:1;
1403
1404 done:
1405 SyStrFree(oURL); oURL = NULL;
1406 SyStrFree(oOutFileName); oOutFileName = NULL;
1407 SyStrFree(oStateFileName); oStateFileName = NULL;
1408 oCanRename = SY_TRUE;
1409 } while (!userBreak);
1410
1411 if (cfgSaveBuf != NULL) free(cfgSaveBuf);
1412 SyFree(state);
1413 SyCfgFree(curcfg);
1414 if (oURLList) SyKVListFree(oURLList);
1415 if (cfgDefaultName) SyStrFree(cfgDefaultName);
1416 if (cfgIFace) SyStrFree(cfgIFace);
1417 if (cfgProxyHTTP) SyStrFree(cfgProxyHTTP);
1418 if (cfgProxyFTP) SyStrFree(cfgProxyFTP);
1419 if (cfgRefererStr) SyStrFree(cfgRefererStr);
1420 if (cfgUAStr) SyStrFree(cfgUAStr);
1421 SyStrFree(oURL);
1422 SyStrFree(oOutFileName);
1423 SyStrFree(oStateFileName);
1424 if (oPostData) SyStrFree(oPostData);
1425 if (oSendCookies) SyStrFree(oSendCookies);
1426 if (oCookieDump) SyStrFree(oCookieDump);
1427 if (oAddonHeaders) SyStrFree(oAddonHeaders);
1428
1429 #ifdef SYREN_USE_SCRIPT
1430 SyScriptDeinit();
1431 #endif
1432
1433 SySocketShutdown();
1434 return mainres;
1435 }
syren