From 69889f01f042d24650acee952ee124f64b0c499e Mon Sep 17 00:00:00 2001 From: ketmar Date: Wed, 31 Jan 2018 21:28:12 +0000 Subject: [PATCH] it is now possible to pass flibusta URL to xreader FossilOrigin-Name: 89e861978441930d83083d3ba42c7e1a38ea94713bc5c35ac29f1b9dd7b684f8 --- xreader.d | 266 +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 265 insertions(+), 1 deletion(-) diff --git a/xreader.d b/xreader.d index 63d300c..cbb79a7 100644 --- a/xreader.d +++ b/xreader.d @@ -14,11 +14,13 @@ * You should have received a copy of the GNU General Public License * along with this program. If not, see . */ -module xreader; +module xreader is aliced; import core.time; import std.concurrency; +import std.net.curl; +import std.regex; import arsd.simpledisplay; import arsd.image; @@ -27,6 +29,7 @@ import iv.nanovg; import iv.nanovg.oui.blendish; import iv.nanovg.perf; +import iv.sdbm; import iv.strex; import iv.cmdcongl; @@ -1227,6 +1230,257 @@ void run () { // ////////////////////////////////////////////////////////////////////////// // +struct FlibustaUrl { + string fullUrl; // onion + string id; + + this (const(char)[] aurl) { + aurl = aurl.xstrip(); + auto flibustaRE = regex(`^(?:https?://)?(?:www\.)?flibusta\.[^/]+/b/(\d+)`); + auto ct = aurl.matchFirst(flibustaRE); + if (!ct.empty) { + import std.format : format; + fullUrl = "http://flibustahezeous3.onion/b/%s/fb2".format(ct[1]); + id = ct[1].idup; + } + } + + @property bool valid () const pure nothrow @safe @nogc { pragma(inline, true); return (fullUrl.length > 0 && id.length > 0); } +} + + +// ////////////////////////////////////////////////////////////////////////// // +__gshared SDBM dbCache = null; + + +void dbOpenCache () { + if (dbCache is null) { + import std.file : exists, mkdirRecurse; + import std.path; + string xfn = buildPath(RcDir, "cache"); + xfn.mkdirRecurse(); + xfn = buildPath(xfn, ".cache.db"); + dbCache = new SDBM(xfn, SDBM.WRITER|SDBM.CREAT|SDBM.NOLCK); + } +} + + +string dbFindInCache() (in auto ref FlibustaUrl furl) { + if (!furl.valid) return null; + dbOpenCache(); + return dbCache.get!string("/files/"~furl.id~"/name"); +} + + +void dbPutToCache() (in auto ref FlibustaUrl furl, const(char)[] fname) { + if (!furl.valid || fname.length == 0) return; + dbOpenCache(); + dbCache.put("/files/"~furl.id~"/name", fname); +} + + +// ////////////////////////////////////////////////////////////////////////// // +// returns file name +string fileDown() (in auto ref FlibustaUrl furl) { + if (!furl.valid) return null; + + auto url = furl.fullUrl.xstrip(); + if (url.length == 0) return null; + + string cachedFName = dbFindInCache(furl); + if (cachedFName.length) return cachedFName; + + auto protoRE = regex(`^([^:/]+):`); + auto hostRE = regex(`^(?:[^:/]+)://([^/]+)`); + + string host; + string realUrl; + + auto protoMt = url.matchFirst(protoRE); + if (protoMt.empty) { + if (url[0] == '/') realUrl = "http://"~url; else realUrl = "http:"~url; + } else { + realUrl = url; + } + + auto hostMt = realUrl.matchFirst(hostRE); + if (hostMt.empty) { + writeln("ERROR: cannot get host from url: ", url); + return null; + } + host = hostMt[1]; + + // content-disposition: attachment; filename="Divov_Sled-zombi.1lzb6Q.96382.fb2.zip" + auto cdRE0 = regex(`^\s*attachment\s*;\s*filename="(.+?)"`, "i"); + auto cdRE1 = regex(`^\s*attachment\s*;\s*filename=([^;]+?)`, "i"); + + auto http = HTTP(host); + http.method = HTTP.Method.get; + http.url = realUrl; + + string fname = null; + string tmpfname = null; + string fnps = null; + bool alreadyDowned = false; + VFile fo; + + http.onReceiveHeader = delegate (in char[] key, in char[] value) { + //writeln(key ~ ": " ~ value); + if (key.strEquCI("content-disposition")) { + auto ct = value.matchFirst(cdRE0); + if (ct.empty) ct = value.matchFirst(cdRE1); + if (ct[1].length) { + auto fnp = ct[1].xstrip; + auto lslpos = fnp.lastIndexOf('/'); + if (lslpos > 0) fnp = fnp[lslpos+1..$]; + if (fnp.length == 0) { + fname = null; + } else { + import std.file : exists, mkdirRecurse; + import std.path; + string xfn = buildPath(RcDir, "cache"); + xfn.mkdirRecurse(); + fname = buildPath(xfn, fnp); + fnps = fnp.idup; + tmpfname = fname~".down.part"; + /* + if (fname.exists) { + alreadyDowned = true; + throw new Exception("already here"); + //throw new FileAlreadyDowned("already here"); + } + */ + write("\r", fnp, " [", realUrl, "]\e[K"); + } + } + } + }; + + http.onReceive = delegate (ubyte[] data) { + if (!fo.isOpen) { + if (fname.length == 0) throw new Exception("no file name found in headers"); + //writeln(" downloading to ", fname); + fo = VFile(tmpfname, "w"); + } + fo.rawWriteExact(data); + return data.length; + }; + + MonoTime lastProgTime = MonoTime.zero; + enum BarLength = 68; + bool doProgUpdate = true; + char[1024] buf = void; + int oldDots = -1, oldPrc = -1; + uint bufpos = 0; + int stickPos = 1; + immutable string stickStr = `|/-\`; + + // will set `doProgUpdate`, and update `oldXXX` + void buildPBar (usize dlTotal, usize dlNow) { + void put (const(char)[] s...) nothrow { + foreach (char ch; s) if (bufpos < buf.length) buf[bufpos++] = ch; + } + void putprc (int prc) { + if (prc < 0) prc = 0; else if (prc > 100) prc = 100; + char[3] xbuf = ' '; + int xpos = cast(int)xbuf.length-1; + do { + xbuf[xpos--] = cast(char)('0'+prc%10); + prc /= 10; + } while (xpos >= 0 && prc != 0); + put(xbuf[]); + put('%'); + } + bufpos = 0; + put("\r"); + put(fnps); + put(" ["); + auto barpos = bufpos; + foreach (immutable _; 0..BarLength) put(" "); + put("] "); + if (dlTotal > 0) { + int prc = cast(int)(cast(usize)100*dlNow/dlTotal); + if (prc < 0) prc = 0; + if (prc > 100) prc = 100; + int dots = cast(int)(cast(usize)BarLength*dlNow/dlTotal); + if (dots < 0) dots = 0; + if (dots > BarLength) dots = BarLength; + if (prc != oldPrc || dots != oldDots) { + doProgUpdate = true; + oldPrc = prc; + oldDots = dots; + } + putprc(prc); + // dots + foreach (immutable dp; 0..dots) if (barpos+dp < buf.length) buf[barpos+dp] = '.'; + } else { + put("? \x08\x08\x08\x08"); + if (oldDots != -1 || oldPrc != -1) doProgUpdate = true; + oldDots = -1; + oldPrc = -1; + } + } + + http.onProgress = delegate (usize dltotal, usize dlnow, usize ultotal, usize ulnow) { + //writeln("Progress ", dltotal, ", ", dlnow, ", ", ultotal, ", ", ulnow); + if (fname.length == 0) { + auto ct = MonoTime.currTime; + if ((ct-lastProgTime).total!"msecs" >= 100) { + write("\x08", stickStr[stickPos]); + stickPos = (stickPos+1)%cast(int)stickStr.length; + lastProgTime = ct; + } + return 0; + } + buildPBar(dltotal, dlnow); + if (doProgUpdate) { write(buf[0..bufpos], "\e[K"); doProgUpdate = false; } + //if (dltotal == 0) return 0; + //auto ct = MonoTime.currTime; + //if ((ct-lastProgTime).total!"msecs" < 1000) return 0; + //lastProgTime = ct; + //writef("\r%s [%s] -- [%s/%s] %3u%%\e[K", fnps, host, intWithCommas(dlnow), intWithCommas(dltotal), 100UL*dlnow/dltotal); + return 0; + }; + + auto ore = regex(`^https?://[^/]*\.onion/`); + if (!realUrl.matchFirst(ore).empty) { + http.proxyType = HTTP.CurlProxy.socks5_hostname; + http.proxy = "127.0.0.1"; + http.proxyPort = 9050; + } + + try { + //write("downloading from [", host, "]: ", realUrl, " ... "); + write("downloading from Flibusta: ", realUrl, " ... ", stickStr[0]); + http.perform(); + buildPBar(100, 100); + writeln(buf[0..bufpos], "\e[K"); + //write("\r\e[K"); + } catch (Exception e) { + if (/*cast(FileAlreadyDowned)e*/alreadyDowned) { + write("\r", fname, " already downloaded.\e[K"); + return fname; // already here + } + import std.exception : collectException; + import std.file : remove; + collectException(tmpfname.remove); + throw e; + } + + if (fo.isOpen) { + // something was downloaded, rename it + import std.file : rename; + fo.close(); + rename(tmpfname, fname); + dbPutToCache(furl, fname); + return fname; + } + + return null; +} + + +// ////////////////////////////////////////////////////////////////////////// // void main (string[] args) { import std.path; @@ -1242,7 +1496,17 @@ void main (string[] args) { } if (lnn.length) args ~= lnn; } catch (Exception) {} + } else { + if (args.length != 2) assert(0, "invalid number of arguments"); + auto furl = FlibustaUrl(args[1]); + if (furl.valid) { + scope(exit) delete dbCache; + string fn = fileDown(furl); + if (fn.length == 0) assert(0, "can't download file"); + args[1] = fn; + } } + if (args.length == 1) assert(0, "no filename"); readConfig(); -- 2.11.4.GIT