compile.lua

   1 #!/usr/bin/lua
   2
   3 local do_zone_map = false
   4 local do_errors = true
   5
   6 local do_compile = true
   7 local do_questtables = true
   8 local do_flight = true
   9
  10 local do_compress = false
  11 local do_serialize = true
  12
  13 local dbg_data = false
  14
  15 --local s = 1048
  16 --local e = 1048
  17 local e = 100
  18
  19 require("compile_lib")
  20
  21 --[[
  22 *****************************************************************
  23 Chain head
  24 ]]
  25
  26 local chainhead = ChainBlock_Create("parse", nil,
  27   function () return {
  28     Data = function (self, key, subkey, value, Output)
  29       local gzx = gzio.open(key, "r")
  30       local gzd = gzx:read("*a")
  31       gzx:close()
  32       gzx = nil
  33       local dat = pluto.unpersist({}, gzd)
  34       gzd = nil
  35       assert(dat)
  36
  37       if do_errors and dat.errors then
  38         for k, v in pairs(dat.errors) do
  39           if k ~= "version" then
  40             for _, d in pairs(v) do
  41               d.key = k
  42               d.fileid = value.fileid
  43               Output(d.local_version, nil, d, "error")
  44             end
  45           end
  46         end
  47       end
  48
  49       local qhv, wowv, locale, faction = string.match(dat.signature, "([0-9.]+) on ([0-9.]+)/([a-zA-Z]+)/([12])")
  50       local v = dat.data
  51       if qhv and wowv and locale and faction
  52         --and not sortversion("0.80", qhv) -- hacky hacky
  53       then
  54         --[[if v.compressed then
  55           local deco = "return " .. LZW.Decompress(v.compressed, 256, 8)
  56           print(#v.compressed, #deco)
  57           local tx = loadstring(deco)()
  58           assert(tx)
  59           v.compressed = nil
  60           for tk, tv in pairs(tx) do
  61             v[tk] = tv
  62           end
  63         end]]
  64         assert(not v.compressed)
  65
  66         -- quests!
  67         if do_compile and do_questtables and v.quest then for qid, qdat in pairs(v.quest) do
  68           qdat.fileid = value.fileid
  69           qdat.locale = locale
  70           Output(string.format("%d", qid), qhv, qdat, "quest")
  71         end end
  72
  73         -- items!
  74         if do_compile and do_questtables and v.item then for iid, idat in pairs(v.item) do
  75           idat.fileid = value.fileid
  76           idat.locale = locale
  77           Output(tostring(iid), qhv, idat, "item")
  78         end end
  79
  80         -- monsters!
  81         if do_compile and do_questtables and v.monster then for mid, mdat in pairs(v.monster) do
  82           mdat.fileid = value.fileid
  83           mdat.locale = locale
  84           Output(tostring(mid), qhv, mdat, "monster")
  85         end end
  86
  87         -- objects!
  88         --[[if do_compile and do_questtables and v.object then for oid, odat in pairs(v.object) do
  89           odat.fileid = value.fileid
  90           Output(string.format("%s@@%s", oid, locale), qhv, odat, "object")
  91         end end]]
  92
  93         -- flight masters!
  94         if do_compile and do_flight and v.flight_master then for fmname, fmdat in pairs(v.flight_master) do
  95           if type(fmdat.master) == "string" then continue end  -- I don't even know how this is possible
  96           Output(string.format("%s@@%s@@%s", faction, fmname, locale), qhv, {dat = fmdat, wowv = wowv}, "flight_master")
  97         end end
  98
  99         -- flight times!
 100         if do_compile and do_flight and v.flight_times then for ftname, ftdat in pairs(v.flight_times) do
 101           Output(string.format("%s@@%s@@%s", ftname, faction, locale), qhv, ftdat, "flight_times")
 102         end end
 103
 104         -- zones!
 105         if locale == "enUS" and do_zone_map and v.zone then for zname, zdat in pairs(v.zone) do
 106           local items = {}
 107           local lv = loc_version(qhv)
 108
 109           for _, key in pairs({"border", "update"}) do
 110             if items and zdat[key] then for idx, chunk in pairs(zdat[key]) do
 111               if math.mod(#chunk, 11) ~= 0 then items = nil end
 112               if not items then break end -- abort, abort
 113
 114               assert(math.mod(#chunk, 11) == 0, tostring(#chunk))
 115               for point = 1, #chunk, 11 do
 116                 local pos = convert_loc(slice_loc(string.sub(chunk, point, point + 10), lv), locale)
 117                 if pos then
 118                   if not zonecolors[zname] then
 119                     local r, g, b = math.ceil(math.random(32, 255)), math.ceil(math.random(32, 255)), math.ceil(math.random(32, 255))
 120                     zonecolors[zname] = r * 65536 + g * 256 + b
 121                   end
 122                   pos.zonecolor = zonecolors[zname]
 123                   if pos.p and pos.x and pos.y then  -- These might be invalid if there are nils embedded in the string. They might still be useful with only one or two nils, but I've got a bunch of data and don't really need more.
 124                     if not valid_pos(pos) then
 125                       items = nil
 126                       break
 127                     end
 128
 129                     pos.c = QuestHelper_ZoneLookup[ite.p][1]
 130                     table.insert(items, pos)
 131                   end
 132                 end
 133               end
 134             end end
 135           end
 136
 137           if items then for _, v in pairs(items) do
 138             v.fileid = value.fileid
 139             Output(string.format("%d@%04d@%04d", v.c, math.floor(v.y / zone_image_chunksize), math.floor(v.x / zone_image_chunksize)), nil, v, "zone") -- This is inverted - it's continent, y, x, for proper sorting.
 140             Output(string.format("%d", v.c), nil, {fileid = value.fileid; math.floor(v.x / zone_image_chunksize), math.floor(v.y / zone_image_chunksize)}, "zone_bounds")
 141           end end
 142         end end
 143       else
 144         --print("Dumped, locale " .. dat.signature)
 145       end
 146     end
 147   } end
 148 )
 149
 150 --[[
 151 *****************************************************************
 152 Object collation
 153 ]]
 154
 155 local object_slurp
 156
 157 if false and do_compile then
 158   local object_locate = ChainBlock_Create("object_locate", {chainhead},
 159     function (key) return {
 160       accum = {name = {}, loc = {}},
 161
 162       fids = {},
 163
 164       -- Here's our actual data
 165       Data = function(self, key, subkey, value, Output)
 166         local name, locale = key:match("(.*)@@(.*)")
 167
 168         if standard_pos_accum(self.accum, value, loc_version(subkey), locale) then return end
 169
 170         while #value > 0 do table.remove(value) end
 171
 172         table.insert(self.accum, value)
 173         self.fids[value.fileid] = true
 174       end,
 175
 176       Finish = function(self, Output, Broadcast)
 177         local fidc = 0
 178         for k, v in pairs(self.fids) do
 179           fidc = fidc + 1
 180         end
 181
 182         if fidc < 3 then return end -- bzzzzzt
 183
 184         local name, locale = key:match("(.*)@@(.*)")
 185
 186         local qout = {}
 187
 188         if position_has(self.accum.loc) then
 189           qout.loc = position_finalize(self.accum.loc)
 190         else
 191           return  -- BZZZZZT
 192         end
 193
 194         if locale == "enUS" then
 195           Broadcast("object", {name = name, loc = qout.loc})
 196           Output("", nil, {type = "data", name = key, data = self.accum}, "reparse")
 197         else
 198           Output(key, nil, qout.loc, "link")
 199           Output("", nil, {type = "data", name = key, data = self.accum}, "reparse")
 200         end
 201       end,
 202     } end,
 203     sortversion, "object"
 204   )
 205
 206   local function find_closest(loc, locblock)
 207     local closest = 5000000000  -- yeah, that's five billion. five fuckin' billion.
 208     --print(#locblock)
 209     for _, ite in ipairs(locblock) do
 210       if loc.p == ite.p then
 211         local tx = loc.x - ite.x
 212         local ty = loc.y - ite.y
 213         local d = tx * tx + ty * ty
 214         if d < closest then
 215           closest = d
 216         end
 217       end
 218     end
 219     return closest
 220   end
 221
 222   local object_link = ChainBlock_Create("object_link", {object_locate},
 223     function (key) return {
 224
 225       compare = {},
 226
 227       -- Here's our actual data
 228       Data = function(self, key, subkey, value, Output)
 229         assert(not self.key)
 230         assert(not self.loc)
 231         assert(key)
 232         assert(value)
 233
 234         self.key = key
 235         self.loc = value
 236       end,
 237
 238       Receive = function(self, id, data)
 239         assert(id == "object")
 240         assert(data)
 241         assert(not self.compare[data.name])
 242
 243         self.compare[data.name] = data.loc
 244       end,
 245
 246       Finish = function(self, Output, Broadcast)
 247         assert(self.key)
 248         assert(self.loc)
 249         assert(self.compare)
 250
 251         local results = {}
 252         local res_size = 0
 253
 254         for enuname, loca in pairs(self.compare) do
 255           local yaku = 0
 256           for _, cl in ipairs(loca) do
 257             yaku = yaku + find_closest(cl, self.loc)
 258           end
 259           for _, cl in ipairs(self.loc) do
 260             yaku = yaku + find_closest(cl, loca)
 261           end
 262           yaku = yaku / (#loca + #self.loc)
 263           assert(not results[enuname])
 264           results[enuname] = yaku
 265           res_size = res_size + 1
 266         end
 267
 268         local nres_size = 0
 269         local nres = {}
 270         for k, v in pairs(results) do
 271           if v < 1000000 then
 272             nres[k] = v
 273             nres_size = nres_size + 1
 274           end
 275         end
 276
 277         print(res_size, nres_size)
 278         Output("", nil, {key = key, data = nres}, "combine")
 279       end,
 280     } end,
 281     nil, "link"
 282   )
 283
 284   local function heap_left(x) return (2*x) end
 285   local function heap_right(x) return (2*x + 1) end
 286
 287   local function heap_sane(heap)
 288     local dmp = ""
 289     local finishbefore = 2
 290     for i = 1, #heap do
 291       if i == finishbefore then
 292         print(dmp)
 293         dmp = ""
 294         finishbefore = finishbefore * 2
 295       end
 296       dmp = dmp .. string.format("%f ", heap[i].c)
 297     end
 298     print(dmp)
 299     print("")
 300     for i = 1, #heap do
 301       assert(not heap[heap_left(i)] or heap[i].c <= heap[heap_left(i)].c)
 302       assert(not heap[heap_right(i)] or heap[i].c <= heap[heap_right(i)].c)
 303     end
 304   end
 305
 306   local function heap_insert(heap, item)
 307     assert(item)
 308     table.insert(heap, item)
 309     local pt = #heap
 310     while pt > 1 do
 311       local ptd2 = math.floor(pt / 2)
 312       if heap[ptd2].c <= heap[pt].c then
 313         break
 314       end
 315       local tmp = heap[pt]
 316       heap[pt] = heap[ptd2]
 317       heap[ptd2] = tmp
 318       pt = ptd2
 319     end
 320     --heap_sane(heap)
 321   end
 322
 323
 324   local function heap_extract(heap)
 325     local rv = heap[1]
 326     if #heap == 1 then table.remove(heap) return rv end
 327     heap[1] = table.remove(heap)
 328     local idx = 1
 329     while idx < #heap do
 330       local minix = idx
 331       if heap[heap_left(idx)] and heap[heap_left(idx)].c < heap[minix].c then minix = heap_left(idx) end
 332       if heap[heap_right(idx)] and heap[heap_right(idx)].c < heap[minix].c then minix = heap_right(idx) end
 333       if minix ~= idx then
 334         local tx = heap[minix]
 335         heap[minix] = heap[idx]
 336         heap[idx] = tx
 337         idx = minix
 338       else
 339         break
 340       end
 341     end
 342     --heap_sane(heap)
 343     return rv
 344   end
 345
 346   --[[
 347   do
 348     local heaptest = {}
 349     for k = 1, 10 do
 350       heap_insert(heaptest, {c = math.random()})
 351     end
 352     while #heaptest > 0 do
 353       heap_extract(heaptest)
 354     end
 355   end]]
 356
 357   local object_combine = ChainBlock_Create("object_combine", {object_link},
 358     function (key) return {
 359
 360       source = {enUS = {}},
 361       heap = {},
 362
 363       Data = function(self, key, subkey, value, Output)
 364         local name, locale = value.key:match("(.*)@@(.*)")  -- boobies regexp
 365         -- insert shit into a heap
 366         if not self.source[locale] then self.source[locale] = {} end
 367         self.source[locale][name] = {}
 368         for k, v in pairs(value.data) do
 369           self.source.enUS[k] = {linkedto = {}}
 370           heap_insert(self.heap, {c = v, dst_locale = locale, dst = name, src = k})
 371         end
 372       end,
 373
 374       Receive = function() end,
 375
 376       Finish = function(self, Output, Broadcast)
 377         print("heap is", #self.heap)
 378
 379         local llst = 0
 380         while #self.heap > 0 do
 381           local ite = heap_extract(self.heap)
 382           assert(ite.c >= llst)
 383           llst = ite.c
 384
 385           if not self.source.enUS[ite.src].linkedto[ite.dst_locale] and not self.source[ite.dst_locale][ite.dst].linked then
 386             self.source.enUS[ite.src].linkedto[ite.dst_locale] = ite.dst
 387             self.source[ite.dst_locale][ite.dst].linked = true
 388             print(string.format("Linked %s to %s/%s (%f)", ite.src, ite.dst_locale, ite.dst, ite.c))
 389           end
 390         end
 391         -- pull shit out of the heap, link things up
 392
 393         -- determine unique IDs for everything we have left
 394
 395         -- output stuff for actual parsing and processing of any remaining data
 396         -- also, output a chart of what we linked
 397         -- remember to output that chart in order-of-linkage
 398       end,
 399     } end,
 400     nil, "combine"
 401   )
 402
 403
 404   -- then, now that we finally have IDs, we do our standard mambo of stuff
 405
 406
 407   --[[object_slurp = ChainBlock_Create("object_slurp", {chainhead},
 408     function (key) return {
 409       accum = {name = {}, loc = {}},
 410
 411       -- Here's our actual data
 412       Data = function(self, key, subkey, value, Output)
 413         if standard_pos_accum(self.accum, value, loc_version(subkey)) then return end
 414         name_accumulate(self.accum.name, key, value.locale)
 415
 416         while #value > 0 do table.remove(value) end
 417         value.locale = nil
 418
 419         table.insert(accum, value)
 420       end,
 421
 422       Finish = function(self, Output)
 423         self.accum.name = name_resolve(self.accum.name)
 424
 425         local qout = {}
 426
 427         if dbg_data then qout.name = self.accum.name end
 428         if position_has(self.accum.loc) then qout.loc = position_finalize(self.accum.loc) end
 429
 430         local has_stuff = false
 431         for k, v in pairs(qout) do
 432           has_stuff = true
 433           break
 434         end
 435         if has_stuff then
 436           Output("", nil, {id="object", key=key, data=qout}, "output")
 437         end
 438       end,
 439     } end,
 440     sortversion, "object"
 441   )]]
 442 end
 443
 444 --[[
 445 *****************************************************************
 446 Monster collation
 447 ]]
 448
 449 local monster_slurp
 450
 451 if do_compile and do_questtables then
 452   monster_slurp = ChainBlock_Create("monster_slurp", {chainhead},
 453     function (key) return {
 454       accum = {name = {}, loc = {}},
 455
 456       -- Here's our actual data
 457       Data = function(self, key, subkey, value, Output)
 458         if standard_pos_accum(self.accum, value, loc_version(subkey), value.locale, 2) then return end
 459         if standard_name_accum(self.accum.name, value) then return end
 460
 461         loot_accumulate(value, {type = "monster", id = tonumber(key)}, Output)
 462       end,
 463
 464       Finish = function(self, Output)
 465         self.accum.name = name_resolve(self.accum.name)
 466
 467         local qout = {}
 468
 469         if dbg_data then qout.dbg_name = self.accum.name.enUS end
 470         if position_has(self.accum.loc) then qout.loc = position_finalize(self.accum.loc) end
 471
 472         local has_stuff = false
 473         for k, v in pairs(qout) do
 474           has_stuff = true
 475           break
 476         end
 477         assert(tonumber(key))
 478         if has_stuff then
 479           Output("*/*", nil, {id="monster", key=tonumber(key), data=qout}, "output")
 480         end
 481         for k, v in pairs(self.accum.name) do
 482           Output(("%s/*"):format(k), nil, {id="monster", key=tonumber(key), data={name=v}}, "output")
 483         end
 484       end,
 485     } end,
 486     sortversion, "monster"
 487   )
 488 end
 489
 490 --[[
 491 local monster_pack
 492 if do_compile then
 493   monster_pack = ChainBlock_Create("monster_pack", {monster_slurp},
 494     function (key) return {
 495       data = {},
 496
 497       Data = function(self, key, subkey, value, Output)
 498         assert(not self.data[value.key])
 499         if not self.data[value.key] then self.data[value.key] = {} end
 500         self.data[value.key] = value.data
 501       end,
 502
 503       Finish = function(self, Output, Broadcast)
 504         Broadcast(nil, {monster=self.data})
 505       end,
 506     } end
 507   )
 508 end]]
 509
 510 --[[
 511 *****************************************************************
 512 Item collation
 513 ]]
 514
 515 local item_name_package
 516 local item_slurp
 517 local item_parse
 518
 519 if do_compile and do_questtables then
 520   item_parse = ChainBlock_Create("item_parse", {chainhead},
 521     function (key) return {
 522       accum = {name = {}},
 523
 524       -- Here's our actual data
 525       Data = function(self, key, subkey, value, Output)
 526         name_accumulate(self.accum.name, value.name, value.locale)
 527
 528         loot_accumulate(value, {type = "item", id = tonumber(key)}, Output)
 529       end,
 530
 531       Finish = function(self, Output)
 532         self.accum.name = name_resolve(self.accum.name)
 533
 534         local qout = {}
 535
 536         -- we don't actually care about the level, so we don't bother to store it. Really, we only care about the name for debug purposes also, so we should probably get rid of it before release.
 537         if dbg_data then qout.dbg_name = self.accum.name.enUS end
 538
 539         --[[Output("", nil, {key = key, name = qout.name}, "name")]]
 540
 541         local has_stuff = false
 542         for k, v in pairs(qout) do
 543           has_stuff = true
 544           break
 545         end
 546
 547         if has_stuff then
 548           Output(key, nil, {type = "core", data = qout}, "item")
 549         end
 550         for k, v in pairs(self.accum.name) do
 551           Output(("%s/*"):format(k), nil, {id="item", key=tonumber(key), data={name=v}}, "output")
 552         end
 553       end,
 554     } end,
 555     sortversion, "item"
 556   )
 557
 558   --[[
 559   item_name_package = ChainBlock_Create("item_name_package", {item_slurp_first},
 560     function (key) return {
 561       accum = {},
 562
 563       -- Here's our actual data
 564       Data = function(self, key, subkey, value, Output)
 565         assert(not self.accum[value.key])
 566         self.accum[value.key] = value.name
 567       end,
 568
 569       Finish = function(self, Output, Broadcast)
 570         Broadcast("item_name_package", self.accum)
 571       end,
 572     } end,
 573     nil, "name"
 574   )]]
 575
 576   -- Input to this module is kind of byzantine, so I'm documenting it here.
 577   -- {Key, Subkey, Value}
 578
 579   -- {999, nil, {source = {type = "monster", id = 12345}, count = 104, type = "skin"}}
 580   -- Means: "We've seen 104 skinnings of item #999 from monster #12345"
 581   local lootables = {}
 582   if monster_slurp then table.insert(lootables, monster_slurp) end
 583   if item_parse then table.insert(lootables, item_parse) end
 584
 585   local loot_merge = ChainBlock_Create("loot_merge", lootables,
 586     function (key) return {
 587       lookup = setmetatable({__exists__ = {}},
 588         {__index = function(self, key)
 589           if not rawget(self, key.sourcetype) then self[key.sourcetype] = {} end
 590           if not self[key.sourcetype][key.sourceid] then self[key.sourcetype][key.sourceid] = {} end
 591           if not self[key.sourcetype][key.sourceid][key.type] then self[key.sourcetype][key.sourceid][key.type] = key  table.insert(self.__exists__, key) end
 592           return self[key.sourcetype][key.sourceid][key.type]
 593         end
 594         }),
 595
 596       dtime = 0,
 597
 598       -- Here's our actual data
 599       Data = function(self, key, subkey, value, Output)
 600         --local st = os.time()
 601         local vx = self.lookup[{sourcetype = value.source.type, sourceid = value.source.id, type = value.type}]
 602         vx.w = (vx.w or 0) + value.count
 603         --self.dtime = self.dtime + os.time() - st
 604       end,
 605
 606       Finish = function(self, Output)
 607         --local st = os.time()
 608         local tacu = {}
 609         for k, v in pairs(self.lookup.__exists__) do
 610           table.insert(tacu, v)
 611         end
 612
 613         --local tacuc = #tacu
 614
 615         Output(key, nil, {type = "loot", data = weighted_concept_finalize(tacu, 0.9, 10)}, "item")
 616       end,
 617     } end,
 618     nil, "loot"
 619   )
 620
 621   item_slurp = ChainBlock_Create("item_slurp", {item_parse, loot_merge},
 622     function (key) return {
 623       accum = {},
 624
 625       -- Here's our actual data
 626       Data = function(self, key, subkey, value, Output)
 627         assert(not self.accum[value.type])
 628         self.accum[value.type] = value.data
 629       end,
 630
 631       Finish = function(self, Output)
 632         local qout = self.accum.core
 633         if not qout then qout = {} end -- Surprisingly, we don't care much about the "core".
 634
 635         if self.accum.loot then for k, v in pairs(self.accum.loot) do
 636           qout[k] = v
 637         end end
 638
 639         if key ~= "gold" then -- okay technically the whole thing could have been ignored, but
 640           assert(tonumber(key))
 641           Output("*/*", nil, {id="item", key=tonumber(key), data=qout}, "output")
 642         end
 643       end,
 644     } end,
 645     nil, "item"
 646   )
 647 end
 648
 649 --[[
 650 *****************************************************************
 651 Quest collation
 652 ]]
 653
 654 local quest_slurp
 655
 656 if do_compile and do_questtables then
 657   local function find_important(dat, count)
 658     local mungedat = {}
 659     local tweight = 0
 660     for k, v in pairs(dat) do
 661       table.insert(mungedat, {d = k, w = v})
 662       tweight = tweight + v
 663     end
 664
 665     if tweight < count / 2 then return end  -- we just don't have enough, something's gone wrong
 666
 667     return weighted_concept_finalize(mungedat, 0.9, 10, count) -- this is not ideal, but it's functional
 668   end
 669
 670   quest_slurp = ChainBlock_Create("quest_slurp", {chainhead --[[, item_name_package]]},
 671     function (key) return {
 672       accum = {name = {}, criteria = {}, level = {}, start = {}, finish = {}},
 673
 674       -- Here's our actual data
 675       Data = function(self, key, subkey, value, Output)
 676         local lv = loc_version(subkey)
 677
 678         -- Split apart the start/end info. This includes locations and possibly the monster that was targeted.
 679         if value.start then
 680           value.start = split_quest_startend(value.start, lv)
 681           convert_multiple_loc(value.start, value.locale)
 682         end
 683         if value["end"] then   --sigh
 684           value.finish = split_quest_startend(value["end"], lv)
 685           convert_multiple_loc(value.finish, value.locale)
 686           value["end"] = nil
 687         end
 688
 689         -- Parse apart the old complicated criteria strings
 690         if not value.criteria then value.criteria = {} end
 691         for k, v in pairs(value) do
 692           local item, token = string.match(k, "criteria_([%d]+)_([a-z]+)")
 693           if token then
 694             assert(item)
 695
 696             if token == "satisfied" then
 697               value[k] = split_quest_satisfied(value[k], lv)
 698               convert_multiple_loc(value[k], value.locale)
 699             end
 700
 701             if not value.criteria[tonumber(item)] then value.criteria[tonumber(item)] = {} end
 702             value.criteria[tonumber(item)][token] = value[k]
 703             value[k] = nil
 704           end
 705         end
 706
 707         -- Accumulate the old criteria strings into our new data
 708         if value.start then for k, v in pairs(value.start) do position_accumulate(self.accum.start, v.loc) end end
 709         if value.finish then for k, v in pairs(value.finish) do position_accumulate(self.accum.finish, v.loc) end end
 710
 711         self.accum.appearances = (self.accum.appearances or 0) + 1
 712         for id, dat in pairs(value.criteria) do
 713           if not self.accum.criteria[id] then self.accum.criteria[id] = {count = 0, loc = {}, monster = {}, item = {}} end
 714           local cid = self.accum.criteria[id]
 715
 716           if dat.satisfied then
 717             for k, v in pairs(dat.satisfied) do
 718               position_accumulate(cid.loc, v.loc)
 719               cid.count = cid.count + (v.c or 1)
 720               list_accumulate(cid, "monster", v.monster)
 721               list_accumulate(cid, "item", v.item)
 722             end
 723           end
 724
 725           cid.appearances = (cid.appearances or 0) + 1
 726
 727           list_accumulate(cid, "type", dat.type)
 728         end
 729
 730         -- Accumulate names and levels
 731         if value.name then
 732           -- Names is a little complicated - we want to get rid of any recommended-level tags that we might have.
 733           local vnx = string.match(value.name, "%b[]%s*(.*)")
 734           if not vnx then vnx = value.name end
 735
 736           name_accumulate(self.accum.name, vnx, value.locale)
 737         end
 738         list_accumulate(self.accum, "level", value.level)
 739       end,
 740
 741       --[[
 742       Receive = function(self, id, data)
 743         self.namedb = data
 744       end,]]
 745
 746       Finish = function(self, Output)
 747         self.accum.name = name_resolve(self.accum.name)
 748         self.accum.level = list_most_common(self.accum.level)
 749
 750         -- First we see if we need to chop out some criteria
 751         do
 752           local appearances = self.accum.appearances * 0.9
 753           appearances = appearances * 0.9
 754           local strips = {}
 755           for k, v in pairs(self.accum.criteria) do
 756             if v.appearances < appearances then
 757               table.insert(strips, k)
 758             end
 759           end
 760           for _, v in pairs(strips) do
 761             self.accum.criteria[v] = nil
 762           end
 763         end
 764
 765         local qout = {}
 766         for k, v in pairs(self.accum.criteria) do
 767
 768           v.type = list_most_common(v.type)
 769
 770           if not qout.criteria then qout.criteria = {} end
 771
 772           -- temp debug output
 773           -- We shouldn't actually be doing this, we should be figuring out which monsters and items this really correlates to.
 774           -- We're currently not. However, this will require correlating with the names for monsters and items.
 775           local snaggy, typ
 776           if v.type == "monster" then
 777             snaggy = find_important(v.monster, v.count)
 778             typ = "kill"
 779           elseif v.type == "item" then
 780             snaggy = find_important(v.item, v.count)
 781             typ = "get"
 782           end
 783
 784           qout.criteria[k] = {}
 785
 786           if dbg_data then
 787             qout.criteria[k].item = v.item
 788             qout.criteria[k].monster = v.monster
 789             qout.criteria[k].count = v.count
 790             qout.criteria[k].type = v.type
 791             qout.criteria[k].appearances = v.appearances
 792
 793             qout.criteria[k].snaggy = snaggy or "(nothin')"
 794           end
 795
 796           if snaggy then
 797             assert(#snaggy > 0)
 798
 799             for _, x in ipairs(snaggy) do
 800               table.insert(qout.criteria[k], {sourcetype = v.type, sourceid = x.d, type = typ})
 801             end
 802           end
 803
 804           if position_has(v) then qout.criteria[k].loc = position_finalize(v.loc) end
 805         end
 806
 807         --if position_has(self.accum.start) then qout.start = { loc = position_finalize(self.accum.start) } end  -- we don't actually care about the start position
 808         if position_has(self.accum.finish) then qout.finish = { loc = position_finalize(self.accum.finish) } end
 809
 810         -- we don't actually care about the level, so we don't bother to store it. Really, we only care about the name for debug purposes also, so we should probably get rid of it before release.
 811         if dbg_data then
 812           qout.dbg_name = self.accum.name.enUS
 813           qout.appearances = self.accum.appearances or "none"
 814         end
 815
 816         local has_stuff = false
 817         for k, v in pairs(qout) do
 818           has_stuff = true
 819           break
 820         end
 821         assert(tonumber(key))
 822         if has_stuff then
 823           --print("Quest output " .. tostring(key))
 824           Output("*/*", nil, {id="quest", key=tonumber(key), data=qout}, "output")
 825         end
 826         for k, v in pairs(self.accum.name) do
 827           Output(("%s/*"):format(k), nil, {id="quest", key=tonumber(key), data={name=v}}, "output")
 828         end
 829       end,
 830     } end,
 831     sortversion, "quest"
 832   )
 833 end
 834
 835 --[[
 836 *****************************************************************
 837 Zone collation
 838 ]]
 839
 840 if do_zone_map then
 841   local zone_draw = ChainBlock_Create("zone_draw", {chainhead},
 842     function (key) return {
 843       imagepiece = Image(zone_image_outchunk, zone_image_outchunk),
 844       ct = 0,
 845
 846       Data = function(self, key, subkey, value, Output)
 847         self.imagepiece:set(math.floor(math.umod(value.x, zone_image_chunksize) / zone_image_descale), math.floor(math.umod(value.y, zone_image_chunksize) / zone_image_descale), value.zonecolor)
 848         self.ct = self.ct + 1
 849       end,
 850
 851       Finish = function(self, Output)
 852         if self.ct > 0 then Output(string.gsub(key, "@.*", ""), key, self.imagepiece, "zone_stitch") end
 853       end,
 854     } end,
 855     nil, "zone"
 856   )
 857
 858   local zone_bounds = ChainBlock_Create("zone_bounds", {chainhead},
 859     function (key) return {
 860       sx = 1000000,
 861       sy = 1000000,
 862       ex = -1000000,
 863       ey = -1000000,
 864
 865       ct = 0,
 866
 867       Data = function(self, key, subkey, value, Output)
 868         self.sx = math.min(self.sx, value[1])
 869         self.sy = math.min(self.sy, value[2])
 870         self.ex = math.max(self.ex, value[1])
 871         self.ey = math.max(self.ey, value[2])
 872         self.ct = self.ct + 1
 873       end,
 874
 875       Finish = function(self, Output)
 876         if self.ct > 1000 then
 877           Output(key, nil, {sx = self.sx, sy = self.sy, ex = self.ex, ey = self.ey}, "zone_stitch")
 878         end
 879       end,
 880     } end,
 881     nil, "zone_bounds"
 882   )
 883
 884   local zone_stitch = ChainBlock_Create("zone_stitch", {zone_draw, zone_bounds},
 885     function (key) return {
 886       Data = function(self, key, subkey, value, Output)
 887         if not subkey then
 888           self.bounds = value
 889           self.imagewriter = ImageTileWriter(string.format("intermed/zone_%s.png", key), self.bounds.ex - self.bounds.sx + 1, self.bounds.ey - self.bounds.sy + 1, zone_image_outchunk)
 890           return
 891         end
 892
 893         if not self.bounds then return end
 894
 895         local yp, xp = string.match(subkey, "[%d-]+@([%d-]+)@([%d-]+)")
 896         if not xp or not yp then print(subkey) end
 897         xp = xp - self.bounds.sx
 898         yp = yp - self.bounds.sy
 899
 900         self.imagewriter:write_tile(xp, yp, value)
 901       end,
 902
 903       Finish = function(self, Output)
 904         if self.imagewriter then self.imagewriter:finish() end
 905       end,
 906     } end,
 907     nil, "zone_stitch"
 908   )
 909 end
 910
 911 --[[
 912 *****************************************************************
 913 Flight paths
 914 ]]
 915
 916 --[[
 917
 918 let us talk about flight paths
 919
 920 sit down
 921
 922 have some tea
 923
 924 very well, let us begin
 925
 926 So, flight masters. First, accumulate each one of each faction/name/locale set. This includes both monsterid (pick most common) and vertex location (simple most-common.)
 927
 928 Then we link together name/locale's of various factions, just so we can get names out and IDs.
 929
 930 After that, we take our routes and determine IDs, with name-lookup for the first and last node, and vertex-lookup for all other nodes, with some really low error threshold. Pick the mean time for each route that has over N tests, then dump those.
 931
 932 For now we'll assume that this will provide sufficiently accurate information.
 933
 934 We'll do this, then start working on the clientside code.
 935
 936 ]]
 937
 938 local flight_data_output
 939 local flight_table_output
 940 local flight_master_name_output
 941
 942 if do_compile and do_flight then
 943   local flight_master_parse = ChainBlock_Create("flight_master_parse", {chainhead},
 944     function (key) return {
 945       mids = {},
 946       locs = {},
 947       newest_version = nil,
 948       count = 0,
 949
 950       -- Here's our actual data
 951       Data = function(self, key, subkey, value, Output)
 952         if not sortversion(self.newest_version, value.wowv) then
 953           self.newest_version = value.wowv
 954         end
 955
 956         list_accumulate(self, "mids", value.dat.master)
 957         list_accumulate(self, "locs", string.format("%s@@%s", value.dat.x, value.dat.y))
 958         self.count = self.count + 1
 959       end,
 960
 961       Finish = function(self, Output)
 962         if self.count < 10 then return end
 963
 964         local faction, name, locale = key:match("(.*)@@(.*)@@(.*)")
 965         assert(faction)
 966         assert(name)
 967         assert(locale)
 968         local mid = list_most_common(self.mids)
 969         local loc = list_most_common(self.locs)
 970
 971         Output(string.format("%s@@%s", loc, faction), nil, {locale = locale, name = name, mid = mid, version = self.newest_version})
 972       end,
 973     } end,
 974     sortversion, "flight_master"
 975   )
 976
 977   local flight_master_accumulate = ChainBlock_Create("flight_master_accumulate", {flight_master_parse},
 978     function (key) return {
 979
 980       names = {},
 981
 982       -- Here's our actual data
 983       Data = function(self, key, subkey, value, Output)
 984         if self.names[value.locale] then
 985           print(key, value.locale, self.names[value.locale], value.name)
 986
 987           print(self.names[value.locale].version, value.version, sortversion(self.names[value.locale].version, value.version), self.names[value.locale].name, value.name)
 988           assert(self.names[value.locale].version ~= value.version)
 989           print(self.names[value.locale].version, value.version, sortversion(self.names[value.locale].version, value.version))
 990
 991           if not sortversion(self.names[value.locale].version, value.version) then
 992             self.names[value.locale] = nil  -- we just blow it away and rebuild it later
 993           else
 994             return
 995           end
 996         end
 997         assert(not self.names[value.locale])
 998         assert(not self.mid or not value.mid or self.mid == value.mid, key)
 999
1000         self.names[value.locale] = {name = value.name, version = value.version}
1001         self.mid = value.mid
1002       end,
1003
1004       Finish = function(self, Output)
1005         local x, y, faction = key:match("(.*)@@(.*)@@(.*)")
1006         local namepack = {}
1007         for k, v in pairs(self.names) do
1008           namepack[k] = v.name
1009         end
1010
1011         Output(tostring(faction), nil, {x = x, y = y, faction = faction, mid = self.mid, names = namepack})
1012       end,
1013     } end
1014   )
1015
1016   if false then
1017     local flight_master_test = ChainBlock_Create("flight_master_test", {flight_master_accumulate},
1018       function (key) return {
1019
1020         data = {},
1021
1022         -- Here's our actual data
1023         Data = function(self, key, subkey, value, Output)
1024           table.insert(self.data, value)
1025         end,
1026
1027         Finish = function(self, Output)
1028           local links = {}
1029           for x = 1, #self.data do
1030             for y = x + 1, #self.data do
1031               local dx = self.data[x].x - self.data[y].x
1032               local dy = self.data[x].y - self.data[y].y
1033               local diff = math.sqrt(dx * dx + dy * dy)
1034               if diff < 0.001 then
1035                 print("------")
1036                 print(diff)
1037                 dbgout(self.data[x])
1038                 dbgout(self.data[y])
1039               end
1040               table.insert(links, diff)
1041             end
1042           end
1043
1044           table.sort(links)
1045
1046           for x = 1, math.min(100, #links) do
1047             print(links[x])
1048           end
1049         end,
1050       } end
1051     )
1052   end
1053
1054   local flight_master_pack = ChainBlock_Create("flight_master_pack", {flight_master_accumulate},
1055     function (key) return {
1056       pack = {},
1057
1058       -- Here's our actual data
1059       Data = function(self, key, subkey, value, Output)
1060         table.insert(self.pack, value)
1061       end,
1062
1063       Finish = function(self, Output, Broadcast)
1064         print("Broadcasting", key)
1065         Broadcast(key, self.pack)
1066
1067         Output(key, nil, "", "name_output") -- just exists to make sure name_output does something
1068       end,
1069     } end
1070   )
1071
1072   local function findname(lookup, dat, locale)
1073     for k, v in ipairs(lookup) do
1074       if v.names[locale] == dat then return k end
1075     end
1076   end
1077
1078   local flight_master_times = ChainBlock_Create("flight_master_times", {flight_master_pack, chainhead},
1079     function (key) local src, dst, faction, locale = key:match("(.*)@@(.*)@@(.*)@@(.*)") assert(faction and src and dst and locale) return {
1080
1081       Data = function(self, key, subkey, value, Output)
1082         if self.fail then return end
1083
1084         if not self.table then if not e or e > 1000 then print("Entire missing faction table!") end return end
1085         assert(self.table)
1086
1087         if not self.src or not self.dst then
1088           self.src = findname(self.table, src, locale)
1089           self.dst = findname(self.table, dst, locale)
1090
1091           --if not self.src then print("failed to find ", src) end
1092           --if not self.dst then print("failed to find ", dst) end
1093           if not self.src or not self.dst then self.fail = true return end
1094         end
1095
1096         assert(self.src)
1097         assert(self.dst)
1098
1099         for k, v in pairs(value) do
1100           if type(v) == "number" and value[k .. "##count"] then
1101             local path = {}
1102             for node in k:gmatch("[^@]+") do
1103               local x, y = node:match("(.*):(.*)")
1104               x, y = tonumber(x), tonumber(y)
1105               local closest, closestval = nil, 0.01
1106               for k, v in ipairs(self.table) do
1107                 local dx, dy = v.x - x, v.y - y
1108                 dx, dy = dx * dx, dy * dy
1109                 local dist = math.sqrt(dx + dy)
1110                 if dist < closestval then
1111                   closestval = dist
1112                   closest = k
1113                 end
1114               end
1115
1116               if not closest then print("Can't find nearby flightpath") return end
1117               assert(closest)
1118               table.insert(path, closest)
1119             end
1120             table.insert(path, self.dst)
1121
1122             local tx = tostring(self.src)
1123             for _, v in ipairs(path) do
1124               tx = tx .. "@" .. tostring(v)
1125             end
1126
1127             Output(faction .. "/" .. tx, nil, v / value[k .. "##count"])
1128           end
1129         end
1130       end,
1131
1132       Receive = function(self, id, value)
1133         if id == faction then self.table = value end
1134       end,
1135     } end,
1136     nil, "flight_times"
1137   )
1138
1139   local flight_master_assemble = ChainBlock_Create("flight_master_assemble", {flight_master_times},
1140     function (key) return {
1141       dat = {},
1142
1143       -- Here's our actual data
1144       Data = function(self, key, subkey, value, Output)
1145         table.insert(self.dat, value)
1146       end,
1147
1148       Finish = function(self, Output, Broadcast)
1149         table.sort(self.dat)
1150
1151         local chop = math.floor(#self.dat / 3)
1152
1153         local acu = 0
1154         local ct = 0
1155         for i = 1 + chop, #self.dat - chop do
1156           acu = acu + self.dat[i]
1157           ct = ct + 1
1158         end
1159
1160         acu = acu / ct
1161
1162         if #self.dat > 10 then
1163           Output(key:match("([%d]+/[%d]+)@.+"), nil, {path = key, distance = acu})
1164         end
1165       end,
1166     } end
1167   )
1168
1169   flight_data_output = ChainBlock_Create("flight_data_output", {flight_master_assemble},
1170     function (key) local faction, src = key:match("([%d]+)/([%d]+)") assert(faction and src) return {
1171       chunky = {},
1172
1173       -- Here's our actual data
1174       Data = function(self, key, subkey, value, Output)
1175         local f, s, m, e = value.path:match("([%d]+)/([%d]+)@(.+)@([%d]+)")
1176         if not f then f, s, e = value.path:match("([%d]+)/([%d]+)@([%d]+)") end
1177         assert(f and s and e)
1178         assert((f .. "/" .. s) == key)
1179         s = tonumber(s)
1180         e = tonumber(e)
1181         assert(s)
1182         assert(e)
1183
1184         if not self.chunky[e] then
1185           self.chunky[e] = {}
1186         end
1187
1188         local dex = {distance = value.distance, path = {}}
1189         if m then for x in m:gmatch("[%d]+") do
1190           assert(tonumber(x))
1191           table.insert(dex.path, tonumber(x))
1192         end end
1193
1194         table.insert(self.chunky[e], dex)
1195       end,
1196
1197       Finish = function(self, Output, Broadcast)
1198         for _, v in pairs(self.chunky) do
1199           table.sort(v, function(a, b) return a.distance < b.distance end)
1200         end
1201
1202         Output(string.format("*/%s", faction), nil, {id = "flightpaths", key = tonumber(src), data = self.chunky}, "output_direct")
1203       end,
1204     } end
1205   )
1206
1207   flight_master_name_output = ChainBlock_Create("flight_master_name_output", {flight_master_pack},
1208     function (key) return {
1209       -- Here's our actual data
1210       Data = function(self, key, subkey, value, Output)
1211       end,
1212
1213       Receive = function(self, id, value)
1214         if id == key then self.table = value end
1215       end,
1216
1217       Finish = function(self, Output, Broadcast)
1218         print("finnish")
1219         for k, v in ipairs(self.table) do
1220           Output(string.format("*/%s", key), nil, {id = "flightmasters", key = k, data = {mid = v.mid}}, "output")
1221           for l, n in pairs(v.names) do
1222             Output(string.format("%s/%s", l, key), nil, {id = "flightmasters", key = k, data = {name = n}}, "output_direct")
1223           end
1224         end
1225       end,
1226     } end,
1227     nil, "name_output"
1228   )
1229 end
1230
1231 --[[
1232 *****************************************************************
1233 Final file generation
1234 ]]
1235
1236 local sources = {}
1237 if quest_slurp then table.insert(sources, quest_slurp) end
1238 if item_slurp then table.insert(sources, item_slurp) end
1239 if item_parse then table.insert(sources, item_parse) end
1240 if monster_slurp then table.insert(sources, monster_slurp) end
1241 if object_slurp then table.insert(sources, object_slurp) end
1242 if flight_data_output then table.insert(sources, flight_data_output) end
1243 if flight_table_output then table.insert(sources, flight_table_output) end
1244 if flight_master_name_output then table.insert(sources, flight_master_name_output) end
1245
1246 local function do_loc_choice(file, item, toplevel, solidity)
1247   local has_linkloc = false
1248   local count = 0
1249
1250   if not solidity then assert(toplevel)  solidity = {} end
1251
1252   do
1253     local loc_obliterate = {}
1254     for k, v in ipairs(item) do
1255       local worked = false
1256       if file[v.sourcetype] and file[v.sourcetype][v.sourceid] and file[v.sourcetype][v.sourceid]["*/*"] then
1257         local valid, tcount = do_loc_choice(file, file[v.sourcetype][v.sourceid]["*/*"], false, solidity)
1258         if valid then
1259           has_linkloc = true
1260           worked = true
1261           count = count + tcount
1262         end
1263       end
1264
1265       if not worked then
1266         table.insert(loc_obliterate, k)
1267       end
1268     end
1269
1270     for i = #loc_obliterate, 1, -1 do
1271       table.remove(item, loc_obliterate[i])
1272     end
1273   end
1274
1275   if dbg_data then
1276     item.full_objective_count = count
1277   end
1278
1279   local reason = string.format("%s, %s, %s", tostring(has_linkloc), tostring(count), (item.loc and tostring(#item.loc) or "(no item.loc)"))
1280
1281   if has_linkloc then
1282     assert(count > 0)
1283     if toplevel and count > 10 and item.loc then
1284       while #item.loc > 10 do
1285         table.remove(item.loc)
1286       end
1287       count = #item.loc
1288       solidity = item.loc.solid   -- reset solidity to just the quest objectives
1289     elseif toplevel and count > 10 then
1290       item.loc = {} -- we're doing this just so we can say "hey, we don't want to use the original locations"
1291       count = 0 -- :(
1292     else
1293       if dbg_data then
1294         item.loc_unused = item.loc_unused or item.loc
1295       end
1296
1297       item.loc = nil
1298     end
1299   else
1300     assert(count == 0)
1301     if item.loc then
1302       count = #item.loc
1303       solids_combine(solidity, item.loc.solid)
1304     end
1305
1306     if dbg_data then
1307       if #item > 0 then
1308         item.link_unused = {}
1309         while #item > 0 do table.insert(item.link_unused, table.remove(item, 1)) end
1310       end
1311     else
1312       while #item > 0 do table.remove(item) end
1313     end
1314   end
1315
1316   local valid = item.loc or #item > 0
1317   --[[if valid then -- technically not necessarily true
1318     assert(count > 0)
1319   else
1320     assert(count == 0)
1321   end]]
1322   return valid, count, reason, solidity
1323 end
1324
1325 local function mark_chains(file, item)
1326   for k, v in ipairs(item) do
1327     if file[v.sourcetype][v.sourceid] then
1328       file[v.sourcetype][v.sourceid].used = true
1329       if file[v.sourcetype][v.sourceid]["*/*"] then mark_chains(file, file[v.sourcetype][v.sourceid]["*/*"]) end
1330     end
1331   end
1332 end
1333
1334 local file_collater = ChainBlock_Create("file_collater", sources,
1335   function (key) return {
1336     Data = function(self, key, subkey, value, Output)
1337       Output("", nil, {fragment = key, value = value})
1338     end
1339   } end,
1340   nil, "output"
1341 )
1342
1343 local file_cull = ChainBlock_Create("file_cull", {file_collater},
1344   function (key) return {
1345     finalfile = {},
1346
1347     Data = function(self, key, subkey, value, Output)
1348       assert(value.value.data)
1349       assert(value.value.id)
1350       assert(value.value.key)
1351       assert(value.fragment)
1352
1353       if not self.finalfile[value.value.id] then self.finalfile[value.value.id] = {} end
1354       if not self.finalfile[value.value.id][value.value.key] then self.finalfile[value.value.id][value.value.key] = {} end
1355       assert(not self.finalfile[value.value.id][value.value.key][value.fragment])
1356       self.finalfile[value.value.id][value.value.key][value.fragment] = value.value.data
1357     end,
1358
1359     Finish = function(self, Output)
1360       -- First we go through and check to see who's got actual locations, and cull either location or linkage
1361       local qct = {}
1362       local solidity = {}
1363
1364       if self.finalfile.quest then for k, v in pairs(self.finalfile.quest) do
1365         if not solidity[k] then solidity[k] = {} end
1366
1367         if v["*/*"] and v["*/*"].criteria then
1368           for cid, crit in pairs(v["*/*"].criteria) do
1369             local _, ct, reason, solids = do_loc_choice(self.finalfile, crit, true)
1370             assert(not solidity[k][cid])
1371             solidity[k][cid] = solids
1372             crit.solid = nil
1373             table.insert(qct, {ct = ct, id = string.format("%d/%d", k, cid), reason = reason})
1374           end
1375
1376           if v["*/*"].finish and v["*/*"].finish.loc and v["*/*"].finish.loc.solid then
1377             assert(not solidity[k].finish)
1378             solidity[k].finish = v["*/*"].finish.loc.solid
1379             v["*/*"].finish.loc.solid = nil
1380           end
1381         end
1382       end end
1383       table.sort(qct, function(a, b) return a.ct < b.ct end)
1384       for _, v in ipairs(qct) do
1385         print("qct", v.ct, v.id, v.reason)
1386       end
1387
1388       -- Then we mark used/unused items
1389       if self.finalfile.quest then for k, v in pairs(self.finalfile.quest) do
1390         v.used = true
1391         if v["*/*"] and v["*/*"].criteria then
1392           for _, crit in pairs(v["*/*"].criteria) do
1393             mark_chains(self.finalfile, crit)
1394           end
1395         end
1396       end end
1397
1398       if self.finalfile.flightmasters then for k, v in pairs(self.finalfile.flightmasters) do
1399         for _, d in pairs(v) do
1400           if d.mid then
1401             mark_chains(self.finalfile, {{sourcetype = "monster", sourceid = d.mid}})
1402           end
1403         end
1404         v.used = true
1405       end end
1406
1407       -- Go through and clear out non-quest solidity
1408       if self.finalfile.quest then for k, v in pairs(self.finalfile.quest) do
1409         if v["*/*"] and v["*/*"].criteria then
1410           for cid, crit in pairs(v["*/*"].criteria) do
1411             if crit.loc then
1412               crit.loc.solid = nil
1413             end
1414           end
1415         end
1416       end end
1417       if self.finalfile.monster then for k, v in pairs(self.finalfile.monster) do
1418         if v["*/*"] and v["*/*"].loc then
1419           v["*/*"].loc.solid = nil
1420         end
1421       end end
1422
1423       -- Then we optionally cull and unmark
1424       for t, d in pairs(self.finalfile) do
1425         for k, v in pairs(d) do
1426           if dbg_data then
1427             for _, tv in pairs(v) do
1428               if type(tv) == "table" then tv.used = v.used or false end
1429             end
1430           end
1431
1432           v.used = nil
1433         end
1434
1435         if not dbg_data then
1436           self.finalfile[t] = d
1437         end
1438       end
1439
1440       for k, v in pairs(solidity) do
1441         Output("solid/testing", nil, {id = "solid", key = k, data = v}, "output_direct")
1442       end
1443
1444       for t, d in pairs(self.finalfile) do
1445         for k, d2 in pairs(d) do
1446           for s, d3 in pairs(d2) do
1447             assert(d3)
1448             Output(s, nil, {id = t, key = k, data = d3}, "output_direct")
1449           end
1450         end
1451       end
1452     end
1453   } end
1454 )
1455
1456 local output_sources = {}
1457 for _, v in ipairs(sources) do
1458   table.insert(output_sources, v)
1459 end
1460 table.insert(output_sources, file_cull)
1461
1462 local function LZW_precompute_table(inputs, tokens)
1463   -- shared init code
1464   local d = {}
1465   local i
1466   for i = 1, #tokens do
1467     d[tokens:sub(i, i)] = 0
1468   end
1469
1470   for _, input in ipairs(inputs) do
1471     local w = ""
1472     for ci = 1, #input do
1473       local c = input:sub(ci, ci)
1474       local wcp = w .. c
1475       if d[wcp] then
1476         w = wcp
1477         d[wcp] = d[wcp] + 1
1478       else
1479         d[wcp] = 1
1480         w = c
1481       end
1482     end
1483   end
1484
1485   local freq = {}
1486   for k, v in pairs(d) do
1487     if #k > 1 then
1488       table.insert(freq, {v, k})
1489     end
1490   end
1491   table.sort(freq, function(a, b) return a[1] < b[1] end)
1492
1493   return freq
1494 end
1495
1496 local function pdump(v)
1497   assert(type(v) == "table")
1498   local writo = {write = function (self, data) Merger.Add(self, data) end}
1499   persistence.store(writo, v)
1500   if not loadstring("return " .. Merger.Finish(writo)) then print(Merger.Finish(writo)) assert(false) end
1501   assert(loadstring("return " .. Merger.Finish(writo)))
1502   local dense = Diet(Merger.Finish(writo))
1503   if not dense then print("Couldn't condense") print(Merger.Finish(writo)) return end  -- wellp
1504   local dist = dense:match("{(.*)}")
1505   assert(dist)
1506   return dist
1507 end
1508
1509
1510 local compress_split = ChainBlock_Create("compress_split", output_sources,
1511   function (key) return {
1512     Data = function(self, key, subkey, value, Output)
1513       Output(key .. "/" .. value.id, subkey, value)
1514     end,
1515   } end, nil, "output_direct")
1516
1517 local compress = ChainBlock_Create("compress", {compress_split},
1518   function (key) return {
1519     finalfile = {},
1520
1521     Data = function(self, key, subkey, value, Output)
1522       assert(value.data, string.format("%s, %s", tostring(value.id), tostring(value.key)))
1523       assert(value.id)
1524       assert(value.key)
1525
1526       assert(not self.finalfile[value.key])
1527       self.finalfile[value.key] = value.data
1528     end,
1529
1530     Finish = function(self, Output)
1531
1532       local fname = "static"
1533
1534       local locale, faction, segment = key:match("(.*)/(.*)/(.*)")
1535       local orig_locale, orig_faction, orig_segment = locale, faction, segment
1536       assert(locale and faction)
1537       if locale == "*" then locale = nil end
1538       if faction == "*" then faction = nil end
1539
1540       if locale then
1541         fname = fname .. "_" .. locale
1542       end
1543       if faction then
1544         fname = fname .. "_" .. faction
1545       end
1546
1547       -- First, compression.
1548       if do_compress then
1549         local d = self.finalfile
1550         local k = segment
1551
1552         local dict = {}
1553
1554         for sk, v in pairs(d) do
1555           assert(type(sk) ~= "string" or not sk:match("__.*"))
1556           assert(type(v) == "table")
1557
1558           dist = pdump(v)
1559           if not dist then continue end
1560
1561           for i = 1, #dist do
1562             dict[dist:byte(i)] = true
1563           end
1564
1565           self.finalfile[sk] = dist
1566         end
1567
1568         local dicto = {}
1569         for k, v in pairs(dict) do
1570           table.insert(dicto, k)
1571         end
1572
1573         table.sort(dicto)
1574
1575         local dictix = string.char(unpack(dicto))
1576         assert(dictix)
1577         d.__dictionary = dictix
1578
1579         -- Now we build the precomputed LZW table
1580         do
1581           -- hackery steakery
1582           if locale == nil or locale == "enUS" or true then
1583             local inps = {}
1584             for _, v in pairs(d) do
1585               table.insert(inps, v)
1586             end
1587             local preco = LZW_precompute_table(inps, dictix)
1588
1589             local total = 0
1590             for _, v in ipairs(preco) do
1591               total = total + v[1] / #v[2]
1592             end
1593
1594             for _, v in ipairs(preco) do
1595               if v[1] > total / 100 then
1596                 --print(locale, faction, v[1], v[2])
1597               end
1598             end
1599
1600             --local ofile = ("final/%s_%s.stats"):format(fname, k)
1601             --fil = io.open(ofile, "w")
1602
1603             --for i = 1, 51, 10 do
1604               --local thresh = total / 100 / i
1605               local thresh = total / 100 / 40 -- this seems about the right threshold
1606
1607               local tix = {}
1608               for _, v in ipairs(preco) do
1609                 if v[1] > thresh then
1610                   table.insert(tix, v[2])
1611                 end
1612               end
1613               table.sort(tix, function(a, b) return #a > #b end)
1614
1615               local fundatoks = {}
1616               local usedtoks = {}
1617               for _, v in ipairs(tix) do
1618                 if usedtoks[v] then continue end
1619
1620                 for i = 1, #v do
1621                   local sub = v:sub(1, i)
1622                   usedtoks[sub] = true
1623                 end
1624                 table.insert(fundatoks, v)
1625               end
1626
1627               if segment ~= "flightmasters" or true then  -- the new decompression is quite a bit slower, and flightmasters are decompressed in large bulk on logon
1628                 local redictix = dictix
1629                 if not redictix:find("\"") then redictix = redictix .. "\"" end
1630                 if not redictix:find(",") then redictix = redictix .. "," end
1631                 if not redictix:find("\\") then redictix = redictix .. "\\" end
1632                 local ftd = pdump(fundatoks)
1633                 self.finalfile.__tokens = LZW.Compress_Dicts(ftd, redictix)
1634                 if LZW.Decompress_Dicts(self.finalfile.__tokens, redictix) ~= ftd then
1635                   print(ftd)
1636                   print(LZW.Decompress_Dicts(self.finalfile.__tokens, redictix))
1637                   print(dictix)
1638                   print(redictix)
1639                 end
1640                 assert(LZW.Decompress_Dicts(self.finalfile.__tokens, redictix) == ftd)
1641
1642                 local prep_id, prep_id_size, prep_is = LZW.Prepare(dictix, fundatoks)
1643
1644                 --local dictsize = #self.finalfile.__tokens
1645                 --local datsize = 0
1646
1647                 for sk, v in pairs(d) do
1648                   if (type(sk) ~= "string" or not sk:match("__.*")) and type(v) == "string" then
1649                     assert(type(v) == "string")
1650                     local compy = LZW.Compress_Dicts_Prepared(v, prep_id, prep_id_size, nil, prep_is)
1651                     --assert(LZW.Decompress_Dicts(compy, dictix, nil, fundatoks) == v)
1652                     --datsize = datsize + #compy
1653
1654                     self.finalfile[sk] = compy
1655                     assert(LZW.Decompress_Dicts_Prepared(self.finalfile[sk], dictix, nil, prep_is) == v)
1656                   end
1657                 end
1658               else
1659                 for sk, v in pairs(d) do
1660                   if (type(sk) ~= "string" or not sk:match("__.*")) and type(v) == "string" then
1661                     assert(type(v) == "string")
1662                     self.finalfile[sk] = LZW.Compress_Dicts(v, dictix)
1663                     assert(LZW.Decompress_Dicts(self.finalfile[sk], dictix) == v)
1664                   end
1665                 end
1666               end
1667
1668               --fil:write(string.format("%d\t%d\t%d\t%d\n", i, dictsize + datsize, dictsize, datsize))
1669
1670               --print(locale, faction, k, i, #fundatoks, dictsize, datsize, dictsize + datsize)
1671             --end
1672
1673             --fil:close()
1674
1675
1676             --[=[fil = io.open(ofile .. ".gnuplot", "w")
1677             fil:write("set term png\n")
1678             fil:write(string.format("set output \"%s.png\"\n", ofile))
1679             fil:write(string.format([[
1680                 plot \
1681                   "%s" using 1:2 with lines title 'Total', \
1682                   "%s" using 1:3 with lines title 'Dict', \
1683                   "%s" using 1:4 with lines title 'Dat']], ofile, ofile, ofile))
1684             fil:write("\n")
1685             fil:close()
1686
1687             os.execute(string.format("gnuplot %s.gnuplot", ofile))]=]
1688           end
1689         end
1690
1691         --[[for sk, v in pairs(d) do
1692           if (type(sk) ~= "string" or not sk:match("__.*")) and type(v) == "string" then
1693             assert(type(v) == "string")
1694             self.finalfile[sk] = LZW.Compress_Dicts(v, dictix)
1695             assert(LZW.Decompress_Dicts(self.finalfile[sk], dictix) == v)
1696           end
1697         end]]
1698       end
1699
1700       if do_compress and do_serialize and segment ~= "flightmasters" then
1701         --[[Header format:
1702
1703           Itemid (0 for endnode)
1704           Offset
1705           Length
1706           Rightlink]]
1707
1708         assert(not self.finalfile.__serialize_index)
1709         assert(not self.finalfile.__serialize_data)
1710
1711         local ntx = {}
1712         local intdat = {}
1713         for k, v in pairs(self.finalfile) do
1714           if type(k) == "number" then
1715             if k <= 0 then
1716               print("Out of bounds:", orig_locale, orig_faction, orig_segment, k)
1717               ntx[k] = v
1718             elseif type(v) ~= "string" then
1719               print("Not a string:", orig_locale, orig_faction, orig_segment, k)
1720               ntx[k] = v
1721             else
1722               assert(#v >= 1)
1723               table.insert(intdat, {key = k, value = v})
1724             end
1725           else
1726             ntx[k] = v
1727           end
1728         end
1729
1730         local data = {}
1731         local dat_len = 1
1732
1733         table.sort(intdat, function(a, b) return a.key < b.key end)
1734
1735         local function write_adaptint(dest, val)
1736           assert(type(val) == "number")
1737           assert(val == math.floor(val))
1738
1739           repeat
1740             dest:append(math.mod(val, 128), 7)
1741             dest:append((val >= 128) and 1 or 0, 1)
1742             val = math.floor(val / 128)
1743           until val == 0
1744         end
1745
1746         local function streamout(st, nd)
1747           local ttx = Bitstream.Output(8)
1748           if st > nd then
1749             write_adaptint(ttx, 0)
1750             return ttx:finish()
1751           else
1752             local tindex = math.floor((st + nd) / 2)
1753             write_adaptint(ttx, intdat[tindex].key)
1754             write_adaptint(ttx, dat_len)
1755             write_adaptint(ttx, #intdat[tindex].value - 1)
1756             Merger.Add(data, intdat[tindex].value)
1757             dat_len = dat_len + #intdat[tindex].value
1758             local lhs = streamout(st, tindex - 1)
1759             local rhs = streamout(tindex + 1, nd)
1760             write_adaptint(ttx, #lhs)
1761             return ttx:finish() .. lhs .. rhs
1762           end
1763         end
1764
1765         ntx.__serialize_index = streamout(1, #intdat)
1766         ntx.__serialize_data = Merger.Finish(data)
1767
1768         print("Index is", #ntx.__serialize_index, "data is", #ntx.__serialize_data)
1769
1770         self.finalfile = ntx
1771       end
1772
1773       Output(string.format("%s/%s", orig_locale, orig_faction), nil, {id = orig_segment, data = self.finalfile})
1774     end,
1775   } end)
1776
1777 local fileout = ChainBlock_Create("fileout", {compress},
1778   function (key) return {
1779     finalfile = {},
1780
1781     Data = function(self, key, subkey, value, Output)
1782       assert(value.data, string.format("%s, %s", tostring(value.id), tostring(value.key)))
1783       assert(value.id)
1784
1785       assert(not self.finalfile[value.id])
1786       self.finalfile[value.id] = value.data
1787     end,
1788
1789     Finish = function(self, Output)
1790
1791       local fname = "static"
1792
1793       local locale, faction = key:match("(.*)/(.*)")
1794       assert(locale and faction)
1795       if locale == "*" then locale = nil end
1796       if faction == "*" then faction = nil end
1797
1798       if locale then
1799         fname = fname .. "_" .. locale
1800       end
1801       if faction then
1802         fname = fname .. "_" .. faction
1803       end
1804
1805       fil = io.open(("final/%s.lua"):format(fname), "w")
1806       fil:write(([=[QuestHelper_File["%s.lua"] = "Development Version"
1807 QuestHelper_Loadtime["%s.lua"] = GetTime()
1808
1809 ]=]):format(fname, fname))
1810
1811       if not locale and not faction then
1812         fil:write("QHDB = {}", "\n")
1813       end
1814       if locale then
1815         fil:write(([[if GetLocale() ~= "%s" then return end]]):format(locale), "\n")
1816       end
1817       if faction then
1818         fil:write(([[if (UnitFactionGroup("player") == "Alliance" and 1 or 2) ~= %s then return end]]):format(faction), "\n")
1819       end
1820       fil:write("\n")
1821
1822       --fil:write("loadstring([[table.insert(QHDB, ")
1823       fil:write("table.insert(QHDB, ")
1824       persistence.store(fil, self.finalfile)
1825       fil:write(")")
1826       --fil:write(")]])()")
1827
1828       fil:close()
1829     end,
1830   } end
1831 )
1832
1833 --[[
1834 *****************************************************************
1835 Error collation
1836 ]]
1837
1838 if do_errors then
1839   local error_collater = ChainBlock_Create("error_collater", {chainhead},
1840     function (key) return {
1841       accum = {},
1842
1843       Data = function (self, key, subkey, value, Output)
1844         assert(value.local_version)
1845         if not value.toc_version or value.local_version ~= value.toc_version then return end
1846         local signature
1847         if value.key ~= "crash" then signature = value.key end
1848         if not signature then signature = value.message end
1849         local v = value.local_version
1850         if not self.accum[v] then self.accum[v] = {} end
1851         if not self.accum[v][signature] then self.accum[v][signature] = {count = 0, dats = {}, sig = signature, ver = v} end
1852         self.accum[v][signature].count = self.accum[v][signature].count + 1
1853         table.insert(self.accum[v][signature].dats, value)
1854       end,
1855
1856       Finish = function (self, Output)
1857         for ver, chunk in pairs(self.accum) do
1858           local tbd = {}
1859           for _, v in pairs(chunk) do
1860             table.insert(tbd, v)
1861           end
1862           table.sort(tbd, function(a, b) return a.count > b.count end)
1863           for i, dat in pairs(tbd) do
1864             dat.count_pos = i
1865             Output("", nil, dat)
1866           end
1867         end
1868       end
1869     } end,
1870     nil, "error"
1871   )
1872
1873   do
1874     local function acuv(tab, ites)
1875       local sit = ""
1876       for _, v in pairs(ites) do
1877         sit = sit .. string.format("%s: %s\n", tostring(v), tostring(tab[v]))
1878         tab[v] = nil
1879       end
1880       return sit
1881     end
1882     local function keez(tab)
1883       local rv = {}
1884       for k, _ in pairs(tab) do
1885         table.insert(rv, k)
1886       end
1887       return rv
1888     end
1889
1890     local error_writer = ChainBlock_Create("error_writer", {error_collater},
1891       function (key) return {
1892         Data = function (self, key, subkey, value, Output)
1893           os.execute("mkdir -p intermed/error/" .. value.ver)
1894           fil = io.open(string.format("intermed/error/%s/%03d-%05d.txt", value.ver, value.count_pos, value.count), "w")
1895           fil:write(value.sig)
1896           fil:write("\n\n\n\n")
1897
1898           for _, tab in pairs(value.dats) do
1899             local prefix = acuv(tab, {"message", "key", "toc_version", "local_version", "game_version", "locale", "timestamp", "mutation"})
1900             local postfix = acuv(tab, {"stack", "addons"})
1901             local midfix = acuv(tab, keez(tab))
1902
1903             fil:write(prefix)
1904             fil:write("\n")
1905             fil:write(midfix)
1906             fil:write("\n")
1907             fil:write(postfix)
1908             fil:write("\n\n\n")
1909           end
1910
1911           fil:close()
1912         end
1913       } end
1914     )
1915   end
1916 end
1917
1918 if ChainBlock_Work() then return end
1919
1920 local count = 1
1921
1922 local function readdir()
1923   local pip = io.popen(("find data/08 -type f | head -n %s | tail -n +%s"):format(e or 1000000000, s or 0))
1924   local flist = pip:read("*a")
1925   pip:close()
1926   local filz = {}
1927   for f in string.gmatch(flist, "[^\n]+") do
1928     if not s or count >= s then table.insert(filz, {fname = f, id = count}) end
1929     count = count + 1
1930     if e and count > e then break end
1931   end
1932   return filz
1933 end
1934
1935 local filout = readdir("data/08")
1936
1937 for k, v in pairs(filout) do
1938   --print(string.format("%d/%d: %s", k, #filz, v.fname))
1939   chainhead:Insert(v.fname, nil, {fileid = v.id})
1940 end
1941
1942 print("Finishing with " .. tostring(count - 1) .. " files")
1943 chainhead:Finish()
1944
1945 check_semiass_failure()