plugins/mod_muc_mam.lua

   1 -- XEP-0313: Message Archive Management for Prosody MUC
   2 -- Copyright (C) 2011-2017 Kim Alvefur
   3 --
   4 -- This file is MIT/X11 licensed.
   5
   6 if module:get_host_type() ~= "component" then
   7         module:log("error", "mod_%s should be loaded only on a MUC component, not normal hosts", module.name);
   8         return;
   9 end
  10
  11 local xmlns_mam     = "urn:xmpp:mam:2";
  12 local xmlns_delay   = "urn:xmpp:delay";
  13 local xmlns_forward = "urn:xmpp:forward:0";
  14 local xmlns_st_id   = "urn:xmpp:sid:0";
  15 local xmlns_muc_user = "http://jabber.org/protocol/muc#user";
  16 local muc_form_enable = "muc#roomconfig_enablearchiving"
  17
  18 local st = require "util.stanza";
  19 local rsm = require "util.rsm";
  20 local jid_bare = require "util.jid".bare;
  21 local jid_split = require "util.jid".split;
  22 local jid_prep = require "util.jid".prep;
  23 local dataform = require "util.dataforms".new;
  24
  25 local mod_muc = module:depends"muc";
  26 local get_room_from_jid = mod_muc.get_room_from_jid;
  27
  28 local is_stanza = st.is_stanza;
  29 local tostring = tostring;
  30 local time_now = os.time;
  31 local m_min = math.min;
  32 local timestamp, timestamp_parse, datestamp = import( "util.datetime", "datetime", "parse", "date");
  33 local default_max_items, max_max_items = 20, module:get_option_number("max_archive_query_results", 50);
  34
  35 local default_history_length = 20;
  36 local max_history_length = module:get_option_number("max_history_messages", math.huge);
  37
  38 local function get_historylength(room)
  39         return math.min(room._data.history_length or default_history_length, max_history_length);
  40 end
  41
  42 function schedule_cleanup()
  43         -- replaced by non-noop later if cleanup is enabled
  44 end
  45
  46 local log_all_rooms = module:get_option_boolean("muc_log_all_rooms", false);
  47 local log_by_default = module:get_option_boolean("muc_log_by_default", true);
  48
  49 local archive_store = "muc_log";
  50 local archive = module:open_store(archive_store, "archive");
  51
  52 if archive.name == "null" or not archive.find then
  53         if not archive.find then
  54                 module:log("error", "Attempt to open archive storage returned a driver without archive API support");
  55                 module:log("error", "mod_%s does not support archiving",
  56                         archive._provided_by or archive.name and "storage_"..archive.name.."(?)" or "<unknown>");
  57         else
  58                 module:log("error", "Attempt to open archive storage returned null driver");
  59         end
  60         module:log("info", "See https://prosody.im/doc/storage and https://prosody.im/doc/archiving for more information");
  61         return false;
  62 end
  63
  64 local function archiving_enabled(room)
  65         if log_all_rooms then
  66                 return true;
  67         end
  68         local enabled = room._data.archiving;
  69         if enabled == nil then
  70                 return log_by_default;
  71         end
  72         return enabled;
  73 end
  74
  75 if not log_all_rooms then
  76         module:hook("muc-config-form", function(event)
  77                 local room, form = event.room, event.form;
  78                 table.insert(form,
  79                 {
  80                         name = muc_form_enable,
  81                         type = "boolean",
  82                         label = "Enable archiving?",
  83                         value = archiving_enabled(room),
  84                 }
  85                 );
  86         end);
  87
  88         module:hook("muc-config-submitted/"..muc_form_enable, function(event)
  89                 event.room._data.archiving = event.value;
  90                 event.status_codes[event.value and "170" or "171"] = true;
  91         end);
  92 end
  93
  94 -- Note: We ignore the 'with' field as this is internally used for stanza types
  95 local query_form = dataform {
  96         { name = "FORM_TYPE"; type = "hidden"; value = xmlns_mam; };
  97         { name = "with"; type = "jid-single"; };
  98         { name = "start"; type = "text-single" };
  99         { name = "end"; type = "text-single"; };
 100 };
 101
 102 -- Serve form
 103 module:hook("iq-get/bare/"..xmlns_mam..":query", function(event)
 104         local origin, stanza = event.origin, event.stanza;
 105         origin.send(st.reply(stanza):tag("query", { xmlns = xmlns_mam }):add_child(query_form:form()));
 106         return true;
 107 end);
 108
 109 -- Handle archive queries
 110 module:hook("iq-set/bare/"..xmlns_mam..":query", function(event)
 111         local origin, stanza = event.origin, event.stanza;
 112         local room_jid = stanza.attr.to;
 113         local room_node = jid_split(room_jid);
 114         local orig_from = stanza.attr.from;
 115         local query = stanza.tags[1];
 116
 117         local room = get_room_from_jid(room_jid);
 118         if not room then
 119                 origin.send(st.error_reply(stanza, "cancel", "item-not-found"))
 120                 return true;
 121         end
 122         local from = jid_bare(orig_from);
 123
 124         -- Banned or not a member of a members-only room?
 125         local from_affiliation = room:get_affiliation(from);
 126         if from_affiliation == "outcast" -- banned
 127                 or room:get_members_only() and not from_affiliation then -- members-only, not a member
 128                 origin.send(st.error_reply(stanza, "auth", "forbidden"))
 129                 return true;
 130         end
 131
 132         local qid = query.attr.queryid;
 133
 134         -- Search query parameters
 135         local qstart, qend;
 136         local form = query:get_child("x", "jabber:x:data");
 137         if form then
 138                 local err;
 139                 form, err = query_form:data(form);
 140                 if err then
 141                         origin.send(st.error_reply(stanza, "modify", "bad-request", select(2, next(err))));
 142                         return true;
 143                 end
 144                 qstart, qend = form["start"], form["end"];
 145         end
 146
 147         if qstart or qend then -- Validate timestamps
 148                 local vstart, vend = (qstart and timestamp_parse(qstart)), (qend and timestamp_parse(qend))
 149                 if (qstart and not vstart) or (qend and not vend) then
 150                         origin.send(st.error_reply(stanza, "modify", "bad-request", "Invalid timestamp"))
 151                         return true;
 152                 end
 153                 qstart, qend = vstart, vend;
 154         end
 155
 156         module:log("debug", "Archive query id %s from %s until %s)",
 157                 tostring(qid),
 158                 qstart and timestamp(qstart) or "the dawn of time",
 159                 qend and timestamp(qend) or "now");
 160
 161         -- RSM stuff
 162         local qset = rsm.get(query);
 163         local qmax = m_min(qset and qset.max or default_max_items, max_max_items);
 164         local reverse = qset and qset.before or false;
 165
 166         local before, after = qset and qset.before, qset and qset.after;
 167         if type(before) ~= "string" then before = nil; end
 168
 169         -- Load all the data!
 170         local data, err = archive:find(room_node, {
 171                 start = qstart; ["end"] = qend; -- Time range
 172                 limit = qmax + 1;
 173                 before = before; after = after;
 174                 reverse = reverse;
 175                 with = "message<groupchat";
 176         });
 177
 178         if not data then
 179                 origin.send(st.error_reply(stanza, "cancel", "internal-server-error"));
 180                 return true;
 181         end
 182         local total = tonumber(err);
 183
 184         local msg_reply_attr = { to = stanza.attr.from, from = stanza.attr.to };
 185
 186         local results = {};
 187
 188         -- Wrap it in stuff and deliver
 189         local first, last;
 190         local count = 0;
 191         local complete = "true";
 192         for id, item, when in data do
 193                 count = count + 1;
 194                 if count > qmax then
 195                         complete = nil;
 196                         break;
 197                 end
 198                 local fwd_st = st.message(msg_reply_attr)
 199                         :tag("result", { xmlns = xmlns_mam, queryid = qid, id = id })
 200                                 :tag("forwarded", { xmlns = xmlns_forward })
 201                                         :tag("delay", { xmlns = xmlns_delay, stamp = timestamp(when) }):up();
 202
 203                 -- Strip <x> tag, containing the original senders JID, unless the room makes this public
 204                 if room:get_whois() ~= "anyone" then
 205                         item:maptags(function (tag)
 206                                 if tag.name == "x" and tag.attr.xmlns == xmlns_muc_user then
 207                                         return nil;
 208                                 end
 209                                 return tag;
 210                         end);
 211                 end
 212                 if not is_stanza(item) then
 213                         item = st.deserialize(item);
 214                 end
 215                 item.attr.to = nil;
 216                 item.attr.xmlns = "jabber:client";
 217                 fwd_st:add_child(item);
 218
 219                 if not first then first = id; end
 220                 last = id;
 221
 222                 if reverse then
 223                         results[count] = fwd_st;
 224                 else
 225                         origin.send(fwd_st);
 226                 end
 227         end
 228
 229         if reverse then
 230                 for i = #results, 1, -1 do
 231                         origin.send(results[i]);
 232                 end
 233                 first, last = last, first;
 234         end
 235
 236         -- That's all folks!
 237         module:log("debug", "Archive query %s completed", tostring(qid));
 238
 239         origin.send(st.reply(stanza)
 240                 :tag("fin", { xmlns = xmlns_mam, queryid = qid, complete = complete })
 241                         :add_child(rsm.generate {
 242                                 first = first, last = last, count = total }));
 243         return true;
 244 end);
 245
 246 module:hook("muc-get-history", function (event)
 247         local room = event.room;
 248         if not archiving_enabled(room) then return end
 249         local room_jid = room.jid;
 250         local maxstanzas = event.maxstanzas;
 251         local maxchars = event.maxchars;
 252         local since = event.since;
 253         local to = event.to;
 254
 255         if maxstanzas == 0 or maxchars == 0 then
 256                 return -- No history requested
 257         end
 258
 259         if not maxstanzas or maxstanzas > get_historylength(room) then
 260                 maxstanzas = get_historylength(room);
 261         end
 262
 263         if room._history and #room._history >= maxstanzas then
 264                 return -- It can deal with this itself
 265         end
 266
 267         -- Load all the data!
 268         local query = {
 269                 limit = maxstanzas;
 270                 start = since;
 271                 reverse = true;
 272                 with = "message<groupchat";
 273         }
 274         local data, err = archive:find(jid_split(room_jid), query);
 275
 276         if not data then
 277                 module:log("error", "Could not fetch history: %s", tostring(err));
 278                 return
 279         end
 280
 281         local history, i = {}, 1;
 282
 283         for id, item, when in data do
 284                 item.attr.to = to;
 285                 item:tag("delay", { xmlns = "urn:xmpp:delay", from = room_jid, stamp = timestamp(when) }):up(); -- XEP-0203
 286                 item:tag("stanza-id", { xmlns = xmlns_st_id, by = room_jid, id = id }):up();
 287                 if room:get_whois() ~= "anyone" then
 288                         item:maptags(function (tag)
 289                                 if tag.name == "x" and tag.attr.xmlns == xmlns_muc_user then
 290                                         return nil;
 291                                 end
 292                                 return tag;
 293                         end);
 294                 end
 295                 if maxchars then
 296                         local chars = #tostring(item);
 297                         if maxchars - chars < 0 then
 298                                 break
 299                         end
 300                         maxchars = maxchars - chars;
 301                 end
 302                 history[i], i = item, i+1;
 303                 -- module:log("debug", tostring(item));
 304         end
 305         function event.next_stanza()
 306                 i = i - 1;
 307                 return history[i];
 308         end
 309         return true;
 310 end, 1);
 311
 312 module:hook("muc-broadcast-messages", function (event)
 313         local room, stanza = event.room, event.stanza;
 314
 315         -- Filter out <stanza-id> that claim to be from us
 316         stanza:maptags(function (tag)
 317                 if tag.name == "stanza-id" and tag.attr.xmlns == xmlns_st_id
 318                 and jid_prep(tag.attr.by) == room.jid then
 319                         return nil;
 320                 end
 321                 if tag.name == "x" and tag.attr.xmlns == xmlns_muc_user then
 322                         return nil;
 323                 end
 324                 return tag;
 325         end);
 326
 327 end, 0);
 328
 329 -- Handle messages
 330 local function save_to_history(self, stanza)
 331         local room_node, room_host = jid_split(self.jid);
 332
 333         local stored_stanza = stanza;
 334
 335         if stanza.name == "message" and self:get_whois() == "anyone" then
 336                 stored_stanza = st.clone(stanza);
 337                 stored_stanza.attr.to = nil;
 338                 local actor = jid_bare(self._occupants[stanza.attr.from].jid);
 339                 local affiliation = self:get_affiliation(actor) or "none";
 340                 local role = self:get_role(actor) or self:get_default_role(affiliation);
 341                 stored_stanza:add_direct_child(st.stanza("x", { xmlns = xmlns_muc_user })
 342                         :tag("item", { affiliation = affiliation; role = role; jid = actor }));
 343         end
 344
 345         -- Policy check
 346         if not archiving_enabled(self) then return end -- Don't log
 347
 348         -- Save the type in the 'with' field, allows storing presence without conflicts
 349         local with = stanza.name
 350         if stanza.attr.type then
 351                 with = with .. "<" .. stanza.attr.type
 352         end
 353
 354         -- And stash it
 355         local id = archive:append(room_node, nil, stored_stanza, time_now(), with);
 356
 357         if id then
 358                 schedule_cleanup(room_node);
 359                 stanza:add_direct_child(st.stanza("stanza-id", { xmlns = xmlns_st_id, by = self.jid, id = id }));
 360         end
 361 end
 362
 363 module:hook("muc-add-history", function (event)
 364         local room, stanza = event.room, event.stanza;
 365         save_to_history(room, stanza);
 366 end);
 367
 368 if module:get_option_boolean("muc_log_presences", false) then
 369         module:hook("muc-occupant-joined", function (event)
 370                 save_to_history(event.room, st.stanza("presence", { from = event.nick }):tag("x", { xmlns = "http://jabber.org/protocol/muc" }));
 371         end);
 372         module:hook("muc-occupant-left", function (event)
 373                 save_to_history(event.room, st.stanza("presence", { type = "unavailable", from = event.nick }));
 374         end);
 375 end
 376
 377 if not archive.delete then
 378         module:log("warn", "Storage driver %s does not support deletion", archive._provided_by);
 379         module:log("warn", "Archived message will persist after a room has been destroyed");
 380 else
 381         module:hook("muc-room-destroyed", function(event)
 382                 local room_node = jid_split(event.room.jid);
 383                 archive:delete(room_node);
 384         end);
 385 end
 386
 387 -- And role/affiliation changes?
 388
 389 module:add_feature(xmlns_mam);
 390
 391 module:hook("muc-disco#info", function(event)
 392         event.reply:tag("feature", {var=xmlns_mam}):up();
 393 end);
 394
 395 -- Cleanup
 396
 397 local cleanup_after = module:get_option_string("muc_log_expires_after", "1w");
 398 local cleanup_interval = module:get_option_number("muc_log_cleanup_interval", 4 * 60 * 60);
 399
 400 if cleanup_after ~= "never" then
 401         local cleanup_storage = module:open_store("muc_log_cleanup");
 402         local cleanup_map = module:open_store("muc_log_cleanup", "map");
 403
 404         local day = 86400;
 405         local multipliers = { d = day, w = day * 7, m = 31 * day, y = 365.2425 * day };
 406         local n, m = cleanup_after:lower():match("(%d+)%s*([dwmy]?)");
 407         if not n then
 408                 module:log("error", "Could not parse muc_log_expires_after string %q", cleanup_after);
 409                 return false;
 410         end
 411
 412         cleanup_after = tonumber(n) * ( multipliers[m] or 1 );
 413
 414         module:log("debug", "muc_log_expires_after = %d -- in seconds", cleanup_after);
 415
 416         if not archive.delete then
 417                 module:log("error", "muc_log_expires_after set but mod_%s does not support deleting", archive._provided_by);
 418                 return false;
 419         end
 420
 421         -- For each day, store a set of rooms that have new messages. To expire
 422         -- messages, we collect the union of sets of rooms from dates that fall
 423         -- outside the cleanup range.
 424
 425         local last_date = require "util.cache".new(module:get_option_number("muc_log_cleanup_date_cache_size", 1000));
 426         function schedule_cleanup(roomname, date)
 427                 date = date or datestamp();
 428                 if last_date:get(roomname) == date then return end
 429                 local ok = cleanup_map:set(date, roomname, true);
 430                 if ok then
 431                         last_date:set(roomname, date);
 432                 end
 433         end
 434
 435         cleanup_runner = require "util.async".runner(function ()
 436                 local rooms = {};
 437                 local cut_off = datestamp(os.time() - cleanup_after);
 438                 for date in cleanup_storage:users() do
 439                         if date <= cut_off then
 440                                 module:log("debug", "Messages from %q should be expired", date);
 441                                 local messages_this_day = cleanup_storage:get(date);
 442                                 if messages_this_day then
 443                                         for room in pairs(messages_this_day) do
 444                                                 rooms[room] = true;
 445                                         end
 446                                         if date < cut_off then
 447                                                 -- Messages from the same day as the cut-off might not have expired yet,
 448                                                 -- but all earlier will have, so clear storage for those days.
 449                                                 cleanup_storage:set(date, nil);
 450                                         end
 451                                 end
 452                         end
 453                 end
 454                 local sum, num_rooms = 0, 0;
 455                 for room in pairs(rooms) do
 456                         local ok, err = archive:delete(room, { ["end"] = os.time() - cleanup_after; })
 457                         if ok then
 458                                 num_rooms = num_rooms + 1;
 459                                 sum = sum + (tonumber(ok) or 0);
 460                         end
 461                 end
 462                 module:log("info", "Deleted %d expired messages for %d rooms", sum, num_rooms);
 463         end);
 464
 465         cleanup_task = module:add_timer(1, function ()
 466                 cleanup_runner:run(true);
 467                 return cleanup_interval;
 468         end);
 469 else
 470         module:log("debug", "Archive expiry disabled");
 471 end