plugins/mod_mam/mod_mam.lua

   1 -- Prosody IM
   2 -- Copyright (C) 2008-2017 Matthew Wild
   3 -- Copyright (C) 2008-2017 Waqas Hussain
   4 -- Copyright (C) 2011-2017 Kim Alvefur
   5 --
   6 -- This project is MIT/X11 licensed. Please see the
   7 -- COPYING file in the source package for more information.
   8 --
   9 -- XEP-0313: Message Archive Management for Prosody
  10 --
  11
  12 local xmlns_mam     = "urn:xmpp:mam:2";
  13 local xmlns_delay   = "urn:xmpp:delay";
  14 local xmlns_forward = "urn:xmpp:forward:0";
  15 local xmlns_st_id   = "urn:xmpp:sid:0";
  16
  17 local um = require "core.usermanager";
  18 local st = require "util.stanza";
  19 local rsm = require "util.rsm";
  20 local get_prefs = module:require"mamprefs".get;
  21 local set_prefs = module:require"mamprefs".set;
  22 local prefs_to_stanza = module:require"mamprefsxml".tostanza;
  23 local prefs_from_stanza = module:require"mamprefsxml".fromstanza;
  24 local jid_bare = require "util.jid".bare;
  25 local jid_split = require "util.jid".split;
  26 local jid_prepped_split = require "util.jid".prepped_split;
  27 local dataform = require "util.dataforms".new;
  28 local host = module.host;
  29
  30 local rm_load_roster = require "core.rostermanager".load_roster;
  31
  32 local is_stanza = st.is_stanza;
  33 local tostring = tostring;
  34 local time_now = os.time;
  35 local m_min = math.min;
  36 local timestamp, timestamp_parse = require "util.datetime".datetime, require "util.datetime".parse;
  37 local default_max_items, max_max_items = 20, module:get_option_number("max_archive_query_results", 50);
  38 local strip_tags = module:get_option_set("dont_archive_namespaces", { "http://jabber.org/protocol/chatstates" });
  39
  40 local archive_store = module:get_option_string("archive_store", "archive");
  41 local archive = module:open_store(archive_store, "archive");
  42
  43 if not archive.find then
  44         error("mod_"..(archive._provided_by or archive.name and "storage_"..archive.name).." does not support archiving\n"
  45                 .."See https://prosody.im/doc/storage and https://prosody.im/doc/archiving for more information");
  46 end
  47 local use_total = module:get_option_boolean("mam_include_total", true);
  48
  49 local cleanup;
  50
  51 local function schedule_cleanup(username)
  52         if cleanup and not cleanup[username] then
  53                 table.insert(cleanup, username);
  54                 cleanup[username] = true;
  55         end
  56 end
  57
  58 -- Handle prefs.
  59 module:hook("iq/self/"..xmlns_mam..":prefs", function(event)
  60         local origin, stanza = event.origin, event.stanza;
  61         local user = origin.username;
  62         if stanza.attr.type == "set" then
  63                 local new_prefs = stanza:get_child("prefs", xmlns_mam);
  64                 local prefs = prefs_from_stanza(new_prefs);
  65                 local ok, err = set_prefs(user, prefs);
  66                 if not ok then
  67                         origin.send(st.error_reply(stanza, "cancel", "internal-server-error", "Error storing preferences: "..tostring(err)));
  68                         return true;
  69                 end
  70         end
  71         local prefs = prefs_to_stanza(get_prefs(user, true));
  72         local reply = st.reply(stanza):add_child(prefs);
  73         origin.send(reply);
  74         return true;
  75 end);
  76
  77 local query_form = dataform {
  78         { name = "FORM_TYPE"; type = "hidden"; value = xmlns_mam; };
  79         { name = "with"; type = "jid-single"; };
  80         { name = "start"; type = "text-single" };
  81         { name = "end"; type = "text-single"; };
  82 };
  83
  84 -- Serve form
  85 module:hook("iq-get/self/"..xmlns_mam..":query", function(event)
  86         local origin, stanza = event.origin, event.stanza;
  87         get_prefs(origin.username, true);
  88         origin.send(st.reply(stanza):query(xmlns_mam):add_child(query_form:form()));
  89         return true;
  90 end);
  91
  92 -- Handle archive queries
  93 module:hook("iq-set/self/"..xmlns_mam..":query", function(event)
  94         local origin, stanza = event.origin, event.stanza;
  95         local query = stanza.tags[1];
  96         local qid = query.attr.queryid;
  97
  98         get_prefs(origin.username, true);
  99         schedule_cleanup(origin.username);
 100
 101         -- Search query parameters
 102         local qwith, qstart, qend;
 103         local form = query:get_child("x", "jabber:x:data");
 104         if form then
 105                 local err;
 106                 form, err = query_form:data(form);
 107                 if err then
 108                         origin.send(st.error_reply(stanza, "modify", "bad-request", select(2, next(err))));
 109                         return true;
 110                 end
 111                 qwith, qstart, qend = form["with"], form["start"], form["end"];
 112                 qwith = qwith and jid_bare(qwith); -- dataforms does jidprep
 113         end
 114
 115         if qstart or qend then -- Validate timestamps
 116                 local vstart, vend = (qstart and timestamp_parse(qstart)), (qend and timestamp_parse(qend));
 117                 if (qstart and not vstart) or (qend and not vend) then
 118                         origin.send(st.error_reply(stanza, "modify", "bad-request", "Invalid timestamp"))
 119                         return true;
 120                 end
 121                 qstart, qend = vstart, vend;
 122         end
 123
 124         module:log("debug", "Archive query, id %s with %s from %s until %s",
 125                 tostring(qid), qwith or "anyone",
 126                 qstart and timestamp(qstart) or "the dawn of time",
 127                 qend and timestamp(qend) or "now");
 128
 129         -- RSM stuff
 130         local qset = rsm.get(query);
 131         local qmax = m_min(qset and qset.max or default_max_items, max_max_items);
 132         local reverse = qset and qset.before or false;
 133         local before, after = qset and qset.before, qset and qset.after;
 134         if type(before) ~= "string" then before = nil; end
 135
 136         -- Load all the data!
 137         local data, err = archive:find(origin.username, {
 138                 start = qstart; ["end"] = qend; -- Time range
 139                 with = qwith;
 140                 limit = qmax == 0 and 0 or qmax + 1;
 141                 before = before; after = after;
 142                 reverse = reverse;
 143                 total = use_total or qmax == 0;
 144         });
 145
 146         if not data then
 147                 origin.send(st.error_reply(stanza, "cancel", "internal-server-error", err));
 148                 return true;
 149         end
 150         local total = tonumber(err);
 151
 152         local msg_reply_attr = { to = stanza.attr.from, from = stanza.attr.to };
 153
 154         local results = {};
 155
 156         -- Wrap it in stuff and deliver
 157         local first, last;
 158         local count = 0;
 159         local complete = "true";
 160         for id, item, when in data do
 161                 count = count + 1;
 162                 if count > qmax then
 163                         -- We requested qmax+1 items. If that many items are retrieved then
 164                         -- there are more results to page through, so:
 165                         complete = nil;
 166                         break;
 167                 end
 168                 local fwd_st = st.message(msg_reply_attr)
 169                         :tag("result", { xmlns = xmlns_mam, queryid = qid, id = id })
 170                                 :tag("forwarded", { xmlns = xmlns_forward })
 171                                         :tag("delay", { xmlns = xmlns_delay, stamp = timestamp(when) }):up();
 172
 173                 if not is_stanza(item) then
 174                         item = st.deserialize(item);
 175                 end
 176                 item.attr.xmlns = "jabber:client";
 177                 fwd_st:add_child(item);
 178
 179                 if not first then first = id; end
 180                 last = id;
 181
 182                 if reverse then
 183                         results[count] = fwd_st;
 184                 else
 185                         origin.send(fwd_st);
 186                 end
 187         end
 188
 189         if reverse then
 190                 for i = #results, 1, -1 do
 191                         origin.send(results[i]);
 192                 end
 193                 first, last = last, first;
 194         end
 195
 196         -- That's all folks!
 197         module:log("debug", "Archive query %s completed", tostring(qid));
 198
 199         origin.send(st.reply(stanza)
 200                 :tag("fin", { xmlns = xmlns_mam, queryid = qid, complete = complete })
 201                         :add_child(rsm.generate {
 202                                 first = first, last = last, count = total }));
 203         return true;
 204 end);
 205
 206 local function has_in_roster(user, who)
 207         local roster = rm_load_roster(user, host);
 208         module:log("debug", "%s has %s in roster? %s", user, who, roster[who] and "yes" or "no");
 209         return roster[who];
 210 end
 211
 212 local function shall_store(user, who)
 213         -- TODO Cache this?
 214         if not um.user_exists(user, host) then
 215                 return false;
 216         end
 217         local prefs = get_prefs(user);
 218         local rule = prefs[who];
 219         module:log("debug", "%s's rule for %s is %s", user, who, tostring(rule));
 220         if rule ~= nil then
 221                 return rule;
 222         end
 223         -- Below could be done by a metatable
 224         local default = prefs[false];
 225         module:log("debug", "%s's default rule is %s", user, tostring(default));
 226         if default == "roster" then
 227                 return has_in_roster(user, who);
 228         end
 229         return default;
 230 end
 231
 232 local function strip_stanza_id(stanza, user)
 233         if stanza:get_child("stanza-id", xmlns_st_id) then
 234                 stanza = st.clone(stanza);
 235                 stanza:maptags(function (tag)
 236                         if tag.name == "stanza-id" and tag.attr.xmlns == xmlns_st_id then
 237                                 local by_user, by_host, res = jid_prepped_split(tag.attr.by);
 238                                 if not res and by_host == host and by_user == user then
 239                                         return nil;
 240                                 end
 241                         end
 242                         return tag;
 243                 end);
 244         end
 245         return stanza;
 246 end
 247
 248 -- Handle messages
 249 local function message_handler(event, c2s)
 250         local origin, stanza = event.origin, event.stanza;
 251         local log = c2s and origin.log or module._log;
 252         local orig_type = stanza.attr.type or "normal";
 253         local orig_from = stanza.attr.from;
 254         local orig_to = stanza.attr.to or orig_from;
 255         -- Stanza without 'to' are treated as if it was to their own bare jid
 256
 257         -- Whos storage do we put it in?
 258         local store_user = c2s and origin.username or jid_split(orig_to);
 259         -- And who are they chatting with?
 260         local with = jid_bare(c2s and orig_to or orig_from);
 261
 262         -- Filter out <stanza-id> that claim to be from us
 263         event.stanza = strip_stanza_id(stanza, store_user);
 264
 265         -- We store chat messages or normal messages that have a body
 266         if not(orig_type == "chat" or (orig_type == "normal" and stanza:get_child("body")) ) then
 267                 log("debug", "Not archiving stanza: %s (type)", stanza:top_tag());
 268                 return;
 269         end
 270
 271         -- or if hints suggest we shouldn't
 272         if not stanza:get_child("store", "urn:xmpp:hints") then -- No hint telling us we should store
 273                 if stanza:get_child("no-permanent-store", "urn:xmpp:hints")
 274                         or stanza:get_child("no-store", "urn:xmpp:hints") then -- Hint telling us we should NOT store
 275                         log("debug", "Not archiving stanza: %s (hint)", stanza:top_tag());
 276                         return;
 277                 end
 278         end
 279
 280         local clone_for_storage;
 281         if not strip_tags:empty() then
 282                 clone_for_storage = st.clone(stanza);
 283                 clone_for_storage:maptags(function (tag)
 284                         if strip_tags:contains(tag.attr.xmlns) then
 285                                 return nil;
 286                         else
 287                                 return tag;
 288                         end
 289                 end);
 290                 if #clone_for_storage.tags == 0 then
 291                         log("debug", "Not archiving stanza: %s (empty when stripped)", stanza:top_tag());
 292                         return;
 293                 end
 294         else
 295                 clone_for_storage = stanza;
 296         end
 297
 298         -- Check with the users preferences
 299         if shall_store(store_user, with) then
 300                 log("debug", "Archiving stanza: %s", stanza:top_tag());
 301
 302                 -- And stash it
 303                 local ok = archive:append(store_user, nil, clone_for_storage, time_now(), with);
 304                 if ok then
 305                         local clone_for_other_handlers = st.clone(stanza);
 306                         local id = ok;
 307                         clone_for_other_handlers:tag("stanza-id", { xmlns = xmlns_st_id, by = store_user.."@"..host, id = id }):up();
 308                         event.stanza = clone_for_other_handlers;
 309                         schedule_cleanup(store_user);
 310                         module:fire_event("archive-message-added", { origin = origin, stanza = clone_for_storage, for_user = store_user, id = id });
 311                 end
 312         else
 313                 log("debug", "Not archiving stanza: %s (prefs)", stanza:top_tag());
 314         end
 315 end
 316
 317 local function c2s_message_handler(event)
 318         return message_handler(event, true);
 319 end
 320
 321 -- Filter out <stanza-id> before the message leaves the server to prevent privacy leak.
 322 local function strip_stanza_id_after_other_events(event)
 323         event.stanza = strip_stanza_id(event.stanza, event.origin.username);
 324 end
 325
 326 module:hook("pre-message/bare", strip_stanza_id_after_other_events, -1);
 327 module:hook("pre-message/full", strip_stanza_id_after_other_events, -1);
 328
 329 local cleanup_after = module:get_option_string("archive_expires_after", "1w");
 330 local cleanup_interval = module:get_option_number("archive_cleanup_interval", 4 * 60 * 60);
 331 if cleanup_after ~= "never" then
 332         local day = 86400;
 333         local multipliers = { d = day, w = day * 7, m = 31 * day, y = 365.2425 * day };
 334         local n, m = cleanup_after:lower():match("(%d+)%s*([dwmy]?)");
 335         if not n then
 336                 module:log("error", "Could not parse archive_expires_after string %q", cleanup_after);
 337                 return false;
 338         end
 339
 340         cleanup_after = tonumber(n) * ( multipliers[m] or 1 );
 341
 342         module:log("debug", "archive_expires_after = %d -- in seconds", cleanup_after);
 343
 344         if not archive.delete then
 345                 module:log("error", "archive_expires_after set but mod_%s does not support deleting", archive._provided_by);
 346                 return false;
 347         end
 348
 349         -- Set of known users to do message expiry for
 350         -- Populated either below or when new messages are added
 351         cleanup = {};
 352
 353         -- Iterating over users is not supported by all authentication modules
 354         -- Catch and ignore error if not supported
 355         pcall(function ()
 356                 -- If this works, then we schedule cleanup for all known users on startup
 357                 for user in um.users(module.host) do
 358                         schedule_cleanup(user);
 359                 end
 360         end);
 361
 362         -- At odd intervals, delete old messages for one user
 363         module:add_timer(math.random(10, 60), function()
 364                 local user = table.remove(cleanup, 1);
 365                 if user then
 366                         module:log("debug", "Removing old messages for user %q", user);
 367                         local ok, err = archive:delete(user, { ["end"] = os.time() - cleanup_after; })
 368                         if not ok then
 369                                 module:log("warn", "Could not expire archives for user %s: %s", user, err);
 370                         elseif type(ok) == "number" then
 371                                 module:log("debug", "Removed %d messages", ok);
 372                         end
 373                         cleanup[user] = nil;
 374                 end
 375                 return math.random(cleanup_interval, cleanup_interval * 2);
 376         end);
 377 else
 378         module:log("debug", "Archive expiry disabled");
 379         -- Don't ask the backend to count the potentially unbounded number of items,
 380         -- it'll get slow.
 381         use_total = module:get_option_boolean("mam_include_total", false);
 382 end
 383
 384 -- Stanzas sent by local clients
 385 module:hook("pre-message/bare", c2s_message_handler, 0);
 386 module:hook("pre-message/full", c2s_message_handler, 0);
 387 -- Stanzas to local clients
 388 module:hook("message/bare", message_handler, 0);
 389 module:hook("message/full", message_handler, 0);
 390
 391 module:hook("account-disco-info", function(event)
 392         (event.reply or event.stanza):tag("feature", {var=xmlns_mam}):up();
 393         (event.reply or event.stanza):tag("feature", {var=xmlns_st_id}):up();
 394 end);
 395