test/t_mapcdb.ml

   1 (*
   2  *    _  _   ____                         _
   3  *  _| || |_/ ___|  ___ _ __  _ __   ___ | |
   4  * |_  ..  _\___ \ / _ \ '_ \| '_ \ / _ \| |
   5  * |_      _|___) |  __/ |_) | |_) | (_) |_|
   6  *   |_||_| |____/ \___| .__/| .__/ \___/(_)
   7  *                     |_|   |_|
   8  *
   9  * Personal Social Web.
  10  *
  11  * Copyright (C) The #Seppo contributors. All rights reserved.
  12  *
  13  * This program is free software: you can redistribute it and/or modify
  14  * it under the terms of the GNU General Public License as published by
  15  * the Free Software Foundation, either version 3 of the License, or
  16  * (at your option) any later version.
  17  *
  18  * This program is distributed in the hope that it will be useful,
  19  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  20  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  21  * GNU General Public License for more details.
  22  *
  23  * You should have received a copy of the GNU General Public License
  24  * along with this program.  If not, see <http://www.gnu.org/licenses/>.
  25  *)
  26
  27 open Seppo_lib
  28
  29 let o i = i |> Optint.of_int
  30
  31 let oo i = i |> Optint.of_int64
  32
  33 let test_hash () =
  34   Logr.info (fun m -> m "mapcdb_test.test_hash");
  35   let open Mapcdb in
  36   "s" |> Bytes.of_string |> Ds_cdb.hash
  37   |> Assrt.equals_int32 __LOC__ 177622l;
  38
  39   "s" |> Bytes.of_string |> hash32_byt |> Assrt.equals_optint __LOC__ (o 177622);
  40
  41   Assrt.equals_optint __LOC__ (o 0x2b5c4) ("a" |> Bytes.of_string |> hash32_byt);
  42   let k0 =
  43     "http://www.traunsteiner-tagblatt.de/region+lokal/landkreis-traunstein/traunstein/pressemitteilungen-der-stadt-traunstein_artikel,-Traunstein-20-%E2%80%93-Neue-Medien-im-Mittelpunkt-_arid,198374.html"
  44   in
  45   Assrt.equals_optint __LOC__ (oo 0xc7410a37L) (k0 |> Bytes.of_string |> hash32_byt);
  46
  47   assert true
  48
  49 let test_find_string_opt () =
  50   Logr.info (fun m -> m "mapcdb_test.test_find_string_opt");
  51   let open Mapcdb in
  52   find_string_opt "s" (Cdb "mini.cdb")
  53   |> Option.get
  54   |> Assrt.equals_string __LOC__ "ß";
  55   assert (None = find_string_opt "zzz" (Cdb "mini.cdb"));
  56 (*
  57   find_string_opt
  58     "http://www.traunsteiner-tagblatt.de/region+lokal/landkreis-traunstein/traunstein/pressemitteilungen-der-stadt-traunstein_artikel,-Traunstein-20-%E2%80%93-Neue-Medien-im-Mittelpunkt-_arid,198374.html"
  59     (Cdb "big.cdb")
  60   |> Option.get |> String.length
  61   |> Assrt.equals_int __LOC__ 1000;
  62 *)
  63   assert true
  64
  65 let mk_db fn =
  66   (try
  67      Unix.mkdir "tmp" File.pDir;
  68    with Unix.Unix_error (Unix.EEXIST, _, _) -> ());
  69   (try
  70      Unix.unlink fn;
  71      Unix.unlink (fn ^ "~")
  72    with Unix.Unix_error (Unix.ENOENT, _, _) -> ());
  73   File.touch fn;
  74   Mapcdb.Cdb fn
  75
  76 let test_update () =
  77   Logr.info (fun m -> m "mapcdb_test.test_update");
  78   let fn = mk_db "tmp/add.cdb" in
  79   let _ = Mapcdb.update_string "a" "1" fn in
  80   Mapcdb.find_string_opt "a" fn |> Option.get |> Assrt.equals_string __LOC__ "1";
  81   let _ = Mapcdb.update_string "a" "2" fn in
  82   Mapcdb.find_string_opt "a" fn |> Option.get |> Assrt.equals_string __LOC__ "2";
  83   let _ = Mapcdb.remove_string "a" fn in
  84   assert (Mapcdb.find_string_opt "a" fn |> Option.is_none);
  85   assert true
  86
  87 let test_fold () =
  88   Logr.info (fun m -> m "%s.%s" "mapcdb_test" "fold_left");
  89   let cdb = Mapcdb.Cdb "data/mini.cdb" in
  90   Mapcdb.fold_left (fun init (k,v) ->
  91       let k = k |> Bytes.to_string
  92       and v = v |> Bytes.to_string in
  93       (* Logr.debug (fun m -> m "  %s->%s" k v); *)
  94       (k,v) :: init) [] cdb
  95   |> List.length |> Assrt.equals_int __LOC__ 3
  96
  97 let test_fold_1 () =
  98   Mapcdb.Cdb "data/2024-04-30-131146-subscribed.cdb"
  99   |> Mapcdb.fold_left (fun c _ -> 1 + c) 0
 100   |> Assrt.equals_int __LOC__ 52
 101
 102 let test_hash32_str_base64 () =
 103   Logr.info (fun m -> m "%s.%s" "Mapcdb_test" "test_hash32_str_base64");
 104   Optint.max_int
 105   |> Optint.to_int64
 106   |> Printf.sprintf "0x%Lx"
 107   |> Assrt.equals_string __LOC__ "0x7fffffff";
 108   Mapcdb._32_0xFFffFFff
 109   |> Optint.to_int64
 110   |> Printf.sprintf "0x%Lx"
 111   |> Assrt.equals_string __LOC__ "0xffffffff";
 112   let h = "https://dev.seppo.social/2023-11-13/activitypub/profile.jlda"
 113           |> Uri.of_string |> Uri.to_string
 114           |> Mapcdb.hash32_str in
 115   h |> Optint.to_string
 116   |> Assrt.equals_string __LOC__ "1316135747";
 117   Optint.encoded_size |> Assrt.equals_int __LOC__ 4;
 118   let b = Bytes.make Optint.encoded_size (Char.chr 0) in
 119   h |> Optint.encode b ~off:0;
 120   b |> Bytes.to_string
 121   |> Base64.encode_string ~pad:false ~alphabet:Base64.uri_safe_alphabet
 122   |> Assrt.equals_string __LOC__ "TnKjQw";
 123   assert true
 124
 125 let test_hash63_str_base64 () =
 126   Logr.info (fun m -> m "%s.%s" "Mapcdb_test" "test_hash63_str_base64");
 127   Optint.Int63.encoded_size |> Assrt.equals_int __LOC__ 8;
 128   let _mask_63 = Optint.Int63.max_int
 129   and _5381_63 = 5381 |> Optint.Int63.of_int in
 130   (* http://cr.yp.to/cdb/cdb.txt *)
 131   let hash63_gen len f_get : Optint.Int63.t =
 132     let ( +. )   = Optint.Int63.add
 133     and ( << )   = Optint.Int63.shift_left
 134     and ( ^ )    = Optint.Int63.logxor
 135     and ( land ) = Optint.Int63.logand in
 136     let rec fkt (idx : int) (h : Optint.Int63.t) =
 137       if idx = len
 138       then h
 139       else
 140         let c = idx |> f_get |> Char.code |> Optint.Int63.of_int in
 141         (((h << 5) +. h) ^ c) land _mask_63
 142         |> fkt (succ idx)
 143     in
 144     fkt 0 _5381_63
 145   in
 146   let hash63_str dat : Optint.Int63.t =
 147     hash63_gen (String.length dat) (String.get dat)
 148   in
 149   let uhash ?(off = 0) ?(buf = Bytes.make (Optint.Int63.encoded_size) (Char.chr 0)) u =
 150     u
 151     |> Uri.to_string
 152     |> hash63_str
 153     |> Optint.Int63.encode buf ~off;
 154     buf
 155     |> Bytes.to_string
 156     |> Base64.encode_string ~pad:false ~alphabet:Base64.uri_safe_alphabet
 157   in
 158   _mask_63
 159   |> Optint.Int63.to_int64
 160   |> Printf.sprintf "0x%Lx"
 161   |> Assrt.equals_string __LOC__ "0x3fffffffffffffff";
 162   let h = "https://dev.seppo.social/2023-11-13/activitypub/profile.jlda"
 163           |> Uri.of_string |> Uri.to_string
 164           |> hash63_str in
 165   h |> Optint.Int63.to_string
 166   |> Assrt.equals_string __LOC__ "4387560302522311491";
 167   let b = Bytes.make Optint.Int63.encoded_size (Char.chr 0) in
 168   h |> Optint.Int63.encode b ~off:0;
 169   b |> Bytes.to_string
 170   |> Base64.encode_string ~pad:false ~alphabet:Base64.uri_safe_alphabet
 171   |> Assrt.equals_string __LOC__ "POO-2U5yo0M";
 172   "https://dev.seppo.social/2023-11-13/activitypub/profile.jlda" |> Uri.of_string
 173   |> uhash |> Assrt.equals_string __LOC__ "POO-2U5yo0M";
 174   "https://digitalcourage.social/users/mro" |> Uri.of_string
 175   |> uhash |> Assrt.equals_string __LOC__ "BlamH6XVcgE";
 176   "https://digitalcourage.social/users/mrp" |> Uri.of_string
 177   |> uhash |> Assrt.equals_string __LOC__ "BlamH6XVch4";
 178   "ittps://digitalcourage.social/users/mrp" |> Uri.of_string
 179   |> uhash |> Assrt.equals_string __LOC__ "LesfhY0ub58";
 180   assert true
 181
 182 let bench_update_1 n =
 183   Logr.info (fun m -> m "mapcdb_test.bench_update_1");
 184   let fn = mk_db "tmp/add_1.cdb" in
 185   let i = ref 1 in
 186   while !i < n do
 187     Printf.printf ".";
 188     let k = Printf.sprintf "%d %f" !i (Sys.time()) in
 189     let _ = Mapcdb.update_string k k fn in
 190     incr i
 191   done;
 192   assert true
 193
 194 let bench_update_2 n =
 195   Logr.info (fun m -> m "mapcdb_test.bench_update_2");
 196   let fn = mk_db "tmp/add_2.cdb" in
 197   let l = List.init n
 198       (fun i ->
 199          let k :bytes = Printf.sprintf "%d" i |> Bytes.of_string in
 200          (k,k)) in
 201   let s = List.to_seq l in
 202   let all _ = true in
 203   let _ = Mapcdb.add_seq all s fn in
 204   assert true
 205
 206 let bench_update_3 n =
 207   Logr.info (fun m -> m "mapcdb_test.bench_update_3");
 208   let fn = mk_db "tmp/add_3.cdb"
 209   and all _ = true
 210   and fkt add =
 211     let i = ref 0 in
 212     while !i < n do
 213       let k = Printf.sprintf "%d" !i |> Bytes.of_string in
 214       let v = Printf.sprintf "%f" (Sys.time()) |> Bytes.of_string in
 215       let _ = add (k,v) in
 216       incr i
 217     done;
 218   in
 219   let _ = Mapcdb.add_many all fkt fn in
 220   assert true
 221
 222 let () =
 223   Unix.chdir "../../../test/";
 224   test_hash ();
 225   test_find_string_opt ();
 226   test_update ();
 227   test_fold ();
 228   test_fold_1 ();
 229   test_hash32_str_base64 ();
 230   test_hash63_str_base64 ();
 231   bench_update_1 1000;
 232   bench_update_2 100_000;
 233   bench_update_3 100_000;
 234   assert true