bridgedb/test/test_Storage.py

   1 #!/usr/bin/env python3
   2 """Unittests for the :mod:`bridgedb.Storage` module."""
   3
   4 import os
   5 import threading
   6 import time
   7 import datetime
   8
   9 from twisted.python import log
  10 from twisted.trial import unittest
  11 from twisted.internet import reactor
  12 from twisted.internet.threads import deferToThread
  13
  14 import bridgedb.Storage as Storage
  15 import bridgedb.main as main
  16 from bridgedb.bridges import Bridge
  17
  18 from bridgedb.test.util import generateFakeBridges
  19
  20 class DatabaseTest(unittest.TestCase):
  21     def setUp(self):
  22         self.fakeBridges = generateFakeBridges()
  23         self.validRings = ['https', 'unallocated', 'email', 'moat']
  24         self.dbfname = 'test-bridgedb.sqlite'
  25         Storage.setDBFilename(self.dbfname)
  26
  27     def tearDown(self):
  28         if os.path.isfile(self.dbfname):
  29             os.unlink(self.dbfname)
  30         Storage.clearGlobalDB()
  31
  32     def _runAndDie(self, timeout, func):
  33         with func():
  34             time.sleep(timeout)
  35
  36     def _cb_assertTrue(self, result):
  37         self.assertTrue(result)
  38
  39     def _cb_assertFalse(self, result):
  40         self.assertFalse(result)
  41
  42     def _eb_Failure(self, failure):
  43         self.fail(failure)
  44
  45     def test_getDB_FalseWhenLocked(self):
  46         Storage._LOCK = threading.Lock()
  47         Storage._LOCK.acquire()
  48         self.assertFalse(Storage._LOCK.acquire(False))
  49
  50     def test_getDB_AcquireLock(self):
  51         Storage.initializeDBLock()
  52         with Storage.getDB() as db:
  53             self.assertIsInstance(db, Storage.Database)
  54             self.assertTrue(Storage.dbIsLocked())
  55             self.assertEqual(db, Storage._OPENED_DB)
  56
  57     def test_getDB_ConcurrencyLock(self):
  58         timeout = 1
  59         d1 = deferToThread(self._runAndDie, timeout, Storage.getDB)
  60         d1.addCallback(self._cb_assertFalse)
  61         d1.addErrback(self._eb_Failure)
  62         d2 = deferToThread(Storage.getDB, False)
  63         d2.addCallback(self._cb_assertFalse)
  64         d2.addErrback(self._eb_Failure)
  65         d2.addCallback(self._cb_assertTrue, Storage.getDB(False))
  66
  67     def test_insertBridgeAndGetRing_new_bridge(self):
  68         bridge = self.fakeBridges[0]
  69         Storage.initializeDBLock()
  70         with Storage.getDB() as db:
  71             ringname = db.insertBridgeAndGetRing(bridge, 'moat',
  72                                                  time.time(),
  73                                                  self.validRings)
  74             self.assertIn(ringname, self.validRings)
  75
  76     def test_insertBridgeAndGetRing_already_seen_bridge(self):
  77         bridge = self.fakeBridges[0]
  78         Storage.initializeDBLock()
  79         with Storage.getDB() as db:
  80             ringname = db.insertBridgeAndGetRing(bridge, 'moat',
  81                                                  time.time(),
  82                                                  self.validRings)
  83             self.assertIn(ringname, self.validRings)
  84             ringname = db.insertBridgeAndGetRing(bridge, 'https',
  85                                                  time.time(),
  86                                                  self.validRings)
  87             self.assertIn(ringname, self.validRings)
  88             self.assertEqual(ringname, 'https')
  89
  90     def test_getBridgeDistributor_recognised(self):
  91         bridge = self.fakeBridges[0]
  92         Storage.initializeDBLock()
  93         with Storage.getDB() as db:
  94             ringname = db.insertBridgeAndGetRing(bridge, 'moat',
  95                                                  time.time(),
  96                                                  self.validRings)
  97             self.assertIn(ringname, self.validRings)
  98             self.assertEqual(ringname, 'moat')
  99             db.commit()
 100
 101         with Storage.getDB() as db:
 102             ringname = db.getBridgeDistributor(bridge, self.validRings)
 103             self.assertEqual(ringname, 'moat')
 104
 105     def test_getBridgeDistributor_unrecognised(self):
 106         bridge = self.fakeBridges[0]
 107         Storage.initializeDBLock()
 108         with Storage.getDB() as db:
 109             ringname = db.insertBridgeAndGetRing(bridge, 'godzilla',
 110                                                  time.time(),
 111                                                  self.validRings)
 112             self.assertIn(ringname, self.validRings)
 113             self.assertEqual(ringname, "unallocated")
 114             db.commit()
 115
 116         with Storage.getDB() as db:
 117             ringname = db.getBridgeDistributor(bridge, self.validRings)
 118             self.assertEqual(ringname, "unallocated")
 119
 120     def test_BridgeMeasurementComparison(self):
 121         m1 = Storage.BridgeMeasurement(0, "", "", "", "", "", "", "",
 122                                        "2020-06-17", 0)
 123         m2 = Storage.BridgeMeasurement(0, "", "", "", "", "", "", "",
 124                                        "2020-06-18", 0)
 125         self.assertTrue(m2.newerThan(m1))
 126         self.assertFalse(m1.newerThan(m2))
 127         self.assertFalse(m1.newerThan(m1))
 128
 129     def test_BridgeMeasurementCompact(self):
 130         m = Storage.BridgeMeasurement(0, "FINGERPRINT", "obfs4", "1.2.3.4",
 131                                       "1234", "ru", "1234", "ooni",
 132                                       "2020-06-17", 0)
 133         self.assertEquals(m.compact(), ("ru", "1.2.3.4", "1234"))
 134
 135     def test_fetchBridgeMeasurements(self):
 136
 137         query = "INSERT INTO BridgeMeasurements (hex_key, bridge_type, " \
 138                 "address, port, blocking_country, blocking_asn, " \
 139                 "measured_by, last_measured, verdict) VALUES ('key', " \
 140                 "'obfs4', '1.2.3.4', '1234', 'RU', '1234', 'OONI', '%s', 1)"
 141         oldMsmt = query % "2017-01-01"
 142         newMsmt = query % datetime.datetime.utcnow().strftime("%Y-%m-%d")
 143
 144         Storage.initializeDBLock()
 145         with Storage.getDB() as db:
 146             db._cur.execute(oldMsmt)
 147             # We're calling _Database__fetchBridgeMeasurements instead of
 148             # __fetchBridgeMeasurements to account for Python's name meddling.
 149             rows = db._Database__fetchBridgeMeasurements()
 150             # Outdated measurements should not be returned.
 151             self.assertEquals(len(rows), 0)
 152
 153             db._cur.execute(newMsmt)
 154             rows = db._Database__fetchBridgeMeasurements()
 155             # Measurements that are "young enough" should be returned.
 156             self.assertEquals(len(rows), 1)
 157
 158     def test_main_loadBlockedBridges(self):
 159         Storage.initializeDBLock()
 160
 161         # Mock configuration object that we use to initialize our bridge rings.
 162         class Cfg(object):
 163             def __init__(self):
 164                 self.FORCE_PORTS = [(443, 1)]
 165                 self.FORCE_FLAGS = [("Stable", 1)]
 166                 self.MOAT_DIST = False
 167                 self.HTTPS_DIST = True
 168                 self.HTTPS_SHARE = 10
 169                 self.N_IP_CLUSTERS = 1
 170                 self.EMAIL_DIST = False
 171                 self.RESERVED_SHARE = 0
 172
 173         bridge = self.fakeBridges[0]
 174         addr, port, _ = bridge.orAddresses[0]
 175         cc= "de"
 176
 177         # Mock object that we use to simulate a database connection.
 178         class DummyDB(object):
 179             def __init__(self):
 180                 pass
 181             def __enter__(self):
 182                 return self
 183             def __exit__(self, type, value, traceback):
 184                 pass
 185             def getBlockedBridges(self):
 186                 return {bridge.fingerprint: [(cc, addr, port)]}
 187             def getBridgeDistributor(self, bridge, validRings):
 188                 return "https"
 189             def insertBridgeAndGetRing(self, bridge, setRing, seenAt, validRings, defaultPool="unallocated"):
 190                 return "https"
 191             def commit(self):
 192                 pass
 193
 194         oldObj = Storage.getDB
 195         Storage.getDB = DummyDB
 196
 197         hashring, _, _, _ = main.createBridgeRings(Cfg(), None, b'key')
 198         hashring.insert(bridge)
 199
 200         self.assertEqual(len(hashring), 1)
 201         self.assertFalse(bridge.isBlockedIn(cc))
 202         self.assertFalse(bridge.isBlockedIn("ab"))
 203         self.assertFalse(bridge.addressIsBlockedIn(cc, addr, port))
 204
 205         main.loadBlockedBridges(hashring)
 206
 207         self.assertTrue(bridge.isBlockedIn(cc))
 208         self.assertFalse(bridge.isBlockedIn("ab"))
 209         self.assertTrue(bridge.addressIsBlockedIn(cc, addr, port))
 210
 211         Storage.getDB = oldObj
 212
 213     def test_getBlockedBridgesFromSql(self):
 214
 215         elems = [(0, "0000000000000000000000000000000000000000", "obfs4",
 216                   "1.2.3.4", "1234", "ru", "4321", "ooni", "2020-06-17",
 217                   Storage.BRIDGE_BLOCKED),
 218                  (1, "1111111111111111111111111111111111111111", "obfs4",
 219                   "1.2.3.4", "1234", "ru", "4321", "ooni", "2020-06-01",
 220                   Storage.BRIDGE_BLOCKED),
 221                  (2, "1111111111111111111111111111111111111111", "obfs4",
 222                   "1.2.3.4", "4321", "ru", "4321", "ooni", "2020-06-01",
 223                   Storage.BRIDGE_BLOCKED),
 224                  (3, "1111111111111111111111111111111111111111", "obfs4",
 225                   "1.2.3.4", "4321", "ru", "4321", "ooni", "2020-05-01",
 226                   Storage.BRIDGE_REACHABLE)]
 227         b = Storage.getBlockedBridgesFromSql(elems)
 228         self.assertEqual(b, {"0000000000000000000000000000000000000000":
 229                              [("ru", "1.2.3.4", "1234")],
 230                              "1111111111111111111111111111111111111111":
 231                              [("ru", "1.2.3.4", "1234"),
 232                               ("ru", "1.2.3.4", "4321")]})
 233
 234         # If multiple measurements disagree, we believe the newest one.
 235         elems = [(0, "0000000000000000000000000000000000000000", "obfs4",
 236                   "1.2.3.4", "1234", "ru", "4321", "ooni", "2020-06-17",
 237                   Storage.BRIDGE_BLOCKED),
 238                  (1, "0000000000000000000000000000000000000000", "obfs4",
 239                   "1.2.3.4", "1234", "ru", "4321", "ooni", "2020-06-01",
 240                   Storage.BRIDGE_REACHABLE)]
 241         b = Storage.getBlockedBridgesFromSql(elems)
 242         self.assertEqual(b, {"0000000000000000000000000000000000000000":
 243                              [("ru", "1.2.3.4", "1234")]})
 244
 245         elems = [(0, "0000000000000000000000000000000000000000", "obfs4",
 246                   "1.2.3.4", "1234", "ru", "4321", "ooni", "2020-06-01",
 247                   Storage.BRIDGE_BLOCKED),
 248                  (1, "0000000000000000000000000000000000000000", "obfs4",
 249                   "1.2.3.4", "1234", "ru", "4321", "ooni", "2020-06-17",
 250                   Storage.BRIDGE_REACHABLE)]
 251         b = Storage.getBlockedBridgesFromSql(elems)
 252         self.assertTrue(len(b) == 0)
 253
 254         # Element ordering must not affect the outcome.
 255         elems = [(1, "0000000000000000000000000000000000000000", "obfs4",
 256                   "1.2.3.4", "1234", "ru", "4321", "ooni", "2020-06-17",
 257                   Storage.BRIDGE_REACHABLE),
 258                  (0, "0000000000000000000000000000000000000000", "obfs4",
 259                   "1.2.3.4", "1234", "ru", "4321", "ooni", "2020-06-01",
 260                   Storage.BRIDGE_BLOCKED)]
 261         b = Storage.getBlockedBridgesFromSql(elems)
 262         self.assertTrue(len(b) == 0)
 263
 264         # Redundant measurements should be discarded.
 265         elems = [(1, "0000000000000000000000000000000000000000", "obfs4",
 266                   "1.2.3.4", "1234", "ru", "4321", "ooni", "2020-06-17",
 267                   Storage.BRIDGE_BLOCKED),
 268                  (0, "0000000000000000000000000000000000000000", "obfs4",
 269                   "1.2.3.4", "1234", "ru", "4321", "ooni", "2020-06-01",
 270                   Storage.BRIDGE_BLOCKED)]
 271         b = Storage.getBlockedBridgesFromSql(elems)
 272         self.assertTrue(len(b) == 1)