src/base/net/geoipdatabase.cpp

   1 /*
   2  * Bittorrent Client using Qt and libtorrent.
   3  * Copyright (C) 2015  Vladimir Golovnev <glassez@yandex.ru>
   4  *
   5  * This program is free software; you can redistribute it and/or
   6  * modify it under the terms of the GNU General Public License
   7  * as published by the Free Software Foundation; either version 2
   8  * of the License, or (at your option) any later version.
   9  *
  10  * This program is distributed in the hope that it will be useful,
  11  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  12  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  13  * GNU General Public License for more details.
  14  *
  15  * You should have received a copy of the GNU General Public License
  16  * along with this program; if not, write to the Free Software
  17  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
  18  *
  19  * In addition, as a special exception, the copyright holders give permission to
  20  * link this program with the OpenSSL project's "OpenSSL" library (or with
  21  * modified versions of it that use the same license as the "OpenSSL" library),
  22  * and distribute the linked executables. You must obey the GNU General Public
  23  * License in all respects for all of the code used other than "OpenSSL".  If you
  24  * modify file(s), you may extend this exception to your version of the file(s),
  25  * but you are not obligated to do so. If you do not wish to do so, delete this
  26  * exception statement from your version.
  27  */
  28
  29 #include "geoipdatabase.h"
  30
  31 #include <QDateTime>
  32 #include <QDebug>
  33 #include <QFile>
  34 #include <QHostAddress>
  35 #include <QVariant>
  36
  37 #include "base/global.h"
  38 #include "base/path.h"
  39
  40 namespace
  41 {
  42     const qint32 MAX_FILE_SIZE = 67108864; // 64MB
  43     const quint32 MAX_METADATA_SIZE = 131072; // 128KB
  44     const char METADATA_BEGIN_MARK[] = "\xab\xcd\xefMaxMind.com";
  45     const char DATA_SECTION_SEPARATOR[16] = {0};
  46
  47     enum class DataType
  48     {
  49         Unknown = 0,
  50         Pointer = 1,
  51         String = 2,
  52         Double = 3,
  53         Bytes = 4,
  54         Integer16 = 5,
  55         Integer32 = 6,
  56         Map = 7,
  57         SignedInteger32 = 8,
  58         Integer64 = 9,
  59         Integer128 = 10,
  60         Array = 11,
  61         DataCacheContainer = 12,
  62         EndMarker = 13,
  63         Boolean = 14,
  64         Float = 15
  65     };
  66 }
  67
  68 struct DataFieldDescriptor
  69 {
  70     DataType fieldType {DataType::Unknown};
  71     union
  72     {
  73         quint32 fieldSize = 0;
  74         quint32 offset; // Pointer
  75     };
  76 };
  77
  78 GeoIPDatabase::GeoIPDatabase(const quint32 size)
  79     : m_size(size)
  80     , m_data(new uchar[size])
  81 {
  82 }
  83
  84 GeoIPDatabase *GeoIPDatabase::load(const Path &filename, QString &error)
  85 {
  86     QFile file {filename.data()};
  87     if (file.size() > MAX_FILE_SIZE)
  88     {
  89         error = tr("Unsupported database file size.");
  90         return nullptr;
  91     }
  92
  93     if (!file.open(QFile::ReadOnly))
  94     {
  95         error = file.errorString();
  96         return nullptr;
  97     }
  98
  99     auto *db = new GeoIPDatabase(file.size());
 100
 101     if (file.read(reinterpret_cast<char *>(db->m_data), db->m_size) != db->m_size)
 102     {
 103         error = file.errorString();
 104         delete db;
 105         return nullptr;
 106     }
 107
 108
 109     if (!db->parseMetadata(db->readMetadata(), error) || !db->loadDB(error))
 110     {
 111         delete db;
 112         return nullptr;
 113     }
 114
 115     return db;
 116 }
 117
 118 GeoIPDatabase *GeoIPDatabase::load(const QByteArray &data, QString &error)
 119 {
 120     if (data.size() > MAX_FILE_SIZE)
 121     {
 122         error = tr("Unsupported database file size.");
 123         return nullptr;
 124     }
 125
 126     auto *db = new GeoIPDatabase(data.size());
 127
 128     memcpy(reinterpret_cast<char *>(db->m_data), data.constData(), db->m_size);
 129
 130     if (!db->parseMetadata(db->readMetadata(), error) || !db->loadDB(error))
 131     {
 132         delete db;
 133         return nullptr;
 134     }
 135
 136     return db;
 137 }
 138
 139 GeoIPDatabase::~GeoIPDatabase()
 140 {
 141     delete [] m_data;
 142 }
 143
 144 QString GeoIPDatabase::type() const
 145 {
 146     return m_dbType;
 147 }
 148
 149 quint16 GeoIPDatabase::ipVersion() const
 150 {
 151     return m_ipVersion;
 152 }
 153
 154 QDateTime GeoIPDatabase::buildEpoch() const
 155 {
 156     return m_buildEpoch;
 157 }
 158
 159 QString GeoIPDatabase::lookup(const QHostAddress &hostAddr) const
 160 {
 161     Q_IPV6ADDR addr = hostAddr.toIPv6Address();
 162
 163     const uchar *ptr = m_data;
 164
 165     for (int i = 0; i < 16; ++i)
 166     {
 167         for (int j = 0; j < 8; ++j)
 168         {
 169             const bool right = static_cast<bool>((addr[i] >> (7 - j)) & 1);
 170             // Interpret the left/right record as number
 171             if (right)
 172                 ptr += m_recordBytes;
 173
 174             quint32 id = 0;
 175             auto *idPtr = reinterpret_cast<uchar *>(&id);
 176             memcpy(&idPtr[4 - m_recordBytes], ptr, m_recordBytes);
 177             fromBigEndian(idPtr, 4);
 178
 179             if (id == m_nodeCount)
 180             {
 181                 return {};
 182             }
 183             if (id > m_nodeCount)
 184             {
 185                 QString country = m_countries.value(id);
 186                 if (country.isEmpty())
 187                 {
 188                     const quint32 offset = id - m_nodeCount - sizeof(DATA_SECTION_SEPARATOR);
 189                     quint32 tmp = offset + m_indexSize + sizeof(DATA_SECTION_SEPARATOR);
 190                     const QVariant val = readDataField(tmp);
 191                     if (val.userType() == QMetaType::QVariantHash)
 192                     {
 193                         country = val.toHash()[u"country"_s].toHash()[u"iso_code"_s].toString();
 194                         m_countries[id] = country;
 195                     }
 196                 }
 197                 return country;
 198             }
 199
 200             ptr = m_data + (id * m_nodeSize);
 201         }
 202     }
 203
 204     return {};
 205 }
 206
 207 #define CHECK_METADATA_REQ(key, type) \
 208 if (!metadata.contains(key)) \
 209 { \
 210     error = errMsgNotFound.arg(key); \
 211     return false; \
 212 } \
 213 if (metadata.value(key).userType() != QMetaType::type) \
 214 { \
 215     error = errMsgInvalid.arg(key);  \
 216     return false; \
 217 }
 218
 219 #define CHECK_METADATA_OPT(key, type) \
 220 if (metadata.contains(key)) \
 221 { \
 222     if (metadata.value(key).userType() != QMetaType::type) \
 223     { \
 224         error = errMsgInvalid.arg(key);  \
 225         return false; \
 226     } \
 227 }
 228
 229 bool GeoIPDatabase::parseMetadata(const QVariantHash &metadata, QString &error)
 230 {
 231     const QString errMsgNotFound = tr("Metadata error: '%1' entry not found.");
 232     const QString errMsgInvalid = tr("Metadata error: '%1' entry has invalid type.");
 233
 234     qDebug() << "Parsing MaxMindDB metadata...";
 235
 236     CHECK_METADATA_REQ(u"binary_format_major_version"_s, UShort);
 237     CHECK_METADATA_REQ(u"binary_format_minor_version"_s, UShort);
 238     const uint versionMajor = metadata.value(u"binary_format_major_version"_s).toUInt();
 239     const uint versionMinor = metadata.value(u"binary_format_minor_version"_s).toUInt();
 240     if (versionMajor != 2)
 241     {
 242         error = tr("Unsupported database version: %1.%2").arg(versionMajor).arg(versionMinor);
 243         return false;
 244     }
 245
 246     CHECK_METADATA_REQ(u"ip_version"_s, UShort);
 247     m_ipVersion = metadata.value(u"ip_version"_s).value<quint16>();
 248     if (m_ipVersion != 6)
 249     {
 250         error = tr("Unsupported IP version: %1").arg(m_ipVersion);
 251         return false;
 252     }
 253
 254     CHECK_METADATA_REQ(u"record_size"_s, UShort);
 255     m_recordSize = metadata.value(u"record_size"_s).value<quint16>();
 256     if (m_recordSize != 24)
 257     {
 258         error = tr("Unsupported record size: %1").arg(m_recordSize);
 259         return false;
 260     }
 261     m_nodeSize = m_recordSize / 4;
 262     m_recordBytes = m_nodeSize / 2;
 263
 264     CHECK_METADATA_REQ(u"node_count"_s, UInt);
 265     m_nodeCount = metadata.value(u"node_count"_s).value<quint32>();
 266     m_indexSize = m_nodeCount * m_nodeSize;
 267
 268     CHECK_METADATA_REQ(u"database_type"_s, QString);
 269     m_dbType = metadata.value(u"database_type"_s).toString();
 270
 271     CHECK_METADATA_REQ(u"build_epoch"_s, ULongLong);
 272     m_buildEpoch = QDateTime::fromSecsSinceEpoch(metadata.value(u"build_epoch"_s).toULongLong());
 273
 274     CHECK_METADATA_OPT(u"languages"_s, QVariantList);
 275     CHECK_METADATA_OPT(u"description"_s, QVariantHash);
 276
 277     return true;
 278 }
 279
 280 bool GeoIPDatabase::loadDB(QString &error) const
 281 {
 282     qDebug() << "Parsing IP geolocation database index tree...";
 283
 284     const int nodeSize = m_recordSize / 4; // in bytes
 285     const int indexSize = m_nodeCount * nodeSize;
 286     if ((m_size < (indexSize + sizeof(DATA_SECTION_SEPARATOR)))
 287         || (memcmp(m_data + indexSize, DATA_SECTION_SEPARATOR, sizeof(DATA_SECTION_SEPARATOR)) != 0))
 288         {
 289         error = tr("Database corrupted: no data section found.");
 290         return false;
 291     }
 292
 293     return true;
 294 }
 295
 296 QVariantHash GeoIPDatabase::readMetadata() const
 297 {
 298     const char *ptr = reinterpret_cast<const char *>(m_data);
 299     quint32 size = m_size;
 300     if (m_size > MAX_METADATA_SIZE)
 301     {
 302         ptr += m_size - MAX_METADATA_SIZE;
 303         size = MAX_METADATA_SIZE;
 304     }
 305
 306     const QByteArray data = QByteArray::fromRawData(ptr, size);
 307     int index = data.lastIndexOf(METADATA_BEGIN_MARK);
 308     if (index >= 0)
 309     {
 310         if (m_size > MAX_METADATA_SIZE)
 311             index += (m_size - MAX_METADATA_SIZE); // from begin of all data
 312         auto offset = static_cast<quint32>(index + strlen(METADATA_BEGIN_MARK));
 313         const QVariant metadata = readDataField(offset);
 314         if (metadata.userType() == QMetaType::QVariantHash)
 315             return metadata.toHash();
 316     }
 317
 318     return {};
 319 }
 320
 321 QVariant GeoIPDatabase::readDataField(quint32 &offset) const
 322 {
 323     DataFieldDescriptor descr;
 324     if (!readDataFieldDescriptor(offset, descr))
 325         return {};
 326
 327     quint32 locOffset = offset;
 328     bool usePointer = false;
 329     if (descr.fieldType == DataType::Pointer)
 330     {
 331         usePointer = true;
 332         // convert offset from data section to global
 333         locOffset = descr.offset + (m_nodeCount * m_recordSize / 4) + sizeof(DATA_SECTION_SEPARATOR);
 334         if (!readDataFieldDescriptor(locOffset, descr))
 335             return {};
 336     }
 337
 338     QVariant fieldValue;
 339     switch (descr.fieldType)
 340     {
 341     case DataType::Pointer:
 342         qDebug() << "* Illegal Pointer using";
 343         break;
 344     case DataType::String:
 345         fieldValue = QString::fromUtf8(reinterpret_cast<const char *>(m_data + locOffset), descr.fieldSize);
 346         locOffset += descr.fieldSize;
 347         break;
 348     case DataType::Double:
 349         if (descr.fieldSize == 8)
 350             fieldValue = readPlainValue<double>(locOffset, descr.fieldSize);
 351         else
 352             qDebug() << "* Invalid field size for type: Double";
 353         break;
 354     case DataType::Bytes:
 355         fieldValue = QByteArray(reinterpret_cast<const char *>(m_data + locOffset), descr.fieldSize);
 356         locOffset += descr.fieldSize;
 357         break;
 358     case DataType::Integer16:
 359         fieldValue = readPlainValue<quint16>(locOffset, descr.fieldSize);
 360         break;
 361     case DataType::Integer32:
 362         fieldValue = readPlainValue<quint32>(locOffset, descr.fieldSize);
 363         break;
 364     case DataType::Map:
 365         fieldValue = readMapValue(locOffset, descr.fieldSize);
 366         break;
 367     case DataType::SignedInteger32:
 368         fieldValue = readPlainValue<qint32>(locOffset, descr.fieldSize);
 369         break;
 370     case DataType::Integer64:
 371         fieldValue = readPlainValue<quint64>(locOffset, descr.fieldSize);
 372         break;
 373     case DataType::Integer128:
 374         qDebug() << "* Unsupported data type: Integer128";
 375         break;
 376     case DataType::Array:
 377         fieldValue = readArrayValue(locOffset, descr.fieldSize);
 378         break;
 379     case DataType::DataCacheContainer:
 380         qDebug() << "* Unsupported data type: DataCacheContainer";
 381         break;
 382     case DataType::EndMarker:
 383         qDebug() << "* Unsupported data type: EndMarker";
 384         break;
 385     case DataType::Boolean:
 386         fieldValue = QVariant::fromValue(static_cast<bool>(descr.fieldSize));
 387         break;
 388     case DataType::Float:
 389         if (descr.fieldSize == 4)
 390             fieldValue = readPlainValue<float>(locOffset, descr.fieldSize);
 391         else
 392             qDebug() << "* Invalid field size for type: Float";
 393         break;
 394     default:
 395         qDebug() << "* Unsupported data type: Unknown";
 396     }
 397
 398     if (!usePointer)
 399         offset = locOffset;
 400     return fieldValue;
 401 }
 402
 403 bool GeoIPDatabase::readDataFieldDescriptor(quint32 &offset, DataFieldDescriptor &out) const
 404 {
 405     const uchar *dataPtr = m_data + offset;
 406     const int availSize = m_size - offset;
 407     if (availSize < 1) return false;
 408
 409     out.fieldType = static_cast<DataType>((dataPtr[0] & 0xE0) >> 5);
 410     if (out.fieldType == DataType::Pointer)
 411     {
 412         const int size = ((dataPtr[0] & 0x18) >> 3);
 413         if (availSize < (size + 2)) return false;
 414
 415         if (size == 0)
 416             out.offset = ((dataPtr[0] & 0x07) << 8) + dataPtr[1];
 417         else if (size == 1)
 418             out.offset = ((dataPtr[0] & 0x07) << 16) + (dataPtr[1] << 8) + dataPtr[2] + 2048;
 419         else if (size == 2)
 420             out.offset = ((dataPtr[0] & 0x07) << 24) + (dataPtr[1] << 16) + (dataPtr[2] << 8) + dataPtr[3] + 526336;
 421         else if (size == 3)
 422             out.offset = (dataPtr[1] << 24) + (dataPtr[2] << 16) + (dataPtr[3] << 8) + dataPtr[4];
 423
 424         offset += size + 2;
 425         return true;
 426     }
 427
 428     out.fieldSize = dataPtr[0] & 0x1F;
 429     if (out.fieldSize <= 28)
 430     {
 431         if (out.fieldType == DataType::Unknown)
 432         {
 433             out.fieldType = static_cast<DataType>(dataPtr[1] + 7);
 434             if ((out.fieldType <= DataType::Map) || (out.fieldType > DataType::Float) || (availSize < 3))
 435                 return false;
 436             offset += 2;
 437         }
 438         else
 439         {
 440             offset += 1;
 441         }
 442     }
 443     else if (out.fieldSize == 29)
 444     {
 445         if (availSize < 2) return false;
 446         out.fieldSize = dataPtr[1] + 29;
 447         offset += 2;
 448     }
 449     else if (out.fieldSize == 30)
 450     {
 451         if (availSize < 3) return false;
 452         out.fieldSize = (dataPtr[1] << 8) + dataPtr[2] + 285;
 453         offset += 3;
 454     }
 455     else if (out.fieldSize == 31)
 456     {
 457         if (availSize < 4) return false;
 458         out.fieldSize = (dataPtr[1] << 16) + (dataPtr[2] << 8) + dataPtr[3] + 65821;
 459         offset += 4;
 460     }
 461
 462     return true;
 463 }
 464
 465 void GeoIPDatabase::fromBigEndian([[maybe_unused]] uchar *buf, [[maybe_unused]] const quint32 len) const
 466 {
 467 #if (Q_BYTE_ORDER == Q_LITTLE_ENDIAN)
 468     std::reverse(buf, buf + len);
 469 #endif
 470 }
 471
 472 QVariant GeoIPDatabase::readMapValue(quint32 &offset, const quint32 count) const
 473 {
 474     QVariantHash map;
 475
 476     for (quint32 i = 0; i < count; ++i)
 477     {
 478         QVariant field = readDataField(offset);
 479         if (field.userType() != QMetaType::QString)
 480             return {};
 481
 482         const QString key = field.toString();
 483         field = readDataField(offset);
 484         if (field.userType() == QMetaType::UnknownType)
 485             return {};
 486
 487         map[key] = field;
 488     }
 489
 490     return map;
 491 }
 492
 493 QVariant GeoIPDatabase::readArrayValue(quint32 &offset, const quint32 count) const
 494 {
 495     QVariantList array;
 496
 497     for (quint32 i = 0; i < count; ++i)
 498     {
 499         const QVariant field = readDataField(offset);
 500         if (field.userType() == QMetaType::UnknownType)
 501             return {};
 502
 503         array.append(field);
 504     }
 505
 506     return array;
 507 }
 508
 509 template <typename T>
 510 QVariant GeoIPDatabase::readPlainValue(quint32 &offset, const quint8 len) const
 511 {
 512     T value = 0;
 513     const uchar *const data = m_data + offset;
 514     const quint32 availSize = m_size - offset;
 515
 516     if ((len > 0) && (len <= sizeof(T) && (availSize >= len)))
 517     {
 518         // copy input data to last 'len' bytes of 'value'
 519         uchar *dst = reinterpret_cast<uchar *>(&value) + (sizeof(T) - len);
 520         memcpy(dst, data, len);
 521         fromBigEndian(reinterpret_cast<uchar *>(&value), sizeof(T));
 522         offset += len;
 523     }
 524
 525     return QVariant::fromValue(value);
 526 }