src/base/net/geoipdatabase.cpp

   1 /*
   2  * Bittorrent Client using Qt and libtorrent.
   3  * Copyright (C) 2015  Vladimir Golovnev <glassez@yandex.ru>
   4  *
   5  * This program is free software; you can redistribute it and/or
   6  * modify it under the terms of the GNU General Public License
   7  * as published by the Free Software Foundation; either version 2
   8  * of the License, or (at your option) any later version.
   9  *
  10  * This program is distributed in the hope that it will be useful,
  11  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  12  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  13  * GNU General Public License for more details.
  14  *
  15  * You should have received a copy of the GNU General Public License
  16  * along with this program; if not, write to the Free Software
  17  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
  18  *
  19  * In addition, as a special exception, the copyright holders give permission to
  20  * link this program with the OpenSSL project's "OpenSSL" library (or with
  21  * modified versions of it that use the same license as the "OpenSSL" library),
  22  * and distribute the linked executables. You must obey the GNU General Public
  23  * License in all respects for all of the code used other than "OpenSSL".  If you
  24  * modify file(s), you may extend this exception to your version of the file(s),
  25  * but you are not obligated to do so. If you do not wish to do so, delete this
  26  * exception statement from your version.
  27  */
  28
  29 #include "geoipdatabase.h"
  30
  31 #include <QDateTime>
  32 #include <QDebug>
  33 #include <QFile>
  34 #include <QHostAddress>
  35 #include <QVariant>
  36
  37 #include "base/global.h"
  38 #include "base/path.h"
  39
  40 namespace
  41 {
  42     const qint32 MAX_FILE_SIZE = 67108864; // 64MB
  43     const quint32 MAX_METADATA_SIZE = 131072; // 128KB
  44     const char METADATA_BEGIN_MARK[] = "\xab\xcd\xefMaxMind.com";
  45     const char DATA_SECTION_SEPARATOR[16] = {0};
  46
  47     enum class DataType
  48     {
  49         Unknown = 0,
  50         Pointer = 1,
  51         String = 2,
  52         Double = 3,
  53         Bytes = 4,
  54         Integer16 = 5,
  55         Integer32 = 6,
  56         Map = 7,
  57         SignedInteger32 = 8,
  58         Integer64 = 9,
  59         Integer128 = 10,
  60         Array = 11,
  61         DataCacheContainer = 12,
  62         EndMarker = 13,
  63         Boolean = 14,
  64         Float = 15
  65     };
  66 }
  67
  68 struct DataFieldDescriptor
  69 {
  70     DataType fieldType {DataType::Unknown};
  71     union
  72     {
  73         quint32 fieldSize = 0;
  74         quint32 offset; // Pointer
  75     };
  76 };
  77
  78 GeoIPDatabase::GeoIPDatabase(const quint32 size)
  79     : m_size(size)
  80     , m_data(new uchar[size])
  81 {
  82 }
  83
  84 GeoIPDatabase *GeoIPDatabase::load(const Path &filename, QString &error)
  85 {
  86     GeoIPDatabase *db = nullptr;
  87     QFile file {filename.data()};
  88     if (file.size() > MAX_FILE_SIZE)
  89     {
  90         error = tr("Unsupported database file size.");
  91         return nullptr;
  92     }
  93
  94     if (!file.open(QFile::ReadOnly))
  95     {
  96         error = file.errorString();
  97         return nullptr;
  98     }
  99
 100     db = new GeoIPDatabase(file.size());
 101
 102     if (file.read(reinterpret_cast<char *>(db->m_data), db->m_size) != db->m_size)
 103     {
 104         error = file.errorString();
 105         delete db;
 106         return nullptr;
 107     }
 108
 109
 110     if (!db->parseMetadata(db->readMetadata(), error) || !db->loadDB(error))
 111     {
 112         delete db;
 113         return nullptr;
 114     }
 115
 116     return db;
 117 }
 118
 119 GeoIPDatabase *GeoIPDatabase::load(const QByteArray &data, QString &error)
 120 {
 121     GeoIPDatabase *db = nullptr;
 122     if (data.size() > MAX_FILE_SIZE)
 123     {
 124         error = tr("Unsupported database file size.");
 125         return nullptr;
 126     }
 127
 128     db = new GeoIPDatabase(data.size());
 129
 130     memcpy(reinterpret_cast<char *>(db->m_data), data.constData(), db->m_size);
 131
 132     if (!db->parseMetadata(db->readMetadata(), error) || !db->loadDB(error))
 133     {
 134         delete db;
 135         return nullptr;
 136     }
 137
 138     return db;
 139 }
 140
 141 GeoIPDatabase::~GeoIPDatabase()
 142 {
 143     delete [] m_data;
 144 }
 145
 146 QString GeoIPDatabase::type() const
 147 {
 148     return m_dbType;
 149 }
 150
 151 quint16 GeoIPDatabase::ipVersion() const
 152 {
 153     return m_ipVersion;
 154 }
 155
 156 QDateTime GeoIPDatabase::buildEpoch() const
 157 {
 158     return m_buildEpoch;
 159 }
 160
 161 QString GeoIPDatabase::lookup(const QHostAddress &hostAddr) const
 162 {
 163     Q_IPV6ADDR addr = hostAddr.toIPv6Address();
 164
 165     const uchar *ptr = m_data;
 166
 167     for (int i = 0; i < 16; ++i)
 168     {
 169         for (int j = 0; j < 8; ++j)
 170         {
 171             const bool right = static_cast<bool>((addr[i] >> (7 - j)) & 1);
 172             // Interpret the left/right record as number
 173             if (right)
 174                 ptr += m_recordBytes;
 175
 176             quint32 id = 0;
 177             auto *idPtr = reinterpret_cast<uchar *>(&id);
 178             memcpy(&idPtr[4 - m_recordBytes], ptr, m_recordBytes);
 179             fromBigEndian(idPtr, 4);
 180
 181             if (id == m_nodeCount)
 182             {
 183                 return {};
 184             }
 185             if (id > m_nodeCount)
 186             {
 187                 QString country = m_countries.value(id);
 188                 if (country.isEmpty())
 189                 {
 190                     const quint32 offset = id - m_nodeCount - sizeof(DATA_SECTION_SEPARATOR);
 191                     quint32 tmp = offset + m_indexSize + sizeof(DATA_SECTION_SEPARATOR);
 192                     const QVariant val = readDataField(tmp);
 193                     if (val.userType() == QMetaType::QVariantHash)
 194                     {
 195                         country = val.toHash()[u"country"_qs].toHash()[u"iso_code"_qs].toString();
 196                         m_countries[id] = country;
 197                     }
 198                 }
 199                 return country;
 200             }
 201
 202             ptr = m_data + (id * m_nodeSize);
 203         }
 204     }
 205
 206     return {};
 207 }
 208
 209 #define CHECK_METADATA_REQ(key, type) \
 210 if (!metadata.contains(key)) \
 211 { \
 212     error = errMsgNotFound.arg(key); \
 213     return false; \
 214 } \
 215 if (metadata.value(key).userType() != QMetaType::type) \
 216 { \
 217     error = errMsgInvalid.arg(key);  \
 218     return false; \
 219 }
 220
 221 #define CHECK_METADATA_OPT(key, type) \
 222 if (metadata.contains(key)) \
 223 { \
 224     if (metadata.value(key).userType() != QMetaType::type) \
 225     { \
 226         error = errMsgInvalid.arg(key);  \
 227         return false; \
 228     } \
 229 }
 230
 231 bool GeoIPDatabase::parseMetadata(const QVariantHash &metadata, QString &error)
 232 {
 233     const QString errMsgNotFound = tr("Metadata error: '%1' entry not found.");
 234     const QString errMsgInvalid = tr("Metadata error: '%1' entry has invalid type.");
 235
 236     qDebug() << "Parsing MaxMindDB metadata...";
 237
 238     CHECK_METADATA_REQ(u"binary_format_major_version"_qs, UShort);
 239     CHECK_METADATA_REQ(u"binary_format_minor_version"_qs, UShort);
 240     const uint versionMajor = metadata.value(u"binary_format_major_version"_qs).toUInt();
 241     const uint versionMinor = metadata.value(u"binary_format_minor_version"_qs).toUInt();
 242     if (versionMajor != 2)
 243     {
 244         error = tr("Unsupported database version: %1.%2").arg(versionMajor).arg(versionMinor);
 245         return false;
 246     }
 247
 248     CHECK_METADATA_REQ(u"ip_version"_qs, UShort);
 249     m_ipVersion = metadata.value(u"ip_version"_qs).value<quint16>();
 250     if (m_ipVersion != 6)
 251     {
 252         error = tr("Unsupported IP version: %1").arg(m_ipVersion);
 253         return false;
 254     }
 255
 256     CHECK_METADATA_REQ(u"record_size"_qs, UShort);
 257     m_recordSize = metadata.value(u"record_size"_qs).value<quint16>();
 258     if (m_recordSize != 24)
 259     {
 260         error = tr("Unsupported record size: %1").arg(m_recordSize);
 261         return false;
 262     }
 263     m_nodeSize = m_recordSize / 4;
 264     m_recordBytes = m_nodeSize / 2;
 265
 266     CHECK_METADATA_REQ(u"node_count"_qs, UInt);
 267     m_nodeCount = metadata.value(u"node_count"_qs).value<quint32>();
 268     m_indexSize = m_nodeCount * m_nodeSize;
 269
 270     CHECK_METADATA_REQ(u"database_type"_qs, QString);
 271     m_dbType = metadata.value(u"database_type"_qs).toString();
 272
 273     CHECK_METADATA_REQ(u"build_epoch"_qs, ULongLong);
 274     m_buildEpoch = QDateTime::fromSecsSinceEpoch(metadata.value(u"build_epoch"_qs).toULongLong());
 275
 276     CHECK_METADATA_OPT(u"languages"_qs, QVariantList);
 277     CHECK_METADATA_OPT(u"description"_qs, QVariantHash);
 278
 279     return true;
 280 }
 281
 282 bool GeoIPDatabase::loadDB(QString &error) const
 283 {
 284     qDebug() << "Parsing IP geolocation database index tree...";
 285
 286     const int nodeSize = m_recordSize / 4; // in bytes
 287     const int indexSize = m_nodeCount * nodeSize;
 288     if ((m_size < (indexSize + sizeof(DATA_SECTION_SEPARATOR)))
 289         || (memcmp(m_data + indexSize, DATA_SECTION_SEPARATOR, sizeof(DATA_SECTION_SEPARATOR)) != 0))
 290         {
 291         error = tr("Database corrupted: no data section found.");
 292         return false;
 293     }
 294
 295     return true;
 296 }
 297
 298 QVariantHash GeoIPDatabase::readMetadata() const
 299 {
 300     const char *ptr = reinterpret_cast<const char *>(m_data);
 301     quint32 size = m_size;
 302     if (m_size > MAX_METADATA_SIZE)
 303     {
 304         ptr += m_size - MAX_METADATA_SIZE;
 305         size = MAX_METADATA_SIZE;
 306     }
 307
 308     const QByteArray data = QByteArray::fromRawData(ptr, size);
 309     int index = data.lastIndexOf(METADATA_BEGIN_MARK);
 310     if (index >= 0)
 311     {
 312         if (m_size > MAX_METADATA_SIZE)
 313             index += (m_size - MAX_METADATA_SIZE); // from begin of all data
 314         auto offset = static_cast<quint32>(index + strlen(METADATA_BEGIN_MARK));
 315         const QVariant metadata = readDataField(offset);
 316         if (metadata.userType() == QMetaType::QVariantHash)
 317             return metadata.toHash();
 318     }
 319
 320     return {};
 321 }
 322
 323 QVariant GeoIPDatabase::readDataField(quint32 &offset) const
 324 {
 325     DataFieldDescriptor descr;
 326     if (!readDataFieldDescriptor(offset, descr))
 327         return {};
 328
 329     quint32 locOffset = offset;
 330     bool usePointer = false;
 331     if (descr.fieldType == DataType::Pointer)
 332     {
 333         usePointer = true;
 334         // convert offset from data section to global
 335         locOffset = descr.offset + (m_nodeCount * m_recordSize / 4) + sizeof(DATA_SECTION_SEPARATOR);
 336         if (!readDataFieldDescriptor(locOffset, descr))
 337             return {};
 338     }
 339
 340     QVariant fieldValue;
 341     switch (descr.fieldType)
 342     {
 343     case DataType::Pointer:
 344         qDebug() << "* Illegal Pointer using";
 345         break;
 346     case DataType::String:
 347         fieldValue = QString::fromUtf8(reinterpret_cast<const char *>(m_data + locOffset), descr.fieldSize);
 348         locOffset += descr.fieldSize;
 349         break;
 350     case DataType::Double:
 351         if (descr.fieldSize == 8)
 352             fieldValue = readPlainValue<double>(locOffset, descr.fieldSize);
 353         else
 354             qDebug() << "* Invalid field size for type: Double";
 355         break;
 356     case DataType::Bytes:
 357         fieldValue = QByteArray(reinterpret_cast<const char *>(m_data + locOffset), descr.fieldSize);
 358         locOffset += descr.fieldSize;
 359         break;
 360     case DataType::Integer16:
 361         fieldValue = readPlainValue<quint16>(locOffset, descr.fieldSize);
 362         break;
 363     case DataType::Integer32:
 364         fieldValue = readPlainValue<quint32>(locOffset, descr.fieldSize);
 365         break;
 366     case DataType::Map:
 367         fieldValue = readMapValue(locOffset, descr.fieldSize);
 368         break;
 369     case DataType::SignedInteger32:
 370         fieldValue = readPlainValue<qint32>(locOffset, descr.fieldSize);
 371         break;
 372     case DataType::Integer64:
 373         fieldValue = readPlainValue<quint64>(locOffset, descr.fieldSize);
 374         break;
 375     case DataType::Integer128:
 376         qDebug() << "* Unsupported data type: Integer128";
 377         break;
 378     case DataType::Array:
 379         fieldValue = readArrayValue(locOffset, descr.fieldSize);
 380         break;
 381     case DataType::DataCacheContainer:
 382         qDebug() << "* Unsupported data type: DataCacheContainer";
 383         break;
 384     case DataType::EndMarker:
 385         qDebug() << "* Unsupported data type: EndMarker";
 386         break;
 387     case DataType::Boolean:
 388         fieldValue = QVariant::fromValue(static_cast<bool>(descr.fieldSize));
 389         break;
 390     case DataType::Float:
 391         if (descr.fieldSize == 4)
 392             fieldValue = readPlainValue<float>(locOffset, descr.fieldSize);
 393         else
 394             qDebug() << "* Invalid field size for type: Float";
 395         break;
 396     default:
 397         qDebug() << "* Unsupported data type: Unknown";
 398     }
 399
 400     if (!usePointer)
 401         offset = locOffset;
 402     return fieldValue;
 403 }
 404
 405 bool GeoIPDatabase::readDataFieldDescriptor(quint32 &offset, DataFieldDescriptor &out) const
 406 {
 407     const uchar *dataPtr = m_data + offset;
 408     const int availSize = m_size - offset;
 409     if (availSize < 1) return false;
 410
 411     out.fieldType = static_cast<DataType>((dataPtr[0] & 0xE0) >> 5);
 412     if (out.fieldType == DataType::Pointer)
 413     {
 414         const int size = ((dataPtr[0] & 0x18) >> 3);
 415         if (availSize < (size + 2)) return false;
 416
 417         if (size == 0)
 418             out.offset = ((dataPtr[0] & 0x07) << 8) + dataPtr[1];
 419         else if (size == 1)
 420             out.offset = ((dataPtr[0] & 0x07) << 16) + (dataPtr[1] << 8) + dataPtr[2] + 2048;
 421         else if (size == 2)
 422             out.offset = ((dataPtr[0] & 0x07) << 24) + (dataPtr[1] << 16) + (dataPtr[2] << 8) + dataPtr[3] + 526336;
 423         else if (size == 3)
 424             out.offset = (dataPtr[1] << 24) + (dataPtr[2] << 16) + (dataPtr[3] << 8) + dataPtr[4];
 425
 426         offset += size + 2;
 427         return true;
 428     }
 429
 430     out.fieldSize = dataPtr[0] & 0x1F;
 431     if (out.fieldSize <= 28)
 432     {
 433         if (out.fieldType == DataType::Unknown)
 434         {
 435             out.fieldType = static_cast<DataType>(dataPtr[1] + 7);
 436             if ((out.fieldType <= DataType::Map) || (out.fieldType > DataType::Float) || (availSize < 3))
 437                 return false;
 438             offset += 2;
 439         }
 440         else
 441         {
 442             offset += 1;
 443         }
 444     }
 445     else if (out.fieldSize == 29)
 446     {
 447         if (availSize < 2) return false;
 448         out.fieldSize = dataPtr[1] + 29;
 449         offset += 2;
 450     }
 451     else if (out.fieldSize == 30)
 452     {
 453         if (availSize < 3) return false;
 454         out.fieldSize = (dataPtr[1] << 8) + dataPtr[2] + 285;
 455         offset += 3;
 456     }
 457     else if (out.fieldSize == 31)
 458     {
 459         if (availSize < 4) return false;
 460         out.fieldSize = (dataPtr[1] << 16) + (dataPtr[2] << 8) + dataPtr[3] + 65821;
 461         offset += 4;
 462     }
 463
 464     return true;
 465 }
 466
 467 void GeoIPDatabase::fromBigEndian(uchar *buf, const quint32 len) const
 468 {
 469 #if (Q_BYTE_ORDER == Q_LITTLE_ENDIAN)
 470     std::reverse(buf, buf + len);
 471 #else
 472     Q_UNUSED(buf);
 473     Q_UNUSED(len);
 474 #endif
 475 }
 476
 477 QVariant GeoIPDatabase::readMapValue(quint32 &offset, const quint32 count) const
 478 {
 479     QVariantHash map;
 480
 481     for (quint32 i = 0; i < count; ++i)
 482     {
 483         QVariant field = readDataField(offset);
 484         if (field.userType() != QMetaType::QString)
 485             return {};
 486
 487         const QString key = field.toString();
 488         field = readDataField(offset);
 489         if (field.userType() == QVariant::Invalid)
 490             return {};
 491
 492         map[key] = field;
 493     }
 494
 495     return map;
 496 }
 497
 498 QVariant GeoIPDatabase::readArrayValue(quint32 &offset, const quint32 count) const
 499 {
 500     QVariantList array;
 501
 502     for (quint32 i = 0; i < count; ++i)
 503     {
 504         const QVariant field = readDataField(offset);
 505         if (field.userType() == QVariant::Invalid)
 506             return {};
 507
 508         array.append(field);
 509     }
 510
 511     return array;
 512 }
 513
 514 template <typename T>
 515 QVariant GeoIPDatabase::readPlainValue(quint32 &offset, const quint8 len) const
 516 {
 517     T value = 0;
 518     const uchar *const data = m_data + offset;
 519     const quint32 availSize = m_size - offset;
 520
 521     if ((len > 0) && (len <= sizeof(T) && (availSize >= len)))
 522     {
 523         // copy input data to last 'len' bytes of 'value'
 524         uchar *dst = reinterpret_cast<uchar *>(&value) + (sizeof(T) - len);
 525         memcpy(dst, data, len);
 526         fromBigEndian(reinterpret_cast<uchar *>(&value), sizeof(T));
 527         offset += len;
 528     }
 529
 530     return QVariant::fromValue(value);
 531 }