[Test] Added tests for CUtil::SplitParams
[xbmc.git] / xbmc / dbwrappers / Database.cpp
blobdc17d4d843cef642d22db2ae14b97ea0a6a13ffb
1 /*
2 * Copyright (C) 2005-2018 Team Kodi
3 * This file is part of Kodi - https://kodi.tv
5 * SPDX-License-Identifier: GPL-2.0-or-later
6 * See LICENSES/README.md for more information.
7 */
9 #include "Database.h"
11 #include "DatabaseManager.h"
12 #include "DbUrl.h"
13 #include "ServiceBroker.h"
14 #include "filesystem/SpecialProtocol.h"
15 #if defined(HAS_MYSQL) || defined(HAS_MARIADB)
16 #include "mysqldataset.h"
17 #endif
18 #include "profiles/ProfileManager.h"
19 #include "settings/AdvancedSettings.h"
20 #include "settings/SettingsComponent.h"
21 #include "sqlitedataset.h"
22 #include "utils/SortUtils.h"
23 #include "utils/StringUtils.h"
24 #include "utils/log.h"
26 #ifdef TARGET_POSIX
27 #include "platform/posix/ConvUtils.h"
28 #endif
30 #include <memory>
32 using namespace dbiplus;
34 #define MAX_COMPRESS_COUNT 20
36 void CDatabase::Filter::AppendField(const std::string& strField)
38 if (strField.empty())
39 return;
41 if (fields.empty() || fields == "*")
42 fields = strField;
43 else
44 fields += ", " + strField;
47 void CDatabase::Filter::AppendJoin(const std::string& strJoin)
49 if (strJoin.empty())
50 return;
52 if (join.empty())
53 join = strJoin;
54 else
55 join += " " + strJoin;
58 void CDatabase::Filter::AppendWhere(const std::string& strWhere, bool combineWithAnd /* = true */)
60 if (strWhere.empty())
61 return;
63 if (where.empty())
64 where = strWhere;
65 else
67 where = "(" + where + ") ";
68 where += combineWithAnd ? "AND" : "OR";
69 where += " (" + strWhere + ")";
73 void CDatabase::Filter::AppendOrder(const std::string& strOrder)
75 if (strOrder.empty())
76 return;
78 if (order.empty())
79 order = strOrder;
80 else
81 order += ", " + strOrder;
84 void CDatabase::Filter::AppendGroup(const std::string& strGroup)
86 if (strGroup.empty())
87 return;
89 if (group.empty())
90 group = strGroup;
91 else
92 group += ", " + strGroup;
95 void CDatabase::ExistsSubQuery::AppendJoin(const std::string& strJoin)
97 if (strJoin.empty())
98 return;
100 if (join.empty())
101 join = strJoin;
102 else
103 join += " " + strJoin;
106 void CDatabase::ExistsSubQuery::AppendWhere(const std::string& strWhere,
107 bool combineWithAnd /* = true */)
109 if (strWhere.empty())
110 return;
112 if (where.empty())
113 where = strWhere;
114 else
116 where += combineWithAnd ? " AND " : " OR ";
117 where += strWhere;
121 bool CDatabase::ExistsSubQuery::BuildSQL(std::string& strSQL)
123 if (tablename.empty())
124 return false;
125 strSQL = "EXISTS (SELECT 1 FROM " + tablename;
126 if (!join.empty())
127 strSQL += " " + join;
128 std::string strWhere;
129 if (!param.empty())
130 strWhere = param;
131 if (!where.empty())
133 if (!strWhere.empty())
134 strWhere += " AND ";
135 strWhere += where;
137 if (!strWhere.empty())
138 strSQL += " WHERE " + strWhere;
140 strSQL += ")";
141 return true;
144 CDatabase::DatasetLayout::DatasetLayout(size_t totalfields)
146 m_fields.resize(totalfields, DatasetFieldInfo(false, false, -1));
149 void CDatabase::DatasetLayout::SetField(int fieldNo,
150 const std::string& strField,
151 bool bOutput /*= false*/)
153 if (fieldNo >= 0 && fieldNo < static_cast<int>(m_fields.size()))
155 m_fields[fieldNo].strField = strField;
156 m_fields[fieldNo].fetch = true;
157 m_fields[fieldNo].output = bOutput;
161 void CDatabase::DatasetLayout::AdjustRecordNumbers(int offset)
163 int recno = 0;
164 for (auto& field : m_fields)
166 if (field.fetch)
168 field.recno = recno + offset;
169 ++recno;
174 bool CDatabase::DatasetLayout::GetFetch(int fieldno)
176 if (fieldno >= 0 && fieldno < static_cast<int>(m_fields.size()))
177 return m_fields[fieldno].fetch;
178 return false;
181 void CDatabase::DatasetLayout::SetFetch(int fieldno, bool bFetch /*= true*/)
183 if (fieldno >= 0 && fieldno < static_cast<int>(m_fields.size()))
184 m_fields[fieldno].fetch = bFetch;
187 bool CDatabase::DatasetLayout::GetOutput(int fieldno)
189 if (fieldno >= 0 && fieldno < static_cast<int>(m_fields.size()))
190 return m_fields[fieldno].output;
191 return false;
194 int CDatabase::DatasetLayout::GetRecNo(int fieldno)
196 if (fieldno >= 0 && fieldno < static_cast<int>(m_fields.size()))
197 return m_fields[fieldno].recno;
198 return -1;
201 const std::string CDatabase::DatasetLayout::GetFields()
203 std::string strSQL;
204 for (const auto& field : m_fields)
206 if (!field.strField.empty() && field.fetch)
208 if (strSQL.empty())
209 strSQL = field.strField;
210 else
211 strSQL += ", " + field.strField;
215 return strSQL;
218 bool CDatabase::DatasetLayout::HasFilterFields()
220 for (const auto& field : m_fields)
222 if (field.fetch)
223 return true;
225 return false;
228 CDatabase::CDatabase()
229 : m_profileManager(*CServiceBroker::GetSettingsComponent()->GetProfileManager())
231 m_openCount = 0;
232 m_sqlite = true;
233 m_multipleExecute = false;
236 CDatabase::~CDatabase(void)
238 Close();
241 void CDatabase::Split(const std::string& strFileNameAndPath,
242 std::string& strPath,
243 std::string& strFileName)
245 strFileName = "";
246 strPath = "";
247 int i = strFileNameAndPath.size() - 1;
248 while (i > 0)
250 char ch = strFileNameAndPath[i];
251 if (ch == ':' || ch == '/' || ch == '\\')
252 break;
253 else
254 i--;
256 strPath = strFileNameAndPath.substr(0, i);
257 strFileName = strFileNameAndPath.substr(i);
260 std::string CDatabase::PrepareSQL(std::string strStmt, ...) const
262 std::string strResult = "";
264 if (nullptr != m_pDB)
266 va_list args;
267 va_start(args, strStmt);
268 strResult = m_pDB->vprepare(strStmt.c_str(), args);
269 va_end(args);
272 return strResult;
275 std::string CDatabase::GetSingleValue(const std::string& query,
276 const std::unique_ptr<Dataset>& ds) const
278 std::string ret;
281 if (!m_pDB || !ds)
282 return ret;
284 if (ds->query(query) && ds->num_rows() > 0)
285 ret = ds->fv(0).get_asString();
287 ds->close();
289 catch (...)
291 CLog::Log(LOGERROR, "{} - failed on query '{}'", __FUNCTION__, query);
293 return ret;
296 std::string CDatabase::GetSingleValue(const std::string& strTable,
297 const std::string& strColumn,
298 const std::string& strWhereClause /* = std::string() */,
299 const std::string& strOrderBy /* = std::string() */) const
301 std::string query = PrepareSQL("SELECT %s FROM %s", strColumn.c_str(), strTable.c_str());
302 if (!strWhereClause.empty())
303 query += " WHERE " + strWhereClause;
304 if (!strOrderBy.empty())
305 query += " ORDER BY " + strOrderBy;
306 query += " LIMIT 1";
307 return GetSingleValue(query, m_pDS);
310 std::string CDatabase::GetSingleValue(const std::string& query) const
312 return GetSingleValue(query, m_pDS);
315 int CDatabase::GetSingleValueInt(const std::string& query, const std::unique_ptr<Dataset>& ds) const
317 int ret = 0;
320 if (!m_pDB || !ds)
321 return ret;
323 if (ds->query(query) && ds->num_rows() > 0)
324 ret = ds->fv(0).get_asInt();
326 ds->close();
328 catch (...)
330 CLog::Log(LOGERROR, "{} - failed on query '{}'", __FUNCTION__, query);
332 return ret;
335 int CDatabase::GetSingleValueInt(const std::string& strTable,
336 const std::string& strColumn,
337 const std::string& strWhereClause /* = std::string() */,
338 const std::string& strOrderBy /* = std::string() */) const
340 std::string strResult = GetSingleValue(strTable, strColumn, strWhereClause, strOrderBy);
341 return static_cast<int>(strtol(strResult.c_str(), NULL, 10));
344 int CDatabase::GetSingleValueInt(const std::string& query) const
346 return GetSingleValueInt(query, m_pDS);
349 bool CDatabase::DeleteValues(const std::string& strTable, const Filter& filter /* = Filter() */)
351 std::string strQuery;
352 BuildSQL(PrepareSQL("DELETE FROM %s ", strTable.c_str()), filter, strQuery);
353 return ExecuteQuery(strQuery);
356 bool CDatabase::BeginMultipleExecute()
358 m_multipleExecute = true;
359 m_multipleQueries.clear();
360 return true;
363 bool CDatabase::CommitMultipleExecute()
365 m_multipleExecute = false;
366 BeginTransaction();
367 for (const auto& i : m_multipleQueries)
369 if (!ExecuteQuery(i))
371 RollbackTransaction();
372 return false;
375 m_multipleQueries.clear();
376 return CommitTransaction();
379 bool CDatabase::ExecuteQuery(const std::string& strQuery)
381 if (m_multipleExecute)
383 m_multipleQueries.push_back(strQuery);
384 return true;
387 bool bReturn = false;
391 if (nullptr == m_pDB)
392 return bReturn;
393 if (nullptr == m_pDS)
394 return bReturn;
395 m_pDS->exec(strQuery);
396 bReturn = true;
398 catch (...)
400 CLog::Log(LOGERROR, "{} - failed to execute query '{}'", __FUNCTION__, strQuery);
403 return bReturn;
406 bool CDatabase::ResultQuery(const std::string& strQuery) const
408 bool bReturn = false;
412 if (nullptr == m_pDB)
413 return bReturn;
414 if (nullptr == m_pDS)
415 return bReturn;
417 std::string strPreparedQuery = PrepareSQL(strQuery);
419 bReturn = m_pDS->query(strPreparedQuery);
421 catch (...)
423 CLog::Log(LOGERROR, "{} - failed to execute query '{}'", __FUNCTION__, strQuery);
426 return bReturn;
429 bool CDatabase::QueueInsertQuery(const std::string& strQuery)
431 if (strQuery.empty())
432 return false;
434 if (!m_bMultiInsert)
436 if (nullptr == m_pDB)
437 return false;
438 if (nullptr == m_pDS2)
439 return false;
441 m_bMultiInsert = true;
442 m_pDS2->insert();
445 m_pDS2->add_insert_sql(strQuery);
447 return true;
450 bool CDatabase::CommitInsertQueries()
452 bool bReturn = true;
454 if (m_bMultiInsert)
458 m_bMultiInsert = false;
459 m_pDS2->post();
460 m_pDS2->clear_insert_sql();
462 catch (...)
464 bReturn = false;
465 CLog::Log(LOGERROR, "{} - failed to execute queries", __FUNCTION__);
469 return bReturn;
472 size_t CDatabase::GetInsertQueriesCount()
474 return m_pDS2->insert_sql_count();
477 bool CDatabase::QueueDeleteQuery(const std::string& strQuery)
479 if (strQuery.empty() || !m_pDB || !m_pDS)
480 return false;
482 m_bMultiDelete = true;
483 m_pDS->del();
484 m_pDS->add_delete_sql(strQuery);
485 return true;
488 bool CDatabase::CommitDeleteQueries()
490 bool bReturn = true;
492 if (m_bMultiDelete)
496 m_bMultiDelete = false;
497 m_pDS->deletion();
498 m_pDS->clear_delete_sql();
500 catch (...)
502 bReturn = false;
503 CLog::Log(LOGERROR, "{} - failed to execute queries", __FUNCTION__);
507 return bReturn;
510 size_t CDatabase::GetDeleteQueriesCount()
512 return m_pDS->delete_sql_count();
515 bool CDatabase::Open()
517 DatabaseSettings db_fallback;
518 return Open(db_fallback);
521 bool CDatabase::Open(const DatabaseSettings& settings)
523 if (IsOpen())
525 m_openCount++;
526 return true;
529 // check our database manager to see if this database can be opened
530 if (!CServiceBroker::GetDatabaseManager().CanOpen(GetBaseDBName()))
531 return false;
533 DatabaseSettings dbSettings = settings;
534 InitSettings(dbSettings);
536 std::string dbName = dbSettings.name;
537 dbName += std::to_string(GetSchemaVersion());
538 return Connect(dbName, dbSettings, false);
541 void CDatabase::InitSettings(DatabaseSettings& dbSettings)
543 m_sqlite = true;
545 #if defined(HAS_MYSQL) || defined(HAS_MARIADB)
546 if (dbSettings.type == "mysql")
548 // check we have all information before we cancel the fallback
549 if (!(dbSettings.host.empty() || dbSettings.user.empty() || dbSettings.pass.empty()))
550 m_sqlite = false;
551 else
552 CLog::Log(LOGINFO, "Essential mysql database information is missing. Require at least host, "
553 "user and pass defined.");
555 else
556 #else
557 if (dbSettings.type == "mysql")
558 CLog::Log(
559 LOGERROR,
560 "MySQL library requested but MySQL support is not compiled in. Falling back to sqlite3.");
561 #endif
563 dbSettings.type = "sqlite3";
564 if (dbSettings.host.empty())
565 dbSettings.host = CSpecialProtocol::TranslatePath(m_profileManager.GetDatabaseFolder());
568 // use separate, versioned database
569 if (dbSettings.name.empty())
570 dbSettings.name = GetBaseDBName();
573 void CDatabase::CopyDB(const std::string& latestDb)
575 m_pDB->copy(latestDb.c_str());
578 void CDatabase::DropAnalytics()
580 m_pDB->drop_analytics();
583 bool CDatabase::Connect(const std::string& dbName, const DatabaseSettings& dbSettings, bool create)
585 // create the appropriate database structure
586 if (dbSettings.type == "sqlite3")
588 m_pDB = std::make_unique<SqliteDatabase>();
590 #if defined(HAS_MYSQL) || defined(HAS_MARIADB)
591 else if (dbSettings.type == "mysql")
593 m_pDB = std::make_unique<MysqlDatabase>();
595 #endif
596 else
598 CLog::Log(LOGERROR, "Unable to determine database type: {}", dbSettings.type);
599 return false;
602 // host name is always required
603 m_pDB->setHostName(dbSettings.host.c_str());
605 if (!dbSettings.port.empty())
606 m_pDB->setPort(dbSettings.port.c_str());
608 if (!dbSettings.user.empty())
609 m_pDB->setLogin(dbSettings.user.c_str());
611 if (!dbSettings.pass.empty())
612 m_pDB->setPasswd(dbSettings.pass.c_str());
614 // database name is always required
615 m_pDB->setDatabase(dbName.c_str());
617 // set configuration regardless if any are empty
618 m_pDB->setConfig(dbSettings.key.c_str(), dbSettings.cert.c_str(), dbSettings.ca.c_str(),
619 dbSettings.capath.c_str(), dbSettings.ciphers.c_str(), dbSettings.compression);
621 // create the datasets
622 m_pDS.reset(m_pDB->CreateDataset());
623 m_pDS2.reset(m_pDB->CreateDataset());
625 if (m_pDB->connect(create) != DB_CONNECTION_OK)
626 return false;
630 // test if db already exists, if not we need to create the tables
631 if (!m_pDB->exists() && create)
633 if (dbSettings.type == "sqlite3")
635 // Modern file systems have a cluster/block size of 4k.
636 // To gain better performance when performing write
637 // operations to the database, set the page size of the
638 // database file to 4k.
639 // This needs to be done before any table is created.
640 m_pDS->exec("PRAGMA page_size=4096\n");
642 // Also set the memory cache size to 16k
643 m_pDS->exec("PRAGMA default_cache_size=4096\n");
645 CreateDatabase();
648 // sqlite3 post connection operations
649 if (dbSettings.type == "sqlite3")
651 m_pDS->exec("PRAGMA cache_size=4096\n");
652 m_pDS->exec("PRAGMA synchronous='NORMAL'\n");
653 m_pDS->exec("PRAGMA count_changes='OFF'\n");
656 catch (DbErrors& error)
658 CLog::Log(LOGERROR, "{} failed with '{}'", __FUNCTION__, error.getMsg());
659 m_openCount = 1; // set to open so we can execute Close()
660 Close();
661 return false;
664 m_openCount = 1; // our database is open
665 return true;
668 int CDatabase::GetDBVersion()
670 m_pDS->query("SELECT idVersion FROM version\n");
671 if (m_pDS->num_rows() > 0)
672 return m_pDS->fv("idVersion").get_asInt();
673 return 0;
676 bool CDatabase::IsOpen()
678 return m_openCount > 0;
681 void CDatabase::Close()
683 if (m_openCount == 0)
684 return;
686 if (m_openCount > 1)
688 m_openCount--;
689 return;
692 m_openCount = 0;
693 m_multipleExecute = false;
695 if (nullptr == m_pDB)
696 return;
697 if (nullptr != m_pDS)
698 m_pDS->close();
699 m_pDB->disconnect();
700 m_pDB.reset();
701 m_pDS.reset();
702 m_pDS2.reset();
705 bool CDatabase::Compress(bool bForce /* =true */)
707 if (!m_sqlite)
708 return true;
712 if (nullptr == m_pDB)
713 return false;
714 if (nullptr == m_pDS)
715 return false;
716 if (!bForce)
718 m_pDS->query("select iCompressCount from version");
719 if (!m_pDS->eof())
721 int iCount = m_pDS->fv(0).get_asInt();
722 if (iCount > MAX_COMPRESS_COUNT)
723 iCount = -1;
724 m_pDS->close();
725 std::string strSQL = PrepareSQL("update version set iCompressCount=%i\n", ++iCount);
726 m_pDS->exec(strSQL);
727 if (iCount != 0)
728 return true;
732 if (!m_pDS->exec("vacuum\n"))
733 return false;
735 catch (...)
737 CLog::Log(LOGERROR, "{} - Compressing the database failed", __FUNCTION__);
738 return false;
740 return true;
743 void CDatabase::Interrupt()
745 m_pDS->interrupt();
748 void CDatabase::BeginTransaction()
752 if (nullptr != m_pDB)
753 m_pDB->start_transaction();
755 catch (...)
757 CLog::Log(LOGERROR, "database:begintransaction failed");
761 bool CDatabase::CommitTransaction()
765 if (nullptr != m_pDB)
766 m_pDB->commit_transaction();
768 catch (...)
770 CLog::Log(LOGERROR, "database:committransaction failed");
771 return false;
773 return true;
776 void CDatabase::RollbackTransaction()
780 if (nullptr != m_pDB)
781 m_pDB->rollback_transaction();
783 catch (...)
785 CLog::Log(LOGERROR, "database:rollbacktransaction failed");
789 bool CDatabase::CreateDatabase()
791 BeginTransaction();
794 CLog::Log(LOGINFO, "creating version table");
795 m_pDS->exec("CREATE TABLE version (idVersion integer, iCompressCount integer)\n");
796 std::string strSQL = PrepareSQL("INSERT INTO version (idVersion,iCompressCount) values(%i,0)\n",
797 GetSchemaVersion());
798 m_pDS->exec(strSQL);
800 CreateTables();
801 CreateAnalytics();
803 catch (...)
805 CLog::Log(LOGERROR, "{} unable to create database:{}", __FUNCTION__, (int)GetLastError());
806 RollbackTransaction();
807 return false;
810 return CommitTransaction();
813 void CDatabase::UpdateVersionNumber()
815 std::string strSQL = PrepareSQL("UPDATE version SET idVersion=%i\n", GetSchemaVersion());
816 m_pDS->exec(strSQL);
819 bool CDatabase::BuildSQL(const std::string& strQuery,
820 const Filter& filter,
821 std::string& strSQL) const
823 strSQL = strQuery;
825 if (!filter.join.empty())
826 strSQL += filter.join;
827 if (!filter.where.empty())
828 strSQL += " WHERE " + filter.where;
829 if (!filter.group.empty())
830 strSQL += " GROUP BY " + filter.group;
831 if (!filter.order.empty())
832 strSQL += " ORDER BY " + filter.order;
833 if (!filter.limit.empty())
834 strSQL += " LIMIT " + filter.limit;
836 return true;
839 bool CDatabase::BuildSQL(const std::string& strBaseDir,
840 const std::string& strQuery,
841 Filter& filter,
842 std::string& strSQL,
843 CDbUrl& dbUrl)
845 SortDescription sorting;
846 return BuildSQL(strBaseDir, strQuery, filter, strSQL, dbUrl, sorting);
849 bool CDatabase::BuildSQL(const std::string& strBaseDir,
850 const std::string& strQuery,
851 Filter& filter,
852 std::string& strSQL,
853 CDbUrl& dbUrl,
854 SortDescription& sorting /* = SortDescription() */)
856 // parse the base path to get additional filters
857 dbUrl.Reset();
858 if (!dbUrl.FromString(strBaseDir) || !GetFilter(dbUrl, filter, sorting))
859 return false;
861 return BuildSQL(strQuery, filter, strSQL);