2 * Copyright (C) 2005-2018 Team Kodi
3 * This file is part of Kodi - https://kodi.tv
5 * SPDX-License-Identifier: GPL-2.0-or-later
6 * See LICENSES/README.md for more information.
11 #include "DatabaseManager.h"
13 #include "ServiceBroker.h"
14 #include "filesystem/SpecialProtocol.h"
15 #if defined(HAS_MYSQL) || defined(HAS_MARIADB)
16 #include "mysqldataset.h"
18 #include "profiles/ProfileManager.h"
19 #include "settings/AdvancedSettings.h"
20 #include "settings/SettingsComponent.h"
21 #include "sqlitedataset.h"
22 #include "utils/SortUtils.h"
23 #include "utils/StringUtils.h"
24 #include "utils/log.h"
27 #include "platform/posix/ConvUtils.h"
32 using namespace dbiplus
;
34 #define MAX_COMPRESS_COUNT 20
36 void CDatabase::Filter::AppendField(const std::string
& strField
)
41 if (fields
.empty() || fields
== "*")
44 fields
+= ", " + strField
;
47 void CDatabase::Filter::AppendJoin(const std::string
& strJoin
)
55 join
+= " " + strJoin
;
58 void CDatabase::Filter::AppendWhere(const std::string
& strWhere
, bool combineWithAnd
/* = true */)
67 where
= "(" + where
+ ") ";
68 where
+= combineWithAnd
? "AND" : "OR";
69 where
+= " (" + strWhere
+ ")";
73 void CDatabase::Filter::AppendOrder(const std::string
& strOrder
)
81 order
+= ", " + strOrder
;
84 void CDatabase::Filter::AppendGroup(const std::string
& strGroup
)
92 group
+= ", " + strGroup
;
95 void CDatabase::ExistsSubQuery::AppendJoin(const std::string
& strJoin
)
103 join
+= " " + strJoin
;
106 void CDatabase::ExistsSubQuery::AppendWhere(const std::string
& strWhere
,
107 bool combineWithAnd
/* = true */)
109 if (strWhere
.empty())
116 where
+= combineWithAnd
? " AND " : " OR ";
121 bool CDatabase::ExistsSubQuery::BuildSQL(std::string
& strSQL
)
123 if (tablename
.empty())
125 strSQL
= "EXISTS (SELECT 1 FROM " + tablename
;
127 strSQL
+= " " + join
;
128 std::string strWhere
;
133 if (!strWhere
.empty())
137 if (!strWhere
.empty())
138 strSQL
+= " WHERE " + strWhere
;
144 CDatabase::DatasetLayout::DatasetLayout(size_t totalfields
)
146 m_fields
.resize(totalfields
, DatasetFieldInfo(false, false, -1));
149 void CDatabase::DatasetLayout::SetField(int fieldNo
,
150 const std::string
& strField
,
151 bool bOutput
/*= false*/)
153 if (fieldNo
>= 0 && fieldNo
< static_cast<int>(m_fields
.size()))
155 m_fields
[fieldNo
].strField
= strField
;
156 m_fields
[fieldNo
].fetch
= true;
157 m_fields
[fieldNo
].output
= bOutput
;
161 void CDatabase::DatasetLayout::AdjustRecordNumbers(int offset
)
164 for (auto& field
: m_fields
)
168 field
.recno
= recno
+ offset
;
174 bool CDatabase::DatasetLayout::GetFetch(int fieldno
)
176 if (fieldno
>= 0 && fieldno
< static_cast<int>(m_fields
.size()))
177 return m_fields
[fieldno
].fetch
;
181 void CDatabase::DatasetLayout::SetFetch(int fieldno
, bool bFetch
/*= true*/)
183 if (fieldno
>= 0 && fieldno
< static_cast<int>(m_fields
.size()))
184 m_fields
[fieldno
].fetch
= bFetch
;
187 bool CDatabase::DatasetLayout::GetOutput(int fieldno
)
189 if (fieldno
>= 0 && fieldno
< static_cast<int>(m_fields
.size()))
190 return m_fields
[fieldno
].output
;
194 int CDatabase::DatasetLayout::GetRecNo(int fieldno
)
196 if (fieldno
>= 0 && fieldno
< static_cast<int>(m_fields
.size()))
197 return m_fields
[fieldno
].recno
;
201 const std::string
CDatabase::DatasetLayout::GetFields()
204 for (const auto& field
: m_fields
)
206 if (!field
.strField
.empty() && field
.fetch
)
209 strSQL
= field
.strField
;
211 strSQL
+= ", " + field
.strField
;
218 bool CDatabase::DatasetLayout::HasFilterFields()
220 for (const auto& field
: m_fields
)
228 CDatabase::CDatabase()
229 : m_profileManager(*CServiceBroker::GetSettingsComponent()->GetProfileManager())
233 m_multipleExecute
= false;
236 CDatabase::~CDatabase(void)
241 void CDatabase::Split(const std::string
& strFileNameAndPath
,
242 std::string
& strPath
,
243 std::string
& strFileName
)
247 int i
= strFileNameAndPath
.size() - 1;
250 char ch
= strFileNameAndPath
[i
];
251 if (ch
== ':' || ch
== '/' || ch
== '\\')
256 strPath
= strFileNameAndPath
.substr(0, i
);
257 strFileName
= strFileNameAndPath
.substr(i
);
260 std::string
CDatabase::PrepareSQL(std::string strStmt
, ...) const
262 std::string strResult
= "";
264 if (nullptr != m_pDB
)
267 va_start(args
, strStmt
);
268 strResult
= m_pDB
->vprepare(strStmt
.c_str(), args
);
275 std::string
CDatabase::GetSingleValue(const std::string
& query
,
276 const std::unique_ptr
<Dataset
>& ds
) const
284 if (ds
->query(query
) && ds
->num_rows() > 0)
285 ret
= ds
->fv(0).get_asString();
291 CLog::Log(LOGERROR
, "{} - failed on query '{}'", __FUNCTION__
, query
);
296 std::string
CDatabase::GetSingleValue(const std::string
& strTable
,
297 const std::string
& strColumn
,
298 const std::string
& strWhereClause
/* = std::string() */,
299 const std::string
& strOrderBy
/* = std::string() */) const
301 std::string query
= PrepareSQL("SELECT %s FROM %s", strColumn
.c_str(), strTable
.c_str());
302 if (!strWhereClause
.empty())
303 query
+= " WHERE " + strWhereClause
;
304 if (!strOrderBy
.empty())
305 query
+= " ORDER BY " + strOrderBy
;
307 return GetSingleValue(query
, m_pDS
);
310 std::string
CDatabase::GetSingleValue(const std::string
& query
) const
312 return GetSingleValue(query
, m_pDS
);
315 int CDatabase::GetSingleValueInt(const std::string
& query
, const std::unique_ptr
<Dataset
>& ds
) const
323 if (ds
->query(query
) && ds
->num_rows() > 0)
324 ret
= ds
->fv(0).get_asInt();
330 CLog::Log(LOGERROR
, "{} - failed on query '{}'", __FUNCTION__
, query
);
335 int CDatabase::GetSingleValueInt(const std::string
& strTable
,
336 const std::string
& strColumn
,
337 const std::string
& strWhereClause
/* = std::string() */,
338 const std::string
& strOrderBy
/* = std::string() */) const
340 std::string strResult
= GetSingleValue(strTable
, strColumn
, strWhereClause
, strOrderBy
);
341 return static_cast<int>(strtol(strResult
.c_str(), NULL
, 10));
344 int CDatabase::GetSingleValueInt(const std::string
& query
) const
346 return GetSingleValueInt(query
, m_pDS
);
349 bool CDatabase::DeleteValues(const std::string
& strTable
, const Filter
& filter
/* = Filter() */)
351 std::string strQuery
;
352 BuildSQL(PrepareSQL("DELETE FROM %s ", strTable
.c_str()), filter
, strQuery
);
353 return ExecuteQuery(strQuery
);
356 bool CDatabase::BeginMultipleExecute()
358 m_multipleExecute
= true;
359 m_multipleQueries
.clear();
363 bool CDatabase::CommitMultipleExecute()
365 m_multipleExecute
= false;
367 for (const auto& i
: m_multipleQueries
)
369 if (!ExecuteQuery(i
))
371 RollbackTransaction();
375 m_multipleQueries
.clear();
376 return CommitTransaction();
379 bool CDatabase::ExecuteQuery(const std::string
& strQuery
)
381 if (m_multipleExecute
)
383 m_multipleQueries
.push_back(strQuery
);
387 bool bReturn
= false;
391 if (nullptr == m_pDB
)
393 if (nullptr == m_pDS
)
395 m_pDS
->exec(strQuery
);
400 CLog::Log(LOGERROR
, "{} - failed to execute query '{}'", __FUNCTION__
, strQuery
);
406 bool CDatabase::ResultQuery(const std::string
& strQuery
) const
408 bool bReturn
= false;
412 if (nullptr == m_pDB
)
414 if (nullptr == m_pDS
)
417 std::string strPreparedQuery
= PrepareSQL(strQuery
);
419 bReturn
= m_pDS
->query(strPreparedQuery
);
423 CLog::Log(LOGERROR
, "{} - failed to execute query '{}'", __FUNCTION__
, strQuery
);
429 bool CDatabase::QueueInsertQuery(const std::string
& strQuery
)
431 if (strQuery
.empty())
436 if (nullptr == m_pDB
)
438 if (nullptr == m_pDS2
)
441 m_bMultiInsert
= true;
445 m_pDS2
->add_insert_sql(strQuery
);
450 bool CDatabase::CommitInsertQueries()
458 m_bMultiInsert
= false;
460 m_pDS2
->clear_insert_sql();
465 CLog::Log(LOGERROR
, "{} - failed to execute queries", __FUNCTION__
);
472 size_t CDatabase::GetInsertQueriesCount()
474 return m_pDS2
->insert_sql_count();
477 bool CDatabase::QueueDeleteQuery(const std::string
& strQuery
)
479 if (strQuery
.empty() || !m_pDB
|| !m_pDS
)
482 m_bMultiDelete
= true;
484 m_pDS
->add_delete_sql(strQuery
);
488 bool CDatabase::CommitDeleteQueries()
496 m_bMultiDelete
= false;
498 m_pDS
->clear_delete_sql();
503 CLog::Log(LOGERROR
, "{} - failed to execute queries", __FUNCTION__
);
510 size_t CDatabase::GetDeleteQueriesCount()
512 return m_pDS
->delete_sql_count();
515 bool CDatabase::Open()
517 DatabaseSettings db_fallback
;
518 return Open(db_fallback
);
521 bool CDatabase::Open(const DatabaseSettings
& settings
)
529 // check our database manager to see if this database can be opened
530 if (!CServiceBroker::GetDatabaseManager().CanOpen(GetBaseDBName()))
533 DatabaseSettings dbSettings
= settings
;
534 InitSettings(dbSettings
);
536 std::string dbName
= dbSettings
.name
;
537 dbName
+= std::to_string(GetSchemaVersion());
538 return Connect(dbName
, dbSettings
, false);
541 void CDatabase::InitSettings(DatabaseSettings
& dbSettings
)
545 #if defined(HAS_MYSQL) || defined(HAS_MARIADB)
546 if (dbSettings
.type
== "mysql")
548 // check we have all information before we cancel the fallback
549 if (!(dbSettings
.host
.empty() || dbSettings
.user
.empty() || dbSettings
.pass
.empty()))
552 CLog::Log(LOGINFO
, "Essential mysql database information is missing. Require at least host, "
553 "user and pass defined.");
557 if (dbSettings
.type
== "mysql")
560 "MySQL library requested but MySQL support is not compiled in. Falling back to sqlite3.");
563 dbSettings
.type
= "sqlite3";
564 if (dbSettings
.host
.empty())
565 dbSettings
.host
= CSpecialProtocol::TranslatePath(m_profileManager
.GetDatabaseFolder());
568 // use separate, versioned database
569 if (dbSettings
.name
.empty())
570 dbSettings
.name
= GetBaseDBName();
573 void CDatabase::CopyDB(const std::string
& latestDb
)
575 m_pDB
->copy(latestDb
.c_str());
578 void CDatabase::DropAnalytics()
580 m_pDB
->drop_analytics();
583 bool CDatabase::Connect(const std::string
& dbName
, const DatabaseSettings
& dbSettings
, bool create
)
585 // create the appropriate database structure
586 if (dbSettings
.type
== "sqlite3")
588 m_pDB
= std::make_unique
<SqliteDatabase
>();
590 #if defined(HAS_MYSQL) || defined(HAS_MARIADB)
591 else if (dbSettings
.type
== "mysql")
593 m_pDB
= std::make_unique
<MysqlDatabase
>();
598 CLog::Log(LOGERROR
, "Unable to determine database type: {}", dbSettings
.type
);
602 // host name is always required
603 m_pDB
->setHostName(dbSettings
.host
.c_str());
605 if (!dbSettings
.port
.empty())
606 m_pDB
->setPort(dbSettings
.port
.c_str());
608 if (!dbSettings
.user
.empty())
609 m_pDB
->setLogin(dbSettings
.user
.c_str());
611 if (!dbSettings
.pass
.empty())
612 m_pDB
->setPasswd(dbSettings
.pass
.c_str());
614 // database name is always required
615 m_pDB
->setDatabase(dbName
.c_str());
617 // set configuration regardless if any are empty
618 m_pDB
->setConfig(dbSettings
.key
.c_str(), dbSettings
.cert
.c_str(), dbSettings
.ca
.c_str(),
619 dbSettings
.capath
.c_str(), dbSettings
.ciphers
.c_str(), dbSettings
.compression
);
621 // create the datasets
622 m_pDS
.reset(m_pDB
->CreateDataset());
623 m_pDS2
.reset(m_pDB
->CreateDataset());
625 if (m_pDB
->connect(create
) != DB_CONNECTION_OK
)
630 // test if db already exists, if not we need to create the tables
631 if (!m_pDB
->exists() && create
)
633 if (dbSettings
.type
== "sqlite3")
635 // Modern file systems have a cluster/block size of 4k.
636 // To gain better performance when performing write
637 // operations to the database, set the page size of the
638 // database file to 4k.
639 // This needs to be done before any table is created.
640 m_pDS
->exec("PRAGMA page_size=4096\n");
642 // Also set the memory cache size to 16k
643 m_pDS
->exec("PRAGMA default_cache_size=4096\n");
648 // sqlite3 post connection operations
649 if (dbSettings
.type
== "sqlite3")
651 m_pDS
->exec("PRAGMA cache_size=4096\n");
652 m_pDS
->exec("PRAGMA synchronous='NORMAL'\n");
653 m_pDS
->exec("PRAGMA count_changes='OFF'\n");
656 catch (DbErrors
& error
)
658 CLog::Log(LOGERROR
, "{} failed with '{}'", __FUNCTION__
, error
.getMsg());
659 m_openCount
= 1; // set to open so we can execute Close()
664 m_openCount
= 1; // our database is open
668 int CDatabase::GetDBVersion()
670 m_pDS
->query("SELECT idVersion FROM version\n");
671 if (m_pDS
->num_rows() > 0)
672 return m_pDS
->fv("idVersion").get_asInt();
676 bool CDatabase::IsOpen()
678 return m_openCount
> 0;
681 void CDatabase::Close()
683 if (m_openCount
== 0)
693 m_multipleExecute
= false;
695 if (nullptr == m_pDB
)
697 if (nullptr != m_pDS
)
705 bool CDatabase::Compress(bool bForce
/* =true */)
712 if (nullptr == m_pDB
)
714 if (nullptr == m_pDS
)
718 m_pDS
->query("select iCompressCount from version");
721 int iCount
= m_pDS
->fv(0).get_asInt();
722 if (iCount
> MAX_COMPRESS_COUNT
)
725 std::string strSQL
= PrepareSQL("update version set iCompressCount=%i\n", ++iCount
);
732 if (!m_pDS
->exec("vacuum\n"))
737 CLog::Log(LOGERROR
, "{} - Compressing the database failed", __FUNCTION__
);
743 void CDatabase::Interrupt()
748 void CDatabase::BeginTransaction()
752 if (nullptr != m_pDB
)
753 m_pDB
->start_transaction();
757 CLog::Log(LOGERROR
, "database:begintransaction failed");
761 bool CDatabase::CommitTransaction()
765 if (nullptr != m_pDB
)
766 m_pDB
->commit_transaction();
770 CLog::Log(LOGERROR
, "database:committransaction failed");
776 void CDatabase::RollbackTransaction()
780 if (nullptr != m_pDB
)
781 m_pDB
->rollback_transaction();
785 CLog::Log(LOGERROR
, "database:rollbacktransaction failed");
789 bool CDatabase::CreateDatabase()
794 CLog::Log(LOGINFO
, "creating version table");
795 m_pDS
->exec("CREATE TABLE version (idVersion integer, iCompressCount integer)\n");
796 std::string strSQL
= PrepareSQL("INSERT INTO version (idVersion,iCompressCount) values(%i,0)\n",
805 CLog::Log(LOGERROR
, "{} unable to create database:{}", __FUNCTION__
, (int)GetLastError());
806 RollbackTransaction();
810 return CommitTransaction();
813 void CDatabase::UpdateVersionNumber()
815 std::string strSQL
= PrepareSQL("UPDATE version SET idVersion=%i\n", GetSchemaVersion());
819 bool CDatabase::BuildSQL(const std::string
& strQuery
,
820 const Filter
& filter
,
821 std::string
& strSQL
) const
825 if (!filter
.join
.empty())
826 strSQL
+= filter
.join
;
827 if (!filter
.where
.empty())
828 strSQL
+= " WHERE " + filter
.where
;
829 if (!filter
.group
.empty())
830 strSQL
+= " GROUP BY " + filter
.group
;
831 if (!filter
.order
.empty())
832 strSQL
+= " ORDER BY " + filter
.order
;
833 if (!filter
.limit
.empty())
834 strSQL
+= " LIMIT " + filter
.limit
;
839 bool CDatabase::BuildSQL(const std::string
& strBaseDir
,
840 const std::string
& strQuery
,
845 SortDescription sorting
;
846 return BuildSQL(strBaseDir
, strQuery
, filter
, strSQL
, dbUrl
, sorting
);
849 bool CDatabase::BuildSQL(const std::string
& strBaseDir
,
850 const std::string
& strQuery
,
854 SortDescription
& sorting
/* = SortDescription() */)
856 // parse the base path to get additional filters
858 if (!dbUrl
.FromString(strBaseDir
) || !GetFilter(dbUrl
, filter
, sorting
))
861 return BuildSQL(strQuery
, filter
, strSQL
);