Merge pull request #25820 from hribz/master
[xbmc.git] / xbmc / URL.cpp
blobb934e7569e27af13570d87d3611095fc77495bd6
1 /*
2 * Copyright (C) 2005-2018 Team Kodi
3 * This file is part of Kodi - https://kodi.tv
5 * SPDX-License-Identifier: GPL-2.0-or-later
6 * See LICENSES/README.md for more information.
7 */
9 #include "URL.h"
11 #include "FileItem.h"
12 #include "FileItemList.h"
13 #include "ServiceBroker.h"
14 #include "Util.h"
15 #include "filesystem/File.h"
16 #include "filesystem/StackDirectory.h"
17 #include "network/Network.h"
18 #include "utils/StringUtils.h"
19 #include "utils/URIUtils.h"
20 #include "utils/log.h"
21 #ifndef TARGET_POSIX
22 #include <sys\stat.h>
23 #endif
25 #include <charconv>
26 #include <iterator>
27 #include <string>
28 #include <system_error>
29 #include <vector>
31 #include <fmt/xchar.h>
33 using namespace ADDON;
35 CURL::~CURL() = default;
37 void CURL::Reset()
39 m_strHostName.clear();
40 m_strDomain.clear();
41 m_strUserName.clear();
42 m_strPassword.clear();
43 m_strShareName.clear();
44 m_strFileName.clear();
45 m_strProtocol.clear();
46 m_strFileType.clear();
47 m_strOptions.clear();
48 m_strProtocolOptions.clear();
49 m_options.Clear();
50 m_protocolOptions.Clear();
51 m_iPort = 0;
54 void CURL::Parse(std::string strURL1)
56 Reset();
57 // start by validating the path
58 std::string strURL = CUtil::ValidatePath(std::move(strURL1));
60 // strURL can be one of the following:
61 // format 1: protocol://[username:password]@hostname[:port]/directoryandfile
62 // format 2: protocol://file
63 // format 3: drive:directoryandfile
65 // first need 2 check if this is a protocol or just a normal drive & path
66 if (!strURL.size()) return ;
67 if (strURL == "?") return;
69 // form is format 1 or 2
70 // format 1: protocol://[domain;][username:password]@hostname[:port]/directoryandfile
71 // format 2: protocol://file
73 // decode protocol
74 size_t iPos = strURL.find("://");
75 if (iPos == std::string::npos)
77 // This is an ugly hack that needs some work.
78 // example: filename /foo/bar.zip/alice.rar/bob.avi
79 // This should turn into zip://rar:///foo/bar.zip/alice.rar/bob.avi
80 iPos = 0;
81 bool is_apk = (strURL.find(".apk/", iPos) != std::string::npos);
82 while (true)
84 if (is_apk)
85 iPos = strURL.find(".apk/", iPos);
86 else
87 iPos = strURL.find(".zip/", iPos);
89 int extLen = 3;
90 if (iPos == std::string::npos)
92 /* set filename and update extension*/
93 SetFileName(std::move(strURL));
94 return ;
96 iPos += extLen + 1;
97 std::string archiveName = strURL.substr(0, iPos);
98 struct __stat64 s;
99 if (XFILE::CFile::Stat(archiveName, &s) == 0)
101 #ifdef TARGET_POSIX
102 if (!S_ISDIR(s.st_mode))
103 #else
104 if (!(s.st_mode & S_IFDIR))
105 #endif
107 archiveName = Encode(archiveName);
108 if (is_apk)
110 CURL c("apk://" + archiveName + "/" + std::move(strURL).substr(iPos + 1));
111 *this = c;
113 else
115 CURL c("zip://" + archiveName + "/" + std::move(strURL).substr(iPos + 1));
116 *this = c;
118 return;
123 else
125 SetProtocol(strURL.substr(0, iPos));
126 iPos += 3;
129 // virtual protocols
130 // why not handle all format 2 (protocol://file) style urls here?
131 // ones that come to mind are iso9660, cdda, musicdb, etc.
132 // they are all local protocols and have no server part, port number, special options, etc.
133 // this removes the need for special handling below.
134 // clang-format off
135 if (
136 IsProtocol("stack") ||
137 IsProtocol("virtualpath") ||
138 IsProtocol("multipath") ||
139 IsProtocol("special") ||
140 IsProtocol("resource") ||
141 IsProtocol("file")
143 // clang-format on
145 SetFileName(std::move(strURL).substr(iPos));
146 return;
149 if (IsProtocol("udf") || IsProtocol("iso9660"))
151 std::string lower(strURL);
152 StringUtils::ToLower(lower);
153 size_t isoPos = lower.find(".iso\\", iPos);
154 if (isoPos == std::string::npos)
155 isoPos = lower.find(".udf\\", iPos);
156 if (isoPos != std::string::npos)
158 strURL.replace(isoPos + 4, 1, "/");
162 // check for username/password - should occur before first /
163 if (iPos == std::string::npos) iPos = 0;
165 // for protocols supporting options, chop that part off here
166 // maybe we should invert this list instead?
167 size_t iEnd = strURL.length();
168 const char* sep = NULL;
170 //! @todo fix all Addon paths
171 std::string strProtocol2 = GetTranslatedProtocol();
172 if(IsProtocol("rss") ||
173 IsProtocol("rsss") ||
174 IsProtocol("rar") ||
175 IsProtocol("apk") ||
176 IsProtocol("xbt") ||
177 IsProtocol("zip") ||
178 IsProtocol("addons") ||
179 IsProtocol("image") ||
180 IsProtocol("videodb") ||
181 IsProtocol("musicdb") ||
182 IsProtocol("androidapp") ||
183 IsProtocol("pvr"))
184 sep = "?";
185 else
186 if( IsProtocolEqual(strProtocol2, "http")
187 || IsProtocolEqual(strProtocol2, "https")
188 || IsProtocolEqual(strProtocol2, "plugin")
189 || IsProtocolEqual(strProtocol2, "addons")
190 || IsProtocolEqual(strProtocol2, "rtsp"))
191 sep = "?;#|";
192 else if(IsProtocolEqual(strProtocol2, "ftp")
193 || IsProtocolEqual(strProtocol2, "ftps"))
194 sep = "?;|";
196 if(sep)
198 size_t iOptions = strURL.find_first_of(sep, iPos);
199 if (iOptions != std::string::npos)
201 // we keep the initial char as it can be any of the above
202 size_t iProto = strURL.find_first_of('|', iOptions);
203 if (iProto != std::string::npos)
205 SetProtocolOptions(strURL.substr(iProto+1));
206 SetOptions(strURL.substr(iOptions,iProto-iOptions));
208 else
209 SetOptions(strURL.substr(iOptions));
210 iEnd = iOptions;
214 size_t iSlash = strURL.find('/', iPos);
215 if(iSlash >= iEnd)
216 iSlash = std::string::npos; // was an invalid slash as it was contained in options
218 // also skip parsing username:password@ for udp/rtp as it not valid
219 // and conflicts with the following example: rtp://sourceip@multicastip
220 size_t iAlphaSign = strURL.find('@', iPos);
221 if (iAlphaSign != std::string::npos && iAlphaSign < iEnd &&
222 (iAlphaSign < iSlash || iSlash == std::string::npos) &&
223 !IsProtocol("udp") && !IsProtocol("rtp"))
225 // username/password found
226 std::string strUserNamePassword = strURL.substr(iPos, iAlphaSign - iPos);
228 // first extract domain, if protocol is smb
229 if (IsProtocol("smb"))
231 size_t iSemiColon = strUserNamePassword.find(';');
233 if (iSemiColon != std::string::npos)
235 m_strDomain = strUserNamePassword.substr(0, iSemiColon);
236 strUserNamePassword.erase(0, iSemiColon + 1);
240 // username:password
241 size_t iColon = strUserNamePassword.find(':');
242 if (iColon != std::string::npos)
244 m_strUserName = strUserNamePassword.substr(0, iColon);
245 m_strPassword = strUserNamePassword.substr(iColon + 1);
247 // username
248 else
250 m_strUserName = std::move(strUserNamePassword);
253 iPos = iAlphaSign + 1;
254 iSlash = strURL.find('/', iAlphaSign);
256 if (iSlash >= iEnd)
257 iSlash = std::string::npos;
260 std::string strHostNameAndPort = strURL.substr(iPos, (iSlash == std::string::npos) ? iEnd - iPos : iSlash - iPos);
261 // check for IPv6 numerical representation inside [].
262 // if [] found, let's store string inside as hostname
263 // and remove that parsed part from strHostNameAndPort
264 size_t iBrk = strHostNameAndPort.rfind(']');
265 if (iBrk != std::string::npos && strHostNameAndPort.starts_with('['))
267 m_strHostName = strHostNameAndPort.substr(1, iBrk-1);
268 strHostNameAndPort.erase(0, iBrk+1);
271 // detect hostname:port/ or just :port/ if previous step found [IPv6] format
272 size_t iColon = strHostNameAndPort.rfind(':');
273 if (iColon != std::string::npos && iColon == strHostNameAndPort.find(':'))
275 if (m_strHostName.empty())
276 m_strHostName = strHostNameAndPort.substr(0, iColon);
277 m_iPort = atoi(strHostNameAndPort.substr(iColon + 1).c_str());
280 // if we still don't have hostname, the strHostNameAndPort substring
281 // is 'just' hostname without :port specification - so use it as is.
282 if (m_strHostName.empty())
283 m_strHostName = std::move(strHostNameAndPort);
285 if (iSlash != std::string::npos)
287 iPos = iSlash + 1;
288 if (iEnd > iPos)
289 m_strFileName = strURL.substr(iPos, iEnd - iPos);
292 if (IsProtocol("musicdb") || IsProtocol("videodb") || IsProtocol("sources") || IsProtocol("pvr"))
294 if (m_strHostName != "" && m_strFileName != "")
296 m_strFileName = StringUtils::Format("{}/{}", m_strHostName, m_strFileName);
297 m_strHostName = "";
299 else
301 if (!m_strHostName.empty() && strURL[iEnd-1]=='/')
302 m_strFileName = m_strHostName + "/";
303 else
304 m_strFileName = m_strHostName;
305 m_strHostName = "";
309 StringUtils::Replace(m_strFileName, '\\', '/');
311 /* update extension + sharename */
312 SetFileName(m_strFileName);
314 /* decode urlencoding on this stuff */
315 if(URIUtils::HasEncodedHostname(*this))
317 m_strHostName = Decode(m_strHostName);
318 SetHostName(m_strHostName);
321 m_strUserName = Decode(m_strUserName);
322 m_strPassword = Decode(m_strPassword);
325 void CURL::SetFileName(std::string strFileName)
327 m_strFileName = std::move(strFileName);
329 size_t slash = m_strFileName.find_last_of(GetDirectorySeparator());
330 size_t period = m_strFileName.find_last_of('.');
331 if(period != std::string::npos && (slash == std::string::npos || period > slash))
332 m_strFileType = m_strFileName.substr(period+1);
333 else
334 m_strFileType = "";
336 slash = m_strFileName.find_first_of(GetDirectorySeparator());
337 if(slash == std::string::npos)
338 m_strShareName = m_strFileName;
339 else
340 m_strShareName = m_strFileName.substr(0, slash);
342 StringUtils::Trim(m_strFileType);
343 StringUtils::ToLower(m_strFileType);
346 void CURL::SetProtocol(std::string strProtocol)
348 m_strProtocol = std::move(strProtocol);
349 StringUtils::ToLower(m_strProtocol);
352 void CURL::SetOptions(std::string strOptions)
354 m_strOptions.clear();
355 m_options.Clear();
356 if( strOptions.length() > 0)
358 if(strOptions[0] == '?' ||
359 strOptions[0] == '#' ||
360 strOptions[0] == ';' ||
361 strOptions.find("xml") != std::string::npos)
363 m_strOptions = std::move(strOptions);
364 m_options.AddOptions(m_strOptions);
366 else
367 CLog::Log(LOGWARNING, "{} - Invalid options specified for url {}", __FUNCTION__, strOptions);
371 void CURL::SetProtocolOptions(std::string strOptions)
373 m_strProtocolOptions.clear();
374 m_protocolOptions.Clear();
375 if (strOptions.length() > 0)
377 if (strOptions[0] == '|')
378 m_strProtocolOptions = std::move(strOptions).substr(1);
379 else
380 m_strProtocolOptions = std::move(strOptions);
381 m_protocolOptions.AddOptions(m_strProtocolOptions);
385 std::string CURL::GetTranslatedProtocol() const
387 if (IsProtocol("shout")
388 || IsProtocol("dav")
389 || IsProtocol("rss"))
390 return "http";
392 if (IsProtocol("davs")
393 || IsProtocol("rsss"))
394 return "https";
396 return GetProtocol();
399 std::string CURL::GetFileNameWithoutPath() const
401 // *.zip and *.rar store the actual zip/rar path in the hostname of the url
402 if ((IsProtocol("rar") ||
403 IsProtocol("zip") ||
404 IsProtocol("xbt") ||
405 IsProtocol("apk")) &&
406 m_strFileName.empty())
407 return URIUtils::GetFileName(m_strHostName);
409 // otherwise, we've already got the filepath, so just grab the filename portion
410 std::string file(m_strFileName);
411 URIUtils::RemoveSlashAtEnd(file);
412 return URIUtils::GetFileName(file);
415 inline
416 void protectIPv6(std::string &hn)
418 if (!hn.empty() && hn.find(':') != hn.rfind(':') && hn.find(':') != std::string::npos)
420 hn = '[' + hn + ']';
424 char CURL::GetDirectorySeparator() const
426 #ifndef TARGET_POSIX
427 //We don't want to use IsLocal here, it can return true
428 //for network protocols that matches localhost or hostname
429 //we only ever want to use \ for win32 local filesystem
430 if ( m_strProtocol.empty() )
431 return '\\';
432 else
433 #endif
434 return '/';
437 std::string CURL::Get() const
439 if (m_strProtocol.empty())
440 return m_strFileName;
442 unsigned int sizeneed = m_strProtocol.length()
443 + m_strDomain.length()
444 + m_strUserName.length()
445 + m_strPassword.length()
446 + m_strHostName.length()
447 + m_strFileName.length()
448 + m_strOptions.length()
449 + m_strProtocolOptions.length()
450 + 10;
452 std::string strURL;
453 strURL.reserve(sizeneed);
455 strURL = GetWithoutOptions();
457 if( !m_strOptions.empty() )
458 strURL += m_strOptions;
460 if (!m_strProtocolOptions.empty())
461 strURL += "|"+m_strProtocolOptions;
463 return strURL;
466 std::string CURL::GetWithoutOptions() const
468 if (m_strProtocol.empty())
469 return m_strFileName;
471 std::string strGet = GetWithoutFilename();
473 // Prevent double slash when concatenating host part and filename part
474 if (!m_strFileName.empty() && (m_strFileName[0] == '/' || m_strFileName[0] == '\\') &&
475 URIUtils::HasSlashAtEnd(strGet) && !(IsProtocol("http") || IsProtocol("https")))
477 URIUtils::RemoveSlashAtEnd(strGet);
480 return strGet + m_strFileName;
483 std::string CURL::GetWithoutUserDetails(bool redact) const
485 std::string strURL;
487 if (IsProtocol("stack"))
489 CFileItemList items;
490 XFILE::CStackDirectory dir;
491 dir.GetDirectory(*this,items);
492 std::vector<std::string> newItems;
493 for (int i=0;i<items.Size();++i)
495 CURL url(items[i]->GetPath());
496 items[i]->SetPath(url.GetWithoutUserDetails(redact));
497 newItems.push_back(items[i]->GetPath());
499 dir.ConstructStackPath(newItems, strURL);
500 return strURL;
503 unsigned int sizeneed = m_strProtocol.length()
504 + m_strHostName.length()
505 + m_strFileName.length()
506 + m_strOptions.length()
507 + m_strProtocolOptions.length()
508 + 10;
510 if (redact && !m_strUserName.empty())
512 sizeneed += sizeof("USERNAME");
513 if (!m_strPassword.empty())
514 sizeneed += sizeof(":PASSWORD@");
515 if (!m_strDomain.empty())
516 sizeneed += sizeof("DOMAIN;");
519 strURL.reserve(sizeneed);
521 if (m_strProtocol.empty())
522 return m_strFileName;
524 strURL = m_strProtocol;
525 strURL += "://";
527 if (redact && !m_strUserName.empty())
529 if (!m_strDomain.empty())
530 strURL += "DOMAIN;";
531 strURL += "USERNAME";
532 if (!m_strPassword.empty())
533 strURL += ":PASSWORD";
534 strURL += "@";
537 if (!m_strHostName.empty())
539 std::string strHostName;
541 if (URIUtils::HasParentInHostname(*this))
542 strHostName = CURL(m_strHostName).GetWithoutUserDetails();
543 else
544 strHostName = m_strHostName;
546 if (URIUtils::HasEncodedHostname(*this))
547 strHostName = Encode(strHostName);
549 if ( HasPort() )
551 protectIPv6(strHostName);
552 strURL += strHostName + StringUtils::Format(":{}", m_iPort);
554 else
555 strURL += strHostName;
557 strURL += "/";
559 strURL += m_strFileName;
561 if( m_strOptions.length() > 0 )
562 strURL += m_strOptions;
563 if( m_strProtocolOptions.length() > 0 )
564 strURL += "|"+m_strProtocolOptions;
566 return strURL;
569 std::string CURL::GetWithoutFilename() const
571 if (m_strProtocol.empty())
572 return "";
574 unsigned int sizeneed = m_strProtocol.length()
575 + m_strDomain.length()
576 + m_strUserName.length()
577 + m_strPassword.length()
578 + m_strHostName.length()
579 + 10;
581 std::string strURL;
582 strURL.reserve(sizeneed);
584 strURL = m_strProtocol;
585 strURL += "://";
587 if (!m_strUserName.empty())
589 if (!m_strDomain.empty())
591 strURL += Encode(m_strDomain);
592 strURL += ";";
594 strURL += Encode(m_strUserName);
595 if (!m_strPassword.empty())
597 strURL += ":";
598 strURL += Encode(m_strPassword);
600 strURL += "@";
603 if (!m_strHostName.empty())
605 std::string hostname;
607 if( URIUtils::HasEncodedHostname(*this) )
608 hostname = Encode(m_strHostName);
609 else
610 hostname = m_strHostName;
612 if (HasPort())
614 protectIPv6(hostname);
615 strURL += hostname + StringUtils::Format(":{}", m_iPort);
617 else
618 strURL += hostname;
620 strURL += "/";
623 return strURL;
626 std::string CURL::GetRedacted() const
628 return GetWithoutUserDetails(true);
631 std::string CURL::GetRedacted(std::string path)
633 return CURL(std::move(path)).GetRedacted();
636 bool CURL::IsLocal() const
638 return (m_strProtocol.empty() || IsLocalHost() || IsProtocol("win-lib"));
641 bool CURL::IsLocalHost() const
643 return CServiceBroker::GetNetwork().IsLocalHost(m_strHostName);
646 bool CURL::IsFileOnly(const std::string &url)
648 return url.find_first_of("/\\") == std::string::npos;
651 bool CURL::IsFullPath(const std::string &url)
653 if (url.size() && url[0] == '/') return true; // /foo/bar.ext
654 if (url.find("://") != std::string::npos) return true; // foo://bar.ext
655 if (url.size() > 1 && url[1] == ':') return true; // c:\\foo\\bar\\bar.ext
656 if (StringUtils::StartsWith(url, "\\\\")) return true; // \\UNC\path\to\file
657 return false;
660 std::string CURL::Decode(std::string_view strURLData)
661 //modified to be more accommodating - if a non hex value follows a % take the characters directly and don't raise an error.
662 // However % characters should really be escaped like any other non safe character (www.rfc-editor.org/rfc/rfc1738.txt)
664 std::string strResult;
666 /* result will always be less than source */
667 strResult.reserve( strURLData.length() );
669 const char* const iterEnd = strURLData.data() + strURLData.size();
670 for (const char* iter = strURLData.data(); iter < iterEnd; ++iter)
672 if (*iter == '+')
673 strResult += ' ';
674 else if (*iter == '%')
676 if (std::distance(iter, iterEnd) >= 3)
678 uint8_t dec_num{};
679 const std::from_chars_result res = std::from_chars(iter + 1, iter + 3, dec_num, 16);
680 if (res.ec != std::errc() || res.ptr != iter + 3)
681 strResult += *iter;
682 else
684 strResult += (char)dec_num;
685 iter += 2;
688 else
689 strResult += *iter;
691 else
692 strResult += *iter;
695 return strResult;
698 std::string CURL::Encode(std::string_view strURLData)
700 std::string strResult;
702 /* wonder what a good value is here is, depends on how often it occurs */
703 strResult.reserve( strURLData.length() * 2 );
705 for (auto kar : strURLData)
707 // Don't URL encode "-_.!()" according to RFC1738
708 //! @todo Update it to "-_.~" after Gotham according to RFC3986
709 if (StringUtils::isasciialphanum(kar) || kar == '-' || kar == '.' || kar == '_' || kar == '!' || kar == '(' || kar == ')')
710 strResult.push_back(kar);
711 else
712 fmt::format_to(std::back_insert_iterator(strResult), "%{:02x}",
713 (unsigned int)((unsigned char)kar));
716 return strResult;
719 bool CURL::IsProtocolEqual(const std::string &protocol, const char *type)
722 NOTE: We're currently using == here as m_strProtocol is assigned as lower-case in SetProtocol(),
723 and we've assumed all other callers are calling with protocol lower-case otherwise.
724 We possibly shouldn't do this (as CURL(foo).Get() != foo, though there are other reasons for this as well)
725 but it handles the requirements of RFC-1738 which allows the scheme to be case-insensitive.
727 if (type)
728 return protocol == type;
729 return false;
732 void CURL::GetOptions(std::map<std::string, std::string> &options) const
734 CUrlOptions::UrlOptions optionsMap = m_options.GetOptions();
735 for (CUrlOptions::UrlOptions::const_iterator option = optionsMap.begin();
736 option != optionsMap.end(); ++option)
737 options[option->first] = option->second.asString();
740 bool CURL::HasOption(const std::string &key) const
742 return m_options.HasOption(key);
745 bool CURL::GetOption(const std::string &key, std::string &value) const
747 CVariant valueObj;
748 if (!m_options.GetOption(key, valueObj))
749 return false;
751 value = valueObj.asString();
752 return true;
755 std::string CURL::GetOption(const std::string &key) const
757 std::string value;
758 if (!GetOption(key, value))
759 return "";
761 return value;
764 void CURL::SetOption(const std::string &key, const std::string &value)
766 m_options.AddOption(key, value);
767 SetOptions(m_options.GetOptionsString(true));
770 void CURL::RemoveOption(const std::string &key)
772 m_options.RemoveOption(key);
773 SetOptions(m_options.GetOptionsString(true));
776 void CURL::GetProtocolOptions(std::map<std::string, std::string> &options) const
778 CUrlOptions::UrlOptions optionsMap = m_protocolOptions.GetOptions();
779 for (CUrlOptions::UrlOptions::const_iterator option = optionsMap.begin();
780 option != optionsMap.end(); ++option)
781 options[option->first] = option->second.asString();
784 bool CURL::HasProtocolOption(const std::string &key) const
786 return m_protocolOptions.HasOption(key);
789 bool CURL::GetProtocolOption(const std::string &key, std::string &value) const
791 CVariant valueObj;
792 if (!m_protocolOptions.GetOption(key, valueObj))
793 return false;
795 value = valueObj.asString();
796 return true;
799 std::string CURL::GetProtocolOption(const std::string &key) const
801 std::string value;
802 if (!GetProtocolOption(key, value))
803 return "";
805 return value;
808 void CURL::SetProtocolOption(const std::string &key, const std::string &value)
810 m_protocolOptions.AddOption(key, value);
811 m_strProtocolOptions = m_protocolOptions.GetOptionsString(false);
814 void CURL::RemoveProtocolOption(const std::string &key)
816 m_protocolOptions.RemoveOption(key);
817 m_strProtocolOptions = m_protocolOptions.GetOptionsString(false);