29#ifndef CPL_VSIL_CURL_CLASS_H_INCLUDED
30#define CPL_VSIL_CURL_CLASS_H_INCLUDED
39#include "cpl_vsil_curl_priv.h"
40#include "cpl_mem_cache.h"
42#include "cpl_curl_priv.h"
45#include <condition_variable>
61#define HAVE_CURLINFO_REDIRECT_URL
63void VSICurlStreamingClearCache(
void);
65struct curl_slist *VSICurlSetOptions(CURL *hCurlHandle,
const char *pszURL,
66 const char *
const *papszOptions);
67struct curl_slist *VSICurlMergeHeaders(
struct curl_slist *poDest,
68 struct curl_slist *poSrcToDestroy);
70struct curl_slist *VSICurlSetContentTypeFromExt(
struct curl_slist *polist,
73struct curl_slist *VSICurlSetCreationHeadersFromOptions(
74 struct curl_slist *headers,
CSLConstList papszOptions,
const char *pszPath);
89 unsigned int nGenerationAuthParameters = 0;
90 ExistStatus eExists = EXIST_UNKNOWN;
94 time_t nExpireTimestampLocal = 0;
95 std::string osRedirectURL{};
96 bool bHasComputedFileSize =
false;
97 bool bIsDirectory =
false;
99 bool bS3LikeRedirect =
false;
105 bool bGotFileList =
false;
106 unsigned int nGenerationAuthParameters = 0;
110struct WriteFuncStruct
112 char *pBuffer =
nullptr;
114 bool bIsHTTP =
false;
115 bool bMultiRange =
false;
120 bool bFoundContentRange =
false;
122 bool bInterruptDownload =
false;
123 bool bDetectRangeDownloadingError =
false;
127 VSICurlReadCbkFunc pfnReadCbk =
nullptr;
128 void *pReadCbkUserData =
nullptr;
129 bool bInterrupted =
false;
134 const GByte *pabyData =
nullptr;
136 size_t nTotalSize = 0;
138 static size_t ReadCallBackBuffer(
char *buffer,
size_t size,
size_t nitems,
141 PutData *poThis =
static_cast<PutData *
>(instream);
142 const size_t nSizeMax = size * nitems;
143 const size_t nSizeToWrite =
144 std::min(nSizeMax, poThis->nTotalSize - poThis->nOff);
145 memcpy(buffer, poThis->pabyData + poThis->nOff, nSizeToWrite);
146 poThis->nOff += nSizeToWrite;
157class VSICurlFilesystemHandlerBase :
public VSIFilesystemHandler
161 struct FilenameOffsetPair
163 std::string filename_;
166 FilenameOffsetPair(
const std::string &filename,
vsi_l_offset offset)
167 : filename_(filename), offset_(offset)
171 bool operator==(
const FilenameOffsetPair &other)
const
173 return filename_ == other.filename_ && offset_ == other.offset_;
177 struct FilenameOffsetPairHasher
179 std::size_t operator()(
const FilenameOffsetPair &k)
const
181 return std::hash<std::string>()(k.filename_) ^
182 std::hash<vsi_l_offset>()(k.offset_);
186 using RegionCacheType = lru11::Cache<
187 FilenameOffsetPair, std::shared_ptr<std::string>, lru11::NullLock,
190 typename std::list<lru11::KeyValuePair<
191 FilenameOffsetPair, std::shared_ptr<std::string>>>::iterator,
192 FilenameOffsetPairHasher>>;
194 std::unique_ptr<RegionCacheType>
195 m_poRegionCacheDoNotUseDirectly{};
197 RegionCacheType *GetRegionCache();
204 lru11::Cache<std::string, bool> oCacheFileProp;
206 int nCachedFilesInDirList = 0;
207 lru11::Cache<std::string, CachedDirList> oCacheDirList;
209 char **ParseHTMLFileList(
const char *pszFilename,
int nMaxFiles,
210 char *pszData,
bool *pbGotFileList);
215 struct RegionInDownload
218 std::condition_variable oCond{};
219 bool bDownloadInProgress =
false;
221 std::string osData{};
224 std::mutex m_oMutex{};
225 std::map<std::string, std::unique_ptr<RegionInDownload>>
226 m_oMapRegionInDownload{};
229 CPLMutex *hMutex =
nullptr;
231 virtual VSICurlHandle *CreateFileHandle(
const char *pszFilename);
232 virtual char **GetFileList(
const char *pszFilename,
int nMaxFiles,
233 bool *pbGotFileList);
235 void RegisterEmptyDir(
const std::string &osDirname);
238 AnalyseS3FileList(
const std::string &osBaseURL,
const char *pszXML,
240 const std::set<std::string> &oSetIgnoredStorageClasses,
243 void AnalyseSwiftFileList(
const std::string &osBaseURL,
244 const std::string &osPrefix,
const char *pszJson,
246 int nMaxFiles,
bool &bIsTruncated,
247 std::string &osNextMarker);
249 static const char *GetOptionsStatic();
251 VSICurlFilesystemHandlerBase();
254 ~VSICurlFilesystemHandlerBase()
override;
256 static bool IsAllowedFilename(
const char *pszFilename);
262 int Stat(
const char *pszFilename,
VSIStatBufL *pStatBuf,
263 int nFlags)
override;
264 int Unlink(
const char *pszFilename)
override;
265 int Rename(
const char *oldpath,
const char *newpath)
override;
266 int Mkdir(
const char *pszDirname,
long nMode)
override;
267 int Rmdir(
const char *pszDirname)
override;
268 char **ReadDirEx(
const char *pszDirname,
int nMaxFiles)
override;
269 char **SiblingFiles(
const char *pszFilename)
override;
271 int HasOptimizedReadMultiRange(
const char * )
override
276 const char *GetActualURL(
const char *pszFilename)
override;
278 const char *GetOptions()
override;
280 char **GetFileMetadata(
const char *pszFilename,
const char *pszDomain,
283 char **ReadDirInternal(
const char *pszDirname,
int nMaxFiles,
284 bool *pbGotFileList);
285 void InvalidateDirContent(
const char *pszDirname);
287 virtual const char *GetDebugKey()
const = 0;
289 virtual std::string GetFSPrefix()
const = 0;
290 virtual bool AllowCachedDataFor(
const char *pszFilename);
292 virtual bool IsLocal(
const char * )
override
298 SupportsSequentialWrite(
const char * ,
304 virtual bool SupportsRandomWrite(
const char * ,
310 std::shared_ptr<std::string> GetRegion(
const char *pszURL,
313 void AddRegion(
const char *pszURL,
vsi_l_offset nFileOffsetStart,
314 size_t nSize,
const char *pData);
316 std::pair<bool, std::string>
317 NotifyStartDownloadRegion(
const std::string &osURL,
319 void NotifyStopDownloadRegion(
const std::string &osURL,
321 const std::string &osData);
323 bool GetCachedFileProp(
const char *pszURL, FileProp &oFileProp);
324 void SetCachedFileProp(
const char *pszURL, FileProp &oFileProp);
325 void InvalidateCachedData(
const char *pszURL);
327 CURLM *GetCurlMultiHandleFor(
const std::string &osURL);
329 virtual void ClearCache();
330 virtual void PartialClearCache(
const char *pszFilename);
332 bool GetCachedDirList(
const char *pszURL, CachedDirList &oCachedDirList);
333 void SetCachedDirList(
const char *pszURL, CachedDirList &oCachedDirList);
334 bool ExistsInCacheDirList(
const std::string &osDirname,
bool *pbIsDir);
336 virtual std::string GetURLFromFilename(
const std::string &osFilename);
339 GetStreamingFilename(
const std::string &osFilename)
const override = 0;
341 static std::set<std::string> GetS3IgnoredStorageClasses();
344class VSICurlFilesystemHandler :
public VSICurlFilesystemHandlerBase
349 VSICurlFilesystemHandler() =
default;
351 const char *GetDebugKey()
const override
356 std::string GetFSPrefix()
const override
362 GetStreamingFilename(
const std::string &osFilename)
const override;
374 VSICurlFilesystemHandlerBase *poFS =
nullptr;
376 bool m_bCached =
true;
378 mutable FileProp oFileProp{};
380 mutable std::mutex m_oMutex{};
381 std::string m_osFilename{};
382 char *m_pszURL =
nullptr;
383 mutable std::string m_osQueryString{};
385 char **m_papszHTTPOptions =
nullptr;
388 int nBlocksToDownload = 1;
390 bool bStopOnInterruptUntilUninstall =
false;
391 bool bInterrupted =
false;
392 VSICurlReadCbkFunc pfnReadCbk =
nullptr;
393 void *pReadCbkUserData =
nullptr;
396 double m_dfRetryDelay = 0.0;
400 void DownloadRegionPostProcess(
const vsi_l_offset startOffset,
401 const int nBlocks,
const char *pBuffer,
409 virtual std::string DownloadRegion(
vsi_l_offset startOffset,
int nBlocks);
411 bool m_bUseHead =
false;
412 bool m_bUseRedirectURLIfNoQueryStringParams =
false;
416 mutable bool m_bPlanetaryComputerURLSigning =
false;
417 mutable std::string m_osPlanetaryComputerCollection{};
418 void ManagePlanetaryComputerSigning()
const;
420 int ReadMultiRangeSingleGet(
int nRanges,
void **ppData,
422 const size_t *panSizes);
423 std::string GetRedirectURLIfValid(
bool &bHasExpired)
const;
425 void UpdateRedirectInfo(CURL *hCurlHandle,
426 const WriteFuncStruct &sWriteFuncHeaderData);
429 struct AdviseReadRange
433 std::condition_variable oCV{};
436 std::vector<GByte> abyData{};
439 std::vector<std::unique_ptr<AdviseReadRange>> m_aoAdviseReadRanges{};
440 std::thread m_oThreadAdviseRead{};
443 virtual struct curl_slist *
444 GetCurlHeaders(
const std::string & ,
445 const struct curl_slist * )
450 virtual bool AllowAutomaticRedirection()
455 virtual bool CanRestartOnError(
const char *,
const char *,
bool)
460 virtual bool UseLimitRangeGetInsteadOfHead()
465 virtual bool IsDirectoryFromExists(
const char * ,
471 virtual void ProcessGetFileSizeResult(
const char * )
475 void SetURL(
const char *pszURL);
477 virtual bool Authenticate(
const char * )
483 VSICurlHandle(VSICurlFilesystemHandlerBase *poFS,
const char *pszFilename,
484 const char *pszURLIn =
nullptr);
485 ~VSICurlHandle()
override;
489 size_t Read(
void *pBuffer,
size_t nSize,
size_t nMemb)
override;
490 int ReadMultiRange(
int nRanges,
void **ppData,
492 const size_t *panSizes)
override;
493 size_t Write(
const void *pBuffer,
size_t nSize,
size_t nMemb)
override;
495 int Flush()
override;
496 int Close()
override;
498 bool HasPRead()
const override
503 size_t PRead(
void *pBuffer,
size_t nSize,
506 void AdviseRead(
int nRanges,
const vsi_l_offset *panOffsets,
507 const size_t *panSizes)
override;
509 size_t GetAdviseReadTotalBytesLimit()
const override;
511 bool IsKnownFileSize()
const
513 return oFileProp.bHasComputedFileSize;
516 vsi_l_offset GetFileSizeOrHeaders(
bool bSetError,
bool bGetHeaders);
520 return GetFileSizeOrHeaders(bSetError,
false);
523 bool Exists(
bool bSetError);
525 bool IsDirectory()
const
527 return oFileProp.bIsDirectory;
532 return oFileProp.nMode;
535 time_t GetMTime()
const
537 return oFileProp.mTime;
545 int InstallReadCbk(VSICurlReadCbkFunc pfnReadCbk,
void *pfnUserData,
546 int bStopOnInterruptUntilUninstall);
547 int UninstallReadCbk();
549 const char *GetURL()
const
559class VSICurlFilesystemHandlerBaseWritable :
public VSICurlFilesystemHandlerBase
564 VSICurlFilesystemHandlerBaseWritable() =
default;
566 virtual VSIVirtualHandleUniquePtr
567 CreateWriteHandle(
const char *pszFilename,
CSLConstList papszOptions) = 0;
573 bool SupportsSequentialWrite(
const char * ,
579 bool SupportsRandomWrite(
const char * ,
587class IVSIS3LikeFSHandler :
public VSICurlFilesystemHandlerBaseWritable
591 virtual int MkdirInternal(
const char *pszDirname,
long nMode,
595 char **GetFileList(
const char *pszFilename,
int nMaxFiles,
596 bool *pbGotFileList)
override;
598 virtual IVSIS3LikeHandleHelper *CreateHandleHelper(
const char *pszURI,
599 bool bAllowNoObject) = 0;
601 virtual int CopyObject(
const char *oldpath,
const char *newpath,
604 int RmdirRecursiveInternal(
const char *pszDirname,
int nBatchSize);
607 IsAllowedHeaderForObjectCreation(
const char * )
612 IVSIS3LikeFSHandler() =
default;
615 int Unlink(
const char *pszFilename)
override;
616 int Mkdir(
const char *pszDirname,
long nMode)
override;
617 int Rmdir(
const char *pszDirname)
override;
618 int Stat(
const char *pszFilename,
VSIStatBufL *pStatBuf,
619 int nFlags)
override;
620 int Rename(
const char *oldpath,
const char *newpath)
override;
622 virtual int CopyFile(
const char *pszSource,
const char *pszTarget,
624 const char *
const *papszOptions,
625 GDALProgressFunc pProgressFunc,
626 void *pProgressData)
override;
628 virtual int DeleteObject(
const char *pszFilename);
630 virtual int *DeleteObjectBatch(
CSLConstList papszFilesOrDirs);
632 bool Sync(
const char *pszSource,
const char *pszTarget,
633 const char *
const *papszOptions, GDALProgressFunc pProgressFunc,
634 void *pProgressData,
char ***ppapszOutputs)
override;
636 VSIDIR *OpenDir(
const char *pszPath,
int nRecurseDepth,
637 const char *
const *papszOptions)
override;
640 virtual bool SupportsParallelMultipartUpload()
const
645 virtual std::string InitiateMultipartUpload(
646 const std::string &osFilename, IVSIS3LikeHandleHelper *poS3HandleHelper,
647 int nMaxRetry,
double dfRetryDelay,
CSLConstList papszOptions);
649 UploadPart(
const std::string &osFilename,
int nPartNumber,
651 const void *pabyBuffer,
size_t nBufferSize,
652 IVSIS3LikeHandleHelper *poS3HandleHelper,
int nMaxRetry,
654 virtual bool CompleteMultipart(
const std::string &osFilename,
655 const std::string &osUploadID,
656 const std::vector<std::string> &aosEtags,
658 IVSIS3LikeHandleHelper *poS3HandleHelper,
659 int nMaxRetry,
double dfRetryDelay);
660 virtual bool AbortMultipart(
const std::string &osFilename,
661 const std::string &osUploadID,
662 IVSIS3LikeHandleHelper *poS3HandleHelper,
663 int nMaxRetry,
double dfRetryDelay);
665 bool AbortPendingUploads(
const char *pszFilename)
override;
672class IVSIS3LikeHandle :
public VSICurlHandle
677 bool UseLimitRangeGetInsteadOfHead()
override
682 bool IsDirectoryFromExists(
const char *pszVerb,
int response_code)
override
685 return response_code == 416 &&
EQUAL(pszVerb,
"GET") &&
686 std::string(m_pszURL).back() ==
'/';
689 void ProcessGetFileSizeResult(
const char *pszContent)
override
691 oFileProp.bIsDirectory =
692 strstr(pszContent,
"ListBucketResult") !=
nullptr;
696 IVSIS3LikeHandle(VSICurlFilesystemHandlerBase *poFSIn,
697 const char *pszFilename,
const char *pszURLIn)
698 : VSICurlHandle(poFSIn, pszFilename, pszURLIn)
702 ~IVSIS3LikeHandle()
override
715 IVSIS3LikeFSHandler *m_poFS =
nullptr;
716 std::string m_osFilename{};
717 IVSIS3LikeHandleHelper *m_poS3HandleHelper =
nullptr;
718 bool m_bUseChunked =
false;
723 int m_nBufferOff = 0;
724 int m_nBufferSize = 0;
725 bool m_bClosed =
false;
726 GByte *m_pabyBuffer =
nullptr;
727 std::string m_osUploadID{};
728 int m_nPartNumber = 0;
729 std::vector<std::string> m_aosEtags{};
730 bool m_bError =
false;
732 CURLM *m_hCurlMulti =
nullptr;
733 CURL *m_hCurl =
nullptr;
734 const void *m_pBuffer =
nullptr;
735 std::string m_osCurlErrBuf{};
736 size_t m_nChunkedBufferOff = 0;
737 size_t m_nChunkedBufferSize = 0;
738 size_t m_nWrittenInPUT = 0;
741 double m_dfRetryDelay = 0.0;
742 WriteFuncStruct m_sWriteFuncHeaderData{};
745 bool DoSinglePartPUT();
747 static size_t ReadCallBackBufferChunked(
char *buffer,
size_t size,
748 size_t nitems,
void *instream);
749 size_t WriteChunked(
const void *pBuffer,
size_t nSize,
size_t nMemb);
750 int FinishChunkedTransfer();
752 void InvalidateParentDirectory();
755 VSIS3WriteHandle(IVSIS3LikeFSHandler *poFS,
const char *pszFilename,
756 IVSIS3LikeHandleHelper *poS3HandleHelper,
bool bUseChunked,
758 ~VSIS3WriteHandle()
override;
762 size_t Read(
void *pBuffer,
size_t nSize,
size_t nMemb)
override;
763 size_t Write(
const void *pBuffer,
size_t nSize,
size_t nMemb)
override;
765 int Close()
override;
769 return m_bUseChunked || m_pabyBuffer !=
nullptr;
782 VSICurlFilesystemHandlerBase *m_poFS =
nullptr;
783 std::string m_osFSPrefix{};
784 std::string m_osFilename{};
787 int m_nBufferOff = 0;
788 int m_nBufferSize = 0;
789 int m_nBufferOffReadCallback = 0;
790 bool m_bClosed =
false;
791 GByte *m_pabyBuffer =
nullptr;
792 bool m_bError =
false;
794 static size_t ReadCallBackBuffer(
char *buffer,
size_t size,
size_t nitems,
796 virtual bool Send(
bool bIsLastBlock) = 0;
799 VSIAppendWriteHandle(VSICurlFilesystemHandlerBase *poFS,
800 const char *pszFSPrefix,
const char *pszFilename,
802 virtual ~VSIAppendWriteHandle();
806 size_t Read(
void *pBuffer,
size_t nSize,
size_t nMemb)
override;
807 size_t Write(
const void *pBuffer,
size_t nSize,
size_t nMemb)
override;
809 int Close()
override;
813 return m_pabyBuffer !=
nullptr;
821struct VSIDIRWithMissingDirSynthesis :
public VSIDIR
823 std::vector<std::unique_ptr<VSIDIREntry>> aoEntries{};
826 std::vector<std::string> m_aosSubpathsStack{};
828 void SynthetizeMissingDirectories(
const std::string &osCurSubdir,
829 bool bAddEntryForThisSubdir);
836struct CurlRequestHelper
838 WriteFuncStruct sWriteFuncData{};
839 WriteFuncStruct sWriteFuncHeaderData{};
840 char szCurlErrBuf[CURL_ERROR_SIZE + 1] = {};
843 ~CurlRequestHelper();
844 long perform(CURL *hCurlHandle,
845 struct curl_slist *headers,
846 VSICurlFilesystemHandlerBase *poFS,
847 IVSIS3LikeHandleHelper *poS3HandleHelper);
854class NetworkStatisticsLogger
856 static int gnEnabled;
857 static NetworkStatisticsLogger gInstance;
859 NetworkStatisticsLogger() =
default;
861 std::mutex m_mutex{};
870 GIntBig nGETDownloadedBytes = 0;
872 GIntBig nPOSTDownloadedBytes = 0;
873 GIntBig nPOSTUploadedBytes = 0;
876 enum class ContextPathType
883 struct ContextPathItem
885 ContextPathType eType;
888 ContextPathItem(ContextPathType eTypeIn,
const std::string &osNameIn)
889 : eType(eTypeIn), osName(osNameIn)
893 bool operator<(
const ContextPathItem &other)
const
895 if (
static_cast<int>(eType) <
static_cast<int>(other.eType))
897 if (
static_cast<int>(eType) >
static_cast<int>(other.eType))
899 return osName < other.osName;
906 std::map<ContextPathItem, Stats> children{};
914 std::map<GIntBig, std::vector<ContextPathItem>>
915 m_mapThreadIdToContextPath{};
917 static void ReadEnabled();
919 std::vector<Counters *> GetCountersForContext();
922 static inline bool IsEnabled()
928 return gnEnabled == TRUE;
931 static void EnterFileSystem(
const char *pszName);
933 static void LeaveFileSystem();
935 static void EnterFile(
const char *pszName);
937 static void LeaveFile();
939 static void EnterAction(
const char *pszName);
941 static void LeaveAction();
943 static void LogHEAD();
945 static void LogGET(
size_t nDownloadedBytes);
947 static void LogPUT(
size_t nUploadedBytes);
949 static void LogPOST(
size_t nUploadedBytes,
size_t nDownloadedBytes);
951 static void LogDELETE();
955 static std::string GetReportAsSerializedJSON();
958struct NetworkStatisticsFileSystem
960 inline explicit NetworkStatisticsFileSystem(
const char *pszName)
962 NetworkStatisticsLogger::EnterFileSystem(pszName);
965 inline ~NetworkStatisticsFileSystem()
967 NetworkStatisticsLogger::LeaveFileSystem();
971struct NetworkStatisticsFile
973 inline explicit NetworkStatisticsFile(
const char *pszName)
975 NetworkStatisticsLogger::EnterFile(pszName);
978 inline ~NetworkStatisticsFile()
980 NetworkStatisticsLogger::LeaveFile();
984struct NetworkStatisticsAction
986 inline explicit NetworkStatisticsAction(
const char *pszName)
988 NetworkStatisticsLogger::EnterAction(pszName);
991 inline ~NetworkStatisticsAction()
993 NetworkStatisticsLogger::LeaveAction();
999int VSICURLGetDownloadChunkSize();
1001void VSICURLInitWriteFuncStruct(cpl::WriteFuncStruct *psStruct,
VSILFILE *fp,
1002 VSICurlReadCbkFunc pfnReadCbk,
1003 void *pReadCbkUserData);
1004size_t VSICurlHandleWriteFunc(
void *buffer,
size_t count,
size_t nmemb,
1006void VSICURLMultiPerform(CURLM *hCurlMultiHandle, CURL *hEasyHandle =
nullptr);
1007void VSICURLResetHeaderAndWriterFunctions(CURL *hCurlHandle);
1009int VSICurlParseUnixPermissions(
const char *pszPermissions);
1012bool VSICURLGetCachedFileProp(
const char *pszURL, cpl::FileProp &oFileProp);
1013void VSICURLSetCachedFileProp(
const char *pszURL, cpl::FileProp &oFileProp);
1014void VSICURLInvalidateCachedFileProp(
const char *pszURL);
1015void VSICURLInvalidateCachedFilePropPrefix(
const char *pszURL);
1016void VSICURLDestroyCacheFileProp();
1018void VSICURLMultiCleanup(CURLM *hCurlMultiHandle);
The CPLJSONArray class holds JSON object from CPLJSONDocument.
Definition: cpl_json.h:57
String list class designed around our use of C "char**" string lists.
Definition: cpl_string.h:449
Interface for read and write JSON documents.
Core portability definitions for CPL.
#define EQUAL(a, b)
Alias for strcasecmp() == 0.
Definition: cpl_port.h:551
#define CPL_DISALLOW_COPY_ASSIGN(ClassName)
Helper to remove the copy and assignment constructors so that the compiler will not generate the defa...
Definition: cpl_port.h:1042
char ** CSLConstList
Type of a constant null-terminated list of nul terminated strings.
Definition: cpl_port.h:1183
unsigned char GByte
Unsigned byte type.
Definition: cpl_port.h:185
long long GIntBig
Large signed integer type (generally 64-bit integer type).
Definition: cpl_port.h:215
Various convenience functions for working with strings and string lists.
#define VSIStatBufL
Type for VSIStatL()
Definition: cpl_vsi.h:203
#define VSI_L_OFFSET_MAX
Maximum value for a file offset.
Definition: cpl_vsi.h:148
struct VSIDIR VSIDIR
Opaque type for a directory iterator.
Definition: cpl_vsi.h:402
GUIntBig vsi_l_offset
Type for a file offset.
Definition: cpl_vsi.h:146
Virtual file handle.
Definition: cpl_vsi_virtual.h:63