GDAL
cpl_vsil_curl_class.h
1 /******************************************************************************
2  *
3  * Project: CPL - Common Portability Library
4  * Purpose: Declarations for /vsicurl/ and related file systems
5  * Author: Even Rouault, even.rouault at spatialys.com
6  *
7  ******************************************************************************
8  * Copyright (c) 2010-2018, Even Rouault <even.rouault at spatialys.com>
9  *
10  * Permission is hereby granted, free of charge, to any person obtaining a
11  * copy of this software and associated documentation files (the "Software"),
12  * to deal in the Software without restriction, including without limitation
13  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
14  * and/or sell copies of the Software, and to permit persons to whom the
15  * Software is furnished to do so, subject to the following conditions:
16  *
17  * The above copyright notice and this permission notice shall be included
18  * in all copies or substantial portions of the Software.
19  *
20  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
21  * OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
22  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
23  * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
24  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
25  * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
26  * DEALINGS IN THE SOFTWARE.
27  ****************************************************************************/
28 
29 #ifndef CPL_VSIL_CURL_CLASS_H_INCLUDED
30 #define CPL_VSIL_CURL_CLASS_H_INCLUDED
31 
32 #ifdef HAVE_CURL
33 
34 #include "cpl_aws.h"
35 #include "cpl_azure.h"
36 #include "cpl_port.h"
37 #include "cpl_json.h"
38 #include "cpl_string.h"
39 #include "cpl_vsil_curl_priv.h"
40 #include "cpl_mem_cache.h"
41 
42 #include "cpl_curl_priv.h"
43 
44 #include <algorithm>
45 #include <condition_variable>
46 #include <set>
47 #include <map>
48 #include <memory>
49 #include <mutex>
50 #include <thread>
51 
52 // To avoid aliasing to CopyFile to CopyFileA on Windows
53 #ifdef CopyFile
54 #undef CopyFile
55 #endif
56 
58 
59 // Leave it for backward compatibility, but deprecate.
60 #define HAVE_CURLINFO_REDIRECT_URL
61 
62 void VSICurlStreamingClearCache(void); // from cpl_vsil_curl_streaming.cpp
63 
64 struct curl_slist *VSICurlSetOptions(CURL *hCurlHandle, const char *pszURL,
65  const char *const *papszOptions);
66 struct curl_slist *VSICurlMergeHeaders(struct curl_slist *poDest,
67  struct curl_slist *poSrcToDestroy);
68 
69 struct curl_slist *VSICurlSetContentTypeFromExt(struct curl_slist *polist,
70  const char *pszPath);
71 
72 struct curl_slist *VSICurlSetCreationHeadersFromOptions(
73  struct curl_slist *headers, CSLConstList papszOptions, const char *pszPath);
74 
75 namespace cpl
76 {
77 
78 typedef enum
79 {
80  EXIST_UNKNOWN = -1,
81  EXIST_NO,
82  EXIST_YES,
83 } ExistStatus;
84 
85 class FileProp
86 {
87  public:
88  unsigned int nGenerationAuthParameters = 0;
89  ExistStatus eExists = EXIST_UNKNOWN;
90  vsi_l_offset fileSize = 0;
91  time_t mTime = 0;
92  time_t nExpireTimestampLocal = 0;
93  CPLString osRedirectURL{};
94  bool bHasComputedFileSize = false;
95  bool bIsDirectory = false;
96  int nMode = 0; // st_mode member of struct stat
97  bool bS3LikeRedirect = false;
98  CPLString ETag{};
99 };
100 
101 struct CachedDirList
102 {
103  bool bGotFileList = false;
104  unsigned int nGenerationAuthParameters = 0;
105  CPLStringList oFileList{}; /* only file name without path */
106 };
107 
108 struct WriteFuncStruct
109 {
110  char *pBuffer = nullptr;
111  size_t nSize = 0;
112  bool bIsHTTP = false;
113  bool bMultiRange = false;
114  vsi_l_offset nStartOffset = 0;
115  vsi_l_offset nEndOffset = 0;
116  int nHTTPCode = 0;
117  vsi_l_offset nContentLength = 0;
118  bool bFoundContentRange = false;
119  bool bError = false;
120  bool bInterruptDownload = false;
121  bool bDetectRangeDownloadingError = false;
122  GIntBig nTimestampDate = 0; // Corresponds to Date: header field
123 
124  VSILFILE *fp = nullptr;
125  VSICurlReadCbkFunc pfnReadCbk = nullptr;
126  void *pReadCbkUserData = nullptr;
127  bool bInterrupted = false;
128 
129 #if !CURL_AT_LEAST_VERSION(7, 54, 0)
130  // Workaround to ignore extra HTTP response headers from
131  // proxies in older versions of curl.
132  // CURLOPT_SUPPRESS_CONNECT_HEADERS fixes this
133  bool bIsProxyConnectHeader = false;
134 #endif
135 };
136 
137 struct PutData
138 {
139  const GByte *pabyData = nullptr;
140  size_t nOff = 0;
141  size_t nTotalSize = 0;
142 
143  static size_t ReadCallBackBuffer(char *buffer, size_t size, size_t nitems,
144  void *instream)
145  {
146  PutData *poThis = static_cast<PutData *>(instream);
147  const size_t nSizeMax = size * nitems;
148  const size_t nSizeToWrite =
149  std::min(nSizeMax, poThis->nTotalSize - poThis->nOff);
150  memcpy(buffer, poThis->pabyData + poThis->nOff, nSizeToWrite);
151  poThis->nOff += nSizeToWrite;
152  return nSizeToWrite;
153  }
154 };
155 
156 /************************************************************************/
157 /* VSICurlFilesystemHandler */
158 /************************************************************************/
159 
160 class VSICurlHandle;
161 
162 class VSICurlFilesystemHandlerBase : public VSIFilesystemHandler
163 {
164  CPL_DISALLOW_COPY_ASSIGN(VSICurlFilesystemHandlerBase)
165 
166  struct FilenameOffsetPair
167  {
168  std::string filename_;
169  vsi_l_offset offset_;
170 
171  FilenameOffsetPair(const std::string &filename, vsi_l_offset offset)
172  : filename_(filename), offset_(offset)
173  {
174  }
175 
176  bool operator==(const FilenameOffsetPair &other) const
177  {
178  return filename_ == other.filename_ && offset_ == other.offset_;
179  }
180  };
181  struct FilenameOffsetPairHasher
182  {
183  std::size_t operator()(const FilenameOffsetPair &k) const
184  {
185  return std::hash<std::string>()(k.filename_) ^
186  std::hash<vsi_l_offset>()(k.offset_);
187  }
188  };
189 
190  using RegionCacheType = lru11::Cache<
191  FilenameOffsetPair, std::shared_ptr<std::string>, lru11::NullLock,
192  std::unordered_map<
193  FilenameOffsetPair,
194  typename std::list<lru11::KeyValuePair<
195  FilenameOffsetPair, std::shared_ptr<std::string>>>::iterator,
196  FilenameOffsetPairHasher>>;
197 
198  std::unique_ptr<RegionCacheType>
199  m_poRegionCacheDoNotUseDirectly{}; // do not access directly. Use
200  // GetRegionCache();
201  RegionCacheType *GetRegionCache();
202 
203  // LRU cache that just keeps in memory if this file system handler is
204  // spposed to know the file properties of a file. The actual cache is a
205  // shared one among all network file systems.
206  // The aim of that design is that invalidating /vsis3/foo results in
207  // /vsis3_streaming/foo to be invalidated as well.
208  lru11::Cache<std::string, bool> oCacheFileProp;
209 
210  int nCachedFilesInDirList = 0;
211  lru11::Cache<std::string, CachedDirList> oCacheDirList;
212 
213  char **ParseHTMLFileList(const char *pszFilename, int nMaxFiles,
214  char *pszData, bool *pbGotFileList);
215 
216  protected:
217  CPLMutex *hMutex = nullptr;
218 
219  virtual VSICurlHandle *CreateFileHandle(const char *pszFilename);
220  virtual char **GetFileList(const char *pszFilename, int nMaxFiles,
221  bool *pbGotFileList);
222 
223  void RegisterEmptyDir(const CPLString &osDirname);
224 
225  bool
226  AnalyseS3FileList(const CPLString &osBaseURL, const char *pszXML,
227  CPLStringList &osFileList, int nMaxFiles,
228  const std::set<std::string> &oSetIgnoredStorageClasses,
229  bool &bIsTruncated);
230 
231  void AnalyseSwiftFileList(const CPLString &osBaseURL,
232  const CPLString &osPrefix, const char *pszJson,
233  CPLStringList &osFileList, int nMaxFilesThisQuery,
234  int nMaxFiles, bool &bIsTruncated,
235  CPLString &osNextMarker);
236 
237  static const char *GetOptionsStatic();
238 
239  static bool IsAllowedFilename(const char *pszFilename);
240 
241  VSICurlFilesystemHandlerBase();
242 
243  public:
244  ~VSICurlFilesystemHandlerBase() override;
245 
246  VSIVirtualHandle *Open(const char *pszFilename, const char *pszAccess,
247  bool bSetError,
248  CSLConstList /* papszOptions */) override;
249 
250  int Stat(const char *pszFilename, VSIStatBufL *pStatBuf,
251  int nFlags) override;
252  int Unlink(const char *pszFilename) override;
253  int Rename(const char *oldpath, const char *newpath) override;
254  int Mkdir(const char *pszDirname, long nMode) override;
255  int Rmdir(const char *pszDirname) override;
256  char **ReadDir(const char *pszDirname) override
257  {
258  return ReadDirEx(pszDirname, 0);
259  }
260  char **ReadDirEx(const char *pszDirname, int nMaxFiles) override;
261  char **SiblingFiles(const char *pszFilename) override;
262 
263  int HasOptimizedReadMultiRange(const char * /* pszPath */) override
264  {
265  return true;
266  }
267 
268  const char *GetActualURL(const char *pszFilename) override;
269 
270  const char *GetOptions() override;
271 
272  char **GetFileMetadata(const char *pszFilename, const char *pszDomain,
273  CSLConstList papszOptions) override;
274 
275  char **ReadDirInternal(const char *pszDirname, int nMaxFiles,
276  bool *pbGotFileList);
277  void InvalidateDirContent(const char *pszDirname);
278 
279  virtual const char *GetDebugKey() const = 0;
280 
281  virtual CPLString GetFSPrefix() const = 0;
282  virtual bool AllowCachedDataFor(const char *pszFilename);
283 
284  virtual bool IsLocal(const char * /* pszPath */) override
285  {
286  return false;
287  }
288  virtual bool
289  SupportsSequentialWrite(const char * /* pszPath */,
290  bool /* bAllowLocalTempFile */) override
291  {
292  return false;
293  }
294  virtual bool SupportsRandomWrite(const char * /* pszPath */,
295  bool /* bAllowLocalTempFile */) override
296  {
297  return false;
298  }
299 
300  std::shared_ptr<std::string> GetRegion(const char *pszURL,
301  vsi_l_offset nFileOffsetStart);
302 
303  void AddRegion(const char *pszURL, vsi_l_offset nFileOffsetStart,
304  size_t nSize, const char *pData);
305 
306  bool GetCachedFileProp(const char *pszURL, FileProp &oFileProp);
307  void SetCachedFileProp(const char *pszURL, FileProp &oFileProp);
308  void InvalidateCachedData(const char *pszURL);
309 
310  CURLM *GetCurlMultiHandleFor(const CPLString &osURL);
311 
312  virtual void ClearCache();
313  virtual void PartialClearCache(const char *pszFilename);
314 
315  bool GetCachedDirList(const char *pszURL, CachedDirList &oCachedDirList);
316  void SetCachedDirList(const char *pszURL, CachedDirList &oCachedDirList);
317  bool ExistsInCacheDirList(const CPLString &osDirname, bool *pbIsDir);
318 
319  virtual CPLString GetURLFromFilename(const CPLString &osFilename);
320 
321  std::string
322  GetStreamingFilename(const std::string &osFilename) const override = 0;
323 
324  static std::set<std::string> GetS3IgnoredStorageClasses();
325 };
326 
327 class VSICurlFilesystemHandler : public VSICurlFilesystemHandlerBase
328 {
329  CPL_DISALLOW_COPY_ASSIGN(VSICurlFilesystemHandler)
330 
331  public:
332  VSICurlFilesystemHandler() = default;
333 
334  const char *GetDebugKey() const override
335  {
336  return "VSICURL";
337  }
338 
339  CPLString GetFSPrefix() const override
340  {
341  return "/vsicurl/";
342  }
343 
344  std::string
345  GetStreamingFilename(const std::string &osFilename) const override;
346 };
347 
348 /************************************************************************/
349 /* VSICurlHandle */
350 /************************************************************************/
351 
352 class VSICurlHandle : public VSIVirtualHandle
353 {
354  CPL_DISALLOW_COPY_ASSIGN(VSICurlHandle)
355 
356  protected:
357  VSICurlFilesystemHandlerBase *poFS = nullptr;
358 
359  bool m_bCached = true;
360 
361  mutable FileProp oFileProp{};
362 
363  mutable std::mutex m_oMutex{};
364  CPLString m_osFilename{}; // e.g "/vsicurl/http://example.com/foo"
365  char *m_pszURL = nullptr; // e.g "http://example.com/foo"
366  mutable std::string m_osQueryString{}; // e.g. an Azure SAS
367 
368  char **m_papszHTTPOptions = nullptr;
369 
370  vsi_l_offset lastDownloadedOffset = VSI_L_OFFSET_MAX;
371  int nBlocksToDownload = 1;
372 
373  bool bStopOnInterruptUntilUninstall = false;
374  bool bInterrupted = false;
375  VSICurlReadCbkFunc pfnReadCbk = nullptr;
376  void *pReadCbkUserData = nullptr;
377 
378  int m_nMaxRetry = 0;
379  double m_dfRetryDelay = 0.0;
380 
381  CPLStringList m_aosHeaders{};
382 
383  void DownloadRegionPostProcess(const vsi_l_offset startOffset,
384  const int nBlocks, const char *pBuffer,
385  size_t nSize);
386 
387  private:
388  vsi_l_offset curOffset = 0;
389 
390  bool bEOF = false;
391 
392  virtual std::string DownloadRegion(vsi_l_offset startOffset, int nBlocks);
393 
394  bool m_bUseHead = false;
395  bool m_bUseRedirectURLIfNoQueryStringParams = false;
396 
397  // Specific to Planetary Computer signing:
398  // https://planetarycomputer.microsoft.com/docs/concepts/sas/
399  mutable bool m_bPlanetaryComputerURLSigning = false;
400  mutable std::string m_osPlanetaryComputerCollection{};
401  void ManagePlanetaryComputerSigning() const;
402 
403  int ReadMultiRangeSingleGet(int nRanges, void **ppData,
404  const vsi_l_offset *panOffsets,
405  const size_t *panSizes);
406  CPLString GetRedirectURLIfValid(bool &bHasExpired) const;
407 
408  void UpdateRedirectInfo(CURL *hCurlHandle,
409  const WriteFuncStruct &sWriteFuncHeaderData);
410 
411  // Used by AdviseRead()
412  struct AdviseReadRange
413  {
414  bool bDone = false;
415  std::mutex oMutex{};
416  std::condition_variable oCV{};
417  vsi_l_offset nStartOffset = 0;
418  size_t nSize = 0;
419  std::vector<GByte> abyData{};
420  };
421  std::vector<std::unique_ptr<AdviseReadRange>> m_aoAdviseReadRanges{};
422  std::thread m_oThreadAdviseRead{};
423 
424  protected:
425  virtual struct curl_slist *
426  GetCurlHeaders(const CPLString & /*osVerb*/,
427  const struct curl_slist * /* psExistingHeaders */)
428  {
429  return nullptr;
430  }
431  virtual bool AllowAutomaticRedirection()
432  {
433  return true;
434  }
435  virtual bool CanRestartOnError(const char *, const char *, bool)
436  {
437  return false;
438  }
439  virtual bool UseLimitRangeGetInsteadOfHead()
440  {
441  return false;
442  }
443  virtual bool IsDirectoryFromExists(const char * /*pszVerb*/,
444  int /*response_code*/)
445  {
446  return false;
447  }
448  virtual void ProcessGetFileSizeResult(const char * /* pszContent */)
449  {
450  }
451  void SetURL(const char *pszURL);
452  virtual bool Authenticate(const char * /* pszFilename */)
453  {
454  return false;
455  }
456 
457  public:
458  VSICurlHandle(VSICurlFilesystemHandlerBase *poFS, const char *pszFilename,
459  const char *pszURLIn = nullptr);
460  ~VSICurlHandle() override;
461 
462  int Seek(vsi_l_offset nOffset, int nWhence) override;
463  vsi_l_offset Tell() override;
464  size_t Read(void *pBuffer, size_t nSize, size_t nMemb) override;
465  int ReadMultiRange(int nRanges, void **ppData,
466  const vsi_l_offset *panOffsets,
467  const size_t *panSizes) override;
468  size_t Write(const void *pBuffer, size_t nSize, size_t nMemb) override;
469  int Eof() override;
470  int Flush() override;
471  int Close() override;
472 
473  bool HasPRead() const override
474  {
475  return true;
476  }
477  size_t PRead(void *pBuffer, size_t nSize,
478  vsi_l_offset nOffset) const override;
479 
480  void AdviseRead(int nRanges, const vsi_l_offset *panOffsets,
481  const size_t *panSizes) override;
482 
483  bool IsKnownFileSize() const
484  {
485  return oFileProp.bHasComputedFileSize;
486  }
487  vsi_l_offset GetFileSizeOrHeaders(bool bSetError, bool bGetHeaders);
488  virtual vsi_l_offset GetFileSize(bool bSetError)
489  {
490  return GetFileSizeOrHeaders(bSetError, false);
491  }
492  bool Exists(bool bSetError);
493  bool IsDirectory() const
494  {
495  return oFileProp.bIsDirectory;
496  }
497  int GetMode() const
498  {
499  return oFileProp.nMode;
500  }
501  time_t GetMTime() const
502  {
503  return oFileProp.mTime;
504  }
505  const CPLStringList &GetHeaders()
506  {
507  return m_aosHeaders;
508  }
509 
510  int InstallReadCbk(VSICurlReadCbkFunc pfnReadCbk, void *pfnUserData,
511  int bStopOnInterruptUntilUninstall);
512  int UninstallReadCbk();
513 
514  const char *GetURL() const
515  {
516  return m_pszURL;
517  }
518 };
519 
520 /************************************************************************/
521 /* IVSIS3LikeFSHandler */
522 /************************************************************************/
523 
524 class IVSIS3LikeFSHandler : public VSICurlFilesystemHandlerBase
525 {
526  CPL_DISALLOW_COPY_ASSIGN(IVSIS3LikeFSHandler)
527 
528  virtual int MkdirInternal(const char *pszDirname, long nMode,
529  bool bDoStatCheck);
530 
531  protected:
532  char **GetFileList(const char *pszFilename, int nMaxFiles,
533  bool *pbGotFileList) override;
534 
535  virtual IVSIS3LikeHandleHelper *CreateHandleHelper(const char *pszURI,
536  bool bAllowNoObject) = 0;
537 
538  virtual int CopyObject(const char *oldpath, const char *newpath,
539  CSLConstList papszMetadata);
540 
541  int RmdirRecursiveInternal(const char *pszDirname, int nBatchSize);
542 
543  virtual bool
544  IsAllowedHeaderForObjectCreation(const char * /* pszHeaderName */)
545  {
546  return false;
547  }
548 
549  IVSIS3LikeFSHandler() = default;
550 
551  public:
552  int Unlink(const char *pszFilename) override;
553  int Mkdir(const char *pszDirname, long nMode) override;
554  int Rmdir(const char *pszDirname) override;
555  int Stat(const char *pszFilename, VSIStatBufL *pStatBuf,
556  int nFlags) override;
557  int Rename(const char *oldpath, const char *newpath) override;
558 
559  virtual int CopyFile(const char *pszSource, const char *pszTarget,
560  VSILFILE *fpSource, vsi_l_offset nSourceSize,
561  const char *const *papszOptions,
562  GDALProgressFunc pProgressFunc,
563  void *pProgressData) override;
564 
565  virtual int DeleteObject(const char *pszFilename);
566 
567  virtual void UpdateMapFromHandle(IVSIS3LikeHandleHelper *)
568  {
569  }
570  virtual void UpdateHandleFromMap(IVSIS3LikeHandleHelper *)
571  {
572  }
573 
574  bool Sync(const char *pszSource, const char *pszTarget,
575  const char *const *papszOptions, GDALProgressFunc pProgressFunc,
576  void *pProgressData, char ***ppapszOutputs) override;
577 
578  VSIDIR *OpenDir(const char *pszPath, int nRecurseDepth,
579  const char *const *papszOptions) override;
580 
581  // Multipart upload
582  virtual bool SupportsParallelMultipartUpload() const
583  {
584  return false;
585  }
586 
587  virtual CPLString InitiateMultipartUpload(
588  const std::string &osFilename, IVSIS3LikeHandleHelper *poS3HandleHelper,
589  int nMaxRetry, double dfRetryDelay, CSLConstList papszOptions);
590  virtual CPLString UploadPart(const CPLString &osFilename, int nPartNumber,
591  const std::string &osUploadID,
592  vsi_l_offset nPosition, const void *pabyBuffer,
593  size_t nBufferSize,
594  IVSIS3LikeHandleHelper *poS3HandleHelper,
595  int nMaxRetry, double dfRetryDelay,
596  CSLConstList papszOptions);
597  virtual bool CompleteMultipart(const CPLString &osFilename,
598  const CPLString &osUploadID,
599  const std::vector<CPLString> &aosEtags,
600  vsi_l_offset nTotalSize,
601  IVSIS3LikeHandleHelper *poS3HandleHelper,
602  int nMaxRetry, double dfRetryDelay);
603  virtual bool AbortMultipart(const CPLString &osFilename,
604  const CPLString &osUploadID,
605  IVSIS3LikeHandleHelper *poS3HandleHelper,
606  int nMaxRetry, double dfRetryDelay);
607 
608  bool AbortPendingUploads(const char *pszFilename) override;
609 };
610 
611 /************************************************************************/
612 /* IVSIS3LikeHandle */
613 /************************************************************************/
614 
615 class IVSIS3LikeHandle : public VSICurlHandle
616 {
617  CPL_DISALLOW_COPY_ASSIGN(IVSIS3LikeHandle)
618 
619  protected:
620  bool UseLimitRangeGetInsteadOfHead() override
621  {
622  return true;
623  }
624  bool IsDirectoryFromExists(const char *pszVerb, int response_code) override
625  {
626  // A bit dirty, but on S3, a GET on a existing directory returns a 416
627  return response_code == 416 && EQUAL(pszVerb, "GET") &&
628  CPLString(m_pszURL).back() == '/';
629  }
630  void ProcessGetFileSizeResult(const char *pszContent) override
631  {
632  oFileProp.bIsDirectory =
633  strstr(pszContent, "ListBucketResult") != nullptr;
634  }
635 
636  public:
637  IVSIS3LikeHandle(VSICurlFilesystemHandlerBase *poFSIn,
638  const char *pszFilename, const char *pszURLIn)
639  : VSICurlHandle(poFSIn, pszFilename, pszURLIn)
640  {
641  }
642  ~IVSIS3LikeHandle() override
643  {
644  }
645 };
646 
647 /************************************************************************/
648 /* VSIS3WriteHandle */
649 /************************************************************************/
650 
651 class VSIS3WriteHandle final : public VSIVirtualHandle
652 {
653  CPL_DISALLOW_COPY_ASSIGN(VSIS3WriteHandle)
654 
655  IVSIS3LikeFSHandler *m_poFS = nullptr;
656  CPLString m_osFilename{};
657  IVSIS3LikeHandleHelper *m_poS3HandleHelper = nullptr;
658  bool m_bUseChunked = false;
659  CPLStringList m_aosOptions{};
660  CPLStringList m_aosHTTPOptions{};
661 
662  vsi_l_offset m_nCurOffset = 0;
663  int m_nBufferOff = 0;
664  int m_nBufferSize = 0;
665  bool m_bClosed = false;
666  GByte *m_pabyBuffer = nullptr;
667  CPLString m_osUploadID{};
668  int m_nPartNumber = 0;
669  std::vector<CPLString> m_aosEtags{};
670  bool m_bError = false;
671 
672  CURLM *m_hCurlMulti = nullptr;
673  CURL *m_hCurl = nullptr;
674  const void *m_pBuffer = nullptr;
675  CPLString m_osCurlErrBuf{};
676  size_t m_nChunkedBufferOff = 0;
677  size_t m_nChunkedBufferSize = 0;
678  size_t m_nWrittenInPUT = 0;
679 
680  int m_nMaxRetry = 0;
681  double m_dfRetryDelay = 0.0;
682  WriteFuncStruct m_sWriteFuncHeaderData{};
683 
684  bool UploadPart();
685  bool DoSinglePartPUT();
686 
687  static size_t ReadCallBackBufferChunked(char *buffer, size_t size,
688  size_t nitems, void *instream);
689  size_t WriteChunked(const void *pBuffer, size_t nSize, size_t nMemb);
690  int FinishChunkedTransfer();
691 
692  void InvalidateParentDirectory();
693 
694  public:
695  VSIS3WriteHandle(IVSIS3LikeFSHandler *poFS, const char *pszFilename,
696  IVSIS3LikeHandleHelper *poS3HandleHelper, bool bUseChunked,
697  CSLConstList papszOptions);
698  ~VSIS3WriteHandle() override;
699 
700  int Seek(vsi_l_offset nOffset, int nWhence) override;
701  vsi_l_offset Tell() override;
702  size_t Read(void *pBuffer, size_t nSize, size_t nMemb) override;
703  size_t Write(const void *pBuffer, size_t nSize, size_t nMemb) override;
704  int Eof() override;
705  int Close() override;
706 
707  bool IsOK()
708  {
709  return m_bUseChunked || m_pabyBuffer != nullptr;
710  }
711 };
712 
713 /************************************************************************/
714 /* VSIAppendWriteHandle */
715 /************************************************************************/
716 
717 class VSIAppendWriteHandle : public VSIVirtualHandle
718 {
719  CPL_DISALLOW_COPY_ASSIGN(VSIAppendWriteHandle)
720 
721  protected:
722  VSICurlFilesystemHandlerBase *m_poFS = nullptr;
723  CPLString m_osFSPrefix{};
724  CPLString m_osFilename{};
725 
726  vsi_l_offset m_nCurOffset = 0;
727  int m_nBufferOff = 0;
728  int m_nBufferSize = 0;
729  int m_nBufferOffReadCallback = 0;
730  bool m_bClosed = false;
731  GByte *m_pabyBuffer = nullptr;
732  bool m_bError = false;
733 
734  static size_t ReadCallBackBuffer(char *buffer, size_t size, size_t nitems,
735  void *instream);
736  virtual bool Send(bool bIsLastBlock) = 0;
737 
738  public:
739  VSIAppendWriteHandle(VSICurlFilesystemHandlerBase *poFS,
740  const char *pszFSPrefix, const char *pszFilename,
741  int nChunkSize);
742  virtual ~VSIAppendWriteHandle();
743 
744  int Seek(vsi_l_offset nOffset, int nWhence) override;
745  vsi_l_offset Tell() override;
746  size_t Read(void *pBuffer, size_t nSize, size_t nMemb) override;
747  size_t Write(const void *pBuffer, size_t nSize, size_t nMemb) override;
748  int Eof() override;
749  int Close() override;
750 
751  bool IsOK()
752  {
753  return m_pabyBuffer != nullptr;
754  }
755 };
756 
757 /************************************************************************/
758 /* VSIDIRWithMissingDirSynthesis */
759 /************************************************************************/
760 
761 struct VSIDIRWithMissingDirSynthesis : public VSIDIR
762 {
763  std::vector<std::unique_ptr<VSIDIREntry>> aoEntries{};
764 
765  protected:
766  std::vector<std::string> m_aosSubpathsStack{};
767 
768  void SynthetizeMissingDirectories(const std::string &osCurSubdir,
769  bool bAddEntryForThisSubdir);
770 };
771 
772 /************************************************************************/
773 /* CurlRequestHelper */
774 /************************************************************************/
775 
776 struct CurlRequestHelper
777 {
778  WriteFuncStruct sWriteFuncData{};
779  WriteFuncStruct sWriteFuncHeaderData{};
780  char szCurlErrBuf[CURL_ERROR_SIZE + 1] = {};
781 
782  CurlRequestHelper();
783  ~CurlRequestHelper();
784  long perform(CURL *hCurlHandle,
785  struct curl_slist *headers, // ownership transferred
786  VSICurlFilesystemHandlerBase *poFS,
787  IVSIS3LikeHandleHelper *poS3HandleHelper);
788 };
789 
790 /************************************************************************/
791 /* NetworkStatisticsLogger */
792 /************************************************************************/
793 
794 class NetworkStatisticsLogger
795 {
796  static int gnEnabled;
797  static NetworkStatisticsLogger gInstance;
798 
799  NetworkStatisticsLogger() = default;
800 
801  std::mutex m_mutex{};
802 
803  struct Counters
804  {
805  GIntBig nHEAD = 0;
806  GIntBig nGET = 0;
807  GIntBig nPUT = 0;
808  GIntBig nPOST = 0;
809  GIntBig nDELETE = 0;
810  GIntBig nGETDownloadedBytes = 0;
811  GIntBig nPUTUploadedBytes = 0;
812  GIntBig nPOSTDownloadedBytes = 0;
813  GIntBig nPOSTUploadedBytes = 0;
814  };
815 
816  enum class ContextPathType
817  {
818  FILESYSTEM,
819  FILE,
820  ACTION,
821  };
822 
823  struct ContextPathItem
824  {
825  ContextPathType eType;
826  CPLString osName;
827 
828  ContextPathItem(ContextPathType eTypeIn, const CPLString &osNameIn)
829  : eType(eTypeIn), osName(osNameIn)
830  {
831  }
832 
833  bool operator<(const ContextPathItem &other) const
834  {
835  if (static_cast<int>(eType) < static_cast<int>(other.eType))
836  return true;
837  if (static_cast<int>(eType) > static_cast<int>(other.eType))
838  return false;
839  return osName < other.osName;
840  }
841  };
842 
843  struct Stats
844  {
845  Counters counters{};
846  std::map<ContextPathItem, Stats> children{};
847 
848  void AsJSON(CPLJSONObject &oJSON) const;
849  };
850 
851  // Workaround bug in Coverity Scan
852  // coverity[generated_default_constructor_used_in_field_initializer]
853  Stats m_stats{};
854  std::map<GIntBig, std::vector<ContextPathItem>>
855  m_mapThreadIdToContextPath{};
856 
857  static void ReadEnabled();
858 
859  std::vector<Counters *> GetCountersForContext();
860 
861  public:
862  static inline bool IsEnabled()
863  {
864  if (gnEnabled < 0)
865  {
866  ReadEnabled();
867  }
868  return gnEnabled == TRUE;
869  }
870 
871  static void EnterFileSystem(const char *pszName);
872 
873  static void LeaveFileSystem();
874 
875  static void EnterFile(const char *pszName);
876 
877  static void LeaveFile();
878 
879  static void EnterAction(const char *pszName);
880 
881  static void LeaveAction();
882 
883  static void LogHEAD();
884 
885  static void LogGET(size_t nDownloadedBytes);
886 
887  static void LogPUT(size_t nUploadedBytes);
888 
889  static void LogPOST(size_t nUploadedBytes, size_t nDownloadedBytes);
890 
891  static void LogDELETE();
892 
893  static void Reset();
894 
895  static CPLString GetReportAsSerializedJSON();
896 };
897 
898 struct NetworkStatisticsFileSystem
899 {
900  inline explicit NetworkStatisticsFileSystem(const char *pszName)
901  {
902  NetworkStatisticsLogger::EnterFileSystem(pszName);
903  }
904 
905  inline ~NetworkStatisticsFileSystem()
906  {
907  NetworkStatisticsLogger::LeaveFileSystem();
908  }
909 };
910 
911 struct NetworkStatisticsFile
912 {
913  inline explicit NetworkStatisticsFile(const char *pszName)
914  {
915  NetworkStatisticsLogger::EnterFile(pszName);
916  }
917 
918  inline ~NetworkStatisticsFile()
919  {
920  NetworkStatisticsLogger::LeaveFile();
921  }
922 };
923 
924 struct NetworkStatisticsAction
925 {
926  inline explicit NetworkStatisticsAction(const char *pszName)
927  {
928  NetworkStatisticsLogger::EnterAction(pszName);
929  }
930 
931  inline ~NetworkStatisticsAction()
932  {
933  NetworkStatisticsLogger::LeaveAction();
934  }
935 };
936 
937 int VSICURLGetDownloadChunkSize();
938 
939 void VSICURLInitWriteFuncStruct(WriteFuncStruct *psStruct, VSILFILE *fp,
940  VSICurlReadCbkFunc pfnReadCbk,
941  void *pReadCbkUserData);
942 size_t VSICurlHandleWriteFunc(void *buffer, size_t count, size_t nmemb,
943  void *req);
944 void MultiPerform(CURLM *hCurlMultiHandle, CURL *hEasyHandle = nullptr);
945 void VSICURLResetHeaderAndWriterFunctions(CURL *hCurlHandle);
946 
947 int VSICurlParseUnixPermissions(const char *pszPermissions);
948 
949 // Cache of file properties (size, etc.)
950 bool VSICURLGetCachedFileProp(const char *pszURL, FileProp &oFileProp);
951 void VSICURLSetCachedFileProp(const char *pszURL, FileProp &oFileProp);
952 void VSICURLInvalidateCachedFileProp(const char *pszURL);
953 void VSICURLInvalidateCachedFilePropPrefix(const char *pszURL);
954 void VSICURLDestroyCacheFileProp();
955 
956 } // namespace cpl
957 
959 
960 #endif // HAVE_CURL
961 
962 #endif // CPL_VSIL_CURL_CLASS_H_INCLUDED
GByte
unsigned char GByte
Unsigned byte type.
Definition: cpl_port.h:205
CPLStringList
String list class designed around our use of C "char**" string lists.
Definition: cpl_string.h:437
CPLString
Convenient string class based on std::string.
Definition: cpl_string.h:311
EQUAL
#define EQUAL(a, b)
Alias for strcasecmp() == 0.
Definition: cpl_port.h:571
CSLConstList
char ** CSLConstList
Type of a constant null-terminated list of nul terminated strings.
Definition: cpl_port.h:1195
CPLJSONObject
The CPLJSONArray class holds JSON object from CPLJSONDocument.
Definition: cpl_json.h:54
cpl_string.h
VSIVirtualHandle
Virtual file handle.
Definition: cpl_vsi_virtual.h:61
cpl_json.h
vsi_l_offset
GUIntBig vsi_l_offset
Type for a file offset.
Definition: cpl_vsi.h:146
GIntBig
long long GIntBig
Large signed integer type (generally 64-bit integer type).
Definition: cpl_port.h:235
VSIDIR
struct VSIDIR VSIDIR
Opaque type for a directory iterator.
Definition: cpl_vsi.h:382
VSIStatBufL
#define VSIStatBufL
Type for VSIStatL()
Definition: cpl_vsi.h:203
cpl_port.h
VSI_L_OFFSET_MAX
#define VSI_L_OFFSET_MAX
Maximum value for a file offset.
Definition: cpl_vsi.h:148
CPL_DISALLOW_COPY_ASSIGN
#define CPL_DISALLOW_COPY_ASSIGN(ClassName)
Helper to remove the copy and assignment constructors so that the compiler will not generate the defa...
Definition: cpl_port.h:1051