GDAL
cpl_vsil_curl_class.h
1 /******************************************************************************
2  *
3  * Project: CPL - Common Portability Library
4  * Purpose: Declarations for /vsicurl/ and related file systems
5  * Author: Even Rouault, even.rouault at spatialys.com
6  *
7  ******************************************************************************
8  * Copyright (c) 2010-2018, Even Rouault <even.rouault at spatialys.com>
9  *
10  * Permission is hereby granted, free of charge, to any person obtaining a
11  * copy of this software and associated documentation files (the "Software"),
12  * to deal in the Software without restriction, including without limitation
13  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
14  * and/or sell copies of the Software, and to permit persons to whom the
15  * Software is furnished to do so, subject to the following conditions:
16  *
17  * The above copyright notice and this permission notice shall be included
18  * in all copies or substantial portions of the Software.
19  *
20  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
21  * OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
22  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
23  * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
24  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
25  * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
26  * DEALINGS IN THE SOFTWARE.
27  ****************************************************************************/
28 
29 #ifndef CPL_VSIL_CURL_CLASS_H_INCLUDED
30 #define CPL_VSIL_CURL_CLASS_H_INCLUDED
31 
32 #ifdef HAVE_CURL
33 
34 #include "cpl_aws.h"
35 #include "cpl_azure.h"
36 #include "cpl_port.h"
37 #include "cpl_json.h"
38 #include "cpl_string.h"
39 #include "cpl_vsil_curl_priv.h"
40 #include "cpl_mem_cache.h"
41 
42 #include "cpl_curl_priv.h"
43 
44 #include <set>
45 #include <map>
46 #include <memory>
47 #include <mutex>
48 
50 
51 // Leave it for backward compatibility, but deprecate.
52 #define HAVE_CURLINFO_REDIRECT_URL
53 
54 void VSICurlStreamingClearCache( void ); // from cpl_vsil_curl_streaming.cpp
55 
56 struct curl_slist* VSICurlSetOptions(CURL* hCurlHandle, const char* pszURL,
57  const char * const* papszOptions);
58 struct curl_slist* VSICurlMergeHeaders( struct curl_slist* poDest,
59  struct curl_slist* poSrcToDestroy );
60 
61 struct curl_slist* VSICurlSetContentTypeFromExt(struct curl_slist* polist,
62  const char *pszPath);
63 
64 struct curl_slist* VSICurlSetCreationHeadersFromOptions(struct curl_slist* headers,
65  CSLConstList papszOptions,
66  const char *pszPath);
67 
68 namespace cpl {
69 
70 typedef enum
71 {
72  EXIST_UNKNOWN = -1,
73  EXIST_NO,
74  EXIST_YES,
75 } ExistStatus;
76 
77 class FileProp
78 {
79  public:
80  unsigned int nGenerationAuthParameters = 0;
81  ExistStatus eExists = EXIST_UNKNOWN;
82  vsi_l_offset fileSize = 0;
83  time_t mTime = 0;
84  time_t nExpireTimestampLocal = 0;
85  CPLString osRedirectURL{};
86  bool bHasComputedFileSize = false;
87  bool bIsDirectory = false;
88  int nMode = 0; // st_mode member of struct stat
89  bool bS3LikeRedirect = false;
90  CPLString ETag{};
91 };
92 
93 struct CachedDirList
94 {
95  bool bGotFileList = false;
96  unsigned int nGenerationAuthParameters = 0;
97  CPLStringList oFileList{}; /* only file name without path */
98 };
99 
100 struct WriteFuncStruct
101 {
102  char* pBuffer = nullptr;
103  size_t nSize = 0;
104  bool bIsHTTP = false;
105  bool bMultiRange = false;
106  vsi_l_offset nStartOffset = 0;
107  vsi_l_offset nEndOffset = 0;
108  int nHTTPCode = 0;
109  vsi_l_offset nContentLength = 0;
110  bool bFoundContentRange = false;
111  bool bError = false;
112  bool bInterruptDownload = false;
113  bool bDetectRangeDownloadingError = false;
114  GIntBig nTimestampDate = 0; // Corresponds to Date: header field
115 
116  VSILFILE *fp = nullptr;
117  VSICurlReadCbkFunc pfnReadCbk = nullptr;
118  void *pReadCbkUserData = nullptr;
119  bool bInterrupted = false;
120 
121 #if !CURL_AT_LEAST_VERSION(7,54,0)
122  // Workaround to ignore extra HTTP response headers from
123  // proxies in older versions of curl.
124  // CURLOPT_SUPPRESS_CONNECT_HEADERS fixes this
125  bool bIsProxyConnectHeader = false;
126 #endif
127 };
128 
129 struct PutData
130 {
131  const GByte* pabyData = nullptr;
132  size_t nOff = 0;
133  size_t nTotalSize = 0;
134 
135  static size_t ReadCallBackBuffer( char *buffer, size_t size,
136  size_t nitems, void *instream )
137  {
138  PutData* poThis = static_cast<PutData *>(instream);
139  const size_t nSizeMax = size * nitems;
140  const size_t nSizeToWrite =
141  std::min(nSizeMax, poThis->nTotalSize - poThis->nOff);
142  memcpy(buffer, poThis->pabyData + poThis->nOff, nSizeToWrite);
143  poThis->nOff += nSizeToWrite;
144  return nSizeToWrite;
145  }
146 };
147 
148 /************************************************************************/
149 /* VSICurlFilesystemHandler */
150 /************************************************************************/
151 
152 class VSICurlHandle;
153 
154 class VSICurlFilesystemHandler : public VSIFilesystemHandler
155 {
156  CPL_DISALLOW_COPY_ASSIGN(VSICurlFilesystemHandler)
157 
158  struct FilenameOffsetPair
159  {
160  std::string filename_;
161  vsi_l_offset offset_;
162 
163  FilenameOffsetPair(const std::string& filename,
164  vsi_l_offset offset) :
165  filename_(filename), offset_(offset) {}
166 
167  bool operator==(const FilenameOffsetPair& other) const
168  {
169  return filename_ == other.filename_ &&
170  offset_ == other.offset_;
171  }
172  };
173  struct FilenameOffsetPairHasher
174  {
175  std::size_t operator()(const FilenameOffsetPair& k) const
176  {
177  return std::hash<std::string>()(k.filename_) ^
178  std::hash<vsi_l_offset>()(k.offset_);
179  }
180  };
181 
182  using RegionCacheType =
183  lru11::Cache<FilenameOffsetPair, std::shared_ptr<std::string>,
184  lru11::NullLock,
185  std::unordered_map<
186  FilenameOffsetPair,
187  typename std::list<lru11::KeyValuePair<FilenameOffsetPair,
188  std::shared_ptr<std::string>>>::iterator,
189  FilenameOffsetPairHasher>>;
190 
191  std::unique_ptr<RegionCacheType> m_poRegionCacheDoNotUseDirectly{}; // do not access directly. Use GetRegionCache();
192  RegionCacheType* GetRegionCache();
193 
194  lru11::Cache<std::string, FileProp> oCacheFileProp;
195 
196  int nCachedFilesInDirList = 0;
197  lru11::Cache<std::string, CachedDirList> oCacheDirList;
198 
199  char** ParseHTMLFileList(const char* pszFilename,
200  int nMaxFiles,
201  char* pszData,
202  bool* pbGotFileList);
203 
204 protected:
205  CPLMutex *hMutex = nullptr;
206 
207  virtual VSICurlHandle* CreateFileHandle(const char* pszFilename);
208  virtual char** GetFileList(const char *pszFilename,
209  int nMaxFiles,
210  bool* pbGotFileList);
211 
212  void RegisterEmptyDir( const CPLString& osDirname );
213 
214  bool AnalyseS3FileList( const CPLString& osBaseURL,
215  const char* pszXML,
216  CPLStringList& osFileList,
217  int nMaxFiles,
218  bool bIgnoreGlacierStorageClass,
219  bool& bIsTruncated );
220 
221  void AnalyseSwiftFileList( const CPLString& osBaseURL,
222  const CPLString& osPrefix,
223  const char* pszJson,
224  CPLStringList& osFileList,
225  int nMaxFilesThisQuery,
226  int nMaxFiles,
227  bool& bIsTruncated,
228  CPLString& osNextMarker );
229 
230  static const char* GetOptionsStatic();
231 
232  static bool IsAllowedFilename( const char* pszFilename );
233 
234 public:
235  VSICurlFilesystemHandler();
236  ~VSICurlFilesystemHandler() override;
237 
238  VSIVirtualHandle *Open( const char *pszFilename,
239  const char *pszAccess,
240  bool bSetError,
241  CSLConstList /* papszOptions */ ) override;
242 
243  int Stat( const char *pszFilename, VSIStatBufL *pStatBuf,
244  int nFlags ) override;
245  int Unlink( const char *pszFilename ) override;
246  int Rename( const char *oldpath, const char *newpath ) override;
247  int Mkdir( const char *pszDirname, long nMode ) override;
248  int Rmdir( const char *pszDirname ) override;
249  char **ReadDir( const char *pszDirname ) override
250  { return ReadDirEx(pszDirname, 0); }
251  char **ReadDirEx( const char *pszDirname, int nMaxFiles ) override;
252  char **SiblingFiles( const char *pszFilename ) override;
253 
254  int HasOptimizedReadMultiRange( const char* /* pszPath */ )
255  override { return true; }
256 
257  const char* GetActualURL(const char* pszFilename) override;
258 
259  const char* GetOptions() override;
260 
261  char** GetFileMetadata( const char * pszFilename, const char* pszDomain,
262  CSLConstList papszOptions ) override;
263 
264  char **ReadDirInternal( const char *pszDirname, int nMaxFiles,
265  bool* pbGotFileList );
266  void InvalidateDirContent( const char *pszDirname );
267 
268  virtual const char* GetDebugKey() const { return "VSICURL"; }
269 
270  virtual CPLString GetFSPrefix() const { return "/vsicurl/"; }
271  virtual bool AllowCachedDataFor(const char* pszFilename);
272 
273  std::shared_ptr<std::string> GetRegion( const char* pszURL,
274  vsi_l_offset nFileOffsetStart );
275 
276  void AddRegion( const char* pszURL,
277  vsi_l_offset nFileOffsetStart,
278  size_t nSize,
279  const char *pData );
280 
281  bool GetCachedFileProp( const char* pszURL,
282  FileProp& oFileProp );
283  void SetCachedFileProp( const char* pszURL,
284  FileProp& oFileProp );
285  void InvalidateCachedData( const char* pszURL );
286 
287  CURLM *GetCurlMultiHandleFor( const CPLString& osURL );
288 
289  virtual void ClearCache();
290  virtual void PartialClearCache(const char* pszFilename);
291 
292 
293  bool GetCachedDirList( const char* pszURL,
294  CachedDirList& oCachedDirList );
295  void SetCachedDirList( const char* pszURL,
296  CachedDirList& oCachedDirList );
297  bool ExistsInCacheDirList( const CPLString& osDirname, bool *pbIsDir );
298 
299  virtual CPLString GetURLFromFilename( const CPLString& osFilename );
300 };
301 
302 /************************************************************************/
303 /* VSICurlHandle */
304 /************************************************************************/
305 
306 class VSICurlHandle : public VSIVirtualHandle
307 {
308  CPL_DISALLOW_COPY_ASSIGN(VSICurlHandle)
309 
310  protected:
311  VSICurlFilesystemHandler* poFS = nullptr;
312 
313  bool m_bCached = true;
314 
315  FileProp oFileProp{};
316 
317  CPLString m_osFilename{}; // e.g "/vsicurl/http://example.com/foo"
318  char* m_pszURL = nullptr; // e.g "http://example.com/foo"
319  std::string m_osQueryString{}; // e.g. an Azure SAS
320 
321  char **m_papszHTTPOptions = nullptr;
322 
323  vsi_l_offset lastDownloadedOffset = VSI_L_OFFSET_MAX;
324  int nBlocksToDownload = 1;
325 
326  bool bStopOnInterruptUntilUninstall = false;
327  bool bInterrupted = false;
328  VSICurlReadCbkFunc pfnReadCbk = nullptr;
329  void *pReadCbkUserData = nullptr;
330 
331  int m_nMaxRetry = 0;
332  double m_dfRetryDelay = 0.0;
333 
334  CPLStringList m_aosHeaders{};
335 
336  void DownloadRegionPostProcess( const vsi_l_offset startOffset,
337  const int nBlocks,
338  const char* pBuffer,
339  size_t nSize );
340 
341  private:
342 
343  vsi_l_offset curOffset = 0;
344 
345  bool bEOF = false;
346 
347  virtual std::string DownloadRegion(vsi_l_offset startOffset, int nBlocks);
348 
349  bool m_bUseHead = false;
350  bool m_bUseRedirectURLIfNoQueryStringParams = false;
351 
352  int ReadMultiRangeSingleGet( int nRanges, void ** ppData,
353  const vsi_l_offset* panOffsets,
354  const size_t* panSizes );
355  CPLString GetRedirectURLIfValid(bool& bHasExpired);
356 
357  protected:
358  virtual struct curl_slist* GetCurlHeaders( const CPLString& /*osVerb*/,
359  const struct curl_slist* /* psExistingHeaders */)
360  { return nullptr; }
361  virtual bool AllowAutomaticRedirection() { return true; }
362  virtual bool CanRestartOnError( const char*, const char*, bool ) { return false; }
363  virtual bool UseLimitRangeGetInsteadOfHead() { return false; }
364  virtual bool IsDirectoryFromExists( const char* /*pszVerb*/, int /*response_code*/ ) { return false; }
365  virtual void ProcessGetFileSizeResult(const char* /* pszContent */ ) {}
366  void SetURL(const char* pszURL);
367  virtual bool Authenticate() { return false; }
368 
369  public:
370 
371  VSICurlHandle( VSICurlFilesystemHandler* poFS,
372  const char* pszFilename,
373  const char* pszURLIn = nullptr );
374  ~VSICurlHandle() override;
375 
376  int Seek( vsi_l_offset nOffset, int nWhence ) override;
377  vsi_l_offset Tell() override;
378  size_t Read( void *pBuffer, size_t nSize, size_t nMemb ) override;
379  int ReadMultiRange( int nRanges, void ** ppData,
380  const vsi_l_offset* panOffsets,
381  const size_t* panSizes ) override;
382  size_t Write( const void *pBuffer, size_t nSize, size_t nMemb ) override;
383  int Eof() override;
384  int Flush() override;
385  int Close() override;
386 
387  bool IsKnownFileSize() const { return oFileProp.bHasComputedFileSize; }
388  vsi_l_offset GetFileSizeOrHeaders(bool bSetError, bool bGetHeaders);
389  virtual vsi_l_offset GetFileSize( bool bSetError ) { return GetFileSizeOrHeaders(bSetError, false); }
390  bool Exists( bool bSetError );
391  bool IsDirectory() const { return oFileProp.bIsDirectory; }
392  int GetMode() const { return oFileProp.nMode; }
393  time_t GetMTime() const { return oFileProp.mTime; }
394  const CPLStringList& GetHeaders() { return m_aosHeaders; }
395 
396  int InstallReadCbk( VSICurlReadCbkFunc pfnReadCbk,
397  void* pfnUserData,
398  int bStopOnInterruptUntilUninstall );
399  int UninstallReadCbk();
400 
401  const char *GetURL() const { return m_pszURL; }
402 };
403 
404 /************************************************************************/
405 /* IVSIS3LikeFSHandler */
406 /************************************************************************/
407 
408 class IVSIS3LikeFSHandler: public VSICurlFilesystemHandler
409 {
410  CPL_DISALLOW_COPY_ASSIGN(IVSIS3LikeFSHandler)
411 
412  bool CopyFile(VSILFILE* fpIn,
413  vsi_l_offset nSourceSize,
414  const char* pszSource,
415  const char* pszTarget,
416  GDALProgressFunc pProgressFunc,
417  void *pProgressData);
418  virtual int MkdirInternal( const char *pszDirname, long nMode, bool bDoStatCheck );
419 
420  protected:
421  char** GetFileList( const char *pszFilename,
422  int nMaxFiles,
423  bool* pbGotFileList ) override;
424 
425  virtual IVSIS3LikeHandleHelper* CreateHandleHelper(
426  const char* pszURI, bool bAllowNoObject) = 0;
427 
428  virtual int CopyObject( const char *oldpath, const char *newpath,
429  CSLConstList papszMetadata );
430 
431  IVSIS3LikeFSHandler() = default;
432 
433  public:
434  int Unlink( const char *pszFilename ) override;
435  int Mkdir( const char *pszDirname, long nMode ) override;
436  int Rmdir( const char *pszDirname ) override;
437  int Stat( const char *pszFilename, VSIStatBufL *pStatBuf,
438  int nFlags ) override;
439  int Rename( const char *oldpath, const char *newpath ) override;
440 
441  virtual int DeleteObject( const char *pszFilename );
442 
443  virtual void UpdateMapFromHandle(IVSIS3LikeHandleHelper*) {}
444  virtual void UpdateHandleFromMap( IVSIS3LikeHandleHelper * ) {}
445 
446  virtual CPLString GetStreamingPath( const char* pszFilename ) const;
447 
448  bool Sync( const char* pszSource, const char* pszTarget,
449  const char* const * papszOptions,
450  GDALProgressFunc pProgressFunc,
451  void *pProgressData,
452  char*** ppapszOutputs ) override;
453 
454  VSIDIR* OpenDir( const char *pszPath, int nRecurseDepth,
455  const char* const *papszOptions) override;
456 
457  // Multipart upload
458  virtual bool SupportsParallelMultipartUpload() const { return false; }
459 
460  virtual CPLString InitiateMultipartUpload(
461  const std::string& osFilename,
462  IVSIS3LikeHandleHelper *poS3HandleHelper,
463  int nMaxRetry,
464  double dfRetryDelay,
465  CSLConstList papszOptions);
466  virtual CPLString UploadPart(const CPLString& osFilename,
467  int nPartNumber,
468  const std::string& osUploadID,
469  vsi_l_offset nPosition,
470  const void* pabyBuffer,
471  size_t nBufferSize,
472  IVSIS3LikeHandleHelper *poS3HandleHelper,
473  int nMaxRetry,
474  double dfRetryDelay);
475  virtual bool CompleteMultipart(const CPLString& osFilename,
476  const CPLString& osUploadID,
477  const std::vector<CPLString>& aosEtags,
478  vsi_l_offset nTotalSize,
479  IVSIS3LikeHandleHelper *poS3HandleHelper,
480  int nMaxRetry,
481  double dfRetryDelay);
482  virtual bool AbortMultipart(const CPLString& osFilename,
483  const CPLString& osUploadID,
484  IVSIS3LikeHandleHelper *poS3HandleHelper,
485  int nMaxRetry,
486  double dfRetryDelay);
487 };
488 
489 /************************************************************************/
490 /* IVSIS3LikeHandle */
491 /************************************************************************/
492 
493 class IVSIS3LikeHandle: public VSICurlHandle
494 {
495  CPL_DISALLOW_COPY_ASSIGN(IVSIS3LikeHandle)
496 
497  protected:
498  bool UseLimitRangeGetInsteadOfHead() override { return true; }
499  bool IsDirectoryFromExists( const char* pszVerb,
500  int response_code ) override
501  {
502  // A bit dirty, but on S3, a GET on a existing directory returns a 416
503  return response_code == 416 && EQUAL(pszVerb, "GET") &&
504  CPLString(m_pszURL).back() == '/';
505  }
506  void ProcessGetFileSizeResult( const char* pszContent ) override
507  {
508  oFileProp.bIsDirectory = strstr(pszContent, "ListBucketResult") != nullptr;
509  }
510 
511  public:
512  IVSIS3LikeHandle( VSICurlFilesystemHandler* poFSIn,
513  const char* pszFilename,
514  const char* pszURLIn ) :
515  VSICurlHandle(poFSIn, pszFilename, pszURLIn) {}
516  ~IVSIS3LikeHandle() override {}
517 };
518 
519 /************************************************************************/
520 /* VSIS3WriteHandle */
521 /************************************************************************/
522 
523 class VSIS3WriteHandle final : public VSIVirtualHandle
524 {
525  CPL_DISALLOW_COPY_ASSIGN(VSIS3WriteHandle)
526 
527  IVSIS3LikeFSHandler *m_poFS = nullptr;
528  CPLString m_osFilename{};
529  IVSIS3LikeHandleHelper *m_poS3HandleHelper = nullptr;
530  bool m_bUseChunked = false;
531  CPLStringList m_aosOptions{};
532 
533  vsi_l_offset m_nCurOffset = 0;
534  int m_nBufferOff = 0;
535  int m_nBufferSize = 0;
536  bool m_bClosed = false;
537  GByte *m_pabyBuffer = nullptr;
538  CPLString m_osUploadID{};
539  int m_nPartNumber = 0;
540  std::vector<CPLString> m_aosEtags{};
541  bool m_bError = false;
542 
543  CURLM *m_hCurlMulti = nullptr;
544  CURL *m_hCurl = nullptr;
545  const void *m_pBuffer = nullptr;
546  CPLString m_osCurlErrBuf{};
547  size_t m_nChunkedBufferOff = 0;
548  size_t m_nChunkedBufferSize = 0;
549  size_t m_nWrittenInPUT = 0;
550 
551  int m_nMaxRetry = 0;
552  double m_dfRetryDelay = 0.0;
553  WriteFuncStruct m_sWriteFuncHeaderData{};
554 
555  bool UploadPart();
556  bool DoSinglePartPUT();
557 
558  static size_t ReadCallBackBufferChunked( char *buffer, size_t size,
559  size_t nitems, void *instream );
560  size_t WriteChunked( const void *pBuffer,
561  size_t nSize, size_t nMemb );
562  int FinishChunkedTransfer();
563 
564  void InvalidateParentDirectory();
565 
566  public:
567  VSIS3WriteHandle( IVSIS3LikeFSHandler* poFS,
568  const char* pszFilename,
569  IVSIS3LikeHandleHelper* poS3HandleHelper,
570  bool bUseChunked,
571  CSLConstList papszOptions );
572  ~VSIS3WriteHandle() override;
573 
574  int Seek( vsi_l_offset nOffset, int nWhence ) override;
575  vsi_l_offset Tell() override;
576  size_t Read( void *pBuffer, size_t nSize, size_t nMemb ) override;
577  size_t Write( const void *pBuffer, size_t nSize, size_t nMemb ) override;
578  int Eof() override;
579  int Close() override;
580 
581  bool IsOK() { return m_bUseChunked || m_pabyBuffer != nullptr; }
582 };
583 
584 /************************************************************************/
585 /* VSIAppendWriteHandle */
586 /************************************************************************/
587 
588 class VSIAppendWriteHandle : public VSIVirtualHandle
589 {
590  CPL_DISALLOW_COPY_ASSIGN(VSIAppendWriteHandle)
591 
592  protected:
593 
594  VSICurlFilesystemHandler* m_poFS = nullptr;
595  CPLString m_osFSPrefix{};
596  CPLString m_osFilename{};
597 
598  vsi_l_offset m_nCurOffset = 0;
599  int m_nBufferOff = 0;
600  int m_nBufferSize = 0;
601  int m_nBufferOffReadCallback = 0;
602  bool m_bClosed = false;
603  GByte *m_pabyBuffer = nullptr;
604  bool m_bError = false;
605 
606  static size_t ReadCallBackBuffer( char *buffer, size_t size,
607  size_t nitems, void *instream );
608  virtual bool Send(bool bIsLastBlock) = 0;
609 
610  public:
611  VSIAppendWriteHandle( VSICurlFilesystemHandler* poFS,
612  const char* pszFSPrefix,
613  const char* pszFilename,
614  int nChunkSize );
615  virtual ~VSIAppendWriteHandle();
616 
617  int Seek( vsi_l_offset nOffset, int nWhence ) override;
618  vsi_l_offset Tell() override;
619  size_t Read( void *pBuffer, size_t nSize, size_t nMemb ) override;
620  size_t Write( const void *pBuffer, size_t nSize, size_t nMemb ) override;
621  int Eof() override;
622  int Close() override;
623 
624  bool IsOK() { return m_pabyBuffer != nullptr; }
625 };
626 
627 /************************************************************************/
628 /* CurlRequestHelper */
629 /************************************************************************/
630 
631 struct CurlRequestHelper
632 {
633  WriteFuncStruct sWriteFuncData{};
634  WriteFuncStruct sWriteFuncHeaderData{};
635  char szCurlErrBuf[CURL_ERROR_SIZE+1] = {};
636 
637  CurlRequestHelper();
638  ~CurlRequestHelper();
639  long perform(CURL* hCurlHandle,
640  struct curl_slist* headers, // ownership transferred
641  VSICurlFilesystemHandler *poFS,
642  IVSIS3LikeHandleHelper *poS3HandleHelper);
643 };
644 
645 /************************************************************************/
646 /* NetworkStatisticsLogger */
647 /************************************************************************/
648 
649 class NetworkStatisticsLogger
650 {
651  static int gnEnabled;
652  static NetworkStatisticsLogger gInstance;
653 
654  NetworkStatisticsLogger() = default;
655 
656  std::mutex m_mutex{};
657 
658  struct Counters
659  {
660  GIntBig nHEAD = 0;
661  GIntBig nGET = 0;
662  GIntBig nPUT = 0;
663  GIntBig nPOST = 0;
664  GIntBig nDELETE = 0;
665  GIntBig nGETDownloadedBytes = 0;
666  GIntBig nPUTUploadedBytes = 0;
667  GIntBig nPOSTDownloadedBytes = 0;
668  GIntBig nPOSTUploadedBytes = 0;
669  };
670 
671  enum class ContextPathType
672  {
673  FILESYSTEM,
674  FILE,
675  ACTION,
676  };
677 
678  struct ContextPathItem
679  {
680  ContextPathType eType;
681  CPLString osName;
682 
683  ContextPathItem(ContextPathType eTypeIn, const CPLString& osNameIn):
684  eType(eTypeIn), osName(osNameIn) {}
685 
686  bool operator< (const ContextPathItem& other ) const
687  {
688  if( static_cast<int>(eType) < static_cast<int>(other.eType) )
689  return true;
690  if( static_cast<int>(eType) > static_cast<int>(other.eType) )
691  return false;
692  return osName < other.osName;
693  }
694  };
695 
696  struct Stats
697  {
698  Counters counters{};
699  std::map<ContextPathItem, Stats> children{};
700 
701  void AsJSON(CPLJSONObject& oJSON) const;
702  };
703 
704  Stats m_stats{};
705  std::map<GIntBig, std::vector<ContextPathItem>> m_mapThreadIdToContextPath{};
706 
707  static void ReadEnabled();
708 
709  std::vector<Counters*> GetCountersForContext();
710 
711 public:
712 
713  static inline bool IsEnabled()
714  {
715  if( gnEnabled < 0)
716  {
717  ReadEnabled();
718  }
719  return gnEnabled == TRUE;
720  }
721 
722  static void EnterFileSystem(const char* pszName);
723 
724  static void LeaveFileSystem();
725 
726  static void EnterFile(const char* pszName);
727 
728  static void LeaveFile();
729 
730  static void EnterAction(const char* pszName);
731 
732  static void LeaveAction();
733 
734  static void LogHEAD();
735 
736  static void LogGET(size_t nDownloadedBytes);
737 
738  static void LogPUT(size_t nUploadedBytes);
739 
740  static void LogPOST(size_t nUploadedBytes,
741  size_t nDownloadedBytes);
742 
743  static void LogDELETE();
744 
745  static void Reset();
746 
747  static CPLString GetReportAsSerializedJSON();
748 };
749 
750 struct NetworkStatisticsFileSystem
751 {
752  inline explicit NetworkStatisticsFileSystem(const char* pszName) {
753  NetworkStatisticsLogger::EnterFileSystem(pszName);
754  }
755 
756  inline ~NetworkStatisticsFileSystem()
757  {
758  NetworkStatisticsLogger::LeaveFileSystem();
759  }
760 };
761 
762 struct NetworkStatisticsFile
763 {
764  inline explicit NetworkStatisticsFile(const char* pszName) {
765  NetworkStatisticsLogger::EnterFile(pszName);
766  }
767 
768  inline ~NetworkStatisticsFile()
769  {
770  NetworkStatisticsLogger::LeaveFile();
771  }
772 };
773 
774 struct NetworkStatisticsAction
775 {
776  inline explicit NetworkStatisticsAction(const char* pszName) {
777  NetworkStatisticsLogger::EnterAction(pszName);
778  }
779 
780  inline ~NetworkStatisticsAction()
781  {
782  NetworkStatisticsLogger::LeaveAction();
783  }
784 };
785 
786 
787 int VSICURLGetDownloadChunkSize();
788 
789 void VSICURLInitWriteFuncStruct( WriteFuncStruct *psStruct,
790  VSILFILE *fp,
791  VSICurlReadCbkFunc pfnReadCbk,
792  void *pReadCbkUserData );
793 size_t VSICurlHandleWriteFunc( void *buffer, size_t count,
794  size_t nmemb, void *req );
795 void MultiPerform(CURLM* hCurlMultiHandle,
796  CURL* hEasyHandle = nullptr);
797 void VSICURLResetHeaderAndWriterFunctions(CURL* hCurlHandle);
798 
799 int VSICurlParseUnixPermissions(const char* pszPermissions);
800 
801 } // namespace cpl
802 
804 
805 #endif // HAVE_CURL
806 
807 #endif // CPL_VSIL_CURL_CLASS_H_INCLUDED
The CPLJSONArray class holds JSON object from CPLJSONDocument.
Definition: cpl_json.h:54
String list class designed around our use of C "char**" string lists.
Definition: cpl_string.h:442
Convenient string class based on std::string.
Definition: cpl_string.h:333
Virtual file handle.
Definition: cpl_vsi_virtual.h:56
Interface for read and write JSON documents.
Core portability definitions for CPL.
#define EQUAL(a, b)
Alias for strcasecmp() == 0.
Definition: cpl_port.h:576
#define CPL_DISALLOW_COPY_ASSIGN(ClassName)
Helper to remove the copy and assignment constructors so that the compiler will not generate the defa...
Definition: cpl_port.h:1007
char ** CSLConstList
Type of a constant null-terminated list of nul terminated strings.
Definition: cpl_port.h:1216
unsigned char GByte
Unsigned byte type.
Definition: cpl_port.h:215
long long GIntBig
Large signed integer type (generally 64-bit integer type).
Definition: cpl_port.h:248
Various convenience functions for working with strings and string lists.
#define VSI_L_OFFSET_MAX
Maximum value for a file offset.
Definition: cpl_vsi.h:142
struct VSIDIR VSIDIR
Opaque type for a directory iterator.
Definition: cpl_vsi.h:319
struct VSI_STAT64_T VSIStatBufL
Type for VSIStatL()
Definition: cpl_vsi.h:195
FILE VSILFILE
Opaque type for a FILE that implements the VSIVirtualHandle API.
Definition: cpl_vsi.h:156
GUIntBig vsi_l_offset
Type for a file offset.
Definition: cpl_vsi.h:140