29 #ifndef CPL_VSIL_CURL_CLASS_H_INCLUDED
30 #define CPL_VSIL_CURL_CLASS_H_INCLUDED
35 #include "cpl_azure.h"
39 #include "cpl_vsil_curl_priv.h"
40 #include "cpl_mem_cache.h"
42 #include "cpl_curl_priv.h"
52 #define HAVE_CURLINFO_REDIRECT_URL
54 void VSICurlStreamingClearCache(
void );
56 struct curl_slist* VSICurlSetOptions(CURL* hCurlHandle,
const char* pszURL,
57 const char *
const* papszOptions);
58 struct curl_slist* VSICurlMergeHeaders(
struct curl_slist* poDest,
59 struct curl_slist* poSrcToDestroy );
61 struct curl_slist* VSICurlSetContentTypeFromExt(
struct curl_slist* polist,
64 struct curl_slist* VSICurlSetCreationHeadersFromOptions(
struct curl_slist* headers,
80 unsigned int nGenerationAuthParameters = 0;
81 ExistStatus eExists = EXIST_UNKNOWN;
84 time_t nExpireTimestampLocal = 0;
86 bool bHasComputedFileSize =
false;
87 bool bIsDirectory =
false;
89 bool bS3LikeRedirect =
false;
95 bool bGotFileList =
false;
96 unsigned int nGenerationAuthParameters = 0;
100 struct WriteFuncStruct
102 char* pBuffer =
nullptr;
104 bool bIsHTTP =
false;
105 bool bMultiRange =
false;
110 bool bFoundContentRange =
false;
112 bool bInterruptDownload =
false;
113 bool bDetectRangeDownloadingError =
false;
117 VSICurlReadCbkFunc pfnReadCbk =
nullptr;
118 void *pReadCbkUserData =
nullptr;
119 bool bInterrupted =
false;
121 #if !CURL_AT_LEAST_VERSION(7,54,0)
125 bool bIsProxyConnectHeader =
false;
131 const GByte* pabyData =
nullptr;
133 size_t nTotalSize = 0;
135 static size_t ReadCallBackBuffer(
char *buffer,
size_t size,
136 size_t nitems,
void *instream )
138 PutData* poThis =
static_cast<PutData *
>(instream);
139 const size_t nSizeMax = size * nitems;
140 const size_t nSizeToWrite =
141 std::min(nSizeMax, poThis->nTotalSize - poThis->nOff);
142 memcpy(buffer, poThis->pabyData + poThis->nOff, nSizeToWrite);
143 poThis->nOff += nSizeToWrite;
154 class VSICurlFilesystemHandler :
public VSIFilesystemHandler
158 struct FilenameOffsetPair
160 std::string filename_;
163 FilenameOffsetPair(
const std::string& filename,
165 filename_(filename), offset_(offset) {}
167 bool operator==(
const FilenameOffsetPair& other)
const
169 return filename_ == other.filename_ &&
170 offset_ == other.offset_;
173 struct FilenameOffsetPairHasher
175 std::size_t operator()(
const FilenameOffsetPair& k)
const
177 return std::hash<std::string>()(k.filename_) ^
178 std::hash<vsi_l_offset>()(k.offset_);
182 using RegionCacheType =
183 lru11::Cache<FilenameOffsetPair, std::shared_ptr<std::string>,
187 typename std::list<lru11::KeyValuePair<FilenameOffsetPair,
188 std::shared_ptr<std::string>>>::iterator,
189 FilenameOffsetPairHasher>>;
191 std::unique_ptr<RegionCacheType> m_poRegionCacheDoNotUseDirectly{};
192 RegionCacheType* GetRegionCache();
194 lru11::Cache<std::string, FileProp> oCacheFileProp;
196 int nCachedFilesInDirList = 0;
197 lru11::Cache<std::string, CachedDirList> oCacheDirList;
199 char** ParseHTMLFileList(
const char* pszFilename,
202 bool* pbGotFileList);
205 CPLMutex *hMutex =
nullptr;
207 virtual VSICurlHandle* CreateFileHandle(
const char* pszFilename);
208 virtual char** GetFileList(
const char *pszFilename,
210 bool* pbGotFileList);
212 void RegisterEmptyDir(
const CPLString& osDirname );
214 bool AnalyseS3FileList(
const CPLString& osBaseURL,
218 bool bIgnoreGlacierStorageClass,
219 bool& bIsTruncated );
221 void AnalyseSwiftFileList(
const CPLString& osBaseURL,
225 int nMaxFilesThisQuery,
230 static const char* GetOptionsStatic();
232 static bool IsAllowedFilename(
const char* pszFilename );
235 VSICurlFilesystemHandler();
236 ~VSICurlFilesystemHandler()
override;
239 const char *pszAccess,
243 int Stat(
const char *pszFilename,
VSIStatBufL *pStatBuf,
244 int nFlags )
override;
245 int Unlink(
const char *pszFilename )
override;
246 int Rename(
const char *oldpath,
const char *newpath )
override;
247 int Mkdir(
const char *pszDirname,
long nMode )
override;
248 int Rmdir(
const char *pszDirname )
override;
249 char **ReadDir(
const char *pszDirname )
override
250 {
return ReadDirEx(pszDirname, 0); }
251 char **ReadDirEx(
const char *pszDirname,
int nMaxFiles )
override;
252 char **SiblingFiles(
const char *pszFilename )
override;
254 int HasOptimizedReadMultiRange(
const char* )
255 override {
return true; }
257 const char* GetActualURL(
const char* pszFilename)
override;
259 const char* GetOptions()
override;
261 char** GetFileMetadata(
const char * pszFilename,
const char* pszDomain,
264 char **ReadDirInternal(
const char *pszDirname,
int nMaxFiles,
265 bool* pbGotFileList );
266 void InvalidateDirContent(
const char *pszDirname );
268 virtual const char* GetDebugKey()
const {
return "VSICURL"; }
270 virtual CPLString GetFSPrefix()
const {
return "/vsicurl/"; }
271 virtual bool AllowCachedDataFor(
const char* pszFilename);
273 std::shared_ptr<std::string> GetRegion(
const char* pszURL,
276 void AddRegion(
const char* pszURL,
281 bool GetCachedFileProp(
const char* pszURL,
282 FileProp& oFileProp );
283 void SetCachedFileProp(
const char* pszURL,
284 FileProp& oFileProp );
285 void InvalidateCachedData(
const char* pszURL );
287 CURLM *GetCurlMultiHandleFor(
const CPLString& osURL );
289 virtual void ClearCache();
290 virtual void PartialClearCache(
const char* pszFilename);
293 bool GetCachedDirList(
const char* pszURL,
294 CachedDirList& oCachedDirList );
295 void SetCachedDirList(
const char* pszURL,
296 CachedDirList& oCachedDirList );
297 bool ExistsInCacheDirList(
const CPLString& osDirname,
bool *pbIsDir );
311 VSICurlFilesystemHandler* poFS =
nullptr;
313 bool m_bCached =
true;
315 FileProp oFileProp{};
318 char* m_pszURL =
nullptr;
319 std::string m_osQueryString{};
321 char **m_papszHTTPOptions =
nullptr;
324 int nBlocksToDownload = 1;
326 bool bStopOnInterruptUntilUninstall =
false;
327 bool bInterrupted =
false;
328 VSICurlReadCbkFunc pfnReadCbk =
nullptr;
329 void *pReadCbkUserData =
nullptr;
332 double m_dfRetryDelay = 0.0;
336 void DownloadRegionPostProcess(
const vsi_l_offset startOffset,
347 virtual std::string DownloadRegion(
vsi_l_offset startOffset,
int nBlocks);
349 bool m_bUseHead =
false;
350 bool m_bUseRedirectURLIfNoQueryStringParams =
false;
352 int ReadMultiRangeSingleGet(
int nRanges,
void ** ppData,
354 const size_t* panSizes );
355 CPLString GetRedirectURLIfValid(
bool& bHasExpired);
358 virtual struct curl_slist* GetCurlHeaders(
const CPLString& ,
359 const struct curl_slist* )
361 virtual bool AllowAutomaticRedirection() {
return true; }
362 virtual bool CanRestartOnError(
const char*,
const char*,
bool ) {
return false; }
363 virtual bool UseLimitRangeGetInsteadOfHead() {
return false; }
364 virtual bool IsDirectoryFromExists(
const char* ,
int ) {
return false; }
365 virtual void ProcessGetFileSizeResult(
const char* ) {}
366 void SetURL(
const char* pszURL);
367 virtual bool Authenticate() {
return false; }
371 VSICurlHandle( VSICurlFilesystemHandler* poFS,
372 const char* pszFilename,
373 const char* pszURLIn =
nullptr );
374 ~VSICurlHandle()
override;
378 size_t Read(
void *pBuffer,
size_t nSize,
size_t nMemb )
override;
379 int ReadMultiRange(
int nRanges,
void ** ppData,
381 const size_t* panSizes )
override;
382 size_t Write(
const void *pBuffer,
size_t nSize,
size_t nMemb )
override;
384 int Flush()
override;
385 int Close()
override;
387 bool IsKnownFileSize()
const {
return oFileProp.bHasComputedFileSize; }
388 vsi_l_offset GetFileSizeOrHeaders(
bool bSetError,
bool bGetHeaders);
389 virtual vsi_l_offset GetFileSize(
bool bSetError ) {
return GetFileSizeOrHeaders(bSetError,
false); }
390 bool Exists(
bool bSetError );
391 bool IsDirectory()
const {
return oFileProp.bIsDirectory; }
392 int GetMode()
const {
return oFileProp.nMode; }
393 time_t GetMTime()
const {
return oFileProp.mTime; }
396 int InstallReadCbk( VSICurlReadCbkFunc pfnReadCbk,
398 int bStopOnInterruptUntilUninstall );
399 int UninstallReadCbk();
401 const char *GetURL()
const {
return m_pszURL; }
408 class IVSIS3LikeFSHandler:
public VSICurlFilesystemHandler
414 const char* pszSource,
415 const char* pszTarget,
416 GDALProgressFunc pProgressFunc,
417 void *pProgressData);
418 virtual int MkdirInternal(
const char *pszDirname,
long nMode,
bool bDoStatCheck );
421 char** GetFileList(
const char *pszFilename,
423 bool* pbGotFileList )
override;
425 virtual IVSIS3LikeHandleHelper* CreateHandleHelper(
426 const char* pszURI,
bool bAllowNoObject) = 0;
428 virtual int CopyObject(
const char *oldpath,
const char *newpath,
431 IVSIS3LikeFSHandler() =
default;
434 int Unlink(
const char *pszFilename )
override;
435 int Mkdir(
const char *pszDirname,
long nMode )
override;
436 int Rmdir(
const char *pszDirname )
override;
437 int Stat(
const char *pszFilename,
VSIStatBufL *pStatBuf,
438 int nFlags )
override;
439 int Rename(
const char *oldpath,
const char *newpath )
override;
441 virtual int DeleteObject(
const char *pszFilename );
443 virtual void UpdateMapFromHandle(IVSIS3LikeHandleHelper*) {}
444 virtual void UpdateHandleFromMap( IVSIS3LikeHandleHelper * ) {}
446 virtual CPLString GetStreamingPath(
const char* pszFilename )
const;
448 bool Sync(
const char* pszSource,
const char* pszTarget,
449 const char*
const * papszOptions,
450 GDALProgressFunc pProgressFunc,
452 char*** ppapszOutputs )
override;
454 VSIDIR* OpenDir(
const char *pszPath,
int nRecurseDepth,
455 const char*
const *papszOptions)
override;
458 virtual bool SupportsParallelMultipartUpload()
const {
return false; }
460 virtual CPLString InitiateMultipartUpload(
461 const std::string& osFilename,
462 IVSIS3LikeHandleHelper *poS3HandleHelper,
468 const std::string& osUploadID,
470 const void* pabyBuffer,
472 IVSIS3LikeHandleHelper *poS3HandleHelper,
474 double dfRetryDelay);
475 virtual bool CompleteMultipart(
const CPLString& osFilename,
477 const std::vector<CPLString>& aosEtags,
479 IVSIS3LikeHandleHelper *poS3HandleHelper,
481 double dfRetryDelay);
482 virtual bool AbortMultipart(
const CPLString& osFilename,
484 IVSIS3LikeHandleHelper *poS3HandleHelper,
486 double dfRetryDelay);
493 class IVSIS3LikeHandle:
public VSICurlHandle
498 bool UseLimitRangeGetInsteadOfHead()
override {
return true; }
499 bool IsDirectoryFromExists(
const char* pszVerb,
500 int response_code )
override
503 return response_code == 416 &&
EQUAL(pszVerb,
"GET") &&
506 void ProcessGetFileSizeResult(
const char* pszContent )
override
508 oFileProp.bIsDirectory = strstr(pszContent,
"ListBucketResult") !=
nullptr;
512 IVSIS3LikeHandle( VSICurlFilesystemHandler* poFSIn,
513 const char* pszFilename,
514 const char* pszURLIn ) :
515 VSICurlHandle(poFSIn, pszFilename, pszURLIn) {}
516 ~IVSIS3LikeHandle()
override {}
527 IVSIS3LikeFSHandler *m_poFS =
nullptr;
529 IVSIS3LikeHandleHelper *m_poS3HandleHelper =
nullptr;
530 bool m_bUseChunked =
false;
534 int m_nBufferOff = 0;
535 int m_nBufferSize = 0;
536 bool m_bClosed =
false;
537 GByte *m_pabyBuffer =
nullptr;
539 int m_nPartNumber = 0;
540 std::vector<CPLString> m_aosEtags{};
541 bool m_bError =
false;
543 CURLM *m_hCurlMulti =
nullptr;
544 CURL *m_hCurl =
nullptr;
545 const void *m_pBuffer =
nullptr;
547 size_t m_nChunkedBufferOff = 0;
548 size_t m_nChunkedBufferSize = 0;
549 size_t m_nWrittenInPUT = 0;
552 double m_dfRetryDelay = 0.0;
553 WriteFuncStruct m_sWriteFuncHeaderData{};
556 bool DoSinglePartPUT();
558 static size_t ReadCallBackBufferChunked(
char *buffer,
size_t size,
559 size_t nitems,
void *instream );
560 size_t WriteChunked(
const void *pBuffer,
561 size_t nSize,
size_t nMemb );
562 int FinishChunkedTransfer();
564 void InvalidateParentDirectory();
567 VSIS3WriteHandle( IVSIS3LikeFSHandler* poFS,
568 const char* pszFilename,
569 IVSIS3LikeHandleHelper* poS3HandleHelper,
572 ~VSIS3WriteHandle()
override;
576 size_t Read(
void *pBuffer,
size_t nSize,
size_t nMemb )
override;
577 size_t Write(
const void *pBuffer,
size_t nSize,
size_t nMemb )
override;
579 int Close()
override;
581 bool IsOK() {
return m_bUseChunked || m_pabyBuffer !=
nullptr; }
594 VSICurlFilesystemHandler* m_poFS =
nullptr;
599 int m_nBufferOff = 0;
600 int m_nBufferSize = 0;
601 int m_nBufferOffReadCallback = 0;
602 bool m_bClosed =
false;
603 GByte *m_pabyBuffer =
nullptr;
604 bool m_bError =
false;
606 static size_t ReadCallBackBuffer(
char *buffer,
size_t size,
607 size_t nitems,
void *instream );
608 virtual bool Send(
bool bIsLastBlock) = 0;
611 VSIAppendWriteHandle( VSICurlFilesystemHandler* poFS,
612 const char* pszFSPrefix,
613 const char* pszFilename,
615 virtual ~VSIAppendWriteHandle();
619 size_t Read(
void *pBuffer,
size_t nSize,
size_t nMemb )
override;
620 size_t Write(
const void *pBuffer,
size_t nSize,
size_t nMemb )
override;
622 int Close()
override;
624 bool IsOK() {
return m_pabyBuffer !=
nullptr; }
631 struct CurlRequestHelper
633 WriteFuncStruct sWriteFuncData{};
634 WriteFuncStruct sWriteFuncHeaderData{};
635 char szCurlErrBuf[CURL_ERROR_SIZE+1] = {};
638 ~CurlRequestHelper();
639 long perform(CURL* hCurlHandle,
640 struct curl_slist* headers,
641 VSICurlFilesystemHandler *poFS,
642 IVSIS3LikeHandleHelper *poS3HandleHelper);
649 class NetworkStatisticsLogger
651 static int gnEnabled;
652 static NetworkStatisticsLogger gInstance;
654 NetworkStatisticsLogger() =
default;
656 std::mutex m_mutex{};
665 GIntBig nGETDownloadedBytes = 0;
667 GIntBig nPOSTDownloadedBytes = 0;
668 GIntBig nPOSTUploadedBytes = 0;
671 enum class ContextPathType
678 struct ContextPathItem
680 ContextPathType eType;
683 ContextPathItem(ContextPathType eTypeIn,
const CPLString& osNameIn):
684 eType(eTypeIn), osName(osNameIn) {}
686 bool operator< (
const ContextPathItem& other )
const
688 if(
static_cast<int>(eType) <
static_cast<int>(other.eType) )
690 if(
static_cast<int>(eType) >
static_cast<int>(other.eType) )
692 return osName < other.osName;
699 std::map<ContextPathItem, Stats> children{};
705 std::map<GIntBig, std::vector<ContextPathItem>> m_mapThreadIdToContextPath{};
707 static void ReadEnabled();
709 std::vector<Counters*> GetCountersForContext();
713 static inline bool IsEnabled()
719 return gnEnabled == TRUE;
722 static void EnterFileSystem(
const char* pszName);
724 static void LeaveFileSystem();
726 static void EnterFile(
const char* pszName);
728 static void LeaveFile();
730 static void EnterAction(
const char* pszName);
732 static void LeaveAction();
734 static void LogHEAD();
736 static void LogGET(
size_t nDownloadedBytes);
738 static void LogPUT(
size_t nUploadedBytes);
740 static void LogPOST(
size_t nUploadedBytes,
741 size_t nDownloadedBytes);
743 static void LogDELETE();
747 static CPLString GetReportAsSerializedJSON();
750 struct NetworkStatisticsFileSystem
752 inline explicit NetworkStatisticsFileSystem(
const char* pszName) {
753 NetworkStatisticsLogger::EnterFileSystem(pszName);
756 inline ~NetworkStatisticsFileSystem()
758 NetworkStatisticsLogger::LeaveFileSystem();
762 struct NetworkStatisticsFile
764 inline explicit NetworkStatisticsFile(
const char* pszName) {
765 NetworkStatisticsLogger::EnterFile(pszName);
768 inline ~NetworkStatisticsFile()
770 NetworkStatisticsLogger::LeaveFile();
774 struct NetworkStatisticsAction
776 inline explicit NetworkStatisticsAction(
const char* pszName) {
777 NetworkStatisticsLogger::EnterAction(pszName);
780 inline ~NetworkStatisticsAction()
782 NetworkStatisticsLogger::LeaveAction();
787 int VSICURLGetDownloadChunkSize();
789 void VSICURLInitWriteFuncStruct( WriteFuncStruct *psStruct,
791 VSICurlReadCbkFunc pfnReadCbk,
792 void *pReadCbkUserData );
793 size_t VSICurlHandleWriteFunc(
void *buffer,
size_t count,
794 size_t nmemb,
void *req );
795 void MultiPerform(CURLM* hCurlMultiHandle,
796 CURL* hEasyHandle =
nullptr);
797 void VSICURLResetHeaderAndWriterFunctions(CURL* hCurlHandle);
799 int VSICurlParseUnixPermissions(
const char* pszPermissions);
The CPLJSONArray class holds JSON object from CPLJSONDocument.
Definition: cpl_json.h:54
String list class designed around our use of C "char**" string lists.
Definition: cpl_string.h:442
Convenient string class based on std::string.
Definition: cpl_string.h:333
Virtual file handle.
Definition: cpl_vsi_virtual.h:56
Interface for read and write JSON documents.
Core portability definitions for CPL.
#define EQUAL(a, b)
Alias for strcasecmp() == 0.
Definition: cpl_port.h:576
#define CPL_DISALLOW_COPY_ASSIGN(ClassName)
Helper to remove the copy and assignment constructors so that the compiler will not generate the defa...
Definition: cpl_port.h:1007
char ** CSLConstList
Type of a constant null-terminated list of nul terminated strings.
Definition: cpl_port.h:1216
unsigned char GByte
Unsigned byte type.
Definition: cpl_port.h:215
long long GIntBig
Large signed integer type (generally 64-bit integer type).
Definition: cpl_port.h:248
Various convenience functions for working with strings and string lists.
#define VSI_L_OFFSET_MAX
Maximum value for a file offset.
Definition: cpl_vsi.h:142
struct VSIDIR VSIDIR
Opaque type for a directory iterator.
Definition: cpl_vsi.h:319
struct VSI_STAT64_T VSIStatBufL
Type for VSIStatL()
Definition: cpl_vsi.h:195
FILE VSILFILE
Opaque type for a FILE that implements the VSIVirtualHandle API.
Definition: cpl_vsi.h:156
GUIntBig vsi_l_offset
Type for a file offset.
Definition: cpl_vsi.h:140