29 #ifndef CPL_VSIL_CURL_CLASS_H_INCLUDED
30 #define CPL_VSIL_CURL_CLASS_H_INCLUDED
37 #include "cpl_vsil_curl_priv.h"
38 #include "cpl_mem_cache.h"
40 #include <curl/curl.h>
49 #if LIBCURL_VERSION_NUM >= 0x071201
50 #define HAVE_CURLINFO_REDIRECT_URL
53 void VSICurlStreamingClearCache(
void );
55 struct curl_slist* VSICurlSetOptions(CURL* hCurlHandle,
const char* pszURL,
56 const char *
const* papszOptions);
57 struct curl_slist* VSICurlMergeHeaders(
struct curl_slist* poDest,
58 struct curl_slist* poSrcToDestroy );
72 unsigned int nGenerationAuthParameters = 0;
73 ExistStatus eExists = EXIST_UNKNOWN;
76 time_t nExpireTimestampLocal = 0;
78 bool bHasComputedFileSize =
false;
79 bool bIsDirectory =
false;
80 bool bS3LikeRedirect =
false;
86 bool bGotFileList =
false;
87 unsigned int nGenerationAuthParameters = 0;
91 struct WriteFuncStruct
93 char* pBuffer =
nullptr;
96 bool bIsInHeader =
false;
97 bool bMultiRange =
false;
102 bool bFoundContentRange =
false;
104 bool bDownloadHeaderOnly =
false;
105 bool bDetectRangeDownloadingError =
false;
109 VSICurlReadCbkFunc pfnReadCbk =
nullptr;
110 void *pReadCbkUserData =
nullptr;
111 bool bInterrupted =
false;
113 #if LIBCURL_VERSION_NUM < 0x073600
117 bool bIsProxyConnectHeader =
false;
127 class VSICurlFilesystemHandler :
public VSIFilesystemHandler
131 struct FilenameOffsetPair
133 std::string filename_;
136 FilenameOffsetPair(
const std::string& filename,
138 filename_(filename), offset_(offset) {}
140 bool operator==(
const FilenameOffsetPair& other)
const
142 return filename_ == other.filename_ &&
143 offset_ == other.offset_;
146 struct FilenameOffsetPairHasher
148 std::size_t operator()(
const FilenameOffsetPair& k)
const
150 return std::hash<std::string>()(k.filename_) ^
151 std::hash<vsi_l_offset>()(k.offset_);
155 using RegionCacheType =
156 lru11::Cache<FilenameOffsetPair, std::shared_ptr<std::string>,
160 typename std::list<lru11::KeyValuePair<FilenameOffsetPair,
161 std::shared_ptr<std::string>>>::iterator,
162 FilenameOffsetPairHasher>>;
164 RegionCacheType oRegionCache;
166 lru11::Cache<std::string, FileProp> oCacheFileProp;
168 int nCachedFilesInDirList = 0;
169 lru11::Cache<std::string, CachedDirList> oCacheDirList;
171 char** ParseHTMLFileList(
const char* pszFilename,
174 bool* pbGotFileList);
177 CPLMutex *hMutex =
nullptr;
179 virtual VSICurlHandle* CreateFileHandle(
const char* pszFilename);
180 virtual char** GetFileList(
const char *pszFilename,
182 bool* pbGotFileList);
184 void RegisterEmptyDir(
const CPLString& osDirname );
186 bool AnalyseS3FileList(
const CPLString& osBaseURL,
190 bool bIgnoreGlacierStorageClass,
191 bool& bIsTruncated );
193 void AnalyseSwiftFileList(
const CPLString& osBaseURL,
197 int nMaxFilesThisQuery,
202 static const char* GetOptionsStatic();
204 static bool IsAllowedFilename(
const char* pszFilename );
207 VSICurlFilesystemHandler();
208 ~VSICurlFilesystemHandler()
override;
211 const char *pszAccess,
212 bool bSetError )
override;
214 int Stat(
const char *pszFilename,
VSIStatBufL *pStatBuf,
215 int nFlags )
override;
216 int Unlink(
const char *pszFilename )
override;
217 int Rename(
const char *oldpath,
const char *newpath )
override;
218 int Mkdir(
const char *pszDirname,
long nMode )
override;
219 int Rmdir(
const char *pszDirname )
override;
220 char **ReadDir(
const char *pszDirname )
override
221 {
return ReadDirEx(pszDirname, 0); }
222 char **ReadDirEx(
const char *pszDirname,
int nMaxFiles )
override;
224 int HasOptimizedReadMultiRange(
const char* )
225 override {
return true; }
227 const char* GetActualURL(
const char* pszFilename)
override;
229 const char* GetOptions()
override;
231 char** GetFileMetadata(
const char * pszFilename,
const char* pszDomain,
234 char **ReadDirInternal(
const char *pszDirname,
int nMaxFiles,
235 bool* pbGotFileList );
236 void InvalidateDirContent(
const char *pszDirname );
238 virtual CPLString GetFSPrefix() {
return "/vsicurl/"; }
239 virtual bool AllowCachedDataFor(
const char* pszFilename);
241 std::shared_ptr<std::string> GetRegion(
const char* pszURL,
244 void AddRegion(
const char* pszURL,
249 bool GetCachedFileProp(
const char* pszURL,
250 FileProp& oFileProp );
251 void SetCachedFileProp(
const char* pszURL,
252 FileProp& oFileProp );
253 void InvalidateCachedData(
const char* pszURL );
255 CURLM *GetCurlMultiHandleFor(
const CPLString& osURL );
257 virtual void ClearCache();
258 virtual void PartialClearCache(
const char* pszFilename);
261 bool GetCachedDirList(
const char* pszURL,
262 CachedDirList& oCachedDirList );
263 void SetCachedDirList(
const char* pszURL,
264 CachedDirList& oCachedDirList );
265 bool ExistsInCacheDirList(
const CPLString& osDirname,
bool *pbIsDir );
279 VSICurlFilesystemHandler* poFS =
nullptr;
281 bool m_bCached =
true;
283 FileProp oFileProp{};
286 char* m_pszURL =
nullptr;
288 char **m_papszHTTPOptions =
nullptr;
291 int nBlocksToDownload = 1;
293 bool bStopOnInterruptUntilUninstall =
false;
294 bool bInterrupted =
false;
295 VSICurlReadCbkFunc pfnReadCbk =
nullptr;
296 void *pReadCbkUserData =
nullptr;
299 double m_dfRetryDelay = 0.0;
303 void DownloadRegionPostProcess(
const vsi_l_offset startOffset,
314 virtual std::string DownloadRegion(
vsi_l_offset startOffset,
int nBlocks);
316 bool m_bUseHead =
false;
318 int ReadMultiRangeSingleGet(
int nRanges,
void ** ppData,
320 const size_t* panSizes );
321 CPLString GetRedirectURLIfValid(
bool& bHasExpired);
324 virtual struct curl_slist* GetCurlHeaders(
const CPLString& ,
325 const struct curl_slist* )
327 virtual bool AllowAutomaticRedirection() {
return true; }
328 virtual bool CanRestartOnError(
const char*,
const char*,
bool ) {
return false; }
329 virtual bool UseLimitRangeGetInsteadOfHead() {
return false; }
330 virtual bool IsDirectoryFromExists(
const char* ,
int ) {
return false; }
331 virtual void ProcessGetFileSizeResult(
const char* ) {}
332 void SetURL(
const char* pszURL);
333 virtual bool Authenticate() {
return false; }
337 VSICurlHandle( VSICurlFilesystemHandler* poFS,
338 const char* pszFilename,
339 const char* pszURLIn =
nullptr );
340 ~VSICurlHandle()
override;
344 size_t Read(
void *pBuffer,
size_t nSize,
size_t nMemb )
override;
345 int ReadMultiRange(
int nRanges,
void ** ppData,
347 const size_t* panSizes )
override;
348 size_t Write(
const void *pBuffer,
size_t nSize,
size_t nMemb )
override;
350 int Flush()
override;
351 int Close()
override;
353 bool IsKnownFileSize()
const {
return oFileProp.bHasComputedFileSize; }
354 vsi_l_offset GetFileSizeOrHeaders(
bool bSetError,
bool bGetHeaders);
355 virtual vsi_l_offset GetFileSize(
bool bSetError ) {
return GetFileSizeOrHeaders(bSetError,
false); }
356 bool Exists(
bool bSetError );
357 bool IsDirectory()
const {
return oFileProp.bIsDirectory; }
358 time_t GetMTime()
const {
return oFileProp.mTime; }
361 int InstallReadCbk( VSICurlReadCbkFunc pfnReadCbk,
363 int bStopOnInterruptUntilUninstall );
364 int UninstallReadCbk();
366 const char *GetURL()
const {
return m_pszURL; }
373 class IVSIS3LikeFSHandler:
public VSICurlFilesystemHandler
379 const char* pszSource,
380 const char* pszTarget,
381 GDALProgressFunc pProgressFunc,
382 void *pProgressData);
383 int MkdirInternal(
const char *pszDirname,
bool bDoStatCheck );
386 char** GetFileList(
const char *pszFilename,
388 bool* pbGotFileList )
override;
390 virtual IVSIS3LikeHandleHelper* CreateHandleHelper(
391 const char* pszURI,
bool bAllowNoObject) = 0;
393 virtual int CopyObject(
const char *oldpath,
const char *newpath,
396 IVSIS3LikeFSHandler() =
default;
399 int Unlink(
const char *pszFilename )
override;
400 int Mkdir(
const char *pszDirname,
long nMode )
override;
401 int Rmdir(
const char *pszDirname )
override;
402 int Stat(
const char *pszFilename,
VSIStatBufL *pStatBuf,
403 int nFlags )
override;
404 int Rename(
const char *oldpath,
const char *newpath )
override;
406 virtual int DeleteObject(
const char *pszFilename );
408 virtual const char* GetDebugKey()
const = 0;
410 virtual void UpdateMapFromHandle(IVSIS3LikeHandleHelper*) {}
411 virtual void UpdateHandleFromMap( IVSIS3LikeHandleHelper * ) {}
413 bool Sync(
const char* pszSource,
const char* pszTarget,
414 const char*
const * papszOptions,
415 GDALProgressFunc pProgressFunc,
417 char*** ppapszOutputs )
override;
419 VSIDIR* OpenDir(
const char *pszPath,
int nRecurseDepth,
420 const char*
const *papszOptions)
override;
424 const std::string& osFilename,
425 IVSIS3LikeHandleHelper *poS3HandleHelper,
427 double dfRetryDelay);
430 const std::string& osUploadID,
431 const void* pabyBuffer,
433 IVSIS3LikeHandleHelper *poS3HandleHelper,
435 double dfRetryDelay);
436 bool CompleteMultipart(
const CPLString& osFilename,
438 const std::vector<CPLString> aosEtags,
439 IVSIS3LikeHandleHelper *poS3HandleHelper,
441 double dfRetryDelay);
442 bool AbortMultipart(
const CPLString& osFilename,
444 IVSIS3LikeHandleHelper *poS3HandleHelper,
446 double dfRetryDelay);
453 class IVSIS3LikeHandle:
public VSICurlHandle
458 bool UseLimitRangeGetInsteadOfHead()
override {
return true; }
459 bool IsDirectoryFromExists(
const char* pszVerb,
460 int response_code )
override
463 return response_code == 416 &&
EQUAL(pszVerb,
"GET") &&
466 void ProcessGetFileSizeResult(
const char* pszContent )
override
468 oFileProp.bIsDirectory = strstr(pszContent,
"ListBucketResult") !=
nullptr;
472 IVSIS3LikeHandle( VSICurlFilesystemHandler* poFSIn,
473 const char* pszFilename,
474 const char* pszURLIn =
nullptr ) :
475 VSICurlHandle(poFSIn, pszFilename, pszURLIn) {}
476 ~IVSIS3LikeHandle()
override {}
487 IVSIS3LikeFSHandler *m_poFS =
nullptr;
489 IVSIS3LikeHandleHelper *m_poS3HandleHelper =
nullptr;
490 bool m_bUseChunked =
false;
493 int m_nBufferOff = 0;
494 int m_nBufferSize = 0;
495 bool m_bClosed =
false;
496 GByte *m_pabyBuffer =
nullptr;
498 int m_nPartNumber = 0;
499 std::vector<CPLString> m_aosEtags{};
500 bool m_bError =
false;
502 CURLM *m_hCurlMulti =
nullptr;
503 CURL *m_hCurl =
nullptr;
504 const void *m_pBuffer =
nullptr;
506 size_t m_nChunkedBufferOff = 0;
507 size_t m_nChunkedBufferSize = 0;
510 double m_dfRetryDelay = 0.0;
511 WriteFuncStruct m_sWriteFuncHeaderData{};
514 bool DoSinglePartPUT();
516 static size_t ReadCallBackBufferChunked(
char *buffer,
size_t size,
517 size_t nitems,
void *instream );
518 size_t WriteChunked(
const void *pBuffer,
519 size_t nSize,
size_t nMemb );
520 int FinishChunkedTransfer();
522 void InvalidateParentDirectory();
525 VSIS3WriteHandle( IVSIS3LikeFSHandler* poFS,
526 const char* pszFilename,
527 IVSIS3LikeHandleHelper* poS3HandleHelper,
529 ~VSIS3WriteHandle()
override;
533 size_t Read(
void *pBuffer,
size_t nSize,
size_t nMemb )
override;
534 size_t Write(
const void *pBuffer,
size_t nSize,
size_t nMemb )
override;
536 int Close()
override;
538 bool IsOK() {
return m_bUseChunked || m_pabyBuffer !=
nullptr; }
551 VSICurlFilesystemHandler* m_poFS =
nullptr;
556 int m_nBufferOff = 0;
557 int m_nBufferSize = 0;
558 int m_nBufferOffReadCallback = 0;
559 bool m_bClosed =
false;
560 GByte *m_pabyBuffer =
nullptr;
561 bool m_bError =
false;
563 static size_t ReadCallBackBuffer(
char *buffer,
size_t size,
564 size_t nitems,
void *instream );
565 virtual bool Send(
bool bIsLastBlock) = 0;
568 VSIAppendWriteHandle( VSICurlFilesystemHandler* poFS,
569 const char* pszFSPrefix,
570 const char* pszFilename,
572 virtual ~VSIAppendWriteHandle();
576 size_t Read(
void *pBuffer,
size_t nSize,
size_t nMemb )
override;
577 size_t Write(
const void *pBuffer,
size_t nSize,
size_t nMemb )
override;
579 int Close()
override;
581 bool IsOK() {
return m_pabyBuffer !=
nullptr; }
588 struct CurlRequestHelper
590 WriteFuncStruct sWriteFuncData{};
591 WriteFuncStruct sWriteFuncHeaderData{};
592 char szCurlErrBuf[CURL_ERROR_SIZE+1] = {};
595 ~CurlRequestHelper();
596 long perform(CURL* hCurlHandle,
597 struct curl_slist* headers,
598 VSICurlFilesystemHandler *poFS,
599 IVSIS3LikeHandleHelper *poS3HandleHelper);
602 int VSICURLGetDownloadChunkSize();
604 void VSICURLInitWriteFuncStruct( WriteFuncStruct *psStruct,
606 VSICurlReadCbkFunc pfnReadCbk,
607 void *pReadCbkUserData );
608 size_t VSICurlHandleWriteFunc(
void *buffer,
size_t count,
609 size_t nmemb,
void *req );
610 void MultiPerform(CURLM* hCurlMultiHandle,
611 CURL* hEasyHandle =
nullptr);
612 void VSICURLResetHeaderAndWriterFunctions(CURL* hCurlHandle);
620 #endif // CPL_VSIL_CURL_CLASS_H_INCLUDED