FreeWRL / FreeX3D 4.3.0
io_http.c
1/*
2
3 FreeWRL support library.
4 IO with HTTP protocol.
5
6*/
7
8/****************************************************************************
9 This file is part of the FreeWRL/FreeX3D Distribution.
10
11 Copyright 2009 CRC Canada. (http://www.crc.gc.ca)
12
13 FreeWRL/FreeX3D is free software: you can redistribute it and/or modify
14 it under the terms of the GNU Lesser Public License as published by
15 the Free Software Foundation, either version 3 of the License, or
16 (at your option) any later version.
17
18 FreeWRL/FreeX3D is distributed in the hope that it will be useful,
19 but WITHOUT ANY WARRANTY; without even the implied warranty of
20 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
21 GNU General Public License for more details.
22
23 You should have received a copy of the GNU General Public License
24 along with FreeWRL/FreeX3D. If not, see <http://www.gnu.org/licenses/>.
25****************************************************************************/
26
27
28
29#include <config.h>
30#include <system.h>
31//#include <resources.h>
32//#include <display.h>
33#include <internal.h>
34
35#include <libFreeWRL.h>
36//#include <list.h>
37//
38//#include <io_files.h>
39//#include <io_http.h>
40//#include <threads.h>
41//#include "scenegraph/Vector.h"
42//#include "main/ProdCon.h"
43
44
45#ifdef _MSC_VER
46#define strncasecmp _strnicmp
47#endif
48
49bool is_url(const char *url)
50{
51#define MAX_PROTOS 4
52 static const char *protos[MAX_PROTOS] = { "ftp", "http", "https", "urn" };
53
54 int i;
55 char *pat;
56 unsigned long delta = 0;
57
58 pat = strstr(url, "://");
59 if (!pat) {
60 return FALSE;
61 }
62
63 delta = (long)pat - (long)url;
64 if (delta > 5) {
65 return FALSE;
66 }
67
68 for (i = 0; i < MAX_PROTOS ; i++) {
69 if (strncasecmp(protos[i], url, strlen(protos[i])) == 0) {
70 return TRUE;
71 }
72 }
73 return FALSE;
74}
75
76// we need to substitute %20 for ' ', as well replace other unsafe chars
77// ftp://ftp.gnu.org/old-gnu/Manuals/wget-1.8.1/html_chapter/wget_2.html#SEC3
78// http://www.rfc-base.org/txt/rfc-1738.txt
79
80static char *RFC1738_unsafe = " <>{}|\\^~[]`#%";
81static int is_unsafe(char c){
82 int j, unsafe = 0;
83 if(c < 32 || c > 126)
84 unsafe = 1;
85 else{
86 for(j=0;j<strlen(RFC1738_unsafe);j++){
87 if( c == RFC1738_unsafe[j]){
88 unsafe = 1;
89 break; //from j loop
90 }
91 }
92 }
93 return unsafe;
94}
95static int count_unsafe(char *str){
96 int i, count, len;
97 len = (int)strlen(str);
98 count = 0;
99 for(i=0;i<len;i++)
100 if(is_unsafe(str[i]))
101 count++;
102 return count;
103}
104static char *hexdigits = "0123456789ABCDEF";
105static char *replace_unsafe(char *str){
106 int i,j,n, len;
107 char *s;
108 len = (int)strlen(str);
109 n = count_unsafe(str);
110 if(n == 0) return strdup(str);
111 //printf("unsafe string=[%s]\n",str);
112 s = malloc(n*3 + len - n +1);
113 j = 0;
114 for(i=0;i<len;i++){
115 if(is_unsafe(str[i])){
116 s[j] = '%';
117 s[j+1] = hexdigits[str[i] >> 4];
118 s[j+2] = hexdigits[str[i] & 0x0f];
119 //if(str[i] != ' ')
120 //printf("unsafe char=%d %c\n",(int)str[i],str[i]);
121 j += 3;
122 }else{
123 s[j] = str[i];
124 j++;
125 }
126 }
127 s[j] = 0;
128 return s;
129}
130
131/*
132 New FreeWRL internal API for HTTP[S] downloads
133
134 - enqueue an URL for download
135 - test if the download is over
136 - open the downloaded file
137 - remove the downloaded file when neede
138 - {same as temp file}
139
140 * structure:
141 - download structure with URL, temp filename, status
142*/
143
144#if defined(HAVE_LIBCURL)
145
146/*
147 To be effectively used, libcurl has to be enabled
148 during configure, then it has to be enabled on
149 the command line (see main program: options.c).
150
151 Instead of downloading files at once with wget,
152 I propose to try libCurl which provides some nice
153 mechanism to reuse open connections and to multi-
154 thread downloads.
155
156 At the moment I've only hacked a basic mise en oeuvre
157 of this library.
158*/
159
160#include <curl/curl.h>
161
162int with_libcurl = TRUE;
163
164static int curl_initialized = 0;
165
166/*
167 libCurl needs to be initialized once.
168 We've choosen the very simple method of curl_easy_init
169 but, in the future we'll use the full features.
170 except with url2file_task_spawn in desktop.c we do 1 download per thread instance,
171 and the haxx multithreading example shows its not threadsafe so
172 one curl instance per thread
173*/
174void init_curl()
175{
176 CURLcode c;
177
178 if ( (c=curl_global_init(CURL_GLOBAL_ALL)) != 0 ) {
179 ERROR_MSG("Curl init failed: %d\n", (int)c);
180 curl_initialized = 0;
181 exit(1);
182 } else {
183 curl_initialized = 1;
184 }
185}
186/* return the temp file where we got the contents of the URL requested */
187/* old char* download_url_curl(const char *url, const char *tmp) */
188char* download_url_curl_OLD(char *parsed_request, char *temp_dir)
189{
190 CURL *curl_h = NULL;
191 CURLcode success;
192 char *temp;
193 FILE *file;
194
195 if (temp_dir) {
196 temp = STRDUP(temp_dir);
197 } else {
198 temp = TEMPNAM(gglobal()->Mainloop.tmpFileLocation, "freewrl_download_curl_XXXXXXXX");
199 if (!temp) {
200 PERROR_MSG("download_url_curl: can't create temporary name.\n");
201 return NULL;
202 }
203 }
204
205 file = fopen(temp, "w");
206 if (!file) {
207 FREE(temp);
208 ERROR_MSG("Cannot create temp file (fopen)\n");
209 return NULL;
210 }
211
212 if (curl_initialized != 0) {
213 init_curl();
214 }
215
216 curl_h = curl_easy_init();
217
218 /*
219 Ask libCurl to download one url at once,
220 and to write it to the specified file.
221 */
222 curl_easy_setopt(curl_h, CURLOPT_URL, parsed_request);
223
224 curl_easy_setopt(curl_h, CURLOPT_WRITEDATA, file);
225
226 success = curl_easy_perform(curl_h);
227
228 if (success != CURLE_OK) {
229 ERROR_MSG("Download failed for url %s\n", parsed_request);
230 fprintf(stderr, "curl_easy_perform() failed: %s\n", curl_easy_strerror(success));
231 fclose(file);
232 unlink(temp);
233 FREE(temp);
234 return NULL;
235 } else {
236#ifdef TRACE_DOWNLOADS
237 TRACE_MSG("Download sucessfull [curl] for url %s\n", parsed_request);
238#endif
239 fclose(file);
240 return temp;
241 }
242}
243
244char* download_url_curl(char *parsed_request, char *temp_dir)
245{
246 CURL *curl_h = NULL;
247 CURLcode success;
248 char *temp, *safe_url;
249 FILE *file;
250
251 if (temp_dir) {
252 temp = STRDUP(temp_dir);
253 } else {
254 temp = TEMPNAM(gglobal()->Mainloop.tmpFileLocation, "freewrl_download_curl_XXXXXXXX");
255 if (!temp) {
256 PERROR_MSG("download_url_curl: can't create temporary name.\n");
257 return NULL;
258 }
259 }
260
261 file = fopen(temp, "w");
262 if (!file) {
263 FREE(temp);
264 ERROR_MSG("Cannot create temp file (fopen)\n");
265 return NULL;
266 }
267
268 // https://curl.haxx.se/libcurl/c/threadsafe.html
269 //https://curl.haxx.se/libcurl/c/multithread.html
270 //- we need a separate handle for each thread which we do with url2file_tactic_spawn in desktop.c
271 curl_h = curl_easy_init();
272
273 // curl_h = curl_easy_init();
274 safe_url = replace_unsafe(parsed_request);
275 /*
276 Ask libCurl to download one url at once,
277 and to write it to the specified file.
278 */
279 curl_easy_setopt(curl_h, CURLOPT_URL, safe_url);
280
281 curl_easy_setopt(curl_h, CURLOPT_WRITEDATA, file);
282
283 success = curl_easy_perform(curl_h);
284
285 if(success == CURLE_OK){
286 // easy_perform returns CURLE_OK even if it couldn't download the file
287 // https://curl.haxx.se/docs/faq.html#Why_do_I_get_downloaded_data_eve
288 //
289 // https://curl.haxx.se/libcurl/c/curl_easy_getinfo.html
290 // CURLcode curl_easy_getinfo(CURL *curl, CURLINFO info, ... ); where ... is int, double ... as per the request
291 long response_code;
292 curl_easy_getinfo(curl_h, CURLINFO_RESPONSE_CODE, &response_code);
293 if(response_code == 200){
294 fclose(file);
295 free(safe_url);
296 curl_easy_cleanup(curl_h);
297 return temp;
298 }
299 }
300 //else if (success != CURLE_OK) or response == 404
301 //if (success != CURLE_OK) fprintf(stderr, "curl_easy_perform() failed: %s\n", curl_easy_strerror(success));
302 fclose(file);
303 unlink(temp);
304 free(safe_url);
305 free(temp);
306 curl_easy_cleanup(curl_h);
307 return NULL;
308}
309
310
311#endif /* HAVE_LIBCURL */
312
313
314#if defined(HAVE_WININET)
315
316/*
317ms windows native methods WinInet - for clients (like freewrl)
318http://msdn.microsoft.com/en-us/library/aa385331(VS.85).aspx C - what I used below
319http://msdn.microsoft.com/en-us/library/sb35xf67.aspx sample browser in C++
320*/
321#include <WinInet.h>
322
323static HINTERNET hWinInet = NULL; //static for an application, although multiple inits OK
324HINTERNET winInetInit()
325{
326//winInet_h = InternetOpen(
327// __in LPCTSTR lpszAgent,
328// __in DWORD dwAccessType,
329// __in LPCTSTR lpszProxyName,
330// __in LPCTSTR lpszProxyBypass,
331// __in DWORD dwFlags
332//);
333
334
335 if(hWinInet == NULL)
336 hWinInet = InternetOpen("freewrl",INTERNET_OPEN_TYPE_DIRECT,NULL,NULL,0); //INTERNET_OPEN_TYPE_PRECONFIG_WITH_NO_AUTOPROXY/*/INTERNET_OPEN_TYPE_PRECONFIG*/,NULL,NULL,0);//INTERNET_FLAG_ASYNC - for callback);
337 return hWinInet;
338}
339void closeWinInetHandle()
340{
341 InternetCloseHandle(hWinInet);
342}
343
344//#define ERROR_MSG ConsoleMessage
345//#define PERROR_MSG ConsoleMessage
346 /* return the temp file where we got the contents of the URL requested */
347/* char* download_url_WinInet(const char *url, const char *tmp) */
348char* download_url_WinInet(char *parsed_request, char *temp_dir)
349{
350 char *temp;
351 temp = NULL;
352 if(!hWinInet)
353 {
354 hWinInet = winInetInit();
355 }
356 if(!hWinInet)
357 return temp;
358 else
359 {
360 DWORD dataLength, len;
361 HINTERNET hOpenUrl;
362 DWORD buflen;
363 //DWORD dwError;
364 DWORD InfoLevel, Index;
365 char buffer[1024];
366
367 if(0){
368 static FILE* fp = NULL;
369 if (!fp) fp = fopen("http_log.txt", "w+");
370 fprintf(fp,"[%s]\n", parsed_request);
371 }
372 hOpenUrl=InternetOpenUrl(hWinInet,parsed_request,NULL,0,0,0); //INTERNET_FLAG_NO_UI|INTERNET_FLAG_RELOAD/*|INTERNET_FLAG_IGNORE_CERT_CN_INVALID install the cert instead*/,0);
373 buflen = 1023;
374 //if(InternetGetLastResponseInfo(&dwError,buffer,&buflen)){
375 // printf("error=%s\n",buffer);
376 //}else{
377 // printf("no error\n");
378 //}
379// BOOL HttpQueryInfo(
380// _In_ HINTERNET hRequest,
381// _In_ DWORD dwInfoLevel,
382// _Inout_ LPVOID lpvBuffer,
383// _Inout_ LPDWORD lpdwBufferLength,
384// _Inout_ LPDWORD lpdwIndex
385//);
386 //http://msdn.microsoft.com/en-us/library/aa384238(v=vs.85).aspx
387 buflen = 1023;
388 Index = 0;
389 InfoLevel = HTTP_QUERY_RAW_HEADERS_CRLF;
390 if(HttpQueryInfo(hOpenUrl,InfoLevel,buffer,&buflen,NULL)){
391 //printf("query buffer=%s\n",buffer);
392 if(strstr(buffer,"404 Not Found")){
393 //HTTP/1.1 404 Not Found
394 ERROR_MSG("Download failed1 for url %s\n", parsed_request);
395 return temp;
396 }
397 //else 200 OK
398 }
399 //else{
400 // printf("no query buffer\n");
401 //}
402
403
404 //DWORD err = GetLastError();
405 if (!(hOpenUrl))
406 {
407 ERROR_MSG("Download failed2 for url %s\n", parsed_request);
408 return temp;
409 }
410 else
411 {
412 FILE *file;
413
414 if (temp_dir) {
415 temp = STRDUP(temp_dir);
416 } else {
417 //temp = _tempnam(gglobal()->Mainloop.tmpFileLocation, "freewrl_download_XXXXXXXX");
418 char *tmp1;
419 tmp1 = _tempnam(NULL, "freewrl_download_XXXXXXXX");
420 if (!tmp1) {
421 PERROR_MSG("download_url: can't create temporary name.\n");
422 return tmp1;
423 }
424 temp = STRDUP(tmp1); //these 2 lines help DEBUG_MALLOC because later we use FREE_IF_NZ on actual_file
425 free(tmp1);
426 }
427
428 file = fopen(temp, "wb");
429 if (!file) {
430 FREE_IF_NZ(temp);
431 ERROR_MSG("Cannot create temp file (fopen)\n");
432 return temp;
433 }
434
435 dataLength=0;
436 len=0;
437
438 while((InternetQueryDataAvailable(hOpenUrl,&dataLength,0,0))&&(dataLength>0))
439 {
440 void *block = MALLOC(void *, dataLength);
441 if ((InternetReadFile(hOpenUrl,(void*)block,dataLength,&len))&&(len>0))
442 {
443 fwrite(block,dataLength,1,file);
444 }
445 FREE(block);
446 }
447 InternetCloseHandle(hOpenUrl);
448 hOpenUrl=NULL;
449 fclose(file);
450 return temp;
451 }
452 }
453 return temp;
454}
455
456#endif
457
458/* lets try this... this should be in the config files */
459#ifdef WGET
460#define HAVE_WGET
461#endif
462
463#ifdef HAVE_WGET
464
465
471char* download_url_wget(char *parsed_request, char *temp_dir)
472{
473 char *temp, *wgetcmd, *safe;
474 int ret;
475
476// OLD_IPHONE_AQUA #if defined (TARGET_AQUA)
477// OLD_IPHONE_AQUA #define WGET_OPTIONS ""
478// OLD_IPHONE_AQUA #define WGET_OUTPUT_DIRECT "-o"
479// OLD_IPHONE_AQUA #else
480
481 // move this to another place (where we check wget options)
482 #define WGET_OPTIONS "--no-check-certificate"
483 #define WGET_OUTPUT_DIRECT "-O"
484
485// OLD_IPHONE_AQUA #endif
486
487 // create temp filename
488 if (temp_dir) {
489 temp = STRDUP(temp_dir);
490 } else {
491 temp = TEMPNAM(gglobal()->Mainloop.tmpFileLocation, "freewrl_download_wget_XXXXXXXX");
492 if (!temp) {
493 PERROR_MSG("download_url_wget: can't create temporary name.\n");
494 return NULL;
495 }
496 }
497
498 // create wget command line
499 safe = replace_unsafe(parsed_request);
500 wgetcmd = MALLOC(void *, strlen(WGET) +
501 strlen(WGET_OPTIONS) +
502 strlen(safe) +
503 strlen(temp) + 6 +1+1);
504
505// OLD_IPHONE_AQUA #if defined (TARGET_AQUA)
506// OLD_IPHONE_AQUA /* AQUA - we DO NOT have the options, but we can not have the space - it screws the freewrlSystem up */
507// OLD_IPHONE_AQUA sprintf(wgetcmd, "%s %s %s %s", WGET, safe, WGET_OUTPUT_DIRECT, temp);
508// OLD_IPHONE_AQUA #else
509
510 sprintf(wgetcmd, "%s %s %s %s %s", WGET, WGET_OPTIONS, safe, WGET_OUTPUT_DIRECT, temp);
511
512// OLD_IPHONE_AQUA #endif
513
514 FREE_IF_NZ(safe);
515 /* printf ("wgetcmd is %s\n",wgetcmd); */
516
517 // call wget
518 ret = freewrlSystem(wgetcmd);
519 if (ret < 0) {
520 ERROR_MSG("Error in wget (%s)\n", wgetcmd);
521 FREE(temp);
522 FREE(wgetcmd);
523 return NULL;
524 }
525 FREE(wgetcmd);
526 return temp;
527}
528#endif /* HAVE_WGET */
529
530
531/* get the file from the network, UNLESS the front end gets files. Old way - the freewrl
532 library tries to get files; new way, the front end gets the files from the network. This
533 allows, for instance, the browser plugin to use proxy servers and cache for getting files.
534
535 So, if the FRONTEND_GETS_FILES is set, we simply pass the http:// filename along....
536*/
537void close_internetHandles()
538{
539#ifdef HAVE_WININET
540 closeWinInetHandle();
541#endif
542}
543void download_url(void *res)
544{
545 char *temp_dir, *parsed_request, *actual_file;
546 parsed_request = fwl_resitem_getURL(res);
547 temp_dir = fwl_resitem_getTempDir(res);
548
549 actual_file = NULL;
550#if defined(HAVE_LIBCURL)
551 if (with_libcurl)
552 actual_file = download_url_curl(parsed_request,temp_dir);
553 #ifdef HAVE_WGET
554 else
555 actual_file = download_url_wget(parsed_request,temp_dir);
556 #endif //HAVE_WGET
557
558#elif defined (HAVE_WGET)
559 actual_file = download_url_wget(parsed_request,temp_dir);
560
561
562#elif defined (HAVE_WININET)
563 actual_file = download_url_WinInet(parsed_request,temp_dir);
564#endif
565
566 /* status indication now */
567 if (actual_file) {
568 /* download succeeded */
569 //res->status = ress_downloaded;
570 fwl_resitem_setStatus(res,ress_downloaded);
571 fwl_resitem_setActualFile(res,actual_file);
572 //moved inside setActualFile:
573 //if(strcmp(actual_file,parsed_request)){
574 // //it's a temp file
575 // s_list_t *item;
576 // item = ml_new(res->actual_file);
577 // if (!res->cached_files)
578 // res->cached_files = (void *)item;
579 // else
580 // res->cached_files = ml_append(res->cached_files,item);
581 //}
582 } else {
583 /* download failed */
584 //res->status = ress_failed;
585 fwl_resitem_setStatus(res,ress_failed);
586 ERROR_MSG("resource_fetch: download failed for url: %s\n", parsed_request);
587 }
588}