rpm  5.4.10
url.c
Go to the documentation of this file.
1 
5 #include "system.h"
6 
7 #include <netinet/in.h>
8 
9 #include <rpmmacro.h>
10 #include <rpmcb.h>
11 #include <rpmio_internal.h>
12 #ifdef WITH_NEON
13 #include <rpmdav.h>
14 #endif
15 
16 #include "debug.h"
17 
18 /*@access FD_t@*/ /* XXX compared with NULL */
19 /*@access urlinfo@*/
20 
21 #ifdef __cplusplus
22 GENfree(urlinfo *)
23 GENfree(rpmop)
24 #endif /* __cplusplus */
25 
26 #ifndef IPPORT_FTP
27 #define IPPORT_FTP 21
28 #endif
29 #ifndef IPPORT_HTTP
30 #define IPPORT_HTTP 80
31 #endif
32 #ifndef IPPORT_HTTPS
33 #define IPPORT_HTTPS 443
34 #endif
35 #ifndef IPPORT_PGPKEYSERVER
36 #define IPPORT_PGPKEYSERVER 11371
37 #endif
38 #ifndef IPPORT_MONGO
39 #define IPPORT_MONGO 27017
40 #endif
41 
42 #ifdef NOTYET
43 #define URL_IS_GIT
44 #define URL_IS_SVN
45 #define URL_IS_SQLITE
46 #define URL_IS_MYSQL
47 #define URL_IS_POSTGRES
48 #define URL_IS_SQLSERVER
49 #endif
50 
53 /*@-redecl@*/
54 int (*urlNotify) (const urlinfo u, unsigned status)
55  /*@*/;
56 /*@=redecl@*/
57 
60 /*@unchecked@*/ /*@null@*/
61 void * urlNotifyArg;
62 
65 /*@unchecked@*/
67 
70 /*@unchecked@*/
71 int _url_debug = 0;
72 
73 #define URLDBG(_f, _m, _x) if ((_url_debug | (_f)) & (_m)) fprintf _x
74 
75 #define URLDBGIO(_f, _x) URLDBG((_f), RPMURL_DEBUG_IO, _x)
76 #define URLDBGREFS(_f, _x) URLDBG((_f), RPMURL_DEBUG_REFS, _x)
77 
80 /*@unchecked@*/
81 /*@only@*/ /*@null@*/
83 
84 static void urlFini(void * _u)
85  /*@globals fileSystem, internalState @*/
86  /*@modifies _u, fileSystem, internalState @*/
87 {
88  urlinfo u = (urlinfo) _u;
89  int xx;
90 
91  if (u->ctrl) {
92 #ifndef NOTYET
93  void * fp = fdGetFp(u->ctrl);
94  if (fp) {
95  fdPush(u->ctrl, fpio, fp, -1); /* Push fpio onto stack */
96  xx = Fclose(u->ctrl);
97  } else if (fdFileno(u->ctrl) >= 0)
98  xx = fdio->close(u->ctrl);
99 #else
100  xx = Fclose(u->ctrl);
101 #endif
102 
103 /*@-usereleased@*/
104  u->ctrl = (FD_t)rpmioFreePoolItem((rpmioItem)u->ctrl, "persist ctrl (urlFree)", __FILE__, __LINE__);
105  if (u->ctrl)
106  fprintf(stderr, _("warning: u %p ctrl %p nrefs != 0 (%s %s)\n"),
107  u, u->ctrl, (u->host ? u->host : ""),
108  (u->scheme ? u->scheme : ""));
109 /*@=usereleased@*/
110  }
111  if (u->data) {
112 #ifndef NOTYET
113  void * fp = fdGetFp(u->data);
114  if (fp) {
115  fdPush(u->data, fpio, fp, -1); /* Push fpio onto stack */
116  xx = Fclose(u->data);
117  } else if (fdFileno(u->data) >= 0)
118  xx = fdio->close(u->data);
119 #else
120  xx = Fclose(u->ctrl);
121 #endif
122 
123 /*@-usereleased@*/
124  u->data = (FD_t)rpmioFreePoolItem((rpmioItem)u->data, "persist data (urlFree)", __FILE__, __LINE__);
125  if (u->data)
126  fprintf(stderr, _("warning: u %p data %p nrefs != 0 (%s %s)\n"),
127  u, u->data, (u->host ? u->host : ""),
128  (u->scheme ? u->scheme : ""));
129 /*@=usereleased@*/
130  }
131 #ifdef WITH_NEON
132  xx = davFree(u);
133 #endif
134  u->etag = _free(u->etag);
135  u->location = _free(u->location);
136  u->rop = _free(u->rop);
137  u->sop = _free(u->sop);
138  u->top = _free(u->top);
139  u->buf = _free(u->buf);
140  u->url = _free(u->url);
141  u->scheme = _free(u->scheme);
142  u->user = _free(u->user);
143  u->password = _free(u->password);
144  u->host = _free(u->host);
145  u->portstr = _free(u->portstr);
146  u->query = _free(u->query);
147  u->fragment = _free(u->fragment);
148  u->proxyu = _free(u->proxyu);
149  u->proxyh = _free(u->proxyh);
150 }
151 
154 /*@unchecked@*/
155 int _url_count = 0;
156 
157 /*@unchecked@*/ /*@only@*/ /*@null@*/
159 
160 static urlinfo urlGetPool(/*@null@*/ rpmioPool pool)
161  /*@globals _urlPool, fileSystem @*/
162  /*@modifies pool, _urlPool, fileSystem @*/
163 {
164  urlinfo u;
165 
166  if (_urlPool == NULL) {
167  _urlPool = rpmioNewPool("u", sizeof(*u), -1, _url_debug,
168  NULL, NULL, urlFini);
169  pool = _urlPool;
170  }
171  u = (urlinfo) rpmioGetPool(pool, sizeof(*u));
172  memset(((char *)u)+sizeof(u->_item), 0, sizeof(*u)-sizeof(u->_item));
173  return u;
174 }
175 
176 urlinfo XurlNew(const char *msg, const char *fn, unsigned ln)
177 {
178  urlinfo u = urlGetPool(_urlPool);
179 
180  u->proxyp = -1;
181  u->port = -1;
182  u->ut = URL_IS_UNKNOWN;
183  u->ctrl = NULL;
184  u->data = NULL;
185  u->location = NULL;
186  u->etag = NULL;
187  u->notify = urlNotify;
188 /*@-assignexpose@*/
189  u->arg = urlNotifyArg;
190 /*@=assignexpose@*/
191  u->rop = (rpmop) xcalloc(1, sizeof(*u->rop));
192  u->sop = (rpmop) xcalloc(1, sizeof(*u->sop));
193  u->top = (rpmop) xcalloc(1, sizeof(*u->top));
194  u->bufAlloced = 0;
195  u->buf = NULL;
197  u->httpVersion = 0;
198  u->magic = URLMAGIC;
199  return (urlinfo) rpmioLinkPoolItem((rpmioItem)u, msg, fn, ln);
200 }
201 
202 void urlFreeCache(void)
203 {
204  if (_url_cache) {
205  int i;
206  for (i = 0; i < _url_count; i++) {
207  if (_url_cache[i] == NULL) continue;
208  _url_cache[i] = urlFree(_url_cache[i], "_url_cache");
209  if (_url_cache[i] == NULL)
210  continue;
211  yarnPossess(_url_cache[i]->_item.use);
212  fprintf(stderr,
213  _("warning: _url_cache[%d] %p nrefs(%ld) != 1 (%s %s)\n"),
214  i, _url_cache[i], yarnPeekLock(_url_cache[i]->_item.use),
215  (_url_cache[i]->host ? _url_cache[i]->host : ""),
216  (_url_cache[i]->scheme ? _url_cache[i]->scheme : ""));
217  yarnRelease(_url_cache[i]->_item.use);
218  }
219  }
220  _url_cache = _free(_url_cache);
221  _url_count = 0;
222 }
223 
224 static int urlStrcmp(/*@null@*/ const char * str1, /*@null@*/ const char * str2)
225  /*@*/
226 {
227  if (str1)
228  if (str2)
229  return strcmp(str1, str2);
230  if (str1 != str2)
231  return -1;
232  return 0;
233 }
234 
235 /*@-mods@*/
236 static void urlFind(/*@null@*/ /*@in@*/ /*@out@*/ urlinfo * uret, int mustAsk)
237  /*@globals rpmGlobalMacroContext, h_errno, fileSystem, internalState @*/
238  /*@modifies *uret, rpmGlobalMacroContext, fileSystem, internalState @*/
239 {
240  urlinfo u;
241  int ucx;
242  int i = 0;
243 
244  if (uret == NULL)
245  return;
246 
247  u = *uret;
248  URLSANE(u);
249 
250  ucx = -1;
251  for (i = 0; i < _url_count; i++) {
252  urlinfo ou = NULL;
253  if (_url_cache == NULL || (ou = _url_cache[i]) == NULL) {
254  if (ucx < 0)
255  ucx = i;
256  continue;
257  }
258 
259  /* Check for cache-miss condition. A cache miss is
260  * a) both items are not NULL and don't compare.
261  * b) either of the items is not NULL.
262  */
263  if (urlStrcmp(u->scheme, ou->scheme))
264  continue;
265  if (urlStrcmp(u->host, ou->host))
266  continue;
267  if (urlStrcmp(u->user, ou->user))
268  continue;
269  if (urlStrcmp(u->portstr, ou->portstr))
270  continue;
271  break; /* Found item in cache */
272  }
273 
274  if (i == _url_count) {
275  if (ucx < 0) {
276  ucx = _url_count++;
277  _url_cache = (urlinfo *) xrealloc(_url_cache, sizeof(*_url_cache) * _url_count);
278  }
279  if (_url_cache) /* XXX always true */
280  _url_cache[ucx] = urlLink(u, "_url_cache (miss)");
281  u = urlFree(u, "urlSplit (urlFind miss)");
282  } else {
283  ucx = i;
284  u = urlFree(u, "urlSplit (urlFind hit)");
285  }
286 
287  /* This URL is now cached. */
288 
289  if (_url_cache) /* XXX always true */
290  u = urlLink(_url_cache[ucx], "_url_cache");
291  *uret = u;
292  /*@-usereleased@*/
293  u = urlFree(u, "_url_cache (urlFind)");
294  /*@=usereleased@*/
295 assert(u != NULL);
296 
297  /* Zap proxy host and port in case they have been reset */
298  u->proxyp = -1;
299  u->proxyh = _free(u->proxyh);
300 
301  /* Perform one-time FTP initialization */
302  if (u->ut == URL_IS_FTP) {
303 
304  if (mustAsk || (u->user != NULL && u->password == NULL)) {
305  const char * host = (u->host ? u->host : "");
306  const char * user = (u->user ? u->user : "");
307  char * prompt;
308  prompt = (char *) alloca(strlen(host) + strlen(user) + 256);
309  sprintf(prompt, _("Password for %s@%s: "), user, host);
310  u->password = _free(u->password);
311 /*@-dependenttrans -moduncon @*/
312  u->password = Getpass(prompt);
313 /*@=dependenttrans =moduncon @*/
314  if (u->password)
315  u->password = xstrdup(u->password);
316  }
317 
318  if (u->proxyh == NULL) {
319  const char *proxy = rpmExpand("%{_ftpproxy}", NULL);
320  if (proxy && *proxy != '%') {
321 /*@observer@*/
322  const char * host = (u->host ? u->host : "");
323  const char *uu = (u->user ? u->user : "anonymous");
324  char *nu = (char *)xmalloc(strlen(uu) + sizeof("@") + strlen(host));
325  (void) stpcpy( stpcpy( stpcpy(nu, uu), "@"), host);
326  u->proxyu = nu;
327  u->proxyh = xstrdup(proxy);
328  }
329  proxy = _free(proxy);
330  }
331 
332  if (u->proxyp < 0) {
333  const char *proxy = rpmExpand("%{_ftpport}", NULL);
334  if (proxy && *proxy != '%') {
335  char *end = NULL;
336  int port = strtol(proxy, &end, 0);
337  if (!(end && *end == '\0')) {
338  fprintf(stderr, _("error: %sport must be a number\n"),
339  (u->scheme ? u->scheme : ""));
340  return;
341  }
342  u->proxyp = port;
343  }
344  proxy = _free(proxy);
345  }
346  }
347 
348  /* Perform one-time HTTP initialization */
349  if (u->ut == URL_IS_HTTP || u->ut == URL_IS_HTTPS || u->ut == URL_IS_HKP) {
350 
351  if (u->proxyh == NULL) {
352  const char *proxy = rpmExpand("%{_httpproxy}", NULL);
353  if (proxy && *proxy != '%')
354  u->proxyh = xstrdup(proxy);
355  proxy = _free(proxy);
356  }
357 
358  if (u->proxyp < 0) {
359  const char *proxy = rpmExpand("%{_httpport}", NULL);
360  if (proxy && *proxy != '%') {
361  char *end;
362  int port = strtol(proxy, &end, 0);
363  if (!(end && *end == '\0')) {
364  fprintf(stderr, _("error: %sport must be a number\n"),
365  (u->scheme ? u->scheme : ""));
366  return;
367  }
368  u->proxyp = port;
369  }
370  proxy = _free(proxy);
371  }
372 
373  }
374 
375  return;
376 }
377 /*@=mods@*/
378 
381 /*@observer@*/ /*@unchecked@*/
382 static struct urlstring {
383 /*@observer@*/ /*@null@*/
384  const char *leadin;
385  size_t len;
387 } urlstrings[] = {
388  { "file://", sizeof("file://")-1, URL_IS_PATH },
389  { "ftp://", sizeof("ftp://")-1, URL_IS_FTP },
390  { "hkp://", sizeof("hkp://")-1, URL_IS_HKP },
391  { "http://", sizeof("http://")-1, URL_IS_HTTP },
392  { "https://", sizeof("https://")-1, URL_IS_HTTPS },
393  { "mongo://", sizeof("mongo://")-1, URL_IS_MONGO },
394  { "mongodb://", sizeof("mongodb://")-1, URL_IS_MONGO },
395 
396  { "git://", sizeof("git://")-1, URL_IS_GIT },
397  { "svn://", sizeof("svn://")-1, URL_IS_SVN },
398  { "sqlite://", sizeof("sqlite://")-1, URL_IS_SQLITE },
399  { "mysql://", sizeof("mysql://")-1, URL_IS_MYSQL },
400  { "postgres://", sizeof("postgres://")-1, URL_IS_POSTGRES },
401  { "mssql://", sizeof("mssql://")-1, URL_IS_SQLSERVER },
402  { "sqlserver://", sizeof("sqlserver://")-1, URL_IS_SQLSERVER },
403 
404  { "-", sizeof("-")-1, URL_IS_DASH },
405  { NULL, 0, URL_IS_UNKNOWN }
406 };
407 
408 urltype urlIsURL(const char * url)
409 {
410  struct urlstring *us;
411  urltype ut = URL_IS_UNKNOWN;;
412 
413  if (url && *url && *url != '/')
414  for (us = urlstrings; us->leadin != NULL; us++) {
415  if (strncmp(url, us->leadin, us->len))
416  continue;
417  ut = us->ret;
418  break;
419  }
420  return ut;
421 }
422 
423 urltype urlType(void * _u)
424 {
425  return (_u != NULL ? ((urlinfo)_u)->ut : URL_IS_UNKNOWN);
426 }
427 
428 /* Return path portion of url (or pointer to NUL if url == NULL) */
429 urltype urlPath(const char * url, const char ** pathp)
430 {
431  static const char empty[] = "";
432  const char *path = (url ? url : empty);
433  urltype ut = URL_IS_UNKNOWN;
434 
435  if (*path != '\0' && *path != '/') {
436  struct urlstring *us;
437  for (us = urlstrings; us->leadin != NULL; us++) {
438  if (strncmp(url, us->leadin, us->len))
439  continue;
440  if ((path = strchr(url+us->len, '/')) == NULL)
441  path = empty;
442  ut = us->ret;
443  break;
444  }
445  }
446 /*@-observertrans@*/
447  if (pathp)
448  *pathp = path;
449 /*@=observertrans@*/
450  return ut;
451 }
452 
456 static const char * urlStrdup(const char * url)
457  /*@*/
458 {
459  size_t nb = strlen(url);
460  char * t = (char *) xmalloc(nb + 1 + 1);
461  const char * nurl = t;
462  while (*url != '\0')
463  *t++ = *url++;
464  *t = '\0';
465  return nurl;
466 }
467 
468 /*
469  * Split URL into components. The URL can look like
470  * scheme://user:password@host:port/path
471  * or as in RFC2732 for IPv6 address
472  * service://user:password@[ip:v6:ad:dr:es:s]:port/path?query#fragment
473  */
474 /*@-modfilesys@*/
475 int urlSplit(const char * url, urlinfo *uret)
476 {
477  urlinfo u;
478  char *myurl;
479  char *s, *se, *f, *fe;
480 
481  if (uret == NULL)
482  return -1;
483  if ((u = urlNew("urlSplit")) == NULL)
484  return -1;
485 
486  myurl = xstrdup(url);
487  if ((se = strrchr(myurl, '#')) != NULL) {
488  *se++ = '\0';
489  u->fragment = xstrdup(se);
490  }
491  if ((se = strrchr(myurl, '?')) != NULL) {
492  *se++ = '\0';
493  u->query = xstrdup(se);
494  }
495 
496  u->url = urlStrdup(myurl); /* XXX +1 byte for pesky trailing '/' */
497  u->ut = urlIsURL(myurl);
498 
499  se = s = myurl;
500  while (1) {
501  /* Point to end of next item */
502  while (*se && *se != '/') se++;
503  /* Item was scheme. Save scheme and go for the rest ...*/
504  if (*se && (se != s) && se[-1] == ':' && se[0] == '/' && se[1] == '/') {
505  se[-1] = '\0';
506  u->scheme = xstrdup(s);
507  se += 2; /* skip over "//" */
508  s = se++;
509  continue;
510  }
511 
512  /* Item was everything-but-path. Continue parse on rest */
513  *se = '\0';
514  break;
515  }
516 
517  /* Look for ...@host... */
518  fe = f = s;
519  while (*fe && *fe != '@') fe++;
520  if (*fe == '@') {
521  s = fe + 1;
522  *fe = '\0';
523  /* Look for user:password@host... */
524  while (fe > f && *fe != ':') fe--;
525  if (*fe == ':') {
526  *fe++ = '\0';
527  u->password = xstrdup(fe);
528  }
529  u->user = xstrdup(f);
530  }
531 
532  /* Look for ...host:port or [v6addr]:port*/
533  fe = f = s;
534  if (strchr(fe, '[') && strchr(fe, ']')) {
535  fe = strchr(f, ']');
536  *f++ = '\0';
537  *fe++ = '\0';
538  }
539 assert(fe != NULL); /* XXX can't happen */
540  while (*fe && *fe != ':') fe++;
541  if (*fe == ':') {
542  *fe++ = '\0';
543  u->portstr = xstrdup(fe);
544  if (u->portstr != NULL && u->portstr[0] != '\0') {
545  char *end;
546  u->port = strtol(u->portstr, &end, 0);
547  if (!(end && *end == '\0')) {
548  rpmlog(RPMLOG_ERR, _("url port must be a number\n"));
549  myurl = _free(myurl);
550  u = urlFree(u, "urlSplit (error #3)");
551  return -1;
552  }
553  }
554  }
555  u->host = xstrdup(f);
556 
557  if (u->port < 0 && u->scheme != NULL) {
558  struct servent *serv;
559 /*@-multithreaded -moduncon @*/
560  /* HACK hkp:// might lookup "pgpkeyserver" */
561  serv = getservbyname(u->scheme, "tcp");
562 /*@=multithreaded =moduncon @*/
563  if (serv != NULL)
564  u->port = (int) ntohs(serv->s_port);
565  else if (u->ut == URL_IS_FTP)
566  u->port = IPPORT_FTP;
567  else if (u->ut == URL_IS_HKP)
569  else if (u->ut == URL_IS_HTTP)
570  u->port = IPPORT_HTTP;
571  else if (u->ut == URL_IS_HTTPS)
572  u->port = IPPORT_HTTPS;
573  else if (u->ut == URL_IS_MONGO)
574  u->port = IPPORT_MONGO;
575 
576 #ifdef NOTYET
577 #define URL_IS_GIT
578 #define URL_IS_SVN
579 #define URL_IS_SQLITE
580 #define URL_IS_MYSQL
581 #define URL_IS_POSTGRES
582 #endif
583 
584  }
585 
586  myurl = _free(myurl);
587  if (uret) {
588  *uret = u;
589 /*@-globs -mods @*/ /* FIX: rpmGlobalMacroContext not in <rpmlib.h> */
590  urlFind(uret, 0);
591 /*@=globs =mods @*/
592  }
593  return 0;
594 }
595 /*@=modfilesys@*/
596 
597 int urlGetFile(const char * url, const char * dest)
598 {
599  int rc;
600  FD_t sfd = NULL;
601  FD_t tfd = NULL;
602  const char * sfuPath = NULL;
603  int urlType = urlPath(url, &sfuPath);
604  char *result;
605 
606  if (*sfuPath == '\0')
607  return FTPERR_UNKNOWN;
608 
609  if (dest == NULL) {
610  if ((dest = strrchr(sfuPath, '/')) != NULL)
611  dest++;
612  else
613  dest = sfuPath;
614  }
615  if (dest == NULL)
616  return FTPERR_UNKNOWN;
617 
618 /*@-globs -mods@*/ /* Avoid including <rpmmacro.h> everywhere for now */
619  if (rpmExpandNumeric("%{?__urlgetfile:1}%{!?__urlgetfile:0}")) {
620  result = rpmExpand("%{__urlgetfile ", url, " ", dest, "}", NULL);
621  if (result != NULL && strcmp(result, "OK") == 0)
622  rc = 0;
623  else {
624  rpmlog(RPMLOG_DEBUG, D_("failed to fetch URL %s via external command\n"), url);
625  rc = FTPERR_UNKNOWN;
626  }
627  result = _free(result);
628  goto exit;
629  }
630 /*@=globs =mods@*/
631 
632  sfd = Fopen(url, "r.ufdio");
633  if (sfd == NULL || Ferror(sfd)) {
634  rpmlog(RPMLOG_DEBUG, D_("failed to open %s: %s\n"), url, Fstrerror(sfd));
635  rc = FTPERR_UNKNOWN;
636  goto exit;
637  }
638 
639  /* XXX this can fail if directory in path does not exist. */
640  tfd = Fopen(dest, "w");
641 if (_url_debug)
642 fprintf(stderr, "*** urlGetFile sfd %p %s tfd %p %s\n", sfd, url, (tfd ? tfd : NULL), dest);
643  if (tfd == NULL || Ferror(tfd)) {
644  rpmlog(RPMLOG_DEBUG, D_("failed to create %s: %s\n"), dest, Fstrerror(tfd));
645  rc = FTPERR_UNKNOWN;
646  goto exit;
647  }
648 
649  switch (urlType) {
650  case URL_IS_HTTPS:
651  case URL_IS_HTTP:
652  case URL_IS_HKP:
653  case URL_IS_FTP:
654  case URL_IS_PATH:
655  case URL_IS_DASH:
656  case URL_IS_UNKNOWN:
657  if ((rc = ufdGetFile(sfd, tfd))) {
658  (void) Unlink(dest);
659  /* XXX FIXME: sfd possibly closed by copyData */
660  /*@-usereleased@*/ (void) Fclose(sfd) /*@=usereleased@*/ ;
661  }
662  sfd = NULL; /* XXX Fclose(sfd) done by ufdGetFile */
663  break;
664  case URL_IS_MONGO: /* XXX FIXME */
665  default:
666  rc = FTPERR_UNKNOWN;
667  break;
668  }
669 
670 exit:
671  if (tfd)
672  (void) Fclose(tfd);
673  if (sfd)
674  (void) Fclose(sfd);
675 
676  return rc;
677 }