Package coprs :: Module helpers
[hide private]
[frames] | no frames]

Source Code for Module coprs.helpers

  1  import math 
  2  import random 
  3  import string 
  4  import html5_parser 
  5   
  6  from os.path import normpath 
  7  from six import with_metaclass 
  8  from six.moves.urllib.parse import urlparse, parse_qs, urlunparse, urlencode 
  9  import re 
 10   
 11  import flask 
 12  import posixpath 
 13  from flask import url_for 
 14  from dateutil import parser as dt_parser 
 15  from netaddr import IPAddress, IPNetwork 
 16  from redis import StrictRedis 
 17  from sqlalchemy.types import TypeDecorator, VARCHAR 
 18  import json 
 19   
 20  from copr_common.enums import EnumType 
 21  from copr_common.rpm import splitFilename 
 22  from coprs import constants 
 23  from coprs import app 
24 25 26 -def generate_api_token(size=30):
27 """ Generate a random string used as token to access the API 28 remotely. 29 30 :kwarg: size, the size of the token to generate, defaults to 30 31 chars. 32 :return: a string, the API token for the user. 33 """ 34 return ''.join(random.choice(string.ascii_lowercase) for x in range(size))
35 36 37 REPO_DL_STAT_FMT = "repo_dl_stat::{copr_user}@{copr_project_name}:{copr_name_release}" 38 CHROOT_REPO_MD_DL_STAT_FMT = "chroot_repo_metadata_dl_stat:hset::{copr_user}@{copr_project_name}:{copr_chroot}" 39 CHROOT_RPMS_DL_STAT_FMT = "chroot_rpms_dl_stat:hset::{copr_user}@{copr_project_name}:{copr_chroot}" 40 PROJECT_RPMS_DL_STAT_FMT = "project_rpms_dl_stat:hset::{copr_user}@{copr_project_name}" 41 42 43 FINISHED_STATUSES = ["succeeded", "forked", "canceled", "skipped", "failed"]
44 45 46 -class CounterStatType(object):
47 REPO_DL = "repo_dl"
48
49 50 -class PermissionEnum(with_metaclass(EnumType, object)):
51 # The text form is part of APIv3! 52 vals = {"nothing": 0, "request": 1, "approved": 2} 53 54 @classmethod
55 - def choices_list(cls, without=-1):
56 return [(n, k) for k, n in cls.vals.items() if n != without]
57
58 59 -class BuildSourceEnum(with_metaclass(EnumType, object)):
60 vals = {"unset": 0, 61 "link": 1, # url 62 "upload": 2, # pkg, tmp, url 63 "pypi": 5, # package_name, version, python_versions 64 "rubygems": 6, # gem_name 65 "scm": 8, # type, clone_url, committish, subdirectory, spec, srpm_build_method 66 "custom": 9, # user-provided script to build sources 67 }
68
69 70 -class JSONEncodedDict(TypeDecorator):
71 """Represents an immutable structure as a json-encoded string. 72 73 Usage:: 74 75 JSONEncodedDict(255) 76 77 """ 78 79 impl = VARCHAR 80
81 - def process_bind_param(self, value, dialect):
82 if value is not None: 83 value = json.dumps(value) 84 85 return value
86
87 - def process_result_value(self, value, dialect):
88 if value is not None: 89 value = json.loads(value) 90 return value
91
92 93 -class Paginator(object):
94 - def __init__(self, query, total_count, page=1, 95 per_page_override=None, urls_count_override=None, 96 additional_params=None):
97 98 self.query = query 99 self.total_count = total_count 100 self.page = page 101 self.per_page = per_page_override or constants.ITEMS_PER_PAGE 102 self.urls_count = urls_count_override or constants.PAGES_URLS_COUNT 103 self.additional_params = additional_params or dict() 104 105 self._sliced_query = None
106
107 - def page_slice(self, page):
108 return (self.per_page * (page - 1), 109 self.per_page * page)
110 111 @property
112 - def sliced_query(self):
113 if not self._sliced_query: 114 self._sliced_query = self.query[slice(*self.page_slice(self.page))] 115 return self._sliced_query
116 117 @property
118 - def pages(self):
119 return int(math.ceil(self.total_count / float(self.per_page)))
120
121 - def border_url(self, request, start):
122 if start: 123 if self.page - 1 > self.urls_count // 2: 124 return self.url_for_other_page(request, 1), 1 125 else: 126 if self.page < self.pages - self.urls_count // 2: 127 return self.url_for_other_page(request, self.pages), self.pages 128 129 return None
130
131 - def get_urls(self, request):
132 left_border = self.page - self.urls_count // 2 133 left_border = 1 if left_border < 1 else left_border 134 right_border = self.page + self.urls_count // 2 135 right_border = self.pages if right_border > self.pages else right_border 136 137 return [(self.url_for_other_page(request, i), i) 138 for i in range(left_border, right_border + 1)]
139
140 - def url_for_other_page(self, request, page):
141 args = request.view_args.copy() 142 args["page"] = page 143 args.update(self.additional_params) 144 return flask.url_for(request.endpoint, **args)
145
146 147 -def chroot_to_branch(chroot):
148 """ 149 Get a git branch name from chroot. Follow the fedora naming standard. 150 """ 151 os, version, arch = chroot.rsplit("-", 2) 152 if os == "fedora": 153 if version == "rawhide": 154 return "master" 155 os = "f" 156 elif os == "epel" and int(version) <= 6: 157 os = "el" 158 elif os == "mageia" and version == "cauldron": 159 os = "cauldron" 160 version = "" 161 elif os == "mageia": 162 os = "mga" 163 return "{}{}".format(os, version)
164
165 166 -def parse_package_name(pkg):
167 """ 168 Parse package name from possibly incomplete nvra string. 169 """ 170 171 if pkg.count(".") >= 3 and pkg.count("-") >= 2: 172 return splitFilename(pkg)[0] 173 174 # doesn"t seem like valid pkg string, try to guess package name 175 result = "" 176 pkg = pkg.replace(".rpm", "").replace(".src", "") 177 178 for delim in ["-", "."]: 179 if delim in pkg: 180 parts = pkg.split(delim) 181 for part in parts: 182 if any(map(lambda x: x.isdigit(), part)): 183 return result[:-1] 184 185 result += part + "-" 186 187 return result[:-1] 188 189 return pkg
190
191 192 -def generate_repo_url(mock_chroot, url):
193 """ Generates url with build results for .repo file. 194 No checks if copr or mock_chroot exists. 195 """ 196 os_version = mock_chroot.os_version 197 198 if mock_chroot.os_release == "fedora": 199 if mock_chroot.os_version != "rawhide": 200 os_version = "$releasever" 201 202 if mock_chroot.os_release == "opensuse-leap": 203 os_version = "$releasever" 204 205 if mock_chroot.os_release == "mageia": 206 if mock_chroot.os_version != "cauldron": 207 os_version = "$releasever" 208 209 url = posixpath.join( 210 url, "{0}-{1}-{2}/".format(mock_chroot.os_release, 211 os_version, "$basearch")) 212 213 return url
214
215 216 -def fix_protocol_for_backend(url):
217 """ 218 Ensure that url either has http or https protocol according to the 219 option in app config "ENFORCE_PROTOCOL_FOR_BACKEND_URL" 220 """ 221 if app.config["ENFORCE_PROTOCOL_FOR_BACKEND_URL"] == "https": 222 return url.replace("http://", "https://") 223 elif app.config["ENFORCE_PROTOCOL_FOR_BACKEND_URL"] == "http": 224 return url.replace("https://", "http://") 225 else: 226 return url
227
228 229 -def fix_protocol_for_frontend(url):
230 """ 231 Ensure that url either has http or https protocol according to the 232 option in app config "ENFORCE_PROTOCOL_FOR_FRONTEND_URL" 233 """ 234 if app.config["ENFORCE_PROTOCOL_FOR_FRONTEND_URL"] == "https": 235 return url.replace("http://", "https://") 236 elif app.config["ENFORCE_PROTOCOL_FOR_FRONTEND_URL"] == "http": 237 return url.replace("https://", "http://") 238 else: 239 return url
240
241 242 -class Serializer(object):
243
244 - def to_dict(self, options=None):
245 """ 246 Usage: 247 248 SQLAlchObject.to_dict() => returns a flat dict of the object 249 SQLAlchObject.to_dict({"foo": {}}) => returns a dict of the object 250 and will include a flat dict of object foo inside of that 251 SQLAlchObject.to_dict({"foo": {"bar": {}}, "spam": {}}) => returns 252 a dict of the object, which will include dict of foo 253 (which will include dict of bar) and dict of spam. 254 255 Options can also contain two special values: __columns_only__ 256 and __columns_except__ 257 258 If present, the first makes only specified fields appear, 259 the second removes specified fields. Both of these fields 260 must be either strings (only works for one field) or lists 261 (for one and more fields). 262 263 SQLAlchObject.to_dict({"foo": {"__columns_except__": ["id"]}, 264 "__columns_only__": "name"}) => 265 266 The SQLAlchObject will only put its "name" into the resulting dict, 267 while "foo" all of its fields except "id". 268 269 Options can also specify whether to include foo_id when displaying 270 related foo object (__included_ids__, defaults to True). 271 This doesn"t apply when __columns_only__ is specified. 272 """ 273 274 result = {} 275 if options is None: 276 options = {} 277 columns = self.serializable_attributes 278 279 if "__columns_only__" in options: 280 columns = options["__columns_only__"] 281 else: 282 columns = set(columns) 283 if "__columns_except__" in options: 284 columns_except = options["__columns_except__"] 285 if not isinstance(options["__columns_except__"], list): 286 columns_except = [options["__columns_except__"]] 287 288 columns -= set(columns_except) 289 290 if ("__included_ids__" in options and 291 options["__included_ids__"] is False): 292 293 related_objs_ids = [ 294 r + "_id" for r, _ in options.items() 295 if not r.startswith("__")] 296 297 columns -= set(related_objs_ids) 298 299 columns = list(columns) 300 301 for column in columns: 302 result[column] = getattr(self, column) 303 304 for related, values in options.items(): 305 if hasattr(self, related): 306 result[related] = getattr(self, related).to_dict(values) 307 return result
308 309 @property
310 - def serializable_attributes(self):
311 return map(lambda x: x.name, self.__table__.columns)
312
313 314 -class RedisConnectionProvider(object):
315 - def __init__(self, config):
316 self.host = config.get("REDIS_HOST", "127.0.0.1") 317 self.port = int(config.get("REDIS_PORT", "6379"))
318
319 - def get_connection(self):
320 return StrictRedis(host=self.host, port=self.port)
321
322 323 -def get_redis_connection():
324 """ 325 Creates connection to redis, now we use default instance at localhost, no config needed 326 """ 327 return StrictRedis()
328
329 330 -def str2bool(v):
331 if v is None: 332 return False 333 return v.lower() in ("yes", "true", "t", "1")
334
335 336 -def copr_url(view, copr, **kwargs):
337 """ 338 Examine given copr and generate proper URL for the `view` 339 340 Values of `username/group_name` and `coprname` are automatically passed as the first two URL parameters, 341 and therefore you should *not* pass them manually. 342 343 Usage: 344 copr_url("coprs_ns.foo", copr) 345 copr_url("coprs_ns.foo", copr, arg1='bar', arg2='baz) 346 """ 347 if copr.is_a_group_project: 348 return url_for(view, group_name=copr.group.name, coprname=copr.name, **kwargs) 349 return url_for(view, username=copr.user.name, coprname=copr.name, **kwargs)
350
351 352 -def url_for_copr_view(view, group_view, copr, **kwargs):
353 if copr.is_a_group_project: 354 return url_for(group_view, group_name=copr.group.name, coprname=copr.name, **kwargs) 355 else: 356 return url_for(view, username=copr.user.name, coprname=copr.name, **kwargs)
357
358 359 -def url_for_copr_builds(copr):
360 return copr_url("coprs_ns.copr_builds", copr)
361 362 363 from sqlalchemy.engine.default import DefaultDialect 364 from sqlalchemy.sql.sqltypes import String, DateTime, NullType 365 366 # python2/3 compatible. 367 PY3 = str is not bytes 368 text = str if PY3 else unicode 369 int_type = int if PY3 else (int, long) 370 str_type = str if PY3 else (str, unicode)
371 372 373 -class StringLiteral(String):
374 """Teach SA how to literalize various things."""
375 - def literal_processor(self, dialect):
376 super_processor = super(StringLiteral, self).literal_processor(dialect) 377 378 def process(value): 379 if isinstance(value, int_type): 380 return text(value) 381 if not isinstance(value, str_type): 382 value = text(value) 383 result = super_processor(value) 384 if isinstance(result, bytes): 385 result = result.decode(dialect.encoding) 386 return result
387 return process
388
389 390 -class LiteralDialect(DefaultDialect):
391 colspecs = { 392 # prevent various encoding explosions 393 String: StringLiteral, 394 # teach SA about how to literalize a datetime 395 DateTime: StringLiteral, 396 # don't format py2 long integers to NULL 397 NullType: StringLiteral, 398 }
399
400 401 -def literal_query(statement):
402 """NOTE: This is entirely insecure. DO NOT execute the resulting strings. 403 This can be used for debuggin - it is not and should not be used in production 404 code. 405 406 It is useful if you want to debug an sqlalchemy query, i.e. copy the 407 resulting SQL query into psql console and try to tweak it so that it 408 actually works or works faster. 409 """ 410 import sqlalchemy.orm 411 if isinstance(statement, sqlalchemy.orm.Query): 412 statement = statement.statement 413 return statement.compile( 414 dialect=LiteralDialect(), 415 compile_kwargs={'literal_binds': True}, 416 ).string
417
418 419 -def stream_template(template_name, **context):
420 app.update_template_context(context) 421 t = app.jinja_env.get_template(template_name) 422 rv = t.stream(context) 423 rv.enable_buffering(2) 424 return rv
425
426 427 -def generate_repo_name(repo_url):
428 """ based on url, generate repo name """ 429 repo_url = re.sub("[^a-zA-Z0-9]", '_', repo_url) 430 repo_url = re.sub("(__*)", '_', repo_url) 431 repo_url = re.sub("(_*$)|^_*", '', repo_url) 432 return repo_url
433
434 435 -def pre_process_repo_url(chroot, repo_url):
436 """ 437 Expands variables and sanitize repo url to be used for mock config 438 """ 439 parsed_url = urlparse(repo_url) 440 query = parse_qs(parsed_url.query) 441 442 if parsed_url.scheme == "copr": 443 user = parsed_url.netloc 444 prj = parsed_url.path.split("/")[1] 445 repo_url = "/".join([ 446 flask.current_app.config["BACKEND_BASE_URL"], 447 "results", user, prj, chroot 448 ]) + "/" 449 450 elif "priority" in query: 451 query.pop("priority") 452 query_string = urlencode(query, doseq=True) 453 parsed_url = parsed_url._replace(query=query_string) 454 repo_url = urlunparse(parsed_url) 455 456 repo_url = repo_url.replace("$chroot", chroot) 457 repo_url = repo_url.replace("$distname", chroot.rsplit("-", 2)[0]) 458 return repo_url
459
460 461 -def parse_repo_params(repo, supported_keys=None):
462 """ 463 :param repo: str repo from Copr/CoprChroot/Build/... 464 :param supported_keys list of supported optional parameters 465 :return: dict of optional parameters parsed from the repo URL 466 """ 467 supported_keys = supported_keys or ["priority"] 468 params = {} 469 qs = parse_qs(urlparse(repo).query) 470 for k, v in qs.items(): 471 if k in supported_keys: 472 # parse_qs returns values as lists, but we allow setting the param only once, 473 # so we can take just first value from it 474 value = int(v[0]) if v[0].isnumeric() else v[0] 475 params[k] = value 476 return params
477
478 479 -def generate_build_config(copr, chroot_id):
480 """ Return dict with proper build config contents """ 481 chroot = None 482 for i in copr.copr_chroots: 483 if i.mock_chroot.name == chroot_id: 484 chroot = i 485 if not chroot: 486 return {} 487 488 packages = "" if not chroot.buildroot_pkgs else chroot.buildroot_pkgs 489 490 repos = [{ 491 "id": "copr_base", 492 "baseurl": copr.repo_url + "/{}/".format(chroot_id), 493 "name": "Copr repository", 494 }] 495 496 if not copr.auto_createrepo: 497 repos.append({ 498 "id": "copr_base_devel", 499 "baseurl": copr.repo_url + "/{}/devel/".format(chroot_id), 500 "name": "Copr buildroot", 501 }) 502 503 def get_additional_repo_views(repos_list): 504 repos = [] 505 for repo in repos_list: 506 params = parse_repo_params(repo) 507 repo_view = { 508 "id": generate_repo_name(repo), 509 "baseurl": pre_process_repo_url(chroot_id, repo), 510 "name": "Additional repo " + generate_repo_name(repo), 511 } 512 repo_view.update(params) 513 repos.append(repo_view) 514 return repos
515 516 repos.extend(get_additional_repo_views(copr.repos_list)) 517 repos.extend(get_additional_repo_views(chroot.repos_list)) 518 519 return { 520 'project_id': copr.repo_id, 521 'additional_packages': packages.split(), 522 'repos': repos, 523 'chroot': chroot_id, 524 'use_bootstrap_container': copr.use_bootstrap_container, 525 'with_opts': chroot.with_opts.split(), 526 'without_opts': chroot.without_opts.split(), 527 } 528
529 530 -def generate_additional_repos(copr_chroot):
531 base_repo = "copr://{}".format(copr_chroot.copr.full_name) 532 repos = [base_repo] + copr_chroot.repos_list + copr_chroot.copr.repos_list 533 if not copr_chroot.copr.auto_createrepo: 534 repos.append("copr://{}/devel".format(copr_chroot.copr.full_name)) 535 return repos
536
537 538 -def trim_git_url(url):
539 if not url: 540 return None 541 542 return re.sub(r'(\.git)?/*$', '', url)
543
544 545 -def get_parsed_git_url(url):
546 if not url: 547 return False 548 549 url = trim_git_url(url) 550 return urlparse(url)
551
552 553 -def get_copr_repo_id(copr_dir):
554 """ 555 We cannot really switch to the new 556 copr:{hostname}:{owner}:{project} format yet, because it is implemented in 557 dnf-plugins-core-3.x which is only on F29+ 558 559 Since the F29+ plugin is able to work with both old and new formats, we can 560 safely stay with the old one until F28 is still supported. Once it goes EOL, 561 we can migrate to the new format. 562 563 New format is: 564 565 return "copr:{0}:{1}:{2}".format(app.config["PUBLIC_COPR_HOSTNAME"].split(":")[0], 566 copr_dir.copr.owner_name.replace("@", "group_"), 567 copr_dir.name) 568 569 """ 570 return copr_dir.repo_id
571
572 573 -class SubdirMatch(object):
574 - def __init__(self, subdir):
575 if not subdir: 576 self.subdir = '.' 577 else: 578 self.subdir = normpath(subdir).strip('/')
579
580 - def match(self, path):
581 if not path: # shouldn't happen 582 return False 583 584 changed = normpath(path).strip('/') 585 if changed == '.': 586 return False # shouldn't happen! 587 588 if self.subdir == '.': 589 return True 590 591 return changed.startswith(self.subdir + '/')
592
593 594 -def pagure_html_diff_changed(html_string):
595 parsed = html5_parser.parse(str(html_string)) 596 elements = parsed.xpath( 597 "//section[contains(@class, 'commit_diff')]" 598 "//div[contains(@class, 'card-header')]" 599 "//a[contains(@class, 'font-weight-bold')]" 600 "/text()") 601 602 return set([str(x) for x in elements])
603
604 605 -def raw_commit_changes(text):
606 changes = set() 607 for line in text.split('\n'): 608 match = re.search(r'^(\+\+\+|---) [ab]/(.*)$', line) 609 if match: 610 changes.add(str(match.group(2))) 611 match = re.search(r'^diff --git a/(.*) b/(.*)$', line) 612 if match: 613 changes.add(str(match.group(1))) 614 changes.add(str(match.group(2))) 615 print(changes) 616 617 return changes
618