1
2
3 import subprocess
4 import argparse
5 import sys
6 import os
7 import json
8 import re
9 import logging
10
11 sys.path.append(
12 os.path.dirname(os.path.dirname(os.path.realpath(__file__)))
13 )
14
15 from coprs import db, app, helpers
16 from coprs.logic.builds_logic import BuildsLogic
17 from coprs.logic.coprs_logic import CoprsLogic
18
19 logging.basicConfig(
20 filename="{0}/check_for_anitya_version_updates.log".format(app.config.get("LOG_DIR")),
21 format='[%(asctime)s][%(levelname)6s]: %(message)s',
22 level=logging.DEBUG)
23 log = logging.getLogger(__name__)
24 log.addHandler(logging.StreamHandler(sys.stdout))
25
26 parser = argparse.ArgumentParser(description='Fetch package version updates by using datagrepper log of anitya emitted messages and issue rebuilds of the respective COPR packages for each such update. Requires httpie package.')
27
28 parser.add_argument('--backend', action='store', default='pypi', choices=['pypi', 'rubygems'],
29 help='only check for updates from backend BACKEND, default pypi')
30 parser.add_argument('--delta', action='store', type=int, metavar='SECONDS', default=86400,
31 help='ignore updates older than SECONDS, default 86400')
32 parser.add_argument('-v', '--version', action='version', version='1.0',
33 help='print program version and exit')
34
35 args = parser.parse_args()
36
37
39 """
40 Run given command in a subprocess
41 """
42 log.info('Executing: '+' '.join(cmd))
43 process = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
44 (stdout, stderr) = process.communicate()
45 if process.returncode != 0:
46 log.error(stderr)
47 sys.exit(1)
48 return stdout
49
51 try:
52 data = data_bytes.decode("utf-8")
53 data_json = json.loads(data)
54 except Exception as e:
55 log.info(data)
56 log.exception(str(e))
57 return data_json
58
60 cmd_binary = 'curl'
61 url_template = 'https://apps.fedoraproject.org/datagrepper/raw?category=anitya&delta={delta}&topic=org.release-monitoring.prod.anitya.project.version.update&rows_per_page=64&order=asc&page={page}'
62 get_updates_cmd = [cmd_binary, url_template.format(delta=args.delta, page=1)]
63 result_json = to_json(run_cmd(get_updates_cmd))
64 messages = result_json['raw_messages']
65 pages = result_json['pages']
66
67 for p in range(2, pages+1):
68 get_updates_cmd = [cmd_binary, url_template.format(delta=args.delta, page=p)]
69 result_json = to_json(run_cmd(get_updates_cmd))
70 messages += result_json['raw_messages']
71
72 return messages
73
83
85 pkg_name_pattern = '(' + '|'.join(updated_packages.keys()) + ')'
86 source_type = helpers.BuildSourceEnum(args.backend.lower())
87 if db.engine.url.drivername == "sqlite":
88 placeholder = '?'
89 true = '1'
90 else:
91 placeholder = '%s'
92 true = 'true'
93 rows = db.engine.execute(
94 """
95 SELECT package.id AS package_id, package.source_json AS source_json, build.pkg_version AS pkg_version, package.copr_id AS copr_id
96 FROM package
97 LEFT OUTER JOIN build ON build.package_id = package.id
98 WHERE package.source_type = {placeholder} AND
99 package.source_json ~* '{pkg_name_pattern}' AND
100 package.webhook_rebuild = {true} AND
101 (build.id is NULL OR build.id = (SELECT MAX(build.id) FROM build WHERE build.package_id = package.id));
102 """.format(placeholder=placeholder, pkg_name_pattern=pkg_name_pattern, true=true), source_type
103 )
104 return rows
105
106
110
111 - def build(self, copr, new_update_version):
113
114
120
121 - def build(self, copr, new_updated_version):
125
126
128 try:
129 return {
130 'pypi': PyPIPackage,
131 'rubygems': RubyGemsPackage,
132 }[backend](source_json)
133 except KeyError:
134 raise Exception('Unsupported backend {0} passed as command-line argument'.format(args.backend))
135
136
138 updated_packages = get_updated_packages(get_updates_messages())
139 log.info('Updated packages according to datagrepper: {0}'.format(updated_packages))
140
141 for row in get_copr_package_info_rows(updated_packages):
142 source_json = json.loads(row.source_json)
143 package = package_from_source(args.backend.lower(), source_json)
144
145 latest_build_version = row.pkg_version
146 log.info('candidate package for rebuild: {0}, package_id: {1}, copr_id: {2}'.format(package.name, row.package_id, row.copr_id))
147 if package.name in updated_packages:
148 new_updated_version = updated_packages[package.name]
149 log.debug('name: {0}, latest_build_version: {1}, new_updated_version {2}'.format(package.name, latest_build_version, new_updated_version))
150
151
152 if not latest_build_version or not re.match(new_updated_version, latest_build_version):
153 try:
154 copr = CoprsLogic.get_by_id(row.copr_id)[0]
155 except Exception as e:
156 log.exception(e)
157 continue
158
159 log.info('Launching {} build for package of source name: {}, package_id: {}, copr_id: {}, user_id: {}'
160 .format(args.backend.lower(), package.name, row.package_id, copr.id, copr.user.id))
161 build = package.build(copr, new_updated_version)
162 db.session.commit()
163 log.info('Launched build id {0}'.format(build.id))
164
165 if __name__ == '__main__':
166 try:
167 main()
168 except Exception as e:
169 log.exception(str(e))
170