Coverage for quibble/commands.py: 72%
773 statements
« prev ^ index » next coverage.py v7.10.7, created at 2026-08-07 07:21 +0000
1"""Encapsulates each step of a job"""
3import contextlib
4import git
5import hashlib
6import importlib.resources
7import io
8import json
9import logging
10import multiprocessing
11import os
12import os.path
13import textwrap
15from concurrent.futures import ThreadPoolExecutor, as_completed
17import requests
18import yaml
20from quibble.gitchangedinhead import GitChangedInHead
21from quibble.util import copylog, isExtOrSkin, ProgressReporter, strtobool
22import quibble.mediawiki.registry
23import quibble.zuul
24import subprocess
25import sys
26import tempfile
29log = logging.getLogger(__name__)
30monitor_interval = 10
33def execute_command(command):
34 '''Shared decorator for execution'''
35 with quibble.Chronometer(str(command), log.info):
36 command.execute()
39def run(cmd: list, cwd: str, shell=False, env=None):
40 """
41 Wrapper around subprocess.Popen(), with default values for
42 stdout, stderr, shell, and env set for convenience and
43 consistency.
44 :param cmd: The command list.
45 :param cwd: The current working directory.
46 :param shell: Whether to set shell=True, default to False.
47 :param env: The optional environment override, default to None.
48 """
50 # We run attached to a terminal (ex: quibble -c bash), in which case there
51 # is no need for capturing output and we want stdout/stderr/stdin to remain
52 # attached to a tty, else the commands would think they run non
53 # interactively (ex: bash shows no prompt)
54 if sys.stdin.isatty() and sys.stdout.isatty(): 54 ↛ 55line 54 didn't jump to line 55 because the condition on line 54 was never true
55 subprocess.check_call(cmd, cwd=cwd, shell=shell, env=env)
56 return
58 collected_output = b''
59 with subprocess.Popen(
60 cmd,
61 stdout=subprocess.PIPE,
62 stderr=subprocess.STDOUT,
63 cwd=cwd,
64 shell=shell,
65 env=env,
66 ) as proc:
67 # The process is opened in binary mode in order to accept invalid
68 # Unicode emitted by the command (see 5c29fb1b and T318029).
69 #
70 # `readline()` let us find new lines from the bytes stream and we
71 # output them immediately in order to keep the output interactive.
72 #
73 # py38: while line := proc.stdout.readline()
74 for line in iter(proc.stdout.readline, b''):
75 sys.stdout.buffer.write(line)
76 sys.stdout.flush()
77 collected_output += line
78 if proc.returncode:
79 raise subprocess.CalledProcessError(
80 proc.returncode,
81 proc.args,
82 # The process is in binary mode, we want to emit valid unicode
83 output=collected_output.decode('utf-8', errors='backslashreplace'),
84 )
87def _npm_install(project_dir, label=None):
88 # A label wraps npm install in its own timed section named after the
89 # caller; without one it runs unwrapped to avoid a duplicate stage.
90 section = (
91 quibble.Chronometer("npm install in '%s'" % label, log.info)
92 if label
93 else contextlib.nullcontext()
94 )
95 with section:
96 if _repo_has_npm_lock(project_dir):
97 cmd = 'ci'
98 if quibble.get_npm_command() == 'pnpm':
99 cmd = 'install'
100 run([quibble.get_npm_command(), cmd], cwd=project_dir)
101 else:
102 run([quibble.get_npm_command(), 'prune'], cwd=project_dir)
103 run(
104 [
105 quibble.get_npm_command(),
106 'install',
107 '--no-progress',
108 '--prefer-offline',
109 ],
110 cwd=project_dir,
111 )
114class ReportVersions:
115 def execute(self):
116 log.info("Python version: %s", sys.version)
118 # Run them all in parallel
119 with ThreadPoolExecutor() as executor:
120 futures = [
121 executor.submit(self._logged_call, cmd)
122 for cmd in self.getCommands()
123 ]
125 for program, logger, message in sorted(
126 [future.result() for future in as_completed(futures)],
127 key=lambda k: k[0],
128 ):
129 for m in message.splitlines():
130 logger(m)
132 def getCommands(self):
133 return [
134 ['chromedriver', '--version'],
135 ['chromium', '--version'],
136 ['composer', '--version'],
137 ['memcached', '--version'],
138 ['mysql', '--version'],
139 ['psql', '--version'],
140 # ['sqlite', '--version'], php-sqlite3 doesn't provide a CLI
141 ['node', '--version'],
142 [quibble.get_npm_command(), '--version'],
143 ['php', '--version'],
144 ]
146 def _logged_call(self, cmd):
147 try:
148 res = subprocess.check_output(cmd, stderr=subprocess.STDOUT)
149 message = '{}: {}'.format(
150 ' '.join(cmd), res.strip().decode('utf-8')
151 )
152 return (cmd[0], log.info, message)
153 except subprocess.CalledProcessError:
154 return (
155 cmd[0],
156 log.warning,
157 'Failed to run command: %s' % ' '.join(cmd),
158 )
159 except FileNotFoundError:
160 return (
161 cmd[0],
162 log.warning,
163 'Command not found: %s' % ' '.join(cmd),
164 )
166 def __str__(self):
167 return 'Versions'
170class ReportDurations:
171 def __init__(self, context_stack, log_dir=None):
172 self.log_dir = log_dir
173 context_stack.enter_context(self)
175 def __enter__(self):
176 pass
178 def __exit__(self, exc_type, exc_value, traceback):
179 if self.log_dir: 179 ↛ 181line 179 didn't jump to line 181 because the condition on line 179 was always true
180 self.writeJsonReport(exc_type, exc_value)
181 self.printReport()
183 def execute(self):
184 pass
186 def printReport(self):
187 print(self._build_report())
189 def _build_report(self):
190 if not quibble.DURATIONS: 190 ↛ 191line 190 didn't jump to line 191 because the condition on line 190 was never true
191 return
193 # We cant use a dict comprehension since it does not retain order of
194 # insertion.
195 formatted = {}
196 for duration, command in quibble.DURATIONS:
197 formatted[command] = '%.03fs' % duration
199 # Width of terminal, or COLUMN, or 80
200 try:
201 term_width = os.get_terminal_size()[0]
202 except OSError:
203 term_width = int(os.environ.get('COLUMNS', 80))
205 d_width = 2 + len(max(formatted.values(), key=len))
207 # No need for a table larger than the longest command
208 term_width = min(
209 term_width, d_width + 7 + len(max(formatted.keys(), key=len))
210 )
211 # Ensure it is not too small though
212 term_width = max(term_width, 72)
213 # Reduce width so the table has a margin
214 term_width -= 2
216 report = (
217 '\n'
218 + '[ REPORT FOR COMMAND DURATIONS ]'.center(term_width).rstrip()
219 + '\n\n'
220 )
222 head = '╒' + '═' * d_width + '╤'
223 tail = '╕'
224 body_len = term_width - len(head) - len(tail)
225 body = '═' * body_len
226 report += head + body + tail + '\n'
228 for command, duration in formatted.items():
229 wrapped = textwrap.wrap(
230 command,
231 width=body_len - 2,
232 )
233 for index, line in enumerate(wrapped):
234 # subsequent_indent = '│' + ' ' * d_width + '│ ',
236 report += '│ '
237 report += (duration if index == 0 else ' ').rjust(d_width - 2)
238 report += ' │ ' + line.ljust(body_len - 2) + ' │\n'
240 report += '╘' + '═' * d_width + '╧' + body + '╛'
242 return report
244 @staticmethod
245 def result_from_exception(exc_type=None):
246 # Set all logical conditions first
247 is_success_cache_hit = exc_type == quibble.commands.SuccessCache.Hit
248 is_success = exc_type is None or is_success_cache_hit
250 # then format and return the result
251 return {
252 'result': 'SUCCESS' if is_success else 'FAILURE',
253 'success_cache_hit': is_success_cache_hit,
254 }
256 def writeJsonReport(self, exc_type=None, exc_value=None):
257 if not os.path.exists(self.log_dir): 257 ↛ 264line 257 didn't jump to line 264 because the condition on line 257 was always true
258 log.warning(
259 'Can not write JSON duration reports: %s does not exist',
260 self.log_dir,
261 )
262 return
264 json_file = os.path.join(self.log_dir, 'quibble-durations.json')
265 # Format the CommandTimingnamed tuple as dict to get a self explanatory
266 # json output.
267 json_report = {}
268 json_report.update(ReportDurations.result_from_exception(exc_type))
269 json_report.update(
270 {
271 'durations': [
272 timing._asdict() for timing in quibble.DURATIONS
273 ],
274 }
275 )
277 json.dump(json_report, open(json_file, 'w'))
279 log.info('Wrote durations to %s', json_file)
281 def __str__(self):
282 return 'Report durations'
285class ZuulClone:
286 def __init__(
287 self,
288 branch,
289 cache_dir,
290 project_branch,
291 projects,
292 workers,
293 workspace,
294 zuul_branch,
295 zuul_newrev,
296 zuul_project,
297 zuul_ref,
298 zuul_url,
299 ):
300 self.branch = branch
301 self.cache_dir = cache_dir
302 self.project_branch = project_branch
303 self.projects = projects
304 self.workers = workers
305 self.workspace = workspace
306 self.zuul_branch = zuul_branch
307 self.zuul_newrev = zuul_newrev
308 self.zuul_project = zuul_project
309 self.zuul_ref = zuul_ref
310 self.zuul_url = zuul_url
312 def execute(self):
313 quibble.zuul.clone(
314 self.branch,
315 self.cache_dir,
316 self.project_branch,
317 self.projects,
318 self.workers,
319 self.workspace,
320 self.zuul_branch,
321 self.zuul_newrev,
322 self.zuul_project,
323 self.zuul_ref,
324 self.zuul_url,
325 )
327 def __str__(self):
328 pruned_params = {
329 k: v for k, v in self.__dict__.items() if v is not None and v != []
330 }
331 return "Zuul clone {}".format(
332 # JSON serialization falls back to a list since projects can be a
333 # set which is not serializable.
334 json.dumps(pruned_params, sort_keys=True, default=list)
335 )
338class ResolveRequires:
339 def __init__(
340 self,
341 mw_install_path,
342 projects,
343 zuul_params,
344 fail_on_extra_requires=False,
345 ):
346 """
347 mw_install_path: root dir of MediaWiki
348 projects: list of Gerrit projects to initially clone
349 zuul_params: other parameters for ZuulClone
350 fail_on_extra_requires: if any repositories has been cloned and has
351 not been given in the initial list of projects, raise an exception.
352 """
353 self.mw_install_path = mw_install_path
354 self.projects = projects
355 self.zuul_params = zuul_params
356 if 'projects' in self.zuul_params: 356 ↛ 357line 356 didn't jump to line 357 because the condition on line 356 was never true
357 del self.zuul_params['projects']
358 self.fail_on_extra_requires = fail_on_extra_requires
360 def execute(self):
361 ext_cloned = set(filter(isExtOrSkin, self.projects))
362 with quibble.logginglevel('zuul.CloneMapper', logging.WARNING):
363 required = self._clone_requires(ext_cloned, ext_cloned)
364 extras = set(required) - set(self.projects)
366 msg = 'Found extra requirements: %s' % ', '.join(extras)
367 if extras and self.fail_on_extra_requires: 367 ↛ 368line 367 didn't jump to line 368 because the condition on line 367 was never true
368 raise Exception(msg)
369 else:
370 log.warning(msg)
372 def _clone_requires(self, new_projects, cloned):
373 to_be_cloned = new_projects - cloned
374 if to_be_cloned:
375 log.info('Cloning: %s', ', '.join(to_be_cloned))
376 execute_command(
377 ZuulClone(projects=to_be_cloned, **self.zuul_params)
378 )
380 found = set()
381 for project in sorted(new_projects):
382 log.info('Looking for requirements of %s', project)
384 project_dir = os.path.join(
385 self.mw_install_path, quibble.zuul.repo_dir(project)
386 )
387 deps = quibble.mediawiki.registry.from_path(project_dir)
388 found.update(deps.getRequiredRepos())
390 if not found:
391 log.debug(
392 'No additional requirements from %s', ', '.join(new_projects)
393 )
394 return set()
396 if found: 396 ↛ exitline 396 didn't return from function '_clone_requires' because the condition on line 396 was always true
397 log.info('Found requirement(s): %s', ', '.join(found))
398 return found.union(self._clone_requires(found, cloned))
400 def __str__(self):
401 return (
402 'Recursively process registration dependencies. '
403 'Fails on extra requires: %s' % self.fail_on_extra_requires
404 )
407class ExtSkinSubmoduleUpdate:
408 def __init__(self, mw_install_path, jobs=None):
409 self.mw_install_path = mw_install_path
410 self.jobs = jobs
412 @staticmethod
413 def getCommands(jobs=None):
414 submodule_update = [
415 'git',
416 'submodule',
417 'update',
418 '--init',
419 '--recursive',
420 ]
422 if jobs is not None:
423 submodule_update.extend(['--jobs', str(jobs)])
425 return [
426 ['git', 'submodule', 'foreach', 'git', 'clean', '-xdff', '-q'],
427 submodule_update,
428 ['git', 'submodule', 'status'],
429 ]
431 def execute(self):
432 log.info('Updating git submodules of extensions and skins')
434 cmds = self.getCommands(jobs=self.jobs)
436 tops = [
437 os.path.join(self.mw_install_path, top)
438 for top in ['extensions', 'skins']
439 ]
441 for top in tops:
442 for dirpath, dirnames, filenames in os.walk(top):
443 if dirpath not in tops:
444 # Only look at the first level
445 dirnames[:] = []
446 if '.gitmodules' not in filenames:
447 continue
449 for cmd in cmds:
450 try:
451 run(cmd, cwd=dirpath)
452 except subprocess.CalledProcessError as e:
453 log.error( # noqa: LOG005, we reraise it
454 "Failed to process git submodules for %s", dirpath
455 )
456 raise e
458 def __str__(self):
459 # TODO: Would be nicer to extract the directory crawl into a subroutine
460 # and print the analysis here.
461 return ("Submodule update: {}").format(self.mw_install_path)
464# Used to be bin/mw-create-composer-local.py
465class CreateComposerLocal:
466 def __init__(self, mw_install_path, dependencies):
467 self.mw_install_path = mw_install_path
468 self.dependencies = dependencies
470 def execute(self):
471 log.info('composer.local.json for merge plugin')
472 composer_local = os.path.join(
473 self.mw_install_path, 'composer.local.json'
474 )
475 with open(composer_local, 'w') as f:
476 json.dump(
477 {
478 "extra": {
479 "merge-plugin": {
480 "include": [
481 "extensions/*/composer.json",
482 "skins/*/composer.json",
483 ]
484 }
485 }
486 },
487 f,
488 )
489 log.info('Created composer.local.json')
491 def __str__(self):
492 return "Create composer.local.json with dependencies {}".format(
493 self.dependencies
494 )
497class ExtSkinComposerTest:
498 def __init__(self, directory):
499 self.directory = directory
501 def execute(self):
502 if _repo_has_composer_script(self.directory, 'test'): 502 ↛ exitline 502 didn't return from function 'execute' because the condition on line 502 was always true
503 cmds = [
504 ['composer', '--ansi', 'validate', '--no-check-publish'],
505 [
506 'composer',
507 '--ansi',
508 'install',
509 '--no-progress',
510 '--prefer-dist',
511 '--profile',
512 '-v',
513 ],
514 ['composer', '--ansi', 'test'],
515 ]
516 for cmd in cmds:
517 run(cmd, cwd=self.directory)
519 def __str__(self):
520 return "composer test in {}".format(self.directory)
523class NpmTest:
524 def __init__(self, directory):
525 self.directory = directory
527 def execute(self):
528 if repo_has_npm_script(self.directory, 'test'):
529 _npm_install(self.directory, label=self.directory)
530 run([quibble.get_npm_command(), 'test'], cwd=self.directory)
531 else:
532 log.warning("%s lacks a package.json", self.directory)
534 def __str__(self):
535 return "npm test in {}".format(self.directory)
538class CoreComposerTest:
539 def __init__(self, mw_install_path):
540 self.mw_install_path = mw_install_path
542 def execute(self):
543 files = []
544 changed = GitChangedInHead([], cwd=self.mw_install_path).changedFiles()
545 if 'composer.json' in changed or '.phpcs.xml' in changed: 545 ↛ 546line 545 didn't jump to line 546 because the condition on line 545 was never true
546 log.info('composer.json or .phpcs.xml changed: linting "."')
547 # '.' is passed to composer lint which then pass it
548 # to parallel-lint and phpcs
549 files = ['.']
550 else:
551 files = GitChangedInHead(
552 ['php'], cwd=self.mw_install_path
553 ).changedFiles()
555 if not files: 555 ↛ 556line 555 didn't jump to line 556 because the condition on line 555 was never true
556 log.info('Skipping composer test (unneeded)')
557 else:
558 log.info("Running composer test for changed files")
560 env = {'COMPOSER_PROCESS_TIMEOUT': '900'}
561 env.update(os.environ)
563 composer_test_cmd = ['composer', 'test-some']
564 composer_test_cmd.extend(files)
565 run(composer_test_cmd, cwd=self.mw_install_path, env=env)
567 def __str__(self):
568 return "composer test for mediawiki/core"
571class NativeComposerDependencies:
572 def __init__(self, mw_install_path):
573 self.mw_install_path = mw_install_path
575 def execute(self):
576 log.info('Running "composer update" for mediawiki/core')
577 cmd = [
578 'composer',
579 'update',
580 '--ansi',
581 '--no-progress',
582 '--prefer-dist',
583 '--profile',
584 '-v',
585 ]
586 run(cmd, cwd=self.mw_install_path)
588 def __str__(self):
589 return "composer update for mediawiki/core"
592class VendorComposerDependencies:
593 def __init__(self, mw_install_path, log_dir):
594 self.mw_install_path = mw_install_path
595 self.log_dir = log_dir
597 def execute(self):
598 log.info('mediawiki/vendor is used, add require-dev dependencies')
599 mw_composer_json = os.path.join(self.mw_install_path, 'composer.json')
600 vendor_dir = os.path.join(self.mw_install_path, 'vendor')
601 with open(mw_composer_json, 'r') as f:
602 composer = json.load(f)
604 reqs = [
605 '='.join([dependency, version])
606 for dependency, version in composer['require-dev'].items()
607 ]
609 log.debug('composer require --dev %s', ' '.join(reqs))
610 composer_require = [
611 'composer',
612 'require',
613 '--dev',
614 '--ansi',
615 '--no-progress',
616 '--no-interaction',
617 '--prefer-dist',
618 '-v',
619 ]
620 composer_require.extend(reqs)
622 run(composer_require, cwd=vendor_dir)
624 # Point composer-merge-plugin to mediawiki/core.
625 # That let us easily merge autoload-dev section and thus complete
626 # the autoloader.
627 # T158674
628 run(
629 [
630 'composer',
631 'config',
632 'extra.merge-plugin.include',
633 mw_composer_json,
634 ],
635 cwd=vendor_dir,
636 )
638 # FIXME integration/composer used to be outdated and broke the
639 # autoloader. Since composer 1.0.0-alpha11 the following might not
640 # be needed anymore.
641 run(['composer', 'dump-autoload', '--optimize'], cwd=vendor_dir)
643 copylog(
644 mw_composer_json,
645 os.path.join(self.log_dir, 'composer.core.json.txt'),
646 )
647 copylog(
648 os.path.join(vendor_dir, 'composer.json'),
649 os.path.join(self.log_dir, 'composer.vendor.json.txt'),
650 )
651 copylog(
652 os.path.join(vendor_dir, 'composer/autoload_files.php'),
653 os.path.join(self.log_dir, 'composer.autoload_files.php.txt'),
654 )
656 def __str__(self):
657 return "Install composer dev-requires for vendor.git"
660class NpmInstall:
661 def __init__(
662 self, mw_install_path, project=None, with_package_command=None
663 ):
664 if not project: 664 ↛ 667line 664 didn't jump to line 667 because the condition on line 664 was always true
665 self.directory = mw_install_path
666 else:
667 self.directory = quibble.commands.get_project_dir(
668 mw_install_path, project
669 )
670 self.with_package_command = with_package_command
671 self.project = project
673 def execute(self):
674 if (
675 self.with_package_command
676 and not quibble.commands.repo_has_npm_script(
677 self.directory, self.with_package_command
678 )
679 ):
680 log.info(
681 '%s command does not exist in project %s package.json, '
682 'skipping npm install',
683 self.with_package_command,
684 self.project,
685 )
686 return
687 _npm_install(self.directory)
689 def __str__(self):
690 return "npm install in {}".format(self.directory)
693class StartBackends:
694 """Start backends and add to a global context stack, to be destroyed in
695 reverse order before application exit.
696 """
698 def __init__(self, context_stack, backends):
699 self.context_stack = context_stack
700 self.backends = backends
702 def execute(self):
703 """Atomically start each backend and add it to the shutdown stack."""
704 for context in self.backends + [self._exit()]:
705 self.context_stack.enter_context(context)
707 def _service_names(self):
708 return " ".join([str(backend) for backend in self.backends])
710 @contextlib.contextmanager
711 def _exit(self):
712 """List which backends will be shut down. This is run before the other
713 shutdown tasks.
714 """
715 yield
716 log.info("Shutting down backends: %s", self._service_names())
718 def __str__(self):
719 return "Start backends: {}".format(self._service_names())
722class InstallMediaWiki:
723 def __init__(
724 self, mw_install_path, db, web_url, log_dir, memcached_port, tmp_dir
725 ):
726 self.mw_install_path = mw_install_path
727 self.db = db
728 self.web_url = web_url
729 self.log_dir = log_dir
730 self.memcached_port = memcached_port
731 self.tmp_dir = tmp_dir
733 def execute(self):
734 self.clearQuibbleLocalSettings()
735 quibble.mediawiki.maintenance.install(
736 args=self._get_install_args(), mwdir=self.mw_install_path
737 )
739 localsettings = os.path.join(self.mw_install_path, 'LocalSettings.php')
740 localsettings_installer = os.path.join(
741 self.mw_install_path, 'LocalSettings-installer.php'
742 )
744 customsettings = self._expand_template(
745 'mediawiki/local_settings.php.tpl',
746 php_constants={
747 'MW_LOG_DIR': self.log_dir,
748 'TMPDIR': self.tmp_dir,
749 # Passed to $wgMemCachedServers
750 'QUIBBLE_MEMCACHED': '127.0.0.1:%s' % self.memcached_port,
751 },
752 )
754 InstallMediaWiki._apply_custom_settings(
755 localsettings=localsettings,
756 installed_copy=localsettings_installer,
757 new_settings=customsettings,
758 log_dir=self.log_dir,
759 )
761 quibble.mediawiki.maintenance.addSite(
762 args=[
763 self.db.dbname, # globalid
764 'CI', # site-group
765 '--filepath=%s/$1' % self.web_url,
766 '--pagepath=%s/index.php?title=$1' % self.web_url,
767 ],
768 mwdir=self.mw_install_path,
769 )
770 quibble.mediawiki.maintenance.update(mwdir=self.mw_install_path)
771 quibble.mediawiki.maintenance.rebuildLocalisationCache(
772 lang=['en'], mwdir=self.mw_install_path
773 )
775 if strtobool(os.getenv('QUIBBLE_OPENSEARCH', 'false')):
776 quibble.mediawiki.maintenance.updateSearchIndexConfig(
777 mwdir=self.mw_install_path
778 )
779 quibble.mediawiki.maintenance.forceSearchIndex(
780 mwdir=self.mw_install_path
781 )
783 def clearQuibbleLocalSettings(self):
784 marker = "# Quibble MediaWiki configuration\n"
785 quibbleLocalSettings = os.path.join(
786 self.mw_install_path, 'LocalSettings.php'
787 )
789 if not os.path.exists(quibbleLocalSettings):
790 return
792 with open(quibbleLocalSettings) as f:
793 if marker in f.readlines():
794 os.unlink(quibbleLocalSettings)
795 return
796 raise Exception(
797 "Unknown configuration file %s\nMarker not found: '%s'"
798 % (quibbleLocalSettings, marker.replace("\n", "\\n"))
799 )
801 def _get_install_args(self):
802 # TODO: Better if we can calculate the install args before
803 # instantiating the database.
804 install_args = [
805 '--scriptpath=',
806 '--server=%s' % self.web_url,
807 '--dbtype=%s' % self.db.type,
808 '--dbname=%s' % self.db.dbname,
809 ]
810 if self.db.type == 'sqlite':
811 install_args.extend(
812 [
813 '--dbpath=%s' % self.db.rootdir,
814 ]
815 )
816 elif self.db.type in ('mysql', 'postgres'):
817 install_args.extend(
818 [
819 '--dbuser=%s' % self.db.user,
820 '--dbpass=%s' % self.db.password,
821 '--dbserver=%s' % self.db.dbserver,
822 ]
823 )
824 else:
825 raise Exception('Unsupported database: %s' % self.db.type)
827 return install_args
829 @staticmethod
830 def _expand_template(template_file, php_constants):
831 ref = (
832 importlib.resources.files(__package__)
833 / 'mediawiki/local_settings.php.tpl'
834 )
835 with importlib.resources.as_file(ref) as quibblesettings_file:
836 customsettings = InstallMediaWiki._expand_localsettings_template(
837 quibblesettings_file, php_constants
838 )
840 return customsettings
842 @staticmethod
843 def _expand_localsettings_template(quibblesettings, php_constants):
844 # Wire variables into settings template.
845 with open(quibblesettings, "r") as f:
846 php_constants_declarations = "\n".join(
847 "const {} = '{}';".format(key, value)
848 for (key, value) in php_constants.items()
849 )
850 customsettings = f.read().replace(
851 '{{constants-declarations}}', php_constants_declarations
852 )
853 return customsettings
855 @staticmethod
856 def _apply_custom_settings(
857 localsettings, installed_copy, new_settings, log_dir
858 ):
859 os.rename(localsettings, installed_copy)
860 with open(localsettings, "w") as f:
861 f.write(new_settings)
863 copylog(localsettings, os.path.join(log_dir, 'LocalSettings.php'))
864 copylog(
865 installed_copy,
866 os.path.join(log_dir, os.path.basename(installed_copy)),
867 )
868 subprocess.check_call(['php', '-l', localsettings, installed_copy])
870 def __str__(self):
871 return "Install MediaWiki, db={}".format(self.db)
874class Phpbench:
875 """
876 See https://github.com/phpbench/phpbench / T291549
877 """
879 def __init__(self, directory, composer_install=False, aggregate=False):
880 self.directory = directory
881 self.composer_install = composer_install
882 self.aggregate = aggregate
884 def execute(self):
885 log.info(self)
886 if not _repo_has_composer_script(self.directory, 'phpbench'):
887 log.info('No phpbench entry found in composer.json')
888 return
889 log.info('Running "composer phpbench" in %s', self.directory)
890 if self.composer_install:
891 cmd = [
892 'composer',
893 '--ansi',
894 'install',
895 '--no-progress',
896 '--prefer-dist',
897 '--profile',
898 '-v',
899 ]
900 run(cmd, cwd=self.directory)
902 if not self.aggregate:
903 run(['composer', '--ansi', 'phpbench'], cwd=self.directory)
904 else:
905 run(['git', 'checkout', 'HEAD~1'], cwd=self.directory)
906 if _repo_has_composer_script(self.directory, 'phpbench'):
907 cmds = [
908 ['composer', '--ansi', 'phpbench', '--', '--tag=original'],
909 # Checkout patch branch again so we can compare against
910 # the HEAD~1 commit
911 ['git', 'checkout', '-'],
912 [
913 'composer',
914 '--ansi',
915 'phpbench',
916 '--',
917 '--ref=original',
918 '--report=aggregate',
919 ],
920 ]
922 for cmd in cmds:
923 run(cmd, cwd=self.directory)
924 else:
925 # HEAD~1 doesn't have phpbench in composer.json, so switch back
926 # to the patch, eventually returning exit code 0
927 run(['git', 'checkout', '-'], cwd=self.directory)
929 if self.composer_install:
930 GitClean(self.directory).execute()
932 def __str__(self):
933 return "Run phpbench"
936class AbstractPhpUnit:
937 def get_phpunit_command(self, repo_path=None):
938 phpunit_command = [
939 'composer',
940 'run',
941 '--timeout=0',
942 'phpunit',
943 '--',
944 ]
946 if repo_path:
947 phpunit_command.append(repo_path)
948 if self.cache_result_file is not None: 948 ↛ 951line 948 didn't jump to line 951 because the condition on line 948 was never true
949 # The path for the file is set in the classes that
950 # extend AbstractPhpUnit
951 phpunit_command.append('--cache-result-file')
952 phpunit_command.append(self.cache_result_file)
953 return phpunit_command
955 def _run_phpunit(self, group=[], exclude_group=[], cmd=None):
956 log.info(self)
958 always_excluded = ['Broken']
959 if not cmd:
960 cmd = self.get_phpunit_command()
961 if self.testsuite:
962 cmd.extend(['--testsuite', self.testsuite])
964 if group:
965 cmd.extend(['--group', ','.join(group)])
967 cmd.extend(
968 ['--exclude-group', ','.join(always_excluded + exclude_group)]
969 )
971 if self.junit and self.junit_file: 971 ↛ 973line 971 didn't jump to line 973 because the condition on line 971 was always true
972 cmd.extend(['--log-junit', self.junit_file])
973 log.info(' '.join(cmd))
975 phpunit_env = {}
976 phpunit_env.update(os.environ)
977 phpunit_env.update({'LANG': 'C.UTF-8'})
979 run(cmd, cwd=self.mw_install_path, env=phpunit_env)
982class PhpUnitDatabaseless(AbstractPhpUnit):
983 def __init__(
984 self,
985 mw_install_path,
986 testsuite,
987 log_dir,
988 junit=False,
989 cache_result_file=None,
990 ):
991 self.mw_install_path = mw_install_path
992 self.testsuite = testsuite
993 self.log_dir = log_dir
994 self.junit_file = os.path.join(self.log_dir, 'junit-dbless.xml')
995 self.junit = junit
996 self.cache_result_file = cache_result_file
998 def execute(self):
999 # XXX might want to run the triggered extension first then the
1000 # other tests.
1001 # XXX some mediawiki/core smoke PHPunit tests should probably
1002 # be run as well.
1003 self._run_phpunit(exclude_group=['Database', 'Standalone'])
1005 def __str__(self):
1006 return "PHPUnit {} suite (without database or standalone)".format(
1007 self.testsuite or 'default'
1008 )
1011class PhpUnitStandalone(AbstractPhpUnit):
1012 def __init__(
1013 self,
1014 mw_install_path,
1015 testsuite,
1016 log_dir,
1017 repo_path,
1018 junit=False,
1019 cache_result_file=None,
1020 ):
1021 self.mw_install_path = mw_install_path
1022 self.testsuite = testsuite
1023 self.log_dir = log_dir
1024 self.repo_path = repo_path
1025 self.junit_file = os.path.join(self.log_dir, 'junit-standalone.xml')
1026 self.junit = junit
1027 self.cache_result_file = cache_result_file
1029 def execute(self):
1030 self._run_phpunit(
1031 group=['Standalone'],
1032 cmd=self.get_phpunit_command(self.repo_path),
1033 )
1035 def __str__(self):
1036 return "PHPUnit {} standalone suite on {}".format(
1037 self.testsuite or 'default', self.repo_path
1038 )
1041class PhpUnitUnit(AbstractPhpUnit):
1042 def __init__(
1043 self, mw_install_path, log_dir, junit=False, cache_result_file=None
1044 ):
1045 self.mw_install_path = mw_install_path
1046 self.log_dir = log_dir
1047 self.testsuite = None
1048 self.junit_file = os.path.join(self.log_dir, 'junit-unit.xml')
1049 self.junit = junit
1050 self.cache_result_file = cache_result_file
1052 def execute(self):
1053 if _repo_has_composer_script(self.mw_install_path, 'phpunit:unit'):
1054 self._run_phpunit(cmd=['composer', 'phpunit:unit', '--'])
1055 else:
1056 log.debug('skipping phpunit:unit stage, script is not present')
1057 return
1059 def __str__(self):
1060 return "PHPUnit unit tests"
1063class PhpUnitDatabase(AbstractPhpUnit):
1064 def __init__(
1065 self,
1066 mw_install_path,
1067 testsuite,
1068 log_dir,
1069 junit=False,
1070 cache_result_file=None,
1071 ):
1072 self.mw_install_path = mw_install_path
1073 self.testsuite = testsuite
1074 self.log_dir = log_dir
1075 self.junit_file = os.path.join(self.log_dir, 'junit-db.xml')
1076 self.junit = junit
1077 self.cache_result_file = cache_result_file
1079 def execute(self):
1080 self._run_phpunit(group=['Database'], exclude_group=['Standalone'])
1082 def __str__(self):
1083 return "PHPUnit {} suite (with database)".format(
1084 self.testsuite or 'default'
1085 )
1088class PhpUnitPrepareParallelRunComposer:
1089 """To run tests in parallel, we need to split the tests that
1090 will be run into smaller suites that we can execute individually
1091 and in parallel. This command runs the phpunit `--list-tests-xml`
1092 function, which dumps out a list of which test classes would be
1093 included in a run of the provided test suite. Composer does this
1094 preparation for us
1096 @see T365978
1097 """
1099 def __init__(
1100 self,
1101 mw_install_path,
1102 testsuite='extensions',
1103 log_dir=None,
1104 junit=False,
1105 ):
1106 self.mw_install_path = mw_install_path
1107 self.testsuite = testsuite
1108 self.log_dir = log_dir
1109 self.junit = junit
1111 def execute(self):
1112 """Use phpunit's `--list-tests-xml` function to generate a
1113 list of test classes that would be included in the suite
1114 and split that list into smaller groups that we can have
1115 composer run in parallel"""
1116 phpunit_env = {}
1117 phpunit_env.update(os.environ)
1118 phpunit_env.update({'LANG': 'C.UTF-8'})
1120 composer_command = 'phpunit:prepare-parallel:default'
1121 if self.testsuite == 'extensions': 1121 ↛ 1124line 1121 didn't jump to line 1124 because the condition on line 1121 was always true
1122 composer_command = 'phpunit:prepare-parallel:extensions'
1124 phpunit_command = ['composer', composer_command]
1126 run(phpunit_command, cwd=self.mw_install_path, env=phpunit_env)
1127 # To support developers in reproducing failed test runs, we
1128 # make a copy of the phpunit*.xml files in the logs folder -
1129 # this adds the file to the artefacts collected by Jenkins
1130 copylog(
1131 os.path.join(self.mw_install_path, 'phpunit-database.xml'),
1132 os.path.join(self.log_dir, 'phpunit-parallel-database.xml'),
1133 )
1134 copylog(
1135 os.path.join(self.mw_install_path, 'phpunit-databaseless.xml'),
1136 os.path.join(self.log_dir, 'phpunit-parallel-databaseless.xml'),
1137 )
1139 def __str__(self):
1140 return "PHPUnit Prepare Parallel Run (Composer)"
1143class AbstractParallelPhpUnit:
1144 """Parent class for running the parallel phpunit test suites."""
1146 def __init__(self, mw_install_path, testsuite, log_dir, junit=False):
1147 self.mw_install_path = mw_install_path
1148 self.testsuite = testsuite
1149 self.log_dir = log_dir
1150 self.junit_file = os.path.join(self.log_dir, 'junit-db.xml')
1151 self.junit = junit
1154class PhpUnitDatabaselessParallelComposer(AbstractParallelPhpUnit):
1155 """Run the tests in the provided suite in parallel, excluding
1156 Database and Standalone tests."""
1158 def execute(self):
1159 """Execute the parallel databaseless test suite"""
1160 phpunit_env = {}
1161 phpunit_env.update(os.environ)
1162 phpunit_env.update({'LANG': 'C.UTF-8'})
1164 phpunit_command = [
1165 'composer',
1166 'run',
1167 '--timeout=0',
1168 'phpunit:parallel:databaseless',
1169 '--',
1170 ]
1172 run(phpunit_command, cwd=self.mw_install_path, env=phpunit_env)
1174 def __str__(self):
1175 return (
1176 "PHPUnit {} suite (without database "
1177 "or standalone) parallel run (Composer)"
1178 ).format(self.testsuite or 'default')
1181class PhpUnitDatabaseParallelComposer(AbstractParallelPhpUnit):
1182 """Run the tests in the provided suite in parallel, excluding
1183 Standalone tests and including the Database tests."""
1185 def execute(self):
1186 """Execute the parallel databaseless test suite"""
1187 phpunit_env = {}
1188 phpunit_env.update(os.environ)
1189 phpunit_env.update({'LANG': 'C.UTF-8'})
1191 phpunit_command = [
1192 'composer',
1193 'run',
1194 '--timeout=0',
1195 'phpunit:parallel:database',
1196 '--',
1197 ]
1199 run(phpunit_command, cwd=self.mw_install_path, env=phpunit_env)
1201 def __str__(self):
1202 return (
1203 "PHPUnit {} suite (with database) parallel run (Composer)"
1204 ).format(self.testsuite or 'default')
1207class PhpUnitParallelNotice:
1208 """Write a notice to the end of the log output so that users
1209 know that this has been a parallel test run and know where to
1210 report issues in the event of failures."""
1212 def execute(self):
1213 log.info(
1214 'NOTICE: These tests have been executed with '
1215 'PHPUnit Parallel enabled.'
1216 )
1217 log.info(
1218 'If you encounter unexpected test failures or '
1219 'notice incomplete execution of test suites, '
1220 'please let us know!'
1221 )
1222 log.info(
1223 'For more information, and to report parallel-'
1224 'testing-related failures, please visit '
1225 'https://phabricator.wikimedia.org/T361190'
1226 )
1228 def __str__(self):
1229 return "PHPUnit Parallel Notice"
1232class QunitTests:
1233 def __init__(self, mw_install_path, web_url):
1234 self.mw_install_path = mw_install_path
1235 self.web_url = web_url
1237 def execute(self):
1238 karma_env = {
1239 'CHROME_BIN': '/usr/bin/chromium',
1240 'MW_SERVER': self.web_url,
1241 'MW_SCRIPT_PATH': '/',
1242 'FORCE_COLOR': '1', # for 'supports-color'
1243 }
1244 karma_env.update(os.environ)
1245 karma_env.update({'CHROMIUM_FLAGS': quibble.chromium_flags()})
1247 run(
1248 ['./node_modules/.bin/grunt', 'qunit'],
1249 cwd=self.mw_install_path,
1250 env=karma_env,
1251 )
1253 def __str__(self):
1254 return "Run QUnit tests"
1257class ApiTesting:
1258 def __init__(self, mw_install_path, projects, url, web_backend):
1259 self.mw_install_path = mw_install_path
1260 self.projects = projects
1261 self.url = url
1262 self.web_backend = web_backend
1264 def execute(self):
1265 settings_in_path = (
1266 self.mw_install_path
1267 + "/tests/api-testing/.api-testing-quibble.json"
1268 )
1269 settings_out_path = self.mw_install_path + "/api-testing-quibble.json"
1270 with open(settings_in_path) as settings_in:
1271 api_settings = json.load(settings_in)
1273 api_settings['base_uri'] = self.url + "/"
1275 with open(settings_out_path, "w") as settings_out:
1276 json.dump(api_settings, settings_out)
1277 quibble_testing_config_env = {
1278 "API_TESTING_CONFIG_FILE": self.mw_install_path
1279 + "/api-testing-quibble.json"
1280 }
1281 quibble_testing_config_env.update(os.environ)
1282 if self.web_backend == 'external':
1283 quibble_testing_config_env.update({'QUIBBLE_APACHE': '1'})
1285 for project in self.projects:
1286 project_dir = os.path.normpath(
1287 os.path.join(
1288 self.mw_install_path, quibble.zuul.repo_dir(project)
1289 )
1290 )
1291 if repo_has_npm_script(project_dir, 'api-testing'):
1292 _npm_install(project_dir, label=project)
1293 run(
1294 [quibble.get_npm_command(), 'run', 'api-testing'],
1295 cwd=project_dir,
1296 env=quibble_testing_config_env,
1297 )
1299 def __str__(self):
1300 return "Run API-Testing"
1303class BrowserTests:
1304 def __init__(
1305 self,
1306 mw_install_path,
1307 projects,
1308 display,
1309 web_url,
1310 web_backend,
1311 parallel_npm_install=False,
1312 ):
1313 self.mw_install_path = mw_install_path
1314 self.projects = projects
1315 self.display = display
1316 self.web_url = web_url
1317 self.web_backend = web_backend
1318 self.parallel_npm_install = parallel_npm_install
1320 def execute(self):
1321 for project in self.projects:
1322 project_dir = get_project_dir(self.mw_install_path, project)
1323 if repo_has_npm_script(project_dir, 'selenium-test'):
1324 chrono_name = "Browser tests in '%s'" % project
1325 with quibble.Chronometer(chrono_name, log.info):
1326 self._run_webdriver(project_dir, project)
1328 def _run_webdriver(self, project_dir, project):
1329 webdriver_env = {}
1330 webdriver_env.update(os.environ)
1331 webdriver_env.update(
1332 {
1333 'MW_SERVER': self.web_url,
1334 'MW_SCRIPT_PATH': '/',
1335 'FORCE_COLOR': '1', # for 'supports-color'
1336 'MEDIAWIKI_USER': 'WikiAdmin',
1337 'MEDIAWIKI_PASSWORD': 'testwikijenkinspass',
1338 'DISPLAY': self.display,
1339 }
1340 )
1341 if self.web_backend == 'external':
1342 webdriver_env.update({'QUIBBLE_APACHE': '1'})
1344 if not self.parallel_npm_install:
1345 _npm_install(project_dir, label=project)
1346 run(
1347 [quibble.get_npm_command(), 'run', 'selenium-test'],
1348 cwd=project_dir,
1349 env=webdriver_env,
1350 )
1352 def __str__(self):
1353 return 'Run all browser tests'
1356class UserScripts:
1357 def __init__(self, mw_install_path, commands, web_url, web_backend):
1358 self.mw_install_path = mw_install_path
1359 self.commands = commands
1360 self.web_url = web_url
1361 self.web_backend = web_backend
1363 def execute(self):
1364 log.info('User commands, working directory: %s', self.mw_install_path)
1365 userscripts_env = {}
1366 userscripts_env.update(os.environ)
1367 userscripts_env.update(
1368 {
1369 'MW_SERVER': self.web_url,
1370 'MW_SCRIPT_PATH': '/',
1371 'MEDIAWIKI_USER': 'WikiAdmin',
1372 'MEDIAWIKI_PASSWORD': 'testwikijenkinspass',
1373 }
1374 )
1375 if self.web_backend == 'external': 1375 ↛ 1378line 1375 didn't jump to line 1378 because the condition on line 1375 was always true
1376 userscripts_env.update({'QUIBBLE_APACHE': '1'})
1378 multiple_commands = len(self.commands) > 1
1379 for cmd in self.commands:
1380 with contextlib.ExitStack() as stack:
1381 if multiple_commands:
1382 stack.enter_context(quibble.Chronometer(cmd, log.info))
1383 run(
1384 cmd,
1385 shell=True,
1386 cwd=self.mw_install_path,
1387 env=userscripts_env,
1388 )
1390 def __str__(self):
1391 return "User commands: {}".format(", ".join(self.commands))
1394class EnsureDirectory:
1395 def __init__(self, directory):
1396 self.directory = directory
1398 def execute(self):
1399 os.makedirs(self.directory, exist_ok=True)
1401 def __str__(self):
1402 return "Ensure dir: '{}'".format(self.directory)
1405class GitClean:
1406 def __init__(self, directory):
1407 self.directory = directory
1409 def execute(self):
1410 subprocess.check_call(['git', 'clean', '-xqdf'], cwd=self.directory)
1411 leftover_files = subprocess.check_output(
1412 ['git', 'status', '--ignored', '--porcelain'],
1413 cwd=self.directory,
1414 text=True,
1415 )
1416 if leftover_files: 1416 ↛ exitline 1416 didn't return from function 'execute' because the condition on line 1416 was always true
1417 log.warning('git clean left behind some files!!! T321795')
1418 for line in leftover_files.rstrip().split('\n'):
1419 log.warning(line)
1420 log.warning(
1421 'Build continuining nonetheless but unexpected '
1422 'failures might happen'
1423 )
1425 def __str__(self):
1426 return "Revert to git clean -xqdf in {}".format(self.directory)
1429class Parallel:
1430 """Run subcommands in parallel.
1432 Steps are an iterable of command objects, to be evaluated
1433 immediately.
1435 Subprocess stdout and stderr, and logging are piped to an interleaved
1436 capture buffer and logged by the parent as each child completes.
1438 Any exceptions are bubbled up.
1439 """
1441 def __init__(self, *, name=None, steps):
1442 self.name = name or "parallel steps"
1443 self.steps = list(steps)
1445 self.workers = max(1, min(len(self.steps), os.cpu_count()))
1447 def execute(self):
1448 # Short-circuit if there aren't enough steps to run in parallel.
1449 if len(self.steps) == 0:
1450 return
1451 elif len(self.steps) == 1:
1452 return execute_command(self.steps[0])
1454 with multiprocessing.Pool(processes=self.workers) as pool:
1455 results = pool.imap_unordered(self._run_child, self.steps)
1456 results_in_progress = ProgressReporter(
1457 desc=self.name,
1458 iterable=results,
1459 sleep_interval=monitor_interval,
1460 total=len(self.steps),
1461 )
1462 for error, capture in results_in_progress:
1463 log.info(capture)
1464 if error:
1465 error.output = capture
1466 raise error
1468 @staticmethod
1469 def _run_child(command):
1470 """Run a command and return its output
1472 This is executed in the child process context, and pipes all of its own
1473 output streams to a single collector. This collected output and any
1474 error are returned in a serializable format.
1476 The child outputs are read as bytes and decoded to Unicode replacing
1477 any potential invalid characters with their hexadecimal form.
1479 Returns
1480 -------
1481 tuple
1482 error : Exception or None
1483 captured : text
1484 Output of the command, with stdout, stderr, and log lines
1485 interleaved.
1486 """
1487 with tempfile.TemporaryFile() as collector, \
1488 quibble.util.redirect_all_streams(collector): # fmt: skip
1489 try:
1490 execute_command(command)
1491 error = None
1492 except Exception as ex:
1493 error = ex
1494 finally:
1495 collector.flush()
1496 collector.seek(0, io.SEEK_SET)
1497 # With Python 3.8 we could use:
1498 # TemporaryFile(errors='backslashreplace')
1499 captured = collector.read().decode(errors='backslashreplace')
1501 return (error, captured)
1503 def __str__(self):
1504 return "Run {} in parallel (concurrency={}):".format(
1505 self.name, self.workers
1506 ) + "".join(["\n* " + step for step in map(str, self.steps)])
1509class SuccessCache:
1510 """
1511 SuccessCache looks for a prior success associated with all Git
1512 working directories under the given path along with additional base cache
1513 key constraints (e.g. project name or build parameters that can determine
1514 which tests are run).
1516 It looks in memcached for the cache key (under a `successcache/` prefix).
1517 The cache key is the SHA256 digest of the given key data and the
1518 `HEAD^{tree}` hash of each of the sorted cloned repos under the given
1519 source path.
1520 """
1522 def __init__(self, client, src_path, projects, key_data=None):
1523 """
1524 client: Quibble cache client.
1525 src_path: Path to the root of the source code being tested
1526 projects: Projects that were cloned.
1527 key_data: Additional data to use when computing a cache key.
1528 """
1529 self.client = client
1530 self.key_data = []
1531 self.src_path = src_path
1532 self.projects = projects
1534 if key_data is not None: 1534 ↛ 1537line 1534 didn't jump to line 1537 because the condition on line 1534 was always true
1535 self.key_data = key_data
1537 self.__digest = None
1539 def check(self):
1540 log.info(
1541 'Checking success cache for all repos under %s '
1542 'and initial key data %s',
1543 self.src_path,
1544 self.key_data,
1545 )
1547 exists = self.client.get(self._key()) is not None
1549 if exists:
1550 log.info('Success cache: HIT')
1551 raise self.Hit()
1553 log.info('Success cache: MISS')
1555 def save(self):
1556 log.info('Saving success cache entry: %s', self._key())
1557 self.client.set(self._key(), '')
1559 def check_command(self):
1560 return self.Check(self)
1562 def save_command(self):
1563 return self.Save(self)
1565 def _key(self):
1566 return 'successcache/%s' % self._digest()
1568 def _digest(self):
1569 if self.__digest is None:
1570 h = hashlib.new('sha256')
1572 for key in self.key_data:
1573 h.update(key.encode('utf8') + b"\x00")
1575 for tree in self._trees():
1576 h.update(tree.encode('ascii') + b"\x00")
1578 self.__digest = h.hexdigest()
1580 return self.__digest
1582 def _trees(self):
1583 for path in sorted(self._repos()):
1584 tree = git.Repo(path).tree('HEAD').hexsha
1585 log.info('Found repo %s with tree %s', path, tree)
1586 yield tree
1588 def _repos(self):
1589 return quibble.zuul.working_trees(
1590 self.src_path, self.projects
1591 ).values()
1593 class Check:
1594 def __init__(self, cache):
1595 self.cache = cache
1597 def execute(self):
1598 self.cache.check()
1600 def __str__(self):
1601 return 'Check success cache'
1603 class Save:
1604 def __init__(self, cache):
1605 self.cache = cache
1607 def execute(self):
1608 self.cache.save()
1610 def __str__(self):
1611 return 'Save success cache'
1613 class Hit(Exception):
1614 pass
1617def _repo_has_composer_script(project_dir, script_name):
1618 composer_path = os.path.join(project_dir, 'composer.json')
1619 return _json_has_script(composer_path, script_name)
1622def _repo_has_npm_lock(project_dir):
1623 lock_path = os.path.join(project_dir, 'package-lock.json')
1624 return os.path.exists(lock_path)
1627def repo_has_quibble_config(project_dir):
1628 return os.path.exists(os.path.join(project_dir, 'quibble.yaml'))
1631def repo_load_quibble_config(project_dir):
1632 with open(os.path.join(project_dir, 'quibble.yaml')) as f:
1633 return yaml.safe_load(f)
1636def repo_has_npm_script(project_dir, script_name):
1637 package_path = os.path.join(project_dir, 'package.json')
1638 return _json_has_script(package_path, script_name)
1641def get_project_dir(mw_install_path, project):
1642 """Get the normalized path for a Zuul project."""
1643 with quibble.logginglevel('zuul.CloneMapper', logging.WARNING):
1644 repo_dir = quibble.zuul.repo_dir(project)
1645 return os.path.normpath(os.path.join(mw_install_path, repo_dir))
1648def _json_has_script(json_file, script_name):
1649 if not os.path.exists(json_file):
1650 return False
1651 with open(json_file) as f:
1652 spec = json.load(f)
1653 return 'scripts' in spec and script_name in spec['scripts']
1656def transmit_error(
1657 should_comment: int,
1658 should_vote: int,
1659 phase: str,
1660 command: str,
1661 reporting_url: str,
1662 api_key: str,
1663 called_process_error: subprocess.CalledProcessError,
1664):
1665 """
1666 Transmit command execution error data to an HTTP endpoint.
1668 :param should_comment: 1 for true, 0 for false.
1669 :param should_vote: 1 for true, 0 for false.
1670 :param phase: The Quibble phase as defined in the command plan.
1671 :param command: The command name from CalledProcessError.cmd.
1672 :param reporting_url: The URL to post the error data to
1673 :param api_key: The API key to use with the outbound request. The
1674 recipient of the data can use this to validate the request.
1675 :param called_process_error: The CalledProcessError object from
1676 the failed command. `output` is set on this object if failed
1677 in a `ParallelCommand` or from `run()`.
1678 """
1680 zuul_pipeline = os.getenv('ZUUL_PIPELINE')
1681 zuul_project = os.getenv('ZUUL_PROJECT')
1682 zuul_change = os.getenv('ZUUL_CHANGE')
1683 zuul_patchset = os.getenv('ZUUL_PATCHSET')
1684 build_url = os.getenv('BUILD_URL')
1686 output = ''
1687 if called_process_error.output:
1688 output = called_process_error.output
1690 try:
1691 data = {
1692 'should_comment': should_comment,
1693 'should_vote': should_vote,
1694 'phase': str(phase),
1695 'command': command,
1696 'pipeline': zuul_pipeline,
1697 'output': output,
1698 'project': zuul_project,
1699 'change': zuul_change,
1700 'patchset': zuul_patchset,
1701 'build_url': build_url + 'consoleFull',
1702 }
1703 log.debug(
1704 'Sending error data for Quibble phase "%s" to %s.',
1705 phase,
1706 reporting_url,
1707 )
1708 requests.post(
1709 reporting_url, json=data, headers={"x-api-key": api_key}, timeout=5
1710 )
1711 except requests.exceptions.RequestException:
1712 log.exception("Failed to POST data")
1713 return
1714 except Exception:
1715 # Deliberately ignore most errors here; we don't want exceptions
1716 # generated while transmitting error data to interfere with
1717 # Quibble's functioning.
1718 log.exception("An exception occurred while sending data")
1719 return