Coverage for quibble/cmd.py: 89%

1#!/usr/bin/env python3 

2# 

3# Copyright 2017-2018, Antoine "hashar" Musso 

4# Copyright 2017, Tyler Cipriani 

5# Copyright 2017-2018, Wikimedia Foundation Inc. 

6# 

7# Licensed under the Apache License, Version 2.0 (the "License"); 

8# you may not use this file except in compliance with the License. 

9# You may obtain a copy of the License at 

10# 

11# http://www.apache.org/licenses/LICENSE-2.0 

12# 

13# Unless required by applicable law or agreed to in writing, software 

14# distributed under the License is distributed on an "AS IS" BASIS, 

15# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 

16# See the License for the specific language governing permissions and 

17# limitations under the License. 

18 

19import argparse 

20import contextlib 

21import logging 

22import os 

23import subprocess 

24import sys 

25import tempfile 

26 

27import quibble 

28import quibble.cache 

29import quibble.mediawiki.maintenance 

30import quibble.backend 

31import quibble.zuul 

32import quibble.commands 

33import quibble.util 

34 

35log = logging.getLogger('quibble.cmd') 

36known_stages = [ 

37 'all', 

38 'phpunit-unit', 

39 'phpbench', 

40 'phpunit', 

41 'phpunit-standalone', 

42 'phpunit-parallel', 

43 'npm-test', 

44 'composer-test', 

45 'qunit', 

46 'selenium', 

47 'api-testing', 

48] 

49default_stages = [ 

50 'phpunit-unit', 

51 'phpbench', 

52 'phpunit', 

53 'phpunit-standalone', 

54 'npm-test', 

55 'composer-test', 

56 'qunit', 

57 'selenium', 

58 'api-testing', 

59] 

60 

61 

62# Used for add_argument(choices=) let us validate multiple choices at once. 

63# >>> 'a' in MultipleChoices(['a', 'b', 'c']) 

64# True 

65# >>> ['a', 'b'] in MultipleChoices(['a', 'b', 'c']) 

66# True 

67class MultipleChoices(list): 

68 def __contains__(self, item): 

69 return set(item).issubset(set(self)) 

70 

71 

72class QuibbleCmd(object): 

73 def __init__(self): 

74 self._context_stack = contextlib.ExitStack() 

75 

76 def _setup_environment( 

77 self, 

78 workspace, 

79 mw_install_path, 

80 log_dir, 

81 tmp_dir, 

82 is_vendor=False, 

83 zuul_env={}, 

84 ): 

85 """ 

86 Set and get needed environment variables. 

87 

88 TODO: Can we deprecate any of these in favor of explicit 

89 parameters? 

90 """ 

91 if quibble.is_in_docker() or 'WORKSPACE' not in os.environ: 

92 # Override WORKSPACE in Docker, we really want /workspace or 

93 # whatever was given from the command line. 

94 # Else set it, since some code might rely on it being set to detect 

95 # whether they are under CI. 

96 os.environ['WORKSPACE'] = workspace 

97 

98 os.environ['MW_INSTALL_PATH'] = mw_install_path 

99 os.environ['MW_LOG_DIR'] = log_dir 

100 os.environ['LOG_DIR'] = log_dir 

101 os.environ['TMPDIR'] = tmp_dir 

102 

103 # The workflow for updating mediawiki/vendor.git is: 

104 # 1. write patch for vendor.git (and its lock file) 

105 # 2. write patch for mediawiki/core.git and its composer.json file 

106 # (with Depends-On #1). 

107 # 

108 # As such, when we run MediaWiki tests for the mediawiki/vendor patch, 

109 # it is impossible for its lock file to match mediawiki/core. 

110 # To prevent this circular dependency, we tell MediaWiki core to skip 

111 # composer lock check (phpunit: TestSetup, and maintenance/update.php) 

112 # during the Jenkins job for the mediawiki/vendor patch. 

113 # 

114 # Instead, we test this during the mediawiki/core commit instead, via 

115 # the "mediawiki-vendor"-quibble job, where checks are NOT skipped. 

116 # 

117 # T333412, T88211, T370380 

118 if is_vendor: 

119 os.environ['MW_SKIP_EXTERNAL_DEPENDENCIES'] = '1' 

120 

121 if zuul_env: 

122 os.environ.update(zuul_env) 

123 

124 def _warn_obsolete_env_deps(self, var): 

125 log.warning( 

126 '%s env variable is deprecated. ' 

127 'Instead pass projects as arguments.', 

128 var, 

129 ) 

130 

131 def _repos_to_clone(self, projects, zuul_project, clone_vendor): 

132 """ 

133 Find repos to clone basedon passed arguments and environment 

134 """ 

135 dependencies = set() 

136 dependencies.add('mediawiki/skins/Vector') 

137 if clone_vendor: 

138 log.info('Adding mediawiki/vendor') 

139 dependencies.add('mediawiki/vendor') 

140 

141 # TODO: Remove this and build a list of additional dependencies. 

142 if zuul_project is not None: 

143 dependencies.add(zuul_project) 

144 

145 if 'SKIN_DEPENDENCIES' in os.environ: 

146 self._warn_obsolete_env_deps('SKIN_DEPENDENCIES') 

147 dependencies.update( 

148 os.environ.get('SKIN_DEPENDENCIES').split('\\n') 

149 ) 

150 

151 if 'EXT_DEPENDENCIES' in os.environ: 

152 self._warn_obsolete_env_deps('EXT_DEPENDENCIES') 

153 dependencies.update( 

154 os.environ.get('EXT_DEPENDENCIES').split('\\n') 

155 ) 

156 

157 dependencies.update(projects) 

158 

159 # mediawiki/core should be first else git clone will fail because the 

160 # destination directory already exists. 

161 if 'mediawiki/core' in dependencies: 

162 dependencies.remove('mediawiki/core') 

163 dependencies = sorted(dependencies) 

164 dependencies.insert(0, 'mediawiki/core') 

165 

166 log.info('Projects: %s', ', '.join(dependencies)) 

167 

168 return dependencies 

169 

170 def _stages_to_run(self, run, skip, commands): 

171 if commands or 'all' in skip: 

172 return [] 

173 

174 stages = default_stages 

175 if skip: 

176 stages = [s for s in stages if s not in skip] 

177 if len(run) > 0: 

178 stages = run 

179 if os.getenv('QUIBBLE_PHPUNIT_PARALLEL') and 'phpunit' in stages: 

180 stages.append('phpunit-parallel') 

181 return stages 

182 

183 def get_zuul_env_from_cli_args(self, args): 

184 """Set the initial ZUUL environment based on command-line 

185 arguments. Currently, only the `--change` argument is 

186 processed here to set ZUUL_REF""" 

187 if not args.change: 

188 return {} 

189 return quibble.util.FetchInfo.change( 

190 *args.change.split(',') 

191 ).asZuulEnv() 

192 

193 def build_execution_plan(self, args): 

194 workspace = args.workspace 

195 mw_install_path = os.path.join(workspace, 'src') 

196 log_dir = os.path.join(workspace, args.log_dir) 

197 memcached_port = 11211 

198 

199 if args.db_dir is not None: 199 ↛ 200line 199 didn't jump to line 200 because the condition on line 199 was never true

200 db_dir = os.path.join(workspace, args.db_dir) 

201 else: 

202 db_dir = None 

203 

204 if args.dump_db_postrun: 204 ↛ 205line 204 didn't jump to line 205 because the condition on line 204 was never true

205 dump_dir = log_dir 

206 else: 

207 dump_dir = None 

208 

209 tmp_dir = tempfile.gettempdir() 

210 

211 cache_client = None 

212 if args.memcached_server is not None: 

213 cache_client = quibble.cache.client(args.memcached_server) 

214 

215 # Set ZUUL variables when given `--change ###` 

216 zuul_env = self.get_zuul_env_from_cli_args(args) 

217 

218 self._setup_environment( 

219 workspace, mw_install_path, log_dir, tmp_dir, zuul_env=zuul_env 

220 ) 

221 

222 zuul_project = os.environ.get('ZUUL_PROJECT', None) 

223 if zuul_project is None: 

224 # TODO: Isn't this default already covered by quibble.zuul, and we 

225 # can remove this code? 

226 log.warning('ZUUL_PROJECT not set. Assuming mediawiki/core') 

227 zuul_project = 'mediawiki/core' 

228 else: 

229 log.debug("ZUUL_PROJECT=%s", zuul_project) 

230 

231 is_core = zuul_project == 'mediawiki/core' 

232 is_extension = ( 

233 zuul_project.startswith('mediawiki/extensions/') 

234 or zuul_project == 'mediawiki/services/parsoid' 

235 ) 

236 is_skin = zuul_project.startswith('mediawiki/skins/') 

237 is_vendor = zuul_project == 'mediawiki/vendor' 

238 

239 use_composer = args.packages_source == 'composer' 

240 use_vendor = args.packages_source == 'vendor' 

241 

242 self._setup_environment( 

243 workspace, mw_install_path, log_dir, tmp_dir, is_vendor=is_vendor 

244 ) 

245 

246 dependencies = self._repos_to_clone( 

247 projects=args.projects, 

248 zuul_project=zuul_project, 

249 clone_vendor=use_vendor, 

250 ) 

251 

252 dependencies_with_project_first = quibble.util.move_item_to_head( 

253 dependencies, zuul_project 

254 ) 

255 

256 repo_path = quibble.zuul.repo_dir(zuul_project) 

257 

258 stages = self._stages_to_run(args.run, args.skip, args.commands) 

259 log.debug('Running stages: %s', ', '.join(stages)) 

260 log.debug( 

261 'QUIBBLE_PHPUNIT_PARALLEL: %s', 

262 os.getenv('QUIBBLE_PHPUNIT_PARALLEL'), 

263 ) 

264 

265 run_composer = 'composer-test' in stages 

266 run_npm = 'npm-test' in stages 

267 

268 database_backend = quibble.backend.getDatabase( 

269 args.db, db_dir, dump_dir, log_dir 

270 ) 

271 

272 web_backend_args = {} 

273 if args.web_backend == 'php' and args.web_php_workers: 273 ↛ 274line 273 didn't jump to line 274 because the condition on line 273 was never true

274 web_backend_args = { 

275 'workers': args.web_php_workers, 

276 } 

277 

278 web_backend = quibble.backend.getWebserver( 

279 args.web_backend, mw_install_path, args.web_url, web_backend_args 

280 ) 

281 

282 plan = [] 

283 

284 # Interactive shell does not need a report 

285 if args.shell is None: 

286 plan.append( 

287 quibble.commands.ReportDurations(self._context_stack, log_dir) 

288 ) 

289 

290 plan.append(quibble.commands.ReportVersions()) 

291 

292 plan.append(quibble.commands.EnsureDirectory(log_dir)) 

293 

294 if not args.skip_zuul: 294 ↛ 331line 294 didn't jump to line 331 because the condition on line 294 was always true

295 zuul_params = { 

296 'branch': args.branch, 

297 'cache_dir': args.git_cache, 

298 'project_branch': args.project_branch, 

299 'workers': args.git_parallel, 

300 'workspace': os.path.join(workspace, 'src'), 

301 'zuul_branch': os.getenv('ZUUL_BRANCH'), 

302 'zuul_newrev': os.getenv('ZUUL_NEWREV'), 

303 'zuul_project': os.getenv('ZUUL_PROJECT'), 

304 'zuul_ref': os.getenv('ZUUL_REF'), 

305 'zuul_url': os.getenv('ZUUL_URL'), 

306 } 

307 

308 plan.append( 

309 quibble.commands.ZuulClone( 

310 projects=dependencies, **zuul_params 

311 ) 

312 ) 

313 

314 if args.resolve_requires: 314 ↛ 315line 314 didn't jump to line 315 because the condition on line 314 was never true

315 plan.append( 

316 quibble.commands.ResolveRequires( 

317 mw_install_path=mw_install_path, 

318 projects=dependencies, 

319 zuul_params=zuul_params, 

320 fail_on_extra_requires=args.fail_on_extra_requires, 

321 ) 

322 ) 

323 

324 plan.append( 

325 quibble.commands.ExtSkinSubmoduleUpdate( 

326 mw_install_path, 

327 jobs=args.git_parallel, 

328 ) 

329 ) 

330 

331 success_cache = None 

332 if cache_client is not None and args.success_cache_key_data: 

333 success_cache = quibble.commands.SuccessCache( 

334 cache_client, 

335 mw_install_path, 

336 dependencies, 

337 key_data=args.success_cache_key_data, 

338 ) 

339 plan.append(success_cache.check_command()) 

340 

341 # Assume project dir is mediawiki/core by default 

342 project_dir = mw_install_path 

343 

344 if is_extension or is_skin: 

345 project_dir = os.path.join(mw_install_path, repo_path) 

346 

347 parallel_steps = [] 

348 if run_composer: 348 ↛ 352line 348 didn't jump to line 352 because the condition on line 348 was always true

349 parallel_steps.append( 

350 quibble.commands.ExtSkinComposerTest(project_dir) 

351 ) 

352 if run_npm: 352 ↛ 354line 352 didn't jump to line 354 because the condition on line 352 was always true

353 parallel_steps.append(quibble.commands.NpmTest(project_dir)) 

354 if parallel_steps: 354 ↛ 365line 354 didn't jump to line 365 because the condition on line 354 was always true

355 plan.extend( 

356 [ 

357 quibble.commands.Parallel( 

358 name="npm and composer tests, if present", 

359 steps=parallel_steps, 

360 ), 

361 quibble.commands.GitClean(project_dir), 

362 ] 

363 ) 

364 

365 if not args.skip_deps and use_composer: 

366 plan.append( 

367 quibble.commands.CreateComposerLocal( 

368 mw_install_path, dependencies 

369 ) 

370 ) 

371 plan.append( 

372 quibble.commands.NativeComposerDependencies(mw_install_path) 

373 ) 

374 

375 if not args.skip_deps: 375 ↛ 388line 375 didn't jump to line 388 because the condition on line 375 was always true

376 if use_vendor and ( 

377 # Stages that do not need dev-requires to work 

378 set(stages) - {'selenium', 'qunit', 'npm-test', 'api-testing'} 

379 or args.commands 

380 ): 

381 plan.append( 

382 quibble.commands.VendorComposerDependencies( 

383 mw_install_path, log_dir 

384 ) 

385 ) 

386 

387 # Post dependency setup, pre database dependent phase. 

388 parallel_steps = [] 

389 

390 if not args.skip_install: 390 ↛ 416line 390 didn't jump to line 416 because the condition on line 390 was always true

391 if not args.db_is_external: 

392 database_backends = [database_backend] 

393 

394 if quibble.util.strtobool( 394 ↛ 397line 394 didn't jump to line 397 because the condition on line 394 was never true

395 os.getenv('QUIBBLE_OPENSEARCH', 'false') 

396 ): 

397 database_backends.append(quibble.backend.OpenSearch()) 

398 

399 plan.append( 

400 quibble.commands.StartBackends( 

401 self._context_stack, database_backends 

402 ) 

403 ) 

404 

405 parallel_steps.append( 

406 quibble.commands.InstallMediaWiki( 

407 mw_install_path=mw_install_path, 

408 db=database_backend, 

409 web_url=web_backend.url, 

410 log_dir=log_dir, 

411 memcached_port=memcached_port, 

412 tmp_dir=tmp_dir, 

413 ) 

414 ) 

415 

416 if ( 

417 not args.skip_deps 

418 and not args.skip_npm_install 

419 and ( 

420 run_npm 

421 or 'qunit' in stages 

422 or 'api-testing' in stages 

423 or args.commands 

424 ) 

425 ): 

426 parallel_steps.append(quibble.commands.NpmInstall(mw_install_path)) 

427 

428 plan.append( 

429 quibble.commands.Parallel( 

430 name="Post-dependency install, pre-database dependent steps", 

431 steps=parallel_steps, 

432 ) 

433 ) 

434 

435 # phpunit-unit needs LocalSettings.php, see T227900#9014246 

436 if 'phpunit-unit' in stages: 

437 plan.append( 

438 quibble.commands.PhpUnitUnit( 

439 mw_install_path, log_dir, args.phpunit_junit 

440 ) 

441 ) 

442 

443 plan.append( 

444 quibble.commands.StartBackends( 

445 self._context_stack, 

446 [quibble.backend.Memcached(port=memcached_port)], 

447 ) 

448 ) 

449 

450 phpunit_testsuite = None 

451 if args.phpunit_testsuite: 451 ↛ 452line 451 didn't jump to line 452 because the condition on line 451 was never true

452 phpunit_testsuite = args.phpunit_testsuite 

453 elif is_extension: 

454 phpunit_testsuite = 'extensions' 

455 elif is_skin: 

456 phpunit_testsuite = 'skins' 

457 

458 if 'phpunit-parallel' in stages: 

459 # We only support parallel for default and extensions suites. 

460 if phpunit_testsuite not in (None, 'extensions'): 460 ↛ 461line 460 didn't jump to line 461 because the condition on line 460 was never true

461 log.warning( 

462 'phpunit-parallel in stages, but only currently supported ' 

463 'for default and extensions test suites - reverting to ' 

464 'serial run' 

465 ) 

466 stages.remove('phpunit-parallel') 

467 

468 # MediaWiki has concurrency issues with SQLite. 

469 # https://phabricator.wikimedia.org/T407954#11690025 

470 if args.db == 'sqlite': 

471 log.warning( 

472 'phpunit-parallel not supported with sqlite (T407954)' 

473 ' - reverting to serial run' 

474 ) 

475 stages.remove('phpunit-parallel') 

476 

477 if 'phpunit-parallel' in stages: 

478 plan.append( 

479 quibble.commands.PhpUnitPrepareParallelRunComposer( 

480 mw_install_path, 

481 phpunit_testsuite, 

482 log_dir, 

483 args.phpunit_junit, 

484 ) 

485 ) 

486 plan.append( 

487 quibble.commands.PhpUnitDatabaselessParallelComposer( 

488 mw_install_path, 

489 phpunit_testsuite, 

490 log_dir, 

491 args.phpunit_junit, 

492 ) 

493 ) 

494 

495 if 'phpunit' in stages and 'phpunit-parallel' not in stages: 

496 plan.append( 

497 quibble.commands.PhpUnitDatabaseless( 

498 mw_install_path, 

499 phpunit_testsuite, 

500 log_dir, 

501 args.phpunit_junit, 

502 ) 

503 ) 

504 

505 if 'phpunit-standalone' in stages and (is_extension or is_skin): 

506 plan.append( 

507 quibble.commands.PhpUnitStandalone( 

508 mw_install_path, 

509 None, 

510 log_dir, 

511 repo_path, 

512 args.phpunit_junit, 

513 ) 

514 ) 

515 

516 if 'phpbench' in stages: 

517 project_dir = mw_install_path 

518 if is_extension or is_skin: 

519 project_dir = os.path.join(mw_install_path, repo_path) 

520 plan.append( 

521 quibble.commands.Phpbench( 

522 project_dir, 

523 composer_install=is_extension or is_skin, 

524 aggregate=args.phpbench_aggregate, 

525 ) 

526 ) 

527 

528 if is_core: 

529 parallel_steps = [] 

530 label = [] 

531 if run_composer: 

532 label.append("'composer test'") 

533 parallel_steps.append( 

534 quibble.commands.CoreComposerTest(mw_install_path) 

535 ) 

536 if run_npm: 

537 label.append("'npm test'") 

538 parallel_steps.append( 

539 quibble.commands.NpmTest(mw_install_path) 

540 ) 

541 if parallel_steps: 

542 plan.append( 

543 quibble.commands.Parallel( 

544 name=" and ".join(label), steps=parallel_steps 

545 ) 

546 ) 

547 

548 if ( 548 ↛ 566line 548 didn't jump to line 566 because the condition on line 548 was always true

549 set(['qunit', 'selenium', 'api-testing']) & set(stages) 

550 or args.commands 

551 ): 

552 backends = [web_backend] 

553 

554 display = os.environ.get('DISPLAY', None) 

555 

556 if not display: 

557 display = ':94' 

558 backends.append(quibble.backend.Xvfb(display)) 

559 

560 backends.append(quibble.backend.ChromeWebDriver(display)) 

561 

562 plan.append( 

563 quibble.commands.StartBackends(self._context_stack, backends) 

564 ) 

565 

566 if 'qunit' in stages: 

567 plan.append( 

568 quibble.commands.QunitTests(mw_install_path, web_backend.url) 

569 ) 

570 

571 if 'selenium' in stages: 

572 selenium_steps = [] 

573 if args.parallel_npm_install: 573 ↛ 574line 573 didn't jump to line 574 because the condition on line 573 was never true

574 parallel_steps = [] 

575 """ 

576 Parallelize the execution of npm install for all browser tests. 

577 Ideally, we'd parallelize the Selenium test execution too, but 

578 since that is going to require quite a bit more work (T226869), 

579 let's start with the part that can be done now. 

580 Note that his has the potential to increase the build time for 

581 projects where a test failure occurs early on (i.e. in core, or 

582 in AbuseFilter), because the tests don't run until npm install 

583 completes for all projects that have browser tests. But for the 

584 common scenario where the tests pass for all repos, this should 

585 result in reducing build time. 

586 """ 

587 for project in dependencies_with_project_first: 

588 parallel_steps.append( 

589 quibble.commands.NpmInstall( 

590 mw_install_path=mw_install_path, 

591 project=project, 

592 with_package_command='selenium-test', 

593 ) 

594 ) 

595 

596 selenium_steps.append( 

597 quibble.commands.Parallel( 

598 name="Parallel npm install for projects with " 

599 "'selenium-test' in package.json", 

600 steps=parallel_steps, 

601 ) 

602 ) 

603 

604 selenium_steps.append( 

605 quibble.commands.BrowserTests( 

606 mw_install_path, 

607 dependencies_with_project_first, 

608 display, 

609 web_backend.url, 

610 args.web_backend, 

611 args.parallel_npm_install, 

612 ) 

613 ) 

614 plan.extend(selenium_steps) 

615 

616 if 'api-testing' in stages: 

617 plan.append( 

618 quibble.commands.ApiTesting( 

619 mw_install_path, 

620 dependencies_with_project_first, 

621 web_backend.url, 

622 args.web_backend, 

623 ) 

624 ) 

625 

626 if 'phpunit' in stages and 'phpunit-parallel' not in stages: 

627 plan.append( 

628 quibble.commands.PhpUnitDatabase( 

629 mw_install_path, 

630 phpunit_testsuite, 

631 log_dir, 

632 args.phpunit_junit, 

633 ) 

634 ) 

635 

636 if 'phpunit-parallel' in stages: 

637 plan.append( 

638 quibble.commands.PhpUnitDatabaseParallelComposer( 

639 mw_install_path, 

640 phpunit_testsuite, 

641 log_dir, 

642 args.phpunit_junit, 

643 ) 

644 ) 

645 

646 if args.commands: 

647 plan.append( 

648 quibble.commands.UserScripts( 

649 mw_install_path, 

650 args.commands, 

651 web_backend.url, 

652 args.web_backend, 

653 ) 

654 ) 

655 

656 if 'phpunit-parallel' in stages: 

657 plan.append(quibble.commands.PhpUnitParallelNotice()) 

658 

659 if success_cache is not None: 

660 plan.append(success_cache.save_command()) 

661 

662 return project_dir, plan 

663 

664 def execute(self, plan, project_dir, reporting_url=None, dry_run=False): 

665 log.debug("Project dir: %s", project_dir) 

666 log.debug("Reporting URL: %s", reporting_url or "not specified") 

667 log.debug("Execution plan:") 

668 for cmd in plan: 

669 log.debug(cmd) 

670 if dry_run: 

671 log.warning("Exiting without execution: --dry-run") 

672 return 

673 

674 with self._context_stack: 

675 for command in plan: 

676 try: 

677 quibble.commands.execute_command(command) 

678 except quibble.commands.SuccessCache.Hit as success_cache_hit: 

679 raise success_cache_hit 

680 except subprocess.CalledProcessError as called_process_error: 

681 # Report exception to a remote service if configured 

682 self.earlywarn( 

683 called_process_error, 

684 command, 

685 project_dir, 

686 reporting_url, 

687 ) 

688 raise called_process_error 

689 

690 def earlywarn( 

691 self, called_process_error, command, project_dir, reporting_url=None 

692 ): 

693 """ 

694 If a command failed, check to see if the repository is configured so 

695 that Quibble should transmit error data to an endpoint for further 

696 processing. 

697 """ 

698 if not quibble.commands.repo_has_quibble_config(project_dir): 698 ↛ 701line 698 didn't jump to line 701 because the condition on line 698 was always true

699 log.debug('No quibble.yaml in %s', project_dir) 

700 return 

701 if not reporting_url: 

702 log.debug('No reporting URL specified.') 

703 return 

704 config = quibble.commands.repo_load_quibble_config(project_dir) 

705 if not config.get('earlywarning'): 

706 log.debug('No earlywarning section found in quibble.yaml') 

707 return 

708 

709 called_process_error_cmd = called_process_error.cmd 

710 if isinstance(called_process_error_cmd, list): 

711 called_process_error_cmd = " ".join(called_process_error_cmd) 

712 quibble.commands.transmit_error( 

713 should_comment=config.get('earlywarning').get('should_comment', 0), 

714 should_vote=config.get('earlywarning').get('should_vote', 0), 

715 phase=str(command), 

716 command=called_process_error_cmd, 

717 reporting_url=reporting_url, 

718 api_key=os.getenv("QUIBBLE_API_KEY"), 

719 called_process_error=called_process_error, 

720 ) 

721 

722 

723def _parse_arguments(args): 

724 args = get_arg_parser().parse_args(args) 

725 if args.shell: 

726 args.commands = args.shell 

727 

728 return args 

729 

730 

731def get_arg_parser(): 

732 """ 

733 Parse arguments 

734 """ 

735 parser = argparse.ArgumentParser( 

736 description='Quibble: the MediaWiki test runner', 

737 prog='quibble', 

738 add_help=False, # added back in the global_opts below 

739 ) 

740 

741 global_opts = parser.add_argument_group('Global options') 

742 global_opts.add_argument( 

743 '-h', 

744 '--help', 

745 action='help', 

746 ) 

747 global_opts.add_argument( 

748 '--color', 

749 dest='color', 

750 action='store_true', 

751 help=( 

752 'Enable colorful output ' 

753 '(or set the FORCE_COLOR environment variable)' 

754 ), 

755 ) 

756 global_opts.add_argument( 

757 '--no-color', 

758 dest='color', 

759 action='store_false', 

760 help='Disable colorful output.', 

761 ) 

762 global_opts.set_defaults( 

763 color=sys.stdin.isatty() or os.getenv('FORCE_COLOR') 

764 ) 

765 

766 global_opts.add_argument( 

767 '-n', 

768 '--dry-run', 

769 action='store_true', 

770 help='Stop before executing any commands.', 

771 ) 

772 global_opts.add_argument( 

773 '--workspace', 

774 default='/workspace' if quibble.is_in_docker() else os.getcwd(), 

775 help='Base path to work from. In Docker: "/workspace", ' 

776 'else current working directory', 

777 ) 

778 global_opts.add_argument( 

779 '--log-dir', 

780 default='log', 

781 help='Where logs and artifacts will be written to. ' 

782 'Default: "log" relatively to workspace', 

783 ) 

784 global_opts.add_argument( 

785 '--reporting-url', 

786 default=None, 

787 help='HTTP endpoint that Quibble will POST error ' 

788 'messages to, for configured repositories.', 

789 ) 

790 global_opts.add_argument( 

791 '--memcached-server', 

792 default=None, 

793 help='Memcached server to use for caching successful results', 

794 ) 

795 global_opts.add_argument( 

796 '--success-cache-key-data', 

797 action='append', 

798 default=[], 

799 help='Data to use when computing a success cache key. Note that the ' 

800 'cache is enabled only when at least one item is given.', 

801 ) 

802 

803 git_ops = parser.add_argument_group('Git operations') 

804 git_ops.add_argument( 

805 '--skip-zuul', 

806 action='store_true', 

807 help='Do not clone/checkout in workspace', 

808 ) 

809 git_ops.add_argument( 

810 '--change', 

811 help=( 

812 'Gerrit Change[,patchset] to act on. Will fetch the change ' 

813 'latest patchset (or the specified one). Overrides ZUUL ' 

814 'environment variables' 

815 ), 

816 ) 

817 git_ops.add_argument( 

818 '--git-cache', 

819 default='/srv/git' if quibble.is_in_docker() else 'ref', 

820 help='Path to bare git repositories to speed up git clone' 

821 'operation. Passed to zuul-cloner as --cache-dir. ' 

822 'In Docker: "/srv/git", else "ref"', 

823 ) 

824 git_ops.add_argument( 

825 '--git-parallel', 

826 default=4, 

827 type=int, 

828 help='Number of workers to clone repositories. Default: 4', 

829 ) 

830 git_ops.add_argument( 

831 '--branch', 

832 default=None, 

833 help=( 

834 'Branch to checkout instead of Zuul selected branch, ' 

835 'for example to specify an alternate branch to test ' 

836 'client library compatibility.' 

837 ), 

838 ) 

839 git_ops.add_argument( 

840 '--project-branch', 

841 nargs=1, 

842 action='append', 

843 default=[], 

844 metavar='PROJECT=BRANCH', 

845 help=( 

846 'project-specific branch to checkout which takes precedence ' 

847 'over --branch if it is provided; may be specified multiple ' 

848 'times.' 

849 ), 

850 ) 

851 

852 deps = parser.add_argument_group('Libraries dependencies') 

853 deps.add_argument( 

854 '--skip-deps', 

855 action='store_true', 

856 help='Do not run composer/npm installs', 

857 ) 

858 deps.add_argument( 

859 '--skip-npm-install', 

860 action='store_true', 

861 help='Do not run the standalone "npm install" dependency step ' 

862 '(the step added automatically for --command/--commands, ' 

863 'npm-test, qunit and api-testing). Intended for jobs whose ' 

864 'commands do not need Node.js dependencies, for example ' 

865 'PHP-only PHPUnit coverage runs. Unlike --skip-deps, composer ' 

866 'dependencies are still installed.', 

867 ) 

868 deps.add_argument( 

869 '--packages-source', 

870 choices=['composer', 'vendor'], 

871 default='vendor', 

872 help='Source to install PHP dependencies from. Default: vendor', 

873 ) 

874 deps.add_argument( 

875 '--parallel-npm-install', 

876 action='store_true', 

877 help='Whether to run "npm install" in parallel for all projects at ' 

878 'the beginning of the BrowserTest stage.', 

879 ) 

880 

881 install = parser.add_argument_group('MediaWiki install') 

882 install.add_argument( 

883 '--skip-install', action='store_true', help='Do not install MediaWiki' 

884 ) 

885 install.add_argument( 

886 '--db', 

887 choices=['sqlite', 'mysql', 'postgres'], 

888 default='mysql', 

889 help='Database backend to use. Default: mysql', 

890 ) 

891 install.add_argument( 

892 '--db-dir', 

893 default=None, 

894 help=( 

895 'Base directory holding database files. A sub directory ' 

896 'prefixed with "quibble-" will be created and deleted ' 

897 'on completion. ' 

898 'If set and relative, relatively to workspace. ' 

899 'Default: %s' % tempfile.gettempdir() 

900 ), 

901 ) 

902 install.add_argument( 

903 '--db-is-external', 

904 action='store_true', 

905 help='If the database is managed externally and not by Quibble. ' 

906 'Default: managed by Quibble', 

907 ) 

908 install.add_argument( 

909 '--dump-db-postrun', 

910 action='store_true', 

911 help='Dump the db before shutting down the server (mysql only)', 

912 ) 

913 

914 ext_requires = parser.add_argument_group('MediaWiki extension requires') 

915 ext_requires.add_argument( 

916 '--resolve-requires', 

917 action='store_true', 

918 help='Whether to process extension.json/skin.json and clone extra ' 

919 'extensions/skins mentioned in the "requires" statement. ' 

920 'This is done recursively.', 

921 ) 

922 ext_requires.add_argument( 

923 '--fail-on-extra-requires', 

924 action='store_true', 

925 help='When --resolve-requires caused Quibble to clone extra ' 

926 'requirements not in the list of projects: fail.' 

927 'Can be used to enforce extensions and skins to declare ' 

928 'their requirements via the extension registry.', 

929 ) 

930 

931 tests = parser.add_argument_group('Stages options') 

932 tests.add_argument( 

933 '--phpbench-aggregate', 

934 action='store_true', 

935 help='If this argument is set, then Quibble will run phpbench in ' 

936 'aggregate mode, comparing the previous commit with the ' 

937 'current one.',