Coverage for quibble/zuul.py: 92%

1# Copyright 2018 Antoine "hashar" Musso 

2# Copyright 2018 Wikimedia Foundation Inc. 

3# 

4# Licensed under the Apache License, Version 2.0 (the "License"); 

5# you may not use this file except in compliance with the License. 

6# You may obtain a copy of the License at 

7# 

8# http://www.apache.org/licenses/LICENSE-2.0 

9# 

10# Unless required by applicable law or agreed to in writing, software 

11# distributed under the License is distributed on an "AS IS" BASIS, 

12# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 

13# See the License for the specific language governing permissions and 

14# limitations under the License. 

15 

16import copy 

17import logging 

18import os 

19import threading 

20 

21from concurrent.futures import ThreadPoolExecutor, as_completed 

22 

23from zuul.lib.cloner import Cloner 

24from zuul.lib.clonemapper import CloneMapper 

25 

26CLONE_MAP = [ 

27 {'name': 'mediawiki/core', 'dest': '.'}, 

28 {'name': 'mediawiki/vendor', 'dest': './vendor'}, 

29 {'name': 'mediawiki/extensions/(.*)', 'dest': './extensions/\\1'}, 

30 {'name': 'mediawiki/skins/(.*)', 'dest': './skins/\\1'}, 

31 {'name': 'mediawiki/services/(.*)', 'dest': './services/\\1'}, 

32] 

33 

34 

35def clone( 

36 branch, 

37 cache_dir, 

38 project_branch, 

39 projects, 

40 workers, 

41 workspace, 

42 zuul_branch, 

43 zuul_newrev, 

44 zuul_project, 

45 zuul_ref, 

46 zuul_url, 

47): 

48 log = logging.getLogger('quibble.zuul.clone') 

49 

50 if isinstance(projects, str): 

51 projects = [projects] 

52 

53 if zuul_ref is not None and zuul_url is None: 

54 raise Exception('Zuul ref requires a Zuul url') 

55 if zuul_newrev is not None and zuul_project is None: 

56 raise Exception('Zuul newrev requires a Zuul project') 

57 

58 project_branches = {} 

59 if project_branch: 

60 for x in project_branch: 

61 p, p_branch = x[0].split('=') 

62 project_branches[p] = p_branch 

63 

64 zuul_cloner = Cloner( 

65 git_base_url='https://gerrit.wikimedia.org/r', 

66 projects=projects, 

67 workspace=workspace, 

68 zuul_branch=zuul_branch, 

69 zuul_ref=zuul_ref, 

70 zuul_url=zuul_url, 

71 branch=branch, 

72 project_branches=project_branches, 

73 cache_dir=cache_dir, 

74 zuul_newrev=zuul_newrev, 

75 zuul_project=zuul_project, 

76 cache_no_hardlinks=False, # False allows hardlink 

77 ) 

78 # The constructor expects a file, set the value directly 

79 zuul_cloner.clone_map = CLONE_MAP 

80 

81 # Reimplement Cloner.execute() to make sure mediawiki/core is cloned first 

82 # and clone the rest in parallel. 

83 dests = working_trees(workspace, projects) 

84 

85 if workers == 1: 

86 return zuul_cloner.execute() 

87 

88 # Reimplement the cloner execute method with parallelism and logging 

89 # suitable for multiplexed output. 

90 log.info("Preparing %d repositories with %s workers", len(dests), workers) 

91 

92 if 'mediawiki/core' in projects: 

93 mw_git_dir = os.path.join(dests['mediawiki/core'], '.git') 

94 if not os.path.exists(mw_git_dir): 94 ↛ 99line 94 didn't jump to line 99 because the condition on line 94 was always true

95 log.info("Cloning mediawiki/core first") 

96 zuul_cloner.prepareRepo('mediawiki/core', dests['mediawiki/core']) 

97 del dests['mediawiki/core'] 

98 

99 can_run = threading.Event() 

100 can_run.set() 

101 

102 with ThreadPoolExecutor(max_workers=workers) as executor: 

103 futures = [ 

104 executor.submit(_clone_worker, can_run, zuul_cloner, project, dest) 

105 for project, dest in dests.items() 

106 ] 

107 # Consume results 

108 for future in as_completed(futures): 

109 future.result() 

110 

111 log.info("Prepared all repositories") 

112 

113 

114def _clone_worker(can_run, cloner, project, dest): 

115 if not can_run.is_set(): 115 ↛ 116line 115 didn't jump to line 116 because the condition on line 115 was never true

116 return 

117 

118 # Forge a new child logger, since repositories might be cloned concurrently 

119 project_cloner = copy.copy(cloner) 

120 project_cloner.log = project_cloner.log.getChild(project) 

121 try: 

122 project_cloner.prepareRepo(project, dest) 

123 except Exception as e: 

124 # Prevent other workers from executing 

125 can_run.clear() 

126 raise e 

127 

128 

129def repo_dir(repo): 

130 mapper = CloneMapper(CLONE_MAP, [repo]) 

131 return mapper.expand(workspace='./')[repo] 

132 

133 

134def working_trees(workspace, projects): 

135 mapper = CloneMapper(CLONE_MAP, projects) 

136 return mapper.expand(workspace=workspace)