18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
|
# File 'modules/profile/manifests/httpbb.pp', line 18
class profile::httpbb (
Optional[Hash[String, Hash[String, String]]] $plain_basicauth_credentials = lookup('profile::httpbb::basicauth_credentials', {default_value => undef}),
Hash[String, Array[String]] $hourly_tests = lookup('profile::httpbb::hourly_tests', {default_value => {}}),
Boolean $test_kubernetes_hourly = lookup('profile::httpbb::test_kubernetes_hourly', {default_value => false}),
){
class {'::httpbb':}
# Walk over the credentials hash and turn "user: password" into "user: base64(...)"
# leaving the structure intact.
if $plain_basicauth_credentials {
$basicauth_credentials = $plain_basicauth_credentials.map |$k, $v| {
{
$k=> $v.map |$user, $password| {
{$user => "Basic ${base64('encode', "${user}:${password}", 'strict') }"}
}.reduce({}) |$m, $v| {
$m.merge($v)
}
}
}.reduce({}) |$mem, $val| {
$mem.merge($val)
}
} else {
$basicauth_credentials = undef
}
file {
[
'/srv/deployment/httpbb-tests/appserver',
'/srv/deployment/httpbb-tests/miscweb',
'/srv/deployment/httpbb-tests/people',
'/srv/deployment/httpbb-tests/releases',
'/srv/deployment/httpbb-tests/gitlab',
'/srv/deployment/httpbb-tests/vrts',
'/srv/deployment/httpbb-tests/contint',
'/srv/deployment/httpbb-tests/etherpad',
'/srv/deployment/httpbb-tests/rt',
'/srv/deployment/httpbb-tests/gerrit',
'/srv/deployment/httpbb-tests/planet',
'/srv/deployment/httpbb-tests/noc',
'/srv/deployment/httpbb-tests/doc',
'/srv/deployment/httpbb-tests/parse',
'/srv/deployment/httpbb-tests/thumbor',
'/srv/deployment/httpbb-tests/docker-registry',
'/srv/deployment/httpbb-tests/query_service',
'/srv/deployment/httpbb-tests/jobrunner',
'/srv/deployment/httpbb-tests/phabricator',
'/srv/deployment/httpbb-tests/liftwing',
'/srv/deployment/httpbb-tests/ores_legacy',
]:
ensure => directory,
purge => true
}
httpbb::test_suite {'appserver/test_foundation.yaml':
source => 'puppet:///modules/profile/httpbb/appserver/test_foundation.yaml'
}
httpbb::test_suite {'appserver/test_main.yaml':
source => 'puppet:///modules/profile/httpbb/appserver/test_main.yaml'
}
httpbb::test_suite {'appserver/test_redirects.yaml':
source => 'puppet:///modules/profile/httpbb/appserver/test_redirects.yaml'
}
httpbb::test_suite {'appserver/test_remnant.yaml':
source => 'puppet:///modules/profile/httpbb/appserver/test_remnant.yaml'
}
httpbb::test_suite {'appserver/test_secure.yaml':
source => 'puppet:///modules/profile/httpbb/appserver/test_secure.yaml'
}
httpbb::test_suite {'appserver/test_wikimania_wikimedia.yaml':
source => 'puppet:///modules/profile/httpbb/appserver/test_wikimania_wikimedia.yaml'
}
httpbb::test_suite {'appserver/test_wwwportals.yaml':
source => 'puppet:///modules/profile/httpbb/appserver/test_wwwportals.yaml'
}
httpbb::test_suite {'miscweb/test_miscweb.yaml':
source => 'puppet:///modules/profile/httpbb/miscweb/test_miscweb.yaml'
}
httpbb::test_suite {'miscweb/test_miscweb-k8s.yaml':
source => 'puppet:///modules/profile/httpbb/miscweb-k8s/test_miscweb-k8s.yaml'
}
httpbb::test_suite {'people/test_people.yaml':
source => 'puppet:///modules/profile/httpbb/people/test_people.yaml'
}
httpbb::test_suite {'releases/test_releases.yaml':
source => 'puppet:///modules/profile/httpbb/releases/test_releases.yaml'
}
httpbb::test_suite {'gitlab/test_gitlab.yaml':
source => 'puppet:///modules/profile/httpbb/gitlab/test_gitlab.yaml'
}
httpbb::test_suite {'vrts/test_vrts.yaml':
source => 'puppet:///modules/profile/httpbb/vrts/test_vrts.yaml'
}
httpbb::test_suite {'contint/test_integration.yaml':
source => 'puppet:///modules/profile/httpbb/contint/test_integration.yaml'
}
httpbb::test_suite {'etherpad/test_etherpad.yaml':
source => 'puppet:///modules/profile/httpbb/etherpad/test_etherpad.yaml'
}
httpbb::test_suite {'rt/test_rt.yaml':
source => 'puppet:///modules/profile/httpbb/rt/test_rt.yaml'
}
httpbb::test_suite {'planet/test_planet.yaml':
source => 'puppet:///modules/profile/httpbb/planet/test_planet.yaml'
}
httpbb::test_suite {'gerrit/test_gerrit.yaml':
source => 'puppet:///modules/profile/httpbb/gerrit/test_gerrit.yaml'
}
httpbb::test_suite {'gerrit/test_gerrit_replica.yaml':
source => 'puppet:///modules/profile/httpbb/gerrit/test_gerrit_replica.yaml'
}
httpbb::test_suite {'phabricator/test_phabricator.yaml':
source => 'puppet:///modules/profile/httpbb/phabricator/test_phabricator.yaml'
}
httpbb::test_suite {'noc/test_noc.yaml':
source => 'puppet:///modules/profile/httpbb/noc/test_noc.yaml'
}
httpbb::test_suite {'doc/test_doc.yaml':
source => 'puppet:///modules/profile/httpbb/doc/test_doc.yaml'
}
httpbb::test_suite {'parse/test_parse.yaml':
source => 'puppet:///modules/profile/httpbb/parse/test_parse.yaml'
}
httpbb::test_suite {'thumbor/test_thumbor.yaml':
source => 'puppet:///modules/profile/httpbb/thumbor/test_thumbor.yaml'
}
httpbb::test_suite {'query_service/test_wdqs.yaml':
source => 'puppet:///modules/profile/httpbb/query_service/test_wdqs.yaml'
}
httpbb::test_suite {'jobrunner/test_endpoint.yaml':
source => 'puppet:///modules/profile/httpbb/jobrunner/test_endpoint.yaml'
}
httpbb::test_suite {'liftwing/test_liftwing_production.yaml':
source => 'puppet:///modules/profile/httpbb/liftwing/test_liftwing_production.yaml'
}
httpbb::test_suite {'liftwing/test_liftwing_staging.yaml':
source => 'puppet:///modules/profile/httpbb/liftwing/test_liftwing_staging.yaml'
}
httpbb::test_suite {'ores_legacy/test_ores_staging.yaml':
source => 'puppet:///modules/profile/httpbb/ores_legacy/test_ores_staging.yaml'
}
httpbb::test_suite {'ores_legacy/test_ores_production.yaml':
source => 'puppet:///modules/profile/httpbb/ores_legacy/test_ores_production.yaml'
}
if $basicauth_credentials and $basicauth_credentials['docker-registry'] {
httpbb::test_suite {'docker-registry/test_docker-registry.yaml':
content => template('profile/httpbb/docker-registry/test_docker-registry.yaml.erb'),
mode => '0400',
}
}
$hourly_tests.each |String $test_dir, Array[String] $hosts| {
$joined_hosts = join($hosts, ',')
$ensure = $hosts ? {
[] => absent,
default => present
}
systemd::timer::job { "httpbb_hourly_${test_dir}":
ensure => $ensure,
description => "Run httpbb ${test_dir}/ tests hourly on ${joined_hosts}",
command => "/bin/sh -c '/usr/bin/httpbb /srv/deployment/httpbb-tests/${test_dir}/*.yaml --hosts ${joined_hosts} --retry_on_timeout'",
interval => {
'start' => 'OnUnitActiveSec',
'interval' => '1 hour',
},
# This doesn't really need access to anything in www-data, but it definitely doesn't need root.
user => 'www-data',
monitoring_enabled => true,
team => 'ServiceOps',
}
}
# Add the hourly Kubernetes test separately, since it needs a different --https_port.
$ensure = $test_kubernetes_hourly.bool2str('present', 'absent')
$kubernetes_services = wmflib::service::fetch().filter |$name, $config| {
$config.has_key('httpbb_dir')
}
$kubernetes_services.each |String $svc_name, Hash $svc| {
$svc_port = $svc['port']
$svc_httpbb_dir = $svc['httpbb_dir']
systemd::timer::job { "httpbb_kubernetes_${svc_name}_hourly":
ensure => $ensure,
description => "Run httpbb ${svc_httpbb_dir} tests hourly on Kubernetes ${svc_name}.",
command => "/bin/sh -c \'/usr/bin/httpbb /srv/deployment/httpbb-tests/${svc_httpbb_dir}/*.yaml --host ${svc_name}.discovery.wmnet --https_port ${svc_port} --retry_on_timeout\'",
interval => {
'start' => 'OnUnitActiveSec',
'interval' => '1 hour',
},
user => 'www-data',
monitoring_enabled => true,
team => 'ServiceOps',
}
}
}
|