Puppet Class: profile::prometheus::cloud

Defined in:
modules/profile/manifests/prometheus/cloud.pp

Overview

SPDX-License-Identifier: Apache-2.0

Parameters:

  • openstack_deployment (String) (defaults to: lookup('profile::prometheus::cloud::openstack_deployment'))
  • alertmanagers (Array) (defaults to: lookup('alertmanagers', {'default_value' => []}))
  • replica_label (String) (defaults to: lookup('prometheus::replica_label'))
  • maintain_dbusers_primary (String) (defaults to: lookup('wmcs_maintain_dbusers_primary')) 


2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
 
# File 'modules/profile/manifests/prometheus/cloud.pp', line 2

class profile::prometheus::cloud (
    String $openstack_deployment = lookup('profile::prometheus::cloud::openstack_deployment'),
    Array $alertmanagers = lookup('alertmanagers', {'default_value' => []}),
    String $replica_label = lookup('prometheus::replica_label'),
    String $maintain_dbusers_primary = lookup('wmcs_maintain_dbusers_primary'),
) {
    $instance = 'cloud'
    $config = prometheus::instance_config($instance)
    $targets_path = $config['targets_path']
    $port = $config['port']
    $storage_retention = $config['retention_time']
    $storage_retention_size = $config['retention_size']
    $thanos_upload = $config['thanos_upload']

    $config_extra = {
        'external_labels' => {
            'site'       => $::site,
            'replica'    => $replica_label,
            'prometheus' => $instance,
        },
    }

    $blackbox_jobs = [
        {
            'job_name'        => 'blackbox_http',
            'metrics_path'    => '/probe',
            'params'          => {
                'module' => [ 'http_200_300_connect' ],
            },
            'file_sd_configs' => [
                { 'files' => [ "${targets_path}/blackbox_http_*.yaml" ] }
            ],
            'relabel_configs' => [
                { 'source_labels' => ['__address__'],
                    'target_label'  => '__param_target',
                    'replacement' => 'http://$1/',
                },
                { 'source_labels' => ['__param_target'],
                    'target_label'  => 'instance',
                },
                { 'target_label' => '__address__',
                    'replacement'  => '127.0.0.1:9115',
                },
            ],
        },
        {
            'job_name'        => 'blackbox_https',
            'metrics_path'    => '/probe',
            'params'          => {
                'module' => [ 'https_200_300_connect' ],
            },
            'file_sd_configs' => [
                { 'files' => [ "${targets_path}/blackbox_https_*.yaml" ] }
            ],
            'relabel_configs' => [
                { 'source_labels' => ['__address__'],
                    'target_label'  => '__param_target',
                    'replacement' => 'https://$1/',
                },
                { 'source_labels' => ['__param_target'],
                    'target_label'  => 'instance',
                },
                { 'target_label' => '__address__',
                    'replacement'  => '127.0.0.1:9115',
                },
            ],
        },
    ]

    $rabbitmq_jobs = [
        {
            'job_name'        => 'rabbitmq',
            'scheme'          => 'http',
            'file_sd_configs' => [
                { 'files' => [ "${targets_path}/rabbitmq_*.yaml" ] }
            ],
        },
    ]

    $pdns_jobs = [
        {
            'job_name'        => 'pdns',
            'scheme'          => 'http',
            'file_sd_configs' => [
                { 'files' => [ "${targets_path}/pdns_*.yaml" ] }
            ],
        },
    ]

    $pdns_rec_jobs = [
        {
            'job_name'        => 'pdns_rec',
            'scheme'          => 'http',
            'file_sd_configs' => [
                { 'files' => [ "${targets_path}/pdns-rec_*.yaml" ] }
            ],
        },
    ]

    $hostname_to_instance_config = {
        'source_labels' => ['hostname', 'instance'],
        'separator'     => ';',
        # This matches either the hostname if it's there, or the instance if it's not.
        # It uses the separator as marker
        'regex'         => '^([^;:]+);.*|^;(.*)',
        'target_label'  => 'instance',
        'replacement'   => '$1',
    }

    $openstack_jobs = [
        {
            'job_name'        => 'openstack',
            'scheme'          => 'http',
            'file_sd_configs' => [
                { 'files' => [ "${targets_path}/openstack_*.yaml" ] }
            ],
            'metric_relabel_configs' => [
                $hostname_to_instance_config,
            ],
            # this number is controversial and may have a high impact on the APIs
            # see T335943
            'scrape_interval' => '4m',
            'scrape_timeout'  => '120s',
        },
    ]

    $ceph_jobs = [
        {
            'job_name'        => "ceph_${::site}",
            'scheme'          => 'http',
            'file_sd_configs' => [
                { 'files' => [ "${targets_path}/ceph_${::site}.yaml" ]}
            ],
            'metric_relabel_configs' => [
                $hostname_to_instance_config,
            ],
            'scrape_timeout'  => '30s',
        },
    ]

    $maintain_dbusers_jobs = [
        {
            'job_name'        => "maintain_dbusers_${::site}",
            'scheme'          => 'http',
            'file_sd_configs' => [
                { 'files' => [ "${targets_path}/maintain_dbusers_*.yaml" ]}
            ],
            'metric_relabel_configs' => [
                $hostname_to_instance_config,
            ],
        },
    ]

    if $maintain_dbusers_primary =~ $::site {
        file { "${targets_path}/maintain_dbusers_${openstack_deployment}.yaml":
            content => to_yaml([{
                'labels'  => {
                    'deployment' => $openstack_deployment,
                },
                'targets' => [
                    "${maintain_dbusers_primary}:9090",
                ],
            }]),
        }
    }

    # https://phabricator.wikimedia.org/T348643#9916509
    file { "${targets_path}/ebpf_exporter_osds.yaml":
        ensure => absent,
    }

    file { "${targets_path}/blackbox_http_keystone.yaml":
        content => to_yaml([{
            'labels'  => {
                'deployment' => $openstack_deployment,
            },
            'targets' => [
                "openstack.${openstack_deployment}.wikimediacloud.org:5000/v3", # keystone
            ],
        }]),
    }

    file { "${targets_path}/blackbox_https_keystone.yaml":
        content => to_yaml([{
            'labels'  => {
                'deployment' => $openstack_deployment,
            },
            'targets' => [
                "openstack.${openstack_deployment}.wikimediacloud.org:25000/v3", # keystone
                "openstack.${openstack_deployment}.wikimediacloud.org:28774", # nova
                "openstack.${openstack_deployment}.wikimediacloud.org:28776", # cinder
                "openstack.${openstack_deployment}.wikimediacloud.org:28778", # placement
                "openstack.${openstack_deployment}.wikimediacloud.org:28779", # trove
                "openstack.${openstack_deployment}.wikimediacloud.org:29001", # designate
                "openstack.${openstack_deployment}.wikimediacloud.org:29292", # glance
                "openstack.${openstack_deployment}.wikimediacloud.org:29696", # neutron
            ],
        }]),
    }

    prometheus::class_config{ "rabbitmq_${::site}":
        dest       => "${targets_path}/rabbitmq_${::site}.yaml",
        class_name => "profile::openstack::${openstack_deployment}::rabbitmq",
        labels     => {'deployment' => $openstack_deployment},
        port       => 15692,
    }

    prometheus::class_config{ "pdns_${::site}":
        dest       => "${targets_path}/pdns_${::site}.yaml",
        class_name => "role::wmcs::openstack::${openstack_deployment}::services",
        labels     => {'deployment' => $openstack_deployment},
        port       => 8081,
    }

    prometheus::class_config{ "pdns-rec_${::site}":
        dest       => "${targets_path}/pdns-rec_${::site}.yaml",
        class_name => "role::wmcs::openstack::${openstack_deployment}::services",
        labels     => {'deployment' => $openstack_deployment},
        port       => 8082,
    }

    prometheus::class_config { "openstack_${::site}":
        dest             => "${targets_path}/openstack_${::site}.yaml",
        class_name       => 'profile::prometheus::openstack_exporter',
        class_parameters => {'ensure' => 'present', 'cloud' => $openstack_deployment},
        labels           => {'deployment' => $openstack_deployment},
        port             => 12345,
    }

    prometheus::class_config{ "ceph_${::site}":
        dest       => "${targets_path}/ceph_${::site}.yaml",
        class_name => 'role::wmcs::ceph::mon',
        port       => 9283,
    }

    prometheus::class_config { "mysql_galera_${openstack_deployment}":
        dest       => "${targets_path}/mysql_galera_${openstack_deployment}.yaml",
        class_name => "role::wmcs::openstack::${openstack_deployment}::control",
        labels     => {'deployment' => $openstack_deployment},
        port       => 9104,
    }

    $galera_jobs = [
        {
            'job_name'        => 'mysql-galera',
            'file_sd_configs' => [
                { 'files' => [ "${targets_path}/mysql_galera_*.yaml"] },
            ],
        }
    ]

    # Job definition for cloudlb haproxy
    $cloudlb_haproxy_jobs = [
        {
            'job_name'        => 'cloudlb-haproxy',
            'scheme'          => 'http',
            'file_sd_configs' => [
                { 'files' => [ "${targets_path}/cloudlb_haproxy_*.yaml"] },
            ],
        },
    ]

    prometheus::class_config { "cloudlb_haproxy_${::site}":
        dest       => "${targets_path}/cloudlb_haproxy_${::site}.yaml",
        class_name => 'profile::wmcs::cloudlb::haproxy',
        port       => 9900,
    }

    prometheus::server { $instance:
        listen_address                 => "127.0.0.1:${port}",
        storage_retention              => $storage_retention,
        storage_retention_size         => $storage_retention_size,
        alertmanagers                  => $alertmanagers.map |$a| { "${a}:9093" },
        alerting_relabel_configs_extra => [
            # Add 'team' label, https://phabricator.wikimedia.org/T302493#7759642
            { 'target_label' => 'team', 'replacement' => 'wmcs', 'action' => 'replace' },
        ],
        scrape_configs_extra           => [
            $blackbox_jobs, $rabbitmq_jobs, $pdns_jobs,
            $pdns_rec_jobs, $openstack_jobs, $ceph_jobs,
            $galera_jobs, $cloudlb_haproxy_jobs,
            $maintain_dbusers_jobs,
        ].flatten,
        global_config_extra            => $config_extra,
        rule_files_extra               => ['/srv/alerts/cloud/*.yaml'],
    }

    profile::thanos::sidecar { $instance:
        prometheus_port     => $port,
        prometheus_instance => $instance,
        enable_upload       => $thanos_upload,
    }

    # Checks for alerting rules, defined in puppet
    prometheus::alert::import { $instance: }

    prometheus::pint::source { $instance:
        port => $port,
    }
}