Puppet Class: profile::prometheus::cloud

Defined in:
modules/profile/manifests/prometheus/cloud.pp

Overview

SPDX-License-Identifier: Apache-2.0

Parameters:

  • openstack_deployment (String) (defaults to: lookup('profile::prometheus::cloud::openstack_deployment'))
  • storage_retention (String) (defaults to: lookup('profile::prometheus::cloud::storage_retention', {'default_value' => '4032h'}))
  • storage_retention_size (Optional[Stdlib::Datasize]) (defaults to: lookup('profile::prometheus::cloud::storage_retention_size', {default_value => undef}))
  • max_chunks_to_persist (Integer) (defaults to: lookup('prometheus::server::max_chunks_to_persist', {'default_value' => 524288}))
  • memory_chunks (Integer) (defaults to: lookup('prometheus::server::memory_chunks', {'default_value' => 1048576}))
  • alertmanagers (Array) (defaults to: lookup('alertmanagers', {'default_value' => []}))
  • enable_thanos_upload (Boolean) (defaults to: lookup('profile::prometheus::enable_thanos_upload', { 'default_value' => false }))
  • thanos_min_time (Optional[String]) (defaults to: lookup('profile::prometheus::thanos::min_time', { 'default_value' => undef }))
  • disable_compaction (Boolean) (defaults to: lookup('profile::prometheus::thanos::disable_compaction', { 'default_value' => false }))
  • replica_label (String) (defaults to: lookup('prometheus::replica_label')) 


2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
 
# File 'modules/profile/manifests/prometheus/cloud.pp', line 2

class profile::prometheus::cloud (
    String $openstack_deployment = lookup('profile::prometheus::cloud::openstack_deployment'),
    String $storage_retention = lookup('profile::prometheus::cloud::storage_retention', {'default_value' => '4032h'}),
    Optional[Stdlib::Datasize] $storage_retention_size = lookup('profile::prometheus::cloud::storage_retention_size', {default_value => undef}),
    Integer $max_chunks_to_persist = lookup('prometheus::server::max_chunks_to_persist', {'default_value' => 524288}),
    Integer $memory_chunks = lookup('prometheus::server::memory_chunks', {'default_value' => 1048576}),
    Array $alertmanagers = lookup('alertmanagers', {'default_value' => []}),
    Boolean $enable_thanos_upload     = lookup('profile::prometheus::enable_thanos_upload', { 'default_value' => false }),
    Optional[String] $thanos_min_time = lookup('profile::prometheus::thanos::min_time', { 'default_value' => undef }),
    Boolean $disable_compaction = lookup('profile::prometheus::thanos::disable_compaction', { 'default_value' => false }),
    String $replica_label = lookup('prometheus::replica_label'),
) {
    $targets_path = '/srv/prometheus/cloud/targets'

    $config_extra = {
        'external_labels' => {
            'site'       => $::site,
            'replica'    => $replica_label,
            'prometheus' => 'cloud',
        },
    }

    $port = 9904

    $blackbox_jobs = [
        {
            'job_name'        => 'blackbox_http',
            'metrics_path'    => '/probe',
            'params'          => {
                'module' => [ 'http_200_300_connect' ],
            },
            'file_sd_configs' => [
                { 'files' => [ "${targets_path}/blackbox_http_*.yaml" ] }
            ],
            'relabel_configs' => [
                { 'source_labels' => ['__address__'],
                    'target_label'  => '__param_target',
                    'replacement' => 'http://$1/',
                },
                { 'source_labels' => ['__param_target'],
                    'target_label'  => 'instance',
                },
                { 'target_label' => '__address__',
                    'replacement'  => '127.0.0.1:9115',
                },
            ],
        },
        {
            'job_name'        => 'blackbox_https',
            'metrics_path'    => '/probe',
            'params'          => {
                'module' => [ 'https_200_300_connect' ],
            },
            'file_sd_configs' => [
                { 'files' => [ "${targets_path}/blackbox_https_*.yaml" ] }
            ],
            'relabel_configs' => [
                { 'source_labels' => ['__address__'],
                    'target_label'  => '__param_target',
                    'replacement' => 'https://$1/',
                },
                { 'source_labels' => ['__param_target'],
                    'target_label'  => 'instance',
                },
                { 'target_label' => '__address__',
                    'replacement'  => '127.0.0.1:9115',
                },
            ],
        },
    ]

    $rabbitmq_jobs = [
        {
            'job_name'        => 'rabbitmq',
            'scheme'          => 'http',
            'file_sd_configs' => [
                { 'files' => [ "${targets_path}/rabbitmq_*.yaml" ] }
            ],
        },
    ]

    $pdns_jobs = [
        {
            'job_name'        => 'pdns',
            'scheme'          => 'http',
            'file_sd_configs' => [
                { 'files' => [ "${targets_path}/pdns_*.yaml" ] }
            ],
        },
    ]

    $pdns_rec_jobs = [
        {
            'job_name'        => 'pdns_rec',
            'scheme'          => 'http',
            'file_sd_configs' => [
                { 'files' => [ "${targets_path}/pdns-rec_*.yaml" ] }
            ],
        },
    ]

    $hostname_to_instance_config = {
        'source_labels' => ['hostname', 'instance'],
        'separator'     => ';',
        # This matches either the hostname if it's there, or the instance if it's not.
        # It uses the separator as marker
        'regex'         => '^([^;:]+);.*|^;(.*)',
        'target_label'  => 'instance',
        'replacement'   => '$1',
    }

    $openstack_jobs = [
        {
            'job_name'        => 'openstack',
            'scheme'          => 'http',
            'file_sd_configs' => [
                { 'files' => [ "${targets_path}/openstack_*.yaml" ] }
            ],
            'metric_relabel_configs' => [
                $hostname_to_instance_config,
            ],
            # this number is controversial and may have a high impact on the APIs
            # see T335943
            'scrape_interval' => '4m',
            'scrape_timeout'  => '120s',
        },
    ]

    $ceph_jobs = [
        {
            'job_name'        => "ceph_${::site}",
            'scheme'          => 'http',
            'file_sd_configs' => [
                { 'files' => [ "${targets_path}/ceph_${::site}.yaml" ]}
            ],
            'metric_relabel_configs' => [
                $hostname_to_instance_config,
            ],
        },
    ]


    file { "${targets_path}/blackbox_http_keystone.yaml":
        content => to_yaml([{
            'labels'  => {
                'deployment' => $openstack_deployment,
            },
            'targets' => [
                "openstack.${openstack_deployment}.wikimediacloud.org:5000/v3", # keystone
            ],
        }]),
    }

    file { "${targets_path}/blackbox_https_keystone.yaml":
        content => to_yaml([{
            'labels'  => {
                'deployment' => $openstack_deployment,
            },
            'targets' => [
                "openstack.${openstack_deployment}.wikimediacloud.org:25000/v3", # keystone
                "openstack.${openstack_deployment}.wikimediacloud.org:28774", # nova
                "openstack.${openstack_deployment}.wikimediacloud.org:28776", # cinder
                "openstack.${openstack_deployment}.wikimediacloud.org:28778", # placement
                "openstack.${openstack_deployment}.wikimediacloud.org:28779", # trove
                "openstack.${openstack_deployment}.wikimediacloud.org:29001", # designate
                "openstack.${openstack_deployment}.wikimediacloud.org:29292", # glance
                "openstack.${openstack_deployment}.wikimediacloud.org:29696", # neutron
            ],
        }]),
    }

    prometheus::class_config{ "rabbitmq_${::site}":
        dest       => "${targets_path}/rabbitmq_${::site}.yaml",
        class_name => "profile::openstack::${openstack_deployment}::rabbitmq",
        labels     => {'deployment' => $openstack_deployment},
        port       => 15692,
    }

    prometheus::class_config{ "pdns_${::site}":
        dest       => "${targets_path}/pdns_${::site}.yaml",
        class_name => "role::wmcs::openstack::${openstack_deployment}::services",
        labels     => {'deployment' => $openstack_deployment},
        port       => 8081,
    }

    prometheus::class_config{ "pdns-rec_${::site}":
        dest       => "${targets_path}/pdns-rec_${::site}.yaml",
        class_name => "role::wmcs::openstack::${openstack_deployment}::services",
        labels     => {'deployment' => $openstack_deployment},
        port       => 8082,
    }

    prometheus::class_config { "openstack_${::site}":
        dest             => "${targets_path}/openstack_${::site}.yaml",
        class_name       => 'profile::prometheus::openstack_exporter',
        class_parameters => {'ensure' => 'present', 'cloud' => $openstack_deployment},
        labels           => {'deployment' => $openstack_deployment},
        port             => 12345,
    }

    prometheus::class_config{ "ceph_${::site}":
        dest       => "${targets_path}/ceph_${::site}.yaml",
        class_name => 'role::wmcs::ceph::mon',
        port       => 9283,
    }

    prometheus::class_config { "mysql_galera_${openstack_deployment}":
        dest       => "${targets_path}/mysql_galera_${openstack_deployment}.yaml",
        class_name => "role::wmcs::openstack::${openstack_deployment}::control",
        labels     => {'deployment' => $openstack_deployment},
        port       => 9104,
    }

    $galera_jobs = [
        {
            'job_name'        => 'mysql-galera',
            'file_sd_configs' => [
                { 'files' => [ "${targets_path}/mysql_galera_*.yaml"] },
            ],
        }
    ]

    # Job definition for cloudlb haproxy
    $cloudlb_haproxy_jobs = [
        {
            'job_name'        => 'cloudlb-haproxy',
            'scheme'          => 'http',
            'file_sd_configs' => [
                { 'files' => [ "${targets_path}/cloudlb_haproxy_*.yaml"] },
            ],
        },
    ]

    prometheus::class_config { "cloudlb_haproxy_${::site}":
        dest       => "${targets_path}/cloudlb_haproxy_${::site}.yaml",
        class_name => 'profile::wmcs::cloudlb::haproxy',
        port       => 9900,
    }

    $max_block_duration = ($enable_thanos_upload and $disable_compaction) ? {
        true    => '2h',
        default => '24h',
    }

    prometheus::server { 'cloud':
        listen_address                 => "127.0.0.1:${port}",
        storage_retention              => $storage_retention,
        storage_retention_size         => $storage_retention_size,
        max_chunks_to_persist          => $max_chunks_to_persist,
        memory_chunks                  => $memory_chunks,
        min_block_duration             => '2h',
        max_block_duration             => $max_block_duration,
        alertmanagers                  => $alertmanagers.map |$a| { "${a}:9093" },
        alerting_relabel_configs_extra => [
            # Add 'team' label, https://phabricator.wikimedia.org/T302493#7759642
            { 'target_label' => 'team', 'replacement' => 'wmcs', 'action' => 'replace' },
        ],
        scrape_configs_extra           => [
            $blackbox_jobs, $rabbitmq_jobs, $pdns_jobs,
            $pdns_rec_jobs, $openstack_jobs, $ceph_jobs,
            $galera_jobs, $cloudlb_haproxy_jobs,
        ].flatten,
        global_config_extra            => $config_extra,
        rule_files_extra               => ['/srv/alerts/cloud/*.yaml'],
    }

    profile::thanos::sidecar { 'cloud':
        prometheus_port     => $port,
        prometheus_instance => 'cloud',
        enable_upload       => $enable_thanos_upload,
        min_time            => $thanos_min_time,
    }

    prometheus::pint::source { 'cloud':
        port => $port,
    }

    prometheus::web { 'cloud':
        proxy_pass => "http://localhost:${port}/cloud",
    }
}