Puppet Class: profile::wmcs::prometheus

Defined in:
modules/profile/manifests/wmcs/prometheus.pp

Overview

Parameters:

  • targets_path (Stdlib::Unixpath) (defaults to: '/srv/prometheus/labs/targets')
  • storage_retention (String) (defaults to: lookup('prometheus::server::storage_retention', {'default_value' => '4032h'}))
  • max_chunks_to_persist (Integer) (defaults to: lookup('prometheus::server::max_chunks_to_persist', {'default_value' => 524288}))
  • memory_chunks (Integer) (defaults to: lookup('prometheus::server::memory_chunks', {'default_value' => 1048576}))
  • storage_retention_size (Optional[Stdlib::Datasize]) (defaults to: lookup('profile::wmcs::prometheus::storage_retention_size', {default_value => undef}))
  • alertmanagers (Array[Stdlib::Host]) (defaults to: lookup('alertmanagers', {'default_value' => []}))


1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
# File 'modules/profile/manifests/wmcs/prometheus.pp', line 1

class profile::wmcs::prometheus(
    Stdlib::Unixpath $targets_path = '/srv/prometheus/labs/targets',
    String $storage_retention = lookup('prometheus::server::storage_retention', {'default_value' => '4032h'}),
    Integer $max_chunks_to_persist = lookup('prometheus::server::max_chunks_to_persist', {'default_value' => 524288}),
    Integer $memory_chunks = lookup('prometheus::server::memory_chunks', {'default_value' => 1048576}),
    Optional[Stdlib::Datasize] $storage_retention_size = lookup('profile::wmcs::prometheus::storage_retention_size',   {default_value => undef}),
    Array[Stdlib::Host] $alertmanagers = lookup('alertmanagers', {'default_value' => []}),
) {
    $config_extra = {
        'external_labels' => {
            # right now cloudmetrics hardware only exists on eqiad1, make sure to update this if that changes
            'deployment' => 'eqiad1',
            'prometheus' => 'cloud',
        },
    }

    include ::prometheus::blackbox_exporter
    $blackbox_jobs = [
        {
            'job_name'        => 'blackbox_http',
            'metrics_path'    => '/probe',
            'params'          => {
                'module' => [ 'http_200_300_connect' ],
            },
            'file_sd_configs' => [
                { 'files' => [ "${targets_path}/blackbox_http_*.yaml" ] }
            ],
            'relabel_configs' => [
                { 'source_labels' => ['__address__'],
                    'target_label'  => '__param_target',
                    'replacement' => 'http://$1/',
                },
                { 'source_labels' => ['__param_target'],
                    'target_label'  => 'instance',
                },
                { 'target_label' => '__address__',
                    'replacement'  => '127.0.0.1:9115',
                },
            ],
        },
        {
            'job_name'        => 'blackbox_https',
            'metrics_path'    => '/probe',
            'params'          => {
                'module' => [ 'https_200_300_connect' ],
            },
            'file_sd_configs' => [
                { 'files' => [ "${targets_path}/blackbox_https_*.yaml" ] }
            ],
            'relabel_configs' => [
                { 'source_labels' => ['__address__'],
                    'target_label'  => '__param_target',
                    'replacement' => 'https://$1/',
                },
                { 'source_labels' => ['__param_target'],
                    'target_label'  => 'instance',
                },
                { 'target_label' => '__address__',
                    'replacement'  => '127.0.0.1:9115',
                },
            ],
        },
    ]

    $rabbitmq_jobs = [
        {
            'job_name'        => 'rabbitmq',
            'scheme'          => 'http',
            'file_sd_configs' => [
                { 'files' => [ "${targets_path}/rabbitmq_*.yaml" ] }
            ],
        },
    ]

    $pdns_jobs = [
        {
            'job_name'        => 'pdns',
            'scheme'          => 'http',
            'file_sd_configs' => [
                { 'files' => [ "${targets_path}/pdns_*.yaml" ] }
            ],
        },
    ]

    $pdns_rec_jobs = [
        {
            'job_name'        => 'pdns_rec',
            'scheme'          => 'http',
            'file_sd_configs' => [
                { 'files' => [ "${targets_path}/pdns-rec_*.yaml" ] }
            ],
        },
    ]

    $openstack_jobs = [
        {
            'job_name'        => 'openstack',
            'scheme'          => 'http',
            'file_sd_configs' => [
                { 'files' => [ "${targets_path}/openstack_*.yaml" ] }
            ],
            'scrape_interval' => '15m',
            'scrape_timeout'  => '120s',
        },
    ]

    $ceph_jobs = [
      {
        'job_name'        => "ceph_${::site}",
        'scheme'          => 'http',
        'file_sd_configs' => [
          { 'files' => [ "${targets_path}/ceph_${::site}.yaml" ]}
        ],
      },
    ]

    file { "${targets_path}/blackbox_http_keystone.yaml":
        content => to_yaml([{
            'targets' => [
                'openstack.eqiad1.wikimediacloud.org:5000/v3', # keystone
            ],
        }]),
    }

    file { "${targets_path}/blackbox_https_keystone.yaml":
        content => to_yaml([{
            'targets' => [
                'openstack.eqiad1.wikimediacloud.org:25000/v3', # keystone
                'openstack.eqiad1.wikimediacloud.org:28774', # nova
                'openstack.eqiad1.wikimediacloud.org:28776', # cinder
                'openstack.eqiad1.wikimediacloud.org:28778', # placement
                'openstack.eqiad1.wikimediacloud.org:28779', # trove
                'openstack.eqiad1.wikimediacloud.org:29001', # designate
                'openstack.eqiad1.wikimediacloud.org:29292', # glance
                'openstack.eqiad1.wikimediacloud.org:29696', # neutron
            ],
        }]),
    }

    prometheus::class_config{ "rabbitmq_${::site}":
        dest       => "${targets_path}/rabbitmq_${::site}.yaml",
        class_name => 'role::wmcs::openstack::eqiad1::control',
        port       => 15692,
    }

    prometheus::class_config{ "pdns_${::site}":
        dest       => "${targets_path}/pdns_${::site}.yaml",
        class_name => 'role::wmcs::openstack::eqiad1::services',
        port       => 8081,
    }

    prometheus::class_config{ "pdns-rec_${::site}":
        dest       => "${targets_path}/pdns-rec_${::site}.yaml",
        class_name => 'role::wmcs::openstack::eqiad1::services',
        port       => 8082,
    }

    file { "${targets_path}/openstack_${::site}.yaml":
        content => to_yaml([{
            'labels'  => {
                'cluster' => 'wmcs',
                'site'    => 'eqiad',
            },
            'targets' => [
                'openstack.eqiad1.wikimediacloud.org:12345',
            ]
        }]),
    }

    file { "${targets_path}/redis_toolforge_hosts.yaml":
        ensure => absent,
    }

    prometheus::class_config{ "ceph_${::site}":
        dest       => "${targets_path}/ceph_${::site}.yaml",
        class_name => 'role::wmcs::ceph::mon',
        port       => 9283,
    }

    # Don't worry about codfw1dev; no cloudmetrics in codfw anyway
    if $::site == eqiad {
        prometheus::class_config{ 'mysql_galera_eqiad1':
            dest       => "${targets_path}/mysql_galera_eqiad1.yaml",
            class_name => 'role::wmcs::openstack::eqiad1::control',
            port       => 9104,
            labels     => {
                'deployment' => 'eqiad1'
            }
        }
    }

    $galera_jobs = [
        {
            'job_name'        => 'mysql-galera',
            'file_sd_configs' => [
                { 'files' => [ "${targets_path}/mysql_galera_*.yaml"] },
            ],
        }
    ]

    prometheus::server { 'labs':
        listen_address                 => '127.0.0.1:9900',
        storage_retention              => $storage_retention,
        storage_retention_size         => $storage_retention_size,
        max_chunks_to_persist          => $max_chunks_to_persist,
        memory_chunks                  => $memory_chunks,
        alertmanagers                  => $alertmanagers.map |$a| { "${a}:9093" },
        alerting_relabel_configs_extra => [
            # Add 'team' label, https://phabricator.wikimedia.org/T302493#7759642
            { 'target_label' => 'team', 'replacement' => 'wmcs', 'action' => 'replace' },
        ],
        scrape_configs_extra           => [
            $blackbox_jobs, $rabbitmq_jobs, $pdns_jobs,
            $pdns_rec_jobs, $openstack_jobs, $ceph_jobs,
            $galera_jobs,
        ].flatten,
        global_config_extra            => $config_extra,
        rule_files_extra               => ['/srv/alerts/cloud/*.yaml'],
    }

    class { 'alerts::deploy::prometheus':
        # let's not introduce new uses of the 'labs' term in operations/alerts.git
        instances => ['cloud'],
    }

    httpd::site{ 'prometheus':
        priority => 10,
        content  => template('profile/wmcs/metricsinfra/prometheus-apache.erb'),
    }

    prometheus::web { 'labs':
        proxy_pass => 'http://localhost:9900/labs',
        require    => Httpd::Site['prometheus'],
    }

    ferm::service { 'prometheus-web':
        proto  => 'tcp',
        port   => '80',
        srange => '$DOMAIN_NETWORKS',
    }
}