Puppet Class: profile::wmcs::paws::prometheus

Defined in:
modules/profile/manifests/wmcs/paws/prometheus.pp

Overview

prometheus instance for PAWS

Parameters:

  • storage_retention_size (Optional[Stdlib::Datasize]) (defaults to: lookup('profile::wmcs::paws::prometheus::storage_retention_size', {default_value => undef}))
  • region (String) (defaults to: lookup('profile::openstack::eqiad1::region'))
  • keystone_api_fqdn (Stdlib::Fqdn) (defaults to: lookup('profile::openstack::eqiad1::keystone_api_fqdn'))
  • observer_user (String) (defaults to: lookup('profile::openstack::base::observer_user'))
  • observer_password (String) (defaults to: lookup('profile::openstack::eqiad1::observer_password'))
  • k8s_apiserver_fqdn (Stdlib::Fqdn) (defaults to: lookup('profile::wmcs::paws::prometheus::k8s_apiserver_fqdn', {default_value => 'k8s.svc.paws.eqiad1.wikimedia.cloud'}))
  • k8s_apiserver_port (Stdlib::Port) (defaults to: lookup('profile::wmcs::paws::prometheus::k8s_apiserver_port', {default_value => 6443}))


2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
# File 'modules/profile/manifests/wmcs/paws/prometheus.pp', line 2

class profile::wmcs::paws::prometheus (
    Optional[Stdlib::Datasize] $storage_retention_size = lookup('profile::wmcs::paws::prometheus::storage_retention_size',   {default_value => undef}),
    String                     $region                 = lookup('profile::openstack::eqiad1::region'),
    Stdlib::Fqdn               $keystone_api_fqdn      = lookup('profile::openstack::eqiad1::keystone_api_fqdn'),
    String                     $observer_user          = lookup('profile::openstack::base::observer_user'),
    String                     $observer_password      = lookup('profile::openstack::eqiad1::observer_password'),
    Stdlib::Fqdn               $k8s_apiserver_fqdn     = lookup('profile::wmcs::paws::prometheus::k8s_apiserver_fqdn', {default_value => 'k8s.svc.paws.eqiad1.wikimedia.cloud'}),
    Stdlib::Port               $k8s_apiserver_port     = lookup('profile::wmcs::paws::prometheus::k8s_apiserver_port',      {default_value => 6443}),
) {
    include ::profile::labs::cindermount::srv

    class { '::httpd':
        modules => ['proxy', 'proxy_http'],
    }

    $k8s_cert_name  = 'paws-k8s-prometheus'
    $k8s_cert_pub  = "/etc/ssl/localcerts/${k8s_cert_name}.crt"
    $k8s_cert_priv = "/etc/ssl/private/${k8s_cert_name}.key"
    sslcert::certificate { $k8s_cert_name:
        ensure  => present,
        chain   => false,
        group   => 'prometheus',
        require => Package['prometheus'], # group is defined by the package?
        notify  => Service['prometheus@paws'],
    }

    $k8s_tls_config = {
        'insecure_skip_verify' => true,
        'cert_file'            => $k8s_cert_pub,
        'key_file'             => $k8s_cert_priv,
    }

    $openstack_jobs = [
        {
            name => 'node-exporter',
            port => 9100,
        },
        {
            name            => 'haproxy',
            port            => 9901,
            instance_filter => 'paws-k8s-haproxy-\\d+',
        },
        {
            name            => 'k8s-apiserver',
            port            => 6443,
            instance_filter => 'paws-k8s-control-\\d+',
            extra_config    => {
                scheme     => 'https',
                tls_config => $k8s_tls_config,
            },
        }
    ].map |Hash $job| {
        if $job['instance_filter'] {
            $relabel_configs = [
                {
                    'source_labels' => ['__meta_openstack_instance_name'],
                    'action'        => 'keep',
                    'regex'         => $job['instance_filter'],
                },
                {
                    'source_labels' => ['__meta_openstack_instance_name'],
                    'target_label'  => 'instance',
                },
                {
                    'source_labels' => ['__meta_openstack_instance_status'],
                    'action'        => 'keep',
                    'regex'         => 'ACTIVE',
                },
            ]
        } else {
            $relabel_configs = [
                {
                    'source_labels' => ['__meta_openstack_instance_name'],
                    'target_label'  => 'instance',
                },
                {
                    'source_labels' => ['__meta_openstack_instance_status'],
                    'action'        => 'keep',
                    'regex'         => 'ACTIVE',
                },
            ]
        }

        $result = {
            'job_name'             => $job['name'],
            'openstack_sd_configs' => [
                {
                    'role'              => 'instance',
                    'region'            => $region,
                    'identity_endpoint' => "https://${keystone_api_fqdn}:25000/v3",
                    'username'          => $observer_user,
                    'password'          => $observer_password,
                    'domain_name'       => 'default',
                    'project_name'      => $::labsproject,
                    'all_tenants'       => false,
                    'refresh_interval'  => '5m',
                    'port'              => $job['port'],
                }
            ],
            'relabel_configs'      => $relabel_configs,
        }

        deep_merge(
            $result,
            pick($job['extra_config'], {})
        )
    }

    $manual_jobs = [
        {
            'job_name'       => 'jupyterhub',
            'scheme'         => 'https',
            'metrics_path'   => '/hub/metrics',
            'static_configs' => [
                {
                    'targets' => ['hub.paws.wmcloud.org'],
                },
            ],
        },

        # this is in manual and not $kubernetes_pod_jobs
        # as it scrapes nodes, not jobs
        {
            'job_name'              => 'k8s-nodes',
            'scheme'                => 'https',
            'tls_config'            => $k8s_tls_config,
            'kubernetes_sd_configs' => [
                {
                    'api_server' => "https://${k8s_apiserver_fqdn}:${k8s_apiserver_port}",
                    'role'       => 'node',
                    'tls_config' => $k8s_tls_config,
                },
            ],
            'relabel_configs'       => [
                {
                    'action' => 'labelmap',
                    'regex'  => '__meta_kubernetes_node_label_(.+)',
                },
                {
                    'target_label' => '__address__',
                    'replacement'  => "${k8s_apiserver_fqdn}:${k8s_apiserver_port}",
                },
                {
                    'source_labels' => ['__meta_kubernetes_node_name'],
                    'regex'         => '(.+)',
                    'target_label'  => '__metrics_path__',
                    # lint:ignore:single_quote_string_with_variables
                    'replacement'   => '/api/v1/nodes/${1}/proxy/metrics',
                    # lint:endignore
                },
            ]
        },
    ]

    $kubernetes_pod_jobs = [
        {
            name      => 'k8s-ingress-nginx',
            namespace => 'ingress-nginx-gen2',
            pod_name  => 'ingress-nginx-gen2-controller-[a-zA-Z0-9]+-[a-zA-Z0-9]+',
            port      => 10254,
        },
        {
            name      => 'k8s-cadvisor',
            namespace => 'metrics',
            pod_name  => 'cadvisor-[a-zA-Z0-9]+',
            port      => 8080,
        },
        {
            name      => 'k8s-kube-state-metrics',
            namespace => 'metrics',
            pod_name  => 'kube-state-metrics-[a-zA-Z0-9]+-[a-zA-Z0-9]+',
            port      => 8080,
        },
    ].map |Hash $job| {
        {
            'job_name'              => $job['name'],
            'scheme'                => 'https',
            'tls_config'            => $k8s_tls_config,
            'kubernetes_sd_configs' => [
                {
                    'api_server' => "https://${k8s_apiserver_fqdn}:${k8s_apiserver_port}",
                    'role'       => 'pod',
                    'tls_config' => $k8s_tls_config,
                },
            ],
            'relabel_configs'       => [
                {
                    'action'        => 'keep',
                    'regex'         => $job['namespace'],
                    'source_labels' => ['__meta_kubernetes_namespace'],
                },
                {
                    'action'        => 'keep',
                    'regex'         => $job['pod_name'],
                    'source_labels' => ['__meta_kubernetes_pod_name'],
                },
                {
                    'action' => 'labelmap',
                    'regex'  => '__meta_kubernetes_pod_label_(.+)',
                },
                {
                    'target_label' => '__address__',
                    'replacement'  => "${k8s_apiserver_fqdn}:${k8s_apiserver_port}",
                },
                {
                    'source_labels' => ['__meta_kubernetes_pod_name'],
                    'regex'         => "(${job['pod_name']})",
                    'target_label'  => '__metrics_path__',
                    'replacement'   => "/api/v1/namespaces/${job['namespace']}/pods/\${1}:${job['port']}/proxy/metrics",
                },
            ]
        }
    }

    $jobs = $openstack_jobs + $manual_jobs + $kubernetes_pod_jobs

    $alertmanager_discovery_extra = [
        {
            'openstack_sd_configs' => [
                {
                    'role'              => 'instance',
                    'region'            => $region,
                    'identity_endpoint' => "https://${keystone_api_fqdn}:25000/v3",
                    'username'          => $observer_user,
                    'password'          => $observer_password,
                    'domain_name'       => 'default',
                    'project_name'      => 'metricsinfra',
                    'all_tenants'       => false,
                    'refresh_interval'  => '5m',
                    'port'              => 8643,
                },
            ],
            'relabel_configs'      => [
                {
                    'source_labels' => ['__meta_openstack_instance_name'],
                    'action'        => 'keep',
                    'regex'         => 'metricsinfra-alertmanager-\d+',
                },
                {
                    'source_labels' => ['__meta_openstack_instance_name'],
                    'target_label'  => 'instance',
                },
                {
                    'source_labels' => ['__meta_openstack_instance_status'],
                    'action'        => 'keep',
                    'regex'         => 'ACTIVE',
                },
            ],
        },
    ]

    prometheus::server { 'paws':
        listen_address                 => '127.0.0.1:9903',
        external_url                   => 'https://prometheus.paws.wmcloud.org/paws',
        storage_retention_size         => $storage_retention_size,
        scrape_configs_extra           => $jobs,
        alertmanager_discovery_extra   => $alertmanager_discovery_extra,
        alerting_relabel_configs_extra => [
            { 'target_label' => 'project', 'replacement' => $::labsproject, 'action' => 'replace' },
        ],
    }

    prometheus::web { 'paws':
        proxy_pass => 'http://localhost:9903/paws',
    }
}