3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
|
# File 'modules/profile/manifests/cache/purge.pp', line 3
class profile::cache::purge(
Optional[String] $host_regex = lookup('profile::cache::purge::host_regex', {'default_value' => undef}),
Array[String] $kafka_topics = lookup('profile::cache::purge::kafka_topics', {'default_value' => []}),
Boolean $kafka_tls = lookup('profile::cache::purge::kafka_tls', {'default_value' => false}),
String $kafka_cluster_name = lookup('profile::cache::purge::kafka_cluster_name', {'default_value' => 'main-eqiad'}),
Optional[String] $tls_key_password = lookup('profile::cache::purge::tls_key_password', {'default_value' => undef}),
){
$kafka_ensure = $kafka_topics ? {
[] => 'absent',
default => 'present'
}
if $kafka_topics != [] {
$kafka_conf = kafka_config($kafka_cluster_name)
$brokers = $kafka_tls ? {
false => $kafka_conf['brokers']['string'].split(','),
default => $kafka_conf['brokers']['ssl_array']
}
} else {
$brokers = []
}
# KAFKA TLS SETUP
$base_dir = '/etc/purged'
$tls_dir = "${base_dir}/ssl"
$tls_private_dir = "${tls_dir}/private"
$tls_key = "${tls_private_dir}/purged.key.pem"
$tls_cert = "${tls_dir}/purged.crt.pem"
file { $base_dir:
ensure => 'directory',
owner => 'root',
group => 'root',
mode => '0555'
}
if $kafka_tls and $kafka_topics != [] {
file { $tls_dir:
ensure => 'directory',
owner => 'root',
group => 'root',
mode => '0555'
}
file { $tls_private_dir:
ensure => 'directory',
owner => 'root',
group => 'root',
mode => '0500'
}
file { $tls_key:
ensure => 'present',
content => secret('certificates/purged/purged.key.private.pem'),
owner => 'root',
group => 'root',
mode => '0400',
before => Service['purged']
}
file { $tls_cert:
ensure => 'present',
content => secret('certificates/purged/purged.crt.pem'),
owner => 'root',
group => 'root',
mode => '0444',
before => Service['purged']
}
$tls_settings = {
'ca_location' => '/etc/ssl/certs/Puppet_Internal_CA.pem',
'key_location' => $tls_key,
'key_password' => $tls_key_password,
'certificate_location' => $tls_cert,
'cipher_suites' => 'ECDHE-ECDSA-AES256-GCM-SHA384',
'curves_list' => 'P-256',
'sigalgs_list' => 'ECDSA+SHA256'
}
} else {
# TODO: uncomment after the transition.
#file { [$tls_key,$tls_cert,$tls_dir,$tls_private_dir]:
# ensure => 'absent'
#}
$tls_settings = undef
}
$prometheus_port = 2112
class { 'purged':
backend_addr => '127.0.0.1:3128',
frontend_addr => '127.0.0.1:3127',
prometheus_addr => ":${prometheus_port}",
frontend_workers => 4,
backend_workers => $::processorcount,
is_active => true,
host_regex => $host_regex,
kafka_topics => $kafka_topics,
brokers => $brokers,
tls => $tls_settings,
kafka_conf_file => "${base_dir}/purged-kafka.conf"
}
nrpe::monitor_systemd_unit_state { 'purged':
require => Service['purged'],
}
monitoring::check_prometheus { 'purged-event-lag':
description => 'Time elapsed since the last kafka event processed by purged',
# HACK: We take the minimum to work around scenarios where one EventGate datacenter is
# depooled and not actively sending messages.
query => "min(purged_event_lag{instance=\"${::hostname}:${prometheus_port}\"}) / 1e6",
method => 'gt',
warning => 3000, # 3 seconds
critical => 5000, # 5 seconds
prometheus_url => "http://prometheus.svc.${::site}.wmnet/ops",
notes_link => 'https://wikitech.wikimedia.org/wiki/Purged#Alerts',
dashboard_links => ["https://grafana.wikimedia.org/dashboard/db/purged?var-datasource=${::site} prometheus/ops&var-instance=${::hostname}"],
}
monitoring::check_prometheus { 'purged-backlog':
description => 'Number of messages locally queued by purged for processing',
query => "purged_backlog{instance=\"${::hostname}:${prometheus_port}\"}",
method => 'gt',
warning => 1000,
critical => 10000,
prometheus_url => "http://prometheus.svc.${::site}.wmnet/ops",
notes_link => 'https://wikitech.wikimedia.org/wiki/Purged#Alerts',
dashboard_links => ["https://grafana.wikimedia.org/dashboard/db/purged?var-datasource=${::site} prometheus/ops&var-instance=${::hostname}"],
}
}
|