1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
|
# File 'modules/profile/manifests/mediawiki/mwlog.pp', line 1
class profile::mediawiki::mwlog (
Stdlib::Unixpath $log_directory = lookup('profile::mediawiki::mwlog::log_directory', {'default_value' => '/srv/mw-log-kafka'}),
Optional[Stdlib::Fqdn] $primary_host = lookup('profile::mediawiki::mwlog::primary_host', {'default_value' => undef}),
Optional[Stdlib::Fqdn] $standby_host = lookup('profile::mediawiki::mwlog::standby_host', {'default_value' => undef}),
) {
$kafka_config = kafka_config('logging-eqiad')
$topic_prefix = 'mwlog-'
# NOTE needs to be updated when adding/removing partitions from topics
$partitions = '0-2'
$archive_directory = "${log_directory}/archive"
file { [$log_directory, $archive_directory]:
ensure => 'directory',
owner => 'kafkatee',
group => 'kafkatee',
require => Package['kafkatee'],
}
file { '/usr/local/bin/mwlog-demux.py':
ensure => present,
source => 'puppet:///modules/profile/mediawiki/mwlog/mwlog-demux.py',
owner => 'root',
group => 'root',
mode => '0555',
}
$mwlog_levels = ['debug', 'err', 'info', 'notice', 'warning']
$mwlog_inputs = $mwlog_levels.map |String $level| {
{
'topic' => "${topic_prefix}${level}",
'partitions' => $partitions,
'options' => { 'encoding' => 'json' },
'offset' => 'end',
}
}
# use logrotate >= 3.21.0 backported from bookworm on mwlog hosts
if debian::codename::eq('bullseye') {
apt::package_from_component { 'logrotate':
component => 'component/logrotate',
priority => 1002,
}
}
kafkatee::instance { 'mwlog':
kafka_brokers => $kafka_config['brokers']['ssl_array'],
output_encoding => 'json',
inputs => $mwlog_inputs,
ssl_enabled => true,
ssl_ca_location => profile::base::certificates::get_trusted_ca_path(),
}
kafkatee::output { 'udp2log-compat':
instance_name => 'mwlog',
destination => "/usr/local/bin/mwlog-demux.py --basedir ${log_directory}",
type => 'pipe',
}
logrotate::rule { 'udp2log-compat':
ensure => present,
file_glob => "${log_directory}/*.log",
old_dir => "${log_directory}/archive",
frequency => 'daily',
not_if_empty => true,
no_create => true,
max_age => 90,
date_ext => true,
compress => true,
missing_ok => true,
post_rotate => ['service kafkatee-mwlog reload'],
}
if $primary_host and $standby_host {
rsync::quickdatacopy { 'srv-mw-log':
source_host => $primary_host,
dest_host => $standby_host,
auto_sync => false,
module_path => '/srv/mw-log',
server_uses_stunnel => true,
}
}
}
|