Puppet Class: role::logging::mediawiki::udp2log

Defined in:
modules/role/manifests/logging/mediawiki/udp2log.pp

Overview

mediawiki udp2log instance. Does not use monitoring.

filtertags: labs-project-deployment-prep

Parameters:

  • logstash_host (Any)
  • monitor (Any) (defaults to: true)
  • log_directory (Any) (defaults to: '/srv/mw-log')
  • rotate (Any) (defaults to: 1000)
  • forward_messages (Any) (defaults to: false)
  • mirror_destinations (Any) (defaults to: undef)


4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
# File 'modules/role/manifests/logging/mediawiki/udp2log.pp', line 4

class role::logging::mediawiki::udp2log(
    $logstash_host,
    $monitor = true,
    $log_directory = '/srv/mw-log',
    $rotate = 1000,
    $forward_messages = false,
    $mirror_destinations = undef,
) {
    system::role { 'logging:mediawiki::udp2log':
        description => 'MediaWiki log collector',
    }

    include ::standard
    include ::profile::base::firewall
    include ::profile::webperf::arclamp

    # Include geoip databases and CLI.
    class { '::geoip': }

    class { '::udp2log':
        monitor          => $monitor,
        default_instance => false,
    }

    if $::hostname == 'mwlog1001' {
        class {'profile::logster_alarm':}
    }

    ferm::rule { 'udp2log_accept_all_wikimedia':
        rule => 'saddr ($DOMAIN_NETWORKS) proto udp ACCEPT;',
    }

    ferm::rule { 'udp2log_notrack':
        table => 'raw',
        chain => 'PREROUTING',
        rule  => 'saddr ($DOMAIN_NETWORKS) proto udp NOTRACK;',
    }

    file { '/usr/local/bin/demux.py':
        mode   => '0555',
        owner  => 'root',
        group  => 'root',
        source => 'puppet:///modules/udp2log/demux.py',
    }

    file { '/usr/local/bin/udpmirror.py':
        mode   => '0555',
        owner  => 'root',
        group  => 'root',
        source => 'puppet:///modules/udp2log/udpmirror.py',
    }

    $logstash_port = 8324

    udp2log::instance { 'mw':
        log_directory       =>   $log_directory,
        monitor_log_age     =>   false,
        monitor_processes   =>   false,
        rotate              =>   $rotate,
        forward_messages    =>   $forward_messages,
        mirror_destinations =>   $mirror_destinations,
        template_variables  => {
            # forwarding to logstash
            logstash_host => $logstash_host,
            logstash_port => $logstash_port,
        },
    }

    # Allow rsyncing of udp2log generated files to
    # analysis hosts.
    class { 'udp2log::rsyncd':
        path        => $log_directory,
        hosts_allow => hiera('statistics_servers', 'stat1007.eqiad.wmnet')
    }

    cron { 'mw-log-cleanup':
        command => '/usr/local/bin/mw-log-cleanup',
        user    => 'root',
        hour    => 2,
        minute  => 0
    }

    file { '/usr/local/bin/mw-log-cleanup':
        owner  => 'root',
        group  => 'root',
        mode   => '0555',
        source => 'puppet:///modules/role/logging/mw-log-cleanup',
    }

    file { '/etc/profile.d/mw-log.sh':
        owner   => 'root',
        group   => 'root',
        mode    => '0555',
        content => "MW_LOG_DIRECTORY=${log_directory}\n",
    }

    file { '/usr/local/bin/fatalmonitor':
        owner  => 'root',
        group  => 'root',
        mode   => '0555',
        source => 'puppet:///modules/role/logging/fatalmonitor',
    }

    # Web server (site added by profile::webperf::arclamp).
    # The httpd class must be here (in a role) instead of in arlamp profile,
    # so other roles (eg. webperf::profiling_tools) may have multiple
    # profiles that add sites.
    class { '::httpd':
        modules => ['mime', 'proxy', 'proxy_http'],
    }

    # Redis is used to receive Xenon stack traces from MediaWiki app servers,
    # for processing by Arc Lamp (see profile::webperf::arclamp).
    redis::instance { '6379':
        settings => {
            maxmemory                   => '1Mb',
            stop_writes_on_bgsave_error => 'no',
            bind                        => '0.0.0.0',
        },
    }

    # The Redis for Arc Lamp and Arc Lamp itself are currently
    # part of the same role (this role), so make sure that
    # Redis starts before Arc Lamp.
    Service['redis-server'] ~> Service['xenon-log']

    ferm::rule { 'xenon_redis':
        rule => 'saddr ($DOMAIN_NETWORKS) proto tcp dport 6379 ACCEPT;',
    }
}