Puppet Class: profile::cache::kafka::eventlogging

Defined in:
modules/profile/manifests/cache/kafka/eventlogging.pp

Overview

SPDX-License-Identifier: Apache-2.0

Class profile::cache::kafka::eventlogging

Sets up a varnishkafka logging endpoint for collecting analytics events coming from external clients.

More info: wikitech.wikimedia.org/wiki/Analytics/EventLogging

Parameters

kafka_cluster_name

Name of the Kafka cluster in the kafka_clusters hash to be passed to the kafka_config() function.

ssl_enabled

If true, the Kafka cluster needs to be configured with SSL support. profile::cache::kafka::certificate will be included, and certs used from it. Default: false

monitoring_enabled

True if the varnishkafka instance should be monitored. Default: false

Parameters:

  • kafka_cluster_name (String) (defaults to: lookup('profile::cache::kafka::eventlogging::kafka_cluster_name'))
  • ssl_enabled (Boolean) (defaults to: lookup('profile::cache::kafka::eventlogging::ssl_enabled', {'default_value' => false}))
  • monitoring_enabled (Boolean) (defaults to: lookup('profile::cache::kafka::eventlogging::monitoring_enabled', {'default_value' => false}))


23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
# File 'modules/profile/manifests/cache/kafka/eventlogging.pp', line 23

class profile::cache::kafka::eventlogging(
    String $kafka_cluster_name = lookup('profile::cache::kafka::eventlogging::kafka_cluster_name'),
    Boolean $ssl_enabled = lookup('profile::cache::kafka::eventlogging::ssl_enabled', {'default_value' => false}),
    Boolean $monitoring_enabled = lookup('profile::cache::kafka::eventlogging::monitoring_enabled', {'default_value' => false}),
) {
    $kafka_config = kafka_config($kafka_cluster_name)

    if $ssl_enabled {
        $kafka_brokers = $kafka_config['brokers']['ssl_array']

        include profile::cache::kafka::certificate

        # Include this class to get key and certificate for varnishkafka
        # to produce to Kafka over SSL/TLS.
        $ssl_ca_location = $profile::cache::kafka::certificate::ssl_ca_location
        $ssl_cipher_suites = $profile::cache::kafka::certificate::ssl_cipher_suites
        $ssl_curves_list = $profile::cache::kafka::certificate::ssl_curves_list
        $ssl_sigalgs_list = $profile::cache::kafka::certificate::ssl_sigalgs_list
        $ssl_keystore_location = $profile::cache::kafka::certificate::ssl_keystore_location
        $ssl_keystore_password = $profile::cache::kafka::certificate::ssl_key_password
        $ssl_key_password = $profile::cache::kafka::certificate::ssl_key_password
        $ssl_key_location = $profile::cache::kafka::certificate::ssl_key_location
        $ssl_certificate_location = $profile::cache::kafka::certificate::ssl_certificate_location
    }
    else {
        $kafka_brokers = $kafka_config['brokers']['array']

        $ssl_ca_location = undef
        $ssl_key_password = undef
        $ssl_key_location = undef
        $ssl_certificate_location = undef
        $ssl_cipher_suites = undef
        $ssl_curves_list = undef
        $ssl_sigalgs_list = undef
        $ssl_keystore_location = undef
        $ssl_keystore_password = undef
    }


    # Sometimes we get junk data sent from bunk user agents.
    # Any user agent regex patterns we want to exclude from eventlogging data can be put in this
    # list, and varnishkafka will be configured not to send this data.
    # See: https://phabricator.wikimedia.org/T266130
    $user_agent_exclude_pattern = '^Fuzz Faster U Fool'

    varnishkafka::instance { 'eventlogging':
        brokers                     => $kafka_brokers,
        # Note that this format uses literal tab characters.
        format                      => '%q	%l	%n	%{%FT%T}t	%{X-Client-IP}o	"%{User-agent}i"',
        format_type                 => 'string',
        compression_codec           => 'snappy',
        topic                       => 'eventlogging-client-side',
        varnish_name                => 'frontend',
        varnish_svc_name            => 'varnish-frontend',
        # Only listen and log requests to /beacon/event(.gif)? that are not from user agents we want to exclude.
        varnish_opts                => { 'q' => "ReqURL ~ \"^/(beacon/)?event(\\.gif)?\\?\" and ReqHeader:user-agent !~ \"${user_agent_exclude_pattern}\"" },
        topic_request_required_acks => '1',
        #TLS/SSL config
        ssl_enabled                 => $ssl_enabled,
        ssl_ca_location             => $ssl_ca_location,
        ssl_key_password            => $ssl_key_password,
        ssl_key_location            => $ssl_key_location,
        ssl_certificate_location    => $ssl_certificate_location,
        ssl_cipher_suites           => $ssl_cipher_suites,
        ssl_curves_list             => $ssl_curves_list,
        ssl_sigalgs_list            => $ssl_sigalgs_list,
        ssl_keystore_location       => $ssl_keystore_location,
        ssl_keystore_password       => $ssl_keystore_password,
    }

    if $monitoring_enabled {
        # Aggregated alarms for delivery errors are defined in icinga::monitor::analytics

        # Generate icinga alert if varnishkafka is not running.
        nrpe::monitor_service { 'varnishkafka-eventlogging':
            description   => 'eventlogging Varnishkafka log producer',
            nrpe_command  => "/usr/lib/nagios/plugins/check_procs -c 1:1 -a '/usr/bin/varnishkafka -S /etc/varnishkafka/eventlogging.conf'",
            contact_group => 'admins,analytics,team-data-platform',
            require       => Varnishkafka::Instance['eventlogging'],
            notes_url     => 'https://wikitech.wikimedia.org/wiki/Analytics/Systems/Varnishkafka',
        }
    }

    # Make sure varnishes are configured and started for the first time
    # before the instances as well, or they fail to start initially...
    Service <| tag == 'varnish_instance' |> -> Varnishkafka::Instance['eventlogging']
}