Puppet Class: profile::webperf::processors

Defined in:
modules/profile/manifests/webperf/processors.pp

Overview

SPDX-License-Identifier: Apache-2.0

Class: profile::webperf::processors

Provision the webperf data processors. Consumes from EventLogging (via Kafka), and produces to Prometheus, and Graphite (via StatsD).

Contact: Performance Team See also: https://wikitech.wikimedia.org/wiki/Webperf

Services:

  • statsv
  • navtiming

Parameters:

  • statsd (String) (defaults to: lookup('statsd'))
  • graphite_host (Stdlib::Fqdn) (defaults to: lookup('graphite_host'))
  • histogram_buckets (Array[Variant[Integer, Float]]) (defaults to: lookup('profile::webperf::processors::histogram_buckets', { 'default_value' => [0.005,0.01,0.025,0.05,0.1,0.25,0.5,1,2.5,5,10,30,60],})) 


15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
 
# File 'modules/profile/manifests/webperf/processors.pp', line 15

class profile::webperf::processors(
    String       $statsd        = lookup('statsd'),
    Stdlib::Fqdn $graphite_host = lookup('graphite_host'),
    Array[Variant[Integer, Float]] $histogram_buckets = lookup('profile::webperf::processors::histogram_buckets', { 'default_value' => [0.005,0.01,0.025,0.05,0.1,0.25,0.5,1,2.5,5,10,30,60],}),
){

    $statsd_parts = split($statsd, ':')
    $statsd_host = $statsd_parts[0]
    $statsd_port = 0 + Integer($statsd_parts[1])

    # statsv is on main kafka, not analytics or jumbo kafka.
    # Note that at any given time, all statsv varnishkafka producers are
    # configured to send to only one kafka cluster (usually main-eqiad).
    # statsv in an inactive datacenter will not process any messages, as
    # varnishkafka will not produce any messages to that DC's kafka cluster.
    # This is configured by the value of the hiera param
    # profile::cache::kafka::statsv::kafka_cluster_name when the statsv varnishkafka
    # profile is included (as of this writing on text caches).
    $kafka_main_config = kafka_config("main-${::site}")
    $kafka_main_brokers = $kafka_main_config['brokers']['ssl_string']

    $kafka_ssl_cafile = profile::base::certificates::get_trusted_ca_path()

    class { '::webperf::statsv':
        kafka_brokers           => $kafka_main_brokers,
        kafka_security_protocol => 'SSL',
        statsd_host             => $statsd_host,
        statsd_port             => $statsd_port,
        # Point --dogstatsd to Prometheus via statsd_exporter (T355837)
        dogstatsd_host          => '127.0.0.1',
        dogstatsd_port          => 9125,
        kafka_ssl_cafile        => $kafka_ssl_cafile,
    }
    class { 'profile::prometheus::statsd_exporter':
        prometheus_instance => 'ext',
        relay_address       => '', # unset to disable relaying
        timer_type          => 'histogram',
        histogram_buckets   => $histogram_buckets
    }

    # EventLogging is on the jumbo kafka. Unlike the main one, this
    # is not yet mirrored to other data centers, so for prod,
    # assume eqiad.
    $kafka_config  = kafka_config('jumbo-eqiad')
    $kafka_brokers = $kafka_config['brokers']['ssl_string']

    # Aggregate client-side latency measurements collected via the
    # NavigationTiming MediaWiki extension and send them to Graphite.
    # See <https://www.mediawiki.org/wiki/Extension:NavigationTiming>
    class { '::webperf::navtiming':
        kafka_brokers           => $kafka_brokers,
        kafka_security_protocol => 'SSL',
        statsd_host             => $statsd_host,
        statsd_port             => $statsd_port,
        kafka_ssl_cafile        => $kafka_ssl_cafile,
    }

    # navtiming exports Prometheus metrics on port 9230.
    if $::realm == 'labs' {
        firewall::service { 'prometheus-navtiming-exporter':
            proto    => 'tcp',
            port     => 9230,
            src_sets => ['LABS_NETWORKS'],
        }
    }
}