Puppet Class: profile::cache::base

Defined in:
modules/profile/manifests/cache/base.pp

Overview

Class profile::cache::base

Sets up some common things for cache instances:

  • conftool

  • monitoring

  • logging/analytics

  • purging

Parameters:

  • cache_cluster (String) (defaults to: lookup('cache::cluster'))
  • logstash_host (Optional[Stdlib::Host]) (defaults to: lookup('logstash_host', {'default_value' => undef}))
  • logstash_syslog_port (Optional[Stdlib::Port]) (defaults to: lookup('logstash_syslog_port', {'default_value' => undef}))
  • logstash_json_lines_port (Optional[Stdlib::Port]) (defaults to: lookup('logstash_json_lines_port', {'default_value' => undef}))
  • log_slow_request_threshold (Float) (defaults to: lookup('profile::cache::base::log_slow_request_threshold', {'default_value' => 60.0}))
  • allow_iptables (Boolean) (defaults to: lookup('profile::cache::base::allow_iptables', {'default_value' => false}))
  • performance_tweaks (Boolean) (defaults to: lookup('profile::cache::base::performance_tweaks', {'default_value' => true}))
  • extra_trust (Array) (defaults to: lookup('profile::cache::base::extra_trust', {'default_value' => []}))
  • wikimedia_domains (Array[Stdlib::Fqdn]) (defaults to: lookup('profile::cache::base::wikimedia_domains'))
  • wmcs_domains (Array[Stdlib::Fqdn]) (defaults to: lookup('profile::cache::base::wmcs_domains'))
  • default_weights (Optional[Hash[String, Integer]]) (defaults to: lookup('profile::cache::base::default_weights', {'default_value' => undef}))
  • conftool_prefix (String) (defaults to: lookup('conftool_prefix'))


9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
# File 'modules/profile/manifests/cache/base.pp', line 9

class profile::cache::base(
    String $cache_cluster                            = lookup('cache::cluster'),
    Optional[Stdlib::Host] $logstash_host            = lookup('logstash_host', {'default_value' => undef}),
    Optional[Stdlib::Port] $logstash_syslog_port     = lookup('logstash_syslog_port', {'default_value' => undef}),
    Optional[Stdlib::Port] $logstash_json_lines_port = lookup('logstash_json_lines_port', {'default_value' => undef}),
    Float $log_slow_request_threshold                = lookup('profile::cache::base::log_slow_request_threshold', {'default_value' => 60.0}),
    Boolean $allow_iptables                          = lookup('profile::cache::base::allow_iptables', {'default_value' => false}),
    Boolean $performance_tweaks                      = lookup('profile::cache::base::performance_tweaks', {'default_value' => true}),
    Array $extra_trust                               = lookup('profile::cache::base::extra_trust', {'default_value' => []}),
    Array[Stdlib::Fqdn] $wikimedia_domains           = lookup('profile::cache::base::wikimedia_domains'),
    Array[Stdlib::Fqdn] $wmcs_domains                = lookup('profile::cache::base::wmcs_domains'),
    Optional[Hash[String, Integer]] $default_weights = lookup('profile::cache::base::default_weights', {'default_value' => undef}),
    String $conftool_prefix                          = lookup('conftool_prefix'),
){

    require network::constants
    # NOTE: Add the public WMCS IP space when T209011 is done
    $wikimedia_nets = flatten(concat($::network::constants::aggregate_networks, '172.16.0.0/12'))
    $wikimedia_trust = flatten(concat($::network::constants::aggregate_networks, $extra_trust))

    # Needed profiles
    require ::profile::conftool::client
    require ::profile::prometheus::cadvisor_exporter
    require ::profile::base::production
    require ::profile::base::systemd

    # FIXME: this cannot be required or it will cause a dependency cycle. It might be a good idea not to include it here
    include ::profile::cache::kafka::webrequest

    include ::profile::prometheus::varnishkafka_exporter

    # Purging
    require ::profile::cache::purge

    # Globals we need to include
    include ::network::constants

    if ! $allow_iptables {
        # Prevent accidental iptables module loads
        kmod::blacklist { 'cp-bl':
            modules => ['x_tables'],
        }
    }

    class { 'conftool::scripts': }

    if $performance_tweaks {
        # Only production needs system perf tweaks
        class { '::cpufrequtils': }
        class { 'cacheproxy::performance': }
    }
    # Basic varnish classes

    class { '::varnish::common':
        log_slow_request_threshold => $log_slow_request_threshold,
        logstash_host              => $logstash_host,
        logstash_json_lines_port   => $logstash_json_lines_port,
    }

    class { [
        '::varnish::common::errorpage',
        '::varnish::common::browsersec',
        '::varnish::common::director_scripts',
    ]:
    }

    class { '::varnish::netmapper_update_common': }
    class { 'varnish::trusted_proxies': }
    # Add /var/netmapper/public_clouds.json from etcd.
    # This file is loaded in wikimedia-frontend.vcl.erb
    confd::file { '/var/netmapper/public_clouds.json':
        ensure     => present,
        watch_keys => ['/request-ipblocks/cloud'],
        prefix     => $conftool_prefix,
        before     => Service['varnish-frontend'],
        content    => template('profile/cache/public_clouds.json.tpl.erb'),
        check      => '/usr/bin/vnm_validate {{.src}}'
    }

    ###########################################################################
    # Analytics/Logging stuff
    ###########################################################################

    # Programs installed on both text and upload nodes
    $common_mtail_programs = ['varnishreqstats', 'varnishttfb', 'varnishxcache']

    # Programs specific to either upload or text
    if $cache_cluster == 'upload' {
        # Media browser cache hit rate and request volume stats.
        $mtail_programs = $common_mtail_programs + [ 'varnishmedia' ]
    } else {
        # ResourceLoader browser cache hit rate and request volume stats.
        $mtail_programs = $common_mtail_programs + [ 'varnishrls' ]
    }

    class { '::varnish::logging':
        default_mtail_programs  => $mtail_programs,
        internal_mtail_programs => [ 'varnishprocessing', 'varnisherrors', 'varnishsli' ],
    }

    # auto-depool on shutdown + conditional one-shot auto-pool on start
    class { 'cacheproxy::traffic_pool': }

    ###########################################################################
    # Purging
    ###########################################################################

    # Node initialization script for conftool
    if $default_weights != undef {
        class { 'conftool::scripts::initialize':
            services => $default_weights,
        }
    }
}