Puppet Class: profile::cache::base

Defined in:
modules/profile/manifests/cache/base.pp

Overview

Class profile::cache::base

Sets up some common things for cache instances:

  • conftool

  • monitoring

  • logging/analytics

  • purging

Parameters:

  • cache_cluster (String) (defaults to: lookup('cache::cluster'))
  • wikimedia_domains (Array[Stdlib::Fqdn]) (defaults to: lookup('wikimedia_domains'))
  • wmcs_domains (Array[Stdlib::Fqdn]) (defaults to: lookup('wmcs_domains'))
  • logstash_host (Optional[Stdlib::Host]) (defaults to: lookup('logstash_host', {'default_value' => undef}))
  • logstash_syslog_port (Optional[Stdlib::Port]) (defaults to: lookup('logstash_syslog_port', {'default_value' => undef}))
  • logstash_json_lines_port (Optional[Stdlib::Port]) (defaults to: lookup('logstash_json_lines_port', {'default_value' => undef}))
  • log_slow_request_threshold (Float) (defaults to: lookup('profile::cache::base::log_slow_request_threshold', {'default_value' => 60.0}))
  • allow_iptables (Boolean) (defaults to: lookup('profile::cache::base::allow_iptables', {'default_value' => false}))
  • performance_tweaks (Boolean) (defaults to: lookup('profile::cache::base::performance_tweaks', {'default_value' => true}))
  • extra_trust (Array) (defaults to: lookup('profile::cache::base::extra_trust', {'default_value' => []}))
  • default_weights (Optional[Hash[String, Integer]]) (defaults to: lookup('profile::cache::base::default_weights', {'default_value' => undef}))
  • conftool_prefix (String) (defaults to: lookup('conftool_prefix'))
  • use_ip_reputation (Boolean) (defaults to: lookup('profile::cache::varnish::frontend::use_ip_reputation'))
  • use_noflow_iface_preup (Boolean) (defaults to: lookup('profile::cache::base::use_noflow_iface_preup', {'default_value' => false})) 


9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
 
# File 'modules/profile/manifests/cache/base.pp', line 9

class profile::cache::base(
    String $cache_cluster                            = lookup('cache::cluster'),
    Array[Stdlib::Fqdn] $wikimedia_domains           = lookup('wikimedia_domains'),
    Array[Stdlib::Fqdn] $wmcs_domains                = lookup('wmcs_domains'),
    Optional[Stdlib::Host] $logstash_host            = lookup('logstash_host', {'default_value' => undef}),
    Optional[Stdlib::Port] $logstash_syslog_port     = lookup('logstash_syslog_port', {'default_value' => undef}),
    Optional[Stdlib::Port] $logstash_json_lines_port = lookup('logstash_json_lines_port', {'default_value' => undef}),
    Float $log_slow_request_threshold                = lookup('profile::cache::base::log_slow_request_threshold', {'default_value' => 60.0}),
    Boolean $allow_iptables                          = lookup('profile::cache::base::allow_iptables', {'default_value' => false}),
    Boolean $performance_tweaks                      = lookup('profile::cache::base::performance_tweaks', {'default_value' => true}),
    Array $extra_trust                               = lookup('profile::cache::base::extra_trust', {'default_value' => []}),
    Optional[Hash[String, Integer]] $default_weights = lookup('profile::cache::base::default_weights', {'default_value' => undef}),
    String $conftool_prefix                          = lookup('conftool_prefix'),
    Boolean $use_ip_reputation                       = lookup('profile::cache::varnish::frontend::use_ip_reputation'),
    Boolean $use_noflow_iface_preup                  = lookup('profile::cache::base::use_noflow_iface_preup', {'default_value' => false}),
){

    require network::constants
    # NOTE: Add the public WMCS IP space when T209011 is done
    $wikimedia_nets = flatten(concat($::network::constants::aggregate_networks, '172.16.0.0/12'))
    $wikimedia_trust = flatten(concat($::network::constants::aggregate_networks, $extra_trust))

    # Needed profiles
    require ::profile::conftool::client
    require ::profile::prometheus::cadvisor
    require ::profile::base::production
    require ::profile::base::systemd

    # FIXME: this cannot be required or it will cause a dependency cycle. It might be a good idea not to include it here
    include ::profile::cache::kafka::webrequest

    include ::profile::prometheus::varnishkafka_exporter

    # Purging
    require ::profile::cache::purge

    # Globals we need to include
    include ::network::constants

    if ! $allow_iptables {
        # Prevent accidental iptables module loads
        kmod::blacklist { 'cp-bl':
            modules => ['x_tables'],
        }
    }

    class { 'conftool::scripts': }

    if $performance_tweaks {
        # Only production needs system perf tweaks
        class { '::cpufrequtils': }
        class { 'cacheproxy::performance':
            use_noflow_iface_preup => $use_noflow_iface_preup,
        }
    }
    # Basic varnish classes

    class { '::varnish::common':
        log_slow_request_threshold => $log_slow_request_threshold,
        logstash_host              => $logstash_host,
        logstash_json_lines_port   => $logstash_json_lines_port,
    }

    class { [
        '::varnish::common::errorpage',
        '::varnish::common::browsersec',
        '::varnish::common::director_scripts',
    ]:
    }

    class { '::varnish::netmapper_update_common': }
    class { 'varnish::trusted_proxies': }
    # Add /var/netmapper/public_clouds.json from etcd.
    # This file is loaded in wikimedia-frontend.vcl.erb
    confd::file { '/var/netmapper/public_clouds.json':
        ensure     => present,
        watch_keys => ['/request-ipblocks/cloud'],
        prefix     => $conftool_prefix,
        before     => Service['varnish-frontend'],
        content    => template('profile/cache/public_clouds.json.tpl.erb'),
        check      => '/usr/bin/vnm_validate {{.src}}'
    }
    # Add /var/netmapper/known_clients.json from etcd.
    # This file is loaded in wikimedia-frontend.vcl.erb
    confd::file { '/var/netmapper/known_clients.json':
        ensure     => present,
        watch_keys => ['/request-ipblocks/known-clients'],
        prefix     => $conftool_prefix,
        before     => Service['varnish-frontend'],
        content    => template('profile/cache/known_clients.json.tpl.erb'),
        check      => '/usr/bin/vnm_validate {{.src}}'
    }
    if ( $use_ip_reputation ) {
        # Add /var/netmapper/vendor_proxies.json
        # This file is loaded in wikimedia-frontend.vcl.erb
        # lint:ignore:puppet_url_without_modules
        file { '/var/netmapper/vendor_proxies.json':
            ensure       => present,
            source       => 'puppet:///volatile/ip_reputation_vendors/proxies.json',
            before       => Service['varnish-frontend'],
            validate_cmd => '/usr/bin/vnm_validate %',
        }
    }
    # lint:endignore

    ###########################################################################
    # Analytics/Logging stuff
    ###########################################################################

    # Programs installed on both text and upload nodes
    $common_mtail_programs = ['varnishreqstats', 'varnishttfb', 'varnishxcache']

    # Programs specific to either upload or text
    if $cache_cluster == 'upload' {
        # Media browser cache hit rate and request volume stats.
        $mtail_programs = $common_mtail_programs + [ 'varnishmedia' ]
    } else {
        # ResourceLoader browser cache hit rate and request volume stats.
        $mtail_programs = $common_mtail_programs + [ 'varnishrls' ]
    }

    class { '::varnish::logging':
        default_mtail_programs  => $mtail_programs,
        internal_mtail_programs => [ 'varnishprocessing', 'varnisherrors', 'varnishsli' ],
    }

    # auto-depool on shutdown + conditional one-shot auto-pool on start
    class { 'cacheproxy::traffic_pool': }

    ###########################################################################
    # Purging
    ###########################################################################

    # Node initialization script for conftool
    if $default_weights != undef {
        class { 'conftool::scripts::initialize':
            services => $default_weights,
        }
    }
}