Puppet Class: profile::cache::base

Defined in:
modules/profile/manifests/cache/base.pp

Overview

SPDX-License-Identifier: Apache-2.0

Class profile::cache::base

Sets up some common things for cache instances:

  • conftool
  • monitoring
  • logging/analytics
  • purging
  • geoip

Parameters:

  • cache_cluster (String) (defaults to: lookup('cache::cluster'))
  • wikimedia_domains (Array[Stdlib::Fqdn]) (defaults to: lookup('wikimedia_domains'))
  • wmcs_domains (Array[Stdlib::Fqdn]) (defaults to: lookup('wmcs_domains'))
  • logstash_host (Optional[Stdlib::Host]) (defaults to: lookup('logstash_host', {'default_value' => undef}))
  • logstash_syslog_port (Optional[Stdlib::Port]) (defaults to: lookup('logstash_syslog_port', {'default_value' => undef}))
  • logstash_json_lines_port (Optional[Stdlib::Port]) (defaults to: lookup('logstash_json_lines_port', {'default_value' => undef}))
  • log_slow_request_threshold (Float) (defaults to: lookup('profile::cache::base::log_slow_request_threshold', {'default_value' => 60.0}))
  • allow_iptables (Boolean) (defaults to: lookup('profile::cache::base::allow_iptables', {'default_value' => false}))
  • performance_tweaks (Boolean) (defaults to: lookup('profile::cache::base::performance_tweaks', {'default_value' => true}))
  • extra_trust (Array) (defaults to: lookup('profile::cache::base::extra_trust', {'default_value' => []}))
  • default_weights (Optional[Hash[String, Integer]]) (defaults to: lookup('profile::cache::base::default_weights', {'default_value' => undef}))
  • conftool_prefix (String) (defaults to: lookup('conftool_prefix'))
  • use_geo_ip (Boolean) (defaults to: lookup('profile::cache::base::use_geo_ip', {'default_value' => true}))
  • use_ip_reputation (Boolean) (defaults to: lookup('profile::cache::varnish::frontend::use_ip_reputation'))
  • enable_monitoring (Boolean) (defaults to: lookup('profile::cache::varnish::frontend::enable_monitoring'))
  • use_noflow_iface_preup (Boolean) (defaults to: lookup('profile::cache::base::use_noflow_iface_preup', {'default_value' => false})) 


11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
 
# File 'modules/profile/manifests/cache/base.pp', line 11

class profile::cache::base(
    String $cache_cluster                            = lookup('cache::cluster'),
    Array[Stdlib::Fqdn] $wikimedia_domains           = lookup('wikimedia_domains'),
    Array[Stdlib::Fqdn] $wmcs_domains                = lookup('wmcs_domains'),
    Optional[Stdlib::Host] $logstash_host            = lookup('logstash_host', {'default_value' => undef}),
    Optional[Stdlib::Port] $logstash_syslog_port     = lookup('logstash_syslog_port', {'default_value' => undef}),
    Optional[Stdlib::Port] $logstash_json_lines_port = lookup('logstash_json_lines_port', {'default_value' => undef}),
    Float $log_slow_request_threshold                = lookup('profile::cache::base::log_slow_request_threshold', {'default_value' => 60.0}),
    Boolean $allow_iptables                          = lookup('profile::cache::base::allow_iptables', {'default_value' => false}),
    Boolean $performance_tweaks                      = lookup('profile::cache::base::performance_tweaks', {'default_value' => true}),
    Array $extra_trust                               = lookup('profile::cache::base::extra_trust', {'default_value' => []}),
    Optional[Hash[String, Integer]] $default_weights = lookup('profile::cache::base::default_weights', {'default_value' => undef}),
    String $conftool_prefix                          = lookup('conftool_prefix'),
    Boolean $use_geo_ip                              = lookup('profile::cache::base::use_geo_ip', {'default_value' => true}),
    Boolean $use_ip_reputation                       = lookup('profile::cache::varnish::frontend::use_ip_reputation'),
    Boolean $enable_monitoring                       = lookup('profile::cache::varnish::frontend::enable_monitoring'),
    Boolean $use_noflow_iface_preup                  = lookup('profile::cache::base::use_noflow_iface_preup', {'default_value' => false}),
){

    require network::constants
    # NOTE: Add the public WMCS IP space when T209011 is done
    $wikimedia_nets = flatten(concat($::network::constants::aggregate_networks, '172.16.0.0/12'))
    $wikimedia_trust = flatten(concat($::network::constants::aggregate_networks, $extra_trust))

    if debian::codename::eq('bullseye') {
        # https://phabricator.wikimedia.org/T380295
        apt::package_from_component { 'lshw-backport':
            component => 'component/lshw',
            packages  => ['lshw'],
        }
    }

    # Needed profiles
    require ::profile::conftool::client
    require ::profile::prometheus::cadvisor
    require ::profile::base::production
    require ::profile::base::systemd

    # Only text uses varnishkafka (statsv ingestion)
    if $cache_cluster == 'text' {
        # FIXME: this cannot be required or it will cause a dependency cycle. It might be a good idea not to include it here
        include ::profile::cache::kafka::webrequest
        include ::profile::prometheus::varnishkafka_exporter
    }

    # Purging
    require ::profile::cache::purge

    # Globals we need to include
    include ::network::constants

    if ! $allow_iptables {
        # Prevent accidental iptables module loads
        kmod::blacklist { 'cp-bl':
            modules => ['x_tables'],
        }
    }

    class { 'conftool::scripts': }

    if $performance_tweaks {
        # Only production needs system perf tweaks
        class { '::cpufrequtils': }
        class { 'cacheproxy::performance':
            use_noflow_iface_preup => $use_noflow_iface_preup,
        }
    }

    # GeoIP data is needed for both Varnish and haproxy.
    class { 'geoip':
        load_data_from_puppetserver => $use_geo_ip,
    }

    class { 'geoip::dev': }

    # Basic varnish classes

    class { '::varnish::common':
        log_slow_request_threshold => $log_slow_request_threshold,
        logstash_host              => $logstash_host,
        logstash_json_lines_port   => $logstash_json_lines_port,
    }

    class { [
        '::varnish::common::errorpage',
        '::varnish::common::browsersec',
    ]:
    }

    if $enable_monitoring {
        class { '::varnish::common::director_scripts': }
    }

    class { '::varnish::netmapper_update_common': }

    if ( $use_ip_reputation ) {
        # Add /var/netmapper/vendor_proxies.json
        # This file is loaded in wikimedia-frontend.vcl.erb
        # lint:ignore:puppet_url_without_modules
        file { '/var/netmapper/vendor_proxies.json':
            ensure       => present,
            source       => 'puppet:///volatile/ip_reputation_vendors/proxies.json',
            before       => Service['varnish-frontend'],
            validate_cmd => '/usr/bin/vnm_validate %',
        }
    }
    # lint:endignore

    ###########################################################################
    # Analytics/Logging stuff
    ###########################################################################

    # Programs installed on both text and upload nodes
    $common_mtail_programs = ['varnishreqstats', 'varnishttfb', 'varnishxcache']

    # Programs specific to either upload or text
    if $cache_cluster == 'upload' {
        # Media browser cache hit rate and request volume stats.
        $mtail_programs = $common_mtail_programs + [ 'varnishmedia' ]
    } else {
        # ResourceLoader browser cache hit rate and request volume stats.
        $mtail_programs = $common_mtail_programs + [ 'varnishrls' ]
    }

    class { '::varnish::logging':
        default_mtail_programs  => $mtail_programs,
        internal_mtail_programs => [ 'varnishprocessing', 'varnisherrors', 'varnishsli' ],
    }

    # auto-depool on shutdown + conditional one-shot auto-pool on start
    class { 'cacheproxy::traffic_pool': }

    ###########################################################################
    # Purging
    ###########################################################################

    # Node initialization script for conftool
    if $default_weights != undef {
        class { 'conftool::scripts::initialize':
            services => $default_weights,
        }
    }
}