Puppet Class: profile::kubernetes::master

Defined in:
modules/profile/manifests/kubernetes/master.pp

Overview

Parameters:

  • etcd_urls (Any) (defaults to: hiera('profile::kubernetes::master::etcd_urls'))
  • accessible_to (Any) (defaults to: hiera('profile::kubernetes::master::accessible_to'))
  • service_cluster_ip_range (Any) (defaults to: hiera('profile::kubernetes::master::service_cluster_ip_range'))
  • service_node_port_range (Any) (defaults to: hiera('profile::kubernetes::master::service_node_port_range', undef))
  • apiserver_count (Any) (defaults to: hiera('profile::kubernetes::master::apiserver_count'))
  • storage_backend (Any) (defaults to: hiera('profile::kubernetes::master::storage_backend', 'etcd2'))
  • admission_controllers (Any) (defaults to: hiera('profile::kubernetes::master::admission_controllers'))
  • expose_puppet_certs (Any) (defaults to: hiera('profile::kubernetes::master::expose_puppet_certs'))
  • service_cert (Any) (defaults to: hiera('profile::kubernetes::master::service_cert', undef))
  • ssl_cert_path (Any) (defaults to: hiera('profile::kubernetes::master::ssl_cert_path'))
  • ssl_key_path (Any) (defaults to: hiera('profile::kubernetes::master::ssl_cert_path'))
  • authz_mode (Any) (defaults to: hiera('profile::kubernetes::master::authz_mode'))
  • service_account_private_key_file (Any) (defaults to: hiera('profile::kubernetes::master::service_account_private_key_file', undef))
  • prometheus_url (Any) (defaults to: hiera('profile::kubernetes::master::prometheus_url', "http://prometheus.svc.${::site}.wmnet/k8s"))
  • runtime_config (Any) (defaults to: hiera('profile::kubernetes::master::runtime_config', undef))


1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
# File 'modules/profile/manifests/kubernetes/master.pp', line 1

class profile::kubernetes::master(
    $etcd_urls=hiera('profile::kubernetes::master::etcd_urls'),
    # List of hosts this is accessible to.
    # SPECIAL VALUE: use 'all' to have this port be open to the world
    $accessible_to=hiera('profile::kubernetes::master::accessible_to'),
    $service_cluster_ip_range=hiera('profile::kubernetes::master::service_cluster_ip_range'),
    $service_node_port_range=hiera('profile::kubernetes::master::service_node_port_range', undef),
    $apiserver_count=hiera('profile::kubernetes::master::apiserver_count'),
    $storage_backend=hiera('profile::kubernetes::master::storage_backend', 'etcd2'),
    $admission_controllers=hiera('profile::kubernetes::master::admission_controllers'),
    $expose_puppet_certs=hiera('profile::kubernetes::master::expose_puppet_certs'),
    $service_cert=hiera('profile::kubernetes::master::service_cert', undef),
    $ssl_cert_path=hiera('profile::kubernetes::master::ssl_cert_path'),
    $ssl_key_path=hiera('profile::kubernetes::master::ssl_cert_path'),
    $authz_mode=hiera('profile::kubernetes::master::authz_mode'),
    $service_account_private_key_file=hiera('profile::kubernetes::master::service_account_private_key_file', undef),
    $prometheus_url=hiera('profile::kubernetes::master::prometheus_url', "http://prometheus.svc.${::site}.wmnet/k8s"),
    $runtime_config=hiera('profile::kubernetes::master::runtime_config', undef),
){
    if $expose_puppet_certs {
        base::expose_puppet_certs { '/etc/kubernetes':
            provide_private => true,
            user            => 'kube',
            group           => 'kube',
        }
    }

    if $service_cert {
        sslcert::certificate { $service_cert:
            ensure       => present,
            group        => 'kube',
            skip_private => false,
            before       => Class['::k8s::apiserver'],
        }
    }

    $etcd_servers = join($etcd_urls, ',')
    class { '::k8s::apiserver':
        etcd_servers             => $etcd_servers,
        ssl_cert_path            => $ssl_cert_path,
        ssl_key_path             => $ssl_key_path,
        authz_mode               => $authz_mode,
        storage_backend          => $storage_backend,
        service_cluster_ip_range => $service_cluster_ip_range,
        service_node_port_range  => $service_node_port_range,
        apiserver_count          => $apiserver_count,
        admission_controllers    => $admission_controllers,
        runtime_config           => $runtime_config,
    }

    class { '::k8s::scheduler': }
    class { '::k8s::controller':
        service_account_private_key_file => $service_account_private_key_file,
    }


    if $accessible_to == 'all' {
        $accessible_range = undef
    } else {
        $accessible_to_ferm = join($accessible_to, ' ')
        $accessible_range = "(@resolve((${accessible_to_ferm})) @resolve((${accessible_to_ferm}), AAAA))"
    }

    ferm::service { 'apiserver-https':
        proto  => 'tcp',
        port   => '6443',
        srange => $accessible_range,
    }

    # Alert us if API requests exceed a certain threshold. TODO: reevaluate
    # after we 've ran a few services
    monitoring::check_prometheus { 'apiserver_request_count':
        description     => 'k8s requests count to the API',
        query           => "scalar(sum(rate(apiserver_request_count{instance=\"${::ipaddress}:6443\"}[5m])))",
        prometheus_url  => $prometheus_url,
        warning         => 50,
        critical        => 100,
        dashboard_links => ['https://grafana.wikimedia.org/dashboard/db/kubernetes-api'],
        notes_link      => 'https://wikitech.wikimedia.org/wiki/Kubernetes',
    }
    # Alert us if API requests latencies exceed a certain threshold. TODO: reevaluate
    # thresholds
    monitoring::check_prometheus { 'apiserver_request_latencies':
        description     => 'k8s API server requests latencies',
        query           => "instance_verb:apiserver_request_latencies_summary:avg5m{verb\\!~\"(CONNECT|WATCH|WATCHLIST)\",instance=\"${::ipaddress}:6443\"}",
        prometheus_url  => $prometheus_url,
        nan_ok          => true,
        warning         => 50000,
        critical        => 100000,
        dashboard_links => ['https://grafana.wikimedia.org/dashboard/db/kubernetes-api'],
        notes_link      => 'https://wikitech.wikimedia.org/wiki/Kubernetes',
    }
    # Alert us if etcd requests latencies exceed a certain threshold. TODO: reevaluate
    # thresholds
    monitoring::check_prometheus { 'etcd_request_latencies':
        description     => 'etcd request latencies',
        query           => "instance_operation:etcd_request_latencies_summary:avg5m{instance=\"${::ipaddress}:6443\"}",
        prometheus_url  => $prometheus_url,
        nan_ok          => true,
        warning         => 30000,
        critical        => 50000,
        dashboard_links => ['https://grafana.wikimedia.org/dashboard/db/kubernetes-api'],
        notes_link      => 'https://wikitech.wikimedia.org/wiki/Etcd/Main_cluster',
    }
}