Puppet Class: profile::statistics::explorer::misc_jobs

Defined in:
modules/profile/manifests/statistics/explorer/misc_jobs.pp

Overview

Class profile::statistics::explorer::misc_jobs

This class is meant to collect crons/timers/scripts from various teams (Discovery, WMDE, etc..) that used to be run only on stat1007. Ideally in the future these jobs will be run on a dedicated VM or similar.

Parameters:

  • statsd_host (String) (defaults to: lookup('statsd'))
  • labstore_hosts (Array[Stdlib::Host]) (defaults to: lookup('labstore_hosts'))
  • graphite_host (Stdlib::Host) (defaults to: lookup('graphite_host'))
  • wmde_secrets (Hash[String,String]) (defaults to: lookup('wmde_secrets'))
  • use_kerberos (Boolean) (defaults to: lookup('profile::statistics::explorer::misc_jobs::use_kerberos'))
  • hosts_with_jobs (Array[String]) (defaults to: lookup('profile::statistics::explorer::misc_jobs::hosts_with_jobs'))


8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
# File 'modules/profile/manifests/statistics/explorer/misc_jobs.pp', line 8

class profile::statistics::explorer::misc_jobs(
    String              $statsd_host     = lookup('statsd'),
    Array[Stdlib::Host] $labstore_hosts  = lookup('labstore_hosts'),
    Stdlib::Host        $graphite_host   = lookup('graphite_host'),
    Hash[String,String] $wmde_secrets    = lookup('wmde_secrets'),
    Boolean             $use_kerberos    = lookup('profile::statistics::explorer::misc_jobs::use_kerberos'),
    Array[String]       $hosts_with_jobs = lookup('profile::statistics::explorer::misc_jobs::hosts_with_jobs'),
) {

    if $::hostname in $hosts_with_jobs {
        # Discovery team statistics scripts and cron jobs
        class { 'statistics::discovery':
            use_kerberos => $use_kerberos
        }

        # Performance team statistics scripts and cron jobs
        class { 'statistics::performance':
            use_kerberos => $use_kerberos
        }

        # WMDE releated statistics & analytics scripts.
        class { 'statistics::wmde':
            statsd_host   => $statsd_host,
            graphite_host => $graphite_host,
            wmde_secrets  => $wmde_secrets,
        }

        # Systemd timers owned by the Search team
        # (leveraging Analytics' refinery)
        include profile::analytics::search::jobs

        # Allowing statistics nodes (mostly clouddb hosts in this case)
        # to push nginx access logs to a specific /srv path. We usually
        # allow only pull based rsyncs, but after T211330 we needed a way
        # to unbreak that use case. This rsync might be removed in the future.
        # TODO: this should be moved to hdfs-rsync.
        rsync::server::module { 'dumps-webrequest':
            path        => '/srv/log/webrequest/archive/dumps.wikimedia.org',
            read_only   => 'no',
            hosts_allow => $labstore_hosts,
            auto_ferm   => true,
        }
    }
}