Puppet Class: dumps::web::enterprise

Defined in:
modules/dumps/manifests/web/enterprise.pp

Overview

Parameters:

  • is_primary_server (Any) (defaults to: false)
  • dumps_web_server (Any) (defaults to: undef)
  • user (Any) (defaults to: undef)
  • group (Any) (defaults to: undef)
  • miscdumpsdir (Any) (defaults to: undef)


1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
# File 'modules/dumps/manifests/web/enterprise.pp', line 1

class dumps::web::enterprise(
    $is_primary_server = false,
    $dumps_web_server = undef,
    $user = undef,
    $group = undef,
    $miscdumpsdir = undef,
){
    $script_path = '/usr/local/bin/wm_enterprise_downloader.py'
    file { $script_path:
        ensure => 'present',
        mode   => '0644',
        owner  => 'root',
        group  => 'root',
        source => 'puppet:///modules/dumps/web/wm_enterprise_downloader.py',
    }

    $creds_path = '/etc/dumps/wm_enterprise_creds'
    file { $creds_path:
        ensure  => 'present',
        mode    => '0640',
        owner   => $user,
        group   => $group,
        content => secret('dumps/wm_enterprise_creds'),
    }

    $settings_path = '/etc/dumps/wm_enterprise_settings'
    file { $settings_path:
        ensure => 'present',
        mode   => '0644',
        owner  => 'root',
        group  => 'root',
        source => 'puppet:///modules/dumps/web/wm_enterprise_settings',
    }

    file { '/srv/dumps/temp':
        ensure => 'directory',
        mode   => '0755',
        owner  => $user,
        group  => $group,
    }

    if $is_primary_server {
        $download_command = "/usr/bin/python3 ${script_path} --creds ${creds_path} --settings ${settings_path} --retries 2"
        systemd::timer::job { 'download_enterprise_htmldumps':
            ensure                  => present,
            description             => 'Twice monthly download of Wikimedia Enterprise HTML dumps',
            user                    => $user,
            monitoring_enabled      => false,
            send_mail               => true,
            send_mail_only_on_error => false,
            environment             => {'MAILTO' => 'ops-dumps@wikimedia.org'},
            command                 => $download_command,
            interval                => {'start' => 'OnCalendar', 'interval' => '*-*-1,20 8:30:0'},
            require                 => [ File[$script_path], File[$creds_path], File[$settings_path] ],
        }
    }

    if ($is_primary_server == false) {
        # rsync the downloaded files to secondary host, allowing the rsync to take a full day
        $rsync_command = "/usr/bin/rsync -a --bwlimit=160000 ${dumps_web_server}::data/xmldatadumps/public/other/enterprise_html/runs ${miscdumpsdir}/enterprise_html/"
        systemd::timer::job { 'rsync_enterprise_htmldumps':
            ensure             => present,
            description        => 'Twice monthly rsync after download of Wikimedia Enterprise HTML dumps',
            user               => root,
            monitoring_enabled => false,
            send_mail          => true,
            environment        => {'MAILTO' => 'ops-dumps@wikimedia.org'},
            command            => $rsync_command,
            interval           => {'start' => 'OnCalendar', 'interval' => '*-*-2,21 8:30:0'},
            require            => [ File[$script_path], File[$creds_path], File[$settings_path] ],
        }
    }
}