1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
|
# File 'modules/dumps/manifests/web/enterprise.pp', line 1
class dumps::web::enterprise(
$is_primary_server = false,
$dumps_web_server = undef,
$user = undef,
$group = undef,
$miscdumpsdir = undef,
){
$script_path = '/usr/local/bin/wm_enterprise_downloader.py'
file { $script_path:
ensure => 'present',
mode => '0644',
owner => 'root',
group => 'root',
source => 'puppet:///modules/dumps/web/wm_enterprise_downloader.py',
}
$creds_path = '/etc/dumps/wm_enterprise_creds'
file { $creds_path:
ensure => 'present',
mode => '0640',
owner => $user,
group => $group,
content => secret('dumps/wm_enterprise_creds'),
}
$settings_path = '/etc/dumps/wm_enterprise_settings'
file { $settings_path:
ensure => 'present',
mode => '0644',
owner => 'root',
group => 'root',
source => 'puppet:///modules/dumps/web/wm_enterprise_settings',
}
file { '/srv/dumps/temp':
ensure => 'directory',
mode => '0755',
owner => $user,
group => $group,
}
$download_command = "/usr/bin/python3 ${script_path} --creds ${creds_path} --settings ${settings_path} --retries 5"
systemd::timer::job { 'download_enterprise_htmldumps':
ensure => $is_primary_server.bool2str('present', 'absent'),
description => 'Twice monthly download of Wikimedia Enterprise HTML dumps',
user => $user,
monitoring_enabled => false,
send_mail => true,
send_mail_only_on_error => false,
environment => {'MAILTO' => 'ops-dumps@wikimedia.org'},
command => $download_command,
interval => {'start' => 'OnCalendar', 'interval' => '*-*-1,20 8:30:0'},
require => [ File[$script_path], File[$creds_path], File[$settings_path] ],
}
# rsync the downloaded files to secondary host, allowing the rsync to take a full day
$rsync_command = "/usr/bin/rsync -a --bwlimit=160000 ${dumps_web_server}::data/xmldatadumps/public/other/enterprise_html/runs ${miscdumpsdir}/enterprise_html/"
systemd::timer::job { 'rsync_enterprise_htmldumps':
ensure => $is_primary_server.bool2str('absent', 'present'),
description => 'Twice monthly rsync after download of Wikimedia Enterprise HTML dumps',
user => root,
monitoring_enabled => false,
send_mail => true,
environment => {'MAILTO' => 'ops-dumps@wikimedia.org'},
command => $rsync_command,
interval => {'start' => 'OnCalendar', 'interval' => '*-*-2,4,6,21,23,25 8:30:0'},
require => [ File[$script_path], File[$creds_path], File[$settings_path] ],
}
}
|