1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
|
# File 'modules/snapshot/manifests/dumps/stagesconfig.pp', line 1
class snapshot::dumps::stagesconfig {
$confsdir = $snapshot::dumps::dirs::confsdir
$firststage_args = '--cutoff {STARTDATE} --date {STARTDATE}'
$rest_args = '--date {STARTDATE} --onepass --prereqs'
$wikiargs = '/bin/bash ./worker --skipdone --exclusive --log'
$args_smallwikis = "${wikiargs} --configfile ${confsdir}/wikidump.conf.dumps"
$args_bigwikis = "${wikiargs} --configfile ${confsdir}/wikidump.conf.dumps:bigwikis"
$args_enwiki = "${wikiargs} --configfile ${confsdir}/wikidump.conf.dumps:en"
$args_wikidatawiki = "${wikiargs} --configfile ${confsdir}/wikidump.conf.dumps:wd"
$jobs_to_skip = join(['metahistorybz2dump',
'metahistorybz2dumprecombine',
'metahistory7zdump',
'metahistory7zdumprecombine',
'xmlflowhistorydump'], ',')
$stages = {
smallwikis => {
firststage => "${args_smallwikis} ${firststage_args}",
rest => "${args_smallwikis} ${rest_args}",
},
bigwikis => {
firststage => "${args_bigwikis} ${firststage_args}",
rest => "${args_bigwikis} ${rest_args}",
},
enwiki => {
firststage => "${args_enwiki} ${firststage_args}",
rest => "${args_enwiki} ${rest_args}",
},
wikidatawiki => {
firststage => "${args_wikidatawiki} ${firststage_args}",
rest => "${args_wikidatawiki} ${rest_args}",
},
skipjob_args => "--skipjobs ${jobs_to_skip}",
}
snapshot::dumps::stagesconf { 'stages_full':
stagestype => 'full',
stages => $stages,
}
snapshot::dumps::stagesconf { 'stages_partial':
stagestype => 'partial',
stages => $stages,
}
snapshot::dumps::stagesconf { 'stages_full_enwiki':
stagestype => 'full_enwiki',
stages => $stages,
}
snapshot::dumps::stagesconf { 'stages_partial_enwiki':
stagestype => 'partial_enwiki',
stages => $stages,
}
snapshot::dumps::stagesconf { 'stages_full_wikidatawiki':
stagestype => 'full_wikidatawiki',
stages => $stages,
}
snapshot::dumps::stagesconf { 'stages_partial_wikidatawiki':
stagestype => 'partial_wikidatawiki',
stages => $stages,
}
snapshot::dumps::stagesconf { 'stages_create_smallwikis':
stagestype => 'create_small',
stages => $stages,
}
snapshot::dumps::stagesconf { 'stages_create_bigwikis':
stagestype => 'create_big',
stages => $stages,
}
snapshot::dumps::stagesconf { 'stages_create_enwiki':
stagestype => 'create_enwiki',
stages => $stages,
}
snapshot::dumps::stagesconf { 'stages_create_wikidatawiki':
stagestype => 'create_wikidatawiki',
stages => $stages,
}
}
|