Puppet Class: mediabackup::worker

Defined in:
modules/mediabackup/manifests/worker.pp

Overview

SPDX-License-Identifier: Apache-2.0 Media backups worker: Install required packages and configures them.

  • sections: hash containing as keys the MediaWiki sections (as they appear on dblists to backup) and as value, another hash with the host, the port, and optionally, a dblist file name containing the list of wikis
  • mw_db_user: user used to authenticate to the mediawiki database (neads only SELECT grants on the image, oldimage and filearchive tables)
  • mw_db_password: password used to authenticate to the mediawiki database
  • dblists_path: Absolute path of where the dblists can be found.
  • mw_db_config_file: Absolute path of where the config file for database connection to MediaWiki production dbs will be stored
  • batchsize: maximum number of rows (files) of metadata to be read into memory and processed for both mw metadata and backups metadata
  • db_config_file: Absolute path of where the config file for database connection to the mediabackup metadata db will be stored
  • db_host: fqdn of the database used to write the media backups metadata backend
  • db_port: port of such database
  • db_user: user used to authenticate to the database (needs read/write grants)
  • db_password: password used to authenticate to the database
  • db_schema: name of the database inside the server where the data is read from and written to
  • storage_hosts: list of host:port where the file backend runs, where the files will be finally stored.
  • encryption_key: String used for encryption and decryption of private files Can be an age secret key or an ssh file
  • storage_root_user: identifier to authenticate on the s3-compatible api for the admin account (all privileges)
  • storage_root_password: password to authenticate on the s3-compatible api for the admin user
  • access_key: identifier to authenticate on the s3-compatible api to store the backup files (it has both read and write permissions)
  • secret_key: password to authenticate on the s3-compatible api for backing up files
  • recovery_access_key: identifier to authenticate on the s3-compatible api to restore the backup files (only has read and list permissions)
  • recovery_secret_key: password to authenticate on the s3-compatible api for recovering files

Parameters:

  • sections (Hash[String, Hash[String, Any]])
  • mw_db_user (String)
  • mw_db_password (String)
  • dblists_path (Stdlib::Unixpath)
  • mw_db_config_file (Stdlib::Unixpath)
  • batchsize (Integer[1])
  • db_config_file (Stdlib::Unixpath)
  • db_host (Stdlib::Fqdn)
  • db_port (Stdlib::Port)
  • db_user (String)
  • db_password (String)
  • storage_hosts (Array[String])
  • encryption_key (String)
  • storage_root_user (String)
  • storage_root_password (String)
  • access_key (String)
  • secret_key (String)
  • recovery_access_key (String)
  • recovery_secret_key (String)
  • db_schema (String) (defaults to: 'mediabackups') 


40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
 
# File 'modules/mediabackup/manifests/worker.pp', line 40

class mediabackup::worker (
    Hash[String, Hash[String, Any]] $sections,
    String                          $mw_db_user,
    String                          $mw_db_password,
    Stdlib::Unixpath                $dblists_path,
    Stdlib::Unixpath                $mw_db_config_file,
    Integer[1]                      $batchsize,
    Stdlib::Unixpath                $db_config_file,
    Stdlib::Fqdn                    $db_host,
    Stdlib::Port                    $db_port,
    String                          $db_user,
    String                          $db_password,
    Array[String]                   $storage_hosts,
    String                          $encryption_key,
    String                          $storage_root_user,
    String                          $storage_root_password,
    String                          $access_key,
    String                          $secret_key,
    String                          $recovery_access_key,
    String                          $recovery_secret_key,
    String                          $db_schema = 'mediabackups',
) {
    # main software
    ensure_packages([ 'mediabackups', ])

    # user and group so we don't run anything as a privileged user
    systemd::sysuser { 'mediabackup':
        home_dir => '/srv/mediabackup',
    }

    # location of temporary storage to download and hash files before
    # sending it to its final location
    file { '/srv/mediabackup':
        ensure => directory,
        mode   => '0750',
        owner  => 'mediabackup',
        group  => 'mediabackup',
    }

    # backup execution configuration dir (including secrets)
    file { '/etc/mediabackup':
        ensure => directory,
        mode   => '0400',
        owner  => 'mediabackup',
        group  => 'mediabackup',
    }

    $mw_db_ssl_ca = '/etc/ssl/certs/wmf-ca-certificates.crt'
    # list of backup source dbs to read mediawiki image metadata
    file { '/etc/mediabackup/mw_db.conf':
        ensure  => present,
        mode    => '0400',
        owner   => 'mediabackup',
        group   => 'mediabackup',
        content => template('mediabackup/mw_db.conf.erb'),
        require => File['/etc/mediabackup'],
    }
    # private data (password) and mysql connection settings
    # (minus host and port)
    file { $mw_db_config_file:
        ensure    => present,
        mode      => '0400',
        owner     => 'mediabackup',
        group     => 'mediabackup',
        content   => template('mediabackup/mw_db.ini.erb'),
        show_diff => false,
        require   => File['/etc/mediabackup'],
    }

    $db_ssl_ca = '/etc/ssl/certs/wmf-ca-certificates.crt'
    # general config for the access to a rw db to write and coordinate
    # backup metadata
    file { '/etc/mediabackup/mediabackups_db.conf':
        ensure  => present,
        mode    => '0400',
        owner   => 'mediabackup',
        group   => 'mediabackup',
        content => template('mediabackup/mediabackups_db.conf.erb'),
        require => File['/etc/mediabackup'],
    }
    # private data (password) and mysql connection settings
    file { '/etc/mediabackup/mediabackups_db.ini':
        ensure    => present,
        mode      => '0400',
        owner     => 'mediabackup',
        group     => 'mediabackup',
        content   => template('mediabackup/mediabackups_db.ini.erb'),
        show_diff => false,
        require   => File['/etc/mediabackup'],
    }

    $tmpdir = '/srv/mediabackup'
    # configuration and credentials to access final storage (S3-compatible
    # cluster on the same dc) for writing (backup generation)
    file { '/etc/mediabackup/mediabackups_storage.conf':
        ensure    => present,
        mode      => '0400',
        owner     => 'mediabackup',
        group     => 'mediabackup',
        content   => template('mediabackup/mediabackups_storage.conf.erb'),
        show_diff => false,
        require   => [ File['/etc/mediabackup'], File['/srv/mediabackup'], ],
    }

    # identity file used for encryption with age
    file { '/etc/mediabackup/encryption.key':
        ensure    => present,
        mode      => '0400',
        owner     => 'mediabackup',
        group     => 'mediabackup',
        content   => $encryption_key,
        show_diff => false,
    }

    # extra read-only policy for the recovery account
    file { '/etc/mediabackup/readandlist.json':
        ensure => present,
        owner  => 'mediabackup',
        group  => 'mediabackup',
        mode   => '0444',
        source => 'puppet:///modules/mediabackup/readandlist.json',
    }
    # extra read and deletion policy for the deletion account
    # Temporarily applied to the read only account until a separate
    # account is created for it.
    file { '/etc/mediabackup/readlistanddelete.json':
        ensure => present,
        owner  => 'mediabackup',
        group  => 'mediabackup',
        mode   => '0444',
        source => 'puppet:///modules/mediabackup/readlistanddelete.json',
    }
    # configuration and credentials to access final storage (S3-compatible
    # cluster on the same dc) for reading and listing (backup recovery)
    file { '/etc/mediabackup/mediabackups_recovery.conf':
        ensure    => present,
        mode      => '0400',
        owner     => 'mediabackup',
        group     => 'mediabackup',
        content   => template('mediabackup/mediabackups_recovery.conf.erb'),
        show_diff => false,
        require   => [
            File['/etc/mediabackup'],
            File['/srv/mediabackup'],
            File['/etc/mediabackup/readandlist.json'],
        ],
    }

    git::clone { 'operations/mediawiki-config':
        ensure    => present,
        directory => '/srv/mediawiki-config',
        owner     => 'mediabackup',
        group     => 'mediabackup',
    }
}