Top Level Namespace

Includes:
RspecPuppetFacts

Defined Under Namespace

Modules: DomainRedirects, Helpers, Kernel, Matchers, Puppet, PuppetDB, PuppetSyntax, SharedData Classes: AlsoString, Apr1Md5, BasicTTLCache, Class, DNSCacheEntry, DNSCached, GitOps, Hiera, Object, PuppetECDSAGen, PuppetECDSAGenError, PybalError, SpecDependencies, String, TaskGen, WMFConfig

Constant Summary collapse

DIR =
ENV['PUPPETDIR'] || File.expand_path('../..', __FILE__)
TEST_ON =
{
  supported_os: [
    {'operatingsystem' => 'Debian',
     'operatingsystemrelease' => ['9', '10']}
  ]
}
PATH_ROOT =
'/usr/local/bin/'
UNSUPPORTED_PLATFORMS =
[].freeze
Log =
Logger.new(STDOUT)

Instance Method Summary collapse

Instance Method Details

#check_pooled_state(ip, port, pool, host, want_pooled) ⇒ Object



70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
# File 'modules/conftool/files/pooler_loop.rb', line 70

def check_pooled_state(ip, port, pool, host, want_pooled)
  # Manage down or unresponsive pybals
  http = Net::HTTP.new(ip, port)
  http.open_timeout = 1
  http.read_timeout = 2

  resp = http.start do |http|
    http.get "/pools/#{pool}/#{host}"
  end

  # ignore 404s
  # rubocop:disable Style/CaseEquality
  return true unless resp === Net::HTTPSuccess
  # rubocop:enable Style/CaseEquality
  enabled, active, pooled = resp.body.strip.split '/'
  if want_pooled
    # A pooled server must be enabled/up/pooled
    # anything else is not ok
    if enabled != 'enabled'
      raise PybalError, "#{host} not logically pooled, lb #{ip}"
    elsif active != 'up'
      raise PybalError, "The service is not up on #{host}, lb #{ip}"
    elsif pooled != 'pooled'
      raise PybalError, "Service not pooled"
    end
  else
    # A disabled server should be disabled/{up,down}/not pooled
    if enabled == 'enabled'
      raise PybalError, "#{host} is still logically pooled, lb #{ip}"
    elsif pooled == 'pooled'
      raise PybalError, "#{host} is still pooled, maybe too many depooled? lb #{ip}"
    end
  end
end

#config_to_hash(conf) ⇒ Object

Function: merge_config(string|hash main_conf, string|hash service_conf)

Merges the service-specific service_conf into main_conf. Both arguments can be either hashes or YAML-formatted strings. It returns the merged configuration hash.



8
9
10
11
# File 'modules/service/lib/puppet/parser/functions/merge_config.rb', line 8

def config_to_hash(conf)
  return YAML.load(conf) unless conf.is_a?(Hash)
  conf
end

#configparser_format(config) ⇒ Object



17
18
19
20
21
22
23
24
25
26
27
28
29
30
# File 'modules/graphite/lib/puppet/parser/functions/configparser_format.rb', line 17

def configparser_format(config)
  # Serialize a hash to Python ConfigParser format.
  config.sort.map { |section, items|
    ["[#{section}]"].concat items.sort.map { |k, v|
      if v.is_a?(Array)
        v = v.join(',')
      else
        v = v == :undef ? '' : v
      end

      "#{k} = #{v}"
    }.push []
  }.join("\n")
end

#count_pending_tests_in(group) ⇒ Object



62
63
64
# File 'modules/stdlib/spec/acceptance/build_csv.rb', line 62

def count_pending_tests_in(group)
  count_test_types_in('pending_tests', group)
end

#count_test_types_in(type, group) ⇒ Object



49
50
51
52
53
54
55
56
# File 'modules/stdlib/spec/acceptance/build_csv.rb', line 49

def count_test_types_in(type, group)
  return 0 if group.nil?
  group.reduce(0) do |m, (k, v)|
    m += v.length if k == type
    m += count_tests_in(v) if v.is_a?(Hash)
    m
  end
end

#count_tests_in(group) ⇒ Object



58
59
60
# File 'modules/stdlib/spec/acceptance/build_csv.rb', line 58

def count_tests_in(group)
  count_test_types_in('tests', group)
end

#dedent_string(string) ⇒ Object



44
45
46
47
48
49
50
# File 'modules/wmflib/lib/puppet/parser/functions/ordered_yaml.rb', line 44

def dedent_string(string)
  lines = string.split("\n")
  return string if lines.empty?
  min_indent = lines.map { |line| line.start_with?(" ") ? line.match(/^ +/).offset(0)[1] : 0 }.min
  return string if min_indent.zero?
  lines.map { |line| line.gsub(/^ {#{min_indent}}/, "") }.join("\n")
end

#determine_scopeObject

Determine the scope from the command line arguments



40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
# File 'utils/hiera_lookup.rb', line 40

def determine_scope
  # Transform the CL arguments in a dictionary in the form "::option" => argument
  scope = Hash[ARGV.map { |kv| "::#{kv.sub('--', '')}".split('=') }]
  abort(usage "Error: --fqdn=FQDN is required.") unless scope['::fqdn']
  scope['::hostname'] = scope['::fqdn'][/^[^.]+/]
  if scope['::fqdn'].end_with?('.wmflabs') || scope['::fqdn'].end_with?('.labtest') || scope['::fqdn'].end_with?('.cloud')
    scope['::realm'] = 'labs'
  else
    scope['::realm'] = 'production'
  end

  # Detect labs project/site
  if scope['::realm'] == 'labs'
    bits = scope['::fqdn'].split('.')
    unless bits.length == 4
      Kernel.abort('labs FQDN must be <hostname>.<project>.<site>.(wmflabs|labtest|cloud)')
    end
    scope['::labsproject'] = bits[1]
    scope['::site'] = bits[2]
  end

  # Attempt at determining the site
  scope['::site'] ||= scope['::fqdn'].split('.')[-2]
  if scope['::site'] == 'wikimedia' || scope['::site'].nil?
    abort(usage "\nError: --site=SITE is required if the FQDN doesn't contain it")
  end

  # Transform roles in the form puppet expects
  if scope['::roles']
    scope['_roles'] = Hash[scope['::roles'].split(',').map { |role| [role.gsub('role::', ''), true] }]
    # Mimic the changes in the role() function
    scope['::_role'] = scope['_roles'].keys[0].gsub(/::/, '/')
    scope.delete('::roles')
  end
  scope
end

#fact(*_args) ⇒ Object



14
15
16
# File 'modules/stdlib/spec/acceptance/build_csv.rb', line 14

def fact(*_args)
  []
end

#filter(event) ⇒ Object



11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
# File 'modules/profile/files/logstash/filter_scripts/loki_event.rb', line 11

def filter(event)
  # event timestamp to nanoseconds
  ts = (event.get('@timestamp').to_i * 1e9).to_s.split('.')[0]

  case @event_type
  when 'alert'
    stream = { :type => 'alert', :host => event.get('host')}
    values = [[ts, "#{event.get('icinga_state')} -- #{event.get('icinga_check_descr')}: #{event.get('icinga_message')}"]]
  when 'sal'
    stream = { :type => 'sal', :project => event['project'] }
    values = [[ts, "#{event.get('nick')}: #{event.get('message')}"]]
  when 'deploy'
    stream = { :type => 'deploy' }
    values = [[ts, "#{event.get('user')}: #{event.get('message')}"]]
  else
    # do nothing
    return [event]
  end

  event.set('streams', [{ :stream => stream, :values => values }])
  [event]
end

#get_tests(children) ⇒ Object

Collect up tests and example groups into a hash



33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
# File 'modules/stdlib/spec/acceptance/build_csv.rb', line 33

def get_tests(children)
  children.each_with_object({}) do |c, memo|
    memo[c.description] = {}
    memo[c.description]['groups'] = get_tests(c.children) unless c.children.empty?
    unless c.examples.empty?
      memo[c.description]['tests'] = c.examples.map { |e|
        e.description unless e.pending?
      }.compact
    end
    next if c.examples.empty?
    memo[c.description]['pending_tests'] = c.examples.map { |e|
      e.description if e.pending?
    }.compact
  end
end

#ini_cast(v) ⇒ Object



24
25
26
27
28
# File 'modules/wmflib/lib/puppet/parser/functions/ini.rb', line 24

def ini_cast(v)
    v.include?('.') ? Float(v) : Integer(v)
  rescue
    v
end

#ini_flatten(map, prefix = nil) ⇒ Object

Function: php_ini( hash $ini_settings [, hash $… ] )

Serialize a hash into php.ini-style format. Takes one or more hashes as arguments. If the argument list contains more than one hash, they are merged together. In case of duplicate keys, hashes to the right win.

Example

php_ini({'server' => {'port' => 80}}) # => server.port = 80


17
18
19
20
21
22
# File 'modules/wmflib/lib/puppet/parser/functions/ini.rb', line 17

def ini_flatten(map, prefix = nil)
  map.reduce({}) do |flat, (k, v)|
    k = [prefix, k].compact.join('.')
    flat.merge! v.is_a?(Hash) ? ini_flatten(v, k) : Hash[k, v]
  end
end

#install_modules(host, modules) ⇒ Object



16
17
18
19
20
21
22
23
24
# File 'modules/cfssl/spec/spec_helper_acceptance.rb', line 16

def install_modules(host, modules)
  module_root = File.expand_path(File.join(__dir__, '..'))
  install_dev_puppet_module_on(
    host, source: module_root, module_name: File.basename(module_root))
  modules.each do |m|
    source = File.expand_path(File.join(module_root, '..', m))
    install_dev_puppet_module_on(host, source: source, module_name: m)
  end
end

#is_correct_format?(agent, volume_group, logical_volume, format_type) ⇒ Boolean

Verify if a filesystem resource type is successfully created

Attributes

  • volume_group - resorce type name, i.e 'VolumeGroup_1234'

  • logical_volume - resorce type name, i.e 'LogicalVolume_a2b3'

  • fromat_type - type of the format of the logical volume, i.e 'ext3'

Returns

nil

Raises

assert_match failure message

Examples

is_correct_format?(agent, VolumeGroup_1234, LogicalVolume_a2b3, ext3)

Returns:

  • (Boolean)


73
74
75
76
77
# File 'modules/lvm/tests/beaker/lib/lvm_helper.rb', line 73

def is_correct_format?(agent, volume_group, logical_volume, format_type)
  on(agent, "file -sL /dev/#{volume_group}/#{logical_volume}") do |result|
    assert_match(/#{format_type}/, result.stdout, "Unexpected error was detected")
  end
end

#nodes_listObject



53
54
55
56
# File 'modules/base/lib/facter/numa.rb', line 53

def nodes_list
  Pathname.glob('/sys/devices/system/node/node[0-9]*')
          .map { |n| /([0-9]+)$/.match(n.to_s)[0].to_i }.sort
end

#nodes_to_htsets(nodes) ⇒ Object



58
59
60
61
62
63
64
65
66
67
68
69
# File 'modules/base/lib/facter/numa.rb', line 58

def nodes_to_htsets(nodes)
  tsl_glob = 'cpu[0-9]*/topology/thread_siblings_list'
  node_to_htset = {}
  nodes.each do |n|
    node_to_htset[n] =
      Pathname.glob("/sys/devices/system/node/node#{n}/#{tsl_glob}")
              .map do |c|
                File.open(c).read.split(',').map(&:to_i).sort
              end.sort.uniq
  end
  node_to_htset
end

#ordered_json(o) ⇒ Object

Function: ordered_json( hash $data [, hash $… ] )

Serialize a hash into JSON with lexicographically sorted keys.

Because the order of keys in Ruby 1.8 hashes is undefined, 'to_pson' is not idempotent: i.e., the serialized form of the same hash object can vary from one invocation to the next. This causes problems whenever a JSON-serialized hash is included in a file template, because the variations in key order are picked up as file updates by Puppet, causing Puppet to replace the file and refresh dependent resources on every run.

Examples

# Render a Puppet hash as a configuration file:
$options = { 'useGraphite' => true, 'minVal' => '0.1' }
file { '/etc/kibana/config.json':
  content => ordered_json($options),
}


21
22
23
24
25
26
27
28
29
30
31
32
33
34
# File 'modules/wmflib/lib/puppet/parser/functions/ordered_json.rb', line 21

def ordered_json(o)
  case o
  when Array
    '[' + o.map { |x| ordered_json(x) }.join(', ') + ']'
  when Hash
    '{' + o.sort.map { |k, v| k.to_pson + ': ' + ordered_json(v) }.join(', ') + '}'
  else
    begin
      o.include?('.') ? Float(o).to_s : Integer(o).to_s
    rescue
      o.to_pson
    end
  end
end

#param_value(subject, type, title, param) ⇒ Object



5
6
7
# File 'modules/nginx/spec/spec_helper.rb', line 5

def param_value(subject, type, title, param)
  subject.resource(type, title).send(:parameters)[param.to_sym]
end

#phpdump(o, level = 1) ⇒ Object

Function: phpdump

Serialize a hash into PHP array with lexicographically sorted keys.



6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
# File 'modules/librenms/lib/puppet/parser/functions/phpdump.rb', line 6

def phpdump(o, level = 1)
  indent = " " * 4

  case o
  when Hash
    contents = ''
    o.sort.each do |k, v|
      contents += indent * level
      contents += k.to_pson + " => " + phpdump(v, level + 1)
      contents += ",\n"
    end
    "array(\n" + contents + indent * (level - 1) + ")"
  when Array
    "array(" + o.map { |x| phpdump(x, level + 1) }.join(', ') + ")"
  when TrueClass
    "TRUE"
  when FalseClass
    "FALSE"
  when nil
    "NULL"
  else
    begin
      o.include?('.') ? Float(o).to_s : Integer(o).to_s
    rescue
      o.to_pson
    end
  end
end

#register(params) ⇒ Object



7
8
9
# File 'modules/profile/files/logstash/filter_scripts/loki_event.rb', line 7

def register(params)
  @event_type = params["type"]
end

#remove_all(agent, pv = nil, vg = nil, lv = nil, aix = false) ⇒ Object

Clean the box after each test, make sure the newly created logical volumes, volume groups, and physical volumes are removed at the end of each test to make the server ready for the next test case.

Attributes

  • pv - physical volume, can be one volume or an array of multiple volumes

  • vg - volume group, can be one group or an array of multiple volume groups

  • lv - logical volume, can be one volume or an array of multiple volumes

  • aix - if the agent is an AIX server.

Returns

nil

Raises

nil

Examples

remove_all(agent, '/dev/sdb', 'VolumeGroup_1234', 'LogicalVolume_fa13')



99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
# File 'modules/lvm/tests/beaker/lib/lvm_helper.rb', line 99

def remove_all(agent, pv=nil, vg=nil, lv=nil, aix=false)
  if aix
    step 'remove aix volume group, physical/logical volume '
    on(agent, "reducevg -d -f #{vg} #{pv}")
    on(agent, "rm -rf /dev/#{vg} /dev/#{lv}")
  else
    step 'remove logical volume if any:'
    if lv
      if lv.kind_of?(Array)
        lv.each do |logical_volume|
          on(agent, "umount /dev/#{vg}/#{logical_volume}", :acceptable_exit_codes => [0,1])
          on(agent, "lvremove /dev/#{vg}/#{logical_volume} --force")
        end
      else
        #note: in some test cases, for example, the test case 'create_vg_property_logical_volume'
        # the logical volume must be unmount before being able to delete it
        on(agent, "umount /dev/#{vg}/#{lv}", :acceptable_exit_codes => [0,1])
        on(agent, "lvremove /dev/#{vg}/#{lv} --force")
      end
    end

    step 'remove volume group if any:'
    if vg
      if vg.kind_of?(Array)
        vg.each do |volume_group|
          on(agent, "vgremove /dev/#{volume_group}")
        end
      else
        on(agent, "vgremove /dev/#{vg}")
      end
    end

    step 'remove logical volume if any:'
    if pv
      if pv.kind_of?(Array)
        pv.each do |physical_volume|
          on(agent, "pvremove #{physical_volume}")
        end
      else
        on(agent, "pvremove #{pv}")
      end
    end
  end
end

#return_puppet_versionObject



20
21
22
# File 'modules/stdlib/spec/spec_helper_acceptance.rb', line 20

def return_puppet_version
  (on default, puppet('--version')).output.chomp
end

#rmerge(*args) ⇒ Object

Function: configparser_format

Serialize a hash to Python ConfigParser format. See <docs.python.org/2/library/configparser.html>



6
7
8
9
10
11
12
13
14
15
# File 'modules/graphite/lib/puppet/parser/functions/configparser_format.rb', line 6

def rmerge(*args)
  # Recursively merge hashes.
  merged = args.shift.clone
  args.each do |hash|
    merged.merge!(hash) do |k, old, new|
      merged[k] = old.is_a?(Hash) && new.is_a?(Hash) ? rmerge(old, new) : new
    end
  end
  merged
end

#sort_keys_recursive(value) ⇒ Object



15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
# File 'modules/wmflib/lib/puppet/parser/functions/ordered_yaml.rb', line 15

def sort_keys_recursive(value)
  # Prepare a value for YAML serialization by sorting its keys (if it is
  # a hash) and the keys of any hash object that is contained within the
  # value. Returns a new value.
  case value
  when Array
    value.map { |elem| sort_keys_recursive(elem) }
  when Hash
    map = {}
    def map.each_pair
      map.sort.each { |p| yield p }
    end
    value.sort.reduce(map) { |h, (k, v)|
      h[k] = sort_keys_recursive(v)
      h
    }
  when 'true', 'false'
    value == 'true'
  when :undef
    nil
  else
    begin
      value.include?('.') ? Float(value) : Integer(value)
    rescue
      value
    end
  end
end

#to_csv(function_list, tests) ⇒ Object

Convert tests hash to csv format



67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
# File 'modules/stdlib/spec/acceptance/build_csv.rb', line 67

def to_csv(function_list, tests)
  function_list.map { |function_name|
    v = tests["#{function_name} function"]
    if v
      positive_tests = count_tests_in(v['groups']['success'])
      negative_tests = count_tests_in(v['groups']['failure'])
      pending_tests  =
        count_pending_tests_in(v['groups']['failure']) +
        count_pending_tests_in(v['groups']['failure'])
    else
      positive_tests = 0
      negative_tests = 0
      pending_tests  = 0
    end
    '%-25s, %-9d, %-9d, %-9d' % [function_name, positive_tests, negative_tests, pending_tests]
  }.compact
end

#usage(error = '') ⇒ Object

hiera_lookup: hiera lookup tool



5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
# File 'utils/hiera_lookup.rb', line 5

def usage(error = '')
  msg = <<-end
hiera_lookup: hiera lookup tool

Usage:
     hiera_lookup [OPTIONS] KEY

Mandatory arguments:

  --fqdn=<FQDN>  Fully Qualified Domain Name
  KEY            Key to lookup in hiera

Optional arguments:

  -v                          Verbose mode
  --site=<SITE>               Wikimedia site (ex: eqiad)
  --roles=<ROLE>[,<ROLE>]...  Puppet role(s) class(es)
  -h, --help                  Display this help and exit

If SITE is not provided, it will be derived from the FQDN whenever possible.
For Wmflabs a FQDN should be: <hostname>.<project>.<site>.wmflabs

Examples:
  hiera_lookup --fqdn=mw1020.eqiad.wmnet --roles=mediawiki::appserver admin::groups
  hiera_lookup --fqdn=host.tools.eqiad1.wikimedia.cloud classes


end
  msg << error
end

#verify_if_created?(agent, resource_type, resource_name, vg = nil, properties = nil) ⇒ Boolean

Verify if a physical volume, volume group, logical volume, or filesystem resource type is created

Attributes

  • resource_type - resorce type, i.e 'physical_volume', 'volume_group', 'logical_volume', 'filesystem',

  • 'aix_physical_volume', 'aix_volume_group', or 'aix_logical_volume'.

  • resource_name - The name of resource type, i.e '/dev/sdb' for physical volume, vg_1234 for volume group

  • vg - volume group name associated with logical volume (if any)

  • properties - a matching string or regular expression in logical volume properties

Returns

nil

Raises

assert_match failure message

Examples

verify_if_created?(agent, 'physical_volume', /dev/sdb', VolumeGroup_123, “Size 7GB”)

Returns:

  • (Boolean)


19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
# File 'modules/lvm/tests/beaker/lib/lvm_helper.rb', line 19

def verify_if_created?(agent, resource_type, resource_name, vg=nil, properties=nil)
  case resource_type
    when 'physical_volume'
      on(agent, "pvdisplay") do |result|
        assert_match(/#{resource_name}/, result.stdout, 'Unexpected error was detected')
      end
    when 'volume_group'
      on(agent, "vgdisplay") do |result|
        assert_match(/#{resource_name}/, result.stdout, 'Unexpected error was detected')
      end
    when 'logical_volume'
      raise ArgumentError, 'Missing volume group that the logical volume is associated with' unless vg
      on(agent, "lvdisplay /dev/#{vg}/#{resource_name}") do |result|
        assert_match(/#{resource_name}/, result.stdout, 'Unexpected error was detected')
        if properties
          assert_match(/#{properties}/, result.stdout, 'Unexpected error was detected')
        end
      end
    when 'aix_physical_volume'
      on(agent, "lspv #{resource_name}") do |result|
        assert_match(/Physical volume #{resource_name} is not assigned to/, result.stdout, 'Unexpected error was detected')
      end
    when 'aix_volume_group'
      on(agent, "lsvg") do |result|
        assert_match(/#{resource_name}/, result.stdout, 'Unexpected error was detected')
      end
    when 'aix_logical_volume'
      raise ArgumentError, 'Missing volume group that the logical volume is associated with' unless vg
      on(agent, "lslv #{resource_name}") do |result|
        assert_match(/#{resource_name}/, result.stdout, 'Unexpected error was detected')
        if properties
          assert_match(/#{properties}/, result.stdout, 'Unexpected error was detected')
        end
      end
  end
end