379 lines
13 KiB
Python
379 lines
13 KiB
Python
# Lint as: python2, python3
|
|
"""This module gives the mkfs creation options for an existing filesystem.
|
|
|
|
tune2fs or xfs_growfs is called according to the filesystem. The results,
|
|
filesystem tunables, are parsed and mapped to corresponding mkfs options.
|
|
"""
|
|
from __future__ import absolute_import
|
|
from __future__ import division
|
|
from __future__ import print_function
|
|
|
|
import os, re, tempfile
|
|
|
|
import six
|
|
|
|
import common
|
|
from autotest_lib.client.common_lib import error, utils
|
|
|
|
|
|
def opt_string2dict(opt_string):
|
|
"""Breaks the mkfs.ext* option string into dictionary."""
|
|
# Example string: '-j -q -i 8192 -b 4096'. There may be extra whitespaces.
|
|
opt_dict = {}
|
|
|
|
for item in opt_string.split('-'):
|
|
item = item.strip()
|
|
if ' ' in item:
|
|
(opt, value) = item.split(' ', 1)
|
|
opt_dict['-%s' % opt] = value
|
|
elif item != '':
|
|
opt_dict['-%s' % item] = None
|
|
# Convert all the digit strings to int.
|
|
for key, value in six.iteritems(opt_dict):
|
|
if value and value.isdigit():
|
|
opt_dict[key] = int(value)
|
|
|
|
return opt_dict
|
|
|
|
|
|
def parse_mke2fs_conf(fs_type, conf_file='/etc/mke2fs.conf'):
|
|
"""Parses mke2fs config file for default settings."""
|
|
# Please see /ect/mke2fs.conf for an example.
|
|
default_opt = {}
|
|
fs_opt = {}
|
|
current_fs_type = ''
|
|
current_section = ''
|
|
f = open(conf_file, 'r')
|
|
for line in f:
|
|
if '[defaults]' == line.strip():
|
|
current_section = '[defaults]'
|
|
elif '[fs_types]' == line.strip():
|
|
current_section = '[fs_types]'
|
|
elif current_section == '[defaults]':
|
|
components = line.split('=', 1)
|
|
if len(components) == 2:
|
|
default_opt[components[0].strip()] = components[1].strip()
|
|
elif current_section == '[fs_types]':
|
|
m = re.search('(\w+) = {', line)
|
|
if m:
|
|
current_fs_type = m.group(1)
|
|
else:
|
|
components = line.split('=', 1)
|
|
if len(components) == 2 and current_fs_type == fs_type:
|
|
default_opt[components[0].strip()] = components[1].strip()
|
|
f.close()
|
|
|
|
# fs_types options override the defaults options
|
|
for key, value in six.iteritems(fs_opt):
|
|
default_opt[key] = value
|
|
|
|
# Convert all the digit strings to int.
|
|
for key, value in six.iteritems(default_opt):
|
|
if value and value.isdigit():
|
|
default_opt[key] = int(value)
|
|
|
|
return default_opt
|
|
|
|
|
|
def convert_conf_opt(default_opt):
|
|
conf_opt_mapping = {'blocksize': '-b',
|
|
'inode_ratio': '-i',
|
|
'inode_size': '-I'}
|
|
mkfs_opt = {}
|
|
|
|
# Here we simply concatenate the feature string while we really need
|
|
# to do the better and/or operations.
|
|
if 'base_features' in default_opt:
|
|
mkfs_opt['-O'] = default_opt['base_features']
|
|
if 'default_features' in default_opt:
|
|
mkfs_opt['-O'] += ',%s' % default_opt['default_features']
|
|
if 'features' in default_opt:
|
|
mkfs_opt['-O'] += ',%s' % default_opt['features']
|
|
|
|
for key, value in six.iteritems(conf_opt_mapping):
|
|
if key in default_opt:
|
|
mkfs_opt[value] = default_opt[key]
|
|
|
|
if '-O' in mkfs_opt:
|
|
mkfs_opt['-O'] = mkfs_opt['-O'].split(',')
|
|
|
|
return mkfs_opt
|
|
|
|
|
|
def merge_ext_features(conf_feature, user_feature):
|
|
user_feature_list = user_feature.split(',')
|
|
|
|
merged_feature = []
|
|
# Removes duplicate entries in conf_list.
|
|
for item in conf_feature:
|
|
if item not in merged_feature:
|
|
merged_feature.append(item)
|
|
|
|
# User options override config options.
|
|
for item in user_feature_list:
|
|
if item[0] == '^':
|
|
if item[1:] in merged_feature:
|
|
merged_feature.remove(item[1:])
|
|
else:
|
|
merged_feature.append(item)
|
|
elif item not in merged_feature:
|
|
merged_feature.append(item)
|
|
return merged_feature
|
|
|
|
|
|
def ext_tunables(dev):
|
|
"""Call tune2fs -l and parse the result."""
|
|
cmd = 'tune2fs -l %s' % dev
|
|
try:
|
|
out = utils.system_output(cmd)
|
|
except error.CmdError:
|
|
tools_dir = os.path.join(os.environ['AUTODIR'], 'tools')
|
|
cmd = '%s/tune2fs.ext4dev -l %s' % (tools_dir, dev)
|
|
out = utils.system_output(cmd)
|
|
# Load option mappings
|
|
tune2fs_dict = {}
|
|
for line in out.splitlines():
|
|
components = line.split(':', 1)
|
|
if len(components) == 2:
|
|
value = components[1].strip()
|
|
option = components[0]
|
|
if value.isdigit():
|
|
tune2fs_dict[option] = int(value)
|
|
else:
|
|
tune2fs_dict[option] = value
|
|
|
|
return tune2fs_dict
|
|
|
|
|
|
def ext_mkfs_options(tune2fs_dict, mkfs_option):
|
|
"""Map the tune2fs options to mkfs options."""
|
|
|
|
def __inode_count(tune_dict, k):
|
|
return (tune_dict['Block count']/tune_dict[k] + 1) * (
|
|
tune_dict['Block size'])
|
|
|
|
def __block_count(tune_dict, k):
|
|
return int(100*tune_dict[k]/tune_dict['Block count'] + 1)
|
|
|
|
def __volume_name(tune_dict, k):
|
|
if tune_dict[k] != '<none>':
|
|
return tune_dict[k]
|
|
else:
|
|
return ''
|
|
|
|
# mappings between fs features and mkfs options
|
|
ext_mapping = {'Blocks per group': '-g',
|
|
'Block size': '-b',
|
|
'Filesystem features': '-O',
|
|
'Filesystem OS type': '-o',
|
|
'Filesystem revision #': '-r',
|
|
'Filesystem volume name': '-L',
|
|
'Flex block group size': '-G',
|
|
'Fragment size': '-f',
|
|
'Inode count': '-i',
|
|
'Inode size': '-I',
|
|
'Journal inode': '-j',
|
|
'Reserved block count': '-m'}
|
|
|
|
conversions = {
|
|
'Journal inode': lambda d, k: None,
|
|
'Filesystem volume name': __volume_name,
|
|
'Reserved block count': __block_count,
|
|
'Inode count': __inode_count,
|
|
'Filesystem features': lambda d, k: re.sub(' ', ',', d[k]),
|
|
'Filesystem revision #': lambda d, k: d[k][0]}
|
|
|
|
for key, value in six.iteritems(ext_mapping):
|
|
if key not in tune2fs_dict:
|
|
continue
|
|
if key in conversions:
|
|
mkfs_option[value] = conversions[key](tune2fs_dict, key)
|
|
else:
|
|
mkfs_option[value] = tune2fs_dict[key]
|
|
|
|
|
|
def xfs_tunables(dev):
|
|
"""Call xfs_grow -n to get filesystem tunables."""
|
|
# Have to mount the filesystem to call xfs_grow.
|
|
tmp_mount_dir = tempfile.mkdtemp()
|
|
cmd = 'mount %s %s' % (dev, tmp_mount_dir)
|
|
utils.system_output(cmd)
|
|
xfs_growfs = os.path.join(os.environ['AUTODIR'], 'tools', 'xfs_growfs')
|
|
cmd = '%s -n %s' % (xfs_growfs, dev)
|
|
try:
|
|
out = utils.system_output(cmd)
|
|
finally:
|
|
# Clean.
|
|
cmd = 'umount %s' % dev
|
|
utils.system_output(cmd, ignore_status=True)
|
|
os.rmdir(tmp_mount_dir)
|
|
|
|
## The output format is given in report_info (xfs_growfs.c)
|
|
## "meta-data=%-22s isize=%-6u agcount=%u, agsize=%u blks\n"
|
|
## " =%-22s sectsz=%-5u attr=%u\n"
|
|
## "data =%-22s bsize=%-6u blocks=%llu, imaxpct=%u\n"
|
|
## " =%-22s sunit=%-6u swidth=%u blks\n"
|
|
## "naming =version %-14u bsize=%-6u\n"
|
|
## "log =%-22s bsize=%-6u blocks=%u, version=%u\n"
|
|
## " =%-22s sectsz=%-5u sunit=%u blks, lazy-count=%u\n"
|
|
## "realtime =%-22s extsz=%-6u blocks=%llu, rtextents=%llu\n"
|
|
|
|
tune2fs_dict = {}
|
|
# Flag for extracting naming version number
|
|
keep_version = False
|
|
for line in out.splitlines():
|
|
m = re.search('^([-\w]+)', line)
|
|
if m:
|
|
main_tag = m.group(1)
|
|
pairs = line.split()
|
|
for pair in pairs:
|
|
# naming: version needs special treatment
|
|
if pair == '=version':
|
|
# 1 means the next pair is the version number we want
|
|
keep_version = True
|
|
continue
|
|
if keep_version:
|
|
tune2fs_dict['naming: version'] = pair
|
|
# Resets the flag since we have logged the version
|
|
keep_version = False
|
|
continue
|
|
# Ignores the strings without '=', such as 'blks'
|
|
if '=' not in pair:
|
|
continue
|
|
key, value = pair.split('=')
|
|
tagged_key = '%s: %s' % (main_tag, key)
|
|
if re.match('[0-9]+', value):
|
|
tune2fs_dict[tagged_key] = int(value.rstrip(','))
|
|
else:
|
|
tune2fs_dict[tagged_key] = value.rstrip(',')
|
|
|
|
return tune2fs_dict
|
|
|
|
|
|
def xfs_mkfs_options(tune2fs_dict, mkfs_option):
|
|
"""Maps filesystem tunables to their corresponding mkfs options."""
|
|
|
|
# Mappings
|
|
xfs_mapping = {'meta-data: isize': '-i size',
|
|
'meta-data: agcount': '-d agcount',
|
|
'meta-data: sectsz': '-s size',
|
|
'meta-data: attr': '-i attr',
|
|
'data: bsize': '-b size',
|
|
'data: imaxpct': '-i maxpct',
|
|
'data: sunit': '-d sunit',
|
|
'data: swidth': '-d swidth',
|
|
'data: unwritten': '-d unwritten',
|
|
'naming: version': '-n version',
|
|
'naming: bsize': '-n size',
|
|
'log: version': '-l version',
|
|
'log: sectsz': '-l sectsize',
|
|
'log: sunit': '-l sunit',
|
|
'log: lazy-count': '-l lazy-count',
|
|
'realtime: extsz': '-r extsize',
|
|
'realtime: blocks': '-r size',
|
|
'realtime: rtextents': '-r rtdev'}
|
|
|
|
mkfs_option['-l size'] = tune2fs_dict['log: bsize'] * (
|
|
tune2fs_dict['log: blocks'])
|
|
|
|
for key, value in six.iteritems(xfs_mapping):
|
|
mkfs_option[value] = tune2fs_dict[key]
|
|
|
|
|
|
def compare_features(needed_feature, current_feature):
|
|
"""Compare two ext* feature lists."""
|
|
if len(needed_feature) != len(current_feature):
|
|
return False
|
|
for feature in current_feature:
|
|
if feature not in needed_feature:
|
|
return False
|
|
return True
|
|
|
|
|
|
def match_ext_options(fs_type, dev, needed_options):
|
|
"""Compare the current ext* filesystem tunables with needed ones."""
|
|
# mkfs.ext* will load default options from /etc/mke2fs.conf
|
|
conf_opt = parse_mke2fs_conf(fs_type)
|
|
# We need to convert the conf options to mkfs options.
|
|
conf_mkfs_opt = convert_conf_opt(conf_opt)
|
|
# Breaks user mkfs option string to dictionary.
|
|
needed_opt_dict = opt_string2dict(needed_options)
|
|
# Removes ignored options.
|
|
ignored_option = ['-c', '-q', '-E', '-F']
|
|
for opt in ignored_option:
|
|
if opt in needed_opt_dict:
|
|
del needed_opt_dict[opt]
|
|
|
|
# User options override config options.
|
|
needed_opt = conf_mkfs_opt
|
|
for key, value in six.iteritems(needed_opt_dict):
|
|
if key == '-N' or key == '-T':
|
|
raise Exception('-N/T is not allowed.')
|
|
elif key == '-O':
|
|
needed_opt[key] = merge_ext_features(needed_opt[key], value)
|
|
else:
|
|
needed_opt[key] = value
|
|
|
|
# '-j' option will add 'has_journal' feature.
|
|
if '-j' in needed_opt and 'has_journal' not in needed_opt['-O']:
|
|
needed_opt['-O'].append('has_journal')
|
|
# 'extents' will be shown as 'extent' in the outcome of tune2fs
|
|
if 'extents' in needed_opt['-O']:
|
|
needed_opt['-O'].append('extent')
|
|
needed_opt['-O'].remove('extents')
|
|
# large_file is a byproduct of resize_inode.
|
|
if 'large_file' not in needed_opt['-O'] and (
|
|
'resize_inode' in needed_opt['-O']):
|
|
needed_opt['-O'].append('large_file')
|
|
|
|
current_opt = {}
|
|
tune2fs_dict = ext_tunables(dev)
|
|
ext_mkfs_options(tune2fs_dict, current_opt)
|
|
|
|
# Does the match
|
|
for key, value in six.iteritems(needed_opt):
|
|
if key == '-O':
|
|
if not compare_features(value, current_opt[key].split(',')):
|
|
return False
|
|
elif key not in current_opt or value != current_opt[key]:
|
|
return False
|
|
return True
|
|
|
|
|
|
def match_xfs_options(dev, needed_options):
|
|
"""Compare the current ext* filesystem tunables with needed ones."""
|
|
tmp_mount_dir = tempfile.mkdtemp()
|
|
cmd = 'mount %s %s' % (dev, tmp_mount_dir)
|
|
utils.system_output(cmd)
|
|
xfs_growfs = os.path.join(os.environ['AUTODIR'], 'tools', 'xfs_growfs')
|
|
cmd = '%s -n %s' % (xfs_growfs, dev)
|
|
try:
|
|
current_option = utils.system_output(cmd)
|
|
finally:
|
|
# Clean.
|
|
cmd = 'umount %s' % dev
|
|
utils.system_output(cmd, ignore_status=True)
|
|
os.rmdir(tmp_mount_dir)
|
|
|
|
# '-N' has the same effect as '-n' in mkfs.ext*. Man mkfs.xfs for details.
|
|
cmd = 'mkfs.xfs %s -N -f %s' % (needed_options, dev)
|
|
needed_out = utils.system_output(cmd)
|
|
# 'mkfs.xfs -N' produces slightly different result than 'xfs_growfs -n'
|
|
needed_out = re.sub('internal log', 'internal ', needed_out)
|
|
if current_option == needed_out:
|
|
return True
|
|
else:
|
|
return False
|
|
|
|
|
|
def match_mkfs_option(fs_type, dev, needed_options):
|
|
"""Compare the current filesystem tunables with needed ones."""
|
|
if fs_type.startswith('ext'):
|
|
ret = match_ext_options(fs_type, dev, needed_options)
|
|
elif fs_type == 'xfs':
|
|
ret = match_xfs_options(dev, needed_options)
|
|
else:
|
|
ret = False
|
|
|
|
return ret
|