Added ability to define disks via 'ephemeralX.Y'.
Modified cc_mounts to identify whether ephermalX is partitioned. Changed datasources for Azure and SmartOS to use 'ephemeralX.Y' format. Added disk remove functionally
This commit is contained in:
Ben Howard
parent
db0fc22e12
commit
738a3472f1
|
|
@ -16,8 +16,8 @@
|
|||
#
|
||||
# You should have received a copy of the GNU General Public License
|
||||
# along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
from cloudinit.settings import PER_INSTANCE
|
||||
from cloudinit import util
|
||||
from cloudinit.settings import PER_INSTANCE
|
||||
import logging
|
||||
import shlex
|
||||
|
||||
|
|
@ -29,13 +29,13 @@ SFDISK_CMD = util.which("sfdisk")
|
|||
LSBLK_CMD = util.which("lsblk")
|
||||
BLKID_CMD = util.which("blkid")
|
||||
BLKDEV_CMD = util.which("blockdev")
|
||||
WIPEFS_CMD = util.which("wipefs")
|
||||
|
||||
LOG = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def handle(_name, cfg, cloud, log, _args):
|
||||
"""
|
||||
Call util.prep_disk for disk_setup cloud-config.
|
||||
See doc/examples/cloud-config_disk-setup.txt for documentation on the
|
||||
format.
|
||||
"""
|
||||
|
|
@ -203,8 +203,38 @@ def is_filesystem(device):
|
|||
return fs_type
|
||||
|
||||
|
||||
def enumerate_disk(device):
|
||||
"""
|
||||
Enumerate the elements of a child device. Return a dict of name,
|
||||
type, fstype, and label
|
||||
"""
|
||||
|
||||
lsblk_cmd = [LSBLK_CMD, '--pairs', '--out', 'NAME,TYPE,FSTYPE,LABEL',
|
||||
device]
|
||||
info = None
|
||||
try:
|
||||
info, _err = util.subp(lsblk_cmd)
|
||||
except Exception as e:
|
||||
raise Exception("Failed during disk check for %s\n%s" % (device, e))
|
||||
|
||||
parts = [x for x in (info.strip()).splitlines() if len(x.split()) > 0]
|
||||
|
||||
for part in parts:
|
||||
d = {'name': None,
|
||||
'type': None,
|
||||
'fstype': None,
|
||||
'label': None,
|
||||
}
|
||||
|
||||
for key, value in value_splitter(part):
|
||||
d[key.lower()] = value
|
||||
|
||||
LOG.info(d)
|
||||
yield d
|
||||
|
||||
|
||||
def find_device_node(device, fs_type=None, label=None, valid_targets=None,
|
||||
label_match=True):
|
||||
label_match=True, replace_fs=None):
|
||||
"""
|
||||
Find a device that is either matches the spec, or the first
|
||||
|
||||
|
|
@ -221,26 +251,12 @@ def find_device_node(device, fs_type=None, label=None, valid_targets=None,
|
|||
if not valid_targets:
|
||||
valid_targets = ['disk', 'part']
|
||||
|
||||
lsblk_cmd = [LSBLK_CMD, '--pairs', '--out', 'NAME,TYPE,FSTYPE,LABEL',
|
||||
device]
|
||||
info = None
|
||||
try:
|
||||
info, _err = util.subp(lsblk_cmd)
|
||||
except Exception as e:
|
||||
raise Exception("Failed during disk check for %s\n%s" % (device, e))
|
||||
|
||||
raw_device_used = False
|
||||
parts = [x for x in (info.strip()).splitlines() if len(x.split()) > 0]
|
||||
for d in enumerate_disk(device):
|
||||
|
||||
for part in parts:
|
||||
d = {'name': None,
|
||||
'type': None,
|
||||
'fstype': None,
|
||||
'label': None,
|
||||
}
|
||||
|
||||
for key, value in value_splitter(part):
|
||||
d[key.lower()] = value
|
||||
if d['fstype'] == replace_fs and label_match == False:
|
||||
# We found a device where we want to replace the FS
|
||||
return ('/dev/%s' % d['name'], False)
|
||||
|
||||
if (d['fstype'] == fs_type and
|
||||
((label_match and d['label'] == label) or not label_match)):
|
||||
|
|
@ -454,6 +470,42 @@ def get_partition_mbr_layout(size, layout):
|
|||
|
||||
return sfdisk_definition
|
||||
|
||||
def purge_disk(device):
|
||||
"""
|
||||
Remove parition table entries
|
||||
"""
|
||||
|
||||
# wipe any file systems first
|
||||
for d in enumerate_disk(device):
|
||||
LOG.info(d)
|
||||
if d['type'] not in ["disk", "crypt"]:
|
||||
wipefs_cmd = [WIPEFS_CMD, "--all", "/dev/%s" % d['name']]
|
||||
try:
|
||||
LOG.info("Purging filesystem on /dev/%s" % d['name'])
|
||||
util.subp(wipefs_cmd)
|
||||
LOG.info("Purged filesystem on /dev/%s" % d['name'])
|
||||
except Exception as e:
|
||||
raise Exception("Failed FS purge of /dev/%s" % d['name'])
|
||||
|
||||
dd_cmd = util.which("dd")
|
||||
last_seek = int(get_hdd_size(device) / 1024) - 2
|
||||
first_mb = [dd_cmd, "if=/dev/zero", "of=%s" % device, "bs=1M", "count=1"]
|
||||
last_mb = [dd_cmd, "if=/dev/zero", "of=%s" % device, "bs=1M", "seek=%s" % last_seek]
|
||||
try:
|
||||
util.subp(first_mb)
|
||||
LOG.info("Purged MBR/Partition table from %s" % device)
|
||||
util.subp(last_mb, rcs=[0,1])
|
||||
LOG.info("Purged any chance of GPT table from %s" % device)
|
||||
|
||||
# Wipe it for good measure
|
||||
wipefs_cmd = [WIPEFS_CMD, "--all", device]
|
||||
util.subp(wipefs_cmd)
|
||||
except Exception as e:
|
||||
LOG.critical(e)
|
||||
raise Exception("Failed to remove MBR/Part from %s" % device)
|
||||
|
||||
read_parttbl(device)
|
||||
|
||||
|
||||
def get_partition_layout(table_type, size, layout):
|
||||
"""
|
||||
|
|
@ -542,6 +594,12 @@ def mkpart(device, definition):
|
|||
if not is_device_valid(device):
|
||||
raise Exception("Device %s is not a disk device!", device)
|
||||
|
||||
# Remove the partition table entries
|
||||
if isinstance(layout, str) and layout.lower() == "remove":
|
||||
LOG.debug("Instructed to remove partition table entries")
|
||||
purge_disk(device)
|
||||
return
|
||||
|
||||
LOG.debug("Checking if device layout matches")
|
||||
if check_partition_layout(table_type, device, layout):
|
||||
LOG.debug("Device partitioning layout matches")
|
||||
|
|
@ -565,6 +623,26 @@ def mkpart(device, definition):
|
|||
LOG.debug("Partition table created for %s", device)
|
||||
|
||||
|
||||
def lookup_force_flag(fs):
|
||||
"""
|
||||
A force flag might be -F or -F, this look it up
|
||||
"""
|
||||
flags = {'ext': '-F',
|
||||
'btrfs': '-f',
|
||||
'xfs': '-f',
|
||||
'reiserfs': '-f',
|
||||
}
|
||||
|
||||
if 'ext' in fs.lower():
|
||||
fs = 'ext'
|
||||
|
||||
if fs.lower() in flags:
|
||||
return flags[fs]
|
||||
|
||||
LOG.warn("Force flag for %s is unknown." % fs)
|
||||
return ''
|
||||
|
||||
|
||||
def mkfs(fs_cfg):
|
||||
"""
|
||||
Create a file system on the device.
|
||||
|
|
@ -592,6 +670,7 @@ def mkfs(fs_cfg):
|
|||
fs_type = fs_cfg.get('filesystem')
|
||||
fs_cmd = fs_cfg.get('cmd', [])
|
||||
fs_opts = fs_cfg.get('extra_opts', [])
|
||||
fs_replace = fs_cfg.get('replace_fs', False)
|
||||
overwrite = fs_cfg.get('overwrite', False)
|
||||
|
||||
# This allows you to define the default ephemeral or swap
|
||||
|
|
@ -632,17 +711,23 @@ def mkfs(fs_cfg):
|
|||
label_match = False
|
||||
|
||||
device, reuse = find_device_node(device, fs_type=fs_type, label=label,
|
||||
label_match=label_match)
|
||||
label_match=label_match,
|
||||
replace_fs=fs_replace)
|
||||
LOG.debug("Automatic device for %s identified as %s", odevice, device)
|
||||
|
||||
if reuse:
|
||||
LOG.debug("Found filesystem match, skipping formating.")
|
||||
return
|
||||
|
||||
if not reuse and fs_replace and device:
|
||||
LOG.debug("Replacing file system on %s as instructed." % device)
|
||||
|
||||
if not device:
|
||||
LOG.debug("No device aviable that matches request. "
|
||||
"Skipping fs creation for %s", fs_cfg)
|
||||
return
|
||||
elif not partition or str(partition).lower() == 'none':
|
||||
LOG.debug("Using the raw device to place filesystem %s on" % label)
|
||||
|
||||
else:
|
||||
LOG.debug("Error in device identification handling.")
|
||||
|
|
@ -682,12 +767,16 @@ def mkfs(fs_cfg):
|
|||
if label:
|
||||
fs_cmd.extend(["-L", label])
|
||||
|
||||
# File systems that support the -F flag
|
||||
if not fs_cmd and (overwrite or device_type(device) == "disk"):
|
||||
fs_cmd.append(lookup_force_flag(fs_type))
|
||||
|
||||
# Add the extends FS options
|
||||
if fs_opts:
|
||||
fs_cmd.extend(fs_opts)
|
||||
|
||||
LOG.debug("Creating file system %s on %s", label, device)
|
||||
LOG.debug(" Using cmd: %s", "".join(fs_cmd))
|
||||
LOG.debug(" Using cmd: %s", " ".join(fs_cmd))
|
||||
try:
|
||||
util.subp(fs_cmd)
|
||||
except Exception as e:
|
||||
|
|
|
|||
|
|
@ -20,6 +20,7 @@
|
|||
|
||||
from string import whitespace # pylint: disable=W0402
|
||||
|
||||
import os.path
|
||||
import re
|
||||
|
||||
from cloudinit import type_utils
|
||||
|
|
@ -75,7 +76,9 @@ def handle(_name, cfg, cloud, log, _args):
|
|||
"name from ephemeral to ephemeral0"), (i + 1))
|
||||
|
||||
if is_mdname(startname):
|
||||
newname = cloud.device_name_to_device(startname)
|
||||
candidate_name = cloud.device_name_to_device(startname)
|
||||
newname = disk_or_part(candidate_name)
|
||||
|
||||
if not newname:
|
||||
log.debug("Ignoring nonexistant named mount %s", startname)
|
||||
cfgmnt[i][1] = None
|
||||
|
|
@ -119,7 +122,8 @@ def handle(_name, cfg, cloud, log, _args):
|
|||
# entry has the same device name
|
||||
for defmnt in defmnts:
|
||||
startname = defmnt[0]
|
||||
devname = cloud.device_name_to_device(startname)
|
||||
candidate_name = cloud.device_name_to_device(startname)
|
||||
devname = disk_or_part(candidate_name)
|
||||
if devname is None:
|
||||
log.debug("Ignoring nonexistant named default mount %s", startname)
|
||||
continue
|
||||
|
|
@ -198,3 +202,32 @@ def handle(_name, cfg, cloud, log, _args):
|
|||
util.subp(("mount", "-a"))
|
||||
except:
|
||||
util.logexc(log, "Activating mounts via 'mount -a' failed")
|
||||
|
||||
|
||||
def disk_or_part(device):
|
||||
"""
|
||||
Find where the file system is on the disk, either on
|
||||
the disk itself or on the first partition. We don't go
|
||||
any deeper than partition 1 though.
|
||||
"""
|
||||
|
||||
if not device:
|
||||
return None
|
||||
|
||||
short_name = device.split('/')[-1]
|
||||
sys_path = "/sys/block/%s" % short_name
|
||||
|
||||
if not os.path.exists(sys_path):
|
||||
LOG.warn("Device %s does not exist in sysfs" % device)
|
||||
return None
|
||||
|
||||
sys_long_path = sys_path + "/" + short_name + "%s"
|
||||
valid_mappings = [ sys_long_path % "1",
|
||||
sys_long_path % "p1",
|
||||
sys_path ]
|
||||
|
||||
for cdisk in valid_mappings:
|
||||
if not os.path.exists(cdisk):
|
||||
continue
|
||||
return "/dev/%s" % cdisk.split('/')[-1]
|
||||
return None
|
||||
|
|
|
|||
|
|
@ -54,8 +54,9 @@ BUILTIN_CLOUD_CONFIG = {
|
|||
'layout': True,
|
||||
'overwrite': False}
|
||||
},
|
||||
'fs_setup': [{'filesystem': 'ext4', 'device': 'ephemeral0',
|
||||
'partition': 'auto'}],
|
||||
'fs_setup': [{'filesystem': 'ext4',
|
||||
'device': 'ephemeral0.1',
|
||||
'replace_fs': 'ntfs'}]
|
||||
}
|
||||
|
||||
DS_CFG_PATH = ['datasource', DS_NAME]
|
||||
|
|
@ -176,7 +177,9 @@ class DataSourceAzureNet(sources.DataSource):
|
|||
return True
|
||||
|
||||
def device_name_to_device(self, name):
|
||||
return self.ds_cfg['disk_aliases'].get(name)
|
||||
device = name.split('.')[0]
|
||||
return util.map_device_alias(self.ds_cfg['disk_aliases'].get(device),
|
||||
alias=name)
|
||||
|
||||
def get_config_obj(self):
|
||||
return self.cfg
|
||||
|
|
|
|||
|
|
@ -81,8 +81,9 @@ BUILTIN_CLOUD_CONFIG = {
|
|||
'layout': False,
|
||||
'overwrite': False}
|
||||
},
|
||||
'fs_setup': [{'label': 'ephemeral0', 'filesystem': 'ext3',
|
||||
'device': 'ephemeral0', 'partition': 'auto'}],
|
||||
'fs_setup': [{'label': 'ephemeral0',
|
||||
'filesystem': 'ext3',
|
||||
'device': 'ephemeral0'}],
|
||||
}
|
||||
|
||||
|
||||
|
|
@ -155,7 +156,9 @@ class DataSourceSmartOS(sources.DataSource):
|
|||
return True
|
||||
|
||||
def device_name_to_device(self, name):
|
||||
return self.ds_cfg['disk_aliases'].get(name)
|
||||
device = name.split('.')[0]
|
||||
return util.map_device_alias(self.ds_cfg['disk_aliases'].get(device),
|
||||
alias=name)
|
||||
|
||||
def get_config_obj(self):
|
||||
return self.cfg
|
||||
|
|
|
|||
|
|
@ -32,6 +32,7 @@ import grp
|
|||
import gzip
|
||||
import hashlib
|
||||
import os
|
||||
import os.path
|
||||
import platform
|
||||
import pwd
|
||||
import random
|
||||
|
|
@ -1826,3 +1827,78 @@ def log_time(logfunc, msg, func, args=None, kwargs=None, get_uptime=False):
|
|||
except:
|
||||
pass
|
||||
return ret
|
||||
|
||||
|
||||
def map_partition(alias):
|
||||
"""
|
||||
Return partition number for devices like ephemeral0.0 or ephemeral0.1
|
||||
|
||||
Parameters:
|
||||
alaias: the alias, i.e. ephemeral0 or swap0
|
||||
device: the actual device to markup
|
||||
|
||||
Rules:
|
||||
- anything after a . is a parittion
|
||||
- device.0 is the same as device
|
||||
"""
|
||||
|
||||
if len(alias.split('.')) == 1:
|
||||
return None
|
||||
|
||||
suffix = alias.split('.')[-1]
|
||||
try:
|
||||
if int(suffix) == 0:
|
||||
return None
|
||||
return int(suffix)
|
||||
except ValueError:
|
||||
pass
|
||||
|
||||
return None
|
||||
|
||||
|
||||
def map_device_alias(device, partition=None, alias=None):
|
||||
"""
|
||||
Find the name of the partition. While this might seem rather
|
||||
straight forward, its not since some devices are '<device><partition>'
|
||||
while others are '<device>p<partition>'. For example, /dev/xvda3 on EC2
|
||||
will present as /dev/xvda3p1 for the first partition since /dev/xvda3 is
|
||||
a block device.
|
||||
|
||||
The primary use is to map 'ephemeral0.1' in the datasource to a
|
||||
real device name
|
||||
"""
|
||||
|
||||
if not device:
|
||||
return None
|
||||
|
||||
if not partition and not alias:
|
||||
raise Exception("partition or alias is required")
|
||||
|
||||
if alias:
|
||||
partition = map_partition(alias)
|
||||
|
||||
# if the partition doesn't map, return the device
|
||||
if not partition:
|
||||
return device
|
||||
|
||||
short_name = device.split('/')[-1]
|
||||
sys_path = "/sys/block/%s" % short_name
|
||||
|
||||
if not os.path.exists(sys_path):
|
||||
return None
|
||||
|
||||
sys_long_path = sys_path + "/" + short_name
|
||||
valid_mappings = [sys_long_path + "%s" % partition,
|
||||
sys_long_path + "p%s" % partition]
|
||||
|
||||
for cdisk in valid_mappings:
|
||||
if not os.path.exists(cdisk):
|
||||
continue
|
||||
|
||||
dev_path = "/dev/%s" % cdisk.split('/')[-1]
|
||||
if os.path.exists(dev_path):
|
||||
return dev_path
|
||||
|
||||
return None
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -30,8 +30,8 @@ disk_setup:
|
|||
fs_setup:
|
||||
- label: ephemeral0
|
||||
filesystem: ext4
|
||||
device: ephemeral0
|
||||
partition: auto
|
||||
device: ephemeral0.1
|
||||
replace_fs: ntfs
|
||||
|
||||
|
||||
Default disk definitions for SmartOS
|
||||
|
|
@ -47,8 +47,7 @@ disk_setup:
|
|||
fs_setup:
|
||||
- label: ephemeral0
|
||||
filesystem: ext3
|
||||
device: ephemeral0
|
||||
partition: auto
|
||||
device: ephemeral0.0
|
||||
|
||||
Cavaut for SmartOS: if ephemeral disk is not defined, then the disk will
|
||||
not be automatically added to the mounts.
|
||||
|
|
@ -187,6 +186,9 @@ Where:
|
|||
label as 'ephemeralX' otherwise there may be issues with the mounting
|
||||
of the ephemeral storage layer.
|
||||
|
||||
If you define the device as 'ephemeralX.Y' then Y will be interpetted
|
||||
as a partition value. However, ephermalX.0 is the _same_ as ephemeralX.
|
||||
|
||||
<PART_VALUE>: The valid options are:
|
||||
"auto|any": tell cloud-init not to care whether there is a partition
|
||||
or not. Auto will use the first partition that does not contain a
|
||||
|
|
|
|||
|
|
@ -328,8 +328,6 @@ class TestAzureDataSource(MockerTestCase):
|
|||
self.assertTrue(ret)
|
||||
cfg = dsrc.get_config_obj()
|
||||
self.assertTrue(cfg)
|
||||
self.assertEquals(dsrc.device_name_to_device("ephemeral0"),
|
||||
"/dev/sdc")
|
||||
|
||||
def test_userdata_arrives(self):
|
||||
userdata = "This is my user-data"
|
||||
|
|
|
|||
Loading…
Reference in New Issue