From 1c242032fbb26fed3a82691abb030583b4f8940b Mon Sep 17 00:00:00 2001 From: Wayne Okuma Date: Thu, 28 Aug 2014 04:33:53 -0400 Subject: [PATCH] Glance Metadata Definitions Catalog - Seed Implements: blueprint metadata-schema-catalog A common API hosted by the Glance service for vendors, admins, services, and users to meaningfully define available key / value pair and tag metadata. The intent is to enable better metadata collaboration across artifacts, services, and projects for OpenStack users. This is about the definition of the available metadata that can be used on different types of resources (images, artifacts, volumes, flavors, aggregates, etc). A definition includes the properties type, its key, it's description, and it's constraints. This catalogue will not store the values for specific instance properties. Change-Id: Ib1c1abf80879fb6dcd5ee30c7d2bc65b0ba720d5 DocImpact Co-Authored-By: Lakshmi N Sampath Co-Authored-By: Wayne Okuma Co-Authored-By: Travis Tripp Co-Authored-By: Pawel Koniszewski Co-Authored-By: Michal Jastrzebski Co-Authored-By: Michal Dulko --- README.rst | 1 - etc/metadefs/README | 4 + etc/metadefs/compute-host-capabilities.json | 181 +++++++++++ etc/metadefs/compute-hypervisor.json | 40 +++ etc/metadefs/compute-libvirt.json | 94 ++++++ etc/metadefs/compute-quota.json | 109 +++++++ etc/metadefs/compute-randomgen.json | 29 ++ etc/metadefs/compute-trust.json | 19 ++ etc/metadefs/compute-vcputopology.json | 54 ++++ etc/metadefs/compute-vmware.json | 31 ++ etc/metadefs/compute-watchdog.json | 33 ++ etc/metadefs/compute-xenapi.json | 29 ++ etc/metadefs/glance-common-image-props.json | 42 +++ glance/cmd/manage.py | 43 +++ glance/db/metadata.py | 45 +++ glance/db/sqlalchemy/metadata.py | 314 ++++++++++++++++++++ glance/tests/unit/test_manage.py | 73 +++++ 17 files changed, 1140 insertions(+), 1 deletion(-) create mode 100644 etc/metadefs/README create mode 100644 etc/metadefs/compute-host-capabilities.json create mode 100644 etc/metadefs/compute-hypervisor.json create mode 100644 etc/metadefs/compute-libvirt.json create mode 100644 etc/metadefs/compute-quota.json create mode 100644 etc/metadefs/compute-randomgen.json create mode 100644 etc/metadefs/compute-trust.json create mode 100644 etc/metadefs/compute-vcputopology.json create mode 100644 etc/metadefs/compute-vmware.json create mode 100644 etc/metadefs/compute-watchdog.json create mode 100644 etc/metadefs/compute-xenapi.json create mode 100644 etc/metadefs/glance-common-image-props.json create mode 100644 glance/db/metadata.py create mode 100644 glance/db/sqlalchemy/metadata.py diff --git a/README.rst b/README.rst index dbbfb5724f..cda9a01b36 100644 --- a/README.rst +++ b/README.rst @@ -5,6 +5,5 @@ Glance Glance is a project that defines services for discovering, registering, retrieving and storing virtual machine images. Use the following resources to learn more: - * `Official Glance documentation `_ * `Official Client documentation `_ diff --git a/etc/metadefs/README b/etc/metadefs/README new file mode 100644 index 0000000000..39d25b3028 --- /dev/null +++ b/etc/metadefs/README @@ -0,0 +1,4 @@ +This directory contains predefined namespaces for Glance Metadata Definitions +Catalog. Files from this directory can be loaded into the database using +db_load_metadefs command for glance-manage. Similarly you can unload the +definitions using db_unload_metadefs command. diff --git a/etc/metadefs/compute-host-capabilities.json b/etc/metadefs/compute-host-capabilities.json new file mode 100644 index 0000000000..a30caca741 --- /dev/null +++ b/etc/metadefs/compute-host-capabilities.json @@ -0,0 +1,181 @@ +{ + "namespace": "OS::Compute::HostCapabilities", + "display_name": "Compute Host Capabilities", + "description": "Capabilities provided by the Compute Host. This provides the ability to fine tune the harware specification required when a new vm is requested.", + "visibility": "public", + "protected": true, + "resource_type_associations": [ + { + "name": "OS::Nova::Flavor", + "prefix": "capabilities:" + }, + { + "name": "OS::Nova::Aggregate", + "prefix": "aggregate_instance_extra_spec:" + } + ], + "properties": { + "cpu_info:vendor": { + "title": "Vendor", + "description": "Specifies the CPU manufacturer.", + "type": "string", + "enum": [ + "Intel", + "AMD" + ] + }, + "cpu_info:model": { + "title": "Model", + "description": "Specifies the CPU model. Use this property to ensure that your vm runs on a a specific cpu model.", + "type": "string", + "enum": [ + "Conroe", + "Core2Duo", + "Penryn", + "Nehalem", + "Westmere", + "SandyBridge", + "IvyBridge", + "Haswell", + "Broadwell", + "Delhi", + "Seoul", + "Abu Dhabi", + "Interlagos", + "Kabini", + "Valencia", + "Zurich", + "Budapest", + "Barcelona", + "Suzuka", + "Shanghai", + "Istanbul", + "Lisbon", + "Magny-Cours", + "Valencia", + "Cortex-A57", + "Cortex-A53", + "Cortex-A12", + "Cortex-A17", + "Cortex-A15", + "Coretx-A7", + "X-Gene" + ] + }, + "cpu_info:arch": { + "title": "Architecture", + "description": "Specifies the CPU architecture. Use this property to specify the architecture supported by the hypervisor.", + "type": "string", + "enum": [ + "x86", + "x86_64", + "i686", + "ia64", + "ARMv8-A", + "ARMv7-A" + ] + }, + "cpu_info:topology:cores": { + "title": "cores", + "description": "Number of cores.", + "type": "integer", + "readonly": false, + "default": 1 + }, + "cpu_info:topology:threads": { + "title": "threads", + "description": "Number of threads.", + "type": "integer", + "readonly": false, + "default": 1 + }, + "cpu_info:topology:sockets": { + "title": "sockets", + "description": "Number of sockets.", + "type": "integer", + "readonly": false, + "default": 1 + }, + "cpu_info:features": { + "title": "Features", + "description": "Specifies CPU flags/features. Using this property you can specify the required set of instructions supported by a vm.", + "type": "array", + "items": { + "type": "string", + "enum": [ + "aes", + "vme", + "de", + "pse", + "tsc", + "msr", + "pae", + "mce", + "cx8", + "apic", + "sep", + "mtrr", + "pge", + "mca", + "cmov", + "pat", + "pse36", + "clflush", + "dts", + "acpi", + "mmx", + "fxsr", + "sse", + "sse2", + "ss", + "ht", + "tm", + "ia64", + "pbe", + "rdtscp", + "pni", + "pclmulqdq", + "dtes64", + "monitor", + "ds_cpl", + "vmx", + "smx", + "est", + "tm2", + "ssse3", + "cid", + "fma", + "cx16", + "xtpr", + "pdcm", + "pcid", + "dca", + "sse4_1", + "sse4_2", + "x2apic", + "movbe", + "popcnt", + "tsc_deadline_timer", + "xsave", + "avx", + "f16c", + "rdrand", + "fsgsbase", + "bmi1", + "hle", + "avx2", + "smep", + "bmi2", + "erms", + "invpcid", + "rtm", + "mpx", + "rdseed", + "adx", + "smap" + ] + } + } + }, + "objects": [] +} diff --git a/etc/metadefs/compute-hypervisor.json b/etc/metadefs/compute-hypervisor.json new file mode 100644 index 0000000000..0397eda500 --- /dev/null +++ b/etc/metadefs/compute-hypervisor.json @@ -0,0 +1,40 @@ +{ + "namespace": "OS::Compute::Hypervisor", + "display_name": "Hypervisor Selection", + "description": "Choose capabilities that should be provided by the Compute Host. This provides the ability to fine tune the harware specification required when a new vm is requested.", + "visibility": "public", + "protected": true, + "resource_type_associations": [ + { + "name": "OS::Glance::Image" + } + ], + "properties": { + "hypervisor_type": { + "title": "Hypervisor Type", + "description": "The hypervisor type. It may be used by the host properties filter for scheduling. The ImagePropertiesFilter filters compute nodes that satisfy any architecture, hypervisor type, or virtual machine mode properties specified on the instance's image properties. Image properties are contained in the image dictionary in the request_spec.", + "type": "string", + "enum": [ + "xen", + "qemu", + "kvm", + "lxc", + "uml", + "vmware", + "hyperv" + ] + }, + "vm_mode": { + "title": "VM Mode", + "description": "The virtual machine mode. This represents the host/guest ABI (application binary interface) used for the virtual machine. It may be used by the host properties filter for scheduling. \n\n hvm — Fully virtualized - This is the virtual machine mode (vm_mode) used by QEMU and KVM. \n\n xen - Xen 3.0 paravirtualized. \n\n uml — User Mode Linux paravirtualized. \n\n exe — Executables in containers. This is the mode used by LXC.", + "type": "string", + "enum": [ + "hvm", + "xen", + "uml", + "exe" + ] + } + }, + "objects": [] +} diff --git a/etc/metadefs/compute-libvirt.json b/etc/metadefs/compute-libvirt.json new file mode 100644 index 0000000000..419c40fec2 --- /dev/null +++ b/etc/metadefs/compute-libvirt.json @@ -0,0 +1,94 @@ +{ + "namespace": "OS::Compute::Libvirt", + "display_name": "libvirt Driver Options", + "description": "The libvirt compute driver options. \n\nThese are properties specific to compute drivers. For a list of all hypervisors, see here: https://wiki.openstack.org/wiki/HypervisorSupportMatrix.", + "visibility": "public", + "protected": true, + "resource_type_associations": [ + { + "name": "OS::Glance::Image" + } + ], + "properties": { + "hw_disk_bus": { + "title": "Disk Bus", + "description": "Specifies the type of disk controller to attach disk devices to.", + "type": "string", + "enum": [ + "scsi", + "virtio", + "uml", + "xen", + "ide", + "usb" + ] + }, + "hw_rng_model": { + "title": "Random Number Generator Device", + "description": "Adds a random-number generator device to the image's instances. The cloud administrator can enable and control device behavior by configuring the instance's flavor. By default: The generator device is disabled. /dev/random is used as the default entropy source. To specify a physical HW RNG device, use the following option in the nova.conf file: rng_dev_path=/dev/hwrng", + "type": "string", + "default": "virtio" + }, + "hw_machine_type": { + "title": "Machine Type", + "description": "Enables booting an ARM system using the specified machine type. By default, if an ARM image is used and its type is not specified, Compute uses vexpress-a15 (for ARMv7) or virt (for AArch64) machine types. Valid types can be viewed by using the virsh capabilities command (machine types are displayed in the machine tag).", + "type": "string" + }, + "hw_scsi_model": { + "title": "SCSI Model", + "description": "Enables the use of VirtIO SCSI (virtio-scsi) to provide block device access for compute instances; by default, instances use VirtIO Block (virtio-blk). VirtIO SCSI is a para-virtualized SCSI controller device that provides improved scalability and performance, and supports advanced SCSI hardware.", + "type": "string", + "default": "virtio-scsi" + }, + "hw_video_model": { + "title": "Video Model", + "description": "The video image driver used.", + "type": "string", + "enum": [ + "vga", + "cirrus", + "vmvga", + "xen", + "qxl" + ] + }, + "hw_video_ram": { + "title": "Max Video Ram", + "description": "Maximum RAM for the video image. Used only if a hw_video:ram_max_mb value has been set in the flavor's extra_specs and that value is higher than the value set in hw_video_ram.", + "type": "integer" + }, + "os_command_line": { + "title": "Kernel Command Line", + "description": "The kernel command line to be used by the libvirt driver, instead of the default. For linux containers (LXC), the value is used as arguments for initialization. This key is valid only for Amazon kernel, ramdisk, or machine images (aki, ari, or ami).", + "type": "string" + }, + "hw_vif_model": { + "title": "Virtual Network Interface", + "description": "Specifies the model of virtual network interface device to use. The valid options depend on the configured hypervisor. KVM and QEMU: e1000, ne2k_pci, pcnet, rtl8139, and virtio. VMware: e1000, e1000e, VirtualE1000, VirtualE1000e, VirtualPCNet32, VirtualSriovEthernetCard, and VirtualVmxnet. Xen: e1000, netfront, ne2k_pci, pcnet, and rtl8139.", + "type": "string", + "enum": [ + "e1000", + "ne2k_pci", + "pcnet", + "rtl8139", + "virtio", + "e1000", + "e1000e", + "VirtualE1000", + "VirtualE1000e", + "VirtualPCNet32", + "VirtualSriovEthernetCard", + "VirtualVmxnet", + "netfront", + "ne2k_pci" + ] + }, + "hw_qemu_guest_agent": { + "title": "QEMU Guest Agent", + "description": "It is a daemon program running inside the domain which is supposed to help management applications with executing functions which need assistance of the guest OS. For example, freezing and thawing filesystems, entering suspend. However, guest agent (GA) is not bullet proof, and hostile guest OS can send spurious replies.", + "type": "string", + "enum": ["yes", "no"] + } + }, + "objects": [] +} diff --git a/etc/metadefs/compute-quota.json b/etc/metadefs/compute-quota.json new file mode 100644 index 0000000000..ca1bd596c7 --- /dev/null +++ b/etc/metadefs/compute-quota.json @@ -0,0 +1,109 @@ +{ + "namespace": "OS::Compute::Quota", + "display_name": "Flavor Quota", + "description": "Compute drivers may enable quotas on CPUs available to a VM, disk tuning, bandwidth I/O, and instance VIF traffic control. See: http://docs.openstack.org/admin-guide-cloud/content/customize-flavors.html", + "visibility": "public", + "protected": true, + "resource_type_associations": [ + { + "name": "OS::Nova::Flavor" + } + ], + "objects": [ + { + "name": "CPU Limits", + "description": "You can configure the CPU limits with control parameters.", + "properties": { + "quota:cpu_shares": { + "title": "Quota: CPU Shares", + "description": "Specifies the proportional weighted share for the domain. If this element is omitted, the service defaults to the OS provided defaults. There is no unit for the value; it is a relative measure based on the setting of other VMs. For example, a VM configured with value 2048 gets twice as much CPU time as a VM configured with value 1024.", + "type": "integer" + }, + "quota:cpu_period": { + "title": "Quota: CPU Period", + "description": "Specifies the enforcement interval (unit: microseconds) for QEMU and LXC hypervisors. Within a period, each VCPU of the domain is not allowed to consume more than the quota worth of runtime. The value should be in range [1000, 1000000]. A period with value 0 means no value.", + "type": "integer", + "minimum": 1000, + "maximum": 1000000 + }, + "quota:cpu_quota": { + "title": "Quota: CPU Quota", + "description": "Specifies the maximum allowed bandwidth (unit: microseconds). A domain with a negative-value quota indicates that the domain has infinite bandwidth, which means that it is not bandwidth controlled. The value should be in range [1000, 18446744073709551] or less than 0. A quota with value 0 means no value. You can use this feature to ensure that all vCPUs run at the same speed.", + "type": "integer" + } + } + }, + { + "name": "Disk QoS", + "description": "Using disk I/O quotas, you can set maximum disk write to 10 MB per second for a VM user.", + "properties": { + "quota:disk_read_bytes_sec": { + "title": "Quota: Disk read bytes / sec", + "description": "Sets disk I/O quota for disk read bytes / sec.", + "type": "integer" + }, + "quota:disk_read_iops_sec": { + "title": "Quota: Disk read IOPS / sec", + "description": "Sets disk I/O quota for disk read IOPS / sec.", + "type": "integer" + }, + "quota:disk_write_bytes_sec": { + "title": "Quota: Disk Write Bytes / sec", + "description": "Sets disk I/O quota for disk write bytes / sec.", + "type": "integer" + }, + "quota:disk_write_iops_sec": { + "title": "Quota: Disk Write IOPS / sec", + "description": "Sets disk I/O quota for disk write IOPS / sec.", + "type": "integer" + }, + "quota:disk_total_bytes_sec": { + "title": "Quota: Disk Total Bytes / sec", + "description": "Sets disk I/O quota for total disk bytes / sec.", + "type": "integer" + }, + "quota:disk_total_iops_sec": { + "title": "Quota: Disk Total IOPS / sec", + "description": "Sets disk I/O quota for disk total IOPS / sec.", + "type": "integer" + } + } + }, + { + "name": "Virtual Interface QoS", + "description": "Bandwidth QoS tuning for instance virtual interfaces (VIFs) may be specified with these properties. Incoming and outgoing traffic can be shaped independently. If not specified, no quality of service (QoS) is applied on that traffic direction. So, if you want to shape only the network's incoming traffic, use inbound only (and vice versa). The OpenStack Networking service abstracts the physical implementation of the network, allowing plugins to configure and manage physical resources. Virtual Interfaces (VIF) in the logical model are analogous to physical network interface cards (NICs). VIFs are typically owned a managed by an external service; for instance when OpenStack Networking is used for building OpenStack networks, VIFs would be created, owned, and managed in Nova. VIFs are connected to OpenStack Networking networks via ports. A port is analogous to a port on a network switch, and it has an administrative state. When a VIF is attached to a port the OpenStack Networking API creates an attachment object, which specifies the fact that a VIF with a given identifier is plugged into the port.", + "properties": { + "quota:vif_inbound_average": { + "title": "Quota: VIF Inbound Average", + "description": "Network Virtual Interface (VIF) inbound average in kilobytes per second. Specifies average bit rate on the interface being shaped.", + "type": "integer" + }, + "quota:vif_inbound_burst": { + "title": "Quota: VIF Inbound Burst", + "description": "Network Virtual Interface (VIF) inbound burst in total kilobytes. Specifies the amount of bytes that can be burst at peak speed.", + "type": "integer" + }, + "quota:vif_inbound_peak": { + "title": "Quota: VIF Inbound Peak", + "description": "Network Virtual Interface (VIF) inbound peak in kilobytes per second. Specifies maximum rate at which an interface can receive data.", + "type": "integer" + }, + "quota:vif_outbound_average": { + "title": "Quota: VIF Outbound Average", + "description": "Network Virtual Interface (VIF) outbound average in kilobytes per second. Specifies average bit rate on the interface being shaped.", + "type": "integer" + }, + "quota:vif_outbound_burst": { + "title": "Quota: VIF Outbound Burst", + "description": "Network Virtual Interface (VIF) outbound burst in total kilobytes. Specifies the amount of bytes that can be burst at peak speed.", + "type": "integer" + }, + "quota:vif_outbound_peak": { + "title": "Quota: VIF Outbound Burst", + "description": "Network Virtual Interface (VIF) outbound peak in kilobytes per second. Specifies maximum rate at which an interface can send data.", + "type": "integer" + } + } + } + ] +} \ No newline at end of file diff --git a/etc/metadefs/compute-randomgen.json b/etc/metadefs/compute-randomgen.json new file mode 100644 index 0000000000..2414b844d5 --- /dev/null +++ b/etc/metadefs/compute-randomgen.json @@ -0,0 +1,29 @@ +{ + "namespace": "OS::Compute::RandomNumberGenerator", + "display_name": "Random Number Generator", + "description": "If a random-number generator device has been added to the instance through its image properties, the device can be enabled and configured.", + "visibility": "public", + "protected": true, + "resource_type_associations": [ + { + "name": "OS::Nova::Flavor" + } + ], + "properties": { + "hw_rng:allowed": { + "title": "Random Number Generator Allowed", + "description": "", + "type": "boolean" + }, + "hw_rng:rate_bytes": { + "title": "Random number generator limits.", + "description": "Allowed amount of bytes that the guest can read from the host's entropy per period.", + "type": "integer" + }, + "hw_rng:rate_period": { + "title": "Random number generator read period.", + "description": "Duration of the read period in seconds.", + "type": "integer" + } + } +} \ No newline at end of file diff --git a/etc/metadefs/compute-trust.json b/etc/metadefs/compute-trust.json new file mode 100644 index 0000000000..58b2cd835f --- /dev/null +++ b/etc/metadefs/compute-trust.json @@ -0,0 +1,19 @@ +{ + "namespace": "OS::Compute::Trust", + "display_name": "Trusted Compute Pools (Intel® TXT)", + "description": "Trusted compute pools with Intel® Trusted Execution Technology (Intel® TXT) support IT compliance by protecting virtualized data centers - private, public, and hybrid clouds against attacks toward hypervisor and BIOS, firmware, and other pre-launch software components.", + "visibility": "public", + "protected": true, + "resource_type_associations": [ + { + "name": "OS::Nova::Flavor" + } + ], + "properties": { + "trust:trusted_host": { + "title": "Intel® TXT attestation", + "description": "Select to ensure that node has been attested by Intel® Trusted Execution Technology (Intel® TXT).", + "type": "boolean" + } + } +} diff --git a/etc/metadefs/compute-vcputopology.json b/etc/metadefs/compute-vcputopology.json new file mode 100644 index 0000000000..345a1c2666 --- /dev/null +++ b/etc/metadefs/compute-vcputopology.json @@ -0,0 +1,54 @@ +{ + "namespace": "OS::Compute::VirtCPUTopology", + "display_name": "Virtual CPU Topology", + "description": "This provides the preferred socket/core/thread counts for the virtual CPU instance exposed to guests. This enables the ability to avoid hitting limitations on vCPU topologies that OS vendors place on their products. See also: http://git.openstack.org/cgit/openstack/nova-specs/tree/specs/juno/virt-driver-vcpu-topology.rst", + "visibility": "public", + "protected": true, + "resource_type_associations": [ + { + "name": "OS::Glance::Image", + "prefix": "hw_" + }, + { + "name": "OS::Cinder::Volume", + "prefix": "hw_", + "properties_target": "image" + }, + { + "name": "OS::Nova::Flavor", + "prefix": "hw:" + } + ], + "properties": { + "cpu_sockets": { + "title": "vCPU Sockets", + "description": "Preferred number of sockets to expose to the guest.", + "type": "integer" + }, + "cpu_cores": { + "title": "vCPU Cores", + "description": "Preferred number of cores to expose to the guest.", + "type": "integer" + }, + "cpu_threads": { + "title": " vCPU Threads", + "description": "Preferred number of threads to expose to the guest.", + "type": "integer" + }, + "cpu_maxsockets": { + "title": "Max vCPU Sockets", + "description": "Maximum number of sockets to expose to the guest.", + "type": "integer" + }, + "cpu_maxcores": { + "title": "Max vCPU Cores", + "description": "Maximum number of cores to expose to the guest.", + "type": "integer" + }, + "cpu_maxthreads": { + "title": "Max vCPU Threads", + "description": "Maximum number of threads to expose to the guest.", + "type": "integer" + } + } +} diff --git a/etc/metadefs/compute-vmware.json b/etc/metadefs/compute-vmware.json new file mode 100644 index 0000000000..90d58d92bb --- /dev/null +++ b/etc/metadefs/compute-vmware.json @@ -0,0 +1,31 @@ +{ + "namespace": "OS::Compute::VMwAre", + "display_name": "VMware Driver Options", + "description": "The VMware compute driver options. \n\nThese are properties specific to compute drivers. For a list of all hypervisors, see here: https://wiki.openstack.org/wiki/HypervisorSupportMatrix.", + "visibility": "public", + "protected": true, + "resource_type_associations": [ + { + "name": "OS::Glance::Image" + } + ], + "properties": { + "vmware_adaptertype": { + "title": "Disk Adapter Type", + "description": "The virtual SCSI or IDE controller used by the hypervisor.", + "type": "string", + "enum": [ + "lsiLogic", + "busLogic", + "ide" + ] + }, + "vmware_ostype": { + "title": "OS Type", + "description": "A VMware GuestID which describes the operating system installed in the image. This value is passed to the hypervisor when creating a virtual machine. If not specified, the key defaults to otherGuest. See thinkvirt.com.", + "type": "string", + "default": "otherGuest" + } + }, + "objects": [] +} diff --git a/etc/metadefs/compute-watchdog.json b/etc/metadefs/compute-watchdog.json new file mode 100644 index 0000000000..7eb32cecc3 --- /dev/null +++ b/etc/metadefs/compute-watchdog.json @@ -0,0 +1,33 @@ +{ + "namespace": "OS::Compute::Watchdog", + "display_name": "Watchdog Behavior", + "description": "Compute drivers may enable watchdog behavior over instances. See: http://docs.openstack.org/admin-guide-cloud/content/customize-flavors.html", + "visibility": "public", + "protected": true, + "resource_type_associations": [ + { + "name": "OS::Glance::Image" + }, + { + "name": "OS::Cinder::Volume", + "properties_target": "image" + }, + { + "name": "OS::Nova::Flavor" + } + ], + "properties": { + "hw_watchdog_action": { + "title": "Watchdog Action", + "description": "For the libvirt driver, you can enable and set the behavior of a virtual hardware watchdog device for each flavor. Watchdog devices keep an eye on the guest server, and carry out the configured action, if the server hangs. The watchdog uses the i6300esb device (emulating a PCI Intel 6300ESB). If hw_watchdog_action is not specified, the watchdog is disabled. Watchdog behavior set using a specific image's properties will override behavior set using flavors.", + "type": "string", + "enum": [ + "disabled", + "reset", + "poweroff", + "pause", + "none" + ] + } + } +} diff --git a/etc/metadefs/compute-xenapi.json b/etc/metadefs/compute-xenapi.json new file mode 100644 index 0000000000..eda7489f1b --- /dev/null +++ b/etc/metadefs/compute-xenapi.json @@ -0,0 +1,29 @@ +{ + "namespace": "OS::Compute::XenAPI", + "display_name": "XenAPI Driver Options", + "description": "The XenAPI compute driver options. \n\nThese are properties specific to compute drivers. For a list of all hypervisors, see here: https://wiki.openstack.org/wiki/HypervisorSupportMatrix.", + "visibility": "public", + "protected": true, + "resource_type_associations": [ + { + "name": "OS::Glance::Image" + } + ], + "properties": { + "os_type": { + "title": "OS Type", + "description": "The operating system installed on the image. The XenAPI driver contains logic that takes different actions depending on the value of the os_type parameter of the image. For example, for os_type=windows images, it creates a FAT32-based swap partition instead of a Linux swap partition, and it limits the injected host name to less than 16 characters.", + "type": "string", + "enum": [ + "linux", + "windows" + ] + }, + "auto_disk_config": { + "title": "Disk Adapter Type", + "description": "If true, the root partition on the disk is automatically resized before the instance boots. This value is only taken into account by the Compute service when using a Xen-based hypervisor with the XenAPI driver. The Compute service will only attempt to resize if there is a single partition on the image, and only if the partition is in ext3 or ext4 format.", + "type": "boolean" + } + }, + "objects": [] +} diff --git a/etc/metadefs/glance-common-image-props.json b/etc/metadefs/glance-common-image-props.json new file mode 100644 index 0000000000..57490d8877 --- /dev/null +++ b/etc/metadefs/glance-common-image-props.json @@ -0,0 +1,42 @@ +{ + "display_name": "Common Image Properties", + "namespace": "OS::Glance:CommonImageProperties", + "description": "When adding an image to Glance, you may specify some common image properties that may prove useful to consumers of your image.", + "protected": true, + "resource_type_associations" : [ + ], + "properties": { + "kernel_id": { + "title": "Kernel ID", + "type": "string", + "pattern": "^([0-9a-fA-F]){8}-([0-9a-fA-F]){4}-([0-9a-fA-F]){4}-([0-9a-fA-F]){4}-([0-9a-fA-F]){12}$", + "description": "ID of image stored in Glance that should be used as the kernel when booting an AMI-style image." + }, + "ramdisk_id": { + "title": "Ramdisk ID", + "type": "string", + "pattern": "^([0-9a-fA-F]){8}-([0-9a-fA-F]){4}-([0-9a-fA-F]){4}-([0-9a-fA-F]){4}-([0-9a-fA-F]){12}$", + "description": "ID of image stored in Glance that should be used as the ramdisk when booting an AMI-style image." + }, + "instance_uuid": { + "title": "Instance ID", + "type": "string", + "description": "ID of instance used to create this image." + }, + "architecture": { + "title": "CPU Architecture", + "description": "The CPU architecture that must be supported by the hypervisor. For example, x86_64, arm, or ppc64. Run uname -m to get the architecture of a machine. We strongly recommend using the architecture data vocabulary defined by the libosinfo project for this purpose.", + "type": "string" + }, + "os_distro": { + "title": "OS Distro", + "description": "The common name of the operating system distribution in lowercase (uses the same data vocabulary as the libosinfo project). Specify only a recognized value for this field. Deprecated values are listed to assist you in searching for the recognized value.", + "type": "string" + }, + "os_version": { + "title": "OS Version", + "description": "Operating system version as specified by the distributor. (for example, '11.10')", + "type": "string" + } + } +} diff --git a/glance/cmd/manage.py b/glance/cmd/manage.py index f5be968860..d7f248c2c4 100755 --- a/glance/cmd/manage.py +++ b/glance/cmd/manage.py @@ -46,6 +46,7 @@ from glance.common import exception from glance.common import utils from glance.db import migration as db_migration from glance.db.sqlalchemy import api as db_api +from glance.db.sqlalchemy import metadata from glance.openstack.common import gettextutils from glance.openstack.common import log from glance.openstack.common import strutils @@ -138,6 +139,25 @@ class DbCommands(object): version, sanity_check=self._need_sanity_check()) + @args('--path', metavar='', help='Path to the directory where ' + 'json metadata files are stored') + def load_metadefs(self, path=None): + """Load metadefinition json files to database""" + metadata.db_load_metadefs(db_api.get_engine(), + path) + + def unload_metadefs(self): + """Unload metadefinitions from database""" + metadata.db_unload_metadefs(db_api.get_engine()) + + @args('--path', metavar='', help='Path to the directory where ' + 'json metadata files should be ' + 'saved.') + def export_metadefs(self, path=None): + """Export metadefinitions data from database to files""" + metadata.db_export_metadefs(db_api.get_engine(), + path) + class DbLegacyCommands(object): """Class for managing the db using legacy commands""" @@ -161,6 +181,15 @@ class DbLegacyCommands(object): self.command_object.sync(CONF.command.version, CONF.command.current_version) + def load_metadefs(self, path=None): + self.command_object.load_metadefs(CONF.command.path) + + def unload_metadefs(self): + self.command_object.unload_metadefs() + + def export_metadefs(self, path=None): + self.command_object.export_metadefs(CONF.command.path) + def add_legacy_command_parsers(command_object, subparsers): @@ -191,6 +220,20 @@ def add_legacy_command_parsers(command_object, subparsers): parser.add_argument('current_version', nargs='?') parser.set_defaults(action='db_sync') + parser = subparsers.add_parser('db_load_metadefs') + parser.set_defaults(action_fn=legacy_command_object.load_metadefs) + parser.add_argument('path', nargs='?') + parser.set_defaults(action='db_load_metadefs') + + parser = subparsers.add_parser('db_unload_metadefs') + parser.set_defaults(action_fn=legacy_command_object.unload_metadefs) + parser.set_defaults(action='db_unload_metadefs') + + parser = subparsers.add_parser('db_export_metadefs') + parser.set_defaults(action_fn=legacy_command_object.export_metadefs) + parser.add_argument('path', nargs='?') + parser.set_defaults(action='db_export_metadefs') + def add_command_parsers(subparsers): command_object = DbCommands() diff --git a/glance/db/metadata.py b/glance/db/metadata.py new file mode 100644 index 0000000000..4917993a22 --- /dev/null +++ b/glance/db/metadata.py @@ -0,0 +1,45 @@ +# Copyright 2010 United States Government as represented by the +# Administrator of the National Aeronautics and Space Administration. +# All Rights Reserved. +# +# Copyright 2013 OpenStack Foundation +# Copyright 2013 Intel Corporation +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +"""Metadata setup commands.""" + +from glance.common import utils +from glance.db.sqlalchemy import api as db_api + +IMPL = utils.LazyPluggable( + 'backend', + config_group='database', + sqlalchemy='glance.db.sqlalchemy.metadata') + + +def load_metadefs(): + """Read metadefinition files and insert data into the database""" + return IMPL.db_load_metadefs(engine=db_api.get_engine(), + metadata_path=None) + + +def unload_metadefs(): + """Unload metadefinitions from database""" + return IMPL.db_unload_metadefs(engine=db_api.get_engine()) + + +def export_metadefs(): + """Export metadefinitions from database to files""" + return IMPL.db_export_metadefs(engine=db_api.get_engine(), + metadata_path=None) diff --git a/glance/db/sqlalchemy/metadata.py b/glance/db/sqlalchemy/metadata.py new file mode 100644 index 0000000000..a8275f8272 --- /dev/null +++ b/glance/db/sqlalchemy/metadata.py @@ -0,0 +1,314 @@ +# Copyright 2010 United States Government as represented by the +# Administrator of the National Aeronautics and Space Administration. +# All Rights Reserved. +# +# Copyright 2013 OpenStack Foundation +# Copyright 2013 Intel Corporation +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import json +import os +from os.path import isfile +from os.path import join +import re + +from oslo.config import cfg +import sqlalchemy +from sqlalchemy.schema import MetaData + +from glance.common import utils +from glance import i18n +import glance.openstack.common.log as logging +from glance.openstack.common import timeutils + +LOG = logging.getLogger(__name__) +_LE = i18n._LE +_LW = i18n._LW +_LI = i18n._LI + +metadata_opts = [ + cfg.StrOpt('metadata_source_path', default='/etc/glance/metadefs/', + help=_('Path to the directory where json metadata ' + 'files are stored')) +] + +CONF = cfg.CONF +CONF.register_opts(metadata_opts) + + +def get_metadef_namespaces_table(meta): + return sqlalchemy.Table('metadef_namespaces', meta, autoload=True) + + +def get_metadef_resource_types_table(meta): + return sqlalchemy.Table('metadef_resource_types', meta, autoload=True) + + +def get_metadef_namespace_resource_types_table(meta): + return sqlalchemy.Table('metadef_namespace_resource_types', meta, + autoload=True) + + +def get_metadef_properties_table(meta): + return sqlalchemy.Table('metadef_properties', meta, autoload=True) + + +def get_metadef_objects_table(meta): + return sqlalchemy.Table('metadef_objects', meta, autoload=True) + + +def _get_resource_type_id(meta, name): + resource_types_table = get_metadef_resource_types_table(meta) + return resource_types_table.select().\ + where(resource_types_table.c.name == name).execute().fetchone().id + + +def _get_resource_type(meta, resource_type_id): + resource_types_table = get_metadef_resource_types_table(meta) + return resource_types_table.select().\ + where(resource_types_table.c.id == resource_type_id).\ + execute().fetchone() + + +def _get_namespace_resource_types(meta, namespace_id): + namespace_resource_types_table =\ + get_metadef_namespace_resource_types_table(meta) + return namespace_resource_types_table.select().\ + where(namespace_resource_types_table.c.namespace_id == namespace_id).\ + execute().fetchall() + + +def _get_properties(meta, namespace_id): + properties_table = get_metadef_properties_table(meta) + return properties_table.select().\ + where(properties_table.c.namespace_id == namespace_id).\ + execute().fetchall() + + +def _get_objects(meta, namespace_id): + objects_table = get_metadef_objects_table(meta) + return objects_table.select().\ + where(objects_table.c.namespace_id == namespace_id).\ + execute().fetchall() + + +def _populate_metadata(meta, metadata_path=None): + if not metadata_path: + metadata_path = CONF.metadata_source_path + + try: + json_schema_files = [f for f in os.listdir(metadata_path) + if isfile(join(metadata_path, f)) + and f.endswith('.json')] + except OSError as e: + LOG.error(utils.exception_to_str(e)) + return + + metadef_namespaces_table = get_metadef_namespaces_table(meta) + metadef_namespace_resource_types_tables =\ + get_metadef_namespace_resource_types_table(meta) + metadef_objects_table = get_metadef_objects_table(meta) + metadef_properties_table = get_metadef_properties_table(meta) + metadef_resource_types_table = get_metadef_resource_types_table(meta) + + if not json_schema_files: + LOG.error(_LE("Json schema files not found in %s. Aborting."), + metadata_path) + return + + for namespace_id, json_schema_file in enumerate(json_schema_files, + start=1): + try: + file = join(metadata_path, json_schema_file) + json_metadata = open(file) + metadata = json.load(json_metadata) + json_metadata.close() + except Exception as e: + LOG.error(utils.exception_to_str(e)) + continue + + values = { + 'id': namespace_id, + 'namespace': metadata.get('namespace', None), + 'display_name': metadata.get('display_name', None), + 'description': metadata.get('description', None), + 'visibility': metadata.get('visibility', None), + 'protected': metadata.get('protected', None), + 'owner': metadata.get('owner', 'admin'), + 'created_at': timeutils.utcnow() + } + _insert_data_to_db(metadef_namespaces_table, values) + + for resource_type in metadata.get('resource_type_associations', []): + try: + resource_type_id = \ + _get_resource_type_id(meta, resource_type['name']) + except AttributeError: + values = { + 'name': resource_type['name'], + 'protected': True, + 'created_at': timeutils.utcnow() + } + _insert_data_to_db(metadef_resource_types_table, + values) + resource_type_id =\ + _get_resource_type_id(meta, resource_type['name']) + + values = { + 'resource_type_id': resource_type_id, + 'namespace_id': namespace_id, + 'created_at': timeutils.utcnow(), + 'properties_target': resource_type.get('properties_target'), + 'prefix': resource_type.get('prefix', None) + } + _insert_data_to_db(metadef_namespace_resource_types_tables, + values) + + for property, schema in metadata.get('properties', {}).iteritems(): + values = { + 'name': property, + 'namespace_id': namespace_id, + 'schema': json.dumps(schema), + 'created_at': timeutils.utcnow() + } + _insert_data_to_db(metadef_properties_table, values) + + for object in metadata.get('objects', []): + values = { + 'name': object.get('name', None), + 'description': object.get('description', None), + 'namespace_id': namespace_id, + 'schema': json.dumps(object.get('properties', None)), + 'created_at': timeutils.utcnow() + } + _insert_data_to_db(metadef_objects_table, values) + + LOG.info(_LI("File %s loaded to database."), file) + + LOG.info(_LI("Metadata loading finished")) + + +def _clear_metadata(meta): + metadef_tables = [get_metadef_properties_table(meta), + get_metadef_objects_table(meta), + get_metadef_namespace_resource_types_table(meta), + get_metadef_namespaces_table(meta)] + + for table in metadef_tables: + table.delete().execute() + LOG.info(_LI("Table %s has been cleared"), table) + + +def _insert_data_to_db(table, values, log_exception=True): + try: + table.insert(values=values).execute() + except sqlalchemy.exc.IntegrityError: + if log_exception: + LOG.warning(_LW("Duplicate entry for values: %s"), values) + + +def _export_data_to_file(meta, path): + if not path: + path = CONF.metadata_source_path + + namespace_table = get_metadef_namespaces_table(meta) + namespaces = namespace_table.select().execute().fetchall() + + pattern = re.compile('[\W_]+', re.UNICODE) + + for id, namespace in enumerate(namespaces, start=1): + namespace_id = namespace['id'] + namespace_file_name = pattern.sub('', namespace['display_name']) + + values = { + 'namespace': namespace['namespace'], + 'display_name': namespace['display_name'], + 'description': namespace['description'], + 'visibility': namespace['visibility'], + 'protected': namespace['protected'], + 'owner': namespace['owner'], + 'resource_type_associations': [], + 'properties': {}, + 'objects': [] + } + + namespace_resource_types = _get_namespace_resource_types(meta, + namespace_id) + db_objects = _get_objects(meta, namespace_id) + db_properties = _get_properties(meta, namespace_id) + + resource_types = [] + for namespace_resource_type in namespace_resource_types: + resource_type =\ + _get_resource_type(meta, + namespace_resource_type['resource_type_id']) + resource_types.append({ + 'name': resource_type['name'], + 'protected': resource_type['protected'] + }) + values.update({ + 'resource_type_associations': resource_types + }) + + objects = [] + for object in db_objects: + objects.append({ + "name": object['name'], + "description": object['description'], + "properties": json.loads(object['schema']) + }) + values.update({ + 'objects': objects + }) + + properties = {} + for property in db_properties: + properties.update({ + property['name']: json.loads(property['schema']) + }) + values.update({ + 'properties': properties + }) + + try: + file_name = ''.join([path, namespace_file_name, '.json']) + json_file = open(file_name, 'w+') + json_file.write(json.dumps(values)) + json_file.close() + + except Exception as e: + LOG.exception(utils.exception_to_str(e)) + LOG.info(_LI("Namespace %s saved in %s"), + namespace_file_name, file_name) + + +def db_load_metadefs(engine, metadata_path=None): + meta = MetaData() + meta.bind = engine + + _populate_metadata(meta, metadata_path) + + +def db_unload_metadefs(engine): + meta = MetaData() + meta.bind = engine + + _clear_metadata(meta) + + +def db_export_metadefs(engine, metadata_path=None): + meta = MetaData() + meta.bind = engine + + _export_data_to_file(meta, metadata_path) diff --git a/glance/tests/unit/test_manage.py b/glance/tests/unit/test_manage.py index ef40c9cdf5..ad7b93e194 100644 --- a/glance/tests/unit/test_manage.py +++ b/glance/tests/unit/test_manage.py @@ -21,6 +21,7 @@ import testtools from glance.cmd import manage from glance.db import migration as db_migration from glance.db.sqlalchemy import api as db_api +from glance.db.sqlalchemy import metadata as db_metadata class TestManageBase(testtools.TestCase): @@ -127,6 +128,42 @@ class TestLegacyManage(TestManageBase): db_migration.MIGRATE_REPO_PATH, '20', sanity_check=False) + def test_db_metadefs_unload(self): + db_metadata.db_unload_metadefs = mock.Mock() + self._main_test_helper(['glance.cmd.manage', 'db_unload_metadefs'], + db_metadata.db_unload_metadefs, + db_api.get_engine()) + + def test_db_metadefs_load(self): + db_metadata.db_load_metadefs = mock.Mock() + self._main_test_helper(['glance.cmd.manage', 'db_load_metadefs'], + db_metadata.db_load_metadefs, + db_api.get_engine(), + None) + + def test_db_metadefs_load_with_specified_path(self): + db_metadata.db_load_metadefs = mock.Mock() + self._main_test_helper(['glance.cmd.manage', 'db_load_metadefs', + '/mock/'], + db_metadata.db_load_metadefs, + db_api.get_engine(), + '/mock/') + + def test_db_metadefs_export(self): + db_metadata.db_export_metadefs = mock.Mock() + self._main_test_helper(['glance.cmd.manage', 'db_export_metadefs'], + db_metadata.db_export_metadefs, + db_api.get_engine(), + None) + + def test_db_metadefs_export_with_specified_path(self): + db_metadata.db_export_metadefs = mock.Mock() + self._main_test_helper(['glance.cmd.manage', 'db_export_metadefs', + '/mock/'], + db_metadata.db_export_metadefs, + db_api.get_engine(), + '/mock/') + class TestManage(TestManageBase): @@ -210,3 +247,39 @@ class TestManage(TestManageBase): db_api.get_engine(), db_migration.MIGRATE_REPO_PATH, '20', sanity_check=False) + + def test_db_metadefs_unload(self): + db_metadata.db_unload_metadefs = mock.Mock() + self._main_test_helper(['glance.cmd.manage', 'db', 'unload_metadefs'], + db_metadata.db_unload_metadefs, + db_api.get_engine()) + + def test_db_metadefs_load(self): + db_metadata.db_load_metadefs = mock.Mock() + self._main_test_helper(['glance.cmd.manage', 'db', 'load_metadefs'], + db_metadata.db_load_metadefs, + db_api.get_engine(), + None) + + def test_db_metadefs_load_with_specified_path(self): + db_metadata.db_load_metadefs = mock.Mock() + self._main_test_helper(['glance.cmd.manage', 'db', 'load_metadefs', + '--path', '/mock/'], + db_metadata.db_load_metadefs, + db_api.get_engine(), + '/mock/') + + def test_db_metadefs_export(self): + db_metadata.db_export_metadefs = mock.Mock() + self._main_test_helper(['glance.cmd.manage', 'db', 'export_metadefs'], + db_metadata.db_export_metadefs, + db_api.get_engine(), + None) + + def test_db_metadefs_export_with_specified_path(self): + db_metadata.db_export_metadefs = mock.Mock() + self._main_test_helper(['glance.cmd.manage', 'db', 'export_metadefs', + '--path', '/mock/'], + db_metadata.db_export_metadefs, + db_api.get_engine(), + '/mock/')