From 36c0b17b51788c867b5aefe269cd860f4ebc7bb4 Mon Sep 17 00:00:00 2001 From: tierno Date: Thu, 12 Jan 2017 18:32:28 +0100 Subject: [PATCH] cloud init parameters at VNFD, allow file tranfer, config-drive, userdata, users, ssh-keys Change-Id: Ice99bfaf3a952dc8b52f3947972f82bb51edff58 Signed-off-by: tierno --- database_utils/migrate_mano_db.sh | 14 ++++ nfvo.py | 99 +++++++++++++++++++---- nfvo_db.py | 6 +- openmano_schemas.py | 34 +++++++- openmanod.py | 4 +- scenarios/examples/simple-cloud-init.yaml | 34 ++++++++ vimconn_openstack.py | 63 +++++++++++---- vnfs/examples/linux-cloud-init.yaml | 67 +++++++++++++++ vnfs/examples/linux.yaml | 1 - 9 files changed, 286 insertions(+), 36 deletions(-) create mode 100644 scenarios/examples/simple-cloud-init.yaml create mode 100644 vnfs/examples/linux-cloud-init.yaml diff --git a/database_utils/migrate_mano_db.sh b/database_utils/migrate_mano_db.sh index 9fc49665..a8ec8f91 100755 --- a/database_utils/migrate_mano_db.sh +++ b/database_utils/migrate_mano_db.sh @@ -186,6 +186,7 @@ DATABASE_TARGET_VER_NUM=0 [ $OPENMANO_VER_NUM -ge 5002 ] && DATABASE_TARGET_VER_NUM=16 #0.5.2 => 16 [ $OPENMANO_VER_NUM -ge 5003 ] && DATABASE_TARGET_VER_NUM=17 #0.5.3 => 17 [ $OPENMANO_VER_NUM -ge 5004 ] && DATABASE_TARGET_VER_NUM=18 #0.5.4 => 18 +[ $OPENMANO_VER_NUM -ge 5005 ] && DATABASE_TARGET_VER_NUM=19 #0.5.5 => 19 #TODO ... put next versions here @@ -716,6 +717,19 @@ function downgrade_from_18(){ echo "DELETE FROM schema_version WHERE version_int='18';" | $DBCMD || ! echo "ERROR. Aborted!" || exit -1 } +function upgrade_to_19(){ + echo " upgrade database from version 0.18 to version 0.19" + echo " add column 'boot_data' at table 'vms'" + echo "ALTER TABLE vms ADD COLUMN boot_data TEXT NULL DEFAULT NULL AFTER image_path;" | $DBCMD || ! echo "ERROR. Aborted!" || exit -1 + echo "INSERT INTO schema_version (version_int, version, openmano_ver, comments, date) VALUES (19, '0.19', '0.5.5', 'Extra Boot-data content at VNFC (vms)', '2017-01-11');" | $DBCMD || ! echo "ERROR. Aborted!" || exit -1 +} +function downgrade_from_19(){ + echo " downgrade database from version 0.19 to version 0.18" + echo " remove column 'boot_data' from table 'vms'" + echo "ALTER TABLE vms DROP COLUMN boot_data;" | $DBCMD || ! echo "ERROR. Aborted!" || exit -1 + echo "DELETE FROM schema_version WHERE version_int='19';" | $DBCMD || ! echo "ERROR. Aborted!" || exit -1 +} + function upgrade_to_X(){ echo " change 'datacenter_nets'" echo "ALTER TABLE datacenter_nets ADD COLUMN vim_tenant_id VARCHAR(36) NOT NULL AFTER datacenter_id, DROP INDEX name_datacenter_id, ADD UNIQUE INDEX name_datacenter_id (name, datacenter_id, vim_tenant_id);" | $DBCMD || ! echo "ERROR. Aborted!" || exit -1 diff --git a/nfvo.py b/nfvo.py index 9a2dd05d..7537c756 100644 --- a/nfvo.py +++ b/nfvo.py @@ -216,7 +216,14 @@ def check_vnf_descriptor(vnf_descriptor): HTTP_Bad_Request) name_list.append( interface["name"] ) vnfc_interfaces[ vnfc["name"] ] = name_list - + # check bood-data info + if "boot-data" in vnfc: + # check that user-data is incompatible with users and config-files + if (vnfc["boot-data"].get("users") or vnfc["boot-data"].get("config-files")) and vnfc["boot-data"].get("user-data"): + raise NfvoException( + "Error at vnf:VNFC:boot-data, fields 'users' and 'config-files' are not compatible with 'user-data'", + HTTP_Bad_Request) + #check if the info in external_connections matches with the one in the vnfcs name_list=[] for external_connection in vnf_descriptor["vnf"].get("external-connections",() ): @@ -605,6 +612,8 @@ def new_vnf(mydb, tenant_id, vnf_descriptor): #print "Image id for VNFC %s: %s" % (vnfc['name'],image_id) VNFCDict[vnfc['name']]["image_id"] = image_id VNFCDict[vnfc['name']]["image_path"] = vnfc.get('VNFC image') + if vnfc.get("boot-data"): + VNFCDict[vnfc['name']]["boot_data"] = yaml.safe_dump(vnfc["boot-data"], default_flow_style=True, width=256) # Step 7. Storing the VNF descriptor in the repository @@ -744,8 +753,9 @@ def new_vnf_v02(mydb, tenant_id, vnf_descriptor): #print "Image id for VNFC %s: %s" % (vnfc['name'],image_id) VNFCDict[vnfc['name']]["image_id"] = image_id VNFCDict[vnfc['name']]["image_path"] = vnfc.get('VNFC image') + if vnfc.get("boot-data"): + VNFCDict[vnfc['name']]["boot_data"] = yaml.safe_dump(vnfc["boot-data"], default_flow_style=True, width=256) - # Step 7. Storing the VNF descriptor in the repository if "descriptor" not in vnf_descriptor["vnf"]: vnf_descriptor["vnf"]["descriptor"] = yaml.safe_dump(vnf_descriptor, indent=4, explicit_start=True, default_flow_style=False) @@ -783,10 +793,15 @@ def get_vnf_id(mydb, tenant_id, vnf_id): data={'vnf' : filtered_content} #GET VM content = mydb.get_rows(FROM='vnfs join vms on vnfs.uuid=vms.vnf_id', - SELECT=('vms.uuid as uuid','vms.name as name', 'vms.description as description'), + SELECT=('vms.uuid as uuid','vms.name as name', 'vms.description as description', 'boot_data'), WHERE={'vnfs.uuid': vnf_id} ) if len(content)==0: raise NfvoException("vnf '{}' not found".format(vnf_id), HTTP_Not_Found) + # change boot_data into boot-data + for vm in content: + if vm.get("boot_data"): + vm["boot-data"] = yaml.safe_load(vm["boot_data"]) + del vm["boot_data"] data['vnf']['VNFC'] = content #TODO: GET all the information from a VNFC and include it in the output. @@ -1545,9 +1560,34 @@ def start_scenario(mydb, tenant_id, scenario_id, instance_scenario_name, instanc #logger.error("start_scenario %s", error_text) raise NfvoException(error_text, e.http_code) -def unify_cloud_config(cloud_config): +def unify_cloud_config(cloud_config_preserve, cloud_config): + ''' join the cloud config information into cloud_config_preserve. + In case of conflict cloud_config_preserve preserves + None is admited + ''' + if not cloud_config_preserve and not cloud_config: + return None + + new_cloud_config = {"key-pairs":[], "users":[]} + # key-pairs + if cloud_config_preserve: + for key in cloud_config_preserve.get("key-pairs", () ): + if key not in new_cloud_config["key-pairs"]: + new_cloud_config["key-pairs"].append(key) + if cloud_config: + for key in cloud_config.get("key-pairs", () ): + if key not in new_cloud_config["key-pairs"]: + new_cloud_config["key-pairs"].append(key) + if not new_cloud_config["key-pairs"]: + del new_cloud_config["key-pairs"] + + # users + if cloud_config: + new_cloud_config["users"] += cloud_config.get("users", () ) + if cloud_config_preserve: + new_cloud_config["users"] += cloud_config_preserve.get("users", () ) index_to_delete = [] - users = cloud_config.get("users", []) + users = new_cloud_config.get("users", []) for index0 in range(0,len(users)): if index0 in index_to_delete: continue @@ -1557,13 +1597,45 @@ def unify_cloud_config(cloud_config): if users[index0]["name"] == users[index1]["name"]: index_to_delete.append(index1) for key in users[index1].get("key-pairs",()): - if "key-pairs" not in users[index0]: + if "key-pairs" not in users[index0]: users[index0]["key-pairs"] = [key] elif key not in users[index0]["key-pairs"]: users[index0]["key-pairs"].append(key) index_to_delete.sort(reverse=True) for index in index_to_delete: del users[index] + if not new_cloud_config["users"]: + del new_cloud_config["users"] + + #boot-data-drive + if cloud_config and cloud_config.get("boot-data-drive") != None: + new_cloud_config["boot-data-drive"] = cloud_config["boot-data-drive"] + if cloud_config_preserve and cloud_config_preserve.get("boot-data-drive") != None: + new_cloud_config["boot-data-drive"] = cloud_config_preserve["boot-data-drive"] + + # user-data + if cloud_config and cloud_config.get("user-data") != None: + new_cloud_config["user-data"] = cloud_config["user-data"] + if cloud_config_preserve and cloud_config_preserve.get("user-data") != None: + new_cloud_config["user-data"] = cloud_config_preserve["user-data"] + + # config files + new_cloud_config["config-files"] = [] + if cloud_config and cloud_config.get("config-files") != None: + new_cloud_config["config-files"] += cloud_config["config-files"] + if cloud_config_preserve: + for file in cloud_config_preserve.get("config-files", ()): + for index in range(0, len(new_cloud_config["config-files"])): + if new_cloud_config["config-files"][index]["dest"] == file["dest"]: + new_cloud_config["config-files"][index] = file + break + else: + new_cloud_config["config-files"].append(file) + if not new_cloud_config["config-files"]: + del new_cloud_config["config-files"] + return new_cloud_config + + def get_datacenter_by_name_uuid(mydb, tenant_id, datacenter_id_name=None, **extra_filter): datacenter_id = None @@ -1779,14 +1851,7 @@ def create_instance(mydb, tenant_id, instance_dict): scenario_vnf["datacenter"] = vnf_instance_desc["datacenter"] #0.1 parse cloud-config parameters - cloud_config = scenarioDict.get("cloud-config", {}) - if instance_dict.get("cloud-config"): - cloud_config.update( instance_dict["cloud-config"]) - if not cloud_config: - cloud_config = None - else: - scenarioDict["cloud-config"] = cloud_config - unify_cloud_config(cloud_config) + cloud_config = unify_cloud_config(instance_dict.get("cloud-config"), scenarioDict.get("cloud-config")) #0.2 merge instance information into scenario #Ideally, the operation should be as simple as: update(scenarioDict,instance_dict) @@ -2041,8 +2106,12 @@ def create_instance(mydb, tenant_id, instance_dict): #print "networks", yaml.safe_dump(myVMDict['networks'], indent=4, default_flow_style=False) #print "interfaces", yaml.safe_dump(vm['interfaces'], indent=4, default_flow_style=False) #print ">>>>>>>>>>>>>>>>>>>>>>>>>>>" + if vm.get("boot_data"): + cloud_config_vm = unify_cloud_config(vm["boot_data"], cloud_config) + else: + cloud_config_vm = cloud_config vm_id = vim.new_vminstance(myVMDict['name'],myVMDict['description'],myVMDict.get('start', None), - myVMDict['imageRef'],myVMDict['flavorRef'],myVMDict['networks'], cloud_config = cloud_config, + myVMDict['imageRef'],myVMDict['flavorRef'],myVMDict['networks'], cloud_config = cloud_config_vm, disk_list = myVMDict['disks']) vm['vim_id'] = vm_id diff --git a/nfvo_db.py b/nfvo_db.py index 5e51c3b1..11ae450a 100644 --- a/nfvo_db.py +++ b/nfvo_db.py @@ -652,7 +652,7 @@ class nfvo_db(db_base.db_base): self.cur.execute(cmd) vnf['interfaces'] = self.cur.fetchall() #vms - cmd = "SELECT vms.uuid as uuid, flavor_id, image_id, vms.name as name, vms.description as description " \ + cmd = "SELECT vms.uuid as uuid, flavor_id, image_id, vms.name as name, vms.description as description, vms.boot_data as boot_data " \ " FROM vnfs join vms on vnfs.uuid=vms.vnf_id " \ " WHERE vnfs.uuid='" + vnf['vnf_id'] +"'" \ " ORDER BY vms.created_at" @@ -660,6 +660,10 @@ class nfvo_db(db_base.db_base): self.cur.execute(cmd) vnf['vms'] = self.cur.fetchall() for vm in vnf['vms']: + if vm["boot_data"]: + vm["boot_data"] = yaml.safe_load(vm["boot_data"]) + else: + del vm["boot_data"] if datacenter_id!=None: cmd = "SELECT vim_id FROM datacenters_images WHERE image_id='{}' AND datacenter_id='{}'".format(vm['image_id'],datacenter_id) self.logger.debug(cmd) diff --git a/openmano_schemas.py b/openmano_schemas.py index a8c92a0f..c325b00d 100644 --- a/openmano_schemas.py +++ b/openmano_schemas.py @@ -495,6 +495,37 @@ numa_schema = { #"required": ["memory"] } +config_files_schema = { + "title": "Config files for cloud init schema", + "$schema": "http://json-schema.org/draft-04/schema#", + "type": "object", + "properties": { + "dest": path_schema, + "encoding": {"type": "string", "enum": ["b64", "base64", "gz", "gz+b64", "gz+base64", "gzip+b64", "gzip+base64"]}, #by default text + "content": {"type": "string"}, + "permissions": {"type": "string"}, # tiypically octal notation '0644' + "owner": {"type": "string"}, # format: owner:group + + }, + "additionalProperties": False, + "required": ["dest", "content"], +} + +boot_data_vdu_schema = { + "title": "Boot data (Cloud-init) configuration schema", + "$schema": "http://json-schema.org/draft-04/schema#", + "type": "object", + "properties":{ + "key-pairs": {"type" : "array", "items": {"type":"string"}}, + "users": {"type" : "array", "items": cloud_config_user_schema}, + "user-data": {"type" : "string"}, # scrip to run + "config-files": {"type": "array", "items": config_files_schema}, + # NOTE: “user-data” are mutually exclusive with users and config-files because user/files are injected using user-data + "boot-data-drive": {"type": "boolean"}, + }, + "additionalProperties": False, +} + vnfc_schema = { "type":"object", "properties":{ @@ -529,7 +560,8 @@ vnfc_schema = { "items": numa_schema }, "bridge-ifaces": bridge_interfaces_schema, - "devices": devices_schema + "devices": devices_schema, + "boot-data" : boot_data_vdu_schema }, "required": ["name"], diff --git a/openmanod.py b/openmanod.py index d00f6182..3d0feb45 100755 --- a/openmanod.py +++ b/openmanod.py @@ -33,9 +33,9 @@ It loads the configuration file and launches the http_server thread that will li ''' __author__="Alfonso Tierno, Gerardo Garcia, Pablo Montes" __date__ ="$26-aug-2014 11:09:29$" -__version__="0.5.4-r512" +__version__="0.5.5-r514" version_date="Jan 2017" -database_version="0.18" #expected database schema version +database_version="0.19" #expected database schema version import httpserver import time diff --git a/scenarios/examples/simple-cloud-init.yaml b/scenarios/examples/simple-cloud-init.yaml new file mode 100644 index 00000000..18ed3e9b --- /dev/null +++ b/scenarios/examples/simple-cloud-init.yaml @@ -0,0 +1,34 @@ +## +# Copyright 2015 Telefónica Investigación y Desarrollo, S.A.U. +# This file is part of openmano +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +# +# For those usages not covered by the Apache License, Version 2.0 please +# contact with: nfvlabs@tid.es +## +--- +schema_version: 2 +scenario: + name: simple-cloud-init + description: Simple network scenario consisting of a single VNF connected to an external network + vnfs: + linux1: # vnf/net name in the scenario + vnf_name: linux-cloud-init # VNF name as introduced in OPENMANO DB + networks: + mgmt: # provide a name for this net or connection + external: true + interfaces: + - linux1: eth0 # Node and its interface + diff --git a/vimconn_openstack.py b/vimconn_openstack.py index 96d89e36..0cd9b193 100644 --- a/vimconn_openstack.py +++ b/vimconn_openstack.py @@ -33,6 +33,7 @@ import yaml import logging import netaddr import time +import yaml from novaclient import client as nClient_v2, exceptions as nvExceptions, api_versions as APIVersion import keystoneclient.v2_0.client as ksClient_v2 @@ -736,26 +737,55 @@ class vimconnector(vimconn.vimconnector): security_groups = self.config.get('security_groups') if type(security_groups) is str: security_groups = ( security_groups, ) + #cloud config + userdata=None + config_drive = None if isinstance(cloud_config, dict): - userdata="#cloud-config\nusers:\n" - #default user - if "key-pairs" in cloud_config: - userdata += " - default:\n ssh-authorized-keys:\n" - for key in cloud_config["key-pairs"]: - userdata += " - '{key}'\n".format(key=key) - for user in cloud_config.get("users",[]): - userdata += " - name: {name}\n sudo: ALL=(ALL) NOPASSWD:ALL\n".format(name=user["name"]) - if "user-info" in user: - userdata += " gecos: {}'\n".format(user["user-info"]) - if user.get("key-pairs"): - userdata += " ssh-authorized-keys:\n" - for key in user["key-pairs"]: - userdata += " - '{key}'\n".format(key=key) + if cloud_config.get("user-data"): + userdata=cloud_config["user-data"] + if cloud_config.get("boot-data-drive") != None: + config_drive = cloud_config["boot-data-drive"] + if cloud_config.get("config-files") or cloud_config.get("users") or cloud_config.get("key-pairs"): + if userdata: + raise vimconn.vimconnConflictException("Cloud-config cannot contain both 'userdata' and 'config-files'/'users'/'key-pairs'") + userdata_dict={} + #default user + if cloud_config.get("key-pairs"): + userdata_dict["ssh-authorized-keys"] = cloud_config["key-pairs"] + userdata_dict["users"] = [{"default": None, "ssh-authorized-keys": cloud_config["key-pairs"] }] + if cloud_config.get("users"): + if "users" not in cloud_config: + userdata_dict["users"] = [ "default" ] + for user in cloud_config["users"]: + user_info = { + "name" : user["name"], + "sudo": "ALL = (ALL)NOPASSWD:ALL" + } + if "user-info" in user: + user_info["gecos"] = user["user-info"] + if user.get("key-pairs"): + user_info["ssh-authorized-keys"] = user["key-pairs"] + userdata_dict["users"].append(user_info) + + if cloud_config.get("config-files"): + userdata_dict["write_files"] = [] + for file in cloud_config["config-files"]: + file_info = { + "path" : file["dest"], + "content": file["content"] + } + if file.get("encoding"): + file_info["encoding"] = file["encoding"] + if file.get("permissions"): + file_info["permissions"] = file["permissions"] + if file.get("owner"): + file_info["owner"] = file["owner"] + userdata_dict["write_files"].append(file_info) + userdata = "#cloud-config\n" + userdata += yaml.safe_dump(userdata_dict, indent=4, default_flow_style=False) self.logger.debug("userdata: %s", userdata) elif isinstance(cloud_config, str): userdata = cloud_config - else: - userdata=None #Create additional volumes in case these are present in disk_list block_device_mapping = None @@ -803,6 +833,7 @@ class vimconnector(vimconn.vimconnector): availability_zone=self.config.get('availability_zone'), key_name=self.config.get('keypair'), userdata=userdata, + config_drive = config_drive, block_device_mapping = block_device_mapping ) # , description=description) #print "DONE :-)", server diff --git a/vnfs/examples/linux-cloud-init.yaml b/vnfs/examples/linux-cloud-init.yaml new file mode 100644 index 00000000..ce8c5de1 --- /dev/null +++ b/vnfs/examples/linux-cloud-init.yaml @@ -0,0 +1,67 @@ +## +# Copyright 2015 Telefónica Investigación y Desarrollo, S.A.U. +# This file is part of openmano +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +# +# For those usages not covered by the Apache License, Version 2.0 please +# contact with: nfvlabs@tid.es +## + +--- +schema_version: "0.2" +vnf: + name: linux-cloud-init + description: Single-VM VNF with a traditional cloud VM based on generic Linux OS + external-connections: + - name: eth0 + type: bridge + description: General purpose interface + VNFC: linux-VM + local_iface_name: eth0 + VNFC: + - name: linux-VM + description: Generic Linux Virtual Machine + #Copy the image to a compute path and edit this path + image name: ubuntu16.04 + vcpus: 1 # Only for traditional cloud VMs. Number of virtual CPUs (oversubscription is allowed). + ram: 2048 # Only for traditional cloud VMs. Memory in MBytes (not from hugepages, oversubscription is allowed) + disk: 20 + bridge-ifaces: + - name: eth0 + vpci: "0000:00:11.0" + numas: [] + boot-data: + key-pairs: + - ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQCy2w9GHMKKNkpCmrDK2ovc3XBYDETuLWwaW24S+feHhLBQiZlzh3gSQoINlA+2ycM9zYbxl4BGzEzpTVyCQFZv5PidG4m6ox7LR+KYkDcITMyjsVuQJKDvt6oZvRt6KbChcCi0n2JJD/oUiJbBFagDBlRslbaFI2mmqmhLlJ5TLDtmYxzBLpjuX4m4tv+pdmQVfg7DYHsoy0hllhjtcDlt1nn05WgWYRTu7mfQTWfVTavu+OjIX3e0WN6NW7yIBWZcE/Q9lC0II3W7PZDE3QaT55se4SPIO2JTdqsx6XGbekdG1n6adlduOI27sOU5m4doiyJ8554yVbuDB/z5lRBD alfonso.tiernosepulveda@telefonica.com + users: + - name: atierno + key-pairs: + - ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQCy2w9GHMKKNkpCmrDK2ovc3XBYDETuLWwaW24S+feHhLBQiZlzh3gSQoINlA+2ycM9zYbxl4BGzEzpTVyCQFZv5PidG4m6ox7LR+KYkDcITMyjsVuQJKDvt6oZvRt6KbChcCi0n2JJD/oUiJbBFagDBlRslbaFI2mmqmhLlJ5TLDtmYxzBLpjuX4m4tv+pdmQVfg7DYHsoy0hllhjtcDlt1nn05WgWYRTu7mfQTWfVTavu+OjIX3e0WN6NW7yIBWZcE/Q9lC0II3W7PZDE3QaT55se4SPIO2JTdqsx6XGbekdG1n6adlduOI27sOU5m4doiyJ8554yVbuDB/z5lRBD alfonso.tiernosepulveda@telefonica.com + boot-data-drive: true + config-files: + - content: | + auto enp0s3 + iface enp0s3 inet dhcp + dest: /etc/network/interfaces.d/enp0s3.cfg + permissions: '0644' + owner: root:root + - content: | + #! /bin/bash + ls -al >> /var/log/osm.log + dest: /etc/rc.local + permissions: '0755' + - content: "file content" + dest: /etc/test_delete + diff --git a/vnfs/examples/linux.yaml b/vnfs/examples/linux.yaml index a80b71a8..8493e505 100644 --- a/vnfs/examples/linux.yaml +++ b/vnfs/examples/linux.yaml @@ -40,4 +40,3 @@ vnf: - name: eth0 vpci: "0000:00:11.0" numas: [] - -- 2.17.1