From eebea06b843c02dc2c456263bbc62c0d796c26ee Mon Sep 17 00:00:00 2001 From: Pablo Montes Moreno Date: Mon, 27 Feb 2017 12:33:10 +0100 Subject: [PATCH 01/16] Implemented basic test for RO. It runs basic RO functionality tests as well as scenario based tests. Tested with different vendors OpenStack installations Change-Id: I3217da96e1ae7e42942b6f5a4e127fab7ed25635 Signed-off-by: Pablo Montes Moreno --- openmanoclient.py | 18 +- ...scenario_additional_disk_empty_volume.yaml | 40 ++ .../vnfd_additional_disk_empty_volume.yaml | 63 ++ .../floating_ip/scenario_floating_ip.yaml | 40 ++ .../floating_ip/vnfd_floating_ip.yaml | 59 ++ .../scenario_additional_disk_based_image.yaml | 40 ++ .../vnfd_additional_disk_based_image.yaml | 64 ++ .../scenario_vnf_no_port_security.yaml | 40 ++ .../vnfd_no_port_security.yaml | 59 ++ .../scenario_simple-cloud-init.yaml | 34 + .../vnfd_linux-cloud-init.yaml | 67 ++ .../simple_linux/scenario_simple_linux.yaml | 34 + test/RO_tests/simple_linux/vnfd_linux.yaml | 42 ++ .../scenario_multi_vnfc.yaml | 34 + .../vnfd_linux_2VMs_v02.yaml | 104 +++ test/test_RO.py | 623 ++++++++++++++++++ 16 files changed, 1356 insertions(+), 5 deletions(-) create mode 100644 test/RO_tests/empy_volume/scenario_additional_disk_empty_volume.yaml create mode 100644 test/RO_tests/empy_volume/vnfd_additional_disk_empty_volume.yaml create mode 100644 test/RO_tests/floating_ip/scenario_floating_ip.yaml create mode 100644 test/RO_tests/floating_ip/vnfd_floating_ip.yaml create mode 100644 test/RO_tests/image_based_volume/scenario_additional_disk_based_image.yaml create mode 100644 test/RO_tests/image_based_volume/vnfd_additional_disk_based_image.yaml create mode 100644 test/RO_tests/no_port_security/scenario_vnf_no_port_security.yaml create mode 100644 test/RO_tests/no_port_security/vnfd_no_port_security.yaml create mode 100644 test/RO_tests/simple_cloud_init/scenario_simple-cloud-init.yaml create mode 100644 test/RO_tests/simple_cloud_init/vnfd_linux-cloud-init.yaml create mode 100644 test/RO_tests/simple_linux/scenario_simple_linux.yaml create mode 100644 test/RO_tests/simple_linux/vnfd_linux.yaml create mode 100644 test/RO_tests/simple_multi_vnfc/scenario_multi_vnfc.yaml create mode 100644 test/RO_tests/simple_multi_vnfc/vnfd_linux_2VMs_v02.yaml create mode 100755 test/test_RO.py diff --git a/openmanoclient.py b/openmanoclient.py index 0d9aa4ae..c11f747e 100644 --- a/openmanoclient.py +++ b/openmanoclient.py @@ -74,7 +74,7 @@ class openmanoclient(): self.datacenter_id = kwargs.get("datacenter_id") self.datacenter_name = kwargs.get("datacenter_name") self.datacenter = None - self.logger = logging.getLogger('manoclient') + self.logger = logging.getLogger(kwargs.get('logger','manoclient')) if kwargs.get("debug"): self.logger.setLevel(logging.DEBUG) @@ -424,7 +424,10 @@ class openmanoclient(): Return: Raises an exception on error, not found, found several, not free Obtain a dictionary with format {'result': text indicating deleted} ''' - return self._del_item("datacenters", uuid, name, all_tenants=True) + if not uuid: + # check that exist + uuid = self._get_item_uuid("datacenters", uuid, name, all_tenants=True) + return self._del_item("datacenters", uuid, name, all_tenants=None) def create_datacenter(self, descriptor=None, descriptor_format=None, name=None, vim_url=None, **kwargs): #, type="openvim", public=False, description=None): @@ -832,9 +835,14 @@ class openmanoclient(): Return: Raises an exception on error Obtain a dictionary with format {'tenant':{new_tenant_info}} ''' - if item not in ("tenants", "networks"): - raise OpenmanoBadParamsException("Unknown value for item '{}', must be 'tenants' or 'nets'".format(str(item))) - + if item not in ("tenants", "networks", "images"): + raise OpenmanoBadParamsException("Unknown value for item '{}', must be 'tenants', 'nets' or " + "images".format(str(item))) + + image_actions = ['list','get','show','delete'] + if item == "images" and action not in image_actions: + raise OpenmanoBadParamsException("Only available actions for item '{}' are {}\n" + "Requested action was '{}'".format(item, ', '.join(image_actions), action)) if all_tenants: tenant_text = "/any" else: diff --git a/test/RO_tests/empy_volume/scenario_additional_disk_empty_volume.yaml b/test/RO_tests/empy_volume/scenario_additional_disk_empty_volume.yaml new file mode 100644 index 00000000..61f5f38a --- /dev/null +++ b/test/RO_tests/empy_volume/scenario_additional_disk_empty_volume.yaml @@ -0,0 +1,40 @@ +## +# Copyright 2015 Telefónica Investigación y Desarrollo, S.A.U. +# This file is part of openmano +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +# +# For those usages not covered by the Apache License, Version 2.0 please +# contact with: nfvlabs@tid.es +## +--- +schema_version: 2 +scenario: + name: vnf_additional_disk_empty_volume + description: Just deploy vnf_2_disks + public: false # if available for other tenants + vnfs: + vnf_2_disks: # vnf name in the scenario + #identify an already openmano uploaded VNF either by vnf_id (uuid, prefered) or vnf_name + #vnf_id: 0c0dcc20-c5d5-11e6-a9fb-fa163e2ae06e #prefered id method + vnf_name: vnf_additional_disk_empty_volume #can fail if several vnfs matches this name + #graph: {"y":399,"x":332,"ifaces":{"left":[["xe0","d"],["xe1","d"]],"bottom":[["eth0","v"],["eth1","m"]]}} + networks: + mgmt: + # Connections based on external networks (datacenter nets) must include the external network in the list of nodes + type: bridge + external: true #this will be connected outside + interfaces: + - vnf_2_disks: mgmt0 + diff --git a/test/RO_tests/empy_volume/vnfd_additional_disk_empty_volume.yaml b/test/RO_tests/empy_volume/vnfd_additional_disk_empty_volume.yaml new file mode 100644 index 00000000..0e0f3eb3 --- /dev/null +++ b/test/RO_tests/empy_volume/vnfd_additional_disk_empty_volume.yaml @@ -0,0 +1,63 @@ +## +# Copyright 2015 Telefónica Investigación y Desarrollo, S.A.U. +# This file is part of openmano +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +# +# For those usages not covered by the Apache License, Version 2.0 please +# contact with: nfvlabs@tid.es +## +--- +vnf: + name: vnf_additional_disk_empty_volume + description: VNF with additional volume based on image + # class: parent # Optional. Used to organize VNFs + external-connections: + - name: mgmt0 + type: mgmt # "mgmt" (autoconnect to management net), "bridge", "data" + VNFC: TEMPLATE-VM # Virtual Machine this interface belongs to + local_iface_name: mgmt0 # interface name inside this Virtual Machine (must be defined in the VNFC section) + description: Management interface + VNFC: # Virtual machine array + - name: TEMPLATE-VM # name of Virtual Machine + description: TEMPLATE description + image name: ubuntu16.04 + # image metadata: {"bus":"ide", "os_type":"windows", "use_incremental": "no" } #Optional + # processor: #Optional + # model: Intel(R) Xeon(R) CPU E5-4620 0 @ 2.20GHz + # features: ["64b", "iommu", "lps", "tlbps", "hwsv", "dioc", "ht"] + # hypervisor: #Optional + # type: QEMU-kvm + # version: "10002|12001|2.6.32-358.el6.x86_64" + vcpus: 1 # Only for traditional cloud VMs. Number of virtual CPUs (oversubscription is allowed). + ram: 1000 # Only for traditional cloud VMs. Memory in MBytes (not from hugepages, oversubscription is allowed) + disk: 5 # disk size in GiB, by default 1 + #numas: + #- paired-threads: 5 # "cores", "paired-threads", "threads" + # paired-threads-id: [ [0,1], [2,3], [4,5], [6,7], [8,9] ] # By default follows incremental order + # memory: 14 # GBytes + # interfaces: [] + bridge-ifaces: + - name: mgmt0 + vpci: "0000:00:0a.0" # Optional. Virtual PCI address + bandwidth: 1 Mbps # Optional. Informative only + # mac_address: '20:33:45:56:77:46' #avoid this option if possible + # model: 'virtio' # ("virtio","e1000","ne2k_pci","pcnet","rtl8139") By default, it is automatically filled by libvirt + devices: # Optional, order determines device letter asignation (hda, hdb, ...) + - type: disk # "disk","cdrom","xml" + size: 1 + # image metadata: {"bus":"ide", "os_type":"windows", "use_incremental": "no" } + # vpci: "0000:00:03.0" # Optional, not for disk or cdrom + # Additional Virtual Machines would be included here + diff --git a/test/RO_tests/floating_ip/scenario_floating_ip.yaml b/test/RO_tests/floating_ip/scenario_floating_ip.yaml new file mode 100644 index 00000000..48470508 --- /dev/null +++ b/test/RO_tests/floating_ip/scenario_floating_ip.yaml @@ -0,0 +1,40 @@ +## +# Copyright 2015 Telefónica Investigación y Desarrollo, S.A.U. +# This file is part of openmano +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +# +# For those usages not covered by the Apache License, Version 2.0 please +# contact with: nfvlabs@tid.es +## +--- +schema_version: 2 +scenario: + name: vnf_floating_ip + description: vnf_floating_ip + public: false # if available for other tenants + vnfs: + vnf_floating_ip: # vnf name in the scenario + #identify an already openmano uploaded VNF either by vnf_id (uuid, prefered) or vnf_name + #vnf_id: 0c0dcc20-c5d5-11e6-a9fb-fa163e2ae06e #prefered id method + vnf_name: vnf_floating_ip #can fail if several vnfs matches this name + #graph: {"y":399,"x":332,"ifaces":{"left":[["xe0","d"],["xe1","d"]],"bottom":[["eth0","v"],["eth1","m"]]}} + networks: + mgmt: + # Connections based on external networks (datacenter nets) must include the external network in the list of nodes + type: bridge + external: true #this will be connected outside + interfaces: + - vnf_floating_ip: mgmt0 + diff --git a/test/RO_tests/floating_ip/vnfd_floating_ip.yaml b/test/RO_tests/floating_ip/vnfd_floating_ip.yaml new file mode 100644 index 00000000..47c73ba1 --- /dev/null +++ b/test/RO_tests/floating_ip/vnfd_floating_ip.yaml @@ -0,0 +1,59 @@ +## +# Copyright 2015 Telefónica Investigación y Desarrollo, S.A.U. +# This file is part of openmano +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +# +# For those usages not covered by the Apache License, Version 2.0 please +# contact with: nfvlabs@tid.es +## +--- +vnf: + name: vnf_floating_ip + description: VNF disabling port_security option in mgmt interface + # class: parent # Optional. Used to organize VNFs + external-connections: + - name: mgmt0 + type: mgmt # "mgmt" (autoconnect to management net), "bridge", "data" + VNFC: vnf_floating_ip # Virtual Machine this interface belongs to + local_iface_name: mgmt0 # interface name inside this Virtual Machine (must be defined in the VNFC section) + description: Management interface + VNFC: # Virtual machine array + - name: vnf_floating_ip # name of Virtual Machine + description: vnf_floating_ip + image name: ubuntu16.04 + # image metadata: {"bus":"ide", "os_type":"windows", "use_incremental": "no" } #Optional + # processor: #Optional + # model: Intel(R) Xeon(R) CPU E5-4620 0 @ 2.20GHz + # features: ["64b", "iommu", "lps", "tlbps", "hwsv", "dioc", "ht"] + # hypervisor: #Optional + # type: QEMU-kvm + # version: "10002|12001|2.6.32-358.el6.x86_64" + vcpus: 1 # Only for traditional cloud VMs. Number of virtual CPUs (oversubscription is allowed). + ram: 1000 # Only for traditional cloud VMs. Memory in MBytes (not from hugepages, oversubscription is allowed) + disk: 5 # disk size in GiB, by default 1 + #numas: + #- paired-threads: 5 # "cores", "paired-threads", "threads" + # paired-threads-id: [ [0,1], [2,3], [4,5], [6,7], [8,9] ] # By default follows incremental order + # memory: 14 # GBytes + # interfaces: [] + bridge-ifaces: + - name: mgmt0 + vpci: "0000:00:0a.0" # Optional. Virtual PCI address + bandwidth: 1 Mbps # Optional. Informative only + floating-ip: True + # mac_address: '20:33:45:56:77:46' #avoid this option if possible + # model: 'virtio' # ("virtio","e1000","ne2k_pci","pcnet","rtl8139") By default, it is automatically filled by libvirt + # Additional Virtual Machines would be included here + diff --git a/test/RO_tests/image_based_volume/scenario_additional_disk_based_image.yaml b/test/RO_tests/image_based_volume/scenario_additional_disk_based_image.yaml new file mode 100644 index 00000000..20f28cb4 --- /dev/null +++ b/test/RO_tests/image_based_volume/scenario_additional_disk_based_image.yaml @@ -0,0 +1,40 @@ +## +# Copyright 2015 Telefónica Investigación y Desarrollo, S.A.U. +# This file is part of openmano +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +# +# For those usages not covered by the Apache License, Version 2.0 please +# contact with: nfvlabs@tid.es +## +--- +schema_version: 2 +scenario: + name: vnf_additional_disk_based_image + description: Just deploy vnf_2_disks + public: false # if available for other tenants + vnfs: + vnf_2_disks: # vnf name in the scenario + #identify an already openmano uploaded VNF either by vnf_id (uuid, prefered) or vnf_name + #vnf_id: 0c0dcc20-c5d5-11e6-a9fb-fa163e2ae06e #prefered id method + vnf_name: vnf_additional_disk_based_image #can fail if several vnfs matches this name + #graph: {"y":399,"x":332,"ifaces":{"left":[["xe0","d"],["xe1","d"]],"bottom":[["eth0","v"],["eth1","m"]]}} + networks: + mgmt: + # Connections based on external networks (datacenter nets) must include the external network in the list of nodes + type: bridge + external: true #this will be connected outside + interfaces: + - vnf_2_disks: mgmt0 + diff --git a/test/RO_tests/image_based_volume/vnfd_additional_disk_based_image.yaml b/test/RO_tests/image_based_volume/vnfd_additional_disk_based_image.yaml new file mode 100644 index 00000000..57898318 --- /dev/null +++ b/test/RO_tests/image_based_volume/vnfd_additional_disk_based_image.yaml @@ -0,0 +1,64 @@ +## +# Copyright 2015 Telefónica Investigación y Desarrollo, S.A.U. +# This file is part of openmano +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +# +# For those usages not covered by the Apache License, Version 2.0 please +# contact with: nfvlabs@tid.es +## +--- +vnf: + name: vnf_additional_disk_based_image + description: VNF with additional volume based on image + # class: parent # Optional. Used to organize VNFs + external-connections: + - name: mgmt0 + type: mgmt # "mgmt" (autoconnect to management net), "bridge", "data" + VNFC: TEMPLATE-VM # Virtual Machine this interface belongs to + local_iface_name: mgmt0 # interface name inside this Virtual Machine (must be defined in the VNFC section) + description: Management interface + VNFC: # Virtual machine array + - name: TEMPLATE-VM # name of Virtual Machine + description: TEMPLATE description + image name: image_name.qcow2 + # image metadata: {"bus":"ide", "os_type":"windows", "use_incremental": "no" } #Optional + # processor: #Optional + # model: Intel(R) Xeon(R) CPU E5-4620 0 @ 2.20GHz + # features: ["64b", "iommu", "lps", "tlbps", "hwsv", "dioc", "ht"] + # hypervisor: #Optional + # type: QEMU-kvm + # version: "10002|12001|2.6.32-358.el6.x86_64" + vcpus: 1 # Only for traditional cloud VMs. Number of virtual CPUs (oversubscription is allowed). + ram: 1000 # Only for traditional cloud VMs. Memory in MBytes (not from hugepages, oversubscription is allowed) + disk: 5 # disk size in GiB, by default 1 + #numas: + #- paired-threads: 5 # "cores", "paired-threads", "threads" + # paired-threads-id: [ [0,1], [2,3], [4,5], [6,7], [8,9] ] # By default follows incremental order + # memory: 14 # GBytes + # interfaces: [] + bridge-ifaces: + - name: mgmt0 + vpci: "0000:00:0a.0" # Optional. Virtual PCI address + bandwidth: 1 Mbps # Optional. Informative only + # mac_address: '20:33:45:56:77:46' #avoid this option if possible + # model: 'virtio' # ("virtio","e1000","ne2k_pci","pcnet","rtl8139") By default, it is automatically filled by libvirt + devices: # Optional, order determines device letter asignation (hda, hdb, ...) + - type: disk # "disk","cdrom","xml" + image name: image_name.qcow2 + size: 2 + # image metadata: {"bus":"ide", "os_type":"windows", "use_incremental": "no" } + # vpci: "0000:00:03.0" # Optional, not for disk or cdrom + # Additional Virtual Machines would be included here + diff --git a/test/RO_tests/no_port_security/scenario_vnf_no_port_security.yaml b/test/RO_tests/no_port_security/scenario_vnf_no_port_security.yaml new file mode 100644 index 00000000..07c85abb --- /dev/null +++ b/test/RO_tests/no_port_security/scenario_vnf_no_port_security.yaml @@ -0,0 +1,40 @@ +## +# Copyright 2015 Telefónica Investigación y Desarrollo, S.A.U. +# This file is part of openmano +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +# +# For those usages not covered by the Apache License, Version 2.0 please +# contact with: nfvlabs@tid.es +## +--- +schema_version: 2 +scenario: + name: vnf_no_port_security + description: vnf_no_port_security + public: false # if available for other tenants + vnfs: + vnf_no_port_security: # vnf name in the scenario + #identify an already openmano uploaded VNF either by vnf_id (uuid, prefered) or vnf_name + #vnf_id: 0c0dcc20-c5d5-11e6-a9fb-fa163e2ae06e #prefered id method + vnf_name: vnf_no_port_security #can fail if several vnfs matches this name + #graph: {"y":399,"x":332,"ifaces":{"left":[["xe0","d"],["xe1","d"]],"bottom":[["eth0","v"],["eth1","m"]]}} + networks: + mgmt: + # Connections based on external networks (datacenter nets) must include the external network in the list of nodes + type: bridge + external: true #this will be connected outside + interfaces: + - vnf_no_port_security: mgmt0 + diff --git a/test/RO_tests/no_port_security/vnfd_no_port_security.yaml b/test/RO_tests/no_port_security/vnfd_no_port_security.yaml new file mode 100644 index 00000000..4815298a --- /dev/null +++ b/test/RO_tests/no_port_security/vnfd_no_port_security.yaml @@ -0,0 +1,59 @@ +## +# Copyright 2015 Telefónica Investigación y Desarrollo, S.A.U. +# This file is part of openmano +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +# +# For those usages not covered by the Apache License, Version 2.0 please +# contact with: nfvlabs@tid.es +## +--- +vnf: + name: vnf_no_port_security + description: VNF disabling port_security option in mgmt interface + # class: parent # Optional. Used to organize VNFs + external-connections: + - name: mgmt0 + type: mgmt # "mgmt" (autoconnect to management net), "bridge", "data" + VNFC: vnf_no_port_security # Virtual Machine this interface belongs to + local_iface_name: mgmt0 # interface name inside this Virtual Machine (must be defined in the VNFC section) + description: Management interface + VNFC: # Virtual machine array + - name: vnf_no_port_security # name of Virtual Machine + description: vnf_no_port_security + image name: ubuntu16.04 + # image metadata: {"bus":"ide", "os_type":"windows", "use_incremental": "no" } #Optional + # processor: #Optional + # model: Intel(R) Xeon(R) CPU E5-4620 0 @ 2.20GHz + # features: ["64b", "iommu", "lps", "tlbps", "hwsv", "dioc", "ht"] + # hypervisor: #Optional + # type: QEMU-kvm + # version: "10002|12001|2.6.32-358.el6.x86_64" + vcpus: 1 # Only for traditional cloud VMs. Number of virtual CPUs (oversubscription is allowed). + ram: 1000 # Only for traditional cloud VMs. Memory in MBytes (not from hugepages, oversubscription is allowed) + disk: 5 # disk size in GiB, by default 1 + #numas: + #- paired-threads: 5 # "cores", "paired-threads", "threads" + # paired-threads-id: [ [0,1], [2,3], [4,5], [6,7], [8,9] ] # By default follows incremental order + # memory: 14 # GBytes + # interfaces: [] + bridge-ifaces: + - name: mgmt0 + vpci: "0000:00:0a.0" # Optional. Virtual PCI address + bandwidth: 1 Mbps # Optional. Informative only + port-security: False + # mac_address: '20:33:45:56:77:46' #avoid this option if possible + # model: 'virtio' # ("virtio","e1000","ne2k_pci","pcnet","rtl8139") By default, it is automatically filled by libvirt + # Additional Virtual Machines would be included here + diff --git a/test/RO_tests/simple_cloud_init/scenario_simple-cloud-init.yaml b/test/RO_tests/simple_cloud_init/scenario_simple-cloud-init.yaml new file mode 100644 index 00000000..18ed3e9b --- /dev/null +++ b/test/RO_tests/simple_cloud_init/scenario_simple-cloud-init.yaml @@ -0,0 +1,34 @@ +## +# Copyright 2015 Telefónica Investigación y Desarrollo, S.A.U. +# This file is part of openmano +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +# +# For those usages not covered by the Apache License, Version 2.0 please +# contact with: nfvlabs@tid.es +## +--- +schema_version: 2 +scenario: + name: simple-cloud-init + description: Simple network scenario consisting of a single VNF connected to an external network + vnfs: + linux1: # vnf/net name in the scenario + vnf_name: linux-cloud-init # VNF name as introduced in OPENMANO DB + networks: + mgmt: # provide a name for this net or connection + external: true + interfaces: + - linux1: eth0 # Node and its interface + diff --git a/test/RO_tests/simple_cloud_init/vnfd_linux-cloud-init.yaml b/test/RO_tests/simple_cloud_init/vnfd_linux-cloud-init.yaml new file mode 100644 index 00000000..92e14fb2 --- /dev/null +++ b/test/RO_tests/simple_cloud_init/vnfd_linux-cloud-init.yaml @@ -0,0 +1,67 @@ +## +# Copyright 2015 Telefónica Investigación y Desarrollo, S.A.U. +# This file is part of openmano +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +# +# For those usages not covered by the Apache License, Version 2.0 please +# contact with: nfvlabs@tid.es +## + +--- +schema_version: "0.2" +vnf: + name: linux-cloud-init + description: Single-VM VNF with a traditional cloud VM based on generic Linux OS + external-connections: + - name: eth0 + type: mgmt + description: General purpose interface + VNFC: linux-VM + local_iface_name: eth0 + VNFC: + - name: linux-VM + description: Generic Linux Virtual Machine + #Copy the image to a compute path and edit this path + image name: ubuntu16.04 + vcpus: 1 # Only for traditional cloud VMs. Number of virtual CPUs (oversubscription is allowed). + ram: 2048 # Only for traditional cloud VMs. Memory in MBytes (not from hugepages, oversubscription is allowed) + disk: 20 + bridge-ifaces: + - name: eth0 + vpci: "0000:00:11.0" + numas: [] + boot-data: + key-pairs: + - ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQCy2w9GHMKKNkpCmrDK2ovc3XBYDETuLWwaW24S+feHhLBQiZlzh3gSQoINlA+2ycM9zYbxl4BGzEzpTVyCQFZv5PidG4m6ox7LR+KYkDcITMyjsVuQJKDvt6oZvRt6KbChcCi0n2JJD/oUiJbBFagDBlRslbaFI2mmqmhLlJ5TLDtmYxzBLpjuX4m4tv+pdmQVfg7DYHsoy0hllhjtcDlt1nn05WgWYRTu7mfQTWfVTavu+OjIX3e0WN6NW7yIBWZcE/Q9lC0II3W7PZDE3QaT55se4SPIO2JTdqsx6XGbekdG1n6adlduOI27sOU5m4doiyJ8554yVbuDB/z5lRBD alfonso.tiernosepulveda@telefonica.com + users: + - name: atierno + key-pairs: + - ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQCy2w9GHMKKNkpCmrDK2ovc3XBYDETuLWwaW24S+feHhLBQiZlzh3gSQoINlA+2ycM9zYbxl4BGzEzpTVyCQFZv5PidG4m6ox7LR+KYkDcITMyjsVuQJKDvt6oZvRt6KbChcCi0n2JJD/oUiJbBFagDBlRslbaFI2mmqmhLlJ5TLDtmYxzBLpjuX4m4tv+pdmQVfg7DYHsoy0hllhjtcDlt1nn05WgWYRTu7mfQTWfVTavu+OjIX3e0WN6NW7yIBWZcE/Q9lC0II3W7PZDE3QaT55se4SPIO2JTdqsx6XGbekdG1n6adlduOI27sOU5m4doiyJ8554yVbuDB/z5lRBD alfonso.tiernosepulveda@telefonica.com + boot-data-drive: true + config-files: + - content: | + auto enp0s3 + iface enp0s3 inet dhcp + dest: /etc/network/interfaces.d/enp0s3.cfg + permissions: '0644' + owner: root:root + - content: | + #! /bin/bash + ls -al >> /var/log/osm.log + dest: /etc/rc.local + permissions: '0755' + - content: "file content" + dest: /etc/test_delete + diff --git a/test/RO_tests/simple_linux/scenario_simple_linux.yaml b/test/RO_tests/simple_linux/scenario_simple_linux.yaml new file mode 100644 index 00000000..a7c2087b --- /dev/null +++ b/test/RO_tests/simple_linux/scenario_simple_linux.yaml @@ -0,0 +1,34 @@ +## +# Copyright 2015 Telefónica Investigación y Desarrollo, S.A.U. +# This file is part of openmano +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +# +# For those usages not covered by the Apache License, Version 2.0 please +# contact with: nfvlabs@tid.es +## +--- +schema_version: 2 +scenario: + name: simple + description: Simple network scenario consisting of a single VNF connected to an external network + vnfs: + linux1: # vnf/net name in the scenario + vnf_name: linux # VNF name as introduced in OPENMANO DB + networks: + mgmt: # provide a name for this net or connection + external: true + interfaces: + - linux1: eth0 # Node and its interface + diff --git a/test/RO_tests/simple_linux/vnfd_linux.yaml b/test/RO_tests/simple_linux/vnfd_linux.yaml new file mode 100644 index 00000000..47c84984 --- /dev/null +++ b/test/RO_tests/simple_linux/vnfd_linux.yaml @@ -0,0 +1,42 @@ +## +# Copyright 2015 Telefónica Investigación y Desarrollo, S.A.U. +# This file is part of openmano +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +# +# For those usages not covered by the Apache License, Version 2.0 please +# contact with: nfvlabs@tid.es +## +--- +vnf: + name: linux + description: Single-VM VNF with a traditional cloud VM based on generic Linux OS + external-connections: + - name: eth0 + type: bridge + VNFC: linux-VM + local_iface_name: eth0 + description: General purpose interface + VNFC: + - name: linux-VM + description: Generic Linux Virtual Machine + #Copy the image to a compute path and edit this path + image name: image_name.qcow2 + vcpus: 1 # Only for traditional cloud VMs. Number of virtual CPUs (oversubscription is allowed). + ram: 1024 # Only for traditional cloud VMs. Memory in MBytes (not from hugepages, oversubscription is allowed) + disk: 10 + bridge-ifaces: + - name: eth0 + vpci: "0000:00:11.0" + numas: [] diff --git a/test/RO_tests/simple_multi_vnfc/scenario_multi_vnfc.yaml b/test/RO_tests/simple_multi_vnfc/scenario_multi_vnfc.yaml new file mode 100644 index 00000000..8de79937 --- /dev/null +++ b/test/RO_tests/simple_multi_vnfc/scenario_multi_vnfc.yaml @@ -0,0 +1,34 @@ +## +# Copyright 2015 Telefónica Investigación y Desarrollo, S.A.U. +# This file is part of openmano +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +# +# For those usages not covered by the Apache License, Version 2.0 please +# contact with: nfvlabs@tid.es +## +--- +schema_version: 2 +scenario: + name: simple_multi_vnfc + description: Simple network scenario consisting of a multi VNFC VNF connected to an external network + vnfs: + linux1: # vnf/net name in the scenario + vnf_name: linux_2VMs_v02 # VNF name as introduced in OPENMANO DB + networks: + mgmt: # provide a name for this net or connection + external: true + interfaces: + - linux1: control0 # Node and its interface + diff --git a/test/RO_tests/simple_multi_vnfc/vnfd_linux_2VMs_v02.yaml b/test/RO_tests/simple_multi_vnfc/vnfd_linux_2VMs_v02.yaml new file mode 100644 index 00000000..3a09672e --- /dev/null +++ b/test/RO_tests/simple_multi_vnfc/vnfd_linux_2VMs_v02.yaml @@ -0,0 +1,104 @@ +## +# Copyright 2015 Telefónica Investigación y Desarrollo, S.A.U. +# This file is part of openmano +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +# +# For those usages not covered by the Apache License, Version 2.0 please +# contact with: nfvlabs@tid.es +## +--- +schema_version: "0.2" +vnf: + name: linux_2VMs_v02 + description: "Example of a linux VNF consisting of two VMs with one internal network" + # class: parent # Optional. Used to organize VNFs + internal-connections: + - name: internalnet + description: internalnet + type: e-lan + implementation: overlay + ip-profile: + ip-version: IPv4 + subnet-address: 192.168.1.0/24 + gateway-address: 192.168.1.1 + dns-address: 8.8.8.8 + dhcp: + enabled: true + start-address: 192.168.1.100 + count: 100 + elements: + - VNFC: linux_2VMs-VM1 + local_iface_name: xe0 + ip_address: 192.168.1.2 + - VNFC: linux_2VMs-VM2 + local_iface_name: xe0 + ip_address: 192.168.1.3 + external-connections: + - name: control0 + type: mgmt + VNFC: linux_2VMs-VM1 + local_iface_name: eth0 + description: control interface VM1 + - name: control1 + type: mgmt + VNFC: linux_2VMs-VM2 + local_iface_name: eth0 + description: control interface VM2 + - name: in + type: bridge + VNFC: linux_2VMs-VM1 + local_iface_name: xe1 + description: data interface input + - name: out + type: bridge + VNFC: linux_2VMs-VM2 + local_iface_name: xe1 + description: data interface output + VNFC: + - name: linux_2VMs-VM1 + description: "Linux VM1 with 4 CPUs, 2 GB RAM and 3 bridge interfaces" + #Copy the image to a compute path and edit this path + image name: TestVM + disk: 10 + vcpus: 4 + ram: 2048 + bridge-ifaces: + - name: eth0 + vpci: "0000:00:09.0" + bandwidth: 1 Mbps # Optional, informative only + - name: xe0 + vpci: "0000:00:11.0" + bandwidth: 1 Mbps + - name: xe1 + vpci: "0000:00:12.0" + bandwidth: 1 Mbps + - name: linux_2VMs-VM2 + description: "Linux VM2 with 2 CPUs, 2 GB RAM and 3 bridge interfaces" + #Copy the image to a compute path and edit this path + image name: TestVM + disk: 10 + vcpus: 2 + ram: 2048 + bridge-ifaces: + - name: eth0 + vpci: "0000:00:09.0" + bandwidth: 1 Mbps # Optional, informative only + - name: xe0 + vpci: "0000:00:11.0" + bandwidth: 1 Mbps + - name: xe1 + vpci: "0000:00:12.0" + bandwidth: 1 Mbps + diff --git a/test/test_RO.py b/test/test_RO.py new file mode 100755 index 00000000..d6667556 --- /dev/null +++ b/test/test_RO.py @@ -0,0 +1,623 @@ +#!/usr/bin/env python2 +# -*- coding: utf-8 -*- + +## +# Copyright 2015 Telefónica Investigación y Desarrollo, S.A.U. +# This file is part of openmano +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +# +# For those usages not covered by the Apache License, Version 2.0 please +# contact with: nfvlabs@tid.es +## + +''' +Module for testing openmano functionality. It uses openmanoclient.py for invoking openmano +''' +__author__="Pablo Montes" +__date__ ="$16-Feb-2017 17:08:16$" +__version__="0.0.1" +version_date="Feb 2017" + +import logging +import imp +import os +from optparse import OptionParser +import unittest +import string +import inspect +import random +import traceback +import glob +import yaml +import sys +import time + +global test_number +global test_directory +global scenario_test_folder +global test_image_name +global management_network + +''' +IMPORTANT NOTE +All unittest classes for code based tests must have prefix 'test_' in order to be taken into account for tests +''' +class test_tenant_operations(unittest.TestCase): + test_index = 1 + tenant_name = None + test_text = None + + @classmethod + def setUpClass(cls): + logger.info("{}. {}".format(test_number, cls.__name__)) + + @classmethod + def tearDownClass(cls): + globals().__setitem__('test_number', globals().__getitem__('test_number') + 1) + + def tearDown(self): + exec_info = sys.exc_info() + if exec_info == (None, None, None): + logger.info(self.__class__.test_text+" -> TEST OK") + else: + logger.warning(self.__class__.test_text+" -> TEST NOK") + error_trace = traceback.format_exception(exec_info[0], exec_info[1], exec_info[2]) + msg = "" + for line in error_trace: + msg = msg + line + logger.critical("{}".format(msg)) + + def test_000_create_RO_tenant(self): + self.__class__.tenant_name = _get_random_string(20) + self.__class__.test_text = "{}.{}. TEST {}".format(test_number, self.__class__.test_index, + inspect.currentframe().f_code.co_name) + self.__class__.test_index += 1 + tenant = client.create_tenant(name=self.__class__.tenant_name, description=self.__class__.tenant_name) + logger.debug("{}".format(tenant)) + self.assertEqual(tenant.get('tenant', {}).get('name', ''), self.__class__.tenant_name) + + def test_010_list_RO_tenant(self): + self.__class__.test_text = "{}.{}. TEST {}".format(test_number, self.__class__.test_index, + inspect.currentframe().f_code.co_name) + self.__class__.test_index += 1 + tenant = client.get_tenant(name=self.__class__.tenant_name) + logger.debug("{}".format(tenant)) + self.assertEqual(tenant.get('tenant', {}).get('name', ''), self.__class__.tenant_name) + + def test_020_delete_RO_tenant(self): + self.__class__.test_text = "{}.{}. TEST {}".format(test_number, self.__class__.test_index, + inspect.currentframe().f_code.co_name) + self.__class__.test_index += 1 + tenant = client.delete_tenant(name=self.__class__.tenant_name) + logger.debug("{}".format(tenant)) + assert('deleted' in tenant.get('result',"")) + +class test_datacenter_operations(unittest.TestCase): + test_index = 1 + datacenter_name = None + test_text = None + + @classmethod + def setUpClass(cls): + logger.info("{}. {}".format(test_number, cls.__name__)) + + @classmethod + def tearDownClass(cls): + globals().__setitem__('test_number', globals().__getitem__('test_number') + 1) + + def tearDown(self): + exec_info = sys.exc_info() + if exec_info == (None, None, None): + logger.info(self.__class__.test_text+" -> TEST OK") + else: + logger.warning(self.__class__.test_text+" -> TEST NOK") + error_trace = traceback.format_exception(exec_info[0], exec_info[1], exec_info[2]) + msg = "" + for line in error_trace: + msg = msg + line + logger.critical("{}".format(msg)) + + def test_000_create_datacenter(self): + self.__class__.test_text = "{}.{}. TEST {}".format(test_number, self.__class__.test_index, + inspect.currentframe().f_code.co_name) + self.__class__.datacenter_name = _get_random_string(20) + self.__class__.test_index += 1 + self.datacenter = client.create_datacenter(name=self.__class__.datacenter_name, vim_url="http://fakeurl/fake") + logger.debug("{}".format(self.datacenter)) + self.assertEqual (self.datacenter.get('datacenter', {}).get('name',''), self.__class__.datacenter_name) + + def test_010_list_datacenter(self): + self.__class__.test_text = "{}.{}. TEST {}".format(test_number, self.__class__.test_index, + inspect.currentframe().f_code.co_name) + + self.__class__.test_index += 1 + self.datacenter = client.get_datacenter(all_tenants=True, name=self.__class__.datacenter_name) + logger.debug("{}".format(self.datacenter)) + self.assertEqual (self.datacenter.get('datacenter', {}).get('name', ''), self.__class__.datacenter_name) + + def test_020_attach_datacenter(self): + self.__class__.test_text = "{}.{}. TEST {}".format(test_number, self.__class__.test_index, + inspect.currentframe().f_code.co_name) + + self.__class__.test_index += 1 + self.datacenter = client.attach_datacenter(name=self.__class__.datacenter_name, vim_tenant_name='fake') + logger.debug("{}".format(self.datacenter)) + assert ('vim_tenants' in self.datacenter.get('datacenter', {})) + + def test_030_list_attached_datacenter(self): + self.__class__.test_text = "{}.{}. TEST {}".format(test_number, self.__class__.test_index, + inspect.currentframe().f_code.co_name) + + self.__class__.test_index += 1 + self.datacenter = client.get_datacenter(all_tenants=False, name=self.__class__.datacenter_name) + logger.debug("{}".format(self.datacenter)) + self.assertEqual (self.datacenter.get('datacenter', {}).get('name', ''), self.__class__.datacenter_name) + + def test_040_detach_datacenter(self): + self.__class__.test_text = "{}.{}. TEST {}".format(test_number, self.__class__.test_index, + inspect.currentframe().f_code.co_name) + + self.__class__.test_index += 1 + self.datacenter = client.detach_datacenter(name=self.__class__.datacenter_name) + logger.debug("{}".format(self.datacenter)) + assert ('detached' in self.datacenter.get('result', "")) + + def test_050_delete_datacenter(self): + self.__class__.test_text = "{}.{}. TEST {}".format(test_number, self.__class__.test_index, + inspect.currentframe().f_code.co_name) + + self.__class__.test_index += 1 + self.datacenter = client.delete_datacenter(name=self.__class__.datacenter_name) + logger.debug("{}".format(self.datacenter)) + assert('deleted' in self.datacenter.get('result',"")) + +class test_VIM_network_operations(unittest.TestCase): + test_index = 1 + vim_network_name = None + test_text = None + vim_network_uuid = None + + @classmethod + def setUpClass(cls): + logger.info("{}. {}".format(test_number, cls.__name__)) + + @classmethod + def tearDownClass(cls): + globals().__setitem__('test_number', globals().__getitem__('test_number') + 1) + + def tearDown(self): + exec_info = sys.exc_info() + if exec_info == (None, None, None): + logger.info(self.__class__.test_text + " -> TEST OK") + else: + logger.warning(self.__class__.test_text + " -> TEST NOK") + error_trace = traceback.format_exception(exec_info[0], exec_info[1], exec_info[2]) + msg = "" + for line in error_trace: + msg = msg + line + logger.critical("{}".format(msg)) + + def test_000_create_VIM_network(self): + self.__class__.test_text = "{}.{}. TEST {}".format(test_number, self.__class__.test_index, + inspect.currentframe().f_code.co_name) + self.__class__.vim_network_name = _get_random_string(20) + self.__class__.test_index += 1 + network = client.vim_action("create", "networks", name=self.__class__.vim_network_name) + logger.debug("{}".format(network)) + self.__class__.vim_network_uuid = network["network"]["id"] + self.assertEqual(network.get('network', {}).get('name', ''), self.__class__.vim_network_name) + + def test_010_list_VIM_networks(self): + self.__class__.test_text = "{}.{}. TEST {}".format(test_number, self.__class__.test_index, + inspect.currentframe().f_code.co_name) + self.__class__.test_index += 1 + networks = client.vim_action("list", "networks") + logger.debug("{}".format(networks)) + + def test_020_get_VIM_network_by_uuid(self): + self.__class__.test_text = "{}.{}. TEST {}".format(test_number, self.__class__.test_index, + inspect.currentframe().f_code.co_name) + + self.__class__.test_index += 1 + network = client.vim_action("show", "networks", uuid=self.__class__.vim_network_uuid) + logger.debug("{}".format(network)) + self.assertEqual(network.get('network', {}).get('name', ''), self.__class__.vim_network_name) + + def test_030_delete_VIM_network_by_uuid(self): + self.__class__.test_text = "{}.{}. TEST {}".format(test_number, self.__class__.test_index, + inspect.currentframe().f_code.co_name) + + self.__class__.test_index += 1 + network = client.vim_action("delete", "networks", uuid=self.__class__.vim_network_uuid) + logger.debug("{}".format(network)) + assert ('deleted' in network.get('result', "")) + +class test_VIM_image_operations(unittest.TestCase): + test_index = 1 + test_text = None + + @classmethod + def setUpClass(cls): + logger.info("{}. {}".format(test_number, cls.__name__)) + + @classmethod + def tearDownClass(cls): + globals().__setitem__('test_number', globals().__getitem__('test_number') + 1) + + def tearDown(self): + exec_info = sys.exc_info() + if exec_info == (None, None, None): + logger.info(self.__class__.test_text + " -> TEST OK") + else: + logger.warning(self.__class__.test_text + " -> TEST NOK") + error_trace = traceback.format_exception(exec_info[0], exec_info[1], exec_info[2]) + msg = "" + for line in error_trace: + msg = msg + line + logger.critical("{}".format(msg)) + + def test_000_list_VIM_images(self): + self.__class__.test_text = "{}.{}. TEST {}".format(test_number, self.__class__.test_index, + inspect.currentframe().f_code.co_name) + self.__class__.test_index += 1 + images = client.vim_action("list", "images") + logger.debug("{}".format(images)) + +''' +The following is a non critical test that will fail most of the times. +In case of OpenStack datacenter these tests will only success if RO has access to the admin endpoint +This test will only be executed in case it is specifically requested by the user +''' +class test_VIM_tenant_operations(unittest.TestCase): + test_index = 1 + vim_tenant_name = None + test_text = None + vim_tenant_uuid = None + + @classmethod + def setUpClass(cls): + logger.info("{}. {}".format(test_number, cls.__name__)) + logger.warning("In case of OpenStack datacenter these tests will only success " + "if RO has access to the admin endpoint") + + @classmethod + def tearDownClass(cls): + globals().__setitem__('test_number', globals().__getitem__('test_number') + 1) + + def tearDown(self): + exec_info = sys.exc_info() + if exec_info == (None, None, None): + logger.info(self.__class__.test_text + " -> TEST OK") + else: + logger.warning(self.__class__.test_text + " -> TEST NOK") + error_trace = traceback.format_exception(exec_info[0], exec_info[1], exec_info[2]) + msg = "" + for line in error_trace: + msg = msg + line + logger.critical("{}".format(msg)) + + def test_000_create_VIM_tenant(self): + self.__class__.test_text = "{}.{}. TEST {}".format(test_number, self.__class__.test_index, + inspect.currentframe().f_code.co_name) + self.__class__.vim_tenant_name = _get_random_string(20) + self.__class__.test_index += 1 + tenant = client.vim_action("create", "tenants", name=self.__class__.vim_tenant_name) + logger.debug("{}".format(tenant)) + self.__class__.vim_tenant_uuid = tenant["tenant"]["id"] + self.assertEqual(tenant.get('tenant', {}).get('name', ''), self.__class__.vim_tenant_name) + + def test_010_list_VIM_tenants(self): + self.__class__.test_text = "{}.{}. TEST {}".format(test_number, self.__class__.test_index, + inspect.currentframe().f_code.co_name) + self.__class__.test_index += 1 + tenants = client.vim_action("list", "tenants") + logger.debug("{}".format(tenants)) + + def test_020_get_VIM_tenant_by_uuid(self): + self.__class__.test_text = "{}.{}. TEST {}".format(test_number, self.__class__.test_index, + inspect.currentframe().f_code.co_name) + + self.__class__.test_index += 1 + tenant = client.vim_action("show", "tenants", uuid=self.__class__.vim_tenant_uuid) + logger.debug("{}".format(tenant)) + self.assertEqual(tenant.get('tenant', {}).get('name', ''), self.__class__.vim_tenant_name) + + def test_030_delete_VIM_tenant_by_uuid(self): + self.__class__.test_text = "{}.{}. TEST {}".format(test_number, self.__class__.test_index, + inspect.currentframe().f_code.co_name) + + self.__class__.test_index += 1 + tenant = client.vim_action("delete", "tenants", uuid=self.__class__.vim_tenant_uuid) + logger.debug("{}".format(tenant)) + assert ('deleted' in tenant.get('result', "")) + +''' +IMPORTANT NOTE +The following unittest class does not have the 'test_' on purpose. This test is the one used for the +scenario based tests. +''' +class descriptor_based_scenario_test(unittest.TestCase): + test_index = 0 + test_text = None + scenario_test_path = None + scenario_uuid = None + instance_scenario_uuid = None + to_delete_list = [] + + @classmethod + def setUpClass(cls): + cls.test_index = 1 + cls.to_delete_list = [] + cls.scenario_test_path = test_directory + '/' + scenario_test_folder + logger.info("{}. {} {}".format(test_number, cls.__name__, scenario_test_folder)) + + @classmethod + def tearDownClass(cls): + globals().__setitem__('test_number', globals().__getitem__('test_number') + 1) + + def tearDown(self): + exec_info = sys.exc_info() + if exec_info == (None, None, None): + logger.info(self.__class__.test_text + " -> TEST OK") + else: + logger.warning(self.__class__.test_text + " -> TEST NOK") + error_trace = traceback.format_exception(exec_info[0], exec_info[1], exec_info[2]) + msg = "" + for line in error_trace: + msg = msg + line + logger.critical("{}".format(msg)) + + + def test_000_load_scenario(self): + self.__class__.test_text = "{}.{}. TEST {} {}".format(test_number, self.__class__.test_index, + inspect.currentframe().f_code.co_name, + scenario_test_folder) + self.__class__.test_index += 1 + vnfd_files = glob.glob(self.__class__.scenario_test_path+'/vnfd_*.yaml') + scenario_file = glob.glob(self.__class__.scenario_test_path + '/scenario_*.yaml') + if len(vnfd_files) == 0 or len(scenario_file) > 1: + raise Exception('Test '+scenario_test_folder+' not valid. It must contain an scenario file and at least one' + ' vnfd file') + + #load all vnfd + for vnfd in vnfd_files: + with open(vnfd, 'r') as stream: + vnf_descriptor = yaml.load(stream) + + vnfc_list = vnf_descriptor['vnf']['VNFC'] + for vnfc in vnfc_list: + vnfc['image name'] = test_image_name + devices = vnfc.get('devices',[]) + for device in devices: + if device['type'] == 'disk' and 'image name' in device: + device['image name'] = test_image_name + + logger.debug("VNF descriptor: {}".format(vnf_descriptor)) + vnf = client.create_vnf(descriptor=vnf_descriptor) + logger.debug(vnf) + self.__class__.to_delete_list.insert(0, {"item": "vnf", "function": client.delete_vnf, + "params": {"uuid": vnf['vnf']['uuid']}}) + + #load the scenario definition + with open(scenario_file[0], 'r') as stream: + scenario_descriptor = yaml.load(stream) + networks = scenario_descriptor['scenario']['networks'] + networks[management_network] = networks.pop('mgmt') + logger.debug("Scenario descriptor: {}".format(scenario_descriptor)) + scenario = client.create_scenario(descriptor=scenario_descriptor) + logger.debug(scenario) + self.__class__.to_delete_list.insert(0,{"item": "scenario", "function": client.delete_scenario, + "params":{"uuid": scenario['scenario']['uuid']} }) + self.__class__.scenario_uuid = scenario['scenario']['uuid'] + + def test_010_instantiate_scenario(self): + self.__class__.test_text = "{}.{}. TEST {} {}".format(test_number, self.__class__.test_index, + inspect.currentframe().f_code.co_name, + scenario_test_folder) + self.__class__.test_index += 1 + + instance = client.create_instance(scenario_id=self.__class__.scenario_uuid, name=self.__class__.test_text) + logger.debug(instance) + self.__class__.to_delete_list.insert(0, {"item": "instance", "function": client.delete_instance, + "params": {"uuid": instance['uuid']}}) + + def test_020_clean_deployment(self): + self.__class__.test_text = "{}.{}. TEST {} {}".format(test_number, self.__class__.test_index, + inspect.currentframe().f_code.co_name, + scenario_test_folder) + self.__class__.test_index += 1 + #At the moment if you delete an scenario right after creating it, in openstack datacenters + #sometimes scenario ports get orphaned. This sleep is just a dirty workaround + time.sleep(5) + for item in self.__class__.to_delete_list: + response = item["function"](**item["params"]) + logger.debug(response) + +def _get_random_string(maxLength): + '''generates a string with random characters string.letters and string.digits + with a random length up to maxLength characters. If maxLength is <15 it will be changed automatically to 15 + ''' + prefix = 'testing_' + min_string = 15 + minLength = min_string - len(prefix) + if maxLength < min_string: maxLength = min_string + maxLength -= len(prefix) + length = random.randint(minLength,maxLength) + return 'testing_'+"".join([random.choice(string.letters+string.digits) for i in xrange(length)]) + +if __name__=="__main__": + sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) + import openmanoclient + + parser = OptionParser() + + #Optional arguments + parser.add_option("-v",'--version', help='Show current version', dest='version', action="store_true", default=False) + parser.add_option('--debug', help='Set logs to debug level', dest='debug', action="store_true", default=False) + parser.add_option('--failed', help='Set logs to show only failed tests. --debug disables this option', + dest='failed', action="store_true", default=False) + parser.add_option('-u', '--url', dest='endpoint_url', help='Set the openmano server url. By default ' + 'http://localhost:9090/openmano', + default='http://localhost:9090/openmano') + default_logger_file = os.path.dirname(__file__)+'/'+os.path.splitext(os.path.basename(__file__))[0]+'.log' + parser.add_option('--logger_file', dest='logger_file', help='Set the logger file. By default '+default_logger_file, + default=default_logger_file) + parser.add_option('--list-tests', help='List all available tests', dest='list-tests', action="store_true", + default=False) + parser.add_option('--test', '--tests', help='Specify the tests to run', dest='tests', default=None) + + #Mandatory arguments + parser.add_option("-t", '--tenant', dest='tenant_name', help='MANDATORY. Set the tenant name to test') + parser.add_option('-d', '--datacenter', dest='datacenter_name', help='MANDATORY, Set the datacenter name to test') + parser.add_option("-i", '--image-name', dest='image-name', help='MANDATORY. Image name of an Ubuntu 16.04 image ' + 'that will be used for testing available in the ' + 'datacenter.') + parser.add_option("-n", '--mgmt-net-name', dest='mgmt-net', help='MANDATORY. Set the tenant name to test') + + (options, args) = parser.parse_args() + + # default logger level is INFO. Options --debug and --failed override this, being --debug prioritary + logger_level = 'INFO' + if options.__dict__['debug']: + logger_level = 'DEBUG' + elif options.__dict__['failed']: + logger_level = 'WARNING' + logger_name = os.path.basename(__file__) + logger = logging.getLogger(logger_name) + logger.setLevel(logger_level) + + # Configure a logging handler to store in a logging file + fileHandler = logging.FileHandler(options.__dict__['logger_file']) + formatter_fileHandler = logging.Formatter('%(asctime)s %(name)s %(levelname)s: %(message)s') + fileHandler.setFormatter(formatter_fileHandler) + logger.addHandler(fileHandler) + + # Configure a handler to print to stdout + consoleHandler = logging.StreamHandler(sys.stdout) + formatter_consoleHandler = logging.Formatter('%(message)s') + consoleHandler.setFormatter(formatter_consoleHandler) + logger.addHandler(consoleHandler) + + logger.debug('Program started with the following arguments: ' + str(options.__dict__)) + + #If version is required print it and exit + if options.__dict__['version']: + logger.info("{}".format((sys.argv[0], __version__+" version", version_date))) + logger.info ("(c) Copyright Telefonica") + sys.exit(0) + + test_directory = os.path.dirname(__file__) + "/RO_tests" + test_directory_content = os.listdir(test_directory) + clsmembers = inspect.getmembers(sys.modules[__name__], inspect.isclass) + + # If only want to obtain a tests list print it and exit + if options.__dict__['list-tests']: + tests_names = [] + for cls in clsmembers: + if cls[0].startswith('test_'): + tests_names.append(cls[0]) + + msg = "The code based tests are:\n\t" + ', '.join(sorted(tests_names))+'\n'+\ + "The descriptor based tests are:\n\t"+ ', '.join(sorted(test_directory_content))+'\n'+\ + "NOTE: The test test_VIM_tenant_operations will fail in case the used datacenter is type OpenStack " \ + "unless RO has access to the admin endpoint. Therefore this test is excluded by default" + + logger.info(msg) + sys.exit(0) + + #Make sure required arguments are present + required = "tenant_name datacenter_name image-name mgmt-net".split() + error = False + for r in required: + if options.__dict__[r] is None: + print "ERROR: parameter "+r+" is required" + error = True + if error: + parser.print_help() + sys.exit(1) + + # set test image name and management network + test_image_name = options.__dict__['image-name'] + management_network = options.__dict__['mgmt-net'] + + #Create the list of tests to be run + descriptor_based_tests = [] + code_based_tests = [] + if options.__dict__['tests'] != None: + tests = sorted(options.__dict__['tests'].split(',')) + for test in tests: + matches_code_based_tests = [item for item in clsmembers if item[0] == test] + if test in test_directory_content: + descriptor_based_tests.append(test) + elif len(matches_code_based_tests) > 0: + code_based_tests.append(matches_code_based_tests[0][1]) + else: + logger.critical("Test {} is not among the possible ones".format(test)) + sys.exit(1) + else: + #include all tests + descriptor_based_tests = test_directory_content + for cls in clsmembers: + #We exclude 'test_VIM_tenant_operations' unless it is specifically requested by the user + if cls[0].startswith('test_') and cls[0] != 'test_VIM_tenant_operations': + code_based_tests.append(cls[1]) + + logger.debug("descriptor_based_tests to be executed: {}".format(descriptor_based_tests)) + logger.debug("code_based_tests to be executed: {}".format(code_based_tests)) + + # import openmanoclient from relative path + client = openmanoclient.openmanoclient( + endpoint_url=options.__dict__['endpoint_url'], + tenant_name=options.__dict__['tenant_name'], + datacenter_name = options.__dict__['datacenter_name'], + debug = options.__dict__['debug'], logger = logger_name) + + # TextTestRunner stream is set to /dev/null in order to avoid the method to directly print the result of tests. + # This is handled in the tests using logging. + stream = open('/dev/null', 'w') + test_number=1 + executed = 0 + failed = 0 + + #Run code based tests + basic_tests_suite = unittest.TestSuite() + for test in code_based_tests: + basic_tests_suite.addTest(unittest.makeSuite(test)) + result = unittest.TextTestRunner(stream=stream).run(basic_tests_suite) + executed += result.testsRun + failed += len(result.failures) + len(result.errors) + if len(result.failures) > 0: + logger.debug("failures : {}".format(result.failures)) + if len(result.errors) > 0: + logger.debug("errors : {}".format(result.errors)) + + # Additionally to the previous tests, scenario based tests will be executed. + # This scenario based tests are defined as directories inside the directory defined in 'test_directory' + for test in descriptor_based_tests: + scenario_test_folder = test + test_suite = unittest.TestSuite() + test_suite.addTest(unittest.makeSuite(descriptor_based_scenario_test)) + result = unittest.TextTestRunner(stream=stream).run(test_suite) + executed += result.testsRun + failed += len(result.failures) + len(result.errors) + if len(result.failures) > 0: + logger.debug("failures : {}".format(result.failures)) + if len(result.errors) > 0: + logger.debug("errors : {}".format(result.errors)) + + #Log summary + logger.warning("Total number of tests: {}; Total number of failures/errors: {}".format(executed, failed)) + + sys.exit(0) \ No newline at end of file -- 2.25.1 From a7d34d04639c12d4260518c6e92f3f5701f5e150 Mon Sep 17 00:00:00 2001 From: tierno Date: Thu, 23 Feb 2017 14:42:07 +0100 Subject: [PATCH 02/16] vimconn.py: Better documentation. Unify new_network method for all vimconns. Add new parameter persistem_info at constructor. New method check_vim_connectivity. Change-Id: Ie2c7900fe12fd5ae01ef2bb4891b79a0f9173285 Signed-off-by: tierno --- nfvo.py | 3 +- vimconn.py | 349 +++++++++++++++++++++++++++------------------ vimconn_openvim.py | 6 +- 3 files changed, 213 insertions(+), 145 deletions(-) diff --git a/nfvo.py b/nfvo.py index 4565f00a..80a606ba 100644 --- a/nfvo.py +++ b/nfvo.py @@ -2855,7 +2855,8 @@ def vim_action_create(mydb, tenant_id, datacenter, item, descriptor): net_type = net.pop("type", "bridge") net_public = net.pop("shared", False) net_ipprofile = net.pop("ip_profile", None) - content = myvim.new_network(net_name, net_type, net_ipprofile, shared=net_public, **net) + net_vlan = net.pop("vlan", None) + content = myvim.new_network(net_name, net_type, net_ipprofile, shared=net_public, vlan=net_vlan) #, **net) elif item=="tenants": tenant = descriptor["tenant"] content = myvim.new_tenant(tenant["name"], tenant.get("description")) diff --git a/vimconn.py b/vimconn.py index ad06dc8b..c1721cdb 100644 --- a/vimconn.py +++ b/vimconn.py @@ -21,10 +21,10 @@ # contact with: nfvlabs@tid.es ## -''' +""" vimconn implement an Abstract class for the vim connector plugins with the definition of the method to be implemented. -''' +""" __author__="Alfonso Tierno" __date__ ="$16-oct-2015 11:09:29$" @@ -42,57 +42,69 @@ HTTP_Service_Unavailable = 503 HTTP_Internal_Server_Error = 500 class vimconnException(Exception): - '''Common and base class Exception for all vimconnector exceptions''' + """Common and base class Exception for all vimconnector exceptions""" def __init__(self, message, http_code=HTTP_Bad_Request): Exception.__init__(self, message) self.http_code = http_code class vimconnConnectionException(vimconnException): - '''Connectivity error with the VIM''' + """Connectivity error with the VIM""" def __init__(self, message, http_code=HTTP_Service_Unavailable): vimconnException.__init__(self, message, http_code) class vimconnUnexpectedResponse(vimconnException): - '''Get an wrong response from VIM''' + """Get an wrong response from VIM""" def __init__(self, message, http_code=HTTP_Service_Unavailable): vimconnException.__init__(self, message, http_code) class vimconnAuthException(vimconnException): - '''Invalid credentials or authorization to perform this action over the VIM''' + """Invalid credentials or authorization to perform this action over the VIM""" def __init__(self, message, http_code=HTTP_Unauthorized): vimconnException.__init__(self, message, http_code) class vimconnNotFoundException(vimconnException): - '''The item is not found at VIM''' + """The item is not found at VIM""" def __init__(self, message, http_code=HTTP_Not_Found): vimconnException.__init__(self, message, http_code) class vimconnConflictException(vimconnException): - '''There is a conflict, e.g. more item found than one''' + """There is a conflict, e.g. more item found than one""" def __init__(self, message, http_code=HTTP_Conflict): vimconnException.__init__(self, message, http_code) +class vimconnNotSupportedException(vimconnException): + """The request is not supported by connector""" + def __init__(self, message, http_code=HTTP_Service_Unavailable): + vimconnException.__init__(self, message, http_code) + class vimconnNotImplemented(vimconnException): - '''The method is not implemented by the connected''' + """The method is not implemented by the connected""" def __init__(self, message, http_code=HTTP_Not_Implemented): vimconnException.__init__(self, message, http_code) class vimconnector(): - '''Abstract base class for all the VIM connector plugins + """Abstract base class for all the VIM connector plugins These plugins must implement a vimconnector class derived from this - and all these methods - ''' - def __init__(self, uuid, name, tenant_id, tenant_name, url, url_admin=None, user=None, passwd=None, log_level=None, config={}): + and all these privated methods + """ + def __init__(self, uuid, name, tenant_id, tenant_name, url, url_admin=None, user=None, passwd=None, log_level=None, + config={}, persitent_info={}): """Constructor of VIM - "uuid": id asigned to this VIM - "name": name assigned to this VIM, can be used for logging - "tenant_id", tenant_name: VIM tenant to be used - "url_admin": optional, url used for administrative tasks - "user", "passwd": credentials of the VIM user - "log_level": if must use a different log_level than the general one - "config": dictionary with extra VIM information. This contains a consolidate version of general VIM config at create - and particular VIM config at attach - Returns can raise an exception if some check is done and fails + Params: + 'uuid': id asigned to this VIM + 'name': name assigned to this VIM, can be used for logging + 'tenant_id', 'tenant_name': (only one of them is mandatory) VIM tenant to be used + 'url_admin': (optional), url used for administrative tasks + 'user', 'passwd': credentials of the VIM user + 'log_level': provider if it should use a different log_level than the general one + 'config': dictionary with extra VIM information. This contains a consolidate version of general VIM config + at creation and particular VIM config at teh attachment + 'persistent_info': dict where the class can store information that will be available among class + destroy/creation cycles. This info is unique per VIM/credential. At first call it will contain an + empty dict. Useful to store login/tokens information for speed up communication + + Returns: Raise an exception is some needed parameter is missing, but it must not do any connectivity + check against the VIM """ self.id = uuid self.name = name @@ -151,6 +163,12 @@ class vimconnector(): else: raise KeyError("Invalid key '%s'" %str(index)) + def check_vim_connectivity(self): + """Checks VIM can be reached and user credentials are ok. + Returns None if success or raised vimconnConnectionException, vimconnAuthException, ... + """ + raise vimconnNotImplemented( "Should have implemented this" ) + def new_tenant(self,tenant_name,tenant_description): """Adds a new tenant to VIM with this name and description, this is done using admin_url if provided "tenant_name": string max lenght 64 @@ -167,90 +185,118 @@ class vimconnector(): raise vimconnNotImplemented( "Should have implemented this" ) def get_tenant_list(self, filter_dict={}): - '''Obtain tenants of VIM + """Obtain tenants of VIM filter_dict dictionary that can contain the following keys: name: filter by tenant name id: filter by tenant uuid/id Returns the tenant list of dictionaries, and empty list if no tenant match all the filers: [{'name':', 'id':', ...}, ...] - ''' + """ raise vimconnNotImplemented( "Should have implemented this" ) - def new_network(self,net_name, net_type, ip_profile=None, shared=False): + def new_network(self, net_name, net_type, ip_profile=None, shared=False, vlan=None): """Adds a tenant network to VIM - net_name is the name - net_type can be 'bridge','data'.'ptp'. TODO: this need to be revised - ip_profile is a dict containing the IP parameters of the network - "ip-version": {"type":"string", "enum":["IPv4","IPv6"]}, - "subnet-address": ip_prefix_schema, - "gateway-address": ip_schema, - "dns-address": ip_schema, - "dhcp": dhcp_schema - - shared is a boolean - Returns the network identifier on success or raises and exeption on failure + Params: + 'net_name': name of the network + 'net_type': one of: + 'bridge': overlay isolated network + 'data': underlay E-LAN network for Passthrough and SRIOV interfaces + 'ptp': underlay E-LINE network for Passthrough and SRIOV interfaces. + 'ip_profile': is a dict containing the IP parameters of the network (Currently only IPv4 is implemented) + 'ip-version': can be one of ["IPv4","IPv6"] + 'subnet-address': ip_prefix_schema, that is X.X.X.X/Y + 'gateway-address': (Optional) ip_schema, that is X.X.X.X + 'dns-address': (Optional) ip_schema, + 'dhcp': (Optional) dict containing + 'enabled': {"type": "boolean"}, + 'start-address': ip_schema, first IP to grant + 'count': number of IPs to grant. + 'shared': if this network can be seen/use by other tenants/organization + 'vlan': in case of a data or ptp net_type, the intended vlan tag to be used for the network + Returns the network identifier on success or raises and exception on failure """ raise vimconnNotImplemented( "Should have implemented this" ) def get_network_list(self, filter_dict={}): - '''Obtain tenant networks of VIM - Filter_dict can be: - name: network name - id: network uuid - shared: boolean - tenant_id: tenant - admin_state_up: boolean - status: 'ACTIVE' - Returns the network list of dictionaries: - [{}, ...] - List can be empty - ''' + """Obtain tenant networks of VIM + Params: + 'filter_dict' (optional) contains entries to return only networks that matches ALL entries: + name: string => returns only networks with this name + id: string => returns networks with this VIM id, this imply returns one network at most + shared: boolean >= returns only networks that are (or are not) shared + tenant_id: sting => returns only networks that belong to this tenant/project + ,#(not used yet) admin_state_up: boolean => returns only networks that are (or are not) in admin state active + #(not used yet) status: 'ACTIVE','ERROR',... => filter networks that are on this status + Returns the network list of dictionaries. each dictionary contains: + 'id': (mandatory) VIM network id + 'name': (mandatory) VIM network name + 'status': (mandatory) can be 'ACTIVE', 'INACTIVE', 'DOWN', 'BUILD', 'ERROR', 'VIM_ERROR', 'OTHER' + 'error_msg': (optional) text that explains the ERROR status + other VIM specific fields: (optional) whenever possible using the same naming of filter_dict param + List can be empty if no network map the filter_dict. Raise an exception only upon VIM connectivity, + authorization, or some other unspecific error + """ raise vimconnNotImplemented( "Should have implemented this" ) def get_network(self, net_id): - '''Obtain network details of net_id VIM network' - Return a dict with the fields at filter_dict (see get_network_list) plus some VIM specific>}, ...]''' + """Obtain network details from the 'net_id' VIM network + Return a dict that contains: + 'id': (mandatory) VIM network id, that is, net_id + 'name': (mandatory) VIM network name + 'status': (mandatory) can be 'ACTIVE', 'INACTIVE', 'DOWN', 'BUILD', 'ERROR', 'VIM_ERROR', 'OTHER' + 'error_msg': (optional) text that explains the ERROR status + other VIM specific fields: (optional) whenever possible using the same naming of filter_dict param + Raises an exception upon error or when network is not found + """ raise vimconnNotImplemented( "Should have implemented this" ) def delete_network(self, net_id): - '''Deletes a tenant network from VIM, provide the network id. - Returns the network identifier or raise an exception''' + """Deletes a tenant network from VIM + Returns the network identifier or raises an exception upon error or when network is not found + """ raise vimconnNotImplemented( "Should have implemented this" ) def refresh_nets_status(self, net_list): - '''Get the status of the networks - Params: the list of network identifiers - Returns a dictionary with: - net_id: #VIM id of this network - status: #Mandatory. Text with one of: - # DELETED (not found at vim) - # VIM_ERROR (Cannot connect to VIM, VIM response error, ...) - # OTHER (Vim reported other status not understood) - # ERROR (VIM indicates an ERROR status) - # ACTIVE, INACTIVE, DOWN (admin down), - # BUILD (on building process) - # - error_msg: #Text with VIM error message, if any. Or the VIM connection ERROR - vim_info: #Text with plain information obtained from vim (yaml.safe_dump) - - ''' + """Get the status of the networks + Params: + 'net_list': a list with the VIM network id to be get the status + Returns a dictionary with: + 'net_id': #VIM id of this network + status: #Mandatory. Text with one of: + # DELETED (not found at vim) + # VIM_ERROR (Cannot connect to VIM, authentication problems, VIM response error, ...) + # OTHER (Vim reported other status not understood) + # ERROR (VIM indicates an ERROR status) + # ACTIVE, INACTIVE, DOWN (admin down), + # BUILD (on building process) + error_msg: #Text with VIM error message, if any. Or the VIM connection ERROR + vim_info: #Text with plain information obtained from vim (yaml.safe_dump) + 'net_id2': ... + """ raise vimconnNotImplemented( "Should have implemented this" ) def get_flavor(self, flavor_id): - '''Obtain flavor details from the VIM - Returns the flavor dict details {'id':<>, 'name':<>, other vim specific } #TODO to concrete - ''' + """Obtain flavor details from the VIM + Returns the flavor dict details {'id':<>, 'name':<>, other vim specific } + Raises an exception upon error or if not found + """ raise vimconnNotImplemented( "Should have implemented this" ) def get_flavor_id_from_data(self, flavor_dict): """Obtain flavor id that match the flavor description - Returns the flavor_id or raises a vimconnNotFoundException + Params: + 'flavor_dict': dictionary that contains: + 'disk': main hard disk in GB + 'ram': meomry in MB + 'vcpus': number of virtual cpus + #TODO: complete parameters for EPA + Returns the flavor_id or raises a vimconnNotFoundException """ raise vimconnNotImplemented( "Should have implemented this" ) def new_flavor(self, flavor_data): - '''Adds a tenant flavor to VIM + """Adds a tenant flavor to VIM flavor_data contains a dictionary with information, keys: name: flavor name ram: memory (cloud type) in MBytes @@ -266,30 +312,24 @@ class vimconnector(): vpci: requested virtual PCI address disk: disk size is_public: - - - #TODO to concrete - Returns the flavor identifier''' + Returns the flavor identifier""" raise vimconnNotImplemented( "Should have implemented this" ) def delete_flavor(self, flavor_id): - '''Deletes a tenant flavor from VIM identify by its id - Returns the used id or raise an exception''' + """Deletes a tenant flavor from VIM identify by its id + Returns the used id or raise an exception""" raise vimconnNotImplemented( "Should have implemented this" ) - def new_image(self,image_dict): - ''' - Adds a tenant image to VIM - Returns: - 200, image-id if the image is created - <0, message if there is an error - ''' + def new_image(self, image_dict): + """ Adds a tenant image to VIM + Returns the image id or raises an exception if failed + """ raise vimconnNotImplemented( "Should have implemented this" ) def delete_image(self, image_id): - '''Deletes a tenant image from VIM''' - '''Returns the HTTP response code and a message indicating details of the success or fail''' + """Deletes a tenant image from VIM + Returns the image_id if image is deleted or raises an exception on error""" raise vimconnNotImplemented( "Should have implemented this" ) def get_image_id_from_path(self, path): @@ -299,7 +339,7 @@ class vimconnector(): raise vimconnNotImplemented( "Should have implemented this" ) def get_image_list(self, filter_dict={}): - '''Obtain tenant images from VIM + """Obtain tenant images from VIM Filter_dict can be: name: image name id: image uuid @@ -308,44 +348,66 @@ class vimconnector(): Returns the image list of dictionaries: [{}, ...] List can be empty - ''' + """ raise vimconnNotImplemented( "Should have implemented this" ) - def new_vminstance(self,name,description,start,image_id,flavor_id,net_list,cloud_config=None,disk_list=None): - '''Adds a VM instance to VIM + def new_vminstance(self, name, description, start, image_id, flavor_id, net_list, cloud_config=None, + disk_list=None): + """Adds a VM instance to VIM Params: - start: indicates if VM must start or boot in pause mode. Ignored - image_id,flavor_id: image and flavor uuid - net_list: list of interfaces, each one is a dictionary with: - name: - net_id: network uuid to connect - vpci: virtual vcpi to assign - model: interface model, virtio, e2000, ... - mac_address: - use: 'data', 'bridge', 'mgmt' - type: 'virtual', 'PF', 'VF', 'VFnotShared' - vim_id: filled/added by this function - cloud_config: can be a text script to be passed directly to cloud-init, - or an object to inject users and ssh keys with format: - key-pairs: [] list of keys to install to the default user - users: [{ name, key-pairs: []}] list of users to add with their key-pair - #TODO ip, security groups - Returns >=0, the instance identifier - <0, error_text - ''' + 'start': (boolean) indicates if VM must start or created in pause mode. + 'image_id','flavor_id': image and flavor VIM id to use for the VM + 'net_list': list of interfaces, each one is a dictionary with: + 'name': (optional) name for the interface. + 'net_id': VIM network id where this interface must be connect to. Mandatory for type==virtual + 'vpci': (optional) virtual vPCI address to assign at the VM. Can be ignored depending on VIM capabilities + 'model': (optional and only have sense for type==virtual) interface model: virtio, e2000, ... + 'mac_address': (optional) mac address to assign to this interface + #TODO: CHECK if an optional 'vlan' parameter is needed for VIMs when type if VF and net_id is not provided, + the VLAN tag to be used. In case net_id is provided, the internal network vlan is used for tagging VF + 'type': (mandatory) can be one of: + 'virtual', in this case always connected to a network of type 'net_type=bridge' + 'PF' (passthrough): depending on VIM capabilities it can be connected to a data/ptp network ot it + can created unconnected + 'VF' (SRIOV with VLAN tag): same as PF for network connectivity. + 'VFnotShared'(SRIOV without VLAN tag) same as PF for network connectivity. VF where no other VFs + are allocated on the same physical NIC + 'bw': (optional) only for PF/VF/VFnotShared. Minimal Bandwidth required for the interface in GBPS + After execution the method will add the key: + 'vim_id': must be filled/added by this method with the VIM identifier generated by the VIM for this + interface. 'net_list' is modified + 'cloud_config': (optional) dictionary with: + 'key-pairs': (optional) list of strings with the public key to be inserted to the default user + 'users': (optional) list of users to be inserted, each item is a dict with: + 'name': (mandatory) user name, + 'key-pairs': (optional) list of strings with the public key to be inserted to the user + 'user-data': (optional) string is a text script to be passed directly to cloud-init + 'config-files': (optional). List of files to be transferred. Each item is a dict with: + 'dest': (mandatory) string with the destination absolute path + 'encoding': (optional, by default text). Can be one of: + 'b64', 'base64', 'gz', 'gz+b64', 'gz+base64', 'gzip+b64', 'gzip+base64' + 'content' (mandatory): string with the content of the file + 'permissions': (optional) string with file permissions, typically octal notation '0644' + 'owner': (optional) file owner, string with the format 'owner:group' + 'boot-data-drive': boolean to indicate if user-data must be passed using a boot drive (hard disk) + 'disk_list': (optional) list with additional disks to the VM. Each item is a dict with: + 'image_id': (optional). VIM id of an existing image. If not provided an empty disk must be mounted + 'size': (mandatory) string with the size of the disk in GB + Returns the instance identifier or raises an exception on error + """ raise vimconnNotImplemented( "Should have implemented this" ) def get_vminstance(self,vm_id): - '''Returns the VM instance information from VIM''' + """Returns the VM instance information from VIM""" raise vimconnNotImplemented( "Should have implemented this" ) def delete_vminstance(self, vm_id): - '''Removes a VM instance from VIM''' - '''Returns the instance identifier''' + """Removes a VM instance from VIM + Returns the instance identifier""" raise vimconnNotImplemented( "Should have implemented this" ) def refresh_vms_status(self, vm_list): - '''Get the status of the virtual machines and their interfaces/ports + """Get the status of the virtual machines and their interfaces/ports Params: the list of VM identifiers Returns a dictionary with: vm_id: #VIM id of this Virtual Machine @@ -355,27 +417,30 @@ class vimconnector(): # OTHER (Vim reported other status not understood) # ERROR (VIM indicates an ERROR status) # ACTIVE, PAUSED, SUSPENDED, INACTIVE (not running), - # CREATING (on building process), ERROR + # BUILD (on building process), ERROR # ACTIVE:NoMgmtIP (Active but any of its interface has an IP address # error_msg: #Text with VIM error message, if any. Or the VIM connection ERROR vim_info: #Text with plain information obtained from vim (yaml.safe_dump) - interfaces: - - vim_info: #Text with plain information obtained from vim (yaml.safe_dump) + interfaces: list with interface info. Each item a dictionary with: + vim_info: #Text with plain information obtained from vim (yaml.safe_dump) mac_address: #Text format XX:XX:XX:XX:XX:XX - vim_net_id: #network id where this interface is connected + vim_net_id: #network id where this interface is connected, if provided at creation vim_interface_id: #interface/port VIM id ip_address: #null, or text with IPv4, IPv6 address - ''' + physical_compute: #identification of compute node where PF,VF interface is allocated + physical_pci: #PCI address of the NIC that hosts the PF,VF + physical_vlan: #physical VLAN used for VF + """ raise vimconnNotImplemented( "Should have implemented this" ) def action_vminstance(self, vm_id, action_dict): - '''Send and action over a VM instance from VIM - Returns the vm_id if the action was successfully sent to the VIM''' + """Send and action over a VM instance from VIM + Returns the vm_id if the action was successfully sent to the VIM""" raise vimconnNotImplemented( "Should have implemented this" ) - def get_vminstance_console(self,vm_id, console_type="vnc"): - ''' + def get_vminstance_console(self, vm_id, console_type="vnc"): + """ Get a console for the virtual machine Params: vm_id: uuid of the VM @@ -387,53 +452,53 @@ class vimconnector(): server: usually ip address port: the http, ssh, ... port suffix: extra text, e.g. the http path and query string - ''' + """ raise vimconnNotImplemented( "Should have implemented this" ) #NOT USED METHODS in current version def host_vim2gui(self, host, server_dict): - '''Transform host dictionary from VIM format to GUI format, + """Transform host dictionary from VIM format to GUI format, and append to the server_dict - ''' + """ raise vimconnNotImplemented( "Should have implemented this" ) def get_hosts_info(self): - '''Get the information of deployed hosts - Returns the hosts content''' + """Get the information of deployed hosts + Returns the hosts content""" raise vimconnNotImplemented( "Should have implemented this" ) def get_hosts(self, vim_tenant): - '''Get the hosts and deployed instances - Returns the hosts content''' + """Get the hosts and deployed instances + Returns the hosts content""" raise vimconnNotImplemented( "Should have implemented this" ) def get_processor_rankings(self): - '''Get the processor rankings in the VIM database''' + """Get the processor rankings in the VIM database""" raise vimconnNotImplemented( "Should have implemented this" ) def new_host(self, host_data): - '''Adds a new host to VIM''' - '''Returns status code of the VIM response''' + """Adds a new host to VIM""" + """Returns status code of the VIM response""" raise vimconnNotImplemented( "Should have implemented this" ) def new_external_port(self, port_data): - '''Adds a external port to VIM''' - '''Returns the port identifier''' + """Adds a external port to VIM""" + """Returns the port identifier""" raise vimconnNotImplemented( "Should have implemented this" ) def new_external_network(self,net_name,net_type): - '''Adds a external network to VIM (shared)''' - '''Returns the network identifier''' + """Adds a external network to VIM (shared)""" + """Returns the network identifier""" raise vimconnNotImplemented( "Should have implemented this" ) def connect_port_network(self, port_id, network_id, admin=False): - '''Connects a external port to a network''' - '''Returns status code of the VIM response''' + """Connects a external port to a network""" + """Returns status code of the VIM response""" raise vimconnNotImplemented( "Should have implemented this" ) def new_vminstancefromJSON(self, vm_data): - '''Adds a VM instance to VIM''' - '''Returns the instance identifier''' + """Adds a VM instance to VIM""" + """Returns the instance identifier""" raise vimconnNotImplemented( "Should have implemented this" ) diff --git a/vimconn_openvim.py b/vimconn_openvim.py index fb59eccc..9d6748a9 100644 --- a/vimconn_openvim.py +++ b/vimconn_openvim.py @@ -481,7 +481,7 @@ class vimconnector(vimconn.vimconnector): except requests.exceptions.RequestException as e: self._format_request_exception(e) - def new_network(self,net_name, net_type, ip_profile=None, shared=False, **vim_specific): + def new_network(self,net_name, net_type, ip_profile=None, shared=False, vlan=None): #, **vim_specific): '''Adds a tenant network to VIM''' '''Returns the network identifier''' try: @@ -489,7 +489,9 @@ class vimconnector(vimconn.vimconnector): if net_type=="bridge": net_type="bridge_data" payload_req = {"name": net_name, "type": net_type, "tenant_id": self.tenant, "shared": shared} - payload_req.update(vim_specific) + if vlan: + payload_req["provider:vlan"] = vlan + # payload_req.update(vim_specific) url = self.url+'/networks' self.logger.info("Adding a new network POST: %s DATA: %s", url, str(payload_req)) vim_response = requests.post(url, headers = self.headers_req, data=json.dumps({"network": payload_req}) ) -- 2.25.1 From b3d367468fb470724f4846bed34af7859e136524 Mon Sep 17 00:00:00 2001 From: tierno Date: Fri, 3 Mar 2017 23:51:05 +0100 Subject: [PATCH 03/16] new persistent_info param at vimconn class Task management at vim_thread Change-Id: Ica8ef928811a973ac8edc669692ad2c867d9518a Signed-off-by: tierno --- nfvo.py | 298 +++++++++++++++++++++++++++++++++++-------- vim_thread.py | 205 +++++++++++++++++++++-------- vimconn.py | 2 + vimconn_openstack.py | 6 +- vimconn_openvim.py | 4 +- vimconn_vmware.py | 3 +- 6 files changed, 405 insertions(+), 113 deletions(-) diff --git a/nfvo.py b/nfvo.py index 80a606ba..14ec54f1 100644 --- a/nfvo.py +++ b/nfvo.py @@ -39,6 +39,9 @@ import vimconn import logging import collections from db_base import db_base_Exception +import nfvo_db +from threading import Lock +from time import time global global_config global vimconn_imported @@ -49,7 +52,13 @@ default_volume_size = '5' #size in GB vimconn_imported = {} # dictionary with VIM type as key, loaded module as value vim_threads = {"running":{}, "deleting": {}, "names": []} # threads running for attached-VIMs +vim_persistent_info = {} logger = logging.getLogger('openmano.nfvo') +task_lock = Lock() +task_dict = {} +last_task_id = 0.0 +db=None +db_lock=Lock() class NfvoException(Exception): def __init__(self, message, http_code): @@ -57,12 +66,35 @@ class NfvoException(Exception): Exception.__init__(self, message) +def get_task_id(): + global last_task_id + task_id = time() + if task_id <= last_task_id: + task_id = last_task_id + 0.000001 + last_task_id = task_id + return "TASK.{:.6f}".format(task_id) + + +def new_task(name, params, store=True, depends=None): + task_id = get_task_id() + task = {"status": "enqueued", "id": task_id, "name": name, "params": params} + if depends: + task["depends"] = depends + if store: + task_dict[task_id] = task + return task + + +def is_task_id(id): + return True if id[:5] == "TASK." else False + + def get_non_used_vim_name(datacenter_name, datacenter_id, tenant_name, tenant_id): name = datacenter_name[:16] if name not in vim_threads["names"]: vim_threads["names"].append(name) return name - name = datatacenter_name[:16] + "." + tenant_name[:16] + name = datacenter_name[:16] + "." + tenant_name[:16] if name not in vim_threads["names"]: vim_threads["names"].append(name) return name @@ -72,6 +104,9 @@ def get_non_used_vim_name(datacenter_name, datacenter_id, tenant_name, tenant_id def start_service(mydb): + global db, global_config + db = nfvo_db.nfvo_db() + db.connect(global_config['db_host'], global_config['db_user'], global_config['db_passwd'], global_config['db_name']) from_= 'tenants_datacenters as td join datacenters as d on td.datacenter_id=d.uuid join datacenter_tenants as dt on td.datacenter_tenant_id=dt.uuid' select_ = ('type','d.config as config','d.uuid as datacenter_id', 'vim_url', 'vim_url_admin', 'd.name as datacenter_name', 'dt.uuid as datacenter_tenant_id','dt.vim_tenant_name as vim_tenant_name','dt.vim_tenant_id as vim_tenant_id', @@ -95,33 +130,37 @@ def start_service(mydb): if module_info and module_info[0]: file.close(module_info[0]) raise NfvoException("Unknown vim type '{}'. Can not open file '{}.py'; {}: {}".format( - vim["type"], module, type(e).__name__, str(e)), HTTP_Bad_Request) + vim["type"], module, type(e).__name__, str(e)), HTTP_Bad_Request) + thread_id = vim["datacenter_id"] + "." + vim['nfvo_tenant_id'] + vim_persistent_info[thread_id] = {} try: #if not tenant: # return -HTTP_Bad_Request, "You must provide a valid tenant name or uuid for VIM %s" % ( vim["type"]) myvim = vimconn_imported[ vim["type"] ].vimconnector( - uuid=vim['datacenter_id'], name=vim['datacenter_name'], - tenant_id=vim['vim_tenant_id'], tenant_name=vim['vim_tenant_name'], - url=vim['vim_url'], url_admin=vim['vim_url_admin'], - user=vim['user'], passwd=vim['passwd'], - config=extra - ) + uuid=vim['datacenter_id'], name=vim['datacenter_name'], + tenant_id=vim['vim_tenant_id'], tenant_name=vim['vim_tenant_name'], + url=vim['vim_url'], url_admin=vim['vim_url_admin'], + user=vim['user'], passwd=vim['passwd'], + config=extra, persistent_info=vim_persistent_info[thread_id] + ) except Exception as e: raise NfvoException("Error at VIM {}; {}: {}".format(vim["type"], type(e).__name__, str(e)), HTTP_Internal_Server_Error) thread_name = get_non_used_vim_name(vim['datacenter_name'], vim['vim_tenant_id'], vim['vim_tenant_name'], vim['vim_tenant_id']) - new_thread = vim_thread.vim_thread(myvim, thread_name) + new_thread = vim_thread.vim_thread(myvim, task_lock, thread_name, vim['datacenter_name'], + vim.get('datacenter_tenant_id'), db=db, db_lock=db_lock) new_thread.start() - thread_id = vim["datacenter_id"] + "-" + vim['nfvo_tenant_id'] vim_threads["running"][thread_id] = new_thread except db_base_Exception as e: raise NfvoException(str(e) + " at nfvo.get_vim", e.http_code) + def stop_service(): for thread_id,thread in vim_threads["running"].items(): - thread.insert_task("exit") + thread.insert_task(new_task("exit", None, store=False)) vim_threads["deleting"][thread_id] = thread - vim_threads["running"]={} + vim_threads["running"] = {} + def get_flavorlist(mydb, vnf_id, nfvo_tenant=None): '''Obtain flavorList @@ -144,6 +183,7 @@ def get_flavorlist(mydb, vnf_id, nfvo_tenant=None): flavorList.append(flavor['flavor_id']) return flavorList + def get_imagelist(mydb, vnf_id, nfvo_tenant=None): '''Obtain imageList return result, content: @@ -162,6 +202,7 @@ def get_imagelist(mydb, vnf_id, nfvo_tenant=None): imageList.append(image['image_id']) return imageList + def get_vim(mydb, nfvo_tenant=None, datacenter_id=None, datacenter_name=None, datacenter_tenant_id=None, vim_tenant=None, vim_tenant_name=None, vim_user=None, vim_passwd=None): '''Obtain a dictionary of VIM (datacenter) classes with some of the input parameters @@ -207,14 +248,22 @@ def get_vim(mydb, nfvo_tenant=None, datacenter_id=None, datacenter_name=None, da vim["type"], module, type(e).__name__, str(e)), HTTP_Bad_Request) try: + if 'nfvo_tenant_id' in vim: + thread_id = vim["datacenter_id"] + "." + vim['nfvo_tenant_id'] + if thread_id not in vim_persistent_info: + vim_persistent_info[thread_id] = {} + persistent_info = vim_persistent_info[thread_id] + else: + persistent_info = {} #if not tenant: # return -HTTP_Bad_Request, "You must provide a valid tenant name or uuid for VIM %s" % ( vim["type"]) vim_dict[ vim['datacenter_id'] ] = vimconn_imported[ vim["type"] ].vimconnector( uuid=vim['datacenter_id'], name=vim['datacenter_name'], - tenant_id=vim.get('vim_tenant_id',vim_tenant), tenant_name=vim.get('vim_tenant_name',vim_tenant_name), + tenant_id=vim.get('vim_tenant_id',vim_tenant), + tenant_name=vim.get('vim_tenant_name',vim_tenant_name), url=vim['vim_url'], url_admin=vim['vim_url_admin'], user=vim.get('user',vim_user), passwd=vim.get('passwd',vim_passwd), - config=extra + config=extra, persistent_info=persistent_info ) except Exception as e: raise NfvoException("Error at VIM {}; {}: {}".format(vim["type"], type(e).__name__, str(e)), HTTP_Internal_Server_Error) @@ -222,6 +271,7 @@ def get_vim(mydb, nfvo_tenant=None, datacenter_id=None, datacenter_name=None, da except db_base_Exception as e: raise NfvoException(str(e) + " at nfvo.get_vim", e.http_code) + def rollback(mydb, vims, rollback_list): undeleted_items=[] #delete things by reverse order @@ -262,6 +312,7 @@ def rollback(mydb, vims, rollback_list): else: return False," Rollback fails to delete: " + str(undeleted_items) + def check_vnf_descriptor(vnf_descriptor, vnf_descriptor_version=1): global global_config #create a dictionary with vnfc-name: vnfc:interface-list key:values pairs @@ -458,6 +509,7 @@ def create_or_use_image(mydb, vims, image_dict, rollback_list, only_create_at_vi return image_vim_id if only_create_at_vim else image_mano_id + def create_or_use_flavor(mydb, vims, flavor_dict, rollback_list, only_create_at_vim=False, return_on_error = None): temp_flavor_dict= {'disk':flavor_dict.get('disk',1), 'ram':flavor_dict.get('ram'), @@ -610,6 +662,7 @@ def create_or_use_flavor(mydb, vims, flavor_dict, rollback_list, only_create_at_ return flavor_vim_id if only_create_at_vim else flavor_mano_id + def new_vnf(mydb, tenant_id, vnf_descriptor): global global_config @@ -744,6 +797,7 @@ def new_vnf(mydb, tenant_id, vnf_descriptor): #logger.error("start_scenario %s", error_text) raise NfvoException(error_text, e.http_code) + def new_vnf_v02(mydb, tenant_id, vnf_descriptor): global global_config @@ -877,6 +931,7 @@ def new_vnf_v02(mydb, tenant_id, vnf_descriptor): #logger.error("start_scenario %s", error_text) raise NfvoException(error_text, e.http_code) + def get_vnf_id(mydb, tenant_id, vnf_id): #check valid tenant_id check_tenant(mydb, tenant_id) @@ -1034,6 +1089,7 @@ def delete_vnf(mydb,tenant_id,vnf_id,datacenter=None,vim_tenant=None): #if undeletedItems: # return "delete_vnf. Undeleted: %s" %(undeletedItems) + def get_hosts_info(mydb, nfvo_tenant_id, datacenter_name=None): result, vims = get_vim(mydb, nfvo_tenant_id, None, datacenter_name) if result < 0: @@ -1047,6 +1103,7 @@ def get_hosts_info(mydb, nfvo_tenant_id, datacenter_name=None): topology = {'name':myvim['name'] , 'servers': servers} return result, topology + def get_hosts(mydb, nfvo_tenant_id): vims = get_vim(mydb, nfvo_tenant_id) if len(vims) == 0: @@ -1082,6 +1139,7 @@ def get_hosts(mydb, nfvo_tenant_id): except vimconn.vimconnException as e: raise NfvoException("Not possible to get_host_list from VIM: {}".format(str(e)), e.http_code) + def new_scenario(mydb, tenant_id, topo): # result, vims = get_vim(mydb, tenant_id) @@ -1365,6 +1423,7 @@ def new_scenario(mydb, tenant_id, topo): return c + def new_scenario_v02(mydb, tenant_id, scenario_dict, version): """ This creates a new scenario for version 0.2 and 0.3""" scenario = scenario_dict["scenario"] @@ -1488,12 +1547,14 @@ def new_scenario_v02(mydb, tenant_id, scenario_dict, version): scenario_id = mydb.new_scenario(scenario) return scenario_id + def edit_scenario(mydb, tenant_id, scenario_id, data): data["uuid"] = scenario_id data["tenant_id"] = tenant_id c = mydb.edit_scenario( data ) return c + def start_scenario(mydb, tenant_id, scenario_id, instance_scenario_name, instance_scenario_description, datacenter=None,vim_tenant=None, startvms=True): #print "Checking that nfvo_tenant_id exists and getting the VIM URI and the VIM tenant_id" datacenter_id, myvim = get_datacenter_by_name_uuid(mydb, tenant_id, datacenter, vim_tenant=vim_tenant) @@ -1696,6 +1757,7 @@ def start_scenario(mydb, tenant_id, scenario_id, instance_scenario_name, instanc #logger.error("start_scenario %s", error_text) raise NfvoException(error_text, e.http_code) + def unify_cloud_config(cloud_config_preserve, cloud_config): ''' join the cloud config information into cloud_config_preserve. In case of conflict cloud_config_preserve preserves @@ -1772,6 +1834,33 @@ def unify_cloud_config(cloud_config_preserve, cloud_config): return new_cloud_config +def get_vim_thread(tenant_id, datacenter_id_name=None, datacenter_tenant_id=None): + datacenter_id = None + datacenter_name = None + thread = None + if datacenter_id_name: + if utils.check_valid_uuid(datacenter_id_name): + datacenter_id = datacenter_id_name + else: + datacenter_name = datacenter_id_name + if datacenter_id: + thread = vim_threads["running"].get(datacenter_id + "." + tenant_id) + else: + for k, v in vim_threads["running"].items(): + datacenter_tenant = k.split(".") + if datacenter_tenant[0] == datacenter_id and datacenter_tenant[1] == tenant_id: + if thread: + raise NfvoException("More than one datacenters found, try to identify with uuid", HTTP_Conflict) + thread = v + elif not datacenter_id and datacenter_tenant[1] == tenant_id: + if thread.datacenter_name == datacenter_name: + if thread: + raise NfvoException("More than one datacenters found, try to identify with uuid", HTTP_Conflict) + thread = v + if not thread: + raise NfvoException("datacenter '{}' not found".format(str(datacenter_id_name)), HTTP_Not_Found) + return thread + def get_datacenter_by_name_uuid(mydb, tenant_id, datacenter_id_name=None, **extra_filter): datacenter_id = None @@ -1789,6 +1878,7 @@ def get_datacenter_by_name_uuid(mydb, tenant_id, datacenter_id_name=None, **extr raise NfvoException("More than one datacenters found, try to identify with uuid", HTTP_Conflict) return vims.keys()[0], vims.values()[0] + def update(d, u): '''Takes dict d and updates it with the values in dict u.''' '''It merges all depth levels''' @@ -1800,17 +1890,20 @@ def update(d, u): d[k] = u[k] return d + def create_instance(mydb, tenant_id, instance_dict): - #print "Checking that nfvo_tenant_id exists and getting the VIM URI and the VIM tenant_id" - #logger.debug("Creating instance...") + # print "Checking that nfvo_tenant_id exists and getting the VIM URI and the VIM tenant_id" + # logger.debug("Creating instance...") scenario = instance_dict["scenario"] #find main datacenter myvims = {} + myvim_threads = {} datacenter2tenant = {} datacenter = instance_dict.get("datacenter") default_datacenter_id, vim = get_datacenter_by_name_uuid(mydb, tenant_id, datacenter) myvims[default_datacenter_id] = vim + myvim_threads[default_datacenter_id] = get_vim_thread(tenant_id, default_datacenter_id) datacenter2tenant[default_datacenter_id] = vim['config']['datacenter_tenant_id'] #myvim_tenant = myvim['tenant_id'] # default_datacenter_name = vim['name'] @@ -1831,10 +1924,11 @@ def create_instance(mydb, tenant_id, instance_dict): logger.debug("Creating instance from scenario-dict:\n%s", yaml.safe_dump(scenarioDict, indent=4, default_flow_style=False)) #TODO remove instance_name = instance_dict["name"] instance_description = instance_dict.get("description") + instance_tasks={} try: - #0 check correct parameters + # 0 check correct parameters for net_name, net_instance_desc in instance_dict.get("networks",{}).iteritems(): - found=False + found = False for scenario_net in scenarioDict['nets']: if net_name == scenario_net["name"]: found = True @@ -1850,13 +1944,14 @@ def create_instance(mydb, tenant_id, instance_dict): #Add this datacenter to myvims d, v = get_datacenter_by_name_uuid(mydb, tenant_id, site["datacenter"]) myvims[d] = v + myvim_threads[d] = get_vim_thread(tenant_id, site["datacenter"]) datacenter2tenant[d] = v['config']['datacenter_tenant_id'] - site["datacenter"] = d #change name to id + site["datacenter"] = d #change name to id else: if site_without_datacenter_field: raise NfvoException("Found more than one entries without datacenter field at instance:networks:{}:sites".format(net_name), HTTP_Bad_Request) site_without_datacenter_field = True - site["datacenter"] = default_datacenter_id #change name to id + site["datacenter"] = default_datacenter_id #change name to id for vnf_name, vnf_instance_desc in instance_dict.get("vnfs",{}).iteritems(): found=False @@ -1867,10 +1962,11 @@ def create_instance(mydb, tenant_id, instance_dict): if not found: raise NfvoException("Invalid vnf name '{}' at instance:vnfs".format(vnf_instance_desc), HTTP_Bad_Request) if "datacenter" in vnf_instance_desc: - #Add this datacenter to myvims + # Add this datacenter to myvims if vnf_instance_desc["datacenter"] not in myvims: d, v = get_datacenter_by_name_uuid(mydb, tenant_id, vnf_instance_desc["datacenter"]) myvims[d] = v + myvim_threads[d] = get_vim_thread(tenant_id, vnf_instance_desc["datacenter"]) datacenter2tenant[d] = v['config']['datacenter_tenant_id'] scenario_vnf["datacenter"] = vnf_instance_desc["datacenter"] @@ -1910,7 +2006,7 @@ def create_instance(mydb, tenant_id, instance_dict): logger.debug("Creating instance scenario-dict MERGED:\n%s", yaml.safe_dump(scenarioDict, indent=4, default_flow_style=False)) - #1. Creating new nets (sce_nets) in the VIM" + # 1. Creating new nets (sce_nets) in the VIM" for sce_net in scenarioDict['nets']: sce_net["vim_id_sites"]={} descriptor_net = instance_dict.get("networks",{}).get(sce_net["name"],{}) @@ -1922,9 +2018,11 @@ def create_instance(mydb, tenant_id, instance_dict): if site.get("datacenter"): vim = myvims[ site["datacenter"] ] datacenter_id = site["datacenter"] + myvim_thread = myvim_threads[ site["datacenter"] ] else: vim = myvims[ default_datacenter_id ] datacenter_id = default_datacenter_id + myvim_thread = myvim_threads[default_datacenter_id] net_type = sce_net['type'] lookfor_filter = {'admin_state_up': True, 'status': 'ACTIVE'} #'shared': True if sce_net["external"]: @@ -1982,48 +2080,58 @@ def create_instance(mydb, tenant_id, instance_dict): create_network = False if create_network: #if network is not external - network_id = vim.new_network(net_vim_name, net_type, sce_net.get('ip_profile',None)) - sce_net["vim_id_sites"][datacenter_id] = network_id - auxNetDict['scenario'][sce_net['uuid']][datacenter_id] = network_id - rollbackList.append({'what':'network', 'where':'vim', 'vim_id':datacenter_id, 'uuid':network_id}) + task = new_task("new-net", (net_vim_name, net_type, sce_net.get('ip_profile',None))) + task_id = myvim_thread.insert_task(task) + instance_tasks[task_id] = task + #network_id = vim.new_network(net_vim_name, net_type, sce_net.get('ip_profile',None)) + sce_net["vim_id_sites"][datacenter_id] = task_id + auxNetDict['scenario'][sce_net['uuid']][datacenter_id] = task_id + rollbackList.append({'what':'network', 'where':'vim', 'vim_id':datacenter_id, 'uuid':task_id}) sce_net["created"] = True - #2. Creating new nets (vnf internal nets) in the VIM" + # 2. Creating new nets (vnf internal nets) in the VIM" #For each vnf net, we create it and we add it to instanceNetlist. for sce_vnf in scenarioDict['vnfs']: for net in sce_vnf['nets']: if sce_vnf.get("datacenter"): vim = myvims[ sce_vnf["datacenter"] ] datacenter_id = sce_vnf["datacenter"] + myvim_thread = myvim_threads[ sce_vnf["datacenter"]] else: vim = myvims[ default_datacenter_id ] datacenter_id = default_datacenter_id + myvim_thread = myvim_threads[default_datacenter_id] descriptor_net = instance_dict.get("vnfs",{}).get(sce_vnf["name"],{}) net_name = descriptor_net.get("name") if not net_name: net_name = "%s.%s" %(instance_name, net["name"]) net_name = net_name[:255] #limit length net_type = net['type'] - network_id = vim.new_network(net_name, net_type, net.get('ip_profile',None)) - net['vim_id'] = network_id + task = new_task("new-net", (net_name, net_type, net.get('ip_profile',None))) + task_id = myvim_thread.insert_task(task) + instance_tasks[task_id] = task + # network_id = vim.new_network(net_name, net_type, net.get('ip_profile',None)) + net['vim_id'] = task_id if sce_vnf['uuid'] not in auxNetDict: auxNetDict[sce_vnf['uuid']] = {} - auxNetDict[sce_vnf['uuid']][net['uuid']] = network_id - rollbackList.append({'what':'network','where':'vim','vim_id':datacenter_id,'uuid':network_id}) + auxNetDict[sce_vnf['uuid']][net['uuid']] = task_id + rollbackList.append({'what':'network','where':'vim','vim_id':datacenter_id,'uuid':task_id}) net["created"] = True #print "auxNetDict:" #print yaml.safe_dump(auxNetDict, indent=4, default_flow_style=False) - #3. Creating new vm instances in the VIM + # 3. Creating new vm instances in the VIM #myvim.new_vminstance(self,vimURI,tenant_id,name,description,image_id,flavor_id,net_dict) for sce_vnf in scenarioDict['vnfs']: if sce_vnf.get("datacenter"): vim = myvims[ sce_vnf["datacenter"] ] + myvim_thread = myvim_threads[ sce_vnf["datacenter"] ] datacenter_id = sce_vnf["datacenter"] else: vim = myvims[ default_datacenter_id ] + myvim_thread = myvim_threads[ default_datacenter_id ] datacenter_id = default_datacenter_id sce_vnf["datacenter_id"] = datacenter_id i = 0 @@ -2046,9 +2154,6 @@ def create_instance(mydb, tenant_id, instance_dict): flavor_dict['extended']= yaml.load(flavor_dict['extended']) flavor_id = create_or_use_flavor(mydb, {datacenter_id: vim}, flavor_dict, rollbackList, True) - - - #Obtain information for additional disks extended_flavor_dict = mydb.get_rows(FROM='datacenters_flavors', SELECT=('extended',), WHERE={'vim_id': flavor_id}) if not extended_flavor_dict: @@ -2063,14 +2168,11 @@ def create_instance(mydb, tenant_id, instance_dict): if 'disks' in extended_flavor_dict_yaml: myVMDict['disks'] = extended_flavor_dict_yaml['disks'] - - - vm['vim_flavor_id'] = flavor_id - myVMDict['imageRef'] = vm['vim_image_id'] myVMDict['flavorRef'] = vm['vim_flavor_id'] myVMDict['networks'] = [] + task_depends = {} #TODO ALF. connect_mgmt_interfaces. Connect management interfaces if this is true for iface in vm['interfaces']: netDict = {} @@ -2121,6 +2223,8 @@ def create_instance(mydb, tenant_id, instance_dict): break else: netDict['net_id'] = auxNetDict[ sce_vnf['uuid'] ][ iface['net_id'] ] + if is_task_id(netDict['net_id']): + task_depends[netDict['net_id']] = instance_tasks[netDict['net_id']] #skip bridge ifaces not connected to any net #if 'net_id' not in netDict or netDict['net_id']==None: # continue @@ -2134,9 +2238,11 @@ def create_instance(mydb, tenant_id, instance_dict): cloud_config_vm = unify_cloud_config(vm["boot_data"], cloud_config) else: cloud_config_vm = cloud_config - vm_id = vim.new_vminstance(myVMDict['name'],myVMDict['description'],myVMDict.get('start', None), - myVMDict['imageRef'],myVMDict['flavorRef'],myVMDict['networks'], cloud_config = cloud_config_vm, - disk_list = myVMDict['disks']) + task = new_task("new-vm", (myVMDict['name'], myVMDict['description'], myVMDict.get('start', None), + myVMDict['imageRef'], myVMDict['flavorRef'], myVMDict['networks'], + cloud_config_vm, myVMDict['disks']), depends=task_depends) + vm_id = myvim_thread.insert_task(task) + instance_tasks[vm_id] = task vm['vim_id'] = vm_id rollbackList.append({'what':'vm','where':'vim','vim_id':datacenter_id,'uuid':vm_id}) @@ -2151,6 +2257,15 @@ def create_instance(mydb, tenant_id, instance_dict): logger.debug("create_instance Deployment done scenarioDict: %s", yaml.safe_dump(scenarioDict, indent=4, default_flow_style=False) ) instance_id = mydb.new_instance_scenario_as_a_whole(tenant_id,instance_name, instance_description, scenarioDict) + # Update database with those ended tasks + for task in instance_tasks.values(): + if task["status"] == "ok": + if task["name"] == "new-vm": + mydb.update_rows("instance_vms", UPDATE={"vim_vm_id": task["result"]}, + WHERE={"vim_vm_id": task["id"]}) + elif task["name"] == "new-net": + mydb.update_rows("instance_nets", UPDATE={"vim_net_id": task["result"]}, + WHERE={"vim_net_id": task["id"]}) return mydb.get_instance_scenario(instance_id) except (NfvoException, vimconn.vimconnException,db_base_Exception) as e: message = rollback(mydb, myvims, rollbackList) @@ -2164,6 +2279,7 @@ def create_instance(mydb, tenant_id, instance_dict): #logger.error("create_instance: %s", error_text) raise NfvoException(error_text, e.http_code) + def delete_instance(mydb, tenant_id, instance_id): #print "Checking that the instance_id exists and getting the instance dictionary" instanceDict = mydb.get_instance_scenario(instance_id, tenant_id) @@ -2176,13 +2292,20 @@ def delete_instance(mydb, tenant_id, instance_id): #2. delete from VIM error_msg = "" - myvims={} + myvims = {} + myvim_threads = {} #2.1 deleting VMs #vm_fail_list=[] for sce_vnf in instanceDict['vnfs']: datacenter_key = (sce_vnf["datacenter_id"], sce_vnf["datacenter_tenant_id"]) if datacenter_key not in myvims: + try: + myvim_thread = get_vim_thread(tenant_id, sce_vnf["datacenter_id"], sce_vnf["datacenter_tenant_id"]) + except NfvoException as e: + logger.error(str(e)) + myvim_thread = None + myvim_threads[datacenter_key] = myvim_thread vims = get_vim(mydb, tenant_id, datacenter_id=sce_vnf["datacenter_id"], datacenter_tenant_id=sce_vnf["datacenter_tenant_id"]) if len(vims) == 0: @@ -2192,12 +2315,32 @@ def delete_instance(mydb, tenant_id, instance_id): else: myvims[datacenter_key] = vims.values()[0] myvim = myvims[datacenter_key] + myvim_thread = myvim_threads[datacenter_key] for vm in sce_vnf['vms']: if not myvim: error_msg += "\n VM id={} cannot be deleted because datacenter={} not found".format(vm['vim_vm_id'], sce_vnf["datacenter_id"]) continue try: - myvim.delete_vminstance(vm['vim_vm_id']) + task=None + if is_task_id(vm['vim_vm_id']): + task_id = vm['vim_vm_id'] + old_task = task_dict.get(task_id) + if not old_task: + error_msg += "\n VM was scheduled for create, but task {} is not found".format(task_id) + continue + with task_lock: + if old_task["status"] == "enqueued": + old_task["status"] = "deleted" + elif old_task["status"] == "error": + continue + elif old_task["status"] == "processing": + task = new_task("del-vm", task_id, depends={task_id: old_task}) + else: #ok + task = new_task("del-vm", old_task["result"]) + else: + task = new_task("del-vm", vm['vim_vm_id'], store=False) + if task: + myvim_thread.insert_task(task) except vimconn.vimconnNotFoundException as e: error_msg+="\n VM VIM_id={} not found at datacenter={}".format(vm['vim_vm_id'], sce_vnf["datacenter_id"]) logger.warn("VM instance '%s'uuid '%s', VIM id '%s', from VNF_id '%s' not found", @@ -2214,6 +2357,12 @@ def delete_instance(mydb, tenant_id, instance_id): continue #skip not created nets datacenter_key = (net["datacenter_id"], net["datacenter_tenant_id"]) if datacenter_key not in myvims: + try: + myvim_thread = get_vim_thread(tenant_id, sce_vnf["datacenter_id"], sce_vnf["datacenter_tenant_id"]) + except NfvoException as e: + logger.error(str(e)) + myvim_thread = None + myvim_threads[datacenter_key] = myvim_thread vims = get_vim(mydb, tenant_id, datacenter_id=net["datacenter_id"], datacenter_tenant_id=net["datacenter_tenant_id"]) if len(vims) == 0: @@ -2222,25 +2371,48 @@ def delete_instance(mydb, tenant_id, instance_id): else: myvims[datacenter_key] = vims.values()[0] myvim = myvims[datacenter_key] + myvim_thread = myvim_threads[datacenter_key] if not myvim: error_msg += "\n Net VIM_id={} cannot be deleted because datacenter={} not found".format(net['vim_net_id'], net["datacenter_id"]) continue try: - myvim.delete_network(net['vim_net_id']) + task = None + if is_task_id(net['vim_net_id']): + task_id = net['vim_net_id'] + old_task = task_dict.get(task_id) + if not old_task: + error_msg += "\n NET was scheduled for create, but task {} is not found".format(task_id) + continue + with task_lock: + if old_task["status"] == "enqueued": + old_task["status"] = "deleted" + elif old_task["status"] == "error": + continue + elif old_task["status"] == "processing": + task = new_task("del-net", task_id, depends={task_id: old_task}) + else: # ok + task = new_task("del-net", old_task["result"]) + else: + task = new_task("del-net", net['vim_net_id'], store=False) + if task: + myvim_thread.insert_task(task) except vimconn.vimconnNotFoundException as e: - error_msg+="\n NET VIM_id={} not found at datacenter={}".format(net['vim_net_id'], net["datacenter_id"]) + error_msg += "\n NET VIM_id={} not found at datacenter={}".format(net['vim_net_id'], net["datacenter_id"]) logger.warn("NET '%s', VIM_id '%s', from VNF_net_id '%s' not found", - net['uuid'], net['vim_net_id'], str(net['vnf_net_id'])) + net['uuid'], net['vim_net_id'], str(net['vnf_net_id'])) except vimconn.vimconnException as e: - error_msg+="\n NET VIM_id={} at datacenter={} Error: {} {}".format(net['vim_net_id'], net["datacenter_id"], e.http_code, str(e)) + error_msg += "\n NET VIM_id={} at datacenter={} Error: {} {}".format(net['vim_net_id'], + net["datacenter_id"], + e.http_code, str(e)) logger.error("Error %d deleting NET '%s', VIM_id '%s', from VNF_net_id '%s': %s", - e.http_code, net['uuid'], net['vim_net_id'], str(net['vnf_net_id']), str(e)) - if len(error_msg)>0: + e.http_code, net['uuid'], net['vim_net_id'], str(net['vnf_net_id']), str(e)) + if len(error_msg) > 0: return 'instance ' + message + ' deleted but some elements could not be deleted, or already deleted (error: 404) from VIM: ' + error_msg else: return 'instance ' + message + ' deleted' + def refresh_instance(mydb, nfvo_tenant, instanceDict, datacenter=None, vim_tenant=None): '''Refreshes a scenario instance. It modifies instanceDict''' '''Returns: @@ -2405,6 +2577,7 @@ def refresh_instance(mydb, nfvo_tenant, instanceDict, datacenter=None, vim_tenan return 0, 'Scenario instance ' + instance_id + ' refreshed.' + def instance_action(mydb,nfvo_tenant,instance_id, action_dict): #print "Checking that the instance_id exists and getting the instance dictionary" instanceDict = mydb.get_instance_scenario(instance_id, nfvo_tenant) @@ -2477,6 +2650,7 @@ def instance_action(mydb,nfvo_tenant,instance_id, action_dict): else: return vm_result + def create_or_use_console_proxy_thread(console_server, console_port): #look for a non-used port console_thread_key = console_server + ":" + str(console_port) @@ -2501,6 +2675,7 @@ def create_or_use_console_proxy_thread(console_server, console_port): raise NfvoException(str(e), HTTP_Bad_Request) raise NfvoException("Not found any free 'http_console_ports'", HTTP_Conflict) + def check_tenant(mydb, tenant_id): '''check that tenant exists at database''' tenant = mydb.get_rows(FROM='nfvo_tenants', SELECT=('uuid',), WHERE={'uuid': tenant_id}) @@ -2508,10 +2683,12 @@ def check_tenant(mydb, tenant_id): raise NfvoException("tenant '{}' not found".format(tenant_id), HTTP_Not_Found) return + def new_tenant(mydb, tenant_dict): tenant_id = mydb.new_row("nfvo_tenants", tenant_dict, add_uuid=True) return tenant_id + def delete_tenant(mydb, tenant): #get nfvo_tenant info @@ -2519,6 +2696,7 @@ def delete_tenant(mydb, tenant): mydb.delete_row_by_id("nfvo_tenants", tenant_dict['uuid']) return tenant_dict['uuid'] + " " + tenant_dict["name"] + def new_datacenter(mydb, datacenter_descriptor): if "config" in datacenter_descriptor: datacenter_descriptor["config"]=yaml.safe_dump(datacenter_descriptor["config"],default_flow_style=True,width=256) @@ -2536,6 +2714,7 @@ def new_datacenter(mydb, datacenter_descriptor): datacenter_id = mydb.new_row("datacenters", datacenter_descriptor, add_uuid=True) return datacenter_id + def edit_datacenter(mydb, datacenter_id_name, datacenter_descriptor): #obtain data, check that only one exist datacenter = mydb.get_table_by_uuid_name('datacenters', datacenter_id_name) @@ -2563,12 +2742,14 @@ def edit_datacenter(mydb, datacenter_id_name, datacenter_descriptor): mydb.update_rows('datacenters', datacenter_descriptor, where) return datacenter_id + def delete_datacenter(mydb, datacenter): #get nfvo_tenant info datacenter_dict = mydb.get_table_by_uuid_name('datacenters', datacenter, 'datacenter') mydb.delete_row_by_id("datacenters", datacenter_dict['uuid']) return datacenter_dict['uuid'] + " " + datacenter_dict['name'] + def associate_datacenter_to_tenant(mydb, nfvo_tenant, datacenter, vim_tenant_id=None, vim_tenant_name=None, vim_username=None, vim_password=None, config=None): #get datacenter info datacenter_id, myvim = get_datacenter_by_name_uuid(mydb, None, datacenter) @@ -2630,12 +2811,13 @@ def associate_datacenter_to_tenant(mydb, nfvo_tenant, datacenter, vim_tenant_id= # create thread datacenter_id, myvim = get_datacenter_by_name_uuid(mydb, tenant_dict['uuid'], datacenter_id) # reload data thread_name = get_non_used_vim_name(datacenter_name, datacenter_id, tenant_dict['name'], tenant_dict['uuid']) - new_thread = vim_thread.vim_thread(myvim, thread_name) + new_thread = vim_thread.vim_thread(myvim, task_lock, thread_name, datacenter_name, db=db, db_lock=db_lock) new_thread.start() - vim_threads["running"][datacenter_id + "-" + tenant_dict['uuid']] = new_thread - + thread_id = datacenter_id + "." + tenant_dict['uuid'] + vim_threads["running"][thread_id] = new_thread return datacenter_id + def deassociate_datacenter_to_tenant(mydb, tenant_id, datacenter, vim_tenant_id=None): #get datacenter info datacenter_id, myvim = get_datacenter_by_name_uuid(mydb, None, datacenter) @@ -2675,12 +2857,13 @@ def deassociate_datacenter_to_tenant(mydb, tenant_id, datacenter, vim_tenant_id= except db_base_Exception as e: logger.error("Cannot delete datacenter_tenants " + str(e)) pass # the error will be caused because dependencies, vim_tenant can not be deleted - thread_id = datacenter_id + "-" + tenant_datacenter_item["nfvo_tenant_id"] + thread_id = datacenter_id + "." + tenant_datacenter_item["nfvo_tenant_id"] thread = vim_threads["running"][thread_id] - thread.insert_task("exit") + thread.insert_task(new_task("exit", None, store=False)) vim_threads["deleting"][thread_id] = thread return "datacenter {} detached. {}".format(datacenter_id, warning) + def datacenter_action(mydb, tenant_id, datacenter, action_dict): #DEPRECATED #get datacenter info @@ -2723,6 +2906,7 @@ def datacenter_action(mydb, tenant_id, datacenter, action_dict): else: raise NfvoException("Unknown action " + str(action_dict), HTTP_Bad_Request) + def datacenter_edit_netmap(mydb, tenant_id, datacenter, netmap, action_dict): #get datacenter info datacenter_id, _ = get_datacenter_by_name_uuid(mydb, tenant_id, datacenter) @@ -2732,6 +2916,7 @@ def datacenter_edit_netmap(mydb, tenant_id, datacenter, netmap, action_dict): WHERE={'datacenter_id':datacenter_id, what: netmap}) return result + def datacenter_new_netmap(mydb, tenant_id, datacenter, action_dict=None): #get datacenter info datacenter_id, myvim = get_datacenter_by_name_uuid(mydb, tenant_id, datacenter) @@ -2778,6 +2963,7 @@ def datacenter_new_netmap(mydb, tenant_id, datacenter, action_dict=None): net_list.append(net_nfvo) return net_list + def vim_action_get(mydb, tenant_id, datacenter, item, name): #get datacenter info datacenter_id, myvim = get_datacenter_by_name_uuid(mydb, tenant_id, datacenter) @@ -2809,6 +2995,7 @@ def vim_action_get(mydb, tenant_id, datacenter, item, name): print "vim_action Not possible to get_%s_list from VIM: %s " % (item, str(e)) raise NfvoException("Not possible to get_{}_list from VIM: {}".format(item, str(e)), e.http_code) + def vim_action_delete(mydb, tenant_id, datacenter, item, name): #get datacenter info if tenant_id == "any": @@ -2842,6 +3029,7 @@ def vim_action_delete(mydb, tenant_id, datacenter, item, name): return "{} {} {} deleted".format(item[:-1], item_id,item_name) + def vim_action_create(mydb, tenant_id, datacenter, item, descriptor): #get datacenter info logger.debug("vim_action_create descriptor %s", str(descriptor)) diff --git a/vim_thread.py b/vim_thread.py index 5460523b..42279a23 100644 --- a/vim_thread.py +++ b/vim_thread.py @@ -33,97 +33,194 @@ import time import Queue import logging import vimconn - +from db_base import db_base_Exception # from logging import Logger # import auxiliary_functions as af -# TODO: insert a logging system + +def is_task_id(id): + return True if id[:5] == "TASK." else False class vim_thread(threading.Thread): - def __init__(self, vimconn, name=None): - '''Init a thread. + def __init__(self, vimconn, task_lock, name=None, datacenter_name=None, datacenter_tenant_id=None, db=None, db_lock=None): + """Init a thread. Arguments: 'id' number of thead 'name' name of thread 'host','user': host ip or name to manage and user 'db', 'db_lock': database class and lock to use it in exclusion - ''' + """ + self.tasksResult = {} + """ It will contain a dictionary with + task_id: + status: enqueued,done,error,deleted,processing + result: VIM result, + """ threading.Thread.__init__(self) self.vim = vimconn + self.datacenter_name = datacenter_name + self.datacenter_tenant_id = datacenter_tenant_id if not name: - self.name = vimconn["id"] + "-" + vimconn["config"]["datacenter_tenant_id"] + self.name = vimconn["id"] + "." + vimconn["config"]["datacenter_tenant_id"] else: self.name = name self.logger = logging.getLogger('openmano.vim.'+self.name) + self.db = db + self.db_lock = db_lock - self.queueLock = threading.Lock() - self.taskQueue = Queue.Queue(2000) + self.task_lock = task_lock + self.task_queue = Queue.Queue(2000) - - def insert_task(self, task, *aditional): + def insert_task(self, task): try: - self.queueLock.acquire() - task = self.taskQueue.put( (task,) + aditional, timeout=5) - self.queueLock.release() - return 1, None + self.task_queue.put(task, False) + return task["id"] except Queue.Full: - return -1, "timeout inserting a task over host " + self.name + raise vimconn.vimconnException(self.name + ": timeout inserting a task") + + def del_task(self, task): + with self.task_lock: + if task["status"] == "enqueued": + task["status"] == "deleted" + return True + else: # task["status"] == "processing" + self.task_lock.release() + return False def run(self): self.logger.debug("Starting") while True: #TODO reload service while True: - self.queueLock.acquire() - if not self.taskQueue.empty(): - task = self.taskQueue.get() + if not self.task_queue.empty(): + task = self.task_queue.get() + self.task_lock.acquire() + if task["status"] == "deleted": + self.task_lock.release() + continue + task["status"] == "processing" + self.task_lock.release() else: - task = None - self.queueLock.release() - - if task is None: now=time.time() time.sleep(1) - continue - - if task[0] == 'instance': - pass - elif task[0] == 'image': - pass - elif task[0] == 'exit': - print self.name, ": processing task exit" - self.terminate() + continue + self.logger.debug("processing task id={} name={} params={}".format(task["id"], task["name"], + str(task["params"]))) + if task["name"] == 'exit' or task["name"] == 'reload': + result, content = self.terminate(task) + elif task["name"] == 'new-vm': + result, content = self.new_vm(task) + elif task["name"] == 'del-vm': + result, content = self.del_vm(task) + elif task["name"] == 'new-net': + result, content = self.new_net(task) + elif task["name"] == 'del-net': + result, content = self.del_net(task) + else: + error_text = "unknown task {}".format(task["name"]) + self.logger.error(error_text) + result = False + content = error_text + + with self.task_lock: + task["status"] = "done" if result else "error" + task["result"] = content + self.task_queue.task_done() + + if task["name"] == 'exit': return 0 - elif task[0] == 'reload': - print self.name, ": processing task reload terminating and relaunching" - self.terminate() + elif task["name"] == 'reload': break - elif task[0] == 'edit-iface': - pass - elif task[0] == 'restore-iface': - pass - elif task[0] == 'new-ovsbridge': - pass - elif task[0] == 'new-vxlan': - pass - elif task[0] == 'del-ovsbridge': - pass - elif task[0] == 'del-vxlan': - pass - elif task[0] == 'create-ovs-bridge-port': - pass - elif task[0] == 'del-ovs-port': - pass - else: - self.logger.error("unknown task %s", str(task)) self.logger.debug("Finishing") - def terminate(self): - pass + def terminate(self, task): + return True, None + + def new_net(self, task): + try: + task_id = task["id"] + params = task["params"] + net_id = self.vim.new_network(*params) + with self.db_lock: + self.db.update_rows("instance_nets", UPDATE={"vim_net_id": net_id}, WHERE={"vim_net_id": task_id}) + return True, net_id + except db_base_Exception as e: + self.logger.error("Error updating database %s", str(e)) + return True, net_id + except vimconn.vimconnException as e: + return False, str(e) + + def new_vm(self, task): + try: + params = task["params"] + task_id = task["id"] + depends = task.get("depends") + net_list = params[5] + for net in net_list: + if is_task_id(net["net_id"]): # change task_id into network_id + try: + task_net = depends[net["net_id"]] + with self.task_lock: + if task_net["status"] == "error": + return False, "Cannot create VM because depends on a network that cannot be created: " + \ + str(task_net["result"]) + elif task_net["status"] == "enqueued" or task_net["status"] == "processing": + return False, "Cannot create VM because depends on a network still not created" + network_id = task_net["result"] + net["net_id"] = network_id + except Exception as e: + return False, "Error trying to map from task_id={} to task result: {}".format(net["net_id"], + str(e)) + vm_id = self.vim.new_vminstance(*params) + with self.db_lock: + self.db.update_rows("instance_vms", UPDATE={"vim_vm_id": vm_id}, WHERE={"vim_vm_id": task_id}) + return True, vm_id + except db_base_Exception as e: + self.logger.error("Error updtaing database %s", str(e)) + return True, vm_id + except vimconn.vimconnException as e: + return False, str(e) + + def del_vm(self, task): + vm_id = task["params"] + if is_task_id(vm_id): + try: + task_create = task["depends"][vm_id] + with self.task_lock: + if task_create["status"] == "error": + return True, "VM was not created. It has error: " + str(task_create["result"]) + elif task_create["status"] == "enqueued" or task_create["status"] == "processing": + return False, "Cannot delete VM because still creating" + vm_id = task_create["result"] + except Exception as e: + return False, "Error trying to get task_id='{}':".format(vm_id, str(e)) + try: + return True, self.vim.delete_vminstance(vm_id) + except vimconn.vimconnException as e: + return False, str(e) + + def del_net(self, task): + net_id = task["params"] + if is_task_id(net_id): + try: + task_create = task["depends"][net_id] + with self.task_lock: + if task_create["status"] == "error": + return True, "net was not created. It has error: " + str(task_create["result"]) + elif task_create["status"] == "enqueued" or task_create["status"] == "processing": + return False, "Cannot delete net because still creating" + net_id = task_create["result"] + except Exception as e: + return False, "Error trying to get task_id='{}':".format(net_id, str(e)) + try: + return True, self.vim.delete_network(net_id) + except vimconn.vimconnException as e: + return False, str(e) + diff --git a/vimconn.py b/vimconn.py index c1721cdb..a9bd9be6 100644 --- a/vimconn.py +++ b/vimconn.py @@ -373,6 +373,8 @@ class vimconnector(): 'VFnotShared'(SRIOV without VLAN tag) same as PF for network connectivity. VF where no other VFs are allocated on the same physical NIC 'bw': (optional) only for PF/VF/VFnotShared. Minimal Bandwidth required for the interface in GBPS + 'port_security': (optional) If False it must avoid any traffic filtering at this interface. If missing + or True, it must apply the default VIM behaviour After execution the method will add the key: 'vim_id': must be filled/added by this method with the VIM identifier generated by the VIM for this interface. 'net_list' is modified diff --git a/vimconn_openstack.py b/vimconn_openstack.py index 7ba280b0..b501d9da 100644 --- a/vimconn_openstack.py +++ b/vimconn_openstack.py @@ -68,7 +68,8 @@ volume_timeout = 60 server_timeout = 60 class vimconnector(vimconn.vimconnector): - def __init__(self, uuid, name, tenant_id, tenant_name, url, url_admin=None, user=None, passwd=None, log_level=None, config={}): + def __init__(self, uuid, name, tenant_id, tenant_name, url, url_admin=None, user=None, passwd=None, + log_level=None, config={}, persistent_info={}): '''using common constructor parameters. In this case 'url' is the keystone authorization url, 'url_admin' is not use @@ -77,7 +78,8 @@ class vimconnector(vimconn.vimconnector): if config.get('APIversion') == 'v3.3': self.osc_api_version = 'v3.3' vimconn.vimconnector.__init__(self, uuid, name, tenant_id, tenant_name, url, url_admin, user, passwd, log_level, config) - + + self.persistent_info = persistent_info self.k_creds={} self.n_creds={} if self.config.get("insecure"): diff --git a/vimconn_openvim.py b/vimconn_openvim.py index 9d6748a9..e86b54db 100644 --- a/vimconn_openvim.py +++ b/vimconn_openvim.py @@ -323,11 +323,13 @@ get_processor_rankings_response_schema = { } class vimconnector(vimconn.vimconnector): - def __init__(self, uuid, name, tenant_id, tenant_name, url, url_admin=None, user=None, passwd=None,log_level="DEBUG",config={}): + def __init__(self, uuid, name, tenant_id, tenant_name, url, url_admin=None, user=None, passwd=None, + log_level="DEBUG", config={}, persistent_info={}): vimconn.vimconnector.__init__(self, uuid, name, tenant_id, tenant_name, url, url_admin, user, passwd, log_level, config) self.tenant = None self.headers_req = {'content-type': 'application/json'} self.logger = logging.getLogger('openmano.vim.openvim') + self.persistent_info = persistent_info if tenant_id: self.tenant = tenant_id diff --git a/vimconn_vmware.py b/vimconn_vmware.py index 3e81f513..cf7445d0 100644 --- a/vimconn_vmware.py +++ b/vimconn_vmware.py @@ -127,7 +127,7 @@ flavorlist = {} class vimconnector(vimconn.vimconnector): def __init__(self, uuid=None, name=None, tenant_id=None, tenant_name=None, - url=None, url_admin=None, user=None, passwd=None, log_level=None, config={}): + url=None, url_admin=None, user=None, passwd=None, log_level=None, config={}, persistent_info={}): """ Constructor create vmware connector to vCloud director. @@ -164,6 +164,7 @@ class vimconnector(vimconn.vimconnector): self.logger = logging.getLogger('openmano.vim.vmware') self.logger.setLevel(10) + self.persistent_info = persistent_info self.name = name self.id = uuid -- 2.25.1 From eb044522c3d119cb594b465380d4979f0988d216 Mon Sep 17 00:00:00 2001 From: kate Date: Mon, 6 Mar 2017 23:54:39 -0800 Subject: [PATCH 04/16] =?utf8?q?=20Changes=20in=20vimconn=5Fvmware.py=20:?= =?utf8?q?=20Modified=20flavor=20logic=20as=20per=20Code=20review=20commen?= =?utf8?q?ts.=20Fixed=20defect=20=E2=80=93=20167=20VMware=20connector=20ne?= =?utf8?q?eds=20to=20resolve=20vCenter=20host=20name.=20Fixed=20defect=20?= =?utf8?q?=E2=80=93=20185=20Plug=20test=20-=20Communication=20to=20VMware?= =?utf8?q?=20VIM=20does=20not=20work=20unless=20hostnames=20are=20added=20?= =?utf8?q?to=20/etc./hosts=20in=20RO=E2=80=9D?= MIME-Version: 1.0 Content-Type: text/plain; charset=utf8 Content-Transfer-Encoding: 8bit Change-Id: Ibd83a870b4b086c4bd2d804e711adbf1c8cbb184 Signed-off-by: kate --- vimconn_vmware.py | 184 +++++++++++++++++++++++++--------------------- 1 file changed, 99 insertions(+), 85 deletions(-) diff --git a/vimconn_vmware.py b/vimconn_vmware.py index cf7445d0..a2242556 100644 --- a/vimconn_vmware.py +++ b/vimconn_vmware.py @@ -121,11 +121,10 @@ netStatus2manoFormat = {'ACTIVE': 'ACTIVE', 'PAUSED': 'PAUSED', 'INACTIVE': 'INA 'ERROR': 'ERROR', 'DELETED': 'DELETED' } -# dict used to store flavor in memory -flavorlist = {} - - class vimconnector(vimconn.vimconnector): + # dict used to store flavor in memory + flavorlist = {} + def __init__(self, uuid=None, name=None, tenant_id=None, tenant_name=None, url=None, url_admin=None, user=None, passwd=None, log_level=None, config={}, persistent_info={}): """ @@ -154,6 +153,7 @@ class vimconnector(vimconn.vimconnector): dict['admin_username'] dict['admin_password'] + config - Provide NSX and vCenter information Returns: Nothing. @@ -181,6 +181,10 @@ class vimconnector(vimconn.vimconnector): self.nsx_manager = None self.nsx_user = None self.nsx_password = None + self.vcenter_ip = None + self.vcenter_port = None + self.vcenter_user = None + self.vcenter_password = None if tenant_name is not None: orgnameandtenant = tenant_name.split(":") @@ -208,6 +212,11 @@ class vimconnector(vimconn.vimconnector): except KeyError: raise vimconn.vimconnException(message="Error: nsx manager or nsx user or nsx password is empty in Config") + self.vcenter_ip = config.get("vcenter_ip", None) + self.vcenter_port = config.get("vcenter_port", None) + self.vcenter_user = config.get("vcenter_user", None) + self.vcenter_password = config.get("vcenter_password", None) + self.org_uuid = None self.vca = None @@ -695,9 +704,9 @@ class vimconnector(vimconn.vimconnector): """Obtain flavor details from the VIM Returns the flavor dict details {'id':<>, 'name':<>, other vim specific } #TODO to concrete """ - if flavor_id not in flavorlist: + if flavor_id not in vimconnector.flavorlist: raise vimconn.vimconnNotFoundException("Flavor not found.") - return flavorlist[flavor_id] + return vimconnector.flavorlist[flavor_id] def new_flavor(self, flavor_data): """Adds a tenant flavor to VIM @@ -745,7 +754,7 @@ class vimconnector(vimconn.vimconnector): new_flavor[FLAVOR_DISK_KEY] = disk # generate a new uuid put to internal dict and return it. flavor_id = uuid.uuid4() - flavorlist[str(flavor_id)] = new_flavor + vimconnector.flavorlist[str(flavor_id)] = new_flavor self.logger.debug("Created flavor - {} : {}".format(flavor_id, new_flavor)) return str(flavor_id) @@ -755,10 +764,10 @@ class vimconnector(vimconn.vimconnector): Returns the used id or raise an exception """ - if flavor_id not in flavorlist: + if flavor_id not in vimconnector.flavorlist: raise vimconn.vimconnNotFoundException("Flavor not found.") - flavorlist.pop(flavor_id, None) + vimconnector.flavorlist.pop(flavor_id, None) return flavor_id def new_image(self, image_dict): @@ -1223,8 +1232,8 @@ class vimconnector(vimconn.vimconnector): """ self.logger.info("Creating new instance for entry {}".format(name)) - self.logger.debug("desc {} boot {} image_id: {} flavor_id: {} net_list: {} cloud_config {}". - format(description, start, image_id, flavor_id, net_list, cloud_config)) + self.logger.debug("desc {} boot {} image_id: {} flavor_id: {} net_list: {} cloud_config {}".format( + description, start, image_id, flavor_id, net_list, cloud_config)) vca = self.connect() if not vca: raise vimconn.vimconnConnectionException("self.connect() is failed.") @@ -1263,13 +1272,13 @@ class vimconnector(vimconn.vimconnector): vm_disk = None pci_devices_info = [] if flavor_id is not None: - if flavor_id not in flavorlist: + if flavor_id not in vimconnector.flavorlist: raise vimconn.vimconnNotFoundException("new_vminstance(): Failed create vApp {}: " "Failed retrieve flavor information " "flavor id {}".format(name, flavor_id)) else: try: - flavor = flavorlist[flavor_id] + flavor = vimconnector.flavorlist[flavor_id] vm_cpus = flavor[FLAVOR_VCPUS_KEY] vm_memory = flavor[FLAVOR_RAM_KEY] vm_disk = flavor[FLAVOR_DISK_KEY] @@ -1281,8 +1290,8 @@ class vimconnector(vimconn.vimconnector): for interface in numa.get("interfaces",() ): if interface["dedicated"].strip()=="yes": pci_devices_info.append(interface) - except KeyError: - raise vimconn.vimconnException("Corrupted flavor. {}".format(flavor_id)) + except Exception as exp: + raise vimconn.vimconnException("Corrupted flavor. {}.Exception: {}".format(flavor_id, exp)) # image upload creates template name as catalog name space Template. templateName = self.get_catalogbyid(catalog_uuid=image_id, catalogs=catalogs) @@ -2907,7 +2916,6 @@ class vimconnector(vimconn.vimconnector): vmext = vim_info.find('vmext:VmVimObjectRef', namespaces) if vmext is not None: vm_vcenter_info["vm_moref_id"] = vmext.find('vmext:MoRef', namespaces).text - vm_vcenter_info["vim_server_href"] = vmext.find('vmext:VimServerRef', namespaces).attrib['href'] parsed_respond["vm_vcenter_info"]= vm_vcenter_info virtual_hardware_section = children_section.find('ovf:VirtualHardwareSection', namespaces) @@ -3075,27 +3083,31 @@ class vimconnector(vimconn.vimconnector): vm_obj = None vcenter_conect = None self.logger.info("Add pci devices {} into vApp {}".format(pci_devices , vapp_uuid)) - #Assuming password of vCenter user is same as password of vCloud user - vm_moref_id , vm_vcenter_host , vm_vcenter_username, vm_vcenter_port = self.get_vcenter_info_rest(vapp_uuid) - self.logger.info("vm_moref_id, {} vm_vcenter_host {} vm_vcenter_username{} "\ - "vm_vcenter_port{}".format( - vm_moref_id, vm_vcenter_host, - vm_vcenter_username, vm_vcenter_port)) - if vm_moref_id and vm_vcenter_host and vm_vcenter_username: + try: + vm_vcenter_info = self.get_vm_vcenter_info(vapp_uuid) + except Exception as exp: + self.logger.error("Error occurred while getting vCenter infromationn"\ + " for VM : {}".format(exp)) + raise vimconn.vimconnException(message=exp) + + if vm_vcenter_info["vm_moref_id"]: context = None if hasattr(ssl, '_create_unverified_context'): context = ssl._create_unverified_context() try: no_of_pci_devices = len(pci_devices) if no_of_pci_devices > 0: - vcenter_conect = SmartConnect(host=vm_vcenter_host, user=vm_vcenter_username, - pwd=self.passwd, port=int(vm_vcenter_port) , - sslContext=context) + vcenter_conect = SmartConnect( + host=vm_vcenter_info["vm_vcenter_ip"], + user=vm_vcenter_info["vm_vcenter_user"], + pwd=vm_vcenter_info["vm_vcenter_password"], + port=int(vm_vcenter_info["vm_vcenter_port"]), + sslContext=context) atexit.register(Disconnect, vcenter_conect) content = vcenter_conect.RetrieveContent() #Get VM and its host - host_obj, vm_obj = self.get_vm_obj(content ,vm_moref_id) + host_obj, vm_obj = self.get_vm_obj(content ,vm_vcenter_info["vm_moref_id"]) self.logger.info("VM {} is currently on host {}".format(vm_obj, host_obj)) if host_obj and vm_obj: #get PCI devies from host on which vapp is currently installed @@ -3133,7 +3145,7 @@ class vimconnector(vimconn.vimconnector): if status: self.logger.info("Added PCI device {} to VM {}".format(pci_device,str(vm_obj))) else: - self.logger.info("Fail to add PCI device {} to VM {}".format(pci_device,str(vm_obj))) + self.logger.error("Fail to add PCI device {} to VM {}".format(pci_device,str(vm_obj))) return True, vm_obj, vcenter_conect else: self.logger.error("Currently there is no host with"\ @@ -3364,10 +3376,9 @@ class vimconnector(vimconn.vimconnector): exp)) return task - def get_vcenter_info_rest(self , vapp_uuid): + def get_vm_vcenter_info(self , vapp_uuid): """ - https://192.169.241.105/api/admin/extension/vimServer/cc82baf9-9f80-4468-bfe9-ce42b3f9dde5 - Method to get details of vCenter + Method to get details of vCenter and vm Args: vapp_uuid - uuid of vApp or VM @@ -3375,46 +3386,39 @@ class vimconnector(vimconn.vimconnector): Returns: Moref Id of VM and deails of vCenter """ - vm_moref_id = None - vm_vcenter = None - vm_vcenter_username = None - vm_vcenter_port = None - - vm_details = self.get_vapp_details_rest(vapp_uuid, need_admin_access=True) - if vm_details and "vm_vcenter_info" in vm_details: - vm_moref_id = vm_details["vm_vcenter_info"]["vm_moref_id"] - vim_server_href = vm_details["vm_vcenter_info"]["vim_server_href"] - - if vim_server_href: - vca = self.connect_as_admin() - if not vca: - raise vimconn.vimconnConnectionException("self.connect() is failed") - if vim_server_href is None: - self.logger.error("No url to get vcenter details") - - if vca.vcloud_session and vca.vcloud_session.organization: - response = Http.get(url=vim_server_href, - headers=vca.vcloud_session.get_vcloud_headers(), - verify=vca.verify, - logger=vca.logger) - - if response.status_code != requests.codes.ok: - self.logger.debug("GET REST API call {} failed. Return status code {}".format(vim_server_href, - response.status_code)) - try: - namespaces={"vmext":"http://www.vmware.com/vcloud/extension/v1.5", - "vcloud":"http://www.vmware.com/vcloud/v1.5" - } - xmlroot_respond = XmlElementTree.fromstring(response.content) - vm_vcenter_username = xmlroot_respond.find('vmext:Username', namespaces).text - vcenter_url = xmlroot_respond.find('vmext:Url', namespaces).text - vm_vcenter_port = vcenter_url.split(":")[2] - vm_vcenter = vcenter_url.split(":")[1].split("//")[1] + vm_vcenter_info = {} + + if self.vcenter_ip is not None: + vm_vcenter_info["vm_vcenter_ip"] = self.vcenter_ip + else: + raise vimconn.vimconnException(message="vCenter IP is not provided."\ + " Please provide vCenter IP while attaching datacenter to tenant in --config") + if self.vcenter_port is not None: + vm_vcenter_info["vm_vcenter_port"] = self.vcenter_port + else: + raise vimconn.vimconnException(message="vCenter port is not provided."\ + " Please provide vCenter port while attaching datacenter to tenant in --config") + if self.vcenter_user is not None: + vm_vcenter_info["vm_vcenter_user"] = self.vcenter_user + else: + raise vimconn.vimconnException(message="vCenter user is not provided."\ + " Please provide vCenter user while attaching datacenter to tenant in --config") + + if self.vcenter_password is not None: + vm_vcenter_info["vm_vcenter_password"] = self.vcenter_password + else: + raise vimconn.vimconnException(message="vCenter user password is not provided."\ + " Please provide vCenter user password while attaching datacenter to tenant in --config") + try: + vm_details = self.get_vapp_details_rest(vapp_uuid, need_admin_access=True) + if vm_details and "vm_vcenter_info" in vm_details: + vm_vcenter_info["vm_moref_id"] = vm_details["vm_vcenter_info"].get("vm_moref_id", None) - except Exception as exp : - self.logger.info("Error occurred calling rest api for vcenter information {}".format(exp)) + return vm_vcenter_info - return vm_moref_id , vm_vcenter , vm_vcenter_username, vm_vcenter_port + except Exception as exp: + self.logger.error("Error occurred while getting vCenter infromationn"\ + " for VM : {}".format(exp)) def get_vm_pci_details(self, vmuuid): @@ -3430,28 +3434,38 @@ class vimconnector(vimconn.vimconnector): """ vm_pci_devices_info = {} try: - vm_moref_id , vm_vcenter_host , vm_vcenter_username, vm_vcenter_port = self.get_vcenter_info_rest(vmuuid) - if vm_moref_id and vm_vcenter_host and vm_vcenter_username: + vm_vcenter_info = self.get_vm_vcenter_info(vmuuid) + if vm_vcenter_info["vm_moref_id"]: context = None if hasattr(ssl, '_create_unverified_context'): context = ssl._create_unverified_context() - vcenter_conect = SmartConnect(host=vm_vcenter_host, user=vm_vcenter_username, - pwd=self.passwd, port=int(vm_vcenter_port), - sslContext=context) + vcenter_conect = SmartConnect(host=vm_vcenter_info["vm_vcenter_ip"], + user=vm_vcenter_info["vm_vcenter_user"], + pwd=vm_vcenter_info["vm_vcenter_password"], + port=int(vm_vcenter_info["vm_vcenter_port"]), + sslContext=context + ) atexit.register(Disconnect, vcenter_conect) content = vcenter_conect.RetrieveContent() #Get VM and its host - host_obj, vm_obj = self.get_vm_obj(content ,vm_moref_id) - for device in vm_obj.config.hardware.device: - if type(device) == vim.vm.device.VirtualPCIPassthrough: - device_details={'devide_id':device.backing.id, - 'pciSlotNumber':device.slotInfo.pciSlotNumber - } - vm_pci_devices_info[device.deviceInfo.label] = device_details + if content: + host_obj, vm_obj = self.get_vm_obj(content ,vm_vcenter_info["vm_moref_id"]) + if host_obj and vm_obj: + vm_pci_devices_info["host_name"]= host_obj.name + vm_pci_devices_info["host_ip"]= host_obj.config.network.vnic[0].spec.ip.ipAddress + for device in vm_obj.config.hardware.device: + if type(device) == vim.vm.device.VirtualPCIPassthrough: + device_details={'devide_id':device.backing.id, + 'pciSlotNumber':device.slotInfo.pciSlotNumber, + } + vm_pci_devices_info[device.deviceInfo.label] = device_details + else: + self.logger.error("Can not connect to vCenter while getting "\ + "PCI devices infromationn") + return vm_pci_devices_info except Exception as exp: - self.logger.info("Error occurred while getting PCI devices infromationn"\ - " for VM {} : {}".format(vm_obj,exp)) - return vm_pci_devices_info - + self.logger.error("Error occurred while getting VM infromationn"\ + " for VM : {}".format(exp)) + raise vimconn.vimconnException(message=exp) -- 2.25.1 From 9453a8a01de39baaacf22abf8fc7a62055983b33 Mon Sep 17 00:00:00 2001 From: garciadeblas Date: Thu, 9 Mar 2017 16:21:24 +0100 Subject: [PATCH 05/16] Makefile to automate tasks: env preparation, build, test, package creation Change-Id: Iaae780a8fbd105902204b630a959aa9e72bf27e7 Signed-off-by: garciadeblas --- Makefile | 40 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 40 insertions(+) create mode 100644 Makefile diff --git a/Makefile b/Makefile new file mode 100644 index 00000000..122fec0a --- /dev/null +++ b/Makefile @@ -0,0 +1,40 @@ +SHELL := /bin/bash +all: connectors build package + +prepare: + mkdir -p build + cp *.py build/ + #cd build; mv openmanod.py openmanod + cp openmano build/ + cp openmanod.cfg build/ + cp openmano.service build/ + cp -r vnfs build/ + cp -r scenarios build/ + cp -r instance-scenarios build/ + cp -r scripts build/ + cd build/scripts; mv service-openmano.sh service-openmano; mv openmano-report.sh openmano-report + cp -r database_utils build/ + +connectors: + rm -f build/openmanolinkervimconn.py + cd build; for i in `ls vimconn_*.py |sed "s/\.py//"` ; do echo "import $$i" >> openmanolinkervimconn.py; done + python build/openmanolinkervimconn.py + rm -f build/openmanolinkervimconn.py + +build: prepare connectors + python -m py_compile build/*.py + +clean: + rm -rf build + #find build -name '*.pyc' -delete + #find build -name '*.pyo' -delete + +pip: + cd build; ./setup.py sdist + #cp dist/* /root/artifacts/... + #fpm -s python -t deb build/setup.py + +test: + ./test/basictest.sh --force --insert-bashrc --install-openvim --init-openvim + + -- 2.25.1 From ea76ecf19940f622caae5fcd2c61eecd7c874299 Mon Sep 17 00:00:00 2001 From: garciadeblas Date: Thu, 9 Mar 2017 16:23:25 +0100 Subject: [PATCH 06/16] New file openmano.service: extracted from install-openmano-service.sh Change-Id: I5f2dc1df0586d73f3f5377b87c57dbbffd61c635 Signed-off-by: garciadeblas --- openmano.service | 11 +++++++++++ 1 file changed, 11 insertions(+) create mode 100644 openmano.service diff --git a/openmano.service b/openmano.service new file mode 100644 index 00000000..cc8966c8 --- /dev/null +++ b/openmano.service @@ -0,0 +1,11 @@ +[Unit] +Description=openmano server (OSM RO) + +[Service] +User=${USER_OWNER} +ExecStart=openmanod -c /etc/osm/openmanod.cfg --log-file=/var/log/osm/openmano.log +Restart=always + +[Install] +WantedBy=multi-user.target + -- 2.25.1 From cd520d35911a27d59c6d375330c1fe3af3d2914e Mon Sep 17 00:00:00 2001 From: garciadeblas Date: Thu, 9 Mar 2017 16:28:59 +0100 Subject: [PATCH 07/16] New file setup.py: builds a python package Change-Id: I60226de64fe73b0603cf46f3b1371309fe7afec3 Signed-off-by: garciadeblas --- setup.py | 44 ++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 44 insertions(+) create mode 100755 setup.py diff --git a/setup.py b/setup.py new file mode 100755 index 00000000..ee9fd52f --- /dev/null +++ b/setup.py @@ -0,0 +1,44 @@ +#!/usr/bin/env python + +from distutils.core import setup +#import glob + +setup(name='osm-ro', + version='1.0', + description='OSM Resource Orchestrator', + author='ETSI OSM', + author_email='alfonso.tiernosepulveda@telefonica.com', + maintainer='garciadeblas', + maintainer_email='gerardo.garciadeblas@telefonica.com', + url='https://osm.etsi.org/gitweb/?p=osm/RO.git;a=summary', + license='Apache 2.0', + #packages=['ro-server', 'ro-client'], + #py_modules=glob.glob('*.py') + py_modules=['console_proxy_thread', + 'db_base', + 'httpserver', + 'nfvo_db', + 'nfvo', + 'openmanoclient', + 'openmanoconfig', + 'openmanod', + 'openmano_schemas', + 'utils', + 'vimconn_openstack', + 'vimconn_openvim', + 'vimconn', + 'vimconn_vmware', + 'vmwarecli', + 'vmwarerecli', + ], + #package_data={'': ['vnfs', 'scenarios', 'instance-scenarios', 'database_utils/mano_db_structure.sql']} + data_files=[('/etc/osm/', ['openmanod.cfg']), + ('/etc/systemd/system/', ['openmano.service']), + ('vnfs', ['vnfs']), + ('scenarios', ['scenarios']), + ('instance-scenarios', ['instance-scenarios']), + ('database_utils', ['database-utils']), + ], + scripts=['openmanod.py', 'openmano', 'scripts/service-openmano', 'scripts/openmano-report',] + ) + -- 2.25.1 From 06e6c396413630640cafae3488442a0869f1642d Mon Sep 17 00:00:00 2001 From: garciadeblas Date: Tue, 28 Mar 2017 15:42:20 +0200 Subject: [PATCH 08/16] New README.rst and requirements.txt files, setup.py and Makefile updated Change-Id: Iefd4fbc2d7076bd62ba84fefaec24c5484f9d8de Signed-off-by: garciadeblas --- Makefile | 28 ++++++++++++++++++---------- README.rst | 8 ++++++++ requirements.txt | 21 +++++++++++++++++++++ setup.py | 26 +++++++++++++++++++++++++- 4 files changed, 72 insertions(+), 11 deletions(-) create mode 100644 README.rst create mode 100644 requirements.txt diff --git a/Makefile b/Makefile index 122fec0a..b5e6960f 100644 --- a/Makefile +++ b/Makefile @@ -1,5 +1,5 @@ SHELL := /bin/bash -all: connectors build package +all: pypackage debpackage prepare: mkdir -p build @@ -7,6 +7,8 @@ prepare: #cd build; mv openmanod.py openmanod cp openmano build/ cp openmanod.cfg build/ + cp requirements.txt build/ + cp README.rst build/ cp openmano.service build/ cp -r vnfs build/ cp -r scenarios build/ @@ -15,26 +17,32 @@ prepare: cd build/scripts; mv service-openmano.sh service-openmano; mv openmano-report.sh openmano-report cp -r database_utils build/ -connectors: +connectors: prepare + # python-novaclient is required for that rm -f build/openmanolinkervimconn.py cd build; for i in `ls vimconn_*.py |sed "s/\.py//"` ; do echo "import $$i" >> openmanolinkervimconn.py; done python build/openmanolinkervimconn.py rm -f build/openmanolinkervimconn.py -build: prepare connectors +build: prepare python -m py_compile build/*.py -clean: - rm -rf build - #find build -name '*.pyc' -delete - #find build -name '*.pyo' -delete - -pip: +pypackage: build cd build; ./setup.py sdist - #cp dist/* /root/artifacts/... + #cp build/dist/* /root/artifacts/... + +debpackage: build + echo "Nothing to be done yet" #fpm -s python -t deb build/setup.py +snappackage: + echo "Nothing to be done yet" + test: ./test/basictest.sh --force --insert-bashrc --install-openvim --init-openvim +clean: + rm -rf build + #find build -name '*.pyc' -delete + #find build -name '*.pyo' -delete diff --git a/README.rst b/README.rst new file mode 100644 index 00000000..3a2be888 --- /dev/null +++ b/README.rst @@ -0,0 +1,8 @@ +=========== +osm-ro +=========== + +osm-ro is the Resource Orchestrator for OSM, dealing with resource operations +against different VIMs such as Openstack, VMware's vCloud Director, openvim +and AWS. + diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 00000000..0f6b6f61 --- /dev/null +++ b/requirements.txt @@ -0,0 +1,21 @@ +--index-url https://pypi.python.org/simple/ + +PyYAML==3.10 +bottle==0.12.7 +PyMySQL==0.7.2 +jsonschema==2.5.1 +paramiko==1.16.0 +argcomplete==0.8.1 +requests==2.13.0 +logutils==0.3.3 +python-cinderclient==1.6.0 +python-glanceclient==2.0.0 +python-keystoneclient==2.3.1 +python-neutronclient==4.1.1 +python-novaclient==3.3.1 +pyvcloud==16 +pyvmomi==6.5 +progressbar==2.3 +prettytable==0.7.2 +boto==2.38.0 + diff --git a/setup.py b/setup.py index ee9fd52f..2d2d07a1 100755 --- a/setup.py +++ b/setup.py @@ -1,11 +1,34 @@ #!/usr/bin/env python -from distutils.core import setup +from setuptools import setup #import glob +requirements = [ + "PyYAML", + "bottle", + "mysqldb", + "jsonschema", + "paramiko", + "argcomplete", + "requests", + "logutils", + "pip", + "novaclient", + "keystoneclient", + "glanceclient", + "neutronclient", + "cinderclient", + "boto", + "pyvcloud", + "progressbar", + "prettytable", + "pyvmomi", +] + setup(name='osm-ro', version='1.0', description='OSM Resource Orchestrator', + long_description=open('README.rst').read(), author='ETSI OSM', author_email='alfonso.tiernosepulveda@telefonica.com', maintainer='garciadeblas', @@ -31,6 +54,7 @@ setup(name='osm-ro', 'vmwarecli', 'vmwarerecli', ], + install_requires=requirements, #package_data={'': ['vnfs', 'scenarios', 'instance-scenarios', 'database_utils/mano_db_structure.sql']} data_files=[('/etc/osm/', ['openmanod.cfg']), ('/etc/systemd/system/', ['openmano.service']), -- 2.25.1 From 2c290ca4088492a3c32bb6ab218d0004da68f6ea Mon Sep 17 00:00:00 2001 From: garciadeblas Date: Thu, 6 Apr 2017 03:12:51 +0200 Subject: [PATCH 09/16] Restructuring code in osm_ro folder, and setup based on MANIFEST Also updated Makefile and service-openmano.sh Change-Id: I60cf49013315efafd73de377452e38faf2f2f1e0 Signed-off-by: garciadeblas --- MANIFEST.in | 9 ++ Makefile | 49 ++++----- openmanod.py | 8 +- openmano.service => osm-ro.service | 2 +- .../console_proxy_thread.py | 0 db_base.py => osm_ro/db_base.py | 0 httpserver.py => osm_ro/httpserver.py | 0 nfvo.py => osm_ro/nfvo.py | 0 nfvo_db.py => osm_ro/nfvo_db.py | 0 .../openmano_schemas.py | 0 openmanoclient.py => osm_ro/openmanoclient.py | 0 utils.py => osm_ro/utils.py | 0 vim_thread.py => osm_ro/vim_thread.py | 0 vimconn.py => osm_ro/vimconn.py | 0 .../vimconn_openstack.py | 0 .../vimconn_openvim.py | 0 vimconn_vmware.py => osm_ro/vimconn_vmware.py | 0 vmwarecli.py => osm_ro/vmwarecli.py | 0 vmwarerecli.py => osm_ro/vmwarerecli.py | 0 requirements.txt | 36 +++---- scripts/service-openmano.sh | 4 +- setup.py | 99 ++++++++++--------- 22 files changed, 110 insertions(+), 97 deletions(-) create mode 100644 MANIFEST.in rename openmano.service => osm-ro.service (59%) rename console_proxy_thread.py => osm_ro/console_proxy_thread.py (100%) rename db_base.py => osm_ro/db_base.py (100%) rename httpserver.py => osm_ro/httpserver.py (100%) rename nfvo.py => osm_ro/nfvo.py (100%) rename nfvo_db.py => osm_ro/nfvo_db.py (100%) rename openmano_schemas.py => osm_ro/openmano_schemas.py (100%) rename openmanoclient.py => osm_ro/openmanoclient.py (100%) rename utils.py => osm_ro/utils.py (100%) rename vim_thread.py => osm_ro/vim_thread.py (100%) rename vimconn.py => osm_ro/vimconn.py (100%) rename vimconn_openstack.py => osm_ro/vimconn_openstack.py (100%) rename vimconn_openvim.py => osm_ro/vimconn_openvim.py (100%) rename vimconn_vmware.py => osm_ro/vimconn_vmware.py (100%) rename vmwarecli.py => osm_ro/vmwarecli.py (100%) rename vmwarerecli.py => osm_ro/vmwarerecli.py (100%) diff --git a/MANIFEST.in b/MANIFEST.in new file mode 100644 index 00000000..154ea275 --- /dev/null +++ b/MANIFEST.in @@ -0,0 +1,9 @@ +#include MANIFEST.in +#include requirements.txt +include README.rst +include openmano +include openmanod.py +include openmanod.cfg +include osm-ro.service +recursive-include osm_ro * + diff --git a/Makefile b/Makefile index b5e6960f..76a29908 100644 --- a/Makefile +++ b/Makefile @@ -2,42 +2,47 @@ SHELL := /bin/bash all: pypackage debpackage prepare: - mkdir -p build - cp *.py build/ - #cd build; mv openmanod.py openmanod - cp openmano build/ - cp openmanod.cfg build/ + mkdir -p build/ + cp MANIFEST.in build/ cp requirements.txt build/ cp README.rst build/ - cp openmano.service build/ - cp -r vnfs build/ - cp -r scenarios build/ - cp -r instance-scenarios build/ - cp -r scripts build/ - cd build/scripts; mv service-openmano.sh service-openmano; mv openmano-report.sh openmano-report - cp -r database_utils build/ + cp setup.py build/ + cp -r osm_ro build/ + cp openmano build/ + cp openmanod.py build/ + cp openmanod.cfg build/ + cp osm-ro.service build/ + cp -r vnfs build/osm_ro + cp -r scenarios build/osm_ro + cp -r instance-scenarios build/osm_ro + cp -r scripts build/osm_ro + cp -r database_utils build/osm_ro connectors: prepare # python-novaclient is required for that - rm -f build/openmanolinkervimconn.py - cd build; for i in `ls vimconn_*.py |sed "s/\.py//"` ; do echo "import $$i" >> openmanolinkervimconn.py; done - python build/openmanolinkervimconn.py - rm -f build/openmanolinkervimconn.py + rm -f build/osm_ro/openmanolinkervimconn.py + cd build/osm_ro; for i in `ls vimconn_*.py |sed "s/\.py//"` ; do echo "import $$i" >> openmanolinkervimconn.py; done + python build/osm_ro/openmanolinkervimconn.py + rm -f build/osm_ro/openmanolinkervimconn.py -build: prepare - python -m py_compile build/*.py +build: connectors prepare + python -m py_compile build/osm_ro/*.py -pypackage: build +pypackage: prepare cd build; ./setup.py sdist - #cp build/dist/* /root/artifacts/... + cd build; ./setup.py bdist_wheel -debpackage: build - echo "Nothing to be done yet" +debpackage: prepare + echo "Nothing to be done" + #cd build; ./setup.py --command-packages=stdeb.command bdist_deb #fpm -s python -t deb build/setup.py snappackage: echo "Nothing to be done yet" +sync: + #cp build/dist/* /root/artifacts/... + test: ./test/basictest.sh --force --insert-bashrc --install-openvim --init-openvim diff --git a/openmanod.py b/openmanod.py index 03901c3f..bb32339c 100755 --- a/openmanod.py +++ b/openmanod.py @@ -37,19 +37,17 @@ __version__="0.5.8-r518" version_date="Jan 2017" database_version="0.19" #expected database schema version -import httpserver import time import sys import getopt import yaml -import nfvo_db from jsonschema import validate as js_v, exceptions as js_e -from openmano_schemas import config_schema -from db_base import db_base_Exception -import nfvo import logging import logging.handlers as log_handlers import socket +from osm_ro import httpserver, nfvo, nfvo_db +from osm_ro.openmano_schemas import config_schema +from osm_ro.db_base import db_base_Exception global global_config global logger diff --git a/openmano.service b/osm-ro.service similarity index 59% rename from openmano.service rename to osm-ro.service index cc8966c8..155150b7 100644 --- a/openmano.service +++ b/osm-ro.service @@ -3,7 +3,7 @@ Description=openmano server (OSM RO) [Service] User=${USER_OWNER} -ExecStart=openmanod -c /etc/osm/openmanod.cfg --log-file=/var/log/osm/openmano.log +ExecStart=openmanod.py -c /etc/osm/openmanod.cfg --log-file=/var/log/osm/openmano.log Restart=always [Install] diff --git a/console_proxy_thread.py b/osm_ro/console_proxy_thread.py similarity index 100% rename from console_proxy_thread.py rename to osm_ro/console_proxy_thread.py diff --git a/db_base.py b/osm_ro/db_base.py similarity index 100% rename from db_base.py rename to osm_ro/db_base.py diff --git a/httpserver.py b/osm_ro/httpserver.py similarity index 100% rename from httpserver.py rename to osm_ro/httpserver.py diff --git a/nfvo.py b/osm_ro/nfvo.py similarity index 100% rename from nfvo.py rename to osm_ro/nfvo.py diff --git a/nfvo_db.py b/osm_ro/nfvo_db.py similarity index 100% rename from nfvo_db.py rename to osm_ro/nfvo_db.py diff --git a/openmano_schemas.py b/osm_ro/openmano_schemas.py similarity index 100% rename from openmano_schemas.py rename to osm_ro/openmano_schemas.py diff --git a/openmanoclient.py b/osm_ro/openmanoclient.py similarity index 100% rename from openmanoclient.py rename to osm_ro/openmanoclient.py diff --git a/utils.py b/osm_ro/utils.py similarity index 100% rename from utils.py rename to osm_ro/utils.py diff --git a/vim_thread.py b/osm_ro/vim_thread.py similarity index 100% rename from vim_thread.py rename to osm_ro/vim_thread.py diff --git a/vimconn.py b/osm_ro/vimconn.py similarity index 100% rename from vimconn.py rename to osm_ro/vimconn.py diff --git a/vimconn_openstack.py b/osm_ro/vimconn_openstack.py similarity index 100% rename from vimconn_openstack.py rename to osm_ro/vimconn_openstack.py diff --git a/vimconn_openvim.py b/osm_ro/vimconn_openvim.py similarity index 100% rename from vimconn_openvim.py rename to osm_ro/vimconn_openvim.py diff --git a/vimconn_vmware.py b/osm_ro/vimconn_vmware.py similarity index 100% rename from vimconn_vmware.py rename to osm_ro/vimconn_vmware.py diff --git a/vmwarecli.py b/osm_ro/vmwarecli.py similarity index 100% rename from vmwarecli.py rename to osm_ro/vmwarecli.py diff --git a/vmwarerecli.py b/osm_ro/vmwarerecli.py similarity index 100% rename from vmwarerecli.py rename to osm_ro/vmwarerecli.py diff --git a/requirements.txt b/requirements.txt index 0f6b6f61..38c055cb 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,21 +1,21 @@ --index-url https://pypi.python.org/simple/ -PyYAML==3.10 -bottle==0.12.7 -PyMySQL==0.7.2 -jsonschema==2.5.1 -paramiko==1.16.0 -argcomplete==0.8.1 -requests==2.13.0 -logutils==0.3.3 -python-cinderclient==1.6.0 -python-glanceclient==2.0.0 -python-keystoneclient==2.3.1 -python-neutronclient==4.1.1 -python-novaclient==3.3.1 -pyvcloud==16 -pyvmomi==6.5 -progressbar==2.3 -prettytable==0.7.2 -boto==2.38.0 +PyYAML +bottle +MySQL-python +jsonschema +paramiko +argcomplete +requests +logutils +python-novaclient +python-keystoneclient +python-glanceclient +python-neutronclient +python-cinderclient +pyvcloud +pyvmomi +progressbar +prettytable +boto diff --git a/scripts/service-openmano.sh b/scripts/service-openmano.sh index 1a48d4a3..0224d20f 100755 --- a/scripts/service-openmano.sh +++ b/scripts/service-openmano.sh @@ -116,7 +116,7 @@ do running=y done #if installed as a service and it is not provided a screen name call service - [[ -f /etc/systemd/system/openmano.service ]] && [[ -z $option_screen_name ]] && running=y #&& service openmano status + [[ -f /etc/systemd/system/osm-ro.service ]] && [[ -z $option_screen_name ]] && running=y #&& service osm-ro status if [ -z "$running" ] then echo -n " $om_name not running" && [[ -n "$option_screen_name" ]] && echo " on screen '$option_screen_name'" || echo @@ -124,7 +124,7 @@ do fi #if installed as a service and it is not provided a screen name call service - [[ -f /etc/systemd/system/openmano.service ]] && [[ -z $option_screen_name ]] && service openmano $om_action && ( [[ $om_action == status ]] || sleep 5 ) && exit $? + [[ -f /etc/systemd/system/osm-ro.service ]] && [[ -z $option_screen_name ]] && service osm-ro $om_action && ( [[ $om_action == status ]] || sleep 5 ) && exit $? #stop diff --git a/setup.py b/setup.py index 2d2d07a1..68781be1 100755 --- a/setup.py +++ b/setup.py @@ -1,68 +1,69 @@ #!/usr/bin/env python from setuptools import setup +from setuptools.command.install import install +from os import system #import glob -requirements = [ +_name = 'osm_ro' +_version = '1.0.0' +_description = 'OSM Resource Orchestrator' +_author = 'ETSI OSM' +_author_email = 'alfonso.tiernosepulveda@telefonica.com' +_maintainer = 'garciadeblas' +_maintainer_email = 'gerardo.garciadeblas@telefonica.com' +_license = 'Apache 2.0' +_url = 'https://osm.etsi.org/gitweb/?p=osm/RO.git;a=summary' +_requirements = [ "PyYAML", "bottle", - "mysqldb", + "MySQL-python", "jsonschema", "paramiko", "argcomplete", "requests", "logutils", - "pip", - "novaclient", - "keystoneclient", - "glanceclient", - "neutronclient", - "cinderclient", - "boto", + "python-novaclient", + "python-keystoneclient", + "python-glanceclient", + "python-neutronclient", + "python-cinderclient", "pyvcloud", "progressbar", "prettytable", "pyvmomi", + "boto", ] -setup(name='osm-ro', - version='1.0', - description='OSM Resource Orchestrator', - long_description=open('README.rst').read(), - author='ETSI OSM', - author_email='alfonso.tiernosepulveda@telefonica.com', - maintainer='garciadeblas', - maintainer_email='gerardo.garciadeblas@telefonica.com', - url='https://osm.etsi.org/gitweb/?p=osm/RO.git;a=summary', - license='Apache 2.0', - #packages=['ro-server', 'ro-client'], - #py_modules=glob.glob('*.py') - py_modules=['console_proxy_thread', - 'db_base', - 'httpserver', - 'nfvo_db', - 'nfvo', - 'openmanoclient', - 'openmanoconfig', - 'openmanod', - 'openmano_schemas', - 'utils', - 'vimconn_openstack', - 'vimconn_openvim', - 'vimconn', - 'vimconn_vmware', - 'vmwarecli', - 'vmwarerecli', - ], - install_requires=requirements, - #package_data={'': ['vnfs', 'scenarios', 'instance-scenarios', 'database_utils/mano_db_structure.sql']} - data_files=[('/etc/osm/', ['openmanod.cfg']), - ('/etc/systemd/system/', ['openmano.service']), - ('vnfs', ['vnfs']), - ('scenarios', ['scenarios']), - ('instance-scenarios', ['instance-scenarios']), - ('database_utils', ['database-utils']), - ], - scripts=['openmanod.py', 'openmano', 'scripts/service-openmano', 'scripts/openmano-report',] - ) +class ROInstaller(install): + def run(self): + cmd = 'echo "Running install script"' + system(cmd) + install.run(self) + +setup(name=_name, + version = _version, + description = _description, + long_description = open('README.rst').read(), + author = _author, + author_email = _author_email, + maintainer = _maintainer, + maintainer_email = _maintainer_email, + url = _url, + license = _license, + packages = [_name], + #packages = ['osm_ro', 'osm_roclient'], + package_dir = {_name: _name}, + package_data = {_name: ['vnfs/*.yaml', 'vnfs/examples/*.yaml', + 'scenarios/*.yaml', 'scenarios/examples/*.yaml', + 'instance-scenarios/examples/*.yaml', 'database_utils/*', + 'scripts/install-openmano*.sh']}, + data_files = [('/etc/osm/', ['openmanod.cfg']), + ('/etc/systemd/system/', ['osm-ro.service']), + ], + scripts=['openmanod.py', 'openmano', 'osm_ro/scripts/service-openmano.sh', 'osm_ro/scripts/openmano-report.sh',], + install_requires=_requirements, + include_package_data=True, + cmdclass = {'install': ROInstaller}, + ) -- 2.25.1 From 89cb0d152c7a052ec868a666a9789056596fbdc7 Mon Sep 17 00:00:00 2001 From: Gennadiy Dubina Date: Mon, 3 Apr 2017 15:46:41 +0300 Subject: [PATCH 10/16] move db server install script to separate file Signed-off-by: Gennadiy Dubina --- scripts/install-db-server.sh | 189 +++++++++++++++++++++++++++++++++++ scripts/install-openmano.sh | 119 ++++------------------ 2 files changed, 207 insertions(+), 101 deletions(-) create mode 100755 scripts/install-db-server.sh diff --git a/scripts/install-db-server.sh b/scripts/install-db-server.sh new file mode 100755 index 00000000..ec0dd6cd --- /dev/null +++ b/scripts/install-db-server.sh @@ -0,0 +1,189 @@ +#!/bin/bash + +function usage(){ + echo -e "usage: sudo $0 [OPTIONS]" + echo -e "Install openmano database server" + echo -e "On a Ubuntu 16.04 it configures openmano as a service" + echo -e " OPTIONS" + echo -e " -u USER: database admin user. 'root' by default. Prompts if needed" + echo -e " -p PASS: database admin password to be used or installed. Prompts if needed" + echo -e " -q --quiet: install in unattended mode" + echo -e " -h --help: show this help" + echo -e " --forcedb: reinstall mano_db DB, deleting previous database if exists and creating a new one" + echo -e " --no-install-packages: use this option to skip updating and installing the requires packages. This avoid wasting time if you are sure requires packages are present e.g. because of a previous installation" +} + +function install_packages(){ + [ -x /usr/bin/apt-get ] && apt-get install -y $* + [ -x /usr/bin/yum ] && yum install -y $* + + #check properly installed + for PACKAGE in $* + do + PACKAGE_INSTALLED="no" + [ -x /usr/bin/apt-get ] && dpkg -l $PACKAGE &>> /dev/null && PACKAGE_INSTALLED="yes" + [ -x /usr/bin/yum ] && yum list installed $PACKAGE &>> /dev/null && PACKAGE_INSTALLED="yes" + if [ "$PACKAGE_INSTALLED" = "no" ] + then + echo "failed to install package '$PACKAGE'. Revise network connectivity and try again" >&2 + exit 1 + fi + done +} + +function db_exists() { + RESULT=`mysqlshow --defaults-extra-file="$2" | grep -v Wildcard | grep -o $1` + if [ "$RESULT" == "$1" ]; then + echo " DB $1 exists" + return 0 + fi + echo " DB $1 does not exist" + return 1 +} + + +DBUSER="root" +DBPASSWD="" +DBPASSWD_PARAM="" +QUIET_MODE="" +FORCEDB="" +NO_PACKAGES="" +while getopts ":u:p:hiq-:" o; do + case "${o}" in + u) + export DBUSER="$OPTARG" + ;; + p) + export DBPASSWD="$OPTARG" + export DBPASSWD_PARAM="-p$OPTARG" + ;; + q) + export QUIET_MODE=yes + export DEBIAN_FRONTEND=noninteractive + ;; + h) + usage && exit 0 + ;; + -) + [ "${OPTARG}" == "help" ] && usage && exit 0 + [ "${OPTARG}" == "forcedb" ] && FORCEDB="y" && continue + [ "${OPTARG}" == "quiet" ] && export QUIET_MODE=yes && export DEBIAN_FRONTEND=noninteractive && continue + [ "${OPTARG}" == "no-install-packages" ] && export NO_PACKAGES=yes && continue + echo -e "Invalid option: '--$OPTARG'\nTry $0 --help for more information" >&2 + exit 1 + ;; + \?) + echo -e "Invalid option: '-$OPTARG'\nTry $0 --help for more information" >&2 + exit 1 + ;; + :) + echo -e "Option '-$OPTARG' requires an argument\nTry $0 --help for more information" >&2 + exit 1 + ;; + *) + usage >&2 + exit 1 + ;; + esac +done + +HERE=$(realpath $(dirname $0)) +OPENMANO_BASEFOLDER=$(dirname $HERE) + +#Discover Linux distribution +#try redhat type +[ -f /etc/redhat-release ] && _DISTRO=$(cat /etc/redhat-release 2>/dev/null | cut -d" " -f1) +#if not assuming ubuntu type +[ -f /etc/redhat-release ] || _DISTRO=$(lsb_release -is 2>/dev/null) + +if [[ -z "$NO_PACKAGES" ]] +then + echo ' +################################################################# +##### INSTALL REQUIRED PACKAGES ##### +#################################################################' + [ "$_DISTRO" == "Ubuntu" ] && install_packages "mysql-server" + [ "$_DISTRO" == "CentOS" -o "$_DISTRO" == "Red" ] && install_packages "mariadb mariadb-server" + + if [[ "$_DISTRO" == "Ubuntu" ]] + then + #start services. By default CentOS does not start services + service mysql start >> /dev/null + # try to set admin password, ignore if fails + [[ -n $DBPASSWD ]] && mysqladmin -u $DBUSER -s password $DBPASSWD + fi + + if [ "$_DISTRO" == "CentOS" -o "$_DISTRO" == "Red" ] + then + #start services. By default CentOS does not start services + service mariadb start + service httpd start + systemctl enable mariadb + systemctl enable httpd + read -e -p "Do you want to configure mariadb (recommended if not done before) (Y/n)" KK + [ "$KK" != "n" -a "$KK" != "no" ] && mysql_secure_installation + + read -e -p "Do you want to set firewall to grant web access port 80,443 (Y/n)" KK + [ "$KK" != "n" -a "$KK" != "no" ] && + firewall-cmd --permanent --zone=public --add-service=http && + firewall-cmd --permanent --zone=public --add-service=https && + firewall-cmd --reload + fi +fi #[[ -z "$NO_PACKAGES" ]] + +#check and ask for database user password. Must be done after database installation +if [[ -n $QUIET_MODE ]] +then + echo -e "\nCheking database connection and ask for credentials" + while ! mysqladmin -s -u$DBUSER $DBPASSWD_PARAM status >/dev/null + do + [ -n "$logintry" ] && echo -e "\nInvalid database credentials!!!. Try again (Ctrl+c to abort)" + [ -z "$logintry" ] && echo -e "\nProvide database credentials" + read -e -p "database user? ($DBUSER) " DBUSER_ + [ -n "$DBUSER_" ] && DBUSER=$DBUSER_ + read -e -s -p "database password? (Enter for not using password) " DBPASSWD_ + [ -n "$DBPASSWD_" ] && DBPASSWD="$DBPASSWD_" && DBPASSWD_PARAM="-p$DBPASSWD_" + [ -z "$DBPASSWD_" ] && DBPASSWD="" && DBPASSWD_PARAM="" + logintry="yes" + done +fi + +echo ' +################################################################# +##### CREATE DATABASE ##### +#################################################################' +echo -e "\nCreating temporary file form MYSQL installation and initialization" +TEMPFILE="$(mktemp -q --tmpdir "installopenmano.XXXXXX")" +trap 'rm -f "$TEMPFILE"' EXIT +chmod 0600 "$TEMPFILE" +echo -e "[client]\n user='$DBUSER'\n password='$DBPASSWD'">"$TEMPFILE" + +if db_exists "mano_db" $TEMPFILE ; then + if [[ -n $FORCEDB ]]; then + echo " Deleting previous database mano_db" + DBDELETEPARAM="" + [[ -n $QUIET_MODE ]] && DBDELETEPARAM="-f" + mysqladmin --defaults-extra-file=$TEMPFILE -s drop mano_db $DBDELETEPARAM || ! echo "Could not delete mano_db database" || exit 1 + #echo "REVOKE ALL PRIVILEGES ON mano_db.* FROM 'mano'@'localhost';" | mysql --defaults-extra-file=$TEMPFILE -s || ! echo "Failed while creating user mano at database" || exit 1 + #echo "DELETE USER 'mano'@'localhost';" | mysql --defaults-extra-file=$TEMPFILE -s || ! echo "Failed while creating user mano at database" || exit 1 + mysqladmin --defaults-extra-file=$TEMPFILE -s create mano_db || ! echo "Error creating mano_db database" || exit 1 + echo "DROP USER 'mano'@'localhost';" | mysql --defaults-extra-file=$TEMPFILE -s || ! echo "Failed while creating user mano at database" || exit 1 + echo "CREATE USER 'mano'@'localhost' identified by 'manopw';" | mysql --defaults-extra-file=$TEMPFILE -s || ! echo "Failed while creating user mano at database" || exit 1 + echo "GRANT ALL PRIVILEGES ON mano_db.* TO 'mano'@'localhost';" | mysql --defaults-extra-file=$TEMPFILE -s || ! echo "Failed while creating user mano at database" || exit 1 + echo " Database 'mano_db' created, user 'mano' password 'manopw'" + else + echo "Database exists. Use option '--forcedb' to force the deletion of the existing one" && exit 1 + fi +else + mysqladmin -u$DBUSER $DBPASSWD_PARAM -s create mano_db || ! echo "Error creating mano_db database" || exit 1 + echo "CREATE USER 'mano'@'localhost' identified by 'manopw';" | mysql --defaults-extra-file=$TEMPFILE -s || ! echo "Failed while creating user mano at database" || exit 1 + echo "GRANT ALL PRIVILEGES ON mano_db.* TO 'mano'@'localhost';" | mysql --defaults-extra-file=$TEMPFILE -s || ! echo "Failed while creating user mano at database" || exit 1 + echo " Database 'mano_db' created, user 'mano' password 'manopw'" +fi + + +echo ' +################################################################# +##### INIT DATABASE ##### +#################################################################' +su $SUDO_USER -c "${OPENMANO_BASEFOLDER}/database_utils/init_mano_db.sh -u mano -p manopw -d mano_db" || ! echo "Failed while initializing database" || exit 1 diff --git a/scripts/install-openmano.sh b/scripts/install-openmano.sh index db678d14..36f59d46 100755 --- a/scripts/install-openmano.sh +++ b/scripts/install-openmano.sh @@ -39,6 +39,7 @@ function usage(){ echo -e " --force: makes idenpotent, delete previous installations folders if needed" echo -e " --noclone: assumes that openmano was cloned previously and that this script is run from the local repo" echo -e " --no-install-packages: use this option to skip updating and installing the requires packages. This avoid wasting time if you are sure requires packages are present e.g. because of a previous installation" + echo -e " --no-db: do not insall mysql server" } function install_packages(){ @@ -59,16 +60,6 @@ function install_packages(){ done } -function db_exists() { - RESULT=`mysqlshow --defaults-extra-file="$2" | grep -v Wildcard | grep -o $1` - if [ "$RESULT" == "$1" ]; then - echo " DB $1 exists" - return 0 - fi - echo " DB $1 does not exist" - return 1 -} - GIT_URL=https://osm.etsi.org/gerrit/osm/RO.git DBUSER="root" DBPASSWD="" @@ -79,6 +70,7 @@ FORCEDB="" FORCE="" NOCLONE="" NO_PACKAGES="" +NO_DB="" while getopts ":u:p:hiq-:" o; do case "${o}" in u) @@ -103,6 +95,7 @@ while getopts ":u:p:hiq-:" o; do [ "${OPTARG}" == "noclone" ] && NOCLONE="y" && continue [ "${OPTARG}" == "quiet" ] && export QUIET_MODE=yes && export DEBIAN_FRONTEND=noninteractive && continue [ "${OPTARG}" == "no-install-packages" ] && export NO_PACKAGES=yes && continue + [ "${OPTARG}" == "no-db" ] && NO_DB="y" && continue echo -e "Invalid option: '--$OPTARG'\nTry $0 --help for more information" >&2 exit 1 ;; @@ -207,54 +200,9 @@ echo ' ################################################################# ##### INSTALL REQUIRED PACKAGES ##### #################################################################' -[ "$_DISTRO" == "Ubuntu" ] && install_packages "git screen wget mysql-server" -[ "$_DISTRO" == "CentOS" -o "$_DISTRO" == "Red" ] && install_packages "git screen wget mariadb mariadb-server" - -if [[ "$_DISTRO" == "Ubuntu" ]] -then - #start services. By default CentOS does not start services - service mysql start >> /dev/null - # try to set admin password, ignore if fails - [[ -n $DBPASSWD ]] && mysqladmin -u $DBUSER -s password $DBPASSWD -fi - -if [ "$_DISTRO" == "CentOS" -o "$_DISTRO" == "Red" ] -then - #start services. By default CentOS does not start services - service mariadb start - service httpd start - systemctl enable mariadb - systemctl enable httpd - read -e -p "Do you want to configure mariadb (recommended if not done before) (Y/n)" KK - [ "$KK" != "n" -a "$KK" != "no" ] && mysql_secure_installation - - read -e -p "Do you want to set firewall to grant web access port 80,443 (Y/n)" KK - [ "$KK" != "n" -a "$KK" != "no" ] && - firewall-cmd --permanent --zone=public --add-service=http && - firewall-cmd --permanent --zone=public --add-service=https && - firewall-cmd --reload -fi -fi #[[ -z "$NO_PACKAGES" ]] - -#check and ask for database user password. Must be done after database installation -if [[ -n $QUIET_MODE ]] -then - echo -e "\nCheking database connection and ask for credentials" - while ! mysqladmin -s -u$DBUSER $DBPASSWD_PARAM status >/dev/null - do - [ -n "$logintry" ] && echo -e "\nInvalid database credentials!!!. Try again (Ctrl+c to abort)" - [ -z "$logintry" ] && echo -e "\nProvide database credentials" - read -e -p "database user? ($DBUSER) " DBUSER_ - [ -n "$DBUSER_" ] && DBUSER=$DBUSER_ - read -e -s -p "database password? (Enter for not using password) " DBPASSWD_ - [ -n "$DBPASSWD_" ] && DBPASSWD="$DBPASSWD_" && DBPASSWD_PARAM="-p$DBPASSWD_" - [ -z "$DBPASSWD_" ] && DBPASSWD="" && DBPASSWD_PARAM="" - logintry="yes" - done -fi +[ "$_DISTRO" == "Ubuntu" ] && install_packages "git screen wget mysql-client" +[ "$_DISTRO" == "CentOS" -o "$_DISTRO" == "Red" ] && install_packages "git screen wget mariadb-client" -if [[ -z "$NO_PACKAGES" ]] -then echo ' ################################################################# ##### INSTALL PYTHON PACKAGES ##### @@ -290,47 +238,6 @@ if [[ -z $NOCLONE ]]; then [[ -z $DEVELOP ]] && su $SUDO_USER -c "git -C ${OPENMANO_BASEFOLDER} checkout tags/v1.0.2" fi -echo ' -################################################################# -##### CREATE DATABASE ##### -#################################################################' -echo -e "\nCreating temporary file form MYSQL installation and initialization" -TEMPFILE="$(mktemp -q --tmpdir "installopenmano.XXXXXX")" -trap 'rm -f "$TEMPFILE"' EXIT -chmod 0600 "$TEMPFILE" -echo -e "[client]\n user='$DBUSER'\n password='$DBPASSWD'">"$TEMPFILE" - -if db_exists "mano_db" $TEMPFILE ; then - if [[ -n $FORCEDB ]]; then - echo " Deleting previous database mano_db" - DBDELETEPARAM="" - [[ -n $QUIET_MODE ]] && DBDELETEPARAM="-f" - mysqladmin --defaults-extra-file=$TEMPFILE -s drop mano_db $DBDELETEPARAM || ! echo "Could not delete mano_db database" || exit 1 - #echo "REVOKE ALL PRIVILEGES ON mano_db.* FROM 'mano'@'localhost';" | mysql --defaults-extra-file=$TEMPFILE -s || ! echo "Failed while creating user mano at database" || exit 1 - #echo "DELETE USER 'mano'@'localhost';" | mysql --defaults-extra-file=$TEMPFILE -s || ! echo "Failed while creating user mano at database" || exit 1 - mysqladmin --defaults-extra-file=$TEMPFILE -s create mano_db || ! echo "Error creating mano_db database" || exit 1 - echo "DROP USER 'mano'@'localhost';" | mysql --defaults-extra-file=$TEMPFILE -s || ! echo "Failed while creating user mano at database" || exit 1 - echo "CREATE USER 'mano'@'localhost' identified by 'manopw';" | mysql --defaults-extra-file=$TEMPFILE -s || ! echo "Failed while creating user mano at database" || exit 1 - echo "GRANT ALL PRIVILEGES ON mano_db.* TO 'mano'@'localhost';" | mysql --defaults-extra-file=$TEMPFILE -s || ! echo "Failed while creating user mano at database" || exit 1 - echo " Database 'mano_db' created, user 'mano' password 'manopw'" - else - echo "Database exists. Use option '--forcedb' to force the deletion of the existing one" && exit 1 - fi -else - mysqladmin -u$DBUSER $DBPASSWD_PARAM -s create mano_db || ! echo "Error creating mano_db database" || exit 1 - echo "CREATE USER 'mano'@'localhost' identified by 'manopw';" | mysql --defaults-extra-file=$TEMPFILE -s || ! echo "Failed while creating user mano at database" || exit 1 - echo "GRANT ALL PRIVILEGES ON mano_db.* TO 'mano'@'localhost';" | mysql --defaults-extra-file=$TEMPFILE -s || ! echo "Failed while creating user mano at database" || exit 1 - echo " Database 'mano_db' created, user 'mano' password 'manopw'" -fi - - -echo ' -################################################################# -##### INIT DATABASE ##### -#################################################################' -su $SUDO_USER -c "${OPENMANO_BASEFOLDER}/database_utils/init_mano_db.sh -u mano -p manopw -d mano_db" || ! echo "Failed while initializing database" || exit 1 - - if [ "$_DISTRO" == "CentOS" -o "$_DISTRO" == "Red" ] then echo ' @@ -404,8 +311,20 @@ then su $SUDO_USER -c 'echo ". ${HOME}/.bash_completion.d/python-argcomplete.sh" >> ~/.bashrc' fi +if [ -z "$NO_DB" ]; then +echo ' +################################################################# +##### INSTALL DATABASE SERVER ##### +#################################################################' - + if [ -n "$QUIET_MODE" ]; then + DB_QUIET='-q' + fi + if [ -n "$FORCEDB" ]; then + DB_FORCE='--forcedb' + fi + ${OPENMANO_BASEFOLDER}/scripts/install-db-server.sh -u $DBUSER $DBPASSWD_PARAM $DB_QUIET $DB_FORCE || exit 1 +fi if [[ -n "$INSTALL_AS_A_SERVICE" ]] then @@ -432,5 +351,3 @@ else fi - - -- 2.25.1 From cf1826b8466193d53981fe6c61d78e9746b816f1 Mon Sep 17 00:00:00 2001 From: garciadeblas Date: Tue, 18 Apr 2017 10:03:17 +0200 Subject: [PATCH 11/16] Updated Makefile with install and develop; renamed openmanod.py to remove extension Change-Id: Ie9f62cd763fa8c74f1222e3533d15b6167a33c4a Signed-off-by: garciadeblas --- MANIFEST.in | 2 +- Makefile | 20 +++++++++++++------- openmanod.py => openmanod | 0 osm-ro.service | 2 +- osm_ro/__init__.py | 0 setup.py | 2 +- 6 files changed, 16 insertions(+), 10 deletions(-) rename openmanod.py => openmanod (100%) create mode 100644 osm_ro/__init__.py diff --git a/MANIFEST.in b/MANIFEST.in index 154ea275..f586ebf3 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -2,7 +2,7 @@ #include requirements.txt include README.rst include openmano -include openmanod.py +include openmanod include openmanod.cfg include osm-ro.service recursive-include osm_ro * diff --git a/Makefile b/Makefile index 76a29908..2c7255b3 100644 --- a/Makefile +++ b/Makefile @@ -1,5 +1,5 @@ SHELL := /bin/bash -all: pypackage debpackage +all: pip deb prepare: mkdir -p build/ @@ -9,7 +9,7 @@ prepare: cp setup.py build/ cp -r osm_ro build/ cp openmano build/ - cp openmanod.py build/ + cp openmanod build/ cp openmanod.cfg build/ cp osm-ro.service build/ cp -r vnfs build/osm_ro @@ -28,18 +28,24 @@ connectors: prepare build: connectors prepare python -m py_compile build/osm_ro/*.py -pypackage: prepare - cd build; ./setup.py sdist - cd build; ./setup.py bdist_wheel +pip: prepare + cd build && ./setup.py sdist + cd build && ./setup.py bdist_wheel -debpackage: prepare +deb: prepare echo "Nothing to be done" #cd build; ./setup.py --command-packages=stdeb.command bdist_deb #fpm -s python -t deb build/setup.py -snappackage: +snap: echo "Nothing to be done yet" +install: + cd build && pip install dist/*.tar.gz + +develop: prepare + cd build && ./setup.py develop + sync: #cp build/dist/* /root/artifacts/... diff --git a/openmanod.py b/openmanod similarity index 100% rename from openmanod.py rename to openmanod diff --git a/osm-ro.service b/osm-ro.service index 155150b7..cc8966c8 100644 --- a/osm-ro.service +++ b/osm-ro.service @@ -3,7 +3,7 @@ Description=openmano server (OSM RO) [Service] User=${USER_OWNER} -ExecStart=openmanod.py -c /etc/osm/openmanod.cfg --log-file=/var/log/osm/openmano.log +ExecStart=openmanod -c /etc/osm/openmanod.cfg --log-file=/var/log/osm/openmano.log Restart=always [Install] diff --git a/osm_ro/__init__.py b/osm_ro/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/setup.py b/setup.py index 68781be1..20b6c3a4 100755 --- a/setup.py +++ b/setup.py @@ -61,7 +61,7 @@ setup(name=_name, data_files = [('/etc/osm/', ['openmanod.cfg']), ('/etc/systemd/system/', ['osm-ro.service']), ], - scripts=['openmanod.py', 'openmano', 'osm_ro/scripts/service-openmano.sh', 'osm_ro/scripts/openmano-report.sh',], + scripts=['openmanod', 'openmano', 'osm_ro/scripts/service-openmano.sh', 'osm_ro/scripts/openmano-report.sh',], install_requires=_requirements, include_package_data=True, cmdclass = {'install': ROInstaller}, -- 2.25.1 From 5b1a664adc703299d10af778802e8ecd12e0cf76 Mon Sep 17 00:00:00 2001 From: garciadeblas Date: Tue, 18 Apr 2017 10:10:31 +0200 Subject: [PATCH 12/16] Fix typo in usage of install-openmano.sh Change-Id: I7f6566f6b73254102d7a3bd79716c7b3d4d8dcc3 Signed-off-by: garciadeblas --- scripts/install-openmano.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/install-openmano.sh b/scripts/install-openmano.sh index 36f59d46..a1981480 100755 --- a/scripts/install-openmano.sh +++ b/scripts/install-openmano.sh @@ -39,7 +39,7 @@ function usage(){ echo -e " --force: makes idenpotent, delete previous installations folders if needed" echo -e " --noclone: assumes that openmano was cloned previously and that this script is run from the local repo" echo -e " --no-install-packages: use this option to skip updating and installing the requires packages. This avoid wasting time if you are sure requires packages are present e.g. because of a previous installation" - echo -e " --no-db: do not insall mysql server" + echo -e " --no-db: do not install mysql server" } function install_packages(){ -- 2.25.1 From 311ea74670ffe316a685d0fd0dcd7102046e9e80 Mon Sep 17 00:00:00 2001 From: Gennadiy Dubina Date: Mon, 3 Apr 2017 20:46:16 +0300 Subject: [PATCH 13/16] add docker build Signed-off-by: Gennadiy Dubina --- Makefile | 11 ++++++++ docker/Dockerfile-local | 50 +++++++++++++++++++++++++++++++++++++ docker/openmano-compose.yml | 31 +++++++++++++++++++++++ docker/scripts/configure.sh | 12 +++++++++ docker/scripts/start.sh | 46 ++++++++++++++++++++++++++++++++++ docker/scripts/wait_db.sh | 14 +++++++++++ requirements.txt | 2 -- 7 files changed, 164 insertions(+), 2 deletions(-) create mode 100644 docker/Dockerfile-local create mode 100644 docker/openmano-compose.yml create mode 100755 docker/scripts/configure.sh create mode 100755 docker/scripts/start.sh create mode 100755 docker/scripts/wait_db.sh diff --git a/Makefile b/Makefile index 2c7255b3..226bcb41 100644 --- a/Makefile +++ b/Makefile @@ -2,6 +2,8 @@ SHELL := /bin/bash all: pip deb prepare: + pip install setuptools + #pip install -r requirements.txt mkdir -p build/ cp MANIFEST.in build/ cp requirements.txt build/ @@ -52,6 +54,15 @@ sync: test: ./test/basictest.sh --force --insert-bashrc --install-openvim --init-openvim +build-docker-from-source: + docker build -t osm/openmano -f docker/Dockerfile-local . + +run-docker: + docker-compose -f docker/openmano-compose.yml up + +stop-docker: + docker-compose -f docker/openmano-compose.yml down + clean: rm -rf build #find build -name '*.pyc' -delete diff --git a/docker/Dockerfile-local b/docker/Dockerfile-local new file mode 100644 index 00000000..3eaa281f --- /dev/null +++ b/docker/Dockerfile-local @@ -0,0 +1,50 @@ +from ubuntu:xenial + +MAINTAINER Gennadiy Dubina + +# script uses it +ENV USER=root + +#avoid extra information from packages +RUN echo 'path-exclude /usr/share/doc/*\n\ +path-include /usr/share/doc/*/copyright\n\ +path-exclude /usr/share/man/*\n\ +path-exclude /usr/share/groff/*\n\ +path-exclude /usr/share/info/*\n\ +path-exclude /usr/share/lintian/*\n\ +path-exclude /usr/share/linda/*\n'\ +> /etc/dpkg/dpkg.cfg.d/01_nodoc && \ + echo 'APT::Install-Recommends "false";\n\ +APT::AutoRemove::RecommendsImportant "false";\n\ +APT::AutoRemove::SuggestsImportant "false";\n'\ +> /etc/apt/apt.conf.d/99_norecommends + +#generate lsb_release stub +RUN echo 'if [ "$1" == "-is" ]; then echo "Ubuntu"; else if [ "$1" == "-rs" ]; then echo "16.04"; fi fi' > /usr/bin/lsb_release && chmod +x /usr/bin/lsb_release + +COPY . /usr/local/src/openmano + +RUN apt update && \ + DEBIAN_FRONTEND=noninteractive apt install -fqy software-properties-common make gcc git python python-dev python-pip sudo wget && \ + pip install --upgrade pip && \ + cd /usr/local/src/openmano && \ + make build && \ + cd build && \ + ./scripts/install-openmano.sh -q --noclone --no-db && \ + cd / && \ + rm -rf /usr/local/src/openmano && \ + rm -rf /usr/include/* && \ + rm -rf /root/.cache && \ + apt purge -y make gcc git curl wget python-dev python-pip && \ + apt autoremove -y && \ + apt clean && \ + rm -rf /var/lib/apt/lists/* + +COPY docker/scripts/ /opt/openmano-docker + +VOLUME /opt/openmano/logs +EXPOSE 9090 + +env DB_USER='' DB_PSWD='' DB_HOST='' DB_PORT=3306 DB_NAME=mano_db + +CMD /opt/openmano-docker/start.sh diff --git a/docker/openmano-compose.yml b/docker/openmano-compose.yml new file mode 100644 index 00000000..253e9574 --- /dev/null +++ b/docker/openmano-compose.yml @@ -0,0 +1,31 @@ +version: '2' +services: + osm-ro-db: + image: mysql + container_name: osm-ro-db + restart: always + environment: + - MYSQL_RANDOM_ROOT_PASSWORD=true + - MYSQL_DATABASE=mano_db + - MYSQL_USER=mano + - MYSQL_PASSWORD=manopw + osm-ro: + build: + context: ../ + dockerfile: docker/Dockerfile-local + image: osm/openmano + container_name: osm-ro + restart: always + environment: + - DB_USER=mano + - DB_PSWD=manopw + - DB_NAME=mano_db + - DB_HOST=osm-ro-db + ports: + - "9090:9090" + volumes: + - /var/log/osm/openmano/logs:/opt/openmano/logs + depends_on: + - osm-ro-db + links: + - osm-ro-db \ No newline at end of file diff --git a/docker/scripts/configure.sh b/docker/scripts/configure.sh new file mode 100755 index 00000000..da7cbba1 --- /dev/null +++ b/docker/scripts/configure.sh @@ -0,0 +1,12 @@ +#!/bin/bash + +#Database parameters +#db_host: localhost +#db_user: mano +#db_passwd: manopw +#db_name: mano_db + +sed -i "s/^db_host:.*/db_host: $DB_HOST/" /opt/openmano/openmanod.cfg +sed -i "s/^db_user:.*/db_user: $DB_USER/" /opt/openmano/openmanod.cfg +sed -i "s/^db_passwd:.*/db_passwd: $DB_PSWD/" /opt/openmano/openmanod.cfg +sed -i "s/^db_name:.*/db_name: $DB_NAME/" /opt/openmano/openmanod.cfg \ No newline at end of file diff --git a/docker/scripts/start.sh b/docker/scripts/start.sh new file mode 100755 index 00000000..3d70c3f3 --- /dev/null +++ b/docker/scripts/start.sh @@ -0,0 +1,46 @@ +#!/bin/bash + +function is_db_created() { + db_host=$1 + db_port=$2 + db_user=$3 + db_pswd=$4 + db_name=$5 + + RESULT=`mysqlshow -h"$db_host" -P"$db_port" -u"$db_user" -p"$db_pswd" | grep -v Wildcard | grep -o $db_name` + if [ "$RESULT" == "$db_name" ]; then + + RESULT=`mysqlshow -h"$db_host" -P"$db_port" -u"$db_user" -p"$db_pswd" "$db_name" | grep -v Wildcard | grep schema_version` + #TODO validate version + if [ -n "$RESULT" ]; then + echo " DB $db_name exists and inited" + return 0 + else + echo " DB $db_name exists BUT not inited" + return 1 + fi + fi + echo " DB $db_name does not exist" + return 1 +} + +echo "1/4 Apply config" +/opt/openmano-docker/configure.sh +[ $? -ne 0 ] && exit 1 +echo "2/4 Wait for db" + +/opt/openmano-docker/wait_db.sh +[ $? -ne 0 ] && exit 1 + +echo "3/4 Init database" && \ +is_db_created "$DB_HOST" "$DB_PORT" "$DB_USER" "$DB_PSWD" "$DB_NAME" +if [ $? -ne 0 ]; then + /opt/openmano/database_utils/init_mano_db.sh -u $DB_USER -p $DB_PSWD -h $DB_HOST -P $DB_PORT -d $DB_NAME + [ $? -ne 0 ] && exit 1 +else + echo " migrage database version" + /opt/openmano/database_utils/migrate_mano_db.sh -u $DB_USER -p $DB_PSWD -h $DB_HOST -P $DB_PORT -d $DB_NAME +fi + +echo "4/4 Try to start" +/opt/openmano/openmanod.py -c /opt/openmano/openmanod.cfg --log-file=/opt/openmano/logs/openmano.log \ No newline at end of file diff --git a/docker/scripts/wait_db.sh b/docker/scripts/wait_db.sh new file mode 100755 index 00000000..f083d76b --- /dev/null +++ b/docker/scripts/wait_db.sh @@ -0,0 +1,14 @@ +#!/bin/bash + +attempt=0 +max_attempts=120 +while ! mysqladmin ping -h"$DB_HOST" -P"$DB_PORT" --silent; do + #wait 120 sec + if [ $attempt -ge $max_attempts ]; then + echo "Can not connect to database during $max_attempts sec" + exit 1 + fi + attempt=$[$attempt+1] + echo "Wait for MySQL Server ${DB_HOST}:${DB_PORT} [attempt $attempt/$max_attempts]" + sleep 1 +done \ No newline at end of file diff --git a/requirements.txt b/requirements.txt index 38c055cb..eb924781 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,5 +1,3 @@ ---index-url https://pypi.python.org/simple/ - PyYAML bottle MySQL-python -- 2.25.1 From d81496cabe6c0dfeadbb4dc1852a3d336032c576 Mon Sep 17 00:00:00 2001 From: garciadeblas Date: Wed, 19 Apr 2017 18:17:11 +0200 Subject: [PATCH 14/16] install_db_server does not drop the user if exists Change-Id: Ia4f3dfa70ba8d1e273fdfda837a5d16a0aacb414 Signed-off-by: garciadeblas --- scripts/install-db-server.sh | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/scripts/install-db-server.sh b/scripts/install-db-server.sh index ec0dd6cd..26738a67 100755 --- a/scripts/install-db-server.sh +++ b/scripts/install-db-server.sh @@ -167,8 +167,7 @@ if db_exists "mano_db" $TEMPFILE ; then #echo "REVOKE ALL PRIVILEGES ON mano_db.* FROM 'mano'@'localhost';" | mysql --defaults-extra-file=$TEMPFILE -s || ! echo "Failed while creating user mano at database" || exit 1 #echo "DELETE USER 'mano'@'localhost';" | mysql --defaults-extra-file=$TEMPFILE -s || ! echo "Failed while creating user mano at database" || exit 1 mysqladmin --defaults-extra-file=$TEMPFILE -s create mano_db || ! echo "Error creating mano_db database" || exit 1 - echo "DROP USER 'mano'@'localhost';" | mysql --defaults-extra-file=$TEMPFILE -s || ! echo "Failed while creating user mano at database" || exit 1 - echo "CREATE USER 'mano'@'localhost' identified by 'manopw';" | mysql --defaults-extra-file=$TEMPFILE -s || ! echo "Failed while creating user mano at database" || exit 1 + echo "CREATE USER 'mano'@'localhost' identified by 'manopw';" | mysql --defaults-extra-file=$TEMPFILE -s || ! echo "Failed while creating user mano at database" echo "GRANT ALL PRIVILEGES ON mano_db.* TO 'mano'@'localhost';" | mysql --defaults-extra-file=$TEMPFILE -s || ! echo "Failed while creating user mano at database" || exit 1 echo " Database 'mano_db' created, user 'mano' password 'manopw'" else -- 2.25.1 From e5ef517abec7db86ce2ee7cba628548ce4b5a6fd Mon Sep 17 00:00:00 2001 From: garciadeblas Date: Thu, 20 Apr 2017 14:46:29 +0200 Subject: [PATCH 15/16] Support of debian packaging via stdeb; postinst script; renaming some files Change-Id: I966a27d84688ee2e7083324033ebea3ade6fc871 Signed-off-by: garciadeblas --- MANIFEST.in | 3 +- Makefile | 26 +++++----- openmanod.cfg => osm_ro/openmanod.cfg | 0 osm-ro.service => osm_ro/osm-ro.service | 0 .../{openmano-report.sh => openmano-report} | 0 scripts/python-osm-ro.postinst | 49 +++++++++++++++++++ .../{service-openmano.sh => service-openmano} | 0 setup.py | 20 +++----- stdeb.cfg | 6 +++ 9 files changed, 76 insertions(+), 28 deletions(-) rename openmanod.cfg => osm_ro/openmanod.cfg (100%) rename osm-ro.service => osm_ro/osm-ro.service (100%) rename scripts/{openmano-report.sh => openmano-report} (100%) create mode 100755 scripts/python-osm-ro.postinst rename scripts/{service-openmano.sh => service-openmano} (100%) create mode 100644 stdeb.cfg diff --git a/MANIFEST.in b/MANIFEST.in index f586ebf3..48790d46 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -1,9 +1,8 @@ #include MANIFEST.in #include requirements.txt include README.rst +include RO_VERSION include openmano include openmanod -include openmanod.cfg -include osm-ro.service recursive-include osm_ro * diff --git a/Makefile b/Makefile index 226bcb41..5bde339b 100644 --- a/Makefile +++ b/Makefile @@ -1,19 +1,19 @@ SHELL := /bin/bash -all: pip deb +all: package install prepare: pip install setuptools - #pip install -r requirements.txt mkdir -p build/ + #git describe | sed -e 's/^v//' > build/RO_VERSION + echo "1.1.5" > build/RO_VERSION cp MANIFEST.in build/ cp requirements.txt build/ cp README.rst build/ cp setup.py build/ + cp stdeb.cfg build/ cp -r osm_ro build/ cp openmano build/ cp openmanod build/ - cp openmanod.cfg build/ - cp osm-ro.service build/ cp -r vnfs build/osm_ro cp -r scenarios build/osm_ro cp -r instance-scenarios build/osm_ro @@ -32,25 +32,25 @@ build: connectors prepare pip: prepare cd build && ./setup.py sdist - cd build && ./setup.py bdist_wheel -deb: prepare - echo "Nothing to be done" - #cd build; ./setup.py --command-packages=stdeb.command bdist_deb - #fpm -s python -t deb build/setup.py +package: prepare + cd build && python setup.py --command-packages=stdeb.command sdist_dsc --with-python2=True + cd build && cp osm_ro/scripts/python-osm-ro.postinst deb_dist/osm-ro*/debian/ + cd build/deb_dist/osm-ro* && dpkg-buildpackage -rfakeroot -uc -us snap: echo "Nothing to be done yet" install: - cd build && pip install dist/*.tar.gz + DEBIAN_FRONTEND=noninteractive apt-get update && \ + DEBIAN_FRONTEND=noninteractive apt-get install -y python-pip && \ + pip install --upgrade pip && \ + dpkg -i build/deb_dist/*.deb develop: prepare + #pip install -r requirements.txt cd build && ./setup.py develop -sync: - #cp build/dist/* /root/artifacts/... - test: ./test/basictest.sh --force --insert-bashrc --install-openvim --init-openvim diff --git a/openmanod.cfg b/osm_ro/openmanod.cfg similarity index 100% rename from openmanod.cfg rename to osm_ro/openmanod.cfg diff --git a/osm-ro.service b/osm_ro/osm-ro.service similarity index 100% rename from osm-ro.service rename to osm_ro/osm-ro.service diff --git a/scripts/openmano-report.sh b/scripts/openmano-report similarity index 100% rename from scripts/openmano-report.sh rename to scripts/openmano-report diff --git a/scripts/python-osm-ro.postinst b/scripts/python-osm-ro.postinst new file mode 100755 index 00000000..00f24002 --- /dev/null +++ b/scripts/python-osm-ro.postinst @@ -0,0 +1,49 @@ +#!/bin/bash + +## +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +# +# For those usages not covered by the Apache License, Version 2.0 please +# contact with: OSM_TECH@list.etsi.org +## + +echo "POST INSTALL OSM-RO" +OSMRO_PATH=`python -c 'import osm_ro; print osm_ro.__path__[0]'` +#OSMLIBOVIM_PATH=`python -c 'import lib_osm_openvim; print lib_osm_openvim.__path__[0]'` + +#Pip packages required for vmware connector +pip install --upgrade pip +pip install pyvcloud +pip install progressbar +pip install prettytable +pip install pyvmomi + +systemctl enable osm-ro.service + +#Creation of log folder +mkdir -p /var/log/osm + +#configure arg-autocomplete for this user +su $SUDO_USER -c 'activate-global-python-argcomplete --user' +if ! su $SUDO_USER -c 'grep -q bash_completion.d/python-argcomplete.sh ${HOME}/.bashrc' +then + echo " inserting .bash_completion.d/python-argcomplete.sh execution at .bashrc" + su $SUDO_USER -c 'echo ". ${HOME}/.bash_completion.d/python-argcomplete.sh" >> ~/.bashrc' +fi + +echo ' +To make OSM RO work, you have to install mysql and a database, and finally start openmano service' +echo ' ${OSMRO_PATH}/scripts/install-db-server.sh -u USER -p ' +echo ' service openmano start' + + diff --git a/scripts/service-openmano.sh b/scripts/service-openmano similarity index 100% rename from scripts/service-openmano.sh rename to scripts/service-openmano diff --git a/setup.py b/setup.py index 20b6c3a4..fb389700 100755 --- a/setup.py +++ b/setup.py @@ -1,12 +1,13 @@ #!/usr/bin/env python +#from distutils.core import setup +#from distutils.command.install_data import install_data from setuptools import setup -from setuptools.command.install import install from os import system #import glob _name = 'osm_ro' -_version = '1.0.0' +_version = open('RO_VERSION').read() _description = 'OSM Resource Orchestrator' _author = 'ETSI OSM' _author_email = 'alfonso.tiernosepulveda@telefonica.com' @@ -35,12 +36,6 @@ _requirements = [ "boto", ] -class ROInstaller(install): - def run(self): - cmd = 'echo "Running install script"' - system(cmd) - install.run(self) - setup(name=_name, version = _version, description = _description, @@ -57,13 +52,12 @@ setup(name=_name, package_data = {_name: ['vnfs/*.yaml', 'vnfs/examples/*.yaml', 'scenarios/*.yaml', 'scenarios/examples/*.yaml', 'instance-scenarios/examples/*.yaml', 'database_utils/*', - 'scripts/install-openmano*.sh']}, - data_files = [('/etc/osm/', ['openmanod.cfg']), - ('/etc/systemd/system/', ['osm-ro.service']), + 'scripts/*']}, + data_files = [('/etc/osm/', ['osm_ro/openmanod.cfg']), + ('/etc/systemd/system/', ['osm_ro/osm-ro.service']), ], - scripts=['openmanod', 'openmano', 'osm_ro/scripts/service-openmano.sh', 'osm_ro/scripts/openmano-report.sh',], + scripts=['openmanod', 'openmano', 'osm_ro/scripts/service-openmano', 'osm_ro/scripts/openmano-report',], install_requires=_requirements, include_package_data=True, - cmdclass = {'install': ROInstaller}, ) diff --git a/stdeb.cfg b/stdeb.cfg new file mode 100644 index 00000000..09787370 --- /dev/null +++ b/stdeb.cfg @@ -0,0 +1,6 @@ +[DEFAULT] +Suite: xenial +XS-Python-Version: >= 2.7 +Maintainer: Gerardo Garcia +Depends: python-pip, libmysqlclient-dev, libssl-dev, libffi-dev, python-argcomplete, python-boto, python-bottle, python-jsonschema, python-logutils, python-cinderclient, python-glanceclient, python-keystoneclient, python-neutronclient, python-novaclient, python-mysqldb, mysql-server + -- 2.25.1 From 3f873ad273104019b98b85483aeb2afdbfd263dd Mon Sep 17 00:00:00 2001 From: garciadeblas Date: Thu, 20 Apr 2017 14:47:30 +0200 Subject: [PATCH 16/16] Minor typo fixed in install-db-server.sh Change-Id: I918f13d2e96287bbd275308162f7da035dfbc5d9 Signed-off-by: garciadeblas --- scripts/install-db-server.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/install-db-server.sh b/scripts/install-db-server.sh index 26738a67..9ada1afa 100755 --- a/scripts/install-db-server.sh +++ b/scripts/install-db-server.sh @@ -152,7 +152,7 @@ echo ' ################################################################# ##### CREATE DATABASE ##### #################################################################' -echo -e "\nCreating temporary file form MYSQL installation and initialization" +echo -e "\nCreating temporary file for MYSQL installation and initialization" TEMPFILE="$(mktemp -q --tmpdir "installopenmano.XXXXXX")" trap 'rm -f "$TEMPFILE"' EXIT chmod 0600 "$TEMPFILE" -- 2.25.1