From 0446cd5df24c38f95cea13b995c553e9b2403f21 Mon Sep 17 00:00:00 2001 From: Anderson Bravalheri Date: Fri, 17 Aug 2018 15:26:19 +0100 Subject: [PATCH] Implement feature 5949 Enable dynamic connectivity setup in multi-site Network Services The code required to implement the feature is contained in `osm_ro/wim` as much as possible. * `wim/engine.py` works together with `nfvo.py` to implement the feature * `wim/persistence.py` is equivalent to `nfvo_db.py` and try to encapsulate most of the SQL-specific code, implementing a persistence layer * `wim/http_handler.py` extends `httpserver.py` adding WIM-related HTTP routes * `wim/wim_thread.py` is similar to `vim_thread.py` and controls the execution of WIM-related tasks * `wim/actions.py` and `wim/wan_link_actions.py` implement the action handling specific code, calling instances of the `wim/wimconn.py` subclasses WIM connectors are still a work in progress Individual change details (newer to older) - Add errors for inconsistent state - Delay re-scheduled tasks - Move lock to inside the persistence object - Better errors for connector failures - Try to cache the wan_link information before it is deleted from the database - Integrate WanLinkDelete to NFVO - Add WanLinkDelete implementation draft with some tests - Add basic wim network creation - Add minimal documentation for actions - Add checks to the create action - Improve documentation, rearrange insert_pending and remove unused functions on WimThread - Integrate Action classes in refresh_tasks - Add Action classes to avoid intricate conditions - Adding Proposed License - Move grouping of actions to persistence - Change WimThread to use SQL to do the heavy lifting - Simplify WimThread reload_actions - Add tests for derive_wan_links - Implement find_common_wim(s) - Add tests for create_wim_account - Add migration scripts for version 33 - Changes to WIM and VIM threads for vim_wim_actions - Implement wim_account management according to the discussion - Add WimHandler integration inside httpserver - Add quick instructions to run the tests - Add WIM functional tests using real database - Add DB WIM port mapping - RO WIM-related console scripts - Add WIM integration to NFVO - Improve database support focusing on tests - RO NBI WIM-related commands in HTTP server - Adding WIM tables to MANO DB - Add wim http handler initial implementation - Move http utility functions to separated files This separation allows the code to be reused more easily and avoids circular dependencies. (The httpserver can import other modules implementing http routes, and those modules can then use the utility functions without having to import back httpserver) - Add a HTTP handler class and custom route decorator These tools can be used to create independent groups of bottle routes/callbacks in a OOP fashion - Extract http error codes and related logic to separated file Change-Id: Icd5fc9fa345852b8cf571e48f427dc10bdbd24c5 Signed-off-by: Anderson Bravalheri --- .gitignore-common | 3 + database_utils/migrate_mano_db.sh | 53 +- .../migrations/down/34_remove_wim_tables.sql | 27 + .../migrations/up/34_add_wim_tables.sql | 165 +++ docker/Dockerfile-local | 2 +- docker/tests.dockerfile | 13 + docker/tests.yml | 37 + openmano | 518 ++++++++- openmanod | 25 +- osm_ro/db_base.py | 143 +-- osm_ro/http_tools/__init__.py | 0 osm_ro/http_tools/errors.py | 67 ++ osm_ro/http_tools/handler.py | 114 ++ osm_ro/http_tools/request_processing.py | 209 ++++ osm_ro/http_tools/tests/__init__.py | 0 osm_ro/http_tools/tests/test_errors.py | 76 ++ osm_ro/http_tools/tests/test_handler.py | 95 ++ osm_ro/http_tools/tox.ini | 49 + osm_ro/httpserver.py | 482 +++----- osm_ro/nfvo.py | 437 ++++--- osm_ro/nfvo_db.py | 202 ++-- osm_ro/openmano_schemas.py | 34 +- osm_ro/openmanoclient.py | 163 ++- osm_ro/openmanod.cfg | 2 + osm_ro/tests/db_helpers.py | 187 +++ osm_ro/tests/helpers.py | 121 ++ osm_ro/utils.py | 153 ++- osm_ro/vim_thread.py | 16 +- osm_ro/vimconn_openstack.py | 82 +- osm_ro/wim/__init__.py | 0 osm_ro/wim/actions.py | 423 +++++++ osm_ro/wim/engine.py | 455 ++++++++ osm_ro/wim/errors.py | 180 +++ osm_ro/wim/failing_connector.py | 82 ++ osm_ro/wim/http_handler.py | 223 ++++ osm_ro/wim/persistence.py | 1018 +++++++++++++++++ osm_ro/wim/schemas.py | 177 +++ osm_ro/wim/tests/__init__.py | 0 osm_ro/wim/tests/fixtures.py | 307 +++++ osm_ro/wim/tests/test_actions.py | 366 ++++++ osm_ro/wim/tests/test_engine.py | 180 +++ osm_ro/wim/tests/test_http_handler.py | 508 ++++++++ osm_ro/wim/tests/test_persistence.py | 265 +++++ osm_ro/wim/tests/test_wim_thread.py | 332 ++++++ osm_ro/wim/tox.ini | 58 + osm_ro/wim/wan_link_actions.py | 315 +++++ osm_ro/wim/wim_thread.py | 437 +++++++ osm_ro/wim/wimconn.py | 236 ++++ osm_ro/wim/wimconn_odl.py | 47 + scripts/install-openmano.sh | 4 +- setup.py | 1 + stdeb.cfg | 2 +- test/test_RO.py | 79 ++ test/test_openmanoclient.py | 75 +- 54 files changed, 8416 insertions(+), 829 deletions(-) create mode 100644 database_utils/migrations/down/34_remove_wim_tables.sql create mode 100644 database_utils/migrations/up/34_add_wim_tables.sql create mode 100644 docker/tests.dockerfile create mode 100644 docker/tests.yml create mode 100644 osm_ro/http_tools/__init__.py create mode 100644 osm_ro/http_tools/errors.py create mode 100644 osm_ro/http_tools/handler.py create mode 100644 osm_ro/http_tools/request_processing.py create mode 100644 osm_ro/http_tools/tests/__init__.py create mode 100644 osm_ro/http_tools/tests/test_errors.py create mode 100644 osm_ro/http_tools/tests/test_handler.py create mode 100644 osm_ro/http_tools/tox.ini create mode 100644 osm_ro/tests/db_helpers.py create mode 100644 osm_ro/tests/helpers.py create mode 100644 osm_ro/wim/__init__.py create mode 100644 osm_ro/wim/actions.py create mode 100644 osm_ro/wim/engine.py create mode 100644 osm_ro/wim/errors.py create mode 100644 osm_ro/wim/failing_connector.py create mode 100644 osm_ro/wim/http_handler.py create mode 100644 osm_ro/wim/persistence.py create mode 100644 osm_ro/wim/schemas.py create mode 100644 osm_ro/wim/tests/__init__.py create mode 100644 osm_ro/wim/tests/fixtures.py create mode 100644 osm_ro/wim/tests/test_actions.py create mode 100644 osm_ro/wim/tests/test_engine.py create mode 100644 osm_ro/wim/tests/test_http_handler.py create mode 100644 osm_ro/wim/tests/test_persistence.py create mode 100644 osm_ro/wim/tests/test_wim_thread.py create mode 100644 osm_ro/wim/tox.ini create mode 100644 osm_ro/wim/wan_link_actions.py create mode 100644 osm_ro/wim/wim_thread.py create mode 100644 osm_ro/wim/wimconn.py create mode 100644 osm_ro/wim/wimconn_odl.py diff --git a/.gitignore-common b/.gitignore-common index 77f6798d..756a1fc5 100644 --- a/.gitignore-common +++ b/.gitignore-common @@ -22,6 +22,9 @@ # This is a template with common files to be igonored, after clone make a copy to .gitignore # cp .gitignore-common .gitignore +.tox/ +.coverage + *.pyc *.pyo diff --git a/database_utils/migrate_mano_db.sh b/database_utils/migrate_mano_db.sh index 31873240..2657cda3 100755 --- a/database_utils/migrate_mano_db.sh +++ b/database_utils/migrate_mano_db.sh @@ -24,6 +24,7 @@ # #Upgrade/Downgrade openmano database preserving the content # +DBUTILS="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" DBUSER="mano" DBPASS="" @@ -33,7 +34,7 @@ DBPORT="3306" DBNAME="mano_db" QUIET_MODE="" #TODO update it with the last database version -LAST_DB_VERSION=32 +LAST_DB_VERSION=34 # Detect paths MYSQL=$(which mysql) @@ -200,6 +201,7 @@ fi #[ $OPENMANO_VER_NUM -ge 5060 ] && DB_VERSION=30 #0.5.60 => 30 #[ $OPENMANO_VER_NUM -ge 5061 ] && DB_VERSION=31 #0.5.61 => 31 #[ $OPENMANO_VER_NUM -ge 5070 ] && DB_VERSION=32 #0.5.70 => 32 +#[ $OPENMANO_VER_NUM -ge 6000 ] && DB_VERSION=34 #0.6.00 => 34 #TODO ... put next versions here function upgrade_to_1(){ @@ -221,8 +223,8 @@ function upgrade_to_1(){ } function downgrade_from_1(){ # echo " downgrade database from version 0.1 to version 0.0" - echo " DROP TABLE \`schema_version\`" - sql "DROP TABLE \`schema_version\`;" + echo " DROP TABLE IF EXISTS \`schema_version\`" + sql "DROP TABLE IF EXISTS \`schema_version\`;" } function upgrade_to_2(){ # echo " upgrade database from version 0.1 to version 0.2" @@ -303,11 +305,11 @@ function downgrade_from_2(){ echo " Delete columns 'user/passwd' from 'vim_tenants'" sql "ALTER TABLE vim_tenants DROP COLUMN user, DROP COLUMN passwd; " echo " delete tables 'datacenter_images', 'images'" - sql "DROP TABLE \`datacenters_images\`;" - sql "DROP TABLE \`images\`;" + sql "DROP TABLE IF EXISTS \`datacenters_images\`;" + sql "DROP TABLE IF EXISTS \`images\`;" echo " delete tables 'datacenter_flavors', 'flavors'" - sql "DROP TABLE \`datacenters_flavors\`;" - sql "DROP TABLE \`flavors\`;" + sql "DROP TABLE IF EXISTS \`datacenters_flavors\`;" + sql "DROP TABLE IF EXISTS \`flavors\`;" sql "DELETE FROM schema_version WHERE version_int='2';" } @@ -621,7 +623,7 @@ function upgrade_to_12(){ function downgrade_from_12(){ # echo " downgrade database from version 0.12 to version 0.11" echo " delete ip_profiles table, and remove ip_address column in 'interfaces' and 'sce_interfaces'" - sql "DROP TABLE ip_profiles;" + sql "DROP TABLE IF EXISTS ip_profiles;" sql "ALTER TABLE interfaces DROP COLUMN ip_address;" sql "ALTER TABLE sce_interfaces DROP COLUMN ip_address;" sql "DELETE FROM schema_version WHERE version_int='12';" @@ -1005,8 +1007,8 @@ function downgrade_from_26(){ "REFERENCES scenarios (uuid);" echo " Delete table instance_actions" - sql "DROP TABLE vim_actions" - sql "DROP TABLE instance_actions" + sql "DROP TABLE IF EXISTS vim_actions" + sql "DROP TABLE IF EXISTS instance_actions" sql "DELETE FROM schema_version WHERE version_int='26';" } @@ -1219,24 +1221,24 @@ function upgrade_to_28(){ function downgrade_from_28(){ echo " [Undo adding the VNFFG tables]" echo " Dropping instance_sfps" - sql "DROP TABLE instance_sfps;" + sql "DROP TABLE IF EXISTS instance_sfps;" echo " Dropping sce_classifications" - sql "DROP TABLE instance_classifications;" + sql "DROP TABLE IF EXISTS instance_classifications;" echo " Dropping instance_sfs" - sql "DROP TABLE instance_sfs;" + sql "DROP TABLE IF EXISTS instance_sfs;" echo " Dropping instance_sfis" - sql "DROP TABLE instance_sfis;" + sql "DROP TABLE IF EXISTS instance_sfis;" echo " Dropping sce_classifier_matches" echo " [Undo adding the VNFFG-SFC instance mapping tables]" - sql "DROP TABLE sce_classifier_matches;" + sql "DROP TABLE IF EXISTS sce_classifier_matches;" echo " Dropping sce_classifiers" - sql "DROP TABLE sce_classifiers;" + sql "DROP TABLE IF EXISTS sce_classifiers;" echo " Dropping sce_rsp_hops" - sql "DROP TABLE sce_rsp_hops;" + sql "DROP TABLE IF EXISTS sce_rsp_hops;" echo " Dropping sce_rsps" - sql "DROP TABLE sce_rsps;" + sql "DROP TABLE IF EXISTS sce_rsps;" echo " Dropping sce_vnffgs" - sql "DROP TABLE sce_vnffgs;" + sql "DROP TABLE IF EXISTS sce_vnffgs;" echo " [Altering vim_actions table]" sql "ALTER TABLE vim_actions MODIFY COLUMN item ENUM('datacenters_flavors','datacenter_images','instance_nets','instance_vms','instance_interfaces') NOT NULL COMMENT 'table where the item is stored'" sql "DELETE FROM schema_version WHERE version_int='28';" @@ -1300,6 +1302,19 @@ function downgrade_from_X(){ echo " Change back 'datacenter_nets'" sql "ALTER TABLE datacenter_nets DROP COLUMN vim_tenant_id, DROP INDEX name_datacenter_id, ADD UNIQUE INDEX name_datacenter_id (name, datacenter_id);" } + +function upgrade_to_34() { + echo " Create databases required for WIM features" + script="$(find "${DBUTILS}/migrations/up" -iname "34*.sql" | tail -1)" + sql "source ${script}" +} + +function downgrade_from_34() { + echo " Drop databases required for WIM features" + script="$(find "${DBUTILS}/migrations/down" -iname "34*.sql" | tail -1)" + sql "source ${script}" +} + #TODO ... put functions here # echo "db version = "${DATABASE_VER_NUM} diff --git a/database_utils/migrations/down/34_remove_wim_tables.sql b/database_utils/migrations/down/34_remove_wim_tables.sql new file mode 100644 index 00000000..c6fa0b43 --- /dev/null +++ b/database_utils/migrations/down/34_remove_wim_tables.sql @@ -0,0 +1,27 @@ +-- +-- Tear down database structure required for integrating OSM with +-- Wide Are Network Infrastructure Managers +-- + +DROP TABLE IF EXISTS wim_port_mappings; +DROP TABLE IF EXISTS wim_nfvo_tenants; +DROP TABLE IF EXISTS instance_wim_nets; + +ALTER TABLE `vim_wim_actions` DROP FOREIGN KEY `FK_actions_wims`; +ALTER TABLE `vim_wim_actions` DROP INDEX `FK_actions_wims`; +ALTER TABLE `vim_wim_actions` DROP INDEX `item_type_id`; +ALTER TABLE `vim_wim_actions` MODIFY `item` enum( + 'datacenters_flavors', + 'datacenter_images', + 'instance_nets', + 'instance_vms', + 'instance_interfaces') NOT NULL + COMMENT 'table where the item is stored'; +ALTER TABLE `vim_wim_actions` MODIFY `datacenter_vim_id` varchar(36) NOT NULL; +ALTER TABLE `vim_wim_actions` DROP `wim_internal_id`, DROP `wim_account_id`; +ALTER TABLE `vim_wim_actions` RENAME TO `vim_actions`; + +DROP TABLE IF EXISTS wim_accounts; +DROP TABLE IF EXISTS wims; + +DELETE FROM schema_version WHERE version_int='34'; diff --git a/database_utils/migrations/up/34_add_wim_tables.sql b/database_utils/migrations/up/34_add_wim_tables.sql new file mode 100644 index 00000000..6c6fc335 --- /dev/null +++ b/database_utils/migrations/up/34_add_wim_tables.sql @@ -0,0 +1,165 @@ +-- +-- Setup database structure required for integrating OSM with +-- Wide Are Network Infrastructure Managers +-- + +DROP TABLE IF EXISTS wims; +CREATE TABLE wims ( + `uuid` varchar(36) NOT NULL, + `name` varchar(255) NOT NULL, + `description` varchar(255) DEFAULT NULL, + `type` varchar(36) NOT NULL DEFAULT 'odl', + `wim_url` varchar(150) NOT NULL, + `config` varchar(4000) DEFAULT NULL, + `created_at` double NOT NULL, + `modified_at` double DEFAULT NULL, + PRIMARY KEY (`uuid`), + UNIQUE KEY `name` (`name`) +) +ENGINE=InnoDB DEFAULT CHARSET=utf8 ROW_FORMAT=COMPACT +COMMENT='WIMs managed by the NFVO.'; + +DROP TABLE IF EXISTS wim_accounts; +CREATE TABLE wim_accounts ( + `uuid` varchar(36) NOT NULL, + `name` varchar(255) DEFAULT NULL, + `wim_id` varchar(36) NOT NULL, + `created` enum('true','false') NOT NULL DEFAULT 'false', + `user` varchar(64) DEFAULT NULL, + `password` varchar(64) DEFAULT NULL, + `config` varchar(4000) DEFAULT NULL, + `created_at` double NOT NULL, + `modified_at` double DEFAULT NULL, + PRIMARY KEY (`uuid`), + UNIQUE KEY `wim_name` (`wim_id`,`name`), + KEY `FK_wim_accounts_wims` (`wim_id`), + CONSTRAINT `FK_wim_accounts_wims` FOREIGN KEY (`wim_id`) + REFERENCES `wims` (`uuid`) ON DELETE CASCADE ON UPDATE CASCADE +) +ENGINE=InnoDB DEFAULT CHARSET=utf8 ROW_FORMAT=COMPACT +COMMENT='WIM accounts by the user'; + +DROP TABLE IF EXISTS `wim_nfvo_tenants`; +CREATE TABLE `wim_nfvo_tenants` ( + `id` integer NOT NULL AUTO_INCREMENT, + `nfvo_tenant_id` varchar(36) NOT NULL, + `wim_id` varchar(36) NOT NULL, + `wim_account_id` varchar(36) NOT NULL, + `created_at` double NOT NULL, + `modified_at` double DEFAULT NULL, + PRIMARY KEY (`id`), + UNIQUE KEY `wim_nfvo_tenant` (`wim_id`,`nfvo_tenant_id`), + KEY `FK_wims_nfvo_tenants` (`wim_id`), + KEY `FK_wim_accounts_nfvo_tenants` (`wim_account_id`), + KEY `FK_nfvo_tenants_wim_accounts` (`nfvo_tenant_id`), + CONSTRAINT `FK_wims_nfvo_tenants` FOREIGN KEY (`wim_id`) + REFERENCES `wims` (`uuid`) ON DELETE CASCADE ON UPDATE CASCADE, + CONSTRAINT `FK_wim_accounts_nfvo_tenants` FOREIGN KEY (`wim_account_id`) + REFERENCES `wim_accounts` (`uuid`) ON DELETE CASCADE ON UPDATE CASCADE, + CONSTRAINT `FK_nfvo_tenants_wim_accounts` FOREIGN KEY (`nfvo_tenant_id`) + REFERENCES `nfvo_tenants` (`uuid`) ON DELETE CASCADE ON UPDATE CASCADE +) +ENGINE=InnoDB AUTO_INCREMENT=86 DEFAULT CHARSET=utf8 ROW_FORMAT=COMPACT +COMMENT='WIM accounts mapping to NFVO tenants'; + +DROP TABLE IF EXISTS `instance_wim_nets`; +CREATE TABLE `instance_wim_nets` ( + `uuid` varchar(36) NOT NULL, + `wim_internal_id` varchar(128) DEFAULT NULL + COMMENT 'Internal ID used by the WIM to refer to the network', + `instance_scenario_id` varchar(36) DEFAULT NULL, + `sce_net_id` varchar(36) DEFAULT NULL, + `wim_id` varchar(36) DEFAULT NULL, + `wim_account_id` varchar(36) NOT NULL, + `status` enum( + 'ACTIVE', + 'INACTIVE', + 'DOWN', + 'BUILD', + 'ERROR', + 'WIM_ERROR', + 'DELETED', + 'SCHEDULED_CREATION', + 'SCHEDULED_DELETION') NOT NULL DEFAULT 'BUILD', + `error_msg` varchar(1024) DEFAULT NULL, + `wim_info` text, + `multipoint` enum('true','false') NOT NULL DEFAULT 'false', + `created` enum('true','false') NOT NULL DEFAULT 'false' + COMMENT 'Created or already exists at WIM', + `created_at` double NOT NULL, + `modified_at` double DEFAULT NULL, + PRIMARY KEY (`uuid`), + KEY `FK_instance_wim_nets_instance_scenarios` (`instance_scenario_id`), + KEY `FK_instance_wim_nets_sce_nets` (`sce_net_id`), + KEY `FK_instance_wim_nets_wims` (`wim_id`), + KEY `FK_instance_wim_nets_wim_accounts` (`wim_account_id`), + CONSTRAINT `FK_instance_wim_nets_wim_accounts` + FOREIGN KEY (`wim_account_id`) REFERENCES `wim_accounts` (`uuid`), + CONSTRAINT `FK_instance_wim_nets_wims` + FOREIGN KEY (`wim_id`) REFERENCES `wims` (`uuid`), + CONSTRAINT `FK_instance_wim_nets_instance_scenarios` + FOREIGN KEY (`instance_scenario_id`) REFERENCES `instance_scenarios` (`uuid`) + ON DELETE CASCADE ON UPDATE CASCADE, + CONSTRAINT `FK_instance_wim_nets_sce_nets` + FOREIGN KEY (`sce_net_id`) REFERENCES `sce_nets` (`uuid`) + ON DELETE SET NULL ON UPDATE CASCADE +) ENGINE=InnoDB DEFAULT CHARSET=utf8 ROW_FORMAT=COMPACT + COMMENT='Instances of wim networks'; + +ALTER TABLE `vim_actions` + RENAME TO `vim_wim_actions`; +ALTER TABLE `vim_wim_actions` + ADD `wim_account_id` varchar(36) DEFAULT NULL AFTER `vim_id`, + ADD `wim_internal_id` varchar(64) DEFAULT NULL AFTER `wim_account_id`, + MODIFY `datacenter_vim_id` varchar(36) DEFAULT NULL, + MODIFY `item` enum( + 'datacenters_flavors', + 'datacenter_images', + 'instance_nets', + 'instance_vms', + 'instance_interfaces', + 'instance_wim_nets') NOT NULL + COMMENT 'table where the item is stored'; +ALTER TABLE `vim_wim_actions` + ADD INDEX `item_type_id` (`item`, `item_id`); +ALTER TABLE `vim_wim_actions` + ADD INDEX `FK_actions_wims` (`wim_account_id`); +ALTER TABLE `vim_wim_actions` + ADD CONSTRAINT `FK_actions_wims` FOREIGN KEY (`wim_account_id`) + REFERENCES `wim_accounts` (`uuid`) + ON UPDATE CASCADE ON DELETE CASCADE; + +DROP TABLE IF EXISTS `wim_port_mappings`; +CREATE TABLE `wim_port_mappings` ( + `id` integer NOT NULL AUTO_INCREMENT, + `wim_id` varchar(36) NOT NULL, + `datacenter_id` varchar(36) NOT NULL, + `pop_switch_dpid` varchar(64) NOT NULL, + `pop_switch_port` varchar(64) NOT NULL, + `wan_service_endpoint_id` varchar(256) NOT NULL + COMMENT 'In case the WIM plugin relies on the wan_service_mapping_info' + COMMENT 'this field contains a unique identifier used to check the mapping_info consistency', + /* In other words: wan_service_endpoint_id = f(wan_service_mapping_info) + * where f is a injective function' + */ + `wan_service_mapping_info` text, + `created_at` double NOT NULL, + `modified_at` double DEFAULT NULL, + PRIMARY KEY (`id`), + UNIQUE KEY `unique_datacenter_port_mapping` + (`datacenter_id`, `pop_switch_dpid`, `pop_switch_port`), + UNIQUE KEY `unique_wim_port_mapping` + (`wim_id`, `wan_service_endpoint_id`), + KEY `FK_wims_wim_physical_connections` (`wim_id`), + KEY `FK_datacenters_wim_port_mappings` (`datacenter_id`), + CONSTRAINT `FK_wims_wim_port_mappings` FOREIGN KEY (`wim_id`) + REFERENCES `wims` (`uuid`) ON DELETE CASCADE ON UPDATE CASCADE, + CONSTRAINT `FK_datacenters_wim_port_mappings` FOREIGN KEY (`datacenter_id`) + REFERENCES `datacenters` (`uuid`) ON DELETE CASCADE ON UPDATE CASCADE +) +ENGINE=InnoDB DEFAULT CHARSET=utf8 +COMMENT='WIM port mappings managed by the WIM.'; + +-- Update Schema with DB version +INSERT INTO schema_version +VALUES (34, '0.34', '0.6.00', 'Added WIM tables', '2018-09-10'); diff --git a/docker/Dockerfile-local b/docker/Dockerfile-local index 61fb0035..3e605cc4 100644 --- a/docker/Dockerfile-local +++ b/docker/Dockerfile-local @@ -15,7 +15,7 @@ RUN apt-get update && \ DEBIAN_FRONTEND=noninteractive pip install -U setuptools setuptools-version-command stdeb && \ DEBIAN_FRONTEND=noninteractive pip install -U pyang pyangbind && \ DEBIAN_FRONTEND=noninteractive pip3 install -U pyang pyangbind && \ - DEBIAN_FRONTEND=noninteractive apt-get -y install python-yaml python-netaddr python-boto && \ + DEBIAN_FRONTEND=noninteractive apt-get -y install python-yaml python-netaddr python-boto python-networkx && \ DEBIAN_FRONTEND=noninteractive apt-get -y install software-properties-common && \ DEBIAN_FRONTEND=noninteractive apt-get -y install python-novaclient python-keystoneclient python-glanceclient python-cinderclient python-neutronclient && \ DEBIAN_FRONTEND=noninteractive pip install -U progressbar pyvmomi pyvcloud==19.1.1 && \ diff --git a/docker/tests.dockerfile b/docker/tests.dockerfile new file mode 100644 index 00000000..fd5a45ca --- /dev/null +++ b/docker/tests.dockerfile @@ -0,0 +1,13 @@ +from ubuntu:xenial + +VOLUME /opt/openmano +VOLUME /var/log/osm + +ENV DEBIAN_FRONTEND=noninteractive + + +RUN apt-get update && \ + apt-get -y install python python-pip mysql-client libmysqlclient-dev && \ + pip install tox + +ENTRYPOINT ["tox"] diff --git a/docker/tests.yml b/docker/tests.yml new file mode 100644 index 00000000..a33f0d43 --- /dev/null +++ b/docker/tests.yml @@ -0,0 +1,37 @@ +# This file is intended to be used by the developer in the local machine +# in order to run the tests in isolation +# To do so, cd into osm_ro and run: +# docker-compose -f ../docker/tests.yml run --rm tox -c +version: '2' +services: + test-db: + image: mysql:5 + container_name: test-db + restart: always + environment: + - MYSQL_ROOT_PASSWORD=osm4u + - MYSQL_USER=mano + - MYSQL_PASSWORD=manopw + - MYSQL_DATABASE=mano_db + tox: + container_name: tox + depends_on: + - test-db + build: + context: ../ + dockerfile: docker/tests.dockerfile + restart: always + environment: + - RO_DB_ROOT_PASSWORD=osm4u + - TEST_DB_HOST=test-db + - TEST_DB_USER=mano + - TEST_DB_PASSWORD=manopw + - TEST_DB_DATABASE=mano_db + ports: + - "9090:9090" + volumes: + - ..:/opt/openmano + - /tmp/osm/openmano/logs:/var/log/osm + entrypoint: + - tox + working_dir: /opt/openmano/osm_ro diff --git a/openmano b/openmano index 9351c81c..98ea191e 100755 --- a/openmano +++ b/openmano @@ -24,7 +24,7 @@ ## """ -openmano client used to interact with openmano-server (openmanod) +openmano client used to interact with openmano-server (openmanod) """ __author__ = "Alfonso Tierno, Gerardo Garcia, Pablo Montes" __date__ = "$09-oct-2014 09:09:48$" @@ -65,6 +65,10 @@ def config(args): mano_tenant_name = "None" mano_datacenter_id = "None" mano_datacenter_name = "None" + # WIM additions + logger.debug("resolving WIM names") + mano_wim_id = "None" + mano_wim_name = "None" try: mano_tenant_id = _get_item_uuid("tenants", mano_tenant) URLrequest = "http://%s:%s/openmano/tenants/%s" %(mano_host, mano_port, mano_tenant_id) @@ -79,17 +83,35 @@ def config(args): if "error" not in content: mano_datacenter_id = content["datacenter"]["uuid"] mano_datacenter_name = content["datacenter"]["name"] + + # WIM + URLrequest = "http://%s:%s/openmano/%s/wims/%s" % ( + mano_host, mano_port, mano_tenant_id, mano_wim) + mano_response = requests.get(URLrequest) + logger.debug("openmano response: %s", mano_response.text) + content = mano_response.json() + if "error" not in content: + mano_wim_id = content["wim"]["uuid"] + mano_wim_name = content["wim"]["name"] + except OpenmanoCLIError: pass print "OPENMANO_TENANT: %s" %mano_tenant print " Id: %s" %mano_tenant_id - print " Name: %s" %mano_tenant_name + print " Name: %s" %mano_tenant_name print "OPENMANO_DATACENTER: %s" %str (mano_datacenter) print " Id: %s" %mano_datacenter_id - print " Name: %s" %mano_datacenter_name + print " Name: %s" %mano_datacenter_name + # WIM + print "OPENMANO_WIM: %s" %str (mano_wim) + print " Id: %s" %mano_wim_id + print " Name: %s" %mano_wim_name + else: print "OPENMANO_TENANT: %s" %mano_tenant print "OPENMANO_DATACENTER: %s" %str (mano_datacenter) + # WIM + print "OPENMANO_WIM: %s" %str (mano_wim) def _print_verbose(mano_response, verbose_level=0): content = mano_response.json() @@ -98,7 +120,7 @@ def _print_verbose(mano_response, verbose_level=0): #print "Non expected format output" print str(content) return result - + val=content.values()[0] if type(val)==str: print val @@ -111,7 +133,7 @@ def _print_verbose(mano_response, verbose_level=0): #print "Non expected dict/list format output" print str(content) return result - + #print content_list if verbose_level==None: verbose_level=0 @@ -163,7 +185,7 @@ def parser_json_yaml(file_name): f.close() except Exception as e: return (False, str(e)) - + #Read and parse file if file_name[-5:]=='.yaml' or file_name[-4:]=='.yml' or (file_name[-5:]!='.json' and '\t' not in text): try: @@ -176,7 +198,7 @@ def parser_json_yaml(file_name): return (False, "Error loading file '"+file_name+"' yaml format error" + error_pos) else: #json try: - config = json.loads(text) + config = json.loads(text) except Exception as e: return (False, "Error loading file '"+file_name+"' json format error " + str(e) ) @@ -234,7 +256,7 @@ def _get_item_uuid(item, item_name_id, tenant=None): elif found > 1: raise OpenmanoCLIError("%d %s found with name '%s'. uuid must be used" %(found, item, item_name_id)) return uuid -# +# # def check_valid_uuid(uuid): # id_schema = {"type" : "string", "pattern": "^[a-fA-F0-9]{8}(-[a-fA-F0-9]{4}){3}-[a-fA-F0-9]{12}$"} # try: @@ -242,7 +264,7 @@ def _get_item_uuid(item, item_name_id, tenant=None): # return True # except js_e.ValidationError: # return False - + def _get_tenant(tenant_name_id = None): if not tenant_name_id: tenant_name_id = mano_tenant @@ -257,6 +279,14 @@ def _get_datacenter(datacenter_name_id = None, tenant = "any"): raise OpenmanoCLIError("neither 'OPENMANO_DATACENTER' environment variable is set nor --datacenter option is used") return _get_item_uuid("datacenters", datacenter_name_id, tenant) +# WIM +def _get_wim(wim_name_id = None, tenant = "any"): + if not wim_name_id: + wim_name_id = mano_wim + if not wim_name_id: + raise OpenmanoCLIError("neither 'OPENMANO_WIM' environment variable is set nor --wim option is used") + return _get_item_uuid("wims", wim_name_id, tenant) + def vnf_create(args): #print "vnf-create",args headers_req = {'Accept': 'application/json', 'content-type': 'application/json'} @@ -356,11 +386,11 @@ def vnf_create(args): elif str(e) == 'image checksum': error_pos= "missing field 'vnf':'VNFC'['image checksum']" else: error_pos="wrong format" print "Wrong VNF descriptor: " + error_pos - return -1 + return -1 payload_req = json.dumps(myvnf) - + #print payload_req - + URLrequest = "http://{}:{}/openmano{}/{}/{token}".format(mano_host, mano_port, api_version, tenant, token=token) logger.debug("openmano request: %s", payload_req) mano_response = requests.post(URLrequest, headers = headers_req, data=payload_req) @@ -485,7 +515,7 @@ def scenario_create(args): nsd['description'] = args.description payload_req = yaml.safe_dump(myscenario, explicit_start=True, indent=4, default_flow_style=False, tags=False, encoding='utf-8', allow_unicode=True) - + # print payload_req URLrequest = "http://{host}:{port}/openmano{api}/{tenant}/{token}".format( host=mano_host, port=mano_port, api=api_version, tenant=tenant, token=token) @@ -606,26 +636,26 @@ def scenario_deploy(args): # action[actionCmd]["datacenter"] = args.datacenter # elif mano_datacenter != None: # action[actionCmd]["datacenter"] = mano_datacenter -# +# # if args.description: # action[actionCmd]["description"] = args.description # payload_req = json.dumps(action, indent=4) # #print payload_req -# +# # URLrequest = "http://%s:%s/openmano/%s/scenarios/%s/action" %(mano_host, mano_port, mano_tenant, args.scenario) # logger.debug("openmano request: %s", payload_req) # mano_response = requests.post(URLrequest, headers = headers_req, data=payload_req) # logger.debug("openmano response: %s", mano_response.text ) # if args.verbose==None: # args.verbose=0 -# +# # result = 0 if mano_response.status_code==200 else mano_response.status_code # content = mano_response.json() # #print json.dumps(content, indent=4) # if args.verbose >= 3: # print yaml.safe_dump(content, indent=4, default_flow_style=False) # return result -# +# # if mano_response.status_code == 200: # myoutput = "%s %s" %(content['uuid'].ljust(38),content['name'].ljust(20)) # if args.verbose >=1: @@ -654,7 +684,7 @@ def scenario_verify(args): logger.debug("openmano request: %s", payload_req) mano_response = requests.post(URLrequest, headers = headers_req, data=payload_req) logger.debug("openmano response: %s", mano_response.text ) - + result = 0 if mano_response.status_code==200 else mano_response.status_code content = mano_response.json() #print json.dumps(content, indent=4) @@ -706,7 +736,7 @@ def instance_create(args): net_scenario = net_tuple[0].strip() net_datacenter = net_tuple[1].strip() if net_scenario not in myInstance["instance"]["networks"]: - myInstance["instance"]["networks"][net_scenario] = {} + myInstance["instance"]["networks"][net_scenario] = {} if "sites" not in myInstance["instance"]["networks"][net_scenario]: myInstance["instance"]["networks"][net_scenario]["sites"] = [ {} ] myInstance["instance"]["networks"][net_scenario]["sites"][0]["netmap-use"] = net_datacenter @@ -727,7 +757,7 @@ def instance_create(args): print "error at netmap-create. Expected net-scenario=net-datacenter or net-scenario. (%s)?" % net_comma return if net_scenario not in myInstance["instance"]["networks"]: - myInstance["instance"]["networks"][net_scenario] = {} + myInstance["instance"]["networks"][net_scenario] = {} if "sites" not in myInstance["instance"]["networks"][net_scenario]: myInstance["instance"]["networks"][net_scenario]["sites"] = [ {} ] myInstance["instance"]["networks"][net_scenario]["sites"][0]["netmap-create"] = net_datacenter @@ -764,7 +794,7 @@ def instance_create(args): except Exception as e: print "Cannot obtain any public ssh key. Error '{}'. Try not using --keymap-auto".format(str(e)) return 1 - + if "cloud-config" not in myInstance["instance"]: myInstance["instance"]["cloud-config"] = {} cloud_config = myInstance["instance"]["cloud-config"] @@ -773,8 +803,8 @@ def instance_create(args): if user: if "users" not in cloud_config: cloud_config["users"] = [] - cloud_config["users"].append({"name": user, "key-pairs": keys }) - + cloud_config["users"].append({"name": user, "key-pairs": keys }) + payload_req = yaml.safe_dump(myInstance, explicit_start=True, indent=4, default_flow_style=False, tags=False, encoding='utf-8', allow_unicode=True) logger.debug("openmano request: %s", payload_req) URLrequest = "http://%s:%s/openmano/%s/instances" %(mano_host, mano_port, tenant) @@ -782,7 +812,7 @@ def instance_create(args): logger.debug("openmano response: %s", mano_response.text ) if args.verbose==None: args.verbose=0 - + result = 0 if mano_response.status_code==200 else mano_response.status_code content = mano_response.json() #print json.dumps(content, indent=4) @@ -931,7 +961,7 @@ def instance_scenario_action(args): action["vnfs"] = args.vnf if args.vm: action["vms"] = args.vm - + headers_req = {'content-type': 'application/json'} payload_req = json.dumps(action, indent=4) URLrequest = "http://%s:%s/openmano/%s/instances/%s/action" %(mano_host, mano_port, tenant, toact) @@ -967,11 +997,11 @@ def tenant_create(args): headers_req = {'Accept': 'application/json', 'content-type': 'application/json'} tenant_dict={"name": args.name} if args.description!=None: - tenant_dict["description"] = args.description + tenant_dict["description"] = args.description payload_req = json.dumps( {"tenant": tenant_dict }) - + #print payload_req - + URLrequest = "http://%s:%s/openmano/tenants" %(mano_host, mano_port) logger.debug("openmano request: %s", payload_req) mano_response = requests.post(URLrequest, headers=headers_req, data=payload_req) @@ -1016,7 +1046,7 @@ def datacenter_attach(args): tenant = _get_tenant() datacenter = _get_datacenter(args.name) headers_req = {'Accept': 'application/json', 'content-type': 'application/json'} - + datacenter_dict={} if args.vim_tenant_id != None: datacenter_dict['vim_tenant'] = args.vim_tenant_id @@ -1032,7 +1062,7 @@ def datacenter_attach(args): payload_req = json.dumps( {"datacenter": datacenter_dict }) #print payload_req - + URLrequest = "http://%s:%s/openmano/%s/datacenters/%s" %(mano_host, mano_port, tenant, datacenter) logger.debug("openmano request: %s", payload_req) mano_response = requests.post(URLrequest, headers=headers_req, data=payload_req) @@ -1101,11 +1131,11 @@ def datacenter_create(args): headers_req = {'Accept': 'application/json', 'content-type': 'application/json'} datacenter_dict={"name": args.name, "vim_url": args.url} if args.description!=None: - datacenter_dict["description"] = args.description + datacenter_dict["description"] = args.description if args.type!=None: - datacenter_dict["type"] = args.type + datacenter_dict["type"] = args.type if args.url!=None: - datacenter_dict["vim_url_admin"] = args.url_admin + datacenter_dict["vim_url_admin"] = args.url_admin if args.config!=None: datacenter_dict["config"] = _load_file_or_yaml(args.config) if args.sdn_controller!=None: @@ -1115,9 +1145,9 @@ def datacenter_create(args): datacenter_dict['config'] = {} datacenter_dict['config']['sdn-controller'] = sdn_controller payload_req = json.dumps( {"datacenter": datacenter_dict }) - + #print payload_req - + URLrequest = "http://%s:%s/openmano/datacenters" %(mano_host, mano_port) logger.debug("openmano request: %s", payload_req) mano_response = requests.post(URLrequest, headers=headers_req, data=payload_req) @@ -1147,9 +1177,9 @@ def datacenter_delete(args): def datacenter_list(args): #print "datacenter-list",args tenant='any' if args.all else _get_tenant() - + if args.name: - toshow = _get_item_uuid("datacenters", args.name, tenant) + toshow = _get_item_uuid("datacenters", args.name, tenant) URLrequest = "http://%s:%s/openmano/%s/datacenters/%s" %(mano_host, mano_port, tenant, toshow) else: URLrequest = "http://%s:%s/openmano/%s/datacenters" %(mano_host, mano_port, tenant) @@ -1500,7 +1530,7 @@ def datacenter_net_action(args): elif args.action == "net-delete": args.netmap = args.net args.all = False - + args.action = "netmap" + args.action[3:] args.vim_name=None args.vim_id=None @@ -1517,13 +1547,13 @@ def datacenter_netmap_action(args): args.verbose=0 headers_req = {'Accept': 'application/json', 'content-type': 'application/json'} URLrequest = "http://%s:%s/openmano/%s/datacenters/%s/netmaps" %(mano_host, mano_port, tenant, datacenter) - + if args.action=="netmap-list": if args.netmap: URLrequest += "/" + args.netmap args.verbose += 1 mano_response = requests.get(URLrequest) - + elif args.action=="netmap-delete": if args.netmap and args.all: print "you can not use a netmap name and the option --all at the same time" @@ -1531,7 +1561,7 @@ def datacenter_netmap_action(args): if args.netmap: force_text= "Delete default netmap '%s' from datacenter '%s' (y/N)? " % (args.netmap, datacenter) URLrequest += "/" + args.netmap - elif args.all: + elif args.all: force_text="Delete all default netmaps from datacenter '%s' (y/N)? " % (datacenter) else: print "you must specify a netmap name or the option --all" @@ -1567,7 +1597,7 @@ def datacenter_netmap_action(args): payload["netmap"]["vim_name"] = args.vim_name payload_req = json.dumps(payload) logger.debug("openmano request: %s", payload_req) - + if args.action=="netmap-edit" and not args.force: if len(payload["netmap"]) == 0: print "You must supply some parameter to edit" @@ -1597,7 +1627,7 @@ def element_edit(args): if args.element[:-1] not in payload: payload = {args.element[:-1]: payload } payload_req = json.dumps(payload) - + #print payload_req if not args.force or (args.name==None and args.filer==None): r = raw_input(" Edit " + args.element[:-1] + " " + args.name + " (y/N)? ") @@ -1663,6 +1693,260 @@ def datacenter_edit(args): return _print_verbose(mano_response, args.verbose) +# WIM +def wim_account_create(args): + tenant = _get_tenant() + wim = _get_wim(args.name) + headers_req = {'Accept': 'application/json', 'content-type': 'application/json'} + + wim_dict = {} + if args.account_name is not None: + wim_dict['name'] = args.account_name + if args.user is not None: + wim_dict['user'] = args.user + if args.password is not None: + wim_dict['password'] = args.password + if args.config is not None: + wim_dict["config"] = _load_file_or_yaml(args.config) + + payload_req = json.dumps({"wim_account": wim_dict}) + + URLrequest = "http://%s:%s/openmano/%s/wims/%s" % (mano_host, mano_port, tenant, wim) + logger.debug("openmano request: %s", payload_req) + mano_response = requests.post(URLrequest, headers=headers_req, data=payload_req) + logger.debug("openmano response: %s", mano_response.text) + result = _print_verbose(mano_response, args.verbose) + # provide addional information if error + if mano_response.status_code != 200: + content = mano_response.json() + if "already in use for 'name'" in content['error']['description'] and \ + "to database wim_tenants table" in content['error']['description']: + print "Try to specify a different name with --wim-tenant-name" + return result + + +def wim_account_delete(args): + if args.all: + tenant = "any" + else: + tenant = _get_tenant() + wim = _get_wim(args.name, tenant) + headers_req = {'Accept': 'application/json', 'content-type': 'application/json'} + URLrequest = "http://%s:%s/openmano/%s/wims/%s" % (mano_host, mano_port, tenant, wim) + mano_response = requests.delete(URLrequest, headers=headers_req) + logger.debug("openmano response: %s", mano_response.text) + content = mano_response.json() + # print json.dumps(content, indent=4) + result = 0 if mano_response.status_code == 200 else mano_response.status_code + if mano_response.status_code == 200: + print content['result'] + else: + print content['error']['description'] + return result + + +def wim_account_edit(args): + tenant = _get_tenant() + wim = _get_wim(args.name) + headers_req = {'Accept': 'application/json', 'content-type': 'application/json'} + + wim_dict = {} + if not args.account_name: + wim_dict['name'] = args.vim_tenant_name + if not args.user: + wim_dict['user'] = args.user + if not args.password: + wim_dict['password'] = args.password + if not args.config: + wim_dict["config"] = _load_file_or_yaml(args.config) + + payload_req = json.dumps({"wim_account": wim_dict}) + + # print payload_req + + URLrequest = "http://%s:%s/openmano/%s/wims/%s" % (mano_host, mano_port, tenant, wim) + logger.debug("openmano request: %s", payload_req) + mano_response = requests.post(URLrequest, headers=headers_req, data=payload_req) + logger.debug("openmano response: %s", mano_response.text) + result = _print_verbose(mano_response, args.verbose) + # provide addional information if error + if mano_response.status_code != 200: + content = mano_response.json() + if "already in use for 'name'" in content['error']['description'] and \ + "to database wim_tenants table" in content['error']['description']: + print "Try to specify a different name with --wim-tenant-name" + return result + +def wim_create(args): + headers_req = {'Accept': 'application/json', 'content-type': 'application/json'} + wim_dict = {"name": args.name, "wim_url": args.url} + if args.description != None: + wim_dict["description"] = args.description + if args.type != None: + wim_dict["type"] = args.type + if args.config != None: + wim_dict["config"] = _load_file_or_yaml(args.config) + + payload_req = json.dumps({"wim": wim_dict}) + + URLrequest = "http://%s:%s/openmano/wims" % (mano_host, mano_port) + logger.debug("openmano request: %s", payload_req) + mano_response = requests.post(URLrequest, headers=headers_req, data=payload_req) + logger.debug("openmano response: %s", mano_response.text) + return _print_verbose(mano_response, args.verbose) + + +def wim_edit(args): + tenant = _get_tenant() + element = _get_item_uuid('wims', args.name, tenant) + headers_req = {'Accept': 'application/json', 'content-type': 'application/json'} + URLrequest = "http://%s:%s/openmano/wims/%s" % (mano_host, mano_port, element) + + has_arguments = False + if args.file != None: + has_arguments = True + payload = _load_file_or_yaml(args.file) + else: + payload = {} + + if not has_arguments: + raise OpenmanoCLIError("At least one argument must be provided to modify the wim") + + if 'wim' not in payload: + payload = {'wim': payload} + payload_req = json.dumps(payload) + + # print payload_req + if not args.force or (args.name == None and args.filer == None): + r = raw_input(" Edit wim " + args.name + " (y/N)? ") + if len(r) > 0 and r[0].lower() == "y": + pass + else: + return 0 + logger.debug("openmano request: %s", payload_req) + mano_response = requests.put(URLrequest, headers=headers_req, data=payload_req) + logger.debug("openmano response: %s", mano_response.text) + if args.verbose == None: + args.verbose = 0 + if args.name != None: + args.verbose += 1 + return _print_verbose(mano_response, args.verbose) + + +def wim_delete(args): + # print "wim-delete",args + todelete = _get_item_uuid("wims", args.name, "any") + if not args.force: + r = raw_input("Delete wim %s (y/N)? " % (args.name)) + if not (len(r) > 0 and r[0].lower() == "y"): + return 0 + URLrequest = "http://%s:%s/openmano/wims/%s" % (mano_host, mano_port, todelete) + mano_response = requests.delete(URLrequest) + logger.debug("openmano response: %s", mano_response.text) + result = 0 if mano_response.status_code == 200 else mano_response.status_code + content = mano_response.json() + # print json.dumps(content, indent=4) + if mano_response.status_code == 200: + print content['result'] + else: + print content['error']['description'] + return result + + +def wim_list(args): + # print "wim-list",args + tenant = 'any' if args.all else _get_tenant() + + if args.name: + toshow = _get_item_uuid("wims", args.name, tenant) + URLrequest = "http://%s:%s/openmano/%s/wims/%s" % (mano_host, mano_port, tenant, toshow) + else: + URLrequest = "http://%s:%s/openmano/%s/wims" % (mano_host, mano_port, tenant) + mano_response = requests.get(URLrequest) + logger.debug("openmano response: %s", mano_response.text) + if args.verbose == None: + args.verbose = 0 + if args.name != None: + args.verbose += 1 + return _print_verbose(mano_response, args.verbose) + + +def wim_port_mapping_set(args): + tenant = _get_tenant() + wim = _get_wim(args.name, tenant) + headers_req = {'Accept': 'application/json', 'content-type': 'application/json'} + + if not args.file: + raise OpenmanoCLIError( + "No yaml/json has been provided specifying the WIM port mapping") + wim_port_mapping = _load_file_or_yaml(args.file) + + payload_req = json.dumps({"wim_port_mapping": wim_port_mapping}) + + # read + URLrequest = "http://%s:%s/openmano/%s/wims/%s/port_mapping" % (mano_host, mano_port, tenant, wim) + mano_response = requests.get(URLrequest) + logger.debug("openmano response: %s", mano_response.text) + port_mapping = mano_response.json() + + if mano_response.status_code != 200: + str(mano_response.json()) + raise OpenmanoCLIError("openmano client error: {}".format(port_mapping['error']['description'])) + # TODO: check this if statement + if len(port_mapping["wim_port_mapping"]) > 0: + if not args.force: + r = raw_input("WIM %s already contains a port mapping. Overwrite? (y/N)? " % (wim)) + if not (len(r) > 0 and r[0].lower() == "y"): + return 0 + + # clear + URLrequest = "http://%s:%s/openmano/%s/wims/%s/port_mapping" % (mano_host, mano_port, tenant, wim) + mano_response = requests.delete(URLrequest) + logger.debug("openmano response: %s", mano_response.text) + if mano_response.status_code != 200: + return _print_verbose(mano_response, args.verbose) + + # set + URLrequest = "http://%s:%s/openmano/%s/wims/%s/port_mapping" % (mano_host, mano_port, tenant, wim) + logger.debug("openmano request: %s", payload_req) + mano_response = requests.post(URLrequest, headers=headers_req, data=payload_req) + logger.debug("openmano response: %s", mano_response.text) + return _print_verbose(mano_response, 4) + + +def wim_port_mapping_list(args): + tenant = _get_tenant() + wim = _get_wim(args.name, tenant) + + URLrequest = "http://%s:%s/openmano/%s/wims/%s/port_mapping" % (mano_host, mano_port, tenant, wim) + mano_response = requests.get(URLrequest) + logger.debug("openmano response: %s", mano_response.text) + + return _print_verbose(mano_response, 4) + + +def wim_port_mapping_clear(args): + tenant = _get_tenant() + wim = _get_wim(args.name, tenant) + + if not args.force: + r = raw_input("Clear WIM port mapping for wim %s (y/N)? " % (wim)) + if not (len(r) > 0 and r[0].lower() == "y"): + return 0 + + URLrequest = "http://%s:%s/openmano/%s/wims/%s/port_mapping" % (mano_host, mano_port, tenant, wim) + mano_response = requests.delete(URLrequest) + logger.debug("openmano response: %s", mano_response.text) + content = mano_response.json() + # print json.dumps(content, indent=4) + result = 0 if mano_response.status_code == 200 else mano_response.status_code + if mano_response.status_code == 200: + print content['result'] + else: + print content['error']['description'] + return result + + def version(args): headers_req = {'Accept': 'application/json', 'content-type': 'application/json'} URLrequest = "http://%s:%s/openmano/version" % (mano_host, mano_port) @@ -1677,19 +1961,21 @@ global mano_port global mano_tenant if __name__=="__main__": - + mano_tenant = os.getenv('OPENMANO_TENANT', None) mano_host = os.getenv('OPENMANO_HOST',"localhost") mano_port = os.getenv('OPENMANO_PORT',"9090") mano_datacenter = os.getenv('OPENMANO_DATACENTER',None) - + # WIM env variable for default WIM + mano_wim = os.getenv('OPENMANO_WIM', None) + main_parser = ThrowingArgumentParser(description='User program to interact with OPENMANO-SERVER (openmanod)') main_parser.add_argument('--version', action='version', help="get version of this client", version='%(prog)s client version ' + __version__ + " (Note: use '%(prog)s version' to get server version)") subparsers = main_parser.add_subparsers(help='commands') - + parent_parser = argparse.ArgumentParser(add_help=False) parent_parser.add_argument('--verbose', '-v', action='count', help="increase verbosity level. Use several times") parent_parser.add_argument('--debug', '-d', action='store_true', help="show debug information") @@ -1715,13 +2001,13 @@ if __name__=="__main__": vnf_list_parser.add_argument("-a", "--all", action="store_true", help="shows all vnfs, not only the owned or public ones") #vnf_list_parser.add_argument('--descriptor', help="prints the VNF descriptor", action="store_true") vnf_list_parser.set_defaults(func=vnf_list) - + vnf_delete_parser = subparsers.add_parser('vnf-delete', parents=[parent_parser], help="deletes a vnf from the catalogue") vnf_delete_parser.add_argument("name", action="store", help="name or uuid of the VNF to be deleted") vnf_delete_parser.add_argument("-f", "--force", action="store_true", help="forces deletion without asking") vnf_delete_parser.add_argument("-a", "--all", action="store_true", help="allow delete not owned or privated one") vnf_delete_parser.set_defaults(func=vnf_delete) - + scenario_create_parser = subparsers.add_parser('scenario-create', parents=[parent_parser], help="adds a scenario into the OPENMANO DB") scenario_create_parser.add_argument("file", action="store", help="location of the YAML file describing the scenario").completer = FilesCompleter scenario_create_parser.add_argument("--name", action="store", help="name of the scenario (if it exists in the YAML scenario, it is overwritten)") @@ -1733,7 +2019,7 @@ if __name__=="__main__": #scenario_list_parser.add_argument('--descriptor', help="prints the scenario descriptor", action="store_true") scenario_list_parser.add_argument("-a", "--all", action="store_true", help="shows all scenarios, not only the owned or public ones") scenario_list_parser.set_defaults(func=scenario_list) - + scenario_delete_parser = subparsers.add_parser('scenario-delete', parents=[parent_parser], help="deletes a scenario from the OPENMANO DB") scenario_delete_parser.add_argument("name", action="store", help="name or uuid of the scenario to be deleted") scenario_delete_parser.add_argument("-f", "--force", action="store_true", help="forces deletion without asking") @@ -1747,12 +2033,12 @@ if __name__=="__main__": scenario_deploy_parser.add_argument("--datacenter", action="store", help="specifies the datacenter. Needed if several datacenters are available") scenario_deploy_parser.add_argument("--description", action="store", help="description of the instance") scenario_deploy_parser.set_defaults(func=scenario_deploy) - + scenario_deploy_parser = subparsers.add_parser('scenario-verify', help="verifies if a scenario can be deployed (deploys it and deletes it)") scenario_deploy_parser.add_argument("scenario", action="store", help="name or uuid of the scenario to be verified") scenario_deploy_parser.add_argument('--debug', '-d', action='store_true', help="show debug information") scenario_deploy_parser.set_defaults(func=scenario_verify) - + instance_scenario_create_parser = subparsers.add_parser('instance-scenario-create', parents=[parent_parser], help="deploys a scenario") instance_scenario_create_parser.add_argument("file", nargs='?', help="descriptor of the instance. Must be a file or yaml/json text") instance_scenario_create_parser.add_argument("--scenario", action="store", help="name or uuid of the scenario to be deployed") @@ -1776,7 +2062,7 @@ if __name__=="__main__": instance_scenario_delete_parser.add_argument("-f", "--force", action="store_true", help="forces deletion without asking") instance_scenario_delete_parser.add_argument("-a", "--all", action="store_true", help="allow delete not owned or privated one") instance_scenario_delete_parser.set_defaults(func=instance_scenario_delete) - + instance_scenario_action_parser = subparsers.add_parser('instance-scenario-action', parents=[parent_parser], help="invoke an action over part or the whole scenario instance") instance_scenario_action_parser.add_argument("name", action="store", help="name or uuid of the scenario instance") instance_scenario_action_parser.add_argument("action", action="store", type=str, \ @@ -1796,7 +2082,7 @@ if __name__=="__main__": #instance_scenario_status_parser = subparsers.add_parser('instance-scenario-status', help="show the status of a scenario instance") #instance_scenario_status_parser.add_argument("name", action="store", help="name or uuid of the scenario instance") #instance_scenario_status_parser.set_defaults(func=instance_scenario_status) - + tenant_create_parser = subparsers.add_parser('tenant-create', parents=[parent_parser], help="creates a new tenant") tenant_create_parser.add_argument("name", action="store", help="name for the tenant") tenant_create_parser.add_argument("--description", action="store", help="description of the tenant") @@ -1956,6 +2242,128 @@ if __name__=="__main__": sdn_controller_delete_parser.set_defaults(func=sdn_controller_delete) # ======================= + # WIM ======================= WIM section================== + + # WIM create + wim_create_parser = subparsers.add_parser('wim-create', + parents=[parent_parser], help="creates a new wim") + wim_create_parser.add_argument("name", action="store", + help="name for the wim") + wim_create_parser.add_argument("url", action="store", + help="url for the wim") + wim_create_parser.add_argument("--type", action="store", + help="wim type: tapi, onos or odl (default)") + wim_create_parser.add_argument("--config", action="store", + help="additional configuration in json/yaml format") + wim_create_parser.add_argument("--description", action="store", + help="description of the wim") + wim_create_parser.set_defaults(func=wim_create) + + # WIM delete + wim_delete_parser = subparsers.add_parser('wim-delete', + parents=[parent_parser], help="deletes a wim from the catalogue") + wim_delete_parser.add_argument("name", action="store", + help="name or uuid of the wim to be deleted") + wim_delete_parser.add_argument("-f", "--force", action="store_true", + help="forces deletion without asking") + wim_delete_parser.set_defaults(func=wim_delete) + + # WIM edit + wim_edit_parser = subparsers.add_parser('wim-edit', + parents=[parent_parser], help="edits a wim") + wim_edit_parser.add_argument("name", help="name or uuid of the wim") + wim_edit_parser.add_argument("--file", + help="json/yaml text or file with the changes")\ + .completer = FilesCompleter + wim_edit_parser.add_argument("-f", "--force", action="store_true", + help="do not prompt for confirmation") + wim_edit_parser.set_defaults(func=wim_edit) + + # WIM list + wim_list_parser = subparsers.add_parser('wim-list', + parents=[parent_parser], + help="lists information about registered wims") + wim_list_parser.add_argument("name", nargs='?', + help="name or uuid of the wim") + wim_list_parser.add_argument("-a", "--all", action="store_true", + help="shows all wims, not only wims attached to tenant") + wim_list_parser.set_defaults(func=wim_list) + + # WIM account create + wim_attach_parser = subparsers.add_parser('wim-account-create', parents= + [parent_parser], help="associates a wim account to the operating tenant") + wim_attach_parser.add_argument("name", help="name or uuid of the wim") + wim_attach_parser.add_argument('--account-name', action='store', + help="specify a name for the wim account.") + wim_attach_parser.add_argument("--user", action="store", + help="user credentials for the wim account") + wim_attach_parser.add_argument("--password", action="store", + help="password credentials for the wim account") + wim_attach_parser.add_argument("--config", action="store", + help="additional configuration in json/yaml format") + wim_attach_parser.set_defaults(func=wim_account_create) + + # WIM account delete + wim_detach_parser = subparsers.add_parser('wim-account-delete', + parents=[parent_parser], + help="removes the association " + "between a wim account and the operating tenant") + wim_detach_parser.add_argument("name", help="name or uuid of the wim") + wim_detach_parser.add_argument("-a", "--all", action="store_true", + help="removes all associations from this wim") + wim_detach_parser.add_argument("-f", "--force", action="store_true", + help="forces delete without asking") + wim_detach_parser.set_defaults(func=wim_account_delete) + + # WIM account edit + wim_attach_edit_parser = subparsers.add_parser('wim-account-edit', parents= + [parent_parser], help="modifies the association of a wim account to the operating tenant") + wim_attach_edit_parser.add_argument("name", help="name or uuid of the wim") + wim_attach_edit_parser.add_argument('--account-name', action='store', + help="specify a name for the wim account.") + wim_attach_edit_parser.add_argument("--user", action="store", + help="user credentials for the wim account") + wim_attach_edit_parser.add_argument("--password", action="store", + help="password credentials for the wim account") + wim_attach_edit_parser.add_argument("--config", action="store", + help="additional configuration in json/yaml format") + wim_attach_edit_parser.set_defaults(func=wim_account_edit) + + # WIM port mapping set + wim_port_mapping_set_parser = subparsers.add_parser('wim-port-mapping-set', + parents=[parent_parser], + help="Load a file with the mappings " + "of ports of a WAN switch that is " + "connected to a PoP and the ports " + "of the switch controlled by the PoP") + wim_port_mapping_set_parser.add_argument("name", action="store", + help="specifies the wim") + wim_port_mapping_set_parser.add_argument("file", + help="json/yaml text or file with the wim port mapping")\ + .completer = FilesCompleter + wim_port_mapping_set_parser.add_argument("-f", "--force", + action="store_true", help="forces overwriting without asking") + wim_port_mapping_set_parser.set_defaults(func=wim_port_mapping_set) + + # WIM port mapping list + wim_port_mapping_list_parser = subparsers.add_parser('wim-port-mapping-list', + parents=[parent_parser], help="Show the port mappings for a wim") + wim_port_mapping_list_parser.add_argument("name", action="store", + help="specifies the wim") + wim_port_mapping_list_parser.set_defaults(func=wim_port_mapping_list) + + # WIM port mapping clear + wim_port_mapping_clear_parser = subparsers.add_parser('wim-port-mapping-clear', + parents=[parent_parser], help="Clean the port mapping in a wim") + wim_port_mapping_clear_parser.add_argument("name", action="store", + help="specifies the wim") + wim_port_mapping_clear_parser.add_argument("-f", "--force", + action="store_true", + help="forces clearing without asking") + wim_port_mapping_clear_parser.set_defaults(func=wim_port_mapping_clear) + + # ======================================================= + action_dict={'net-update': 'retrieves external networks from datacenter', 'net-edit': 'edits an external network', 'net-delete': 'deletes an external network', @@ -2060,7 +2468,7 @@ if __name__=="__main__": vim_item_create_parser.set_defaults(func=vim_action, item=item, action="create") argcomplete.autocomplete(main_parser) - + try: args = main_parser.parse_args() #logging info @@ -2086,7 +2494,7 @@ if __name__=="__main__": except OpenmanoCLIError as e: print str(e) result = -5 - + #print result exit(result) diff --git a/openmanod b/openmanod index 3fd1ae0c..4ebac750 100755 --- a/openmanod +++ b/openmanod @@ -27,7 +27,7 @@ openmano server. Main program that implements a reference NFVO (Network Functions Virtualisation Orchestrator). It interfaces with an NFV VIM through its API and offers a northbound interface, based on REST (openmano API), where NFV services are offered including the creation and deletion of VNF templates, VNF instances, -network service templates and network service instances. +network service templates and network service instances. It loads the configuration file and launches the http_server thread that will listen requests using openmano API. """ @@ -44,13 +44,15 @@ import socket from osm_ro import httpserver, nfvo, nfvo_db from osm_ro.openmano_schemas import config_schema from osm_ro.db_base import db_base_Exception +from osm_ro.wim.engine import WimEngine +from osm_ro.wim.persistence import WimPersistence import osm_ro __author__ = "Alfonso Tierno, Gerardo Garcia, Pablo Montes" __date__ = "$26-aug-2014 11:09:29$" -__version__ = "0.5.78-r588" +__version__ = "0.6.00" version_date = "Sep 2018" -database_version = 32 # expected database schema version +database_version = 34 # expected database schema version global global_config @@ -106,7 +108,7 @@ def load_configuration(configuration_file): def console_port_iterator(): - '''this iterator deals with the http_console_ports + '''this iterator deals with the http_console_ports returning the ports one by one ''' index = 0 @@ -287,7 +289,7 @@ if __name__=="__main__": logger.critical("Starting openmano server version: '%s %s' command: '%s'", __version__, version_date, " ".join(sys.argv)) - for log_module in ("nfvo", "http", "vim", "db", "console", "ovim"): + for log_module in ("nfvo", "http", "vim", "wim", "db", "console", "ovim"): log_level_module = "log_level_" + log_module log_file_module = "log_file_" + log_module logger_module = logging.getLogger('openmano.' + log_module) @@ -334,9 +336,18 @@ if __name__=="__main__": pass # if tenant exist (NfvoException error 409), ignore else: # otherwise print and error and continue logger.error("Cannot create tenant '{}': {}".format(create_tenant, e)) - nfvo.start_service(mydb) - httpthread = httpserver.httpserver(mydb, False, global_config['http_host'], global_config['http_port']) + # WIM module + wim_persistence = WimPersistence(mydb) + wim_engine = WimEngine(wim_persistence) + # --- + nfvo.start_service(mydb, wim_persistence, wim_engine) + + httpthread = httpserver.httpserver( + mydb, False, + global_config['http_host'], global_config['http_port'], + wim_persistence, wim_engine + ) httpthread.start() if 'http_admin_port' in global_config: diff --git a/osm_ro/db_base.py b/osm_ro/db_base.py index 3b12f745..2bfa4cde 100644 --- a/osm_ro/db_base.py +++ b/osm_ro/db_base.py @@ -37,14 +37,7 @@ import logging import datetime from jsonschema import validate as js_v, exceptions as js_e -HTTP_Bad_Request = 400 -HTTP_Unauthorized = 401 -HTTP_Not_Found = 404 -HTTP_Method_Not_Allowed = 405 -HTTP_Request_Timeout = 408 -HTTP_Conflict = 409 -HTTP_Service_Unavailable = 503 -HTTP_Internal_Server_Error = 500 +from .http_tools import errors as httperrors def _check_valid_uuid(uuid): id_schema = {"type" : "string", "pattern": "^[a-fA-F0-9]{8}(-[a-fA-F0-9]{4}){3}-[a-fA-F0-9]{12}$"} @@ -68,7 +61,7 @@ def _convert_datetime2str(var): for k,v in var.items(): if type(v) is datetime.datetime: var[k]= v.strftime('%Y-%m-%dT%H:%M:%S') - elif type(v) is dict or type(v) is list or type(v) is tuple: + elif type(v) is dict or type(v) is list or type(v) is tuple: _convert_datetime2str(v) if len(var) == 0: return True elif type(var) is list or type(var) is tuple: @@ -76,7 +69,7 @@ def _convert_datetime2str(var): _convert_datetime2str(v) def _convert_bandwidth(data, reverse=False, logger=None): - '''Check the field bandwidth recursivelly and when found, it removes units and convert to number + '''Check the field bandwidth recursivelly and when found, it removes units and convert to number It assumes that bandwidth is well formed Attributes: 'data': dictionary bottle.FormsDict variable to be checked. None or empty is consideted valid @@ -111,7 +104,7 @@ def _convert_bandwidth(data, reverse=False, logger=None): _convert_bandwidth(k, reverse, logger) def _convert_str2boolean(data, items): - '''Check recursively the content of data, and if there is an key contained in items, convert value from string to boolean + '''Check recursively the content of data, and if there is an key contained in items, convert value from string to boolean Done recursively Attributes: 'data': dictionary variable to be checked. None or empty is considered valid @@ -135,16 +128,15 @@ def _convert_str2boolean(data, items): if type(k) is dict or type(k) is tuple or type(k) is list: _convert_str2boolean(k, items) -class db_base_Exception(Exception): +class db_base_Exception(httperrors.HttpMappedError): '''Common Exception for all database exceptions''' - - def __init__(self, message, http_code=HTTP_Bad_Request): - Exception.__init__(self, message) - self.http_code = http_code + + def __init__(self, message, http_code=httperrors.Bad_Request): + super(db_base_Exception, self).__init__(message, http_code) class db_base(): tables_with_created_field=() - + def __init__(self, host=None, user=None, passwd=None, database=None, log_name='db', log_level=None): self.host = host self.user = user @@ -155,9 +147,9 @@ class db_base(): self.logger = logging.getLogger(log_name) if self.log_level: self.logger.setLevel( getattr(logging, log_level) ) - + def connect(self, host=None, user=None, passwd=None, database=None): - '''Connect to specific data base. + '''Connect to specific data base. The first time a valid host, user, passwd and database must be provided, Following calls can skip this parameters ''' @@ -172,8 +164,16 @@ class db_base(): except mdb.Error as e: raise db_base_Exception("Cannot connect to DataBase '{}' at '{}@{}' Error {}: {}".format( self.database, self.user, self.host, e.args[0], e.args[1]), - http_code = HTTP_Unauthorized ) - + http_code = httperrors.Unauthorized ) + + def escape(self, value): + return self.con.escape(value) + + + def escape_string(self, value): + return self.con.escape_string(value) + + def get_db_version(self): ''' Obtain the database schema version. Return: (negative, text) if error or version 0.0 where schema_version table is missing @@ -212,10 +212,10 @@ class db_base(): if e[0][-5:] == "'con'": self.logger.warn("while disconnecting from DB: Error %d: %s",e.args[0], e.args[1]) return - else: + else: raise - def _format_error(self, e, tries=1, command=None, extra=None): + def _format_error(self, e, tries=1, command=None, extra=None, table=None): '''Creates a text error base on the produced exception Params: e: mdb exception @@ -227,7 +227,8 @@ class db_base(): HTTP error in negative, formatted error text ''' if isinstance(e,AttributeError ): - raise db_base_Exception("DB Exception " + str(e), HTTP_Internal_Server_Error) + self.logger.debug(str(e), exc_info=True) + raise db_base_Exception("DB Exception " + str(e), httperrors.Internal_Server_Error) if e.args[0]==2006 or e.args[0]==2013 : #MySQL server has gone away (((or))) Exception 2013: Lost connection to MySQL server during query if tries>1: self.logger.warn("DB Exception '%s'. Retry", str(e)) @@ -235,32 +236,45 @@ class db_base(): self.connect() return else: - raise db_base_Exception("Database connection timeout Try Again", HTTP_Request_Timeout) - + raise db_base_Exception("Database connection timeout Try Again", httperrors.Request_Timeout) + fk=e.args[1].find("foreign key constraint fails") if fk>=0: if command=="update": - raise db_base_Exception("tenant_id '{}' not found.".format(extra), HTTP_Not_Found) + raise db_base_Exception("tenant_id '{}' not found.".format(extra), httperrors.Not_Found) elif command=="delete": - raise db_base_Exception("Resource is not free. There are {} that prevent deleting it.".format(extra), HTTP_Conflict) + raise db_base_Exception("Resource is not free. There are {} that prevent deleting it.".format(extra), httperrors.Conflict) de = e.args[1].find("Duplicate entry") fk = e.args[1].find("for key") uk = e.args[1].find("Unknown column") wc = e.args[1].find("in 'where clause'") fl = e.args[1].find("in 'field list'") #print de, fk, uk, wc,fl + table_info = ' (table `{}`)'.format(table) if table else '' if de>=0: if fk>=0: #error 1062 - raise db_base_Exception("Value {} already in use for {}".format(e.args[1][de+15:fk], e.args[1][fk+7:]), HTTP_Conflict) + raise db_base_Exception( + "Value {} already in use for {}{}".format( + e.args[1][de+15:fk], e.args[1][fk+7:], table_info), + httperrors.Conflict) if uk>=0: if wc>=0: - raise db_base_Exception("Field {} can not be used for filtering".format(e.args[1][uk+14:wc]), HTTP_Bad_Request) + raise db_base_Exception( + "Field {} can not be used for filtering{}".format( + e.args[1][uk+14:wc], table_info), + httperrors.Bad_Request) if fl>=0: - raise db_base_Exception("Field {} does not exist".format(e.args[1][uk+14:wc]), HTTP_Bad_Request) - raise db_base_Exception("Database internal Error {}: {}".format(e.args[0], e.args[1]), HTTP_Internal_Server_Error) - + raise db_base_Exception( + "Field {} does not exist{}".format( + e.args[1][uk+14:wc], table_info), + httperrors.Bad_Request) + raise db_base_Exception( + "Database internal Error{} {}: {}".format( + table_info, e.args[0], e.args[1]), + httperrors.Internal_Server_Error) + def __str2db_format(self, data): - '''Convert string data to database format. + '''Convert string data to database format. If data is None it returns the 'Null' text, otherwise it returns the text surrounded by quotes ensuring internal quotes are escaped. ''' @@ -270,10 +284,10 @@ class db_base(): return json.dumps(data) else: return json.dumps(str(data)) - + def __tuple2db_format_set(self, data): """Compose the needed text for a SQL SET, parameter 'data' is a pair tuple (A,B), - and it returns the text 'A="B"', where A is a field of a table and B is the value + and it returns the text 'A="B"', where A is a field of a table and B is the value If B is None it returns the 'A=Null' text, without surrounding Null by quotes If B is not None it returns the text "A='B'" or 'A="B"' where B is surrounded by quotes, and it ensures internal quotes of B are escaped. @@ -287,10 +301,10 @@ class db_base(): elif isinstance(data[1], dict): if "INCREMENT" in data[1]: return "{A}={A}{N:+d}".format(A=data[0], N=data[1]["INCREMENT"]) - raise db_base_Exception("Format error for UPDATE field") + raise db_base_Exception("Format error for UPDATE field: {!r}".format(data[0])) else: return str(data[0]) + '=' + json.dumps(str(data[1])) - + def __create_where(self, data, use_or=None): """ Compose the needed text for a SQL WHERE, parameter 'data' can be a dict or a list of dict. By default lists are @@ -347,9 +361,9 @@ class db_base(): '''remove single quotes ' of any string content of data dictionary''' for k,v in data.items(): if type(v) == str: - if "'" in v: + if "'" in v: data[k] = data[k].replace("'","_") - + def _update_rows(self, table, UPDATE, WHERE, modified_time=0): """ Update one or several rows of a table. :param UPDATE: dictionary with the changes. dict keys are database columns that will be set with the dict values @@ -369,7 +383,7 @@ class db_base(): values += ",modified_at={:f}".format(modified_time) cmd= "UPDATE " + table + " SET " + values + " WHERE " + self.__create_where(WHERE) self.logger.debug(cmd) - self.cur.execute(cmd) + self.cur.execute(cmd) return self.cur.rowcount def _new_uuid(self, root_uuid=None, used_table=None, created_time=0): @@ -398,7 +412,7 @@ class db_base(): def _new_row_internal(self, table, INSERT, add_uuid=False, root_uuid=None, created_time=0, confidential_data=False): ''' Add one row into a table. It DOES NOT begin or end the transaction, so self.con.cursor must be created - Attribute + Attribute INSERT: dictionary with the key:value to insert table: table where to insert add_uuid: if True, it will create an uuid key entry at INSERT if not provided @@ -411,7 +425,7 @@ class db_base(): #create uuid if not provided if 'uuid' not in INSERT: uuid = INSERT['uuid'] = str(myUuid.uuid1()) # create_uuid - else: + else: uuid = str(INSERT['uuid']) else: uuid=None @@ -429,7 +443,7 @@ class db_base(): self.cur.execute(cmd) #insertion cmd= "INSERT INTO " + table +" SET " + \ - ",".join(map(self.__tuple2db_format_set, INSERT.iteritems() )) + ",".join(map(self.__tuple2db_format_set, INSERT.iteritems() )) if created_time: cmd += ",created_at=%f" % created_time if confidential_data: @@ -448,16 +462,16 @@ class db_base(): self.cur.execute(cmd) rows = self.cur.fetchall() return rows - + def new_row(self, table, INSERT, add_uuid=False, created_time=0, confidential_data=False): ''' Add one row into a table. - Attribute + Attribute INSERT: dictionary with the key: value to insert table: table where to insert tenant_id: only useful for logs. If provided, logs will use this tenant_id add_uuid: if True, it will create an uuid key entry at INSERT if not provided It checks presence of uuid and add one automatically otherwise - Return: (result, uuid) where result can be 0 if error, or 1 if ok + Return: uuid ''' if table in self.tables_with_created_field and created_time==0: created_time=time.time() @@ -467,9 +481,9 @@ class db_base(): with self.con: self.cur = self.con.cursor() return self._new_row_internal(table, INSERT, add_uuid, None, created_time, confidential_data) - + except (mdb.Error, AttributeError) as e: - self._format_error(e, tries) + self._format_error(e, tries, table=table) tries -= 1 def update_rows(self, table, UPDATE, WHERE, modified_time=0): @@ -493,10 +507,10 @@ class db_base(): try: with self.con: self.cur = self.con.cursor() - return self._update_rows(table, UPDATE, WHERE) - + return self._update_rows( + table, UPDATE, WHERE, modified_time) except (mdb.Error, AttributeError) as e: - self._format_error(e, tries) + self._format_error(e, tries, table=table) tries -= 1 def _delete_row_by_id_internal(self, table, uuid): @@ -519,7 +533,8 @@ class db_base(): self.cur = self.con.cursor() return self._delete_row_by_id_internal(table, uuid) except (mdb.Error, AttributeError) as e: - self._format_error(e, tries, "delete", "dependencies") + self._format_error( + e, tries, "delete", "dependencies", table=table) tries -= 1 def delete_row(self, **sql_dict): @@ -567,9 +582,9 @@ class db_base(): rows = self.cur.fetchall() return rows except (mdb.Error, AttributeError) as e: - self._format_error(e, tries) + self._format_error(e, tries, table=table) tries -= 1 - + def get_rows(self, **sql_dict): """ Obtain rows from a table. :param SELECT: list or tuple of fields to retrieve) (by default all) @@ -581,7 +596,7 @@ class db_base(): keys can be suffixed by >,<,<>,>=,<= so that this is used to compare key and value instead of "=" The special keys "OR", "AND" with a dict value is used to create a nested WHERE If a list, each item will be a dictionary that will be concatenated with OR - :param LIMIT: limit the number of obtianied entries (Optional) + :param LIMIT: limit the number of obtained entries (Optional) :param ORDER_BY: list or tuple of fields to order, add ' DESC' to each item if inverse order is required :return: a list with dictionaries at each row, raises exception upon error """ @@ -628,10 +643,10 @@ class db_base(): Attribute: table: string of table name uuid_name: name or uuid. If not uuid format is found, it is considered a name - allow_severeral: if False return ERROR if more than one row are founded - error_item_text: in case of error it identifies the 'item' name for a proper output text + allow_severeral: if False return ERROR if more than one row are founded + error_item_text: in case of error it identifies the 'item' name for a proper output text 'WHERE_OR': dict of key:values, translated to key=value OR ... (Optional) - 'WHERE_AND_OR: str 'AND' or 'OR'(by default) mark the priority to 'WHERE AND (WHERE_OR)' or (WHERE) OR WHERE_OR' (Optional + 'WHERE_AND_OR: str 'AND' or 'OR'(by default) mark the priority to 'WHERE AND (WHERE_OR)' or (WHERE) OR WHERE_OR' (Optional Return: if allow_several==False, a dictionary with this row, or error if no item is found or more than one is found if allow_several==True, a list of dictionaries with the row or rows, error if no item is found ''' @@ -656,16 +671,16 @@ class db_base(): self.cur.execute(cmd) number = self.cur.rowcount if number == 0: - raise db_base_Exception("No {} found with {} '{}'".format(error_item_text, what, uuid_name), http_code=HTTP_Not_Found) + raise db_base_Exception("No {} found with {} '{}'".format(error_item_text, what, uuid_name), http_code=httperrors.Not_Found) elif number > 1 and not allow_serveral: - raise db_base_Exception("More than one {} found with {} '{}'".format(error_item_text, what, uuid_name), http_code=HTTP_Conflict) + raise db_base_Exception("More than one {} found with {} '{}'".format(error_item_text, what, uuid_name), http_code=httperrors.Conflict) if allow_serveral: rows = self.cur.fetchall() else: rows = self.cur.fetchone() return rows except (mdb.Error, AttributeError) as e: - self._format_error(e, tries) + self._format_error(e, tries, table=table) tries -= 1 def get_uuid(self, uuid): @@ -684,7 +699,7 @@ class db_base(): def get_uuid_from_name(self, table, name): '''Searchs in table the name and returns the uuid - ''' + ''' tries = 2 while tries: try: @@ -699,6 +714,6 @@ class db_base(): return self.cur.rowcount, "More than one VNF with name %s found in table %s" %(name, table) return self.cur.rowcount, rows[0]["uuid"] except (mdb.Error, AttributeError) as e: - self._format_error(e, tries) + self._format_error(e, tries, table=table) tries -= 1 diff --git a/osm_ro/http_tools/__init__.py b/osm_ro/http_tools/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/osm_ro/http_tools/errors.py b/osm_ro/http_tools/errors.py new file mode 100644 index 00000000..552e85b1 --- /dev/null +++ b/osm_ro/http_tools/errors.py @@ -0,0 +1,67 @@ +# -*- coding: utf-8 -*- +import logging +from functools import wraps + +import bottle +import yaml + +Bad_Request = 400 +Unauthorized = 401 +Not_Found = 404 +Forbidden = 403 +Method_Not_Allowed = 405 +Not_Acceptable = 406 +Request_Timeout = 408 +Conflict = 409 +Service_Unavailable = 503 +Internal_Server_Error = 500 + + +class HttpMappedError(Exception): + """Base class for a new hierarchy that translate HTTP error codes + to python exceptions + + This class accepts an extra argument ``http_code`` (integer + representing HTTP error codes). + """ + + def __init__(self, message, http_code=Internal_Server_Error): + Exception.__init__(self, message) + self.http_code = http_code + + +class ErrorHandler(object): + """Defines a default strategy for handling HttpMappedError. + + This class implements a wrapper (can also be used as decorator), that + watches out for different exceptions and log them accordingly. + + Arguments: + logger(logging.Logger): logger object to be used to report errors + """ + def __init__(self, logger=None): + self.logger = logger or logging.getLogger('openmano.http') + + def __call__(self, function): + @wraps(function) + def _wraped(*args, **kwargs): + try: + return function(*args, **kwargs) + except bottle.HTTPError: + raise + except HttpMappedError as ex: + self.logger.error( + "%s error %s", + function.__name__, ex.http_code, exc_info=True) + bottle.abort(ex.http_code, str(ex)) + except yaml.YAMLError as ex: + self.logger.error( + "YAML error while trying to serialize/unserialize fields", + exc_info=True) + bottle.abort(Bad_Request, type(ex).__name__ + ": " + str(ex)) + except Exception as ex: + self.logger.error("Unexpected exception: ", exc_info=True) + bottle.abort(Internal_Server_Error, + type(ex).__name__ + ": " + str(ex)) + + return _wraped diff --git a/osm_ro/http_tools/handler.py b/osm_ro/http_tools/handler.py new file mode 100644 index 00000000..49249a8c --- /dev/null +++ b/osm_ro/http_tools/handler.py @@ -0,0 +1,114 @@ +# -*- coding: utf-8 -*- +## +# Copyright 2018 University of Bristol - High Performance Networks Research +# Group +# All Rights Reserved. +# +# Contributors: Anderson Bravalheri, Dimitrios Gkounis, Abubakar Siddique +# Muqaddas, Navdeep Uniyal, Reza Nejabati and Dimitra Simeonidou +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +# +# For those usages not covered by the Apache License, Version 2.0 please +# contact with: +# +# Neither the name of the University of Bristol nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# This work has been performed in the context of DCMS UK 5G Testbeds +# & Trials Programme and in the framework of the Metro-Haul project - +# funded by the European Commission under Grant number 761727 through the +# Horizon 2020 and 5G-PPP programmes. +## + +from types import MethodType + +from bottle import Bottle + + +class route(object): + """Decorator that stores route information, so creating the routes can be + postponed. + + This allows methods (OOP) with bottle. + + Arguments: + method: HTTP verb (e.g. ``'get'``, ``'post'``, ``'put'``, ...) + path: URL path that will be handled by the callback + """ + def __init__(self, method, path, **kwargs): + kwargs['method'] = method.upper() + self.route_info = (path, kwargs) + + def __call__(self, function): + function.route_info = self.route_info + return function + + +class BaseHandler(object): + """Base class that allows isolated webapp implementation using Bottle, + when used in conjunction with the ``route`` decorator. + + In this context, a ``Handler`` is meant to be a collection of Bottle + routes/callbacks related to a specific topic. + + A ``Handler`` instance can produce a WSGI app that can be mounted or merged + inside another more general bottle app. + + Example: + + from http_tools.handler import Handler, route + from http_tools.errors import ErrorHandler + + class MyHandler(Handler): + plugins = [ErrorHandler()] + url_base = '/my/url/base' + + @route('GET', '/some/path/') + def get_var(self, var): + return var + + app = MyHandler.wsgi_app + # ^ Webapp with a `GET /my/url/base/some/path/` route + """ + _wsgi_app = None + + url_base = '' + """String representing a path fragment to be prepended to the routes""" + + plugins = [] + """Bottle plugins to be installed when creating the WSGI app""" + + @property + def wsgi_app(self): + """Create a WSGI app based on the implemented callbacks""" + + if self._wsgi_app: + # Return if cached + return self._wsgi_app + + app = Bottle() + + members = (getattr(self, m) for m in dir(self) if m != 'wsgi_app') + callbacks = (m for m in members + if isinstance(m, MethodType) and hasattr(m, 'route_info')) + + for callback in callbacks: + path, kwargs = callback.route_info + kwargs.update(callback=callback, apply=self.plugins) + app.route(self.url_base + path, **kwargs) + + self._wsgi_app = app + + return app diff --git a/osm_ro/http_tools/request_processing.py b/osm_ro/http_tools/request_processing.py new file mode 100644 index 00000000..0b8a6ca2 --- /dev/null +++ b/osm_ro/http_tools/request_processing.py @@ -0,0 +1,209 @@ +# -*- coding: utf-8 -*- + +# +# Util functions previously in `httpserver` +# + +__author__ = "Alfonso Tierno, Gerardo Garcia" + +import json +import logging + +import bottle +import yaml +from jsonschema import exceptions as js_e +from jsonschema import validate as js_v + +from . import errors as httperrors + +logger = logging.getLogger('openmano.http') + + +def remove_clear_passwd(data): + """ + Removes clear passwords from the data received + :param data: data with clear password + :return: data without the password information + """ + + passw = ['password: ', 'passwd: '] + + for pattern in passw: + init = data.find(pattern) + while init != -1: + end = data.find('\n', init) + data = data[:init] + '{}******'.format(pattern) + data[end:] + init += 1 + init = data.find(pattern, init) + return data + + +def change_keys_http2db(data, http_db, reverse=False): + '''Change keys of dictionary data acording to the key_dict values + This allow change from http interface names to database names. + When reverse is True, the change is otherwise + Attributes: + data: can be a dictionary or a list + http_db: is a dictionary with hhtp names as keys and database names as value + reverse: by default change is done from http api to database. + If True change is done otherwise. + Return: None, but data is modified''' + if type(data) is tuple or type(data) is list: + for d in data: + change_keys_http2db(d, http_db, reverse) + elif type(data) is dict or type(data) is bottle.FormsDict: + if reverse: + for k,v in http_db.items(): + if v in data: data[k]=data.pop(v) + else: + for k,v in http_db.items(): + if k in data: data[v]=data.pop(k) + + +def format_out(data): + '''Return string of dictionary data according to requested json, yaml, xml. + By default json + ''' + logger.debug("OUT: " + yaml.safe_dump(data, explicit_start=True, indent=4, default_flow_style=False, tags=False, encoding='utf-8', allow_unicode=True) ) + accept = bottle.request.headers.get('Accept') + if accept and 'application/yaml' in accept: + bottle.response.content_type='application/yaml' + return yaml.safe_dump( + data, explicit_start=True, indent=4, default_flow_style=False, + tags=False, encoding='utf-8', allow_unicode=True) #, canonical=True, default_style='"' + else: #by default json + bottle.response.content_type='application/json' + #return data #json no style + return json.dumps(data, indent=4) + "\n" + + +def format_in(default_schema, version_fields=None, version_dict_schema=None, confidential_data=False): + """ + Parse the content of HTTP request against a json_schema + + :param default_schema: The schema to be parsed by default + if no version field is found in the client data. + In None no validation is done + :param version_fields: If provided it contains a tuple or list with the + fields to iterate across the client data to obtain the version + :param version_dict_schema: It contains a dictionary with the version as key, + and json schema to apply as value. + It can contain a None as key, and this is apply + if the client data version does not match any key + :return: user_data, used_schema: if the data is successfully decoded and + matches the schema. + + Launch a bottle abort if fails + """ + #print "HEADERS :" + str(bottle.request.headers.items()) + try: + error_text = "Invalid header format " + format_type = bottle.request.headers.get('Content-Type', 'application/json') + if 'application/json' in format_type: + error_text = "Invalid json format " + #Use the json decoder instead of bottle decoder because it informs about the location of error formats with a ValueError exception + client_data = json.load(bottle.request.body) + #client_data = bottle.request.json() + elif 'application/yaml' in format_type: + error_text = "Invalid yaml format " + client_data = yaml.load(bottle.request.body) + elif 'application/xml' in format_type: + bottle.abort(501, "Content-Type: application/xml not supported yet.") + else: + logger.warning('Content-Type ' + str(format_type) + ' not supported.') + bottle.abort(httperrors.Not_Acceptable, 'Content-Type ' + str(format_type) + ' not supported.') + return + # if client_data == None: + # bottle.abort(httperrors.Bad_Request, "Content error, empty") + # return + if confidential_data: + logger.debug('IN: %s', remove_clear_passwd (yaml.safe_dump(client_data, explicit_start=True, indent=4, default_flow_style=False, + tags=False, encoding='utf-8', allow_unicode=True))) + else: + logger.debug('IN: %s', yaml.safe_dump(client_data, explicit_start=True, indent=4, default_flow_style=False, + tags=False, encoding='utf-8', allow_unicode=True) ) + # look for the client provider version + error_text = "Invalid content " + if not default_schema and not version_fields: + return client_data, None + client_version = None + used_schema = None + if version_fields != None: + client_version = client_data + for field in version_fields: + if field in client_version: + client_version = client_version[field] + else: + client_version=None + break + if client_version == None: + used_schema = default_schema + elif version_dict_schema != None: + if client_version in version_dict_schema: + used_schema = version_dict_schema[client_version] + elif None in version_dict_schema: + used_schema = version_dict_schema[None] + if used_schema==None: + bottle.abort(httperrors.Bad_Request, "Invalid schema version or missing version field") + + js_v(client_data, used_schema) + return client_data, used_schema + except (TypeError, ValueError, yaml.YAMLError) as exc: + error_text += str(exc) + logger.error(error_text, exc_info=True) + bottle.abort(httperrors.Bad_Request, error_text) + except js_e.ValidationError as exc: + logger.error( + "validate_in error, jsonschema exception", exc_info=True) + error_pos = "" + if len(exc.path)>0: error_pos=" at " + ":".join(map(json.dumps, exc.path)) + bottle.abort(httperrors.Bad_Request, error_text + exc.message + error_pos) + #except: + # bottle.abort(httperrors.Bad_Request, "Content error: Failed to parse Content-Type", error_pos) + # raise + +def filter_query_string(qs, http2db, allowed): + '''Process query string (qs) checking that contains only valid tokens for avoiding SQL injection + Attributes: + 'qs': bottle.FormsDict variable to be processed. None or empty is considered valid + 'http2db': dictionary with change from http API naming (dictionary key) to database naming(dictionary value) + 'allowed': list of allowed string tokens (API http naming). All the keys of 'qs' must be one of 'allowed' + Return: A tuple with the (select,where,limit) to be use in a database query. All of then transformed to the database naming + select: list of items to retrieve, filtered by query string 'field=token'. If no 'field' is present, allowed list is returned + where: dictionary with key, value, taken from the query string token=value. Empty if nothing is provided + limit: limit dictated by user with the query string 'limit'. 100 by default + abort if not permited, using bottel.abort + ''' + where={} + limit=100 + select=[] + #if type(qs) is not bottle.FormsDict: + # bottle.abort(httperrors.Internal_Server_Error, '!!!!!!!!!!!!!!invalid query string not a dictionary') + # #bottle.abort(httperrors.Internal_Server_Error, "call programmer") + for k in qs: + if k=='field': + select += qs.getall(k) + for v in select: + if v not in allowed: + bottle.abort(httperrors.Bad_Request, "Invalid query string at 'field="+v+"'") + elif k=='limit': + try: + limit=int(qs[k]) + except: + bottle.abort(httperrors.Bad_Request, "Invalid query string at 'limit="+qs[k]+"'") + else: + if k not in allowed: + bottle.abort(httperrors.Bad_Request, "Invalid query string at '"+k+"="+qs[k]+"'") + if qs[k]!="null": where[k]=qs[k] + else: where[k]=None + if len(select)==0: select += allowed + #change from http api to database naming + for i in range(0,len(select)): + k=select[i] + if http2db and k in http2db: + select[i] = http2db[k] + if http2db: + change_keys_http2db(where, http2db) + #print "filter_query_string", select,where,limit + + return select,where,limit diff --git a/osm_ro/http_tools/tests/__init__.py b/osm_ro/http_tools/tests/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/osm_ro/http_tools/tests/test_errors.py b/osm_ro/http_tools/tests/test_errors.py new file mode 100644 index 00000000..a968e768 --- /dev/null +++ b/osm_ro/http_tools/tests/test_errors.py @@ -0,0 +1,76 @@ +# -*- coding: utf-8 -*- +import unittest + +import bottle + +from .. import errors as httperrors +from ...tests.helpers import TestCaseWithLogging + + +class TestHttpErrors(TestCaseWithLogging): + def test_http_error_base(self): + # When an error code is passed as argument + ex = httperrors.HttpMappedError(http_code=1226324) + # then it should be set in the exception object + self.assertEqual(ex.http_code, 1226324) + # When an error code is not passed as argument + ex = httperrors.HttpMappedError() + # then the default error code (internal server error) should be used + self.assertEqual(ex.http_code, httperrors.Internal_Server_Error) + + def test_error_handler_should_log_unexpected_errors(self): + # Given a error handler wraps a function + error_handler = httperrors.ErrorHandler(self.logger) + + # and the function raises an unexpected error + @error_handler + def _throw(): + raise AttributeError('some error') + + # when the function is called + with self.assertRaises(bottle.HTTPError): + _throw() + logs = self.caplog.getvalue() + # then the exception should be contained by bottle + # and a proper message should be logged + assert "Unexpected exception:" in logs + + def test_error_handler_should_log_http_based_errors(self): + # Given a error handler wraps a function + error_handler = httperrors.ErrorHandler(self.logger) + + # and the function raises an error that is considered by the + # application + @error_handler + def _throw(): + raise httperrors.HttpMappedError(http_code=404) + + # when the function is called + with self.assertRaises(bottle.HTTPError): + _throw() + logs = self.caplog.getvalue() + # then the exception should be contained by bottle + # and a proper message should be logged + assert "_throw error 404" in logs + + def test_error_handler_should_ignore_bottle_errors(self): + # Given a error handler wraps a function + error_handler = httperrors.ErrorHandler(self.logger) + + # and the function raises an error that is considered by the + # application + exception = bottle.HTTPError() + + @error_handler + def _throw(): + raise exception + + # when the function is called + with self.assertRaises(bottle.HTTPError) as context: + _throw() + # then the exception should bypass the error handler + self.assertEqual(context.exception, exception) + + +if __name__ == '__main__': + unittest.main() diff --git a/osm_ro/http_tools/tests/test_handler.py b/osm_ro/http_tools/tests/test_handler.py new file mode 100644 index 00000000..af325450 --- /dev/null +++ b/osm_ro/http_tools/tests/test_handler.py @@ -0,0 +1,95 @@ +# -*- coding: utf-8 -*- +import unittest + +from mock import MagicMock, patch +from webtest import TestApp + +from .. import handler +from ..handler import BaseHandler, route + + +class TestIntegration(unittest.TestCase): + def test_wsgi_app(self): + # Given a Handler class that implements a route + some_plugin = MagicMock() + + class MyHandler(BaseHandler): + url_base = '/42' + plugins = [some_plugin] + + @route('get', '/some/path') + def callback(self): + return 'some content' + + route_mock = MagicMock() + with patch(handler.__name__+'.Bottle.route', route_mock): + # When we try to access wsgi_app for the first time + my_handler = MyHandler() + assert my_handler.wsgi_app + # then bottle.route should be called with the right arguments + route_mock.assert_called_once_with('/42/some/path', method='GET', + callback=my_handler.callback, + apply=[some_plugin]) + + # When we try to access wsgi_app for the second time + assert my_handler.wsgi_app + # then the result should be cached + # and bottle.route should not be called again + self.assertEqual(route_mock.call_count, 1) + + def test_route_created(self): + # Given a Handler class, as in the example documentation + class MyHandler(BaseHandler): + def __init__(self): + self.value = 42 + + @route('GET', '/some/path/') + def callback(self, param): + return '{} + {}'.format(self.value, param) + + # when this class is used to generate a webapp + app = TestApp(MyHandler().wsgi_app) + + # then the defined URLs should be available + response = app.get('/some/path/0') + self.assertEqual(response.status_code, 200) + # and the callbacks should have access to ``self`` + response.mustcontain('42 + 0') + + def test_url_base(self): + # Given a Handler class that allows url_base customization + class MyHandler(BaseHandler): + def __init__(self, url_base): + self.url_base = url_base + + @route('GET', '/some/path/') + def callback(self, param): + return param + + # when this class is used to generate a webapp + app = TestApp(MyHandler('/prefix').wsgi_app) + + # then the prefixed URLs should be available + response = app.get('/prefix/some/path/content') + self.assertEqual(response.status_code, 200) + response.mustcontain('content') + + def test_starting_param(self): + # Given a Handler class with a route beginning with a param + class MyHandler(BaseHandler): + @route('GET', '//some/path') + def callback(self, param): + return '**{}**'.format(param) + + # is used to generate a webapp + app = TestApp(MyHandler().wsgi_app) + + # when the defined URLs is accessed + response = app.get('/42/some/path') + # Then no error should happen + self.assertEqual(response.status_code, 200) + response.mustcontain('**42**') + + +if __name__ == '__main__': + unittest.main() diff --git a/osm_ro/http_tools/tox.ini b/osm_ro/http_tools/tox.ini new file mode 100644 index 00000000..43055c26 --- /dev/null +++ b/osm_ro/http_tools/tox.ini @@ -0,0 +1,49 @@ +# This tox file allows the devs to run unit tests only for this subpackage. +# In order to do so, cd into the directory and run `tox` + +[tox] +minversion = 1.8 +envlist = py27,py36,flake8,radon +skipsdist = True + +[testenv] +changedir = {toxinidir} +commands = + nosetests -d --with-coverage --cover-package=. {posargs:tests} +deps = + WebTest + bottle + coverage + mock + nose + six + PyYaml + +[testenv:flake8] +changedir = {toxinidir} +deps = flake8 +commands = flake8 {posargs:.} + +[testenv:radon] +changedir = {toxinidir} +deps = radon +commands = + radon cc --show-complexity --total-average {posargs:.} + radon mi -s {posargs:.} + +[coverage:run] +branch = True +source = {toxinidir} +omit = + tests + tests/* + */test_* + .tox/* + +[coverage:report] +show_missing = True + +[flake8] +exclude = + request_processing.py + .tox diff --git a/osm_ro/httpserver.py b/osm_ro/httpserver.py index b3b55b2f..11aee157 100644 --- a/osm_ro/httpserver.py +++ b/osm_ro/httpserver.py @@ -24,19 +24,16 @@ ''' HTTP server implementing the openmano API. It will answer to POST, PUT, GET methods in the appropriate URLs and will use the nfvo.py module to run the appropriate method. -Every YAML/JSON file is checked against a schema in openmano_schemas.py module. +Every YAML/JSON file is checked against a schema in openmano_schemas.py module. ''' __author__="Alfonso Tierno, Gerardo Garcia" __date__ ="$17-sep-2014 09:07:15$" import bottle import yaml -import json import threading -import time import logging -from jsonschema import validate as js_v, exceptions as js_e from openmano_schemas import vnfd_schema_v01, vnfd_schema_v02, \ nsd_schema_v01, nsd_schema_v02, nsd_schema_v03, scenario_edit_schema, \ scenario_action_schema, instance_scenario_action_schema, instance_scenario_create_schema_v01, \ @@ -45,6 +42,14 @@ from openmano_schemas import vnfd_schema_v01, vnfd_schema_v02, \ object_schema, netmap_new_schema, netmap_edit_schema, sdn_controller_schema, sdn_controller_edit_schema, \ sdn_port_mapping_schema, sdn_external_port_schema +from .http_tools import errors as httperrors +from .http_tools.request_processing import ( + format_out, + format_in, + filter_query_string +) +from .wim.http_handler import WimHandler + import nfvo import utils from db_base import db_base_Exception @@ -56,42 +61,6 @@ global logger url_base="/openmano" logger = None -HTTP_Bad_Request = 400 -HTTP_Unauthorized = 401 -HTTP_Not_Found = 404 -HTTP_Forbidden = 403 -HTTP_Method_Not_Allowed = 405 -HTTP_Not_Acceptable = 406 -HTTP_Service_Unavailable = 503 -HTTP_Internal_Server_Error= 500 - -def delete_nulls(var): - if type(var) is dict: - for k in var.keys(): - if var[k] is None: del var[k] - elif type(var[k]) is dict or type(var[k]) is list or type(var[k]) is tuple: - if delete_nulls(var[k]): del var[k] - if len(var) == 0: return True - elif type(var) is list or type(var) is tuple: - for k in var: - if type(k) is dict: delete_nulls(k) - if len(var) == 0: return True - return False - -def convert_datetime2str(var): - '''Converts a datetime variable to a string with the format '%Y-%m-%dT%H:%i:%s' - It enters recursively in the dict var finding this kind of variables - ''' - if type(var) is dict: - for k,v in var.items(): - if type(v) is float and k in ("created_at", "modified_at"): - var[k] = time.strftime("%Y-%m-%dT%H:%M:%S", time.localtime(v) ) - elif type(v) is dict or type(v) is list or type(v) is tuple: - convert_datetime2str(v) - if len(var) == 0: return True - elif type(var) is list or type(var) is tuple: - for v in var: - convert_datetime2str(v) def log_to_logger(fn): ''' @@ -102,15 +71,16 @@ def log_to_logger(fn): def _log_to_logger(*args, **kwargs): actual_response = fn(*args, **kwargs) # modify this to log exactly what you need: - logger.info('FROM %s %s %s %s' % (bottle.request.remote_addr, + logger.info('FROM %s %s %s %s', bottle.request.remote_addr, bottle.request.method, bottle.request.url, - bottle.response.status)) + bottle.response.status) return actual_response return _log_to_logger class httpserver(threading.Thread): - def __init__(self, db, admin=False, host='localhost', port=9090): + def __init__(self, db, admin=False, host='localhost', port=9090, + wim_persistence=None, wim_engine=None): #global url_base global mydb global logger @@ -127,186 +97,37 @@ class httpserver(threading.Thread): #self.url_preffix = 'http://' + host + ':' + str(port) + url_base mydb = db #self.first_usable_connection_index = 10 - #self.next_connection_index = self.first_usable_connection_index #The next connection index to be used + #self.next_connection_index = self.first_usable_connection_index #The next connection index to be used #Ensure that when the main program exits the thread will also exit + + self.handlers = [ + WimHandler(db, wim_persistence, wim_engine, url_base) + ] + self.daemon = True self.setDaemon(True) - - def run(self): + + def run(self, debug=False, quiet=True): bottle.install(log_to_logger) - bottle.run(host=self.host, port=self.port, debug=False, quiet=True) - + default_app = bottle.app() + + for handler in self.handlers: + default_app.merge(handler.wsgi_app) + + bottle.run(host=self.host, port=self.port, debug=debug, quiet=quiet) + + def run_bottle(db, host_='localhost', port_=9090): - '''used for launching in main thread, so that it can be debugged''' - global mydb - mydb = db - bottle.run(host=host_, port=port_, debug=True) #quiet=True - + '''Used for launching in main thread, so that it can be debugged''' + server = httpserver(db, host=host_, port=port_) + server.run(debug=True) # quiet=True + @bottle.route(url_base + '/', method='GET') def http_get(): - #print + #print return 'works' #TODO: to be completed -# -# Util functions -# - -def change_keys_http2db(data, http_db, reverse=False): - '''Change keys of dictionary data acording to the key_dict values - This allow change from http interface names to database names. - When reverse is True, the change is otherwise - Attributes: - data: can be a dictionary or a list - http_db: is a dictionary with hhtp names as keys and database names as value - reverse: by default change is done from http api to database. If True change is done otherwise - Return: None, but data is modified''' - if type(data) is tuple or type(data) is list: - for d in data: - change_keys_http2db(d, http_db, reverse) - elif type(data) is dict or type(data) is bottle.FormsDict: - if reverse: - for k,v in http_db.items(): - if v in data: data[k]=data.pop(v) - else: - for k,v in http_db.items(): - if k in data: data[v]=data.pop(k) - -def format_out(data): - '''return string of dictionary data according to requested json, yaml, xml. By default json''' - logger.debug("OUT: " + yaml.safe_dump(data, explicit_start=True, indent=4, default_flow_style=False, tags=False, encoding='utf-8', allow_unicode=True) ) - if 'application/yaml' in bottle.request.headers.get('Accept'): - bottle.response.content_type='application/yaml' - return yaml.safe_dump(data, explicit_start=True, indent=4, default_flow_style=False, tags=False, encoding='utf-8', allow_unicode=True) #, canonical=True, default_style='"' - else: #by default json - bottle.response.content_type='application/json' - #return data #json no style - return json.dumps(data, indent=4) + "\n" - -def format_in(default_schema, version_fields=None, version_dict_schema=None, confidential_data=False): - """ - Parse the content of HTTP request against a json_schema - :param default_schema: The schema to be parsed by default if no version field is found in the client data. In None - no validation is done - :param version_fields: If provided it contains a tuple or list with the fields to iterate across the client data to - obtain the version - :param version_dict_schema: It contains a dictionary with the version as key, and json schema to apply as value. - It can contain a None as key, and this is apply if the client data version does not match any key - :return: user_data, used_schema: if the data is successfully decoded and matches the schema. - Launch a bottle abort if fails - """ - #print "HEADERS :" + str(bottle.request.headers.items()) - try: - error_text = "Invalid header format " - format_type = bottle.request.headers.get('Content-Type', 'application/json') - if 'application/json' in format_type: - error_text = "Invalid json format " - #Use the json decoder instead of bottle decoder because it informs about the location of error formats with a ValueError exception - client_data = json.load(bottle.request.body) - #client_data = bottle.request.json() - elif 'application/yaml' in format_type: - error_text = "Invalid yaml format " - client_data = yaml.load(bottle.request.body) - elif 'application/xml' in format_type: - bottle.abort(501, "Content-Type: application/xml not supported yet.") - else: - logger.warning('Content-Type ' + str(format_type) + ' not supported.') - bottle.abort(HTTP_Not_Acceptable, 'Content-Type ' + str(format_type) + ' not supported.') - return - # if client_data == None: - # bottle.abort(HTTP_Bad_Request, "Content error, empty") - # return - if confidential_data: - logger.debug('IN: %s', remove_clear_passwd (yaml.safe_dump(client_data, explicit_start=True, indent=4, default_flow_style=False, - tags=False, encoding='utf-8', allow_unicode=True))) - else: - logger.debug('IN: %s', yaml.safe_dump(client_data, explicit_start=True, indent=4, default_flow_style=False, - tags=False, encoding='utf-8', allow_unicode=True) ) - # look for the client provider version - error_text = "Invalid content " - if not default_schema and not version_fields: - return client_data, None - client_version = None - used_schema = None - if version_fields != None: - client_version = client_data - for field in version_fields: - if field in client_version: - client_version = client_version[field] - else: - client_version=None - break - if client_version == None: - used_schema = default_schema - elif version_dict_schema != None: - if client_version in version_dict_schema: - used_schema = version_dict_schema[client_version] - elif None in version_dict_schema: - used_schema = version_dict_schema[None] - if used_schema==None: - bottle.abort(HTTP_Bad_Request, "Invalid schema version or missing version field") - - js_v(client_data, used_schema) - return client_data, used_schema - except (TypeError, ValueError, yaml.YAMLError) as exc: - error_text += str(exc) - logger.error(error_text) - bottle.abort(HTTP_Bad_Request, error_text) - except js_e.ValidationError as exc: - logger.error("validate_in error, jsonschema exception at '%s' '%s' ", str(exc.path), str(exc.message)) - error_pos = "" - if len(exc.path)>0: error_pos=" at " + ":".join(map(json.dumps, exc.path)) - bottle.abort(HTTP_Bad_Request, error_text + exc.message + error_pos) - #except: - # bottle.abort(HTTP_Bad_Request, "Content error: Failed to parse Content-Type", error_pos) - # raise - -def filter_query_string(qs, http2db, allowed): - '''Process query string (qs) checking that contains only valid tokens for avoiding SQL injection - Attributes: - 'qs': bottle.FormsDict variable to be processed. None or empty is considered valid - 'http2db': dictionary with change from http API naming (dictionary key) to database naming(dictionary value) - 'allowed': list of allowed string tokens (API http naming). All the keys of 'qs' must be one of 'allowed' - Return: A tuple with the (select,where,limit) to be use in a database query. All of then transformed to the database naming - select: list of items to retrieve, filtered by query string 'field=token'. If no 'field' is present, allowed list is returned - where: dictionary with key, value, taken from the query string token=value. Empty if nothing is provided - limit: limit dictated by user with the query string 'limit'. 100 by default - abort if not permited, using bottel.abort - ''' - where={} - limit=100 - select=[] - #if type(qs) is not bottle.FormsDict: - # bottle.abort(HTTP_Internal_Server_Error, '!!!!!!!!!!!!!!invalid query string not a dictionary') - # #bottle.abort(HTTP_Internal_Server_Error, "call programmer") - for k in qs: - if k=='field': - select += qs.getall(k) - for v in select: - if v not in allowed: - bottle.abort(HTTP_Bad_Request, "Invalid query string at 'field="+v+"'") - elif k=='limit': - try: - limit=int(qs[k]) - except: - bottle.abort(HTTP_Bad_Request, "Invalid query string at 'limit="+qs[k]+"'") - else: - if k not in allowed: - bottle.abort(HTTP_Bad_Request, "Invalid query string at '"+k+"="+qs[k]+"'") - if qs[k]!="null": where[k]=qs[k] - else: where[k]=None - if len(select)==0: select += allowed - #change from http api to database naming - for i in range(0,len(select)): - k=select[i] - if http2db and k in http2db: - select[i] = http2db[k] - if http2db: - change_keys_http2db(where, http2db) - #print "filter_query_string", select,where,limit - - return select,where,limit - @bottle.hook('after_request') def enable_cors(): '''Don't know yet if really needed. Keep it just in case''' @@ -327,7 +148,7 @@ def http_get_tenants(): try: tenants = mydb.get_rows(FROM='nfvo_tenants', SELECT=select_,WHERE=where_,LIMIT=limit_) #change_keys_http2db(content, http2db_tenant, reverse=True) - convert_datetime2str(tenants) + utils.convert_float_timestamp2str(tenants) data={'tenants' : tenants} return format_out(data) except bottle.HTTPError: @@ -337,7 +158,7 @@ def http_get_tenants(): bottle.abort(e.http_code, str(e)) except Exception as e: logger.error("Unexpected exception: ", exc_info=True) - bottle.abort(HTTP_Internal_Server_Error, type(e).__name__ + ": " + str(e)) + bottle.abort(httperrors.Internal_Server_Error, type(e).__name__ + ": " + str(e)) @bottle.route(url_base + '/tenants/', method='GET') @@ -354,10 +175,10 @@ def http_get_tenant_id(tenant_id): tenants = mydb.get_rows(FROM=from_, SELECT=select_,WHERE=where_) #change_keys_http2db(content, http2db_tenant, reverse=True) if len(tenants) == 0: - bottle.abort(HTTP_Not_Found, "No tenant found with {}='{}'".format(what, tenant_id)) + bottle.abort(httperrors.Not_Found, "No tenant found with {}='{}'".format(what, tenant_id)) elif len(tenants) > 1: - bottle.abort(HTTP_Bad_Request, "More than one tenant found with {}='{}'".format(what, tenant_id)) - convert_datetime2str(tenants[0]) + bottle.abort(httperrors.Bad_Request, "More than one tenant found with {}='{}'".format(what, tenant_id)) + utils.convert_float_timestamp2str(tenants[0]) data = {'tenant': tenants[0]} return format_out(data) except bottle.HTTPError: @@ -367,7 +188,7 @@ def http_get_tenant_id(tenant_id): bottle.abort(e.http_code, str(e)) except Exception as e: logger.error("Unexpected exception: ", exc_info=True) - bottle.abort(HTTP_Internal_Server_Error, type(e).__name__ + ": " + str(e)) + bottle.abort(httperrors.Internal_Server_Error, type(e).__name__ + ": " + str(e)) @bottle.route(url_base + '/tenants', method='POST') @@ -379,7 +200,7 @@ def http_post_tenants(): r = utils.remove_extra_items(http_content, tenant_schema) if r: logger.debug("Remove received extra items %s", str(r)) - try: + try: data = nfvo.new_tenant(mydb, http_content['tenant']) return http_get_tenant_id(data) except bottle.HTTPError: @@ -389,7 +210,7 @@ def http_post_tenants(): bottle.abort(e.http_code, str(e)) except Exception as e: logger.error("Unexpected exception: ", exc_info=True) - bottle.abort(HTTP_Internal_Server_Error, type(e).__name__ + ": " + str(e)) + bottle.abort(httperrors.Internal_Server_Error, type(e).__name__ + ": " + str(e)) @bottle.route(url_base + '/tenants/', method='PUT') @@ -401,11 +222,11 @@ def http_edit_tenant_id(tenant_id): r = utils.remove_extra_items(http_content, tenant_edit_schema) if r: logger.debug("Remove received extra items %s", str(r)) - + #obtain data, check that only one exist - try: + try: tenant = mydb.get_table_by_uuid_name('nfvo_tenants', tenant_id) - #edit data + #edit data tenant_id = tenant['uuid'] where={'uuid': tenant['uuid']} mydb.update_rows('nfvo_tenants', http_content['tenant'], where) @@ -417,7 +238,7 @@ def http_edit_tenant_id(tenant_id): bottle.abort(e.http_code, str(e)) except Exception as e: logger.error("Unexpected exception: ", exc_info=True) - bottle.abort(HTTP_Internal_Server_Error, type(e).__name__ + ": " + str(e)) + bottle.abort(httperrors.Internal_Server_Error, type(e).__name__ + ": " + str(e)) @bottle.route(url_base + '/tenants/', method='DELETE') @@ -434,7 +255,7 @@ def http_delete_tenant_id(tenant_id): bottle.abort(e.http_code, str(e)) except Exception as e: logger.error("Unexpected exception: ", exc_info=True) - bottle.abort(HTTP_Internal_Server_Error, type(e).__name__ + ": " + str(e)) + bottle.abort(httperrors.Internal_Server_Error, type(e).__name__ + ": " + str(e)) @bottle.route(url_base + '//datacenters', method='GET') @@ -458,7 +279,7 @@ def http_get_datacenters(tenant_id): datacenters = mydb.get_rows(FROM='datacenters', SELECT=select_,WHERE=where_,LIMIT=limit_) #change_keys_http2db(content, http2db_tenant, reverse=True) - convert_datetime2str(datacenters) + utils.convert_float_timestamp2str(datacenters) data={'datacenters' : datacenters} return format_out(data) except bottle.HTTPError: @@ -468,7 +289,7 @@ def http_get_datacenters(tenant_id): bottle.abort(e.http_code, str(e)) except Exception as e: logger.error("Unexpected exception: ", exc_info=True) - bottle.abort(HTTP_Internal_Server_Error, type(e).__name__ + ": " + str(e)) + bottle.abort(httperrors.Internal_Server_Error, type(e).__name__ + ": " + str(e)) @bottle.route(url_base + '//vim_accounts', method='GET') @@ -542,11 +363,11 @@ def http_get_datacenter_id(tenant_id, datacenter_id): SELECT=select_, FROM=from_, WHERE=where_) - + if len(datacenters)==0: - bottle.abort( HTTP_Not_Found, "No datacenter found for tenant with {} '{}'".format(what, datacenter_id) ) - elif len(datacenters)>1: - bottle.abort( HTTP_Bad_Request, "More than one datacenter found for tenant with {} '{}'".format(what, datacenter_id) ) + bottle.abort( httperrors.Not_Found, "No datacenter found for tenant with {} '{}'".format(what, datacenter_id) ) + elif len(datacenters)>1: + bottle.abort( httperrors.Bad_Request, "More than one datacenter found for tenant with {} '{}'".format(what, datacenter_id) ) datacenter = datacenters[0] if tenant_id != 'any': #get vim tenant info @@ -586,7 +407,7 @@ def http_get_datacenter_id(tenant_id, datacenter_id): except Exception as e: logger.error("Exception '%s' while trying to load config information", str(e)) #change_keys_http2db(content, http2db_datacenter, reverse=True) - convert_datetime2str(datacenter) + utils.convert_float_timestamp2str(datacenter) data={'datacenter' : datacenter} return format_out(data) except bottle.HTTPError: @@ -596,7 +417,7 @@ def http_get_datacenter_id(tenant_id, datacenter_id): bottle.abort(e.http_code, str(e)) except Exception as e: logger.error("Unexpected exception: ", exc_info=True) - bottle.abort(HTTP_Internal_Server_Error, type(e).__name__ + ": " + str(e)) + bottle.abort(httperrors.Internal_Server_Error, type(e).__name__ + ": " + str(e)) @bottle.route(url_base + '/datacenters', method='POST') @@ -618,7 +439,7 @@ def http_post_datacenters(): bottle.abort(e.http_code, str(e)) except Exception as e: logger.error("Unexpected exception: ", exc_info=True) - bottle.abort(HTTP_Internal_Server_Error, type(e).__name__ + ": " + str(e)) + bottle.abort(httperrors.Internal_Server_Error, type(e).__name__ + ": " + str(e)) @bottle.route(url_base + '/datacenters/', method='PUT') @@ -630,7 +451,7 @@ def http_edit_datacenter_id(datacenter_id_name): r = utils.remove_extra_items(http_content, datacenter_edit_schema) if r: logger.debug("Remove received extra items %s", str(r)) - + try: datacenter_id = nfvo.edit_datacenter(mydb, datacenter_id_name, http_content['datacenter']) return http_get_datacenter_id('any', datacenter_id) @@ -641,7 +462,7 @@ def http_edit_datacenter_id(datacenter_id_name): bottle.abort(e.http_code, str(e)) except Exception as e: logger.error("Unexpected exception: ", exc_info=True) - bottle.abort(HTTP_Internal_Server_Error, type(e).__name__ + ": " + str(e)) + bottle.abort(httperrors.Internal_Server_Error, type(e).__name__ + ": " + str(e)) @bottle.route(url_base + '//sdn_controllers', method='POST') def http_post_sdn_controller(tenant_id): @@ -662,7 +483,7 @@ def http_post_sdn_controller(tenant_id): bottle.abort(e.http_code, str(e)) except Exception as e: logger.error("Unexpected exception: ", exc_info=True) - bottle.abort(HTTP_Internal_Server_Error, type(e).__name__ + ": " + str(e)) + bottle.abort(httperrors.Internal_Server_Error, type(e).__name__ + ": " + str(e)) @bottle.route(url_base + '//sdn_controllers/', method='PUT') def http_put_sdn_controller_update(tenant_id, controller_id): @@ -687,7 +508,7 @@ def http_put_sdn_controller_update(tenant_id, controller_id): bottle.abort(e.http_code, str(e)) except Exception as e: logger.error("Unexpected exception: ", exc_info=True) - bottle.abort(HTTP_Internal_Server_Error, type(e).__name__ + ": " + str(e)) + bottle.abort(httperrors.Internal_Server_Error, type(e).__name__ + ": " + str(e)) @bottle.route(url_base + '//sdn_controllers', method='GET') def http_get_sdn_controller(tenant_id): @@ -704,7 +525,7 @@ def http_get_sdn_controller(tenant_id): bottle.abort(e.http_code, str(e)) except Exception as e: logger.error("Unexpected exception: ", exc_info=True) - bottle.abort(HTTP_Internal_Server_Error, type(e).__name__ + ": " + str(e)) + bottle.abort(httperrors.Internal_Server_Error, type(e).__name__ + ": " + str(e)) @bottle.route(url_base + '//sdn_controllers/', method='GET') def http_get_sdn_controller_id(tenant_id, controller_id): @@ -720,7 +541,7 @@ def http_get_sdn_controller_id(tenant_id, controller_id): bottle.abort(e.http_code, str(e)) except Exception as e: logger.error("Unexpected exception: ", exc_info=True) - bottle.abort(HTTP_Internal_Server_Error, type(e).__name__ + ": " + str(e)) + bottle.abort(httperrors.Internal_Server_Error, type(e).__name__ + ": " + str(e)) @bottle.route(url_base + '//sdn_controllers/', method='DELETE') def http_delete_sdn_controller_id(tenant_id, controller_id): @@ -736,7 +557,7 @@ def http_delete_sdn_controller_id(tenant_id, controller_id): bottle.abort(e.http_code, str(e)) except Exception as e: logger.error("Unexpected exception: ", exc_info=True) - bottle.abort(HTTP_Internal_Server_Error, type(e).__name__ + ": " + str(e)) + bottle.abort(httperrors.Internal_Server_Error, type(e).__name__ + ": " + str(e)) @bottle.route(url_base + '//datacenters//sdn_mapping', method='POST') def http_post_datacenter_sdn_port_mapping(tenant_id, datacenter_id): @@ -757,7 +578,7 @@ def http_post_datacenter_sdn_port_mapping(tenant_id, datacenter_id): bottle.abort(e.http_code, str(e)) except Exception as e: logger.error("Unexpected exception: ", exc_info=True) - bottle.abort(HTTP_Internal_Server_Error, type(e).__name__ + ": " + str(e)) + bottle.abort(httperrors.Internal_Server_Error, type(e).__name__ + ": " + str(e)) @bottle.route(url_base + '//datacenters//sdn_mapping', method='GET') def http_get_datacenter_sdn_port_mapping(tenant_id, datacenter_id): @@ -774,7 +595,7 @@ def http_get_datacenter_sdn_port_mapping(tenant_id, datacenter_id): bottle.abort(e.http_code, str(e)) except Exception as e: logger.error("Unexpected exception: ", exc_info=True) - bottle.abort(HTTP_Internal_Server_Error, type(e).__name__ + ": " + str(e)) + bottle.abort(httperrors.Internal_Server_Error, type(e).__name__ + ": " + str(e)) @bottle.route(url_base + '//datacenters//sdn_mapping', method='DELETE') def http_delete_datacenter_sdn_port_mapping(tenant_id, datacenter_id): @@ -790,7 +611,7 @@ def http_delete_datacenter_sdn_port_mapping(tenant_id, datacenter_id): bottle.abort(e.http_code, str(e)) except Exception as e: logger.error("Unexpected exception: ", exc_info=True) - bottle.abort(HTTP_Internal_Server_Error, type(e).__name__ + ": " + str(e)) + bottle.abort(httperrors.Internal_Server_Error, type(e).__name__ + ": " + str(e)) @bottle.route(url_base + '//datacenters//networks', method='GET') #deprecated @bottle.route(url_base + '//datacenters//netmaps', method='GET') @@ -800,7 +621,7 @@ def http_getnetmap_datacenter_id(tenant_id, datacenter_id, netmap_id=None): logger.debug('FROM %s %s %s', bottle.request.remote_addr, bottle.request.method, bottle.request.url) #obtain data try: - datacenter_dict = mydb.get_table_by_uuid_name('datacenters', datacenter_id, "datacenter") + datacenter_dict = mydb.get_table_by_uuid_name('datacenters', datacenter_id, "datacenter") where_= {"datacenter_id":datacenter_dict['uuid']} if netmap_id: if utils.check_valid_uuid(netmap_id): @@ -809,14 +630,14 @@ def http_getnetmap_datacenter_id(tenant_id, datacenter_id, netmap_id=None): where_["name"] = netmap_id netmaps =mydb.get_rows(FROM='datacenter_nets', SELECT=('name','vim_net_id as vim_id', 'uuid', 'type','multipoint','shared','description', 'created_at'), - WHERE=where_ ) - convert_datetime2str(netmaps) + WHERE=where_ ) + utils.convert_float_timestamp2str(netmaps) utils.convert_str2boolean(netmaps, ('shared', 'multipoint') ) if netmap_id and len(netmaps)==1: data={'netmap' : netmaps[0]} elif netmap_id and len(netmaps)==0: - bottle.abort(HTTP_Not_Found, "No netmap found with " + " and ".join(map(lambda x: str(x[0])+": "+str(x[1]), where_.iteritems())) ) - return + bottle.abort(httperrors.Not_Found, "No netmap found with " + " and ".join(map(lambda x: str(x[0])+": "+str(x[1]), where_.iteritems())) ) + return else: data={'netmaps' : netmaps} return format_out(data) @@ -827,7 +648,7 @@ def http_getnetmap_datacenter_id(tenant_id, datacenter_id, netmap_id=None): bottle.abort(e.http_code, str(e)) except Exception as e: logger.error("Unexpected exception: ", exc_info=True) - bottle.abort(HTTP_Internal_Server_Error, type(e).__name__ + ": " + str(e)) + bottle.abort(httperrors.Internal_Server_Error, type(e).__name__ + ": " + str(e)) @bottle.route(url_base + '//datacenters//netmaps', method='DELETE') @@ -837,7 +658,7 @@ def http_delnetmap_datacenter_id(tenant_id, datacenter_id, netmap_id=None): logger.debug('FROM %s %s %s', bottle.request.remote_addr, bottle.request.method, bottle.request.url) #obtain data try: - datacenter_dict = mydb.get_table_by_uuid_name('datacenters', datacenter_id, "datacenter") + datacenter_dict = mydb.get_table_by_uuid_name('datacenters', datacenter_id, "datacenter") where_= {"datacenter_id":datacenter_dict['uuid']} if netmap_id: if utils.check_valid_uuid(netmap_id): @@ -845,9 +666,9 @@ def http_delnetmap_datacenter_id(tenant_id, datacenter_id, netmap_id=None): else: where_["name"] = netmap_id #change_keys_http2db(content, http2db_tenant, reverse=True) - deleted = mydb.delete_row(FROM='datacenter_nets', WHERE= where_) + deleted = mydb.delete_row(FROM='datacenter_nets', WHERE= where_) if deleted == 0 and netmap_id: - bottle.abort(HTTP_Not_Found, "No netmap found with " + " and ".join(map(lambda x: str(x[0])+": "+str(x[1]), where_.iteritems())) ) + bottle.abort(httperrors.Not_Found, "No netmap found with " + " and ".join(map(lambda x: str(x[0])+": "+str(x[1]), where_.iteritems())) ) if netmap_id: return format_out({"result": "netmap %s deleted" % netmap_id}) else: @@ -859,7 +680,7 @@ def http_delnetmap_datacenter_id(tenant_id, datacenter_id, netmap_id=None): bottle.abort(e.http_code, str(e)) except Exception as e: logger.error("Unexpected exception: ", exc_info=True) - bottle.abort(HTTP_Internal_Server_Error, type(e).__name__ + ": " + str(e)) + bottle.abort(httperrors.Internal_Server_Error, type(e).__name__ + ": " + str(e)) @bottle.route(url_base + '//datacenters//netmaps/upload', method='POST') @@ -867,7 +688,7 @@ def http_uploadnetmap_datacenter_id(tenant_id, datacenter_id): logger.debug('FROM %s %s %s', bottle.request.remote_addr, bottle.request.method, bottle.request.url) try: netmaps = nfvo.datacenter_new_netmap(mydb, tenant_id, datacenter_id, None) - convert_datetime2str(netmaps) + utils.convert_float_timestamp2str(netmaps) utils.convert_str2boolean(netmaps, ('shared', 'multipoint') ) data={'netmaps' : netmaps} return format_out(data) @@ -878,7 +699,7 @@ def http_uploadnetmap_datacenter_id(tenant_id, datacenter_id): bottle.abort(e.http_code, str(e)) except Exception as e: logger.error("Unexpected exception: ", exc_info=True) - bottle.abort(HTTP_Internal_Server_Error, type(e).__name__ + ": " + str(e)) + bottle.abort(httperrors.Internal_Server_Error, type(e).__name__ + ": " + str(e)) @bottle.route(url_base + '//datacenters//netmaps', method='POST') @@ -893,7 +714,7 @@ def http_postnetmap_datacenter_id(tenant_id, datacenter_id): try: #obtain data, check that only one exist netmaps = nfvo.datacenter_new_netmap(mydb, tenant_id, datacenter_id, http_content) - convert_datetime2str(netmaps) + utils.convert_float_timestamp2str(netmaps) utils.convert_str2boolean(netmaps, ('shared', 'multipoint') ) data={'netmaps' : netmaps} return format_out(data) @@ -904,7 +725,7 @@ def http_postnetmap_datacenter_id(tenant_id, datacenter_id): bottle.abort(e.http_code, str(e)) except Exception as e: logger.error("Unexpected exception: ", exc_info=True) - bottle.abort(HTTP_Internal_Server_Error, type(e).__name__ + ": " + str(e)) + bottle.abort(httperrors.Internal_Server_Error, type(e).__name__ + ": " + str(e)) @bottle.route(url_base + '//datacenters//netmaps/', method='PUT') @@ -916,7 +737,7 @@ def http_putnettmap_datacenter_id(tenant_id, datacenter_id, netmap_id): r = utils.remove_extra_items(http_content, netmap_edit_schema) if r: logger.debug("Remove received extra items %s", str(r)) - + #obtain data, check that only one exist try: nfvo.datacenter_edit_netmap(mydb, tenant_id, datacenter_id, netmap_id, http_content) @@ -928,8 +749,8 @@ def http_putnettmap_datacenter_id(tenant_id, datacenter_id, netmap_id): bottle.abort(e.http_code, str(e)) except Exception as e: logger.error("Unexpected exception: ", exc_info=True) - bottle.abort(HTTP_Internal_Server_Error, type(e).__name__ + ": " + str(e)) - + bottle.abort(httperrors.Internal_Server_Error, type(e).__name__ + ": " + str(e)) + @bottle.route(url_base + '//datacenters//action', method='POST') def http_action_datacenter_id(tenant_id, datacenter_id): @@ -954,13 +775,13 @@ def http_action_datacenter_id(tenant_id, datacenter_id): bottle.abort(e.http_code, str(e)) except Exception as e: logger.error("Unexpected exception: ", exc_info=True) - bottle.abort(HTTP_Internal_Server_Error, type(e).__name__ + ": " + str(e)) + bottle.abort(httperrors.Internal_Server_Error, type(e).__name__ + ": " + str(e)) @bottle.route(url_base + '/datacenters/', method='DELETE') def http_delete_datacenter_id( datacenter_id): '''delete a tenant from database, can use both uuid or name''' - + logger.debug('FROM %s %s %s', bottle.request.remote_addr, bottle.request.method, bottle.request.url) try: data = nfvo.delete_datacenter(mydb, datacenter_id) @@ -972,7 +793,7 @@ def http_delete_datacenter_id( datacenter_id): bottle.abort(e.http_code, str(e)) except Exception as e: logger.error("Unexpected exception: ", exc_info=True) - bottle.abort(HTTP_Internal_Server_Error, type(e).__name__ + ": " + str(e)) + bottle.abort(httperrors.Internal_Server_Error, type(e).__name__ + ": " + str(e)) @bottle.route(url_base + '//datacenters/', method='POST') @@ -996,7 +817,7 @@ def http_associate_datacenters(tenant_id, datacenter_id=None): bottle.abort(e.http_code, str(e)) except Exception as e: logger.error("Unexpected exception: ", exc_info=True) - bottle.abort(HTTP_Internal_Server_Error, type(e).__name__ + ": " + str(e)) + bottle.abort(httperrors.Internal_Server_Error, type(e).__name__ + ": " + str(e)) @bottle.route(url_base + '//vim_accounts/', method='PUT') @bottle.route(url_base + '//datacenters/', method='PUT') @@ -1019,7 +840,7 @@ def http_vim_account_edit(tenant_id, vim_account_id=None, datacenter_id=None): bottle.abort(e.http_code, str(e)) except Exception as e: logger.error("Unexpected exception: ", exc_info=True) - bottle.abort(HTTP_Internal_Server_Error, type(e).__name__ + ": " + str(e)) + bottle.abort(httperrors.Internal_Server_Error, type(e).__name__ + ": " + str(e)) @bottle.route(url_base + '//datacenters/', method='DELETE') @@ -1037,7 +858,7 @@ def http_deassociate_datacenters(tenant_id, datacenter_id=None, vim_account_id=N bottle.abort(e.http_code, str(e)) except Exception as e: logger.error("Unexpected exception: ", exc_info=True) - bottle.abort(HTTP_Internal_Server_Error, type(e).__name__ + ": " + str(e)) + bottle.abort(httperrors.Internal_Server_Error, type(e).__name__ + ": " + str(e)) @bottle.route(url_base + '//vim//network//attach', method='POST') def http_post_vim_net_sdn_attach(tenant_id, datacenter_id, network_id): @@ -1053,7 +874,7 @@ def http_post_vim_net_sdn_attach(tenant_id, datacenter_id, network_id): bottle.abort(e.http_code, str(e)) except Exception as e: logger.error("Unexpected exception: ", exc_info=True) - bottle.abort(HTTP_Internal_Server_Error, type(e).__name__ + ": " + str(e)) + bottle.abort(httperrors.Internal_Server_Error, type(e).__name__ + ": " + str(e)) @bottle.route(url_base + '//vim//network//detach', method='DELETE') @bottle.route(url_base + '//vim//network//detach/', method='DELETE') @@ -1069,8 +890,8 @@ def http_delete_vim_net_sdn_detach(tenant_id, datacenter_id, network_id, port_id bottle.abort(e.http_code, str(e)) except Exception as e: logger.error("Unexpected exception: ", exc_info=True) - bottle.abort(HTTP_Internal_Server_Error, type(e).__name__ + ": " + str(e)) - + bottle.abort(httperrors.Internal_Server_Error, type(e).__name__ + ": " + str(e)) + @bottle.route(url_base + '//vim//', method='GET') @bottle.route(url_base + '//vim///', method='GET') def http_get_vim_items(tenant_id, datacenter_id, item, name=None): @@ -1085,7 +906,7 @@ def http_get_vim_items(tenant_id, datacenter_id, item, name=None): bottle.abort(e.http_code, str(e)) except Exception as e: logger.error("Unexpected exception: ", exc_info=True) - bottle.abort(HTTP_Internal_Server_Error, type(e).__name__ + ": " + str(e)) + bottle.abort(httperrors.Internal_Server_Error, type(e).__name__ + ": " + str(e)) @bottle.route(url_base + '//vim///', method='DELETE') @@ -1101,7 +922,7 @@ def http_del_vim_items(tenant_id, datacenter_id, item, name): bottle.abort(e.http_code, str(e)) except Exception as e: logger.error("Unexpected exception: ", exc_info=True) - bottle.abort(HTTP_Internal_Server_Error, type(e).__name__ + ": " + str(e)) + bottle.abort(httperrors.Internal_Server_Error, type(e).__name__ + ": " + str(e)) @bottle.route(url_base + '//vim//', method='POST') @@ -1118,7 +939,7 @@ def http_post_vim_items(tenant_id, datacenter_id, item): bottle.abort(e.http_code, str(e)) except Exception as e: logger.error("Unexpected exception: ", exc_info=True) - bottle.abort(HTTP_Internal_Server_Error, type(e).__name__ + ": " + str(e)) + bottle.abort(httperrors.Internal_Server_Error, type(e).__name__ + ": " + str(e)) @bottle.route(url_base + '//vnfs', method='GET') @@ -1135,7 +956,7 @@ def http_get_vnfs(tenant_id): vnfs = mydb.get_rows(FROM='vnfs', SELECT=select_, WHERE=where_, LIMIT=limit_) # change_keys_http2db(content, http2db_vnf, reverse=True) utils.convert_str2boolean(vnfs, ('public',)) - convert_datetime2str(vnfs) + utils.convert_float_timestamp2str(vnfs) data={'vnfs': vnfs} return format_out(data) except bottle.HTTPError: @@ -1145,7 +966,7 @@ def http_get_vnfs(tenant_id): bottle.abort(e.http_code, str(e)) except Exception as e: logger.error("Unexpected exception: ", exc_info=True) - bottle.abort(HTTP_Internal_Server_Error, type(e).__name__ + ": " + str(e)) + bottle.abort(httperrors.Internal_Server_Error, type(e).__name__ + ": " + str(e)) @bottle.route(url_base + '//vnfs/', method='GET') @@ -1155,7 +976,7 @@ def http_get_vnf_id(tenant_id,vnf_id): try: vnf = nfvo.get_vnf_id(mydb,tenant_id,vnf_id) utils.convert_str2boolean(vnf, ('public',)) - convert_datetime2str(vnf) + utils.convert_float_timestamp2str(vnf) return format_out(vnf) except bottle.HTTPError: raise @@ -1164,7 +985,7 @@ def http_get_vnf_id(tenant_id,vnf_id): bottle.abort(e.http_code, str(e)) except Exception as e: logger.error("Unexpected exception: ", exc_info=True) - bottle.abort(HTTP_Internal_Server_Error, type(e).__name__ + ": " + str(e)) + bottle.abort(httperrors.Internal_Server_Error, type(e).__name__ + ": " + str(e)) @bottle.route(url_base + '//vnfs', method='POST') @@ -1187,7 +1008,7 @@ def http_post_vnfs(tenant_id): vnf_id = nfvo.new_vnf_v02(mydb,tenant_id,http_content) else: logger.warning('Unexpected schema_version: %s', http_content.get("schema_version")) - bottle.abort(HTTP_Bad_Request, "Invalid schema version") + bottle.abort(httperrors.Bad_Request, "Invalid schema version") return http_get_vnf_id(tenant_id, vnf_id) except bottle.HTTPError: raise @@ -1196,7 +1017,7 @@ def http_post_vnfs(tenant_id): bottle.abort(e.http_code, str(e)) except Exception as e: logger.error("Unexpected exception: ", exc_info=True) - bottle.abort(HTTP_Internal_Server_Error, type(e).__name__ + ": " + str(e)) + bottle.abort(httperrors.Internal_Server_Error, type(e).__name__ + ": " + str(e)) @bottle.route(url_base + '/v3//vnfd', method='POST') @@ -1214,7 +1035,7 @@ def http_post_vnfs_v3(tenant_id): for vnfd_uuid in vnfd_uuid_list: vnf = nfvo.get_vnf_id(mydb, tenant_id, vnfd_uuid) utils.convert_str2boolean(vnf, ('public',)) - convert_datetime2str(vnf) + utils.convert_float_timestamp2str(vnf) vnfd_list.append(vnf["vnf"]) return format_out({"vnfd": vnfd_list}) except bottle.HTTPError: @@ -1224,13 +1045,13 @@ def http_post_vnfs_v3(tenant_id): bottle.abort(e.http_code, str(e)) except Exception as e: logger.error("Unexpected exception: ", exc_info=True) - bottle.abort(HTTP_Internal_Server_Error, type(e).__name__ + ": " + str(e)) + bottle.abort(httperrors.Internal_Server_Error, type(e).__name__ + ": " + str(e)) @bottle.route(url_base + '//vnfs/', method='DELETE') def http_delete_vnf_id(tenant_id, vnf_id): '''delete a vnf from database, and images and flavors in VIM when appropriate, can use both uuid or name''' logger.debug('FROM %s %s %s', bottle.request.remote_addr, bottle.request.method, bottle.request.url) - #check valid tenant_id and deletes the vnf, including images, + #check valid tenant_id and deletes the vnf, including images, try: data = nfvo.delete_vnf(mydb,tenant_id,vnf_id) #print json.dumps(data, indent=4) @@ -1242,7 +1063,7 @@ def http_delete_vnf_id(tenant_id, vnf_id): bottle.abort(e.http_code, str(e)) except Exception as e: logger.error("Unexpected exception: ", exc_info=True) - bottle.abort(HTTP_Internal_Server_Error, type(e).__name__ + ": " + str(e)) + bottle.abort(httperrors.Internal_Server_Error, type(e).__name__ + ": " + str(e)) #@bottle.route(url_base + '//hosts/topology', method='GET') @@ -1258,12 +1079,12 @@ def http_get_hosts(tenant_id, datacenter): else: #openmano-gui is using a hardcoded value for the datacenter result, data = nfvo.get_hosts_info(mydb, tenant_id) #, datacenter) - + if result < 0: #print "http_get_hosts error %d %s" % (-result, data) bottle.abort(-result, data) else: - convert_datetime2str(data) + utils.convert_float_timestamp2str(data) #print json.dumps(data, indent=4) return format_out(data) except bottle.HTTPError: @@ -1273,7 +1094,7 @@ def http_get_hosts(tenant_id, datacenter): bottle.abort(e.http_code, str(e)) except Exception as e: logger.error("Unexpected exception: ", exc_info=True) - bottle.abort(HTTP_Internal_Server_Error, type(e).__name__ + ": " + str(e)) + bottle.abort(httperrors.Internal_Server_Error, type(e).__name__ + ": " + str(e)) @bottle.route(url_base + '/', method='OPTIONS') @@ -1297,7 +1118,7 @@ def http_post_deploy(tenant_id): #r = utils.remove_extra_items(http_content, used_schema) #if r is not None: print "http_post_deploy: Warning: remove extra items ", r #print "http_post_deploy input: ", http_content - + try: scenario_id = nfvo.new_scenario(mydb, tenant_id, http_content) instance = nfvo.start_scenario(mydb, tenant_id, scenario_id, http_content['name'], http_content['name']) @@ -1310,7 +1131,7 @@ def http_post_deploy(tenant_id): bottle.abort(e.http_code, str(e)) except Exception as e: logger.error("Unexpected exception: ", exc_info=True) - bottle.abort(HTTP_Internal_Server_Error, type(e).__name__ + ": " + str(e)) + bottle.abort(httperrors.Internal_Server_Error, type(e).__name__ + ": " + str(e)) @bottle.route(url_base + '//topology/verify', method='POST') @@ -1319,7 +1140,7 @@ def http_post_verify(tenant_id): # '''post topology verify''' # print "http_post_verify by tenant " + tenant_id + ' datacenter ' + datacenter logger.debug('FROM %s %s %s', bottle.request.remote_addr, bottle.request.method, bottle.request.url) - return + return # # SCENARIOS @@ -1342,7 +1163,7 @@ def http_post_scenarios(tenant_id): scenario_id = nfvo.new_scenario_v02(mydb, tenant_id, http_content, "0.3") else: logger.warning('Unexpected schema_version: %s', http_content.get("schema_version")) - bottle.abort(HTTP_Bad_Request, "Invalid schema version") + bottle.abort(httperrors.Bad_Request, "Invalid schema version") #print json.dumps(data, indent=4) #return format_out(data) return http_get_scenario_id(tenant_id, scenario_id) @@ -1353,7 +1174,7 @@ def http_post_scenarios(tenant_id): bottle.abort(e.http_code, str(e)) except Exception as e: logger.error("Unexpected exception: ", exc_info=True) - bottle.abort(HTTP_Internal_Server_Error, type(e).__name__ + ": " + str(e)) + bottle.abort(httperrors.Internal_Server_Error, type(e).__name__ + ": " + str(e)) @bottle.route(url_base + '/v3//nsd', method='POST') def http_post_nsds_v3(tenant_id): @@ -1369,7 +1190,7 @@ def http_post_nsds_v3(tenant_id): nsd_list = [] for nsd_uuid in nsd_uuid_list: scenario = mydb.get_scenario(nsd_uuid, tenant_id) - convert_datetime2str(scenario) + utils.convert_float_timestamp2str(scenario) nsd_list.append(scenario) data = {'nsd': nsd_list} return format_out(data) @@ -1380,7 +1201,7 @@ def http_post_nsds_v3(tenant_id): bottle.abort(e.http_code, str(e)) except Exception as e: logger.error("Unexpected exception: ", exc_info=True) - bottle.abort(HTTP_Internal_Server_Error, type(e).__name__ + ": " + str(e)) + bottle.abort(httperrors.Internal_Server_Error, type(e).__name__ + ": " + str(e)) @bottle.route(url_base + '//scenarios//action', method='POST') @@ -1424,7 +1245,7 @@ def http_post_scenario_action(tenant_id, scenario_id): bottle.abort(e.http_code, str(e)) except Exception as e: logger.error("Unexpected exception: ", exc_info=True) - bottle.abort(HTTP_Internal_Server_Error, type(e).__name__ + ": " + str(e)) + bottle.abort(httperrors.Internal_Server_Error, type(e).__name__ + ": " + str(e)) @bottle.route(url_base + '//scenarios', method='GET') @@ -1434,14 +1255,14 @@ def http_get_scenarios(tenant_id): try: #check valid tenant_id if tenant_id != "any": - nfvo.check_tenant(mydb, tenant_id) + nfvo.check_tenant(mydb, tenant_id) #obtain data s,w,l=filter_query_string(bottle.request.query, None, ('uuid', 'name', 'osm_id', 'description', 'tenant_id', 'created_at', 'public')) if tenant_id != "any": w["OR"] = {"tenant_id": tenant_id, "public": True} scenarios = mydb.get_rows(SELECT=s, WHERE=w, LIMIT=l, FROM='scenarios') - convert_datetime2str(scenarios) + utils.convert_float_timestamp2str(scenarios) utils.convert_str2boolean(scenarios, ('public',) ) data={'scenarios':scenarios} #print json.dumps(scenarios, indent=4) @@ -1453,7 +1274,7 @@ def http_get_scenarios(tenant_id): bottle.abort(e.http_code, str(e)) except Exception as e: logger.error("Unexpected exception: ", exc_info=True) - bottle.abort(HTTP_Internal_Server_Error, type(e).__name__ + ": " + str(e)) + bottle.abort(httperrors.Internal_Server_Error, type(e).__name__ + ": " + str(e)) @bottle.route(url_base + '//scenarios/', method='GET') @@ -1463,10 +1284,10 @@ def http_get_scenario_id(tenant_id, scenario_id): try: #check valid tenant_id if tenant_id != "any": - nfvo.check_tenant(mydb, tenant_id) + nfvo.check_tenant(mydb, tenant_id) #obtain data scenario = mydb.get_scenario(scenario_id, tenant_id) - convert_datetime2str(scenario) + utils.convert_float_timestamp2str(scenario) data={'scenario' : scenario} return format_out(data) except bottle.HTTPError: @@ -1476,7 +1297,7 @@ def http_get_scenario_id(tenant_id, scenario_id): bottle.abort(e.http_code, str(e)) except Exception as e: logger.error("Unexpected exception: ", exc_info=True) - bottle.abort(HTTP_Internal_Server_Error, type(e).__name__ + ": " + str(e)) + bottle.abort(httperrors.Internal_Server_Error, type(e).__name__ + ": " + str(e)) @bottle.route(url_base + '//scenarios/', method='DELETE') @@ -1498,7 +1319,7 @@ def http_delete_scenario_id(tenant_id, scenario_id): bottle.abort(e.http_code, str(e)) except Exception as e: logger.error("Unexpected exception: ", exc_info=True) - bottle.abort(HTTP_Internal_Server_Error, type(e).__name__ + ": " + str(e)) + bottle.abort(httperrors.Internal_Server_Error, type(e).__name__ + ": " + str(e)) @bottle.route(url_base + '//scenarios/', method='PUT') @@ -1521,7 +1342,7 @@ def http_put_scenario_id(tenant_id, scenario_id): bottle.abort(e.http_code, str(e)) except Exception as e: logger.error("Unexpected exception: ", exc_info=True) - bottle.abort(HTTP_Internal_Server_Error, type(e).__name__ + ": " + str(e)) + bottle.abort(httperrors.Internal_Server_Error, type(e).__name__ + ": " + str(e)) @bottle.route(url_base + '//instances', method='POST') def http_post_instances(tenant_id): @@ -1535,17 +1356,17 @@ def http_post_instances(tenant_id): try: #check valid tenant_id if tenant_id != "any": - nfvo.check_tenant(mydb, tenant_id) + nfvo.check_tenant(mydb, tenant_id) data = nfvo.create_instance(mydb, tenant_id, http_content["instance"]) return format_out(data) except bottle.HTTPError: raise except (nfvo.NfvoException, db_base_Exception) as e: - logger.error("http_post_instances error {}: {}".format(e.http_code, str(e))) + logger.error("http_post_instances error {}: {}".format(e.http_code, str(e)), exc_info=True) bottle.abort(e.http_code, str(e)) except Exception as e: logger.error("Unexpected exception: ", exc_info=True) - bottle.abort(HTTP_Internal_Server_Error, type(e).__name__ + ": " + str(e)) + bottle.abort(httperrors.Internal_Server_Error, type(e).__name__ + ": " + str(e)) # # INSTANCES @@ -1556,13 +1377,13 @@ def http_get_instances(tenant_id): try: #check valid tenant_id if tenant_id != "any": - nfvo.check_tenant(mydb, tenant_id) + nfvo.check_tenant(mydb, tenant_id) #obtain data s,w,l=filter_query_string(bottle.request.query, None, ('uuid', 'name', 'scenario_id', 'tenant_id', 'description', 'created_at')) if tenant_id != "any": w['tenant_id'] = tenant_id instances = mydb.get_rows(SELECT=s, WHERE=w, LIMIT=l, FROM='instance_scenarios') - convert_datetime2str(instances) + utils.convert_float_timestamp2str(instances) utils.convert_str2boolean(instances, ('public',) ) data={'instances':instances} return format_out(data) @@ -1573,7 +1394,7 @@ def http_get_instances(tenant_id): bottle.abort(e.http_code, str(e)) except Exception as e: logger.error("Unexpected exception: ", exc_info=True) - bottle.abort(HTTP_Internal_Server_Error, type(e).__name__ + ": " + str(e)) + bottle.abort(httperrors.Internal_Server_Error, type(e).__name__ + ": " + str(e)) @bottle.route(url_base + '//instances/', method='GET') @@ -1584,7 +1405,7 @@ def http_get_instance_id(tenant_id, instance_id): #check valid tenant_id if tenant_id != "any": - nfvo.check_tenant(mydb, tenant_id) + nfvo.check_tenant(mydb, tenant_id) if tenant_id == "any": tenant_id = None @@ -1598,7 +1419,7 @@ def http_get_instance_id(tenant_id, instance_id): index = iface["ip_address"].find(";") if index >= 0: iface["ip_address"] = iface["ip_address"][:index] - convert_datetime2str(instance) + utils.convert_float_timestamp2str(instance) # print json.dumps(instance, indent=4) return format_out(instance) except bottle.HTTPError: @@ -1608,7 +1429,7 @@ def http_get_instance_id(tenant_id, instance_id): bottle.abort(e.http_code, str(e)) except Exception as e: logger.error("Unexpected exception: ", exc_info=True) - bottle.abort(HTTP_Internal_Server_Error, type(e).__name__ + ": " + str(e)) + bottle.abort(httperrors.Internal_Server_Error, type(e).__name__ + ": " + str(e)) @bottle.route(url_base + '//instances/', method='DELETE') @@ -1618,7 +1439,7 @@ def http_delete_instance_id(tenant_id, instance_id): try: #check valid tenant_id if tenant_id != "any": - nfvo.check_tenant(mydb, tenant_id) + nfvo.check_tenant(mydb, tenant_id) if tenant_id == "any": tenant_id = None #obtain data @@ -1631,7 +1452,7 @@ def http_delete_instance_id(tenant_id, instance_id): bottle.abort(e.http_code, str(e)) except Exception as e: logger.error("Unexpected exception: ", exc_info=True) - bottle.abort(HTTP_Internal_Server_Error, type(e).__name__ + ": " + str(e)) + bottle.abort(httperrors.Internal_Server_Error, type(e).__name__ + ": " + str(e)) @bottle.route(url_base + '//instances//action', method='POST') @@ -1651,13 +1472,13 @@ def http_post_instance_scenario_action(tenant_id, instance_id): try: #check valid tenant_id if tenant_id != "any": - nfvo.check_tenant(mydb, tenant_id) + nfvo.check_tenant(mydb, tenant_id) #print "http_post_instance_scenario_action input: ", http_content #obtain data instance = mydb.get_instance_scenario(instance_id, tenant_id) instance_id = instance["uuid"] - + data = nfvo.instance_action(mydb, tenant_id, instance_id, http_content) return format_out(data) except bottle.HTTPError: @@ -1667,7 +1488,7 @@ def http_post_instance_scenario_action(tenant_id, instance_id): bottle.abort(e.http_code, str(e)) except Exception as e: logger.error("Unexpected exception: ", exc_info=True) - bottle.abort(HTTP_Internal_Server_Error, type(e).__name__ + ": " + str(e)) + bottle.abort(httperrors.Internal_Server_Error, type(e).__name__ + ": " + str(e)) @bottle.route(url_base + '//instances//action', method='GET') @@ -1693,34 +1514,17 @@ def http_get_instance_scenario_action(tenant_id, instance_id, action_id=None): bottle.abort(e.http_code, str(e)) except Exception as e: logger.error("Unexpected exception: ", exc_info=True) - bottle.abort(HTTP_Internal_Server_Error, type(e).__name__ + ": " + str(e)) - -def remove_clear_passwd(data): - """ - Removes clear passwords from the data received - :param data: data with clear password - :return: data without the password information - """ - - passw = ['password: ', 'passwd: '] + bottle.abort(httperrors.Internal_Server_Error, type(e).__name__ + ": " + str(e)) - for pattern in passw: - init = data.find(pattern) - while init != -1: - end = data.find('\n', init) - data = data[:init] + '{}******'.format(pattern) + data[end:] - init += 1 - init = data.find(pattern, init) - return data @bottle.error(400) -@bottle.error(401) -@bottle.error(404) +@bottle.error(401) +@bottle.error(404) @bottle.error(403) -@bottle.error(405) +@bottle.error(405) @bottle.error(406) @bottle.error(409) -@bottle.error(503) +@bottle.error(503) @bottle.error(500) def error400(error): e={"error":{"code":error.status_code, "type":error.status, "description":error.body}} diff --git a/osm_ro/nfvo.py b/osm_ro/nfvo.py index c89ea796..bc5e624a 100644 --- a/osm_ro/nfvo.py +++ b/osm_ro/nfvo.py @@ -28,13 +28,11 @@ __author__="Alfonso Tierno, Gerardo Garcia, Pablo Montes" __date__ ="$16-sep-2014 22:05:01$" # import imp -# import json +import json import yaml import utils from utils import deprecated import vim_thread -from db_base import HTTP_Unauthorized, HTTP_Bad_Request, HTTP_Internal_Server_Error, HTTP_Not_Found,\ - HTTP_Conflict, HTTP_Method_Not_Allowed import console_proxy_thread as cli import vimconn import logging @@ -56,8 +54,22 @@ from pyangbind.lib.serialise import pybindJSONDecoder from copy import deepcopy +# WIM +import wim.wimconn as wimconn +import wim.wim_thread as wim_thread +from .http_tools import errors as httperrors +from .wim.engine import WimEngine +from .wim.persistence import WimPersistence +from copy import deepcopy +# + global global_config global vimconn_imported +# WIM +global wim_engine +wim_engine = None +global wimconn_imported +# global logger global default_volume_size default_volume_size = '5' #size in GB @@ -68,16 +80,21 @@ global_config = None vimconn_imported = {} # dictionary with VIM type as key, loaded module as value vim_threads = {"running":{}, "deleting": {}, "names": []} # threads running for attached-VIMs vim_persistent_info = {} +# WIM +wimconn_imported = {} # dictionary with WIM type as key, loaded module as value +wim_threads = {"running":{}, "deleting": {}, "names": []} # threads running for attached-WIMs +wim_persistent_info = {} +# + logger = logging.getLogger('openmano.nfvo') task_lock = Lock() last_task_id = 0.0 db = None db_lock = Lock() -class NfvoException(Exception): - def __init__(self, message, http_code): - self.http_code = http_code - Exception.__init__(self, message) + +class NfvoException(httperrors.HttpMappedError): + """Common Class for NFVO errors""" def get_task_id(): @@ -116,13 +133,32 @@ def get_non_used_vim_name(datacenter_name, datacenter_id, tenant_name, tenant_id vim_threads["names"].append(name) return name +# -- Move +def get_non_used_wim_name(wim_name, wim_id, tenant_name, tenant_id): + name = wim_name[:16] + if name not in wim_threads["names"]: + wim_threads["names"].append(name) + return name + name = wim_name[:16] + "." + tenant_name[:16] + if name not in wim_threads["names"]: + wim_threads["names"].append(name) + return name + name = wim_id + "-" + tenant_id + wim_threads["names"].append(name) + return name -def start_service(mydb): + +def start_service(mydb, persistence=None, wim=None): global db, global_config db = nfvo_db.nfvo_db() db.connect(global_config['db_host'], global_config['db_user'], global_config['db_passwd'], global_config['db_name']) global ovim + if persistence: + persistence.lock = db_lock + else: + persistence = WimPersistence(db, lock=db_lock) + # Initialize openvim for SDN control # TODO: Avoid static configuration by adding new parameters to openmanod.cfg # TODO: review ovim.py to delete not needed configuration @@ -143,9 +179,14 @@ def start_service(mydb): try: # starts ovim library ovim = ovim_module.ovim(ovim_configuration) + + global wim_engine + wim_engine = wim or WimEngine(persistence) + wim_engine.ovim = ovim + ovim.start_service() - #delete old unneeded vim_actions + #delete old unneeded vim_wim_actions clean_db(mydb) # starts vim_threads @@ -176,13 +217,13 @@ def start_service(mydb): # if module_info and module_info[0]: # file.close(module_info[0]) raise NfvoException("Unknown vim type '{}'. Cannot open file '{}.py'; {}: {}".format( - vim["type"], module, type(e).__name__, str(e)), HTTP_Bad_Request) + vim["type"], module, type(e).__name__, str(e)), httperrors.Bad_Request) thread_id = vim['datacenter_tenant_id'] vim_persistent_info[thread_id] = {} try: #if not tenant: - # return -HTTP_Bad_Request, "You must provide a valid tenant name or uuid for VIM %s" % ( vim["type"]) + # return -httperrors.Bad_Request, "You must provide a valid tenant name or uuid for VIM %s" % ( vim["type"]) myvim = vimconn_imported[ vim["type"] ].vimconnector( uuid=vim['datacenter_id'], name=vim['datacenter_name'], tenant_id=vim['vim_tenant_id'], tenant_name=vim['vim_tenant_name'], @@ -196,13 +237,15 @@ def start_service(mydb): vim['datacenter_id'], e)) except Exception as e: raise NfvoException("Error at VIM {}; {}: {}".format(vim["type"], type(e).__name__, e), - HTTP_Internal_Server_Error) + httperrors.Internal_Server_Error) thread_name = get_non_used_vim_name(vim['datacenter_name'], vim['vim_tenant_id'], vim['vim_tenant_name'], vim['vim_tenant_id']) new_thread = vim_thread.vim_thread(task_lock, thread_name, vim['datacenter_name'], vim['datacenter_tenant_id'], db=db, db_lock=db_lock, ovim=ovim) new_thread.start() vim_threads["running"][thread_id] = new_thread + + wim_engine.start_threads() except db_base_Exception as e: raise NfvoException(str(e) + " at nfvo.get_vim", e.http_code) except ovim_module.ovimException as e: @@ -213,17 +256,21 @@ def start_service(mydb): msg=message[22:-3], dbname=global_config["db_ovim_name"], dbuser=global_config["db_ovim_user"], dbpass=global_config["db_ovim_passwd"], ver=message[-3:-1], dbhost=global_config["db_ovim_host"]) - raise NfvoException(message, HTTP_Bad_Request) + raise NfvoException(message, httperrors.Bad_Request) def stop_service(): global ovim, global_config if ovim: ovim.stop_service() - for thread_id,thread in vim_threads["running"].items(): + for thread_id, thread in vim_threads["running"].items(): thread.insert_task("exit") vim_threads["deleting"][thread_id] = thread vim_threads["running"] = {} + + if wim_engine: + wim_engine.stop_threads() + if global_config and global_config.get("console_thread"): for thread in global_config["console_thread"]: thread.terminate = True @@ -238,21 +285,21 @@ def clean_db(mydb): :param mydb: database connector :return: None """ - # get and delete unused vim_actions: all elements deleted, one week before, instance not present + # get and delete unused vim_wim_actions: all elements deleted, one week before, instance not present now = t.time()-3600*24*7 instance_action_id = None nb_deleted = 0 while True: actions_to_delete = mydb.get_rows( SELECT=("item", "item_id", "instance_action_id"), - FROM="vim_actions as va join instance_actions as ia on va.instance_action_id=ia.uuid " + FROM="vim_wim_actions as va join instance_actions as ia on va.instance_action_id=ia.uuid " "left join instance_scenarios as i on ia.instance_id=i.uuid", WHERE={"va.action": "DELETE", "va.modified_at<": now, "i.uuid": None, "va.status": ("DONE", "SUPERSEDED")}, LIMIT=100 ) for to_delete in actions_to_delete: - mydb.delete_row(FROM="vim_actions", WHERE=to_delete) + mydb.delete_row(FROM="vim_wim_actions", WHERE=to_delete) if instance_action_id != to_delete["instance_action_id"]: instance_action_id = to_delete["instance_action_id"] mydb.delete_row(FROM="instance_actions", WHERE={"uuid": instance_action_id}) @@ -260,8 +307,7 @@ def clean_db(mydb): if len(actions_to_delete) < 100: break if nb_deleted: - logger.debug("Removed {} unused vim_actions".format(nb_deleted)) - + logger.debug("Removed {} unused vim_wim_actions".format(nb_deleted)) def get_flavorlist(mydb, vnf_id, nfvo_tenant=None): @@ -357,7 +403,7 @@ def get_vim(mydb, nfvo_tenant=None, datacenter_id=None, datacenter_name=None, da vim["type"], module, type(e).__name__, str(e))) continue raise NfvoException("Unknown vim type '{}'. Can not open file '{}.py'; {}: {}".format( - vim["type"], module, type(e).__name__, str(e)), HTTP_Bad_Request) + vim["type"], module, type(e).__name__, str(e)), httperrors.Bad_Request) try: if 'datacenter_tenant_id' in vim: @@ -368,7 +414,7 @@ def get_vim(mydb, nfvo_tenant=None, datacenter_id=None, datacenter_name=None, da else: persistent_info = {} #if not tenant: - # return -HTTP_Bad_Request, "You must provide a valid tenant name or uuid for VIM %s" % ( vim["type"]) + # return -httperrors.Bad_Request, "You must provide a valid tenant name or uuid for VIM %s" % ( vim["type"]) vim_dict[ vim['datacenter_id'] ] = vimconn_imported[ vim["type"] ].vimconnector( uuid=vim['datacenter_id'], name=vim['datacenter_name'], tenant_id=vim.get('vim_tenant_id',vim_tenant), @@ -381,7 +427,7 @@ def get_vim(mydb, nfvo_tenant=None, datacenter_id=None, datacenter_name=None, da if ignore_errors: logger.error("Error at VIM {}; {}: {}".format(vim["type"], type(e).__name__, str(e))) continue - http_code = HTTP_Internal_Server_Error + http_code = httperrors.Internal_Server_Error if isinstance(e, vimconn.vimconnException): http_code = e.http_code raise NfvoException("Error at VIM {}; {}: {}".format(vim["type"], type(e).__name__, str(e)), http_code) @@ -446,7 +492,7 @@ def check_vnf_descriptor(vnf_descriptor, vnf_descriptor_version=1): raise NfvoException( "Error at vnf:VNFC[name:'{}']:numas:interfaces:name, interface name '{}' already used in this VNFC".format( vnfc["name"], interface["name"]), - HTTP_Bad_Request) + httperrors.Bad_Request) name_dict[ interface["name"] ] = "underlay" #bridge interfaces for interface in vnfc.get("bridge-ifaces",() ): @@ -454,7 +500,7 @@ def check_vnf_descriptor(vnf_descriptor, vnf_descriptor_version=1): raise NfvoException( "Error at vnf:VNFC[name:'{}']:bridge-ifaces:name, interface name '{}' already used in this VNFC".format( vnfc["name"], interface["name"]), - HTTP_Bad_Request) + httperrors.Bad_Request) name_dict[ interface["name"] ] = "overlay" vnfc_interfaces[ vnfc["name"] ] = name_dict # check bood-data info @@ -463,7 +509,7 @@ def check_vnf_descriptor(vnf_descriptor, vnf_descriptor_version=1): # if (vnfc["boot-data"].get("users") or vnfc["boot-data"].get("config-files")) and vnfc["boot-data"].get("user-data"): # raise NfvoException( # "Error at vnf:VNFC:boot-data, fields 'users' and 'config-files' are not compatible with 'user-data'", - # HTTP_Bad_Request) + # httperrors.Bad_Request) #check if the info in external_connections matches with the one in the vnfcs name_list=[] @@ -472,20 +518,20 @@ def check_vnf_descriptor(vnf_descriptor, vnf_descriptor_version=1): raise NfvoException( "Error at vnf:external-connections:name, value '{}' already used as an external-connection".format( external_connection["name"]), - HTTP_Bad_Request) + httperrors.Bad_Request) name_list.append(external_connection["name"]) if external_connection["VNFC"] not in vnfc_interfaces: raise NfvoException( "Error at vnf:external-connections[name:'{}']:VNFC, value '{}' does not match any VNFC".format( external_connection["name"], external_connection["VNFC"]), - HTTP_Bad_Request) + httperrors.Bad_Request) if external_connection["local_iface_name"] not in vnfc_interfaces[ external_connection["VNFC"] ]: raise NfvoException( "Error at vnf:external-connections[name:'{}']:local_iface_name, value '{}' does not match any interface of this VNFC".format( external_connection["name"], external_connection["local_iface_name"]), - HTTP_Bad_Request ) + httperrors.Bad_Request ) #check if the info in internal_connections matches with the one in the vnfcs name_list=[] @@ -494,7 +540,7 @@ def check_vnf_descriptor(vnf_descriptor, vnf_descriptor_version=1): raise NfvoException( "Error at vnf:internal-connections:name, value '%s' already used as an internal-connection".format( internal_connection["name"]), - HTTP_Bad_Request) + httperrors.Bad_Request) name_list.append(internal_connection["name"]) #We should check that internal-connections of type "ptp" have only 2 elements @@ -504,7 +550,7 @@ def check_vnf_descriptor(vnf_descriptor, vnf_descriptor_version=1): internal_connection["name"], 'ptp' if vnf_descriptor_version==1 else 'e-line', 'data' if vnf_descriptor_version==1 else "e-lan"), - HTTP_Bad_Request) + httperrors.Bad_Request) for port in internal_connection["elements"]: vnf = port["VNFC"] iface = port["local_iface_name"] @@ -512,13 +558,13 @@ def check_vnf_descriptor(vnf_descriptor, vnf_descriptor_version=1): raise NfvoException( "Error at vnf:internal-connections[name:'{}']:elements[]:VNFC, value '{}' does not match any VNFC".format( internal_connection["name"], vnf), - HTTP_Bad_Request) + httperrors.Bad_Request) if iface not in vnfc_interfaces[ vnf ]: raise NfvoException( "Error at vnf:internal-connections[name:'{}']:elements[]:local_iface_name, value '{}' does not match any interface of this VNFC".format( internal_connection["name"], iface), - HTTP_Bad_Request) - return -HTTP_Bad_Request, + httperrors.Bad_Request) + return -httperrors.Bad_Request, if vnf_descriptor_version==1 and "type" not in internal_connection: if vnfc_interfaces[vnf][iface] == "overlay": internal_connection["type"] = "bridge" @@ -536,7 +582,7 @@ def check_vnf_descriptor(vnf_descriptor, vnf_descriptor_version=1): internal_connection["name"], iface, 'bridge' if vnf_descriptor_version==1 else 'overlay', 'data' if vnf_descriptor_version==1 else 'underlay'), - HTTP_Bad_Request) + httperrors.Bad_Request) if (internal_connection.get("type") == "bridge" or internal_connection.get("implementation") == "overlay") and \ vnfc_interfaces[vnf][iface] == "underlay": raise NfvoException( @@ -544,7 +590,7 @@ def check_vnf_descriptor(vnf_descriptor, vnf_descriptor_version=1): internal_connection["name"], iface, 'data' if vnf_descriptor_version==1 else 'underlay', 'bridge' if vnf_descriptor_version==1 else 'overlay'), - HTTP_Bad_Request) + httperrors.Bad_Request) def create_or_use_image(mydb, vims, image_dict, rollback_list, only_create_at_vim=False, return_on_error=None): @@ -589,7 +635,7 @@ def create_or_use_image(mydb, vims, image_dict, rollback_list, only_create_at_vi vim_images = vim.get_image_list(filter_dict) #logger.debug('>>>>>>>> VIM images: %s', str(vim_images)) if len(vim_images) > 1: - raise vimconn.vimconnException("More than one candidate VIM image found for filter: {}".format(str(filter_dict)), HTTP_Conflict) + raise vimconn.vimconnException("More than one candidate VIM image found for filter: {}".format(str(filter_dict)), httperrors.Conflict) elif len(vim_images) == 0: raise vimconn.vimconnNotFoundException("Image not found at VIM with filter: '{}'".format(str(filter_dict))) else: @@ -845,7 +891,7 @@ def new_vnfd_v3(mydb, tenant_id, vnf_descriptor): try: pybindJSONDecoder.load_ietf_json(vnf_descriptor, None, None, obj=myvnfd, path_helper=True) except Exception as e: - raise NfvoException("Error. Invalid VNF descriptor format " + str(e), HTTP_Bad_Request) + raise NfvoException("Error. Invalid VNF descriptor format " + str(e), httperrors.Bad_Request) db_vnfs = [] db_nets = [] db_vms = [] @@ -929,7 +975,7 @@ def new_vnfd_v3(mydb, tenant_id, vnf_descriptor): raise NfvoException("Error. Invalid VNF descriptor at 'vnfd[{}]':'vld[{}]':'ip-profile-ref':" "'{}'. Reference to a non-existing 'ip_profiles'".format( str(vnfd["id"]), str(vld["id"]), str(vld["ip-profile-ref"])), - HTTP_Bad_Request) + httperrors.Bad_Request) db_ip_profiles[ip_profile_name2db_table_index[ip_profile_name]]["net_id"] = net_uuid else: #check no ip-address has been defined for icp in vld.get("internal-connection-point").itervalues(): @@ -937,7 +983,7 @@ def new_vnfd_v3(mydb, tenant_id, vnf_descriptor): raise NfvoException("Error at 'vnfd[{}]':'vld[{}]':'internal-connection-point[{}]' " "contains an ip-address but no ip-profile has been defined at VLD".format( str(vnfd["id"]), str(vld["id"]), str(icp["id"])), - HTTP_Bad_Request) + httperrors.Bad_Request) # connection points vaiable declaration cp_name2iface_uuid = {} @@ -1085,7 +1131,7 @@ def new_vnfd_v3(mydb, tenant_id, vnf_descriptor): raise NfvoException("Error. Invalid VNF descriptor at 'vnfd[{}]':'vdu[{}]':'interface':'virtual" "-interface':'type':'{}'. Interface type is not supported".format( vnfd_id, vdu_id, iface.get("virtual-interface").get("type")), - HTTP_Bad_Request) + httperrors.Bad_Request) if iface.get("mgmt-interface"): db_interface["type"] = "mgmt" @@ -1119,7 +1165,7 @@ def new_vnfd_v3(mydb, tenant_id, vnf_descriptor): " at connection-point".format( vnf=vnfd_id, vdu=vdu_id, iface=iface["name"], cp=iface.get("vnfd-connection-point-ref")), - HTTP_Bad_Request) + httperrors.Bad_Request) elif iface.get("internal-connection-point-ref"): try: for icp_descriptor in vdu_descriptor["internal-connection-point"]: @@ -1154,7 +1200,7 @@ def new_vnfd_v3(mydb, tenant_id, vnf_descriptor): " {msg}".format( vnf=vnfd_id, vdu=vdu_id, iface=iface["name"], cp=iface.get("internal-connection-point-ref"), msg=str(e)), - HTTP_Bad_Request) + httperrors.Bad_Request) if iface.get("position"): db_interface["created_at"] = int(iface.get("position")) * 50 if iface.get("mac-address"): @@ -1239,11 +1285,11 @@ def new_vnfd_v3(mydb, tenant_id, vnf_descriptor): raise NfvoException("Error. Invalid VNF descriptor at 'vnfd[{vnf}]':'placement-groups[{pg}]':" "'member-vdus':'{vdu}'. Reference to a non-existing vdu".format( vnf=vnfd_id, pg=pg_name, vdu=vdu_id), - HTTP_Bad_Request) + httperrors.Bad_Request) db_vms[vdu_id2db_table_index[vdu_id]]["availability_zone"] = pg_name # TODO consider the case of isolation and not colocation # if pg.get("strategy") == "ISOLATION": - + # VNF mgmt configuration mgmt_access = {} if vnfd["mgmt-interface"].get("vdu-id"): @@ -1252,7 +1298,7 @@ def new_vnfd_v3(mydb, tenant_id, vnf_descriptor): raise NfvoException("Error. Invalid VNF descriptor at 'vnfd[{vnf}]':'mgmt-interface':'vdu-id':" "'{vdu}'. Reference to a non-existing vdu".format( vnf=vnfd_id, vdu=mgmt_vdu_id), - HTTP_Bad_Request) + httperrors.Bad_Request) mgmt_access["vm_id"] = vdu_id2uuid[vnfd["mgmt-interface"]["vdu-id"]] # if only one cp is defined by this VDU, mark this interface as of type "mgmt" if vdu_id2cp_name.get(mgmt_vdu_id): @@ -1265,7 +1311,7 @@ def new_vnfd_v3(mydb, tenant_id, vnf_descriptor): raise NfvoException("Error. Invalid VNF descriptor at 'vnfd[{vnf}]':'mgmt-interface':'cp':'{cp}'. " "Reference to a non-existing connection-point".format( vnf=vnfd_id, cp=vnfd["mgmt-interface"]["cp"]), - HTTP_Bad_Request) + httperrors.Bad_Request) mgmt_access["vm_id"] = cp_name2vm_uuid[vnfd["mgmt-interface"]["cp"]] mgmt_access["interface_id"] = cp_name2iface_uuid[vnfd["mgmt-interface"]["cp"]] # mark this interface as of type mgmt @@ -1303,7 +1349,7 @@ def new_vnfd_v3(mydb, tenant_id, vnf_descriptor): raise except Exception as e: logger.error("Exception {}".format(e)) - raise # NfvoException("Exception {}".format(e), HTTP_Bad_Request) + raise # NfvoException("Exception {}".format(e), httperrors.Bad_Request) @deprecated("Use new_vnfd_v3") @@ -1319,7 +1365,7 @@ def new_vnf(mydb, tenant_id, vnf_descriptor): if "tenant_id" in vnf_descriptor["vnf"]: if vnf_descriptor["vnf"]["tenant_id"] != tenant_id: raise NfvoException("VNF can not have a different tenant owner '{}', must be '{}'".format(vnf_descriptor["vnf"]["tenant_id"], tenant_id), - HTTP_Unauthorized) + httperrors.Unauthorized) else: vnf_descriptor['vnf']['tenant_id'] = tenant_id # Step 3. Get the URL of the VIM from the nfvo_tenant and the datacenter @@ -1377,9 +1423,9 @@ def new_vnf(mydb, tenant_id, vnf_descriptor): # result2, message = rollback(myvim, myvimURL, myvim_tenant, flavorList, imageList) # if result2: # print "Error creating flavor: unknown processor model. Rollback successful." - # return -HTTP_Bad_Request, "Error creating flavor: unknown processor model. Rollback successful." + # return -httperrors.Bad_Request, "Error creating flavor: unknown processor model. Rollback successful." # else: - # return -HTTP_Bad_Request, "Error creating flavor: unknown processor model. Rollback fail: you need to access VIM and delete the following %s" % message + # return -httperrors.Bad_Request, "Error creating flavor: unknown processor model. Rollback fail: you need to access VIM and delete the following %s" % message myflavorDict['extended']['processor_ranking'] = 100 #Hardcoded value, while we decide when the mapping is done if 'numas' in vnfc and len(vnfc['numas'])>0: @@ -1436,7 +1482,7 @@ def new_vnf(mydb, tenant_id, vnf_descriptor): error_text = "Exception at database" elif isinstance(e, KeyError): error_text = "KeyError exception " - e.http_code = HTTP_Internal_Server_Error + e.http_code = httperrors.Internal_Server_Error else: error_text = "Exception at VIM" error_text += " {} {}. {}".format(type(e).__name__, str(e), message) @@ -1457,7 +1503,7 @@ def new_vnf_v02(mydb, tenant_id, vnf_descriptor): if "tenant_id" in vnf_descriptor["vnf"]: if vnf_descriptor["vnf"]["tenant_id"] != tenant_id: raise NfvoException("VNF can not have a different tenant owner '{}', must be '{}'".format(vnf_descriptor["vnf"]["tenant_id"], tenant_id), - HTTP_Unauthorized) + httperrors.Unauthorized) else: vnf_descriptor['vnf']['tenant_id'] = tenant_id # Step 3. Get the URL of the VIM from the nfvo_tenant and the datacenter @@ -1514,9 +1560,9 @@ def new_vnf_v02(mydb, tenant_id, vnf_descriptor): # result2, message = rollback(myvim, myvimURL, myvim_tenant, flavorList, imageList) # if result2: # print "Error creating flavor: unknown processor model. Rollback successful." - # return -HTTP_Bad_Request, "Error creating flavor: unknown processor model. Rollback successful." + # return -httperrors.Bad_Request, "Error creating flavor: unknown processor model. Rollback successful." # else: - # return -HTTP_Bad_Request, "Error creating flavor: unknown processor model. Rollback fail: you need to access VIM and delete the following %s" % message + # return -httperrors.Bad_Request, "Error creating flavor: unknown processor model. Rollback fail: you need to access VIM and delete the following %s" % message myflavorDict['extended']['processor_ranking'] = 100 #Hardcoded value, while we decide when the mapping is done if 'numas' in vnfc and len(vnfc['numas'])>0: @@ -1572,7 +1618,7 @@ def new_vnf_v02(mydb, tenant_id, vnf_descriptor): error_text = "Exception at database" elif isinstance(e, KeyError): error_text = "KeyError exception " - e.http_code = HTTP_Internal_Server_Error + e.http_code = httperrors.Internal_Server_Error else: error_text = "Exception at VIM" error_text += " {} {}. {}".format(type(e).__name__, str(e), message) @@ -1601,7 +1647,7 @@ def get_vnf_id(mydb, tenant_id, vnf_id): 'boot_data'), WHERE={'vnfs.uuid': vnf_id} ) if len(content)==0: - raise NfvoException("vnf '{}' not found".format(vnf_id), HTTP_Not_Found) + raise NfvoException("vnf '{}' not found".format(vnf_id), httperrors.Not_Found) # change boot_data into boot-data for vm in content: if vm.get("boot_data"): @@ -1625,7 +1671,7 @@ def get_vnf_id(mydb, tenant_id, vnf_id): if len(ipprofiles)==1: net["ip_profile"] = ipprofiles[0] elif len(ipprofiles)>1: - raise NfvoException("More than one ip-profile found with this criteria: net_id='{}'".format(net['uuid']), HTTP_Bad_Request) + raise NfvoException("More than one ip-profile found with this criteria: net_id='{}'".format(net['uuid']), httperrors.Bad_Request) #TODO: For each net, GET its elements and relevant info per element (VNFC, iface, ip_address) and include them in the output. @@ -1669,7 +1715,7 @@ def delete_vnf(mydb,tenant_id,vnf_id,datacenter=None,vim_tenant=None): deleted = mydb.delete_row_by_id('vnfs', vnf_id) if deleted == 0: - raise NfvoException("vnf '{}' not found".format(vnf_id), HTTP_Not_Found) + raise NfvoException("vnf '{}' not found".format(vnf_id), httperrors.Not_Found) undeletedItems = [] for flavor in flavorList: @@ -1751,7 +1797,7 @@ def get_hosts_info(mydb, nfvo_tenant_id, datacenter_name=None): if result < 0: return result, vims elif result == 0: - return -HTTP_Not_Found, "datacenter '%s' not found" % datacenter_name + return -httperrors.Not_Found, "datacenter '%s' not found" % datacenter_name myvim = vims.values()[0] result,servers = myvim.get_hosts_info() if result < 0: @@ -1763,10 +1809,10 @@ def get_hosts_info(mydb, nfvo_tenant_id, datacenter_name=None): def get_hosts(mydb, nfvo_tenant_id): vims = get_vim(mydb, nfvo_tenant_id) if len(vims) == 0: - raise NfvoException("No datacenter found for tenant '{}'".format(str(nfvo_tenant_id)), HTTP_Not_Found) + raise NfvoException("No datacenter found for tenant '{}'".format(str(nfvo_tenant_id)), httperrors.Not_Found) elif len(vims)>1: #print "nfvo.datacenter_action() error. Several datacenters found" - raise NfvoException("More than one datacenters found, try to identify with uuid", HTTP_Conflict) + raise NfvoException("More than one datacenters found, try to identify with uuid", httperrors.Conflict) myvim = vims.values()[0] try: hosts = myvim.get_hosts() @@ -1808,7 +1854,7 @@ def new_scenario(mydb, tenant_id, topo): if "tenant_id" in topo: if topo["tenant_id"] != tenant_id: raise NfvoException("VNF can not have a different tenant owner '{}', must be '{}'".format(topo["tenant_id"], tenant_id), - HTTP_Unauthorized) + httperrors.Unauthorized) else: tenant_id=None @@ -1840,15 +1886,15 @@ def new_scenario(mydb, tenant_id, topo): error_text += " 'VNF model' " + vnf['VNF model'] where['name'] = vnf['VNF model'] if len(where) == 1: - raise NfvoException("Descriptor need a 'vnf_id' or 'VNF model' field at " + error_pos, HTTP_Bad_Request) + raise NfvoException("Descriptor need a 'vnf_id' or 'VNF model' field at " + error_pos, httperrors.Bad_Request) vnf_db = mydb.get_rows(SELECT=('uuid','name','description'), FROM='vnfs', WHERE=where) if len(vnf_db)==0: - raise NfvoException("unknown" + error_text + " at " + error_pos, HTTP_Not_Found) + raise NfvoException("unknown" + error_text + " at " + error_pos, httperrors.Not_Found) elif len(vnf_db)>1: - raise NfvoException("more than one" + error_text + " at " + error_pos + " Concrete with 'vnf_id'", HTTP_Conflict) + raise NfvoException("more than one" + error_text + " at " + error_pos + " Concrete with 'vnf_id'", httperrors.Conflict) vnf['uuid']=vnf_db[0]['uuid'] vnf['description']=vnf_db[0]['description'] #get external interfaces @@ -1874,7 +1920,7 @@ def new_scenario(mydb, tenant_id, topo): con_type = conections[k].get("type", "link") if con_type != "link": if k in other_nets: - raise NfvoException("Format error. Reapeted network name at 'topology':'connections':'{}'".format(str(k)), HTTP_Bad_Request) + raise NfvoException("Format error. Reapeted network name at 'topology':'connections':'{}'".format(str(k)), httperrors.Bad_Request) other_nets[k] = {'external': False} if conections[k].get("graph"): other_nets[k]["graph"] = conections[k]["graph"] @@ -1897,10 +1943,10 @@ def new_scenario(mydb, tenant_id, topo): for iface in ifaces_list: if iface[0] not in vnfs and iface[0] not in other_nets : raise NfvoException("format error. Invalid VNF name at 'topology':'connections':'{}':'nodes':'{}'".format( - str(k), iface[0]), HTTP_Not_Found) + str(k), iface[0]), httperrors.Not_Found) if iface[0] in vnfs and iface[1] not in vnfs[ iface[0] ]['ifaces']: raise NfvoException("format error. Invalid interface name at 'topology':'connections':'{}':'nodes':'{}':'{}'".format( - str(k), iface[0], iface[1]), HTTP_Not_Found) + str(k), iface[0], iface[1]), httperrors.Not_Found) #1.5 unify connections from the pair list to a consolidated list index=0 @@ -1937,13 +1983,13 @@ def new_scenario(mydb, tenant_id, topo): if 'name' not in net: net['name']=k if 'model' not in net: - raise NfvoException("needed a 'model' at " + error_pos, HTTP_Bad_Request) + raise NfvoException("needed a 'model' at " + error_pos, httperrors.Bad_Request) if net['model']=='bridge_net': net['type']='bridge'; elif net['model']=='dataplane_net': net['type']='data'; else: - raise NfvoException("unknown 'model' '"+ net['model'] +"' at " + error_pos, HTTP_Not_Found) + raise NfvoException("unknown 'model' '"+ net['model'] +"' at " + error_pos, httperrors.Not_Found) else: #external #IF we do not want to check that external network exist at datacenter pass @@ -1957,17 +2003,17 @@ def new_scenario(mydb, tenant_id, topo): # error_text += " 'model' " + net['model'] # WHERE_['name'] = net['model'] # if len(WHERE_) == 0: -# return -HTTP_Bad_Request, "needed a 'net_id' or 'model' at " + error_pos +# return -httperrors.Bad_Request, "needed a 'net_id' or 'model' at " + error_pos # r,net_db = mydb.get_table(SELECT=('uuid','name','description','type','shared'), # FROM='datacenter_nets', WHERE=WHERE_ ) # if r<0: # print "nfvo.new_scenario Error getting datacenter_nets",r,net_db # elif r==0: # print "nfvo.new_scenario Error" +error_text+ " is not present at database" -# return -HTTP_Bad_Request, "unknown " +error_text+ " at " + error_pos +# return -httperrors.Bad_Request, "unknown " +error_text+ " at " + error_pos # elif r>1: # print "nfvo.new_scenario Error more than one external_network for " +error_text+ " is present at database" -# return -HTTP_Bad_Request, "more than one external_network for " +error_text+ "at "+ error_pos + " Concrete with 'net_id'" +# return -httperrors.Bad_Request, "more than one external_network for " +error_text+ "at "+ error_pos + " Concrete with 'net_id'" # other_nets[k].update(net_db[0]) #ENDIF net_list={} @@ -1984,7 +2030,7 @@ def new_scenario(mydb, tenant_id, topo): if other_net_index>=0: error_text="There is some interface connected both to net '%s' and net '%s'" % (con[other_net_index][0], net_key) #print "nfvo.new_scenario " + error_text - raise NfvoException(error_text, HTTP_Bad_Request) + raise NfvoException(error_text, httperrors.Bad_Request) else: other_net_index = index net_target = net_key @@ -2008,7 +2054,7 @@ def new_scenario(mydb, tenant_id, topo): # if type_=='data' and other_nets[net_target]['type']=="ptp": # error_text = "Error connecting %d nodes on a not multipoint net %s" % (len(con), net_target) # print "nfvo.new_scenario " + error_text -# return -HTTP_Bad_Request, error_text +# return -httperrors.Bad_Request, error_text #ENDIF for iface in con: vnfs[ iface[0] ]['ifaces'][ iface[1] ]['net_key'] = net_target @@ -2030,7 +2076,7 @@ def new_scenario(mydb, tenant_id, topo): if net_type_bridge and net_type_data: error_text = "Error connection interfaces of bridge type with data type. Firs node %s, iface %s" % (iface[0], iface[1]) #print "nfvo.new_scenario " + error_text - raise NfvoException(error_text, HTTP_Bad_Request) + raise NfvoException(error_text, httperrors.Bad_Request) elif net_type_bridge: type_='bridge' else: @@ -2041,7 +2087,7 @@ def new_scenario(mydb, tenant_id, topo): error_text = "Error connection node %s : %s does not match any VNF or interface" % (iface[0], iface[1]) #print "nfvo.new_scenario " + error_text #raise e - raise NfvoException(error_text, HTTP_Bad_Request) + raise NfvoException(error_text, httperrors.Bad_Request) #1.8: Connect to management net all not already connected interfaces of type 'mgmt' #1.8.1 obtain management net @@ -2088,7 +2134,7 @@ def new_scenario_v02(mydb, tenant_id, scenario_dict, version): if scenario["tenant_id"] != tenant_id: # print "nfvo.new_scenario_v02() tenant '%s' not found" % tenant_id raise NfvoException("VNF can not have a different tenant owner '{}', must be '{}'".format( - scenario["tenant_id"], tenant_id), HTTP_Unauthorized) + scenario["tenant_id"], tenant_id), httperrors.Unauthorized) else: tenant_id=None @@ -2104,14 +2150,14 @@ def new_scenario_v02(mydb, tenant_id, scenario_dict, version): error_text += " 'vnf_name' " + vnf['vnf_name'] where['name'] = vnf['vnf_name'] if len(where) == 1: - raise NfvoException("Needed a 'vnf_id' or 'vnf_name' at " + error_pos, HTTP_Bad_Request) + raise NfvoException("Needed a 'vnf_id' or 'vnf_name' at " + error_pos, httperrors.Bad_Request) vnf_db = mydb.get_rows(SELECT=('uuid', 'name', 'description'), FROM='vnfs', WHERE=where) if len(vnf_db) == 0: - raise NfvoException("Unknown" + error_text + " at " + error_pos, HTTP_Not_Found) + raise NfvoException("Unknown" + error_text + " at " + error_pos, httperrors.Not_Found) elif len(vnf_db) > 1: - raise NfvoException("More than one" + error_text + " at " + error_pos + " Concrete with 'vnf_id'", HTTP_Conflict) + raise NfvoException("More than one" + error_text + " at " + error_pos + " Concrete with 'vnf_id'", httperrors.Conflict) vnf['uuid'] = vnf_db[0]['uuid'] vnf['description'] = vnf_db[0]['description'] vnf['ifaces'] = {} @@ -2139,17 +2185,17 @@ def new_scenario_v02(mydb, tenant_id, scenario_dict, version): error_text = "Error at 'networks':'{}':'interfaces' VNF '{}' not match any VNF at 'vnfs'".format( net_name, vnf) # logger.debug("nfvo.new_scenario_v02 " + error_text) - raise NfvoException(error_text, HTTP_Not_Found) + raise NfvoException(error_text, httperrors.Not_Found) if iface not in scenario["vnfs"][vnf]['ifaces']: error_text = "Error at 'networks':'{}':'interfaces':'{}' interface not match any VNF interface"\ .format(net_name, iface) # logger.debug("nfvo.new_scenario_v02 " + error_text) - raise NfvoException(error_text, HTTP_Bad_Request) + raise NfvoException(error_text, httperrors.Bad_Request) if "net_key" in scenario["vnfs"][vnf]['ifaces'][iface]: error_text = "Error at 'networks':'{}':'interfaces':'{}' interface already connected at network"\ "'{}'".format(net_name, iface,scenario["vnfs"][vnf]['ifaces'][iface]['net_key']) # logger.debug("nfvo.new_scenario_v02 " + error_text) - raise NfvoException(error_text, HTTP_Bad_Request) + raise NfvoException(error_text, httperrors.Bad_Request) scenario["vnfs"][vnf]['ifaces'][ iface ]['net_key'] = net_name scenario["vnfs"][vnf]['ifaces'][iface]['ip_address'] = ip_address iface_type = scenario["vnfs"][vnf]['ifaces'][iface]['type'] @@ -2162,7 +2208,7 @@ def new_scenario_v02(mydb, tenant_id, scenario_dict, version): error_text = "Error connection interfaces of 'bridge' type and 'data' type at 'networks':'{}':'interfaces'"\ .format(net_name) # logger.debug("nfvo.new_scenario " + error_text) - raise NfvoException(error_text, HTTP_Bad_Request) + raise NfvoException(error_text, httperrors.Bad_Request) elif net_type_bridge: type_ = 'bridge' else: @@ -2173,19 +2219,19 @@ def new_scenario_v02(mydb, tenant_id, scenario_dict, version): error_text = "Error connecting interfaces of data type to a network declared as 'underlay' at "\ "'network':'{}'".format(net_name) # logger.debug(error_text) - raise NfvoException(error_text, HTTP_Bad_Request) + raise NfvoException(error_text, httperrors.Bad_Request) elif type_ != "bridge" and net["implementation"] == "overlay": error_text = "Error connecting interfaces of data type to a network declared as 'overlay' at "\ "'network':'{}'".format(net_name) # logger.debug(error_text) - raise NfvoException(error_text, HTTP_Bad_Request) + raise NfvoException(error_text, httperrors.Bad_Request) net.pop("implementation") if "type" in net and version == "0.3": # for v0.3 if type_ == "data" and net["type"] == "e-line": error_text = "Error connecting more than 2 interfaces of data type to a network declared as type "\ "'e-line' at 'network':'{}'".format(net_name) # logger.debug(error_text) - raise NfvoException(error_text, HTTP_Bad_Request) + raise NfvoException(error_text, httperrors.Bad_Request) elif type_ == "ptp" and net["type"] == "e-lan": type_ = "data" @@ -2213,7 +2259,7 @@ def new_nsd_v3(mydb, tenant_id, nsd_descriptor): try: pybindJSONDecoder.load_ietf_json(nsd_descriptor, None, None, obj=mynsd) except Exception as e: - raise NfvoException("Error. Invalid NS descriptor format: " + str(e), HTTP_Bad_Request) + raise NfvoException("Error. Invalid NS descriptor format: " + str(e), httperrors.Bad_Request) db_scenarios = [] db_sce_nets = [] db_sce_vnfs = [] @@ -2256,7 +2302,7 @@ def new_nsd_v3(mydb, tenant_id, nsd_descriptor): raise NfvoException("Error. Invalid NS descriptor at 'nsd[{}]':'constituent-vnfd':'vnfd-id-ref':" "'{}'. Reference to a non-existing VNFD in the catalog".format( str(nsd["id"]), str(vnf["vnfd-id-ref"])[:255]), - HTTP_Bad_Request) + httperrors.Bad_Request) sce_vnf_uuid = str(uuid4()) uuid_list.append(sce_vnf_uuid) db_sce_vnf = { @@ -2325,7 +2371,7 @@ def new_nsd_v3(mydb, tenant_id, nsd_descriptor): raise NfvoException("Error. Invalid NS descriptor at 'nsd[{}]':'vld[{}]':'ip-profile-ref':'{}'." " Reference to a non-existing 'ip_profiles'".format( str(nsd["id"]), str(vld["id"]), str(vld["ip-profile-ref"])), - HTTP_Bad_Request) + httperrors.Bad_Request) db_ip_profiles[ip_profile_name2db_table_index[ip_profile_name]]["sce_net_id"] = sce_net_uuid elif vld.get("vim-network-name"): db_sce_net["vim_network_name"] = get_str(vld, "vim-network-name", 255) @@ -2339,7 +2385,7 @@ def new_nsd_v3(mydb, tenant_id, nsd_descriptor): "-ref':'member-vnf-index-ref':'{}'. Reference to a non-existing index at " "'nsd':'constituent-vnfd'".format( str(nsd["id"]), str(vld["id"]), str(iface["member-vnf-index-ref"])), - HTTP_Bad_Request) + httperrors.Bad_Request) existing_ifaces = mydb.get_rows(SELECT=('i.uuid as uuid', 'i.type as iface_type'), FROM="interfaces as i join vms on i.vm_id=vms.uuid", @@ -2352,7 +2398,7 @@ def new_nsd_v3(mydb, tenant_id, nsd_descriptor): "connection-point name at VNFD '{}'".format( str(nsd["id"]), str(vld["id"]), str(iface["vnfd-connection-point-ref"]), str(iface.get("vnfd-id-ref"))[:255]), - HTTP_Bad_Request) + httperrors.Bad_Request) interface_uuid = existing_ifaces[0]["uuid"] if existing_ifaces[0]["iface_type"] == "data" and not db_sce_net["type"]: db_sce_net["type"] = "data" @@ -2407,7 +2453,7 @@ def new_nsd_v3(mydb, tenant_id, nsd_descriptor): "-ref':'member-vnf-index-ref':'{}'. Reference to a non-existing index at " "'nsd':'constituent-vnfd'".format( str(nsd["id"]), str(rsp["id"]), str(iface["member-vnf-index-ref"])), - HTTP_Bad_Request) + httperrors.Bad_Request) existing_ifaces = mydb.get_rows(SELECT=('i.uuid as uuid',), FROM="interfaces as i join vms on i.vm_id=vms.uuid", @@ -2420,7 +2466,7 @@ def new_nsd_v3(mydb, tenant_id, nsd_descriptor): "connection-point name at VNFD '{}'".format( str(nsd["id"]), str(rsp["id"]), str(iface["vnfd-connection-point-ref"]), str(iface.get("vnfd-id-ref"))[:255]), - HTTP_Bad_Request) + httperrors.Bad_Request) interface_uuid = existing_ifaces[0]["uuid"] sce_rsp_hop_uuid = str(uuid4()) uuid_list.append(sce_rsp_hop_uuid) @@ -2446,7 +2492,7 @@ def new_nsd_v3(mydb, tenant_id, nsd_descriptor): "-ref':'member-vnf-index-ref':'{}'. Reference to a non-existing index at " "'nsd':'constituent-vnfd'".format( str(nsd["id"]), str(classifier["id"]), str(classifier["member-vnf-index-ref"])), - HTTP_Bad_Request) + httperrors.Bad_Request) existing_ifaces = mydb.get_rows(SELECT=('i.uuid as uuid',), FROM="interfaces as i join vms on i.vm_id=vms.uuid", WHERE={'vnf_id': vnf_index2vnf_uuid[vnf_index], @@ -2458,7 +2504,7 @@ def new_nsd_v3(mydb, tenant_id, nsd_descriptor): "connection-point name at VNFD '{}'".format( str(nsd["id"]), str(rsp["id"]), str(iface["vnfd-connection-point-ref"]), str(iface.get("vnfd-id-ref"))[:255]), - HTTP_Bad_Request) + httperrors.Bad_Request) interface_uuid = existing_ifaces[0]["uuid"] db_sce_classifier = { @@ -2514,7 +2560,7 @@ def new_nsd_v3(mydb, tenant_id, nsd_descriptor): raise except Exception as e: logger.error("Exception {}".format(e)) - raise # NfvoException("Exception {}".format(e), HTTP_Bad_Request) + raise # NfvoException("Exception {}".format(e), httperrors.Bad_Request) def edit_scenario(mydb, tenant_id, scenario_id, data): @@ -2575,7 +2621,7 @@ def start_scenario(mydb, tenant_id, scenario_id, instance_scenario_name, instanc error_text = "Error, datacenter '%s' does not have external network '%s'." % (datacenter_name, sce_net['name']) _, message = rollback(mydb, vims, rollbackList) logger.error("nfvo.start_scenario: %s", error_text) - raise NfvoException(error_text, HTTP_Bad_Request) + raise NfvoException(error_text, httperrors.Bad_Request) logger.debug("Using existent VIM network for scenario %s. Network id %s", scenarioDict['name'],sce_net['vim_id']) auxNetDict['scenario'][sce_net['uuid']] = sce_net['vim_id'] @@ -2622,7 +2668,7 @@ def start_scenario(mydb, tenant_id, scenario_id, instance_scenario_name, instanc # check if there is enough availability zones available at vim level. if myvims[datacenter_id].availability_zone and vnf_availability_zones: if len(vnf_availability_zones) > len(myvims[datacenter_id].availability_zone): - raise NfvoException('No enough availability zones at VIM for this deployment', HTTP_Bad_Request) + raise NfvoException('No enough availability zones at VIM for this deployment', httperrors.Bad_Request) for vm in sce_vnf['vms']: i += 1 @@ -2678,9 +2724,9 @@ def start_scenario(mydb, tenant_id, scenario_id, instance_scenario_name, instanc e_text = "Cannot determine the interface type PF or VF of VNF '%s' VM '%s' iface '%s'" %(sce_vnf['name'], vm['name'], iface['internal_name']) if flavor_dict.get('extended')==None: raise NfvoException(e_text + "After database migration some information is not available. \ - Try to delete and create the scenarios and VNFs again", HTTP_Conflict) + Try to delete and create the scenarios and VNFs again", httperrors.Conflict) else: - raise NfvoException(e_text, HTTP_Internal_Server_Error) + raise NfvoException(e_text, httperrors.Internal_Server_Error) if netDict["use"]=="mgmt" or netDict["use"]=="bridge": netDict["type"]="virtual" if "vpci" in iface and iface["vpci"] is not None: @@ -2857,12 +2903,12 @@ def get_vim_thread(mydb, tenant_id, datacenter_id_name=None, datacenter_tenant_i "join datacenters as d on d.uuid=dt.datacenter_id", WHERE=where_) if len(datacenters) > 1: - raise NfvoException("More than one datacenters found, try to identify with uuid", HTTP_Conflict) + raise NfvoException("More than one datacenters found, try to identify with uuid", httperrors.Conflict) elif datacenters: thread_id = datacenters[0]["datacenter_tenant_id"] thread = vim_threads["running"].get(thread_id) if not thread: - raise NfvoException("datacenter '{}' not found".format(str(datacenter_id_name)), HTTP_Not_Found) + raise NfvoException("datacenter '{}' not found".format(str(datacenter_id_name)), httperrors.Not_Found) return thread_id, thread except db_base_Exception as e: raise NfvoException("{} {}".format(type(e).__name__ , str(e)), e.http_code) @@ -2883,10 +2929,10 @@ def get_datacenter_uuid(mydb, tenant_id, datacenter_id_name): from_ = 'datacenters as d' vimaccounts = mydb.get_rows(FROM=from_, SELECT=("d.uuid as uuid, d.name as name",), WHERE=WHERE_dict ) if len(vimaccounts) == 0: - raise NfvoException("datacenter '{}' not found".format(str(datacenter_id_name)), HTTP_Not_Found) + raise NfvoException("datacenter '{}' not found".format(str(datacenter_id_name)), httperrors.Not_Found) elif len(vimaccounts)>1: #print "nfvo.datacenter_action() error. Several datacenters found" - raise NfvoException("More than one datacenters found, try to identify with uuid", HTTP_Conflict) + raise NfvoException("More than one datacenters found, try to identify with uuid", httperrors.Conflict) return vimaccounts[0]["uuid"], vimaccounts[0]["name"] @@ -2900,10 +2946,10 @@ def get_datacenter_by_name_uuid(mydb, tenant_id, datacenter_id_name=None, **extr datacenter_name = datacenter_id_name vims = get_vim(mydb, tenant_id, datacenter_id, datacenter_name, **extra_filter) if len(vims) == 0: - raise NfvoException("datacenter '{}' not found".format(str(datacenter_id_name)), HTTP_Not_Found) + raise NfvoException("datacenter '{}' not found".format(str(datacenter_id_name)), httperrors.Not_Found) elif len(vims)>1: #print "nfvo.datacenter_action() error. Several datacenters found" - raise NfvoException("More than one datacenters found, try to identify with uuid", HTTP_Conflict) + raise NfvoException("More than one datacenters found, try to identify with uuid", httperrors.Conflict) return vims.keys()[0], vims.values()[0] @@ -3010,7 +3056,7 @@ def create_instance(mydb, tenant_id, instance_dict): break else: raise NfvoException("Invalid scenario network name or id '{}' at instance:networks".format(net_name), - HTTP_Bad_Request) + httperrors.Bad_Request) if "sites" not in net_instance_desc: net_instance_desc["sites"] = [ {} ] site_without_datacenter_field = False @@ -3026,7 +3072,7 @@ def create_instance(mydb, tenant_id, instance_dict): else: if site_without_datacenter_field: raise NfvoException("Found more than one entries without datacenter field at " - "instance:networks:{}:sites".format(net_name), HTTP_Bad_Request) + "instance:networks:{}:sites".format(net_name), httperrors.Bad_Request) site_without_datacenter_field = True site["datacenter"] = default_datacenter_id # change name to id @@ -3035,7 +3081,7 @@ def create_instance(mydb, tenant_id, instance_dict): if vnf_name == scenario_vnf['member_vnf_index'] or vnf_name == scenario_vnf['uuid'] or vnf_name == scenario_vnf['name']: break else: - raise NfvoException("Invalid vnf name '{}' at instance:vnfs".format(vnf_name), HTTP_Bad_Request) + raise NfvoException("Invalid vnf name '{}' at instance:vnfs".format(vnf_name), httperrors.Bad_Request) if "datacenter" in vnf_instance_desc: # Add this datacenter to myvims vnf_instance_desc["datacenter"], _ = get_datacenter_uuid(mydb, tenant_id, vnf_instance_desc["datacenter"]) @@ -3050,7 +3096,7 @@ def create_instance(mydb, tenant_id, instance_dict): if net_id == scenario_net['osm_id'] or net_id == scenario_net['uuid'] or net_id == scenario_net["name"]: break else: - raise NfvoException("Invalid net id or name '{}' at instance:vnfs:networks".format(net_id), HTTP_Bad_Request) + raise NfvoException("Invalid net id or name '{}' at instance:vnfs:networks".format(net_id), httperrors.Bad_Request) if net_instance_desc.get("vim-network-name"): scenario_net["vim-network-name"] = net_instance_desc["vim-network-name"] if net_instance_desc.get("name"): @@ -3067,7 +3113,7 @@ def create_instance(mydb, tenant_id, instance_dict): if vdu_id == scenario_vm['osm_id'] or vdu_id == scenario_vm["name"]: break else: - raise NfvoException("Invalid vdu id or name '{}' at instance:vnfs:vdus".format(vdu_id), HTTP_Bad_Request) + raise NfvoException("Invalid vdu id or name '{}' at instance:vnfs:vdus".format(vdu_id), httperrors.Bad_Request) scenario_vm["instance_parameters"] = vdu_instance_desc for iface_id, iface_instance_desc in vdu_instance_desc.get("interfaces", {}).iteritems(): for scenario_interface in scenario_vm['interfaces']: @@ -3075,7 +3121,7 @@ def create_instance(mydb, tenant_id, instance_dict): scenario_interface.update(iface_instance_desc) break else: - raise NfvoException("Invalid vdu id or name '{}' at instance:vnfs:vdus".format(vdu_id), HTTP_Bad_Request) + raise NfvoException("Invalid vdu id or name '{}' at instance:vnfs:vdus".format(vdu_id), httperrors.Bad_Request) # 0.1 parse cloud-config parameters cloud_config = unify_cloud_config(instance_dict.get("cloud-config"), scenarioDict.get("cloud-config")) @@ -3181,7 +3227,7 @@ def create_instance(mydb, tenant_id, instance_dict): if number_mgmt_networks > 1: raise NfvoException("Found several VLD of type mgmt. " "You must concrete what vim-network must be use for each one", - HTTP_Bad_Request) + httperrors.Bad_Request) create_network = False lookfor_network = True if vim["config"].get("management_network_id"): @@ -3431,10 +3477,17 @@ def create_instance(mydb, tenant_id, instance_dict): } task_index += 1 db_vim_actions.append(db_vim_action) + db_instance_action["number_tasks"] = task_index + + # --> WIM + wan_links = wim_engine.derive_wan_links(db_instance_nets, tenant_id) + wim_actions = wim_engine.create_actions(wan_links) + wim_actions, db_instance_action = ( + wim_engine.incorporate_actions(wim_actions, db_instance_action)) + # <-- WIM scenarioDict["datacenter2tenant"] = myvim_threads_id - db_instance_action["number_tasks"] = task_index db_instance_scenario['datacenter_tenant_id'] = myvim_threads_id[default_datacenter_id] db_instance_scenario['datacenter_id'] = default_datacenter_id db_tables=[ @@ -3449,7 +3502,8 @@ def create_instance(mydb, tenant_id, instance_dict): {"instance_sfs": db_instance_sfs}, {"instance_classifications": db_instance_classifications}, {"instance_sfps": db_instance_sfps}, - {"vim_actions": db_vim_actions} + {"instance_wim_nets": wan_links}, + {"vim_wim_actions": db_vim_actions + wim_actions} ] logger.debug("create_instance done DB tables: %s", @@ -3458,6 +3512,8 @@ def create_instance(mydb, tenant_id, instance_dict): for myvim_thread_id in myvim_threads_id.values(): vim_threads["running"][myvim_thread_id].insert_task(db_vim_actions) + wim_engine.dispatch(wim_actions) + returned_instance = mydb.get_instance_scenario(instance_uuid) returned_instance["action_id"] = instance_action_id return returned_instance @@ -3471,6 +3527,7 @@ def create_instance(mydb, tenant_id, instance_dict): error_text = "Exception" error_text += " {} {}. {}".format(type(e).__name__, str(e), message) # logger.error("create_instance: %s", error_text) + logger.exception(e) raise NfvoException(error_text, e.http_code) @@ -3587,7 +3644,7 @@ def instantiate_vnf(mydb, sce_vnf, params, params_out, rollbackList): # check if there is enough availability zones available at vim level. if myvims[datacenter_id].availability_zone and vnf_availability_zones: if len(vnf_availability_zones) > len(myvims[datacenter_id].availability_zone): - raise NfvoException('No enough availability zones at VIM for this deployment', HTTP_Bad_Request) + raise NfvoException('No enough availability zones at VIM for this deployment', httperrors.Bad_Request) if sce_vnf.get("datacenter"): vim = myvims[sce_vnf["datacenter"]] @@ -3643,7 +3700,7 @@ def instantiate_vnf(mydb, sce_vnf, params, params_out, rollbackList): extended_flavor_dict = mydb.get_rows(FROM='datacenters_flavors', SELECT=('extended',), WHERE={'vim_id': flavor_id}) if not extended_flavor_dict: - raise NfvoException("flavor '{}' not found".format(flavor_id), HTTP_Not_Found) + raise NfvoException("flavor '{}' not found".format(flavor_id), httperrors.Not_Found) # extended_flavor_dict_yaml = yaml.load(extended_flavor_dict[0]) myVMDict['disks'] = None @@ -3693,9 +3750,9 @@ def instantiate_vnf(mydb, sce_vnf, params, params_out, rollbackList): sce_vnf['name'], vm['name'], iface['internal_name']) if flavor_dict.get('extended') == None: raise NfvoException(e_text + "After database migration some information is not available. \ - Try to delete and create the scenarios and VNFs again", HTTP_Conflict) + Try to delete and create the scenarios and VNFs again", httperrors.Conflict) else: - raise NfvoException(e_text, HTTP_Internal_Server_Error) + raise NfvoException(e_text, httperrors.Internal_Server_Error) if netDict["use"] == "mgmt" or netDict["use"] == "bridge": netDict["type"] = "virtual" if iface.get("vpci"): @@ -3829,6 +3886,14 @@ def delete_instance(mydb, tenant_id, instance_id): instanceDict = mydb.get_instance_scenario(instance_id, tenant_id) # print yaml.safe_dump(instanceDict, indent=4, default_flow_style=False) tenant_id = instanceDict["tenant_id"] + + # --> WIM + # We need to retrieve the WIM Actions now, before the instance_scenario is + # deleted. The reason for that is that: ON CASCADE rules will delete the + # instance_wim_nets record in the database + wim_actions = wim_engine.delete_actions(instance_scenario_id=instance_id) + # <-- WIM + # print "Checking that nfvo_tenant_id exists and getting the VIM URI and the VIM tenant_id" # 1. Delete from Database message = mydb.delete_instance_scenario(instance_id, tenant_id) @@ -4089,9 +4154,15 @@ def delete_instance(mydb, tenant_id, instance_id): db_vim_actions.append(db_vim_action) db_instance_action["number_tasks"] = task_index + + # --> WIM + wim_actions, db_instance_action = ( + wim_engine.incorporate_actions(wim_actions, db_instance_action)) + # <-- WIM + db_tables = [ {"instance_actions": db_instance_action}, - {"vim_actions": db_vim_actions} + {"vim_wim_actions": db_vim_actions + wim_actions} ] logger.debug("delete_instance done DB tables: %s", @@ -4100,6 +4171,8 @@ def delete_instance(mydb, tenant_id, instance_id): for myvim_thread_id in vimthread_affected.keys(): vim_threads["running"][myvim_thread_id].insert_task(db_vim_actions) + wim_engine.dispatch(wim_actions) + if len(error_msg) > 0: return 'action_id={} instance {} deleted but some elements could not be deleted, or already deleted '\ '(error: 404) from VIM: {}'.format(instance_action_id, message, error_msg) @@ -4300,7 +4373,7 @@ def instance_action(mydb,nfvo_tenant,instance_id, action_dict): #print "Checking that nfvo_tenant_id exists and getting the VIM URI and the VIM tenant_id" vims = get_vim(mydb, nfvo_tenant, instanceDict['datacenter_id']) if len(vims) == 0: - raise NfvoException("datacenter '{}' not found".format(str(instanceDict['datacenter_id'])), HTTP_Not_Found) + raise NfvoException("datacenter '{}' not found".format(str(instanceDict['datacenter_id'])), httperrors.Not_Found) myvim = vims.values()[0] vm_result = {} vm_error = 0 @@ -4332,7 +4405,7 @@ def instance_action(mydb,nfvo_tenant,instance_id, action_dict): ORDER_BY="vms.created_at" ) if not target_vm: - raise NfvoException("Cannot find the vdu with id {}".format(vdu_id), HTTP_Not_Found) + raise NfvoException("Cannot find the vdu with id {}".format(vdu_id), httperrors.Not_Found) else: if not osm_vdu_id and not member_vnf_index: raise NfvoException("Invalid imput vdu parameters. Must supply either 'vdu-id' of 'osm_vdu_id','member-vnf-index'") @@ -4345,7 +4418,7 @@ def instance_action(mydb,nfvo_tenant,instance_id, action_dict): ORDER_BY="ivms.created_at" ) if not target_vm: - raise NfvoException("Cannot find the vdu with osm_vdu_id {} and member-vnf-index {}".format(osm_vdu_id, member_vnf_index), HTTP_Not_Found) + raise NfvoException("Cannot find the vdu with osm_vdu_id {} and member-vnf-index {}".format(osm_vdu_id, member_vnf_index), httperrors.Not_Found) vdu_id = target_vm[-1]["uuid"] vm_result[vdu_id] = {"created": [], "deleted": [], "description": "scheduled"} target_vm = target_vm[-1] @@ -4383,7 +4456,7 @@ def instance_action(mydb,nfvo_tenant,instance_id, action_dict): vim_action_to_clone = mydb.get_rows(FROM="vim_actions", WHERE=where) if not vim_action_to_clone: - raise NfvoException("Cannot find the vim_action at database with {}".format(where), HTTP_Internal_Server_Error) + raise NfvoException("Cannot find the vim_action at database with {}".format(where), httperrors.Internal_Server_Error) vim_action_to_clone = vim_action_to_clone[0] extra = yaml.safe_load(vim_action_to_clone["extra"]) @@ -4516,13 +4589,13 @@ def instance_action(mydb,nfvo_tenant,instance_id, action_dict): password=password, ro_key=priv_RO_key) else: raise NfvoException("Unable to inject ssh key in vm: {} - Aborting".format(vm['uuid']), - HTTP_Internal_Server_Error) + httperrors.Internal_Server_Error) except KeyError: raise NfvoException("Unable to inject ssh key in vm: {} - Aborting".format(vm['uuid']), - HTTP_Internal_Server_Error) + httperrors.Internal_Server_Error) else: raise NfvoException("Unable to inject ssh key in vm: {} - Aborting".format(vm['uuid']), - HTTP_Internal_Server_Error) + httperrors.Internal_Server_Error) else: data = myvim.action_vminstance(vm['vim_vm_id'], action_dict) if "console" in action_dict: @@ -4537,7 +4610,7 @@ def instance_action(mydb,nfvo_tenant,instance_id, action_dict): } vm_ok +=1 elif data["server"]=="127.0.0.1" or data["server"]=="localhost": - vm_result[ vm['uuid'] ] = {"vim_result": -HTTP_Unauthorized, + vm_result[ vm['uuid'] ] = {"vim_result": -httperrors.Unauthorized, "description": "this console is only reachable by local interface", "name":vm['name'] } @@ -4582,9 +4655,9 @@ def instance_action_get(mydb, nfvo_tenant, instance_id, action_id): rows = mydb.get_rows(FROM="instance_actions", WHERE=filter) if action_id: if not rows: - raise NfvoException("Not found any action with this criteria", HTTP_Not_Found) - vim_actions = mydb.get_rows(FROM="vim_actions", WHERE={"instance_action_id": action_id}) - rows[0]["vim_actions"] = vim_actions + raise NfvoException("Not found any action with this criteria", httperrors.Not_Found) + vim_wim_actions = mydb.get_rows(FROM="vim_wim_actions", WHERE={"instance_action_id": action_id}) + rows[0]["vim_wim_actions"] = vim_wim_actions return {"actions": rows} @@ -4609,15 +4682,15 @@ def create_or_use_console_proxy_thread(console_server, console_port): #port used, try with onoher continue except cli.ConsoleProxyException as e: - raise NfvoException(str(e), HTTP_Bad_Request) - raise NfvoException("Not found any free 'http_console_ports'", HTTP_Conflict) + raise NfvoException(str(e), httperrors.Bad_Request) + raise NfvoException("Not found any free 'http_console_ports'", httperrors.Conflict) def check_tenant(mydb, tenant_id): '''check that tenant exists at database''' tenant = mydb.get_rows(FROM='nfvo_tenants', SELECT=('uuid',), WHERE={'uuid': tenant_id}) if not tenant: - raise NfvoException("tenant '{}' not found".format(tenant_id), HTTP_Not_Found) + raise NfvoException("tenant '{}' not found".format(tenant_id), httperrors.Not_Found) return def new_tenant(mydb, tenant_dict): @@ -4660,7 +4733,7 @@ def new_datacenter(mydb, datacenter_descriptor): # file.close(module_info[0]) raise NfvoException("Incorrect datacenter type '{}'. Plugin '{}.py' not installed".format(datacenter_type, module), - HTTP_Bad_Request) + httperrors.Bad_Request) datacenter_id = mydb.new_row("datacenters", datacenter_descriptor, add_uuid=True, confidential_data=True) if sdn_port_mapping: @@ -4705,7 +4778,7 @@ def edit_datacenter(mydb, datacenter_id_name, datacenter_descriptor): for k in to_delete: del config_dict[k] except Exception as e: - raise NfvoException("Bad format at datacenter:config " + str(e), HTTP_Bad_Request) + raise NfvoException("Bad format at datacenter:config " + str(e), httperrors.Bad_Request) if config_dict: datacenter_descriptor["config"] = yaml.safe_dump(config_dict, default_flow_style=True, width=256) else: @@ -4744,7 +4817,7 @@ def create_vim_account(mydb, nfvo_tenant, datacenter_id, name=None, vim_id=None, try: if not datacenter_id: if not vim_id: - raise NfvoException("You must provide 'vim_id", http_code=HTTP_Bad_Request) + raise NfvoException("You must provide 'vim_id", http_code=httperrors.Bad_Request) datacenter_id = vim_id datacenter_id, datacenter_name = get_datacenter_uuid(mydb, None, datacenter_id) @@ -4759,7 +4832,7 @@ def create_vim_account(mydb, nfvo_tenant, datacenter_id, name=None, vim_id=None, # #check that this association does not exist before # tenants_datacenters = mydb.get_rows(FROM='tenants_datacenters', WHERE=tenants_datacenter_dict) # if len(tenants_datacenters)>0: - # raise NfvoException("datacenter '{}' and tenant'{}' are already attached".format(datacenter_id, tenant_dict['uuid']), HTTP_Conflict) + # raise NfvoException("datacenter '{}' and tenant'{}' are already attached".format(datacenter_id, tenant_dict['uuid']), httperrors.Conflict) vim_tenant_id_exist_atdb=False if not create_vim_tenant: @@ -4785,7 +4858,7 @@ def create_vim_account(mydb, nfvo_tenant, datacenter_id, name=None, vim_id=None, datacenter_name = myvim["name"] vim_tenant = myvim.new_tenant(vim_tenant_name, "created by openmano for datacenter "+datacenter_name) except vimconn.vimconnException as e: - raise NfvoException("Not possible to create vim_tenant {} at VIM: {}".format(vim_tenant, str(e)), HTTP_Internal_Server_Error) + raise NfvoException("Not possible to create vim_tenant {} at VIM: {}".format(vim_tenant_id, str(e)), httperrors.Internal_Server_Error) datacenter_tenants_dict = {} datacenter_tenants_dict["created"]="true" @@ -4819,7 +4892,7 @@ def create_vim_account(mydb, nfvo_tenant, datacenter_id, name=None, vim_id=None, vim_threads["running"][thread_id] = new_thread return thread_id except vimconn.vimconnException as e: - raise NfvoException(str(e), HTTP_Bad_Request) + raise NfvoException(str(e), httperrors.Bad_Request) def edit_vim_account(mydb, nfvo_tenant, datacenter_tenant_id, datacenter_id=None, name=None, vim_tenant=None, @@ -4834,9 +4907,9 @@ def edit_vim_account(mydb, nfvo_tenant, datacenter_tenant_id, datacenter_id=None where_["dt.datacenter_id"] = datacenter_id vim_accounts = mydb.get_rows(SELECT="dt.uuid as uuid, config", FROM=from_, WHERE=where_) if not vim_accounts: - raise NfvoException("vim_account not found for this tenant", http_code=HTTP_Not_Found) + raise NfvoException("vim_account not found for this tenant", http_code=httperrors.Not_Found) elif len(vim_accounts) > 1: - raise NfvoException("found more than one vim_account for this tenant", http_code=HTTP_Conflict) + raise NfvoException("found more than one vim_account for this tenant", http_code=httperrors.Conflict) datacenter_tenant_id = vim_accounts[0]["uuid"] original_config = vim_accounts[0]["config"] @@ -4880,7 +4953,7 @@ def delete_vim_account(mydb, tenant_id, vim_account_id, datacenter=None): tenants_datacenter_dict["nfvo_tenant_id"] = tenant_uuid tenant_datacenter_list = mydb.get_rows(FROM='tenants_datacenters', WHERE=tenants_datacenter_dict) if len(tenant_datacenter_list)==0 and tenant_uuid: - raise NfvoException("datacenter '{}' and tenant '{}' are not attached".format(datacenter_id, tenant_dict['uuid']), HTTP_Not_Found) + raise NfvoException("datacenter '{}' and tenant '{}' are not attached".format(datacenter_id, tenant_dict['uuid']), httperrors.Not_Found) #delete this association mydb.delete_row(FROM='tenants_datacenters', WHERE=tenants_datacenter_dict) @@ -4922,7 +4995,7 @@ def datacenter_action(mydb, tenant_id, datacenter, action_dict): #print content except vimconn.vimconnException as e: #logger.error("nfvo.datacenter_action() Not possible to get_network_list from VIM: %s ", str(e)) - raise NfvoException(str(e), HTTP_Internal_Server_Error) + raise NfvoException(str(e), httperrors.Internal_Server_Error) #update nets Change from VIM format to NFVO format net_list=[] for net in nets: @@ -4951,7 +5024,7 @@ def datacenter_action(mydb, tenant_id, datacenter, action_dict): return result else: - raise NfvoException("Unknown action " + str(action_dict), HTTP_Bad_Request) + raise NfvoException("Unknown action " + str(action_dict), httperrors.Bad_Request) def datacenter_edit_netmap(mydb, tenant_id, datacenter, netmap, action_dict): @@ -4981,11 +5054,11 @@ def datacenter_new_netmap(mydb, tenant_id, datacenter, action_dict=None): vim_nets = myvim.get_network_list(filter_dict=filter_dict) except vimconn.vimconnException as e: #logger.error("nfvo.datacenter_new_netmap() Not possible to get_network_list from VIM: %s ", str(e)) - raise NfvoException(str(e), HTTP_Internal_Server_Error) + raise NfvoException(str(e), httperrors.Internal_Server_Error) if len(vim_nets)>1 and action_dict: - raise NfvoException("more than two networks found, specify with vim_id", HTTP_Conflict) + raise NfvoException("more than two networks found, specify with vim_id", httperrors.Conflict) elif len(vim_nets)==0: # and action_dict: - raise NfvoException("Not found a network at VIM with " + str(filter_dict), HTTP_Not_Found) + raise NfvoException("Not found a network at VIM with " + str(filter_dict), httperrors.Not_Found) net_list=[] for net in vim_nets: net_nfvo={'datacenter_id': datacenter_id} @@ -5026,11 +5099,11 @@ def get_sdn_net_id(mydb, tenant_id, datacenter, network_id): # ensure the network is defined if len(network) == 0: raise NfvoException("Network {} is not present in the system".format(network_id), - HTTP_Bad_Request) + httperrors.Bad_Request) # ensure there is only one network with the provided name if len(network) > 1: - raise NfvoException("Multiple networks present in vim identified by {}".format(network_id), HTTP_Bad_Request) + raise NfvoException("Multiple networks present in vim identified by {}".format(network_id), httperrors.Bad_Request) # ensure it is a dataplane network if network[0]['type'] != 'data': @@ -5063,7 +5136,7 @@ def get_sdn_net_id(mydb, tenant_id, datacenter, network_id): return sdn_net_id else: raise NfvoException("More than one SDN network is associated to vim network {}".format( - network_id), HTTP_Internal_Server_Error) + network_id), httperrors.Internal_Server_Error) def get_sdn_controller_id(mydb, datacenter): # Obtain sdn controller id @@ -5077,12 +5150,12 @@ def vim_net_sdn_attach(mydb, tenant_id, datacenter, network_id, descriptor): try: sdn_network_id = get_sdn_net_id(mydb, tenant_id, datacenter, network_id) if not sdn_network_id: - raise NfvoException("No SDN network is associated to vim-network {}".format(network_id), HTTP_Internal_Server_Error) + raise NfvoException("No SDN network is associated to vim-network {}".format(network_id), httperrors.Internal_Server_Error) #Obtain sdn controller id controller_id = get_sdn_controller_id(mydb, datacenter) if not controller_id: - raise NfvoException("No SDN controller is set for datacenter {}".format(datacenter), HTTP_Internal_Server_Error) + raise NfvoException("No SDN controller is set for datacenter {}".format(datacenter), httperrors.Internal_Server_Error) #Obtain sdn controller info sdn_controller = ovim.show_of_controller(controller_id) @@ -5103,7 +5176,7 @@ def vim_net_sdn_attach(mydb, tenant_id, datacenter, network_id, descriptor): result = ovim.new_port(port_data) except ovimException as e: raise NfvoException("ovimException attaching SDN network {} to vim network {}".format( - sdn_network_id, network_id) + str(e), HTTP_Internal_Server_Error) + sdn_network_id, network_id) + str(e), httperrors.Internal_Server_Error) except db_base_Exception as e: raise NfvoException("db_base_Exception attaching SDN network to vim network {}".format( network_id) + str(e), e.http_code) @@ -5117,7 +5190,7 @@ def vim_net_sdn_detach(mydb, tenant_id, datacenter, network_id, port_id=None): sdn_network_id = get_sdn_net_id(mydb, tenant_id, datacenter, network_id) if not sdn_network_id: raise NfvoException("No SDN network is associated to vim-network {}".format(network_id), - HTTP_Internal_Server_Error) + httperrors.Internal_Server_Error) #in case no port_id is specified only ports marked as 'external_port' will be detached filter = {'name': 'external_port', 'net_id': sdn_network_id} @@ -5125,11 +5198,11 @@ def vim_net_sdn_detach(mydb, tenant_id, datacenter, network_id, port_id=None): port_list = ovim.get_ports(columns={'uuid'}, filter=filter) except ovimException as e: raise NfvoException("ovimException obtaining external ports for net {}. ".format(network_id) + str(e), - HTTP_Internal_Server_Error) + httperrors.Internal_Server_Error) if len(port_list) == 0: raise NfvoException("No ports attached to the network {} were found with the requested criteria".format(network_id), - HTTP_Bad_Request) + httperrors.Bad_Request) port_uuid_list = [] for port in port_list: @@ -5137,7 +5210,7 @@ def vim_net_sdn_detach(mydb, tenant_id, datacenter, network_id, port_id=None): port_uuid_list.append(port['uuid']) ovim.delete_port(port['uuid']) except ovimException as e: - raise NfvoException("ovimException deleting port {} for net {}. ".format(port['uuid'], network_id) + str(e), HTTP_Internal_Server_Error) + raise NfvoException("ovimException deleting port {} for net {}. ".format(port['uuid'], network_id) + str(e), httperrors.Internal_Server_Error) return 'Detached ports uuid: {}'.format(','.join(port_uuid_list)) @@ -5157,7 +5230,7 @@ def vim_action_get(mydb, tenant_id, datacenter, item, name): if len(content) == 0: raise NfvoException("Network {} is not present in the system. ".format(name), - HTTP_Bad_Request) + httperrors.Bad_Request) #Update the networks with the attached ports for net in content: @@ -5167,7 +5240,7 @@ def vim_action_get(mydb, tenant_id, datacenter, item, name): #port_list = ovim.get_ports(columns={'uuid', 'switch_port', 'vlan'}, filter={'name': 'external_port', 'net_id': sdn_network_id}) port_list = ovim.get_ports(columns={'uuid', 'switch_port', 'vlan','name'}, filter={'net_id': sdn_network_id}) except ovimException as e: - raise NfvoException("ovimException obtaining external ports for net {}. ".format(network_id) + str(e), HTTP_Internal_Server_Error) + raise NfvoException("ovimException obtaining external ports for net {}. ".format(network_id) + str(e), httperrors.Internal_Server_Error) #Remove field name and if port name is external_port save it as 'type' for port in port_list: if port['name'] == 'external_port': @@ -5182,7 +5255,7 @@ def vim_action_get(mydb, tenant_id, datacenter, item, name): content = myvim.get_image_list(filter_dict=filter_dict) else: - raise NfvoException(item + "?", HTTP_Method_Not_Allowed) + raise NfvoException(item + "?", httperrors.Method_Not_Allowed) logger.debug("vim_action response %s", content) #update nets Change from VIM format to NFVO format if name and len(content)==1: return {item[:-1]: content[0]} @@ -5207,9 +5280,9 @@ def vim_action_delete(mydb, tenant_id, datacenter, item, name): logger.debug("vim_action_delete vim response: " + str(content)) items = content.values()[0] if type(items)==list and len(items)==0: - raise NfvoException("Not found " + item, HTTP_Not_Found) + raise NfvoException("Not found " + item, httperrors.Not_Found) elif type(items)==list and len(items)>1: - raise NfvoException("Found more than one {} with this name. Use uuid.".format(item), HTTP_Not_Found) + raise NfvoException("Found more than one {} with this name. Use uuid.".format(item), httperrors.Not_Found) else: # it is a dict item_id = items["id"] item_name = str(items.get("name")) @@ -5225,7 +5298,7 @@ def vim_action_delete(mydb, tenant_id, datacenter, item, name): except ovimException as e: raise NfvoException( "ovimException obtaining external ports for net {}. ".format(network_id) + str(e), - HTTP_Internal_Server_Error) + httperrors.Internal_Server_Error) # By calling one by one all ports to be detached we ensure that not only the external_ports get detached for port in port_list: @@ -5244,7 +5317,7 @@ def vim_action_delete(mydb, tenant_id, datacenter, item, name): except ovimException as e: logger.error("ovimException deleting SDN network={} ".format(sdn_network_id) + str(e), exc_info=True) raise NfvoException("ovimException deleting SDN network={} ".format(sdn_network_id) + str(e), - HTTP_Internal_Server_Error) + httperrors.Internal_Server_Error) content = myvim.delete_network(item_id) elif item=="tenants": @@ -5252,7 +5325,7 @@ def vim_action_delete(mydb, tenant_id, datacenter, item, name): elif item == "images": content = myvim.delete_image(item_id) else: - raise NfvoException(item + "?", HTTP_Method_Not_Allowed) + raise NfvoException(item + "?", httperrors.Method_Not_Allowed) except vimconn.vimconnException as e: #logger.error( "vim_action Not possible to delete_{} {}from VIM: {} ".format(item, name, str(e))) raise NfvoException("Not possible to delete_{} {} from VIM: {}".format(item, name, str(e)), e.http_code) @@ -5293,7 +5366,7 @@ def vim_action_create(mydb, tenant_id, datacenter, item, descriptor): logger.error("ovimException creating SDN network={} ".format( sdn_network) + str(e), exc_info=True) raise NfvoException("ovimException creating SDN network={} ".format(sdn_network) + str(e), - HTTP_Internal_Server_Error) + httperrors.Internal_Server_Error) # Save entry in in dabase mano_db in table instance_nets to stablish a dictionary vim_net_id <->sdn_net_id # use instance_scenario_id=None to distinguish from real instaces of nets @@ -5311,7 +5384,7 @@ def vim_action_create(mydb, tenant_id, datacenter, item, descriptor): tenant = descriptor["tenant"] content = myvim.new_tenant(tenant["name"], tenant.get("description")) else: - raise NfvoException(item + "?", HTTP_Method_Not_Allowed) + raise NfvoException(item + "?", httperrors.Method_Not_Allowed) except vimconn.vimconnException as e: raise NfvoException("Not possible to create {} at VIM: {}".format(item, str(e)), e.http_code) @@ -5345,7 +5418,7 @@ def sdn_controller_delete(mydb, tenant_id, controller_id): if datacenter['config']: config = yaml.load(datacenter['config']) if 'sdn-controller' in config and config['sdn-controller'] == controller_id: - raise NfvoException("SDN controller {} is in use by datacenter {}".format(controller_id, datacenter['uuid']), HTTP_Conflict) + raise NfvoException("SDN controller {} is in use by datacenter {}".format(controller_id, datacenter['uuid']), httperrors.Conflict) data = ovim.delete_of_controller(controller_id) msg = 'SDN controller {} deleted'.format(data) @@ -5355,12 +5428,12 @@ def sdn_controller_delete(mydb, tenant_id, controller_id): def datacenter_sdn_port_mapping_set(mydb, tenant_id, datacenter_id, sdn_port_mapping): controller = mydb.get_rows(FROM="datacenters", SELECT=("config",), WHERE={"uuid":datacenter_id}) if len(controller) < 1: - raise NfvoException("Datacenter {} not present in the database".format(datacenter_id), HTTP_Not_Found) + raise NfvoException("Datacenter {} not present in the database".format(datacenter_id), httperrors.Not_Found) try: sdn_controller_id = yaml.load(controller[0]["config"])["sdn-controller"] except: - raise NfvoException("The datacenter {} has not an SDN controller associated".format(datacenter_id), HTTP_Bad_Request) + raise NfvoException("The datacenter {} has not an SDN controller associated".format(datacenter_id), httperrors.Bad_Request) sdn_controller = ovim.show_of_controller(sdn_controller_id) switch_dpid = sdn_controller["dpid"] @@ -5376,7 +5449,7 @@ def datacenter_sdn_port_mapping_set(mydb, tenant_id, datacenter_id, sdn_port_map element["switch_mac"] = port.get("switch_mac") if not pci or not (element["switch_port"] or element["switch_mac"]): raise NfvoException ("The mapping must contain the 'pci' and at least one of the elements 'switch_port'" - " or 'switch_mac'", HTTP_Bad_Request) + " or 'switch_mac'", httperrors.Bad_Request) for pci_expanded in utils.expand_brackets(pci): element["pci"] = pci_expanded maps.append(dict(element)) @@ -5403,10 +5476,10 @@ def datacenter_sdn_port_mapping_list(mydb, tenant_id, datacenter_id): result["dpid"] = sdn_controller["dpid"] if result["sdn-controller"] == None: - raise NfvoException("SDN controller is not defined for datacenter {}".format(datacenter_id), HTTP_Bad_Request) + raise NfvoException("SDN controller is not defined for datacenter {}".format(datacenter_id), httperrors.Bad_Request) if result["dpid"] == None: raise NfvoException("It was not possible to determine DPID for SDN controller {}".format(result["sdn-controller"]), - HTTP_Internal_Server_Error) + httperrors.Internal_Server_Error) if len(maps) == 0: return result @@ -5414,9 +5487,9 @@ def datacenter_sdn_port_mapping_list(mydb, tenant_id, datacenter_id): ports_correspondence_dict = dict() for link in maps: if result["sdn-controller"] != link["ofc_id"]: - raise NfvoException("The sdn-controller specified for different port mappings differ", HTTP_Internal_Server_Error) + raise NfvoException("The sdn-controller specified for different port mappings differ", httperrors.Internal_Server_Error) if result["dpid"] != link["switch_dpid"]: - raise NfvoException("The dpid specified for different port mappings differ", HTTP_Internal_Server_Error) + raise NfvoException("The dpid specified for different port mappings differ", httperrors.Internal_Server_Error) element = dict() element["pci"] = link["pci"] if link["switch_port"]: @@ -5455,10 +5528,10 @@ def create_RO_keypair(tenant_id): try: public_key = key.publickey().exportKey('OpenSSH') if isinstance(public_key, ValueError): - raise NfvoException("Unable to create public key: {}".format(public_key), HTTP_Internal_Server_Error) + raise NfvoException("Unable to create public key: {}".format(public_key), httperrors.Internal_Server_Error) private_key = key.exportKey(passphrase=tenant_id, pkcs=8) except (ValueError, NameError) as e: - raise NfvoException("Unable to create private key: {}".format(e), HTTP_Internal_Server_Error) + raise NfvoException("Unable to create private key: {}".format(e), httperrors.Internal_Server_Error) return public_key, private_key def decrypt_key (key, tenant_id): @@ -5474,7 +5547,7 @@ def decrypt_key (key, tenant_id): key = RSA.importKey(key,tenant_id) unencrypted_key = key.exportKey('PEM') if isinstance(unencrypted_key, ValueError): - raise NfvoException("Unable to decrypt the private key: {}".format(unencrypted_key), HTTP_Internal_Server_Error) + raise NfvoException("Unable to decrypt the private key: {}".format(unencrypted_key), httperrors.Internal_Server_Error) except ValueError as e: - raise NfvoException("Unable to decrypt the private key: {}".format(e), HTTP_Internal_Server_Error) + raise NfvoException("Unable to decrypt the private key: {}".format(e), httperrors.Internal_Server_Error) return unencrypted_key diff --git a/osm_ro/nfvo_db.py b/osm_ro/nfvo_db.py index 0418de49..b1224436 100644 --- a/osm_ro/nfvo_db.py +++ b/osm_ro/nfvo_db.py @@ -34,12 +34,16 @@ import yaml import time #import sys, os +from .http_tools import errors as httperrors + tables_with_createdat_field=["datacenters","instance_nets","instance_scenarios","instance_vms","instance_vnfs", "interfaces","nets","nfvo_tenants","scenarios","sce_interfaces","sce_nets", "sce_vnfs","tenants_datacenters","datacenter_tenants","vms","vnfs", "datacenter_nets", - "instance_actions", "vim_actions", "sce_vnffgs", "sce_rsps", "sce_rsp_hops", + "instance_actions", "sce_vnffgs", "sce_rsps", "sce_rsp_hops", "sce_classifiers", "sce_classifier_matches", "instance_sfis", "instance_sfs", - "instance_classifications", "instance_sfps"] + "instance_classifications", "instance_sfps", "wims", "wim_accounts", "wim_nfvo_tenants", + "wim_port_mappings", "vim_wim_actions", + "instance_wim_nets"] class nfvo_db(db_base.db_base): @@ -55,7 +59,7 @@ class nfvo_db(db_base.db_base): created_time = time.time() try: with self.con: - + myVNFDict = {} myVNFDict["name"] = vnf_name myVNFDict["descriptor"] = vnf_descriptor['vnf'].get('descriptor') @@ -63,22 +67,22 @@ class nfvo_db(db_base.db_base): myVNFDict["description"] = vnf_descriptor['vnf']['description'] myVNFDict["class"] = vnf_descriptor['vnf'].get('class',"MISC") myVNFDict["tenant_id"] = vnf_descriptor['vnf'].get("tenant_id") - + vnf_id = self._new_row_internal('vnfs', myVNFDict, add_uuid=True, root_uuid=None, created_time=created_time) #print "Adding new vms to the NFVO database" #For each vm, we must create the appropriate vm in the NFVO database. vmDict = {} for _,vm in VNFCDict.iteritems(): #This code could make the name of the vms grow and grow. - #If we agree to follow this convention, we should check with a regex that the vnfc name is not including yet the vnf name + #If we agree to follow this convention, we should check with a regex that the vnfc name is not including yet the vnf name #vm['name'] = "%s-%s" % (vnf_name,vm['name']) #print "VM name: %s. Description: %s" % (vm['name'], vm['description']) vm["vnf_id"] = vnf_id created_time += 0.00001 - vm_id = self._new_row_internal('vms', vm, add_uuid=True, root_uuid=vnf_id, created_time=created_time) + vm_id = self._new_row_internal('vms', vm, add_uuid=True, root_uuid=vnf_id, created_time=created_time) #print "Internal vm id in NFVO DB: %s" % vm_id vmDict[vm['name']] = vm_id - + #Collect the bridge interfaces of each VM/VNFC under the 'bridge-ifaces' field bridgeInterfacesDict = {} for vm in vnf_descriptor['vnf']['VNFC']: @@ -124,19 +128,19 @@ class nfvo_db(db_base.db_base): if 'internal-connections' in vnf_descriptor['vnf']: for net in vnf_descriptor['vnf']['internal-connections']: #print "Net name: %s. Description: %s" % (net['name'], net['description']) - + myNetDict = {} myNetDict["name"] = net['name'] myNetDict["description"] = net['description'] myNetDict["type"] = net['type'] myNetDict["vnf_id"] = vnf_id - + created_time += 0.00001 net_id = self._new_row_internal('nets', myNetDict, add_uuid=True, root_uuid=vnf_id, created_time=created_time) - + for element in net['elements']: ifaceItem = {} - #ifaceItem["internal_name"] = "%s-%s-%s" % (net['name'],element['VNFC'], element['local_iface_name']) + #ifaceItem["internal_name"] = "%s-%s-%s" % (net['name'],element['VNFC'], element['local_iface_name']) ifaceItem["internal_name"] = element['local_iface_name'] #ifaceItem["vm_id"] = vmDict["%s-%s" % (vnf_name,element['VNFC'])] ifaceItem["vm_id"] = vmDict[element['VNFC']] @@ -159,17 +163,17 @@ class nfvo_db(db_base.db_base): created_time_iface = bridgeiface['created_time'] internalconnList.append(ifaceItem) #print "Internal net id in NFVO DB: %s" % net_id - + #print "Adding internal interfaces to the NFVO database (if any)" for iface in internalconnList: #print "Iface name: %s" % iface['internal_name'] iface_id = self._new_row_internal('interfaces', iface, add_uuid=True, root_uuid=vnf_id, created_time = created_time_iface) #print "Iface id in NFVO DB: %s" % iface_id - + #print "Adding external interfaces to the NFVO database" for iface in vnf_descriptor['vnf']['external-connections']: myIfaceDict = {} - #myIfaceDict["internal_name"] = "%s-%s-%s" % (vnf_name,iface['VNFC'], iface['local_iface_name']) + #myIfaceDict["internal_name"] = "%s-%s-%s" % (vnf_name,iface['VNFC'], iface['local_iface_name']) myIfaceDict["internal_name"] = iface['local_iface_name'] #myIfaceDict["vm_id"] = vmDict["%s-%s" % (vnf_name,iface['VNFC'])] myIfaceDict["vm_id"] = vmDict[iface['VNFC']] @@ -193,13 +197,13 @@ class nfvo_db(db_base.db_base): #print "Iface name: %s" % iface['name'] iface_id = self._new_row_internal('interfaces', myIfaceDict, add_uuid=True, root_uuid=vnf_id, created_time = created_time_iface) #print "Iface id in NFVO DB: %s" % iface_id - + return vnf_id - + except (mdb.Error, AttributeError) as e: self._format_error(e, tries) tries -= 1 - + def new_vnf_as_a_whole2(self,nfvo_tenant,vnf_name,vnf_descriptor,VNFCDict): self.logger.debug("Adding new vnf to the NFVO database") tries = 2 @@ -207,7 +211,7 @@ class nfvo_db(db_base.db_base): created_time = time.time() try: with self.con: - + myVNFDict = {} myVNFDict["name"] = vnf_name myVNFDict["descriptor"] = vnf_descriptor['vnf'].get('descriptor') @@ -222,15 +226,15 @@ class nfvo_db(db_base.db_base): vmDict = {} for _,vm in VNFCDict.iteritems(): #This code could make the name of the vms grow and grow. - #If we agree to follow this convention, we should check with a regex that the vnfc name is not including yet the vnf name + #If we agree to follow this convention, we should check with a regex that the vnfc name is not including yet the vnf name #vm['name'] = "%s-%s" % (vnf_name,vm['name']) #print "VM name: %s. Description: %s" % (vm['name'], vm['description']) vm["vnf_id"] = vnf_id created_time += 0.00001 - vm_id = self._new_row_internal('vms', vm, add_uuid=True, root_uuid=vnf_id, created_time=created_time) + vm_id = self._new_row_internal('vms', vm, add_uuid=True, root_uuid=vnf_id, created_time=created_time) #print "Internal vm id in NFVO DB: %s" % vm_id vmDict[vm['name']] = vm_id - + #Collect the bridge interfaces of each VM/VNFC under the 'bridge-ifaces' field bridgeInterfacesDict = {} for vm in vnf_descriptor['vnf']['VNFC']: @@ -274,13 +278,13 @@ class nfvo_db(db_base.db_base): if 'internal-connections' in vnf_descriptor['vnf']: for net in vnf_descriptor['vnf']['internal-connections']: #print "Net name: %s. Description: %s" % (net['name'], net['description']) - + myNetDict = {} myNetDict["name"] = net['name'] myNetDict["description"] = net['description'] if (net["implementation"] == "overlay"): net["type"] = "bridge" - #It should give an error if the type is e-line. For the moment, we consider it as a bridge + #It should give an error if the type is e-line. For the moment, we consider it as a bridge elif (net["implementation"] == "underlay"): if (net["type"] == "e-line"): net["type"] = "ptp" @@ -289,10 +293,10 @@ class nfvo_db(db_base.db_base): net.pop("implementation") myNetDict["type"] = net['type'] myNetDict["vnf_id"] = vnf_id - + created_time += 0.00001 net_id = self._new_row_internal('nets', myNetDict, add_uuid=True, root_uuid=vnf_id, created_time=created_time) - + if "ip-profile" in net: ip_profile = net["ip-profile"] myIPProfileDict = {} @@ -305,13 +309,13 @@ class nfvo_db(db_base.db_base): myIPProfileDict["dhcp_enabled"] = ip_profile["dhcp"].get('enabled',"true") myIPProfileDict["dhcp_start_address"] = ip_profile["dhcp"].get('start-address',None) myIPProfileDict["dhcp_count"] = ip_profile["dhcp"].get('count',None) - + created_time += 0.00001 ip_profile_id = self._new_row_internal('ip_profiles', myIPProfileDict) - + for element in net['elements']: ifaceItem = {} - #ifaceItem["internal_name"] = "%s-%s-%s" % (net['name'],element['VNFC'], element['local_iface_name']) + #ifaceItem["internal_name"] = "%s-%s-%s" % (net['name'],element['VNFC'], element['local_iface_name']) ifaceItem["internal_name"] = element['local_iface_name'] #ifaceItem["vm_id"] = vmDict["%s-%s" % (vnf_name,element['VNFC'])] ifaceItem["vm_id"] = vmDict[element['VNFC']] @@ -335,11 +339,11 @@ class nfvo_db(db_base.db_base): #print "Iface name: %s" % iface['internal_name'] iface_id = self._new_row_internal('interfaces', ifaceItem, add_uuid=True, root_uuid=vnf_id, created_time=created_time_iface) #print "Iface id in NFVO DB: %s" % iface_id - + #print "Adding external interfaces to the NFVO database" for iface in vnf_descriptor['vnf']['external-connections']: myIfaceDict = {} - #myIfaceDict["internal_name"] = "%s-%s-%s" % (vnf_name,iface['VNFC'], iface['local_iface_name']) + #myIfaceDict["internal_name"] = "%s-%s-%s" % (vnf_name,iface['VNFC'], iface['local_iface_name']) myIfaceDict["internal_name"] = iface['local_iface_name'] #myIfaceDict["vm_id"] = vmDict["%s-%s" % (vnf_name,iface['VNFC'])] myIfaceDict["vm_id"] = vmDict[iface['VNFC']] @@ -363,9 +367,9 @@ class nfvo_db(db_base.db_base): #print "Iface name: %s" % iface['name'] iface_id = self._new_row_internal('interfaces', myIfaceDict, add_uuid=True, root_uuid=vnf_id, created_time=created_time_iface) #print "Iface id in NFVO DB: %s" % iface_id - + return vnf_id - + except (mdb.Error, AttributeError) as e: self._format_error(e, tries) # except KeyError as e2: @@ -388,7 +392,7 @@ class nfvo_db(db_base.db_base): 'name': scenario_dict['name'], 'description': scenario_dict['description'], 'public': scenario_dict.get('public', "false")} - + scenario_uuid = self._new_row_internal('scenarios', INSERT_, add_uuid=True, root_uuid=None, created_time=created_time) #sce_nets for net in scenario_dict['nets'].values(): @@ -447,9 +451,9 @@ class nfvo_db(db_base.db_base): created_time += 0.00001 iface_uuid = self._new_row_internal('sce_interfaces', INSERT_, add_uuid=True, root_uuid=scenario_uuid, created_time=created_time) - + return scenario_uuid - + except (mdb.Error, AttributeError) as e: self._format_error(e, tries) tries -= 1 @@ -465,7 +469,7 @@ class nfvo_db(db_base.db_base): #check that scenario exist tenant_id = scenario_dict.get('tenant_id') scenario_uuid = scenario_dict['uuid'] - + where_text = "uuid='{}'".format(scenario_uuid) if not tenant_id and tenant_id != "any": where_text += " AND (tenant_id='{}' OR public='True')".format(tenant_id) @@ -474,9 +478,9 @@ class nfvo_db(db_base.db_base): self.cur.execute(cmd) self.cur.fetchall() if self.cur.rowcount==0: - raise db_base.db_base_Exception("No scenario found with this criteria " + where_text, db_base.HTTP_Bad_Request) + raise db_base.db_base_Exception("No scenario found with this criteria " + where_text, httperrors.Bad_Request) elif self.cur.rowcount>1: - raise db_base.db_base_Exception("More than one scenario found with this criteria " + where_text, db_base.HTTP_Bad_Request) + raise db_base.db_base_Exception("More than one scenario found with this criteria " + where_text, httperrors.Bad_Request) #scenario nodes = {} @@ -500,7 +504,7 @@ class nfvo_db(db_base.db_base): item_changed += self._update_rows('sce_nets', node, WHERE_) item_changed += self._update_rows('sce_vnfs', node, WHERE_, modified_time=modified_time) return item_changed - + except (mdb.Error, AttributeError) as e: self._format_error(e, tries) tries -= 1 @@ -509,7 +513,7 @@ class nfvo_db(db_base.db_base): # '''Obtain the scenario instance information, filtering by one or serveral of the tenant, uuid or name # instance_scenario_id is the uuid or the name if it is not a valid uuid format # Only one scenario isntance must mutch the filtering or an error is returned -# ''' +# ''' # print "1******************************************************************" # try: # with self.con: @@ -525,11 +529,11 @@ class nfvo_db(db_base.db_base): # self.cur.execute("SELECT * FROM instance_scenarios WHERE "+ where_text) # rows = self.cur.fetchall() # if self.cur.rowcount==0: -# return -HTTP_Bad_Request, "No scenario instance found with this criteria " + where_text +# return -httperrors.Bad_Request, "No scenario instance found with this criteria " + where_text # elif self.cur.rowcount>1: -# return -HTTP_Bad_Request, "More than one scenario instance found with this criteria " + where_text +# return -httperrors.Bad_Request, "More than one scenario instance found with this criteria " + where_text # instance_scenario_dict = rows[0] -# +# # #instance_vnfs # self.cur.execute("SELECT uuid,vnf_id FROM instance_vnfs WHERE instance_scenario_id='"+ instance_scenario_dict['uuid'] + "'") # instance_scenario_dict['instance_vnfs'] = self.cur.fetchall() @@ -537,17 +541,17 @@ class nfvo_db(db_base.db_base): # #instance_vms # self.cur.execute("SELECT uuid, vim_vm_id "+ # "FROM instance_vms "+ -# "WHERE instance_vnf_id='" + vnf['uuid'] +"'" +# "WHERE instance_vnf_id='" + vnf['uuid'] +"'" # ) # vnf['instance_vms'] = self.cur.fetchall() # #instance_nets # self.cur.execute("SELECT uuid, vim_net_id FROM instance_nets WHERE instance_scenario_id='"+ instance_scenario_dict['uuid'] + "'") # instance_scenario_dict['instance_nets'] = self.cur.fetchall() -# +# # #instance_interfaces # self.cur.execute("SELECT uuid, vim_interface_id, instance_vm_id, instance_net_id FROM instance_interfaces WHERE instance_scenario_id='"+ instance_scenario_dict['uuid'] + "'") # instance_scenario_dict['instance_interfaces'] = self.cur.fetchall() -# +# # db_base._convert_datetime2str(instance_scenario_dict) # db_base._convert_str2boolean(instance_scenario_dict, ('public','shared','external') ) # print "2******************************************************************" @@ -561,7 +565,7 @@ class nfvo_db(db_base.db_base): scenario_id is the uuid or the name if it is not a valid uuid format if datacenter_vim_id,d datacenter_id is provided, it supply aditional vim_id fields with the matching vim uuid Only one scenario must mutch the filtering or an error is returned - ''' + ''' tries = 2 while tries: try: @@ -575,9 +579,9 @@ class nfvo_db(db_base.db_base): self.cur.execute(cmd) rows = self.cur.fetchall() if self.cur.rowcount==0: - raise db_base.db_base_Exception("No scenario found with this criteria " + where_text, db_base.HTTP_Bad_Request) + raise db_base.db_base_Exception("No scenario found with this criteria " + where_text, httperrors.Bad_Request) elif self.cur.rowcount>1: - raise db_base.db_base_Exception("More than one scenario found with this criteria " + where_text, db_base.HTTP_Bad_Request) + raise db_base.db_base_Exception("More than one scenario found with this criteria " + where_text, httperrors.Bad_Request) scenario_dict = rows[0] if scenario_dict["cloud_config"]: scenario_dict["cloud-config"] = yaml.load(scenario_dict["cloud_config"]) @@ -627,17 +631,17 @@ class nfvo_db(db_base.db_base): if datacenter_vim_id!=None: cmd = "SELECT vim_id FROM datacenters_images WHERE image_id='{}' AND datacenter_vim_id='{}'".format(vm['image_id'],datacenter_vim_id) self.logger.debug(cmd) - self.cur.execute(cmd) + self.cur.execute(cmd) if self.cur.rowcount==1: vim_image_dict = self.cur.fetchone() vm['vim_image_id']=vim_image_dict['vim_id'] cmd = "SELECT vim_id FROM datacenters_flavors WHERE flavor_id='{}' AND datacenter_vim_id='{}'".format(vm['flavor_id'],datacenter_vim_id) self.logger.debug(cmd) - self.cur.execute(cmd) + self.cur.execute(cmd) if self.cur.rowcount==1: vim_flavor_dict = self.cur.fetchone() vm['vim_flavor_id']=vim_flavor_dict['vim_id'] - + #interfaces cmd = "SELECT uuid,internal_name,external_name,net_id,type,vpci,mac,bw,model,ip_address," \ "floating_ip, port_security" \ @@ -662,15 +666,15 @@ class nfvo_db(db_base.db_base): vnf['nets'] = self.cur.fetchall() for vnf_net in vnf['nets']: SELECT_ = "ip_version,subnet_address,gateway_address,dns_address,dhcp_enabled,dhcp_start_address,dhcp_count" - cmd = "SELECT {} FROM ip_profiles WHERE net_id='{}'".format(SELECT_,vnf_net['uuid']) + cmd = "SELECT {} FROM ip_profiles WHERE net_id='{}'".format(SELECT_,vnf_net['uuid']) self.logger.debug(cmd) self.cur.execute(cmd) ipprofiles = self.cur.fetchall() if self.cur.rowcount==1: vnf_net["ip_profile"] = ipprofiles[0] elif self.cur.rowcount>1: - raise db_base.db_base_Exception("More than one ip-profile found with this criteria: net_id='{}'".format(vnf_net['uuid']), db_base.HTTP_Bad_Request) - + raise db_base.db_base_Exception("More than one ip-profile found with this criteria: net_id='{}'".format(vnf_net['uuid']), httperrors.Bad_Request) + #sce_nets cmd = "SELECT uuid,name,type,external,description,vim_network_name, osm_id" \ " FROM sce_nets WHERE scenario_id='{}'" \ @@ -682,28 +686,28 @@ class nfvo_db(db_base.db_base): for net in scenario_dict['nets']: if str(net['external']) == 'false': SELECT_ = "ip_version,subnet_address,gateway_address,dns_address,dhcp_enabled,dhcp_start_address,dhcp_count" - cmd = "SELECT {} FROM ip_profiles WHERE sce_net_id='{}'".format(SELECT_,net['uuid']) + cmd = "SELECT {} FROM ip_profiles WHERE sce_net_id='{}'".format(SELECT_,net['uuid']) self.logger.debug(cmd) self.cur.execute(cmd) ipprofiles = self.cur.fetchall() if self.cur.rowcount==1: net["ip_profile"] = ipprofiles[0] elif self.cur.rowcount>1: - raise db_base.db_base_Exception("More than one ip-profile found with this criteria: sce_net_id='{}'".format(net['uuid']), db_base.HTTP_Bad_Request) + raise db_base.db_base_Exception("More than one ip-profile found with this criteria: sce_net_id='{}'".format(net['uuid']), httperrors.Bad_Request) continue WHERE_=" WHERE name='{}'".format(net['name']) if datacenter_id!=None: WHERE_ += " AND datacenter_id='{}'".format(datacenter_id) cmd = "SELECT vim_net_id FROM datacenter_nets" + WHERE_ self.logger.debug(cmd) - self.cur.execute(cmd) + self.cur.execute(cmd) d_net = self.cur.fetchone() if d_net==None or datacenter_vim_id==None: #print "nfvo_db.get_scenario() WARNING external net %s not found" % net['name'] net['vim_id']=None else: net['vim_id']=d_net['vim_net_id'] - + db_base._convert_datetime2str(scenario_dict) db_base._convert_str2boolean(scenario_dict, ('public','shared','external','port-security','floating-ip') ) @@ -745,13 +749,13 @@ class nfvo_db(db_base.db_base): '''Deletes a scenario, filtering by one or several of the tenant, uuid or name scenario_id is the uuid or the name if it is not a valid uuid format Only one scenario must mutch the filtering or an error is returned - ''' + ''' tries = 2 while tries: try: with self.con: self.cur = self.con.cursor(mdb.cursors.DictCursor) - + #scenario table where_text = "uuid='{}'".format(scenario_id) if not tenant_id and tenant_id != "any": @@ -761,12 +765,12 @@ class nfvo_db(db_base.db_base): self.cur.execute(cmd) rows = self.cur.fetchall() if self.cur.rowcount==0: - raise db_base.db_base_Exception("No scenario found where " + where_text, db_base.HTTP_Not_Found) + raise db_base.db_base_Exception("No scenario found where " + where_text, httperrors.Not_Found) elif self.cur.rowcount>1: - raise db_base.db_base_Exception("More than one scenario found where " + where_text, db_base.HTTP_Conflict) + raise db_base.db_base_Exception("More than one scenario found where " + where_text, httperrors.Conflict) scenario_uuid = rows[0]["uuid"] scenario_name = rows[0]["name"] - + #sce_vnfs cmd = "DELETE FROM scenarios WHERE uuid='{}'".format(scenario_uuid) self.logger.debug(cmd) @@ -777,7 +781,7 @@ class nfvo_db(db_base.db_base): self._format_error(e, tries, "delete", "instances running") tries -= 1 - def new_rows(self, tables, uuid_list=None): + def new_rows(self, tables, uuid_list=None, confidential_data=False): """ Make a transactional insertion of rows at several tables. Can be also a deletion :param tables: list with dictionary where the keys are the table names and the values are a row or row list @@ -795,6 +799,7 @@ class nfvo_db(db_base.db_base): :return: None if success, raise exception otherwise """ tries = 2 + table_name = None while tries: created_time = time.time() try: @@ -819,10 +824,11 @@ class nfvo_db(db_base.db_base): else: created_time_param = 0 self._new_row_internal(table_name, row, add_uuid=False, root_uuid=None, - created_time=created_time_param) + confidential_data=confidential_data, + created_time=created_time_param) return except (mdb.Error, AttributeError) as e: - self._format_error(e, tries) + self._format_error(e, tries, table=table_name) tries -= 1 def new_instance_scenario_as_a_whole(self,tenant_id,instance_scenario_name,instance_scenario_description,scenarioDict): @@ -845,7 +851,7 @@ class nfvo_db(db_base.db_base): INSERT_["cloud_config"] = yaml.safe_dump(scenarioDict["cloud-config"], default_flow_style=True, width=256) instance_uuid = self._new_row_internal('instance_scenarios', INSERT_, add_uuid=True, root_uuid=None, created_time=created_time) - + net_scene2instance={} #instance_nets #nets interVNF for net in scenarioDict['nets']: @@ -855,10 +861,10 @@ class nfvo_db(db_base.db_base): net["vim_id_sites"] ={datacenter_site_id: net['vim_id']} net["vim_id_sites"]["datacenter_site_id"] = {datacenter_site_id: net['vim_id']} sce_net_id = net.get("uuid") - + for datacenter_site_id,vim_id in net["vim_id_sites"].iteritems(): INSERT_={'vim_net_id': vim_id, 'created': net.get('created', False), 'instance_scenario_id':instance_uuid } #, 'type': net['type'] - INSERT_['datacenter_id'] = datacenter_site_id + INSERT_['datacenter_id'] = datacenter_site_id INSERT_['datacenter_tenant_id'] = scenarioDict["datacenter2tenant"][datacenter_site_id] if not net.get('created', False): INSERT_['status'] = "ACTIVE" @@ -868,31 +874,31 @@ class nfvo_db(db_base.db_base): instance_net_uuid = self._new_row_internal('instance_nets', INSERT_, True, instance_uuid, created_time) net_scene2instance[ sce_net_id ][datacenter_site_id] = instance_net_uuid net['uuid'] = instance_net_uuid #overwrite scnario uuid by instance uuid - + if 'ip_profile' in net: net['ip_profile']['net_id'] = None net['ip_profile']['sce_net_id'] = None net['ip_profile']['instance_net_id'] = instance_net_uuid created_time += 0.00001 ip_profile_id = self._new_row_internal('ip_profiles', net['ip_profile']) - + #instance_vnfs for vnf in scenarioDict['vnfs']: datacenter_site_id = vnf.get('datacenter_id', datacenter_id) INSERT_={'instance_scenario_id': instance_uuid, 'vnf_id': vnf['vnf_id'] } - INSERT_['datacenter_id'] = datacenter_site_id + INSERT_['datacenter_id'] = datacenter_site_id INSERT_['datacenter_tenant_id'] = scenarioDict["datacenter2tenant"][datacenter_site_id] if vnf.get("uuid"): INSERT_['sce_vnf_id'] = vnf['uuid'] created_time += 0.00001 instance_vnf_uuid = self._new_row_internal('instance_vnfs', INSERT_, True, instance_uuid, created_time) vnf['uuid'] = instance_vnf_uuid #overwrite scnario uuid by instance uuid - + #instance_nets #nets intraVNF for net in vnf['nets']: net_scene2instance[ net['uuid'] ] = {} INSERT_={'vim_net_id': net['vim_id'], 'created': net.get('created', False), 'instance_scenario_id':instance_uuid } #, 'type': net['type'] - INSERT_['datacenter_id'] = net.get('datacenter_id', datacenter_site_id) + INSERT_['datacenter_id'] = net.get('datacenter_id', datacenter_site_id) INSERT_['datacenter_tenant_id'] = scenarioDict["datacenter2tenant"][datacenter_id] if net.get("uuid"): INSERT_['net_id'] = net['uuid'] @@ -900,7 +906,7 @@ class nfvo_db(db_base.db_base): instance_net_uuid = self._new_row_internal('instance_nets', INSERT_, True, instance_uuid, created_time) net_scene2instance[ net['uuid'] ][datacenter_site_id] = instance_net_uuid net['uuid'] = instance_net_uuid #overwrite scnario uuid by instance uuid - + if 'ip_profile' in net: net['ip_profile']['net_id'] = None net['ip_profile']['sce_net_id'] = None @@ -914,7 +920,7 @@ class nfvo_db(db_base.db_base): created_time += 0.00001 instance_vm_uuid = self._new_row_internal('instance_vms', INSERT_, True, instance_uuid, created_time) vm['uuid'] = instance_vm_uuid #overwrite scnario uuid by instance uuid - + #instance_interfaces for interface in vm['interfaces']: net_id = interface.get('net_id', None) @@ -945,7 +951,7 @@ class nfvo_db(db_base.db_base): '''Obtain the instance information, filtering by one or several of the tenant, uuid or name instance_id is the uuid or the name if it is not a valid uuid format Only one instance must mutch the filtering or an error is returned - ''' + ''' tries = 2 while tries: try: @@ -969,17 +975,17 @@ class nfvo_db(db_base.db_base): self.logger.debug(cmd) self.cur.execute(cmd) rows = self.cur.fetchall() - + if self.cur.rowcount == 0: - raise db_base.db_base_Exception("No instance found where " + where_text, db_base.HTTP_Not_Found) + raise db_base.db_base_Exception("No instance found where " + where_text, httperrors.Not_Found) elif self.cur.rowcount > 1: raise db_base.db_base_Exception("More than one instance found where " + where_text, - db_base.HTTP_Bad_Request) + httperrors.Bad_Request) instance_dict = rows[0] if instance_dict["cloud_config"]: instance_dict["cloud-config"] = yaml.load(instance_dict["cloud_config"]) del instance_dict["cloud_config"] - + # instance_vnfs cmd = "SELECT iv.uuid as uuid, iv.vnf_id as vnf_id, sv.name as vnf_name, sce_vnf_id, datacenter_id"\ ", datacenter_tenant_id, v.mgmt_access, sv.member_vnf_index, v.osm_id as vnfd_osm_id "\ @@ -1035,7 +1041,7 @@ class nfvo_db(db_base.db_base): del vm["vm_uuid"] #instance_nets - #select_text = "instance_nets.uuid as uuid,sce_nets.name as net_name,instance_nets.vim_net_id as net_id,instance_nets.status as status,instance_nets.external as external" + #select_text = "instance_nets.uuid as uuid,sce_nets.name as net_name,instance_nets.vim_net_id as net_id,instance_nets.status as status,instance_nets.external as external" #from_text = "instance_nets join instance_scenarios on instance_nets.instance_scenario_id=instance_scenarios.uuid " + \ # "join sce_nets on instance_scenarios.scenario_id=sce_nets.scenario_id" #where_text = "instance_nets.instance_scenario_id='"+ instance_dict['uuid'] + "'" @@ -1092,18 +1098,18 @@ class nfvo_db(db_base.db_base): except (mdb.Error, AttributeError) as e: self._format_error(e, tries) tries -= 1 - + def delete_instance_scenario(self, instance_id, tenant_id=None): '''Deletes a instance_Scenario, filtering by one or serveral of the tenant, uuid or name instance_id is the uuid or the name if it is not a valid uuid format Only one instance_scenario must mutch the filtering or an error is returned - ''' + ''' tries = 2 while tries: try: with self.con: self.cur = self.con.cursor(mdb.cursors.DictCursor) - + #instance table where_list=[] if tenant_id is not None: where_list.append( "tenant_id='" + tenant_id +"'" ) @@ -1116,24 +1122,24 @@ class nfvo_db(db_base.db_base): self.logger.debug(cmd) self.cur.execute(cmd) rows = self.cur.fetchall() - + if self.cur.rowcount==0: - raise db_base.db_base_Exception("No instance found where " + where_text, db_base.HTTP_Bad_Request) + raise db_base.db_base_Exception("No instance found where " + where_text, httperrors.Bad_Request) elif self.cur.rowcount>1: - raise db_base.db_base_Exception("More than one instance found where " + where_text, db_base.HTTP_Bad_Request) + raise db_base.db_base_Exception("More than one instance found where " + where_text, httperrors.Bad_Request) instance_uuid = rows[0]["uuid"] instance_name = rows[0]["name"] - + #sce_vnfs cmd = "DELETE FROM instance_scenarios WHERE uuid='{}'".format(instance_uuid) self.logger.debug(cmd) self.cur.execute(cmd) - + return instance_uuid + " " + instance_name except (mdb.Error, AttributeError) as e: self._format_error(e, tries, "delete", "No dependences can avoid deleting!!!!") tries -= 1 - + def new_instance_scenario(self, instance_scenario_dict, tenant_id): #return self.new_row('vnfs', vnf_dict, None, tenant_id, True, True) return self._new_row_internal('instance_scenarios', instance_scenario_dict, tenant_id, add_uuid=True, root_uuid=None, log=True) @@ -1149,7 +1155,7 @@ class nfvo_db(db_base.db_base): def update_instance_vnf(self, instance_vnf_dict): #TODO: return - + def delete_instance_vnf(self, instance_vnf_id): #TODO: return @@ -1161,14 +1167,14 @@ class nfvo_db(db_base.db_base): def update_instance_vm(self, instance_vm_dict): #TODO: return - + def delete_instance_vm(self, instance_vm_id): #TODO: return def new_instance_net(self, instance_net_dict, tenant_id, instance_scenario_id = None): return self._new_row_internal('instance_nets', instance_net_dict, tenant_id, add_uuid=True, root_uuid=instance_scenario_id, log=True) - + def update_instance_net(self, instance_net_dict): #TODO: return @@ -1176,7 +1182,7 @@ class nfvo_db(db_base.db_base): def delete_instance_net(self, instance_net_id): #TODO: return - + def new_instance_interface(self, instance_interface_dict, tenant_id, instance_scenario_id = None): return self._new_row_internal('instance_interfaces', instance_interface_dict, tenant_id, add_uuid=True, root_uuid=instance_scenario_id, log=True) @@ -1190,7 +1196,7 @@ class nfvo_db(db_base.db_base): def update_datacenter_nets(self, datacenter_id, new_net_list=[]): ''' Removes the old and adds the new net list at datacenter list for one datacenter. - Attribute + Attribute datacenter_id: uuid of the datacenter to act upon table: table where to insert new_net_list: the new values to be inserted. If empty it only deletes the existing nets @@ -1216,4 +1222,4 @@ class nfvo_db(db_base.db_base): self._format_error(e, tries) tries -= 1 - + diff --git a/osm_ro/openmano_schemas.py b/osm_ro/openmano_schemas.py index 41b65be3..2d9dfae8 100644 --- a/osm_ro/openmano_schemas.py +++ b/osm_ro/openmano_schemas.py @@ -22,7 +22,7 @@ ## ''' -JSON schemas used by openmano httpserver.py module to parse the different files and messages sent through the API +JSON schemas used by openmano httpserver.py module to parse the different files and messages sent through the API ''' __author__="Alfonso Tierno, Gerardo Garcia, Pablo Montes" __date__ ="$09-oct-2014 09:09:48$" @@ -47,7 +47,7 @@ integer1_schema={"type":"integer","minimum":1} path_schema={"type":"string", "pattern":"^(\.){0,2}(/[^/\"':{}\(\)]+)+$"} vlan_schema={"type":"integer","minimum":1,"maximum":4095} vlan1000_schema={"type":"integer","minimum":1000,"maximum":4095} -mac_schema={"type":"string", "pattern":"^[0-9a-fA-F][02468aceACE](:[0-9a-fA-F]{2}){5}$"} #must be unicast LSB bit of MSB byte ==0 +mac_schema={"type":"string", "pattern":"^[0-9a-fA-F][02468aceACE](:[0-9a-fA-F]{2}){5}$"} #must be unicast LSB bit of MSB byte ==0 #mac_schema={"type":"string", "pattern":"^([0-9a-fA-F]{2}:){5}[0-9a-fA-F]{2}$"} ip_schema={"type":"string","pattern":"^((25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\.){3}(25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)$"} ip_prefix_schema={"type":"string","pattern":"^((25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\.){3}(25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)/(30|[12]?[0-9])$"} @@ -98,13 +98,13 @@ config_schema = { "vim_name": nameshort_schema, "vim_tenant_name": nameshort_schema, "mano_tenant_name": nameshort_schema, - "mano_tenant_id": id_schema, + "mano_tenant_id": id_schema, "http_console_proxy": {"type":"boolean"}, "http_console_host": nameshort_schema, "http_console_ports": { - "type": "array", + "type": "array", "items": {"OneOf": [ - port_schema, + port_schema, {"type": "object", "properties": {"from": port_schema, "to": port_schema}, "required": ["from", "to"]} ]} }, @@ -112,12 +112,14 @@ config_schema = { "log_socket_level": log_level_schema, "log_level_db": log_level_schema, "log_level_vim": log_level_schema, + "log_level_wim": log_level_schema, "log_level_nfvo": log_level_schema, "log_level_http": log_level_schema, "log_level_console": log_level_schema, "log_level_ovim": log_level_schema, "log_file_db": path_schema, "log_file_vim": path_schema, + "log_file_wim": path_schema, "log_file_nfvo": path_schema, "log_file_http": path_schema, "log_file_console": path_schema, @@ -429,7 +431,7 @@ external_connection_schema_v02 = { "properties":{ "name": name_schema, "mgmt": {"type":"boolean"}, - "type": {"type": "string", "enum":["e-line", "e-lan"]}, + "type": {"type": "string", "enum":["e-line", "e-lan"]}, "implementation": {"type": "string", "enum":["overlay", "underlay"]}, "VNFC": name_schema, "local_iface_name": name_schema , @@ -579,7 +581,7 @@ vnfc_schema = { "bridge-ifaces": bridge_interfaces_schema, "devices": devices_schema, "boot-data" : boot_data_vdu_schema - + }, "required": ["name"], "oneOf": [ @@ -766,7 +768,7 @@ nsd_schema_v02 = { }, } }, - + }, "required": ["vnfs", "name"], "additionalProperties": False @@ -860,7 +862,7 @@ nsd_schema_v03 = { }, } }, - + }, "required": ["vnfs", "networks","name"], "additionalProperties": False @@ -1027,7 +1029,7 @@ instance_scenario_create_schema_v01 = { "vim-network-name": name_schema, "ip-profile": ip_profile_schema, "name": name_schema, - } + } } } }, @@ -1057,7 +1059,7 @@ instance_scenario_create_schema_v01 = { } }, "ip-profile": ip_profile_schema, - #if the network connects VNFs deployed at different sites, you must specify one entry per site that this network connect to + #if the network connects VNFs deployed at different sites, you must specify one entry per site that this network connect to "sites": { "type":"array", "minLength":1, @@ -1066,16 +1068,16 @@ instance_scenario_create_schema_v01 = { "properties":{ # By default for an scenario 'external' network openmano looks for an existing VIM network to map this external scenario network, # for other networks openamno creates at VIM - # Use netmap-create to force to create an external scenario network + # Use netmap-create to force to create an external scenario network "netmap-create": {"oneOf":[name_schema,{"type": "null"}]}, #datacenter network to use. Null if must be created as an internal net - #netmap-use: Indicates an existing VIM network that must be used for this scenario network. + #netmap-use: Indicates an existing VIM network that must be used for this scenario network. #Can use both the VIM network name (if it is not ambiguous) or the VIM net UUID #If both 'netmap-create' and 'netmap-use'are supplied, netmap-use precedes, but if fails openmano follows the netmap-create #In oder words, it is the same as 'try to map to the VIM network (netmap-use) if exist, and if not create the network (netmap-create) - "netmap-use": name_schema, # + "netmap-use": name_schema, # "vim-network-name": name_schema, #override network name #"ip-profile": ip_profile_schema, - "datacenter": name_schema, + "datacenter": name_schema, } } }, @@ -1215,4 +1217,4 @@ sdn_external_port_schema = { "mac": mac_schema }, "required": ["port"] -} \ No newline at end of file +} diff --git a/osm_ro/openmanoclient.py b/osm_ro/openmanoclient.py index 15662037..ece30aad 100644 --- a/osm_ro/openmanoclient.py +++ b/osm_ro/openmanoclient.py @@ -565,7 +565,168 @@ class openmanoclient(): if mano_response.status_code==200: return content else: - raise OpenmanoResponseException(str(content)) + raise OpenmanoResponseException(str(content)) + + # WIMS + + def list_wims(self, all_tenants=False, **kwargs): + '''Obtain a list of wims, that are the WIM information at openmano + Params: can be filtered by 'uuid','name','wim_url','type' + Return: Raises an exception on error + Obtain a dictionary with format {'wims':[{wim1_info},{wim2_info},...]}} + ''' + return self._list_item("wims", all_tenants, filter_dict=kwargs) + + def get_wim(self, uuid=None, name=None, all_tenants=False): + '''Obtain the information of a wim + Params: uuid or/and name. If only name is supplied, there must be only one or an exception is raised + Return: Raises an exception on error, not found, found several + Obtain a dictionary with format {'wim':{wim_info}} + ''' + return self._get_item("wims", uuid, name, all_tenants) + + def delete_wim(self, uuid=None, name=None): + '''Delete a wim + Params: uuid or/and name. If only name is supplied, there must be only one or an exception is raised + Return: Raises an exception on error, not found, found several, not free + Obtain a dictionary with format {'result': text indicating deleted} + ''' + if not uuid: + # check that exist + uuid = self._get_item_uuid("wims", uuid, name, all_tenants=True) + return self._del_item("wims", uuid, name, all_tenants=None) + + def create_wim(self, descriptor=None, descriptor_format=None, name=None, wim_url=None, **kwargs): + # , type="openvim", public=False, description=None): + '''Creates a wim + Params: must supply a descriptor or/and just a name and a wim_url + descriptor: with format {'wim':{new_wim_info}} + new_wim_info must contain 'name', 'wim_url', and optionally 'description' + must be a dictionary or a json/yaml text. + name: the wim name. Overwrite descriptor name if any + wim_url: the wim URL. Overwrite descriptor vim_url if any + wim_type: the WIM type, can be tapi, odl, onos. Overwrite descriptor type if any + public: boolean, by default not public + description: wim description. Overwrite descriptor description if any + config: dictionary with extra configuration for the concrete wim + Return: Raises an exception on error + Obtain a dictionary with format {'wim:{new_wim_info}} + ''' + if isinstance(descriptor, str): + descriptor = self.parse(descriptor, descriptor_format) + elif descriptor: + pass + elif name and wim_url: + descriptor = {"wim": {"name": name, "wim_url": wim_url}} + else: + raise OpenmanoBadParamsException("Missing descriptor, or name and wim_url") + + if 'wim' not in descriptor or len(descriptor) != 1: + raise OpenmanoBadParamsException("Descriptor must contain only one 'wim' field") + if name: + descriptor['wim']['name'] = name + if wim_url: + descriptor['wim']['wim_url'] = wim_url + for param in kwargs: + descriptor['wim'][param] = kwargs[param] + + return self._create_item("wims", descriptor, all_tenants=None) + + def edit_wim(self, uuid=None, name=None, descriptor=None, descriptor_format=None, all_tenants=False, + **kwargs): + '''Edit the parameters of a wim + Params: must supply a descriptor or/and a parameter to change + uuid or/and name. If only name is supplied, there must be only one or an exception is raised + descriptor: with format {'wim':{params to change info}} + must be a dictionary or a json/yaml text. + parameters to change can be supplied by the descriptor or as parameters: + new_name: the wim name + wim_url: the wim URL + wim_type: the wim type, can be tapi, onos, odl + public: boolean, available to other tenants + description: wim description + Return: Raises an exception on error, not found or found several + Obtain a dictionary with format {'wim':{new_wim_info}} + ''' + if isinstance(descriptor, str): + descriptor = self.parse(descriptor, descriptor_format) + elif descriptor: + pass + elif kwargs: + descriptor = {"wim": {}} + else: + raise OpenmanoBadParamsException("Missing descriptor") + + if 'wim' not in descriptor or len(descriptor) != 1: + raise OpenmanoBadParamsException("Descriptor must contain only one 'wim' field") + for param in kwargs: + if param == 'new_name': + descriptor['wim']['name'] = kwargs[param] + else: + descriptor['wim'][param] = kwargs[param] + return self._edit_item("wims", descriptor, uuid, name, all_tenants=None) + + def attach_wim(self, uuid=None, name=None, descriptor=None, descriptor_format=None, wim_user=None, + wim_password=None, wim_tenant_name=None, wim_tenant_id=None): + # check that exist + uuid = self._get_item_uuid("wims", uuid, name, all_tenants=True) + tenant_text = "/" + self._get_tenant() + + if isinstance(descriptor, str): + descriptor = self.parse(descriptor, descriptor_format) + elif descriptor: + pass + elif wim_user or wim_password or wim_tenant_name or wim_tenant_id: + descriptor = {"wim": {}} + else: + raise OpenmanoBadParamsException("Missing descriptor or params") + + if wim_user or wim_password or wim_tenant_name or wim_tenant_id: + # print args.name + try: + if wim_user: + descriptor['wim']['wim_user'] = wim_user + if wim_password: + descriptor['wim']['wim_password'] = wim_password + if wim_tenant_name: + descriptor['wim']['wim_tenant_name'] = wim_tenant_name + if wim_tenant_id: + descriptor['wim']['wim_tenant'] = wim_tenant_id + except (KeyError, TypeError) as e: + if str(e) == 'wim': + error_pos = "missing field 'wim'" + else: + error_pos = "wrong format" + raise OpenmanoBadParamsException("Wrong wim descriptor: " + error_pos) + + payload_req = yaml.safe_dump(descriptor) + # print payload_req + URLrequest = "{}{}/wims/{}".format(self.endpoint_url, tenant_text, uuid) + self.logger.debug("openmano POST %s %s", URLrequest, payload_req) + mano_response = requests.post(URLrequest, headers=self.headers_req, data=payload_req) + self.logger.debug("openmano response: %s", mano_response.text) + + content = self._parse_yaml(mano_response.text, response=True) + if mano_response.status_code == 200: + return content + else: + raise OpenmanoResponseException(str(content)) + + def detach_wim(self, uuid=None, name=None): + if not uuid: + # check that exist + uuid = self._get_item_uuid("wims", uuid, name, all_tenants=False) + tenant_text = "/" + self._get_tenant() + URLrequest = "{}{}/wims/{}".format(self.endpoint_url, tenant_text, uuid) + self.logger.debug("openmano DELETE %s", URLrequest) + mano_response = requests.delete(URLrequest, headers=self.headers_req) + self.logger.debug("openmano response: %s", mano_response.text) + + content = self._parse_yaml(mano_response.text, response=True) + if mano_response.status_code == 200: + return content + else: + raise OpenmanoResponseException(str(content)) #VNFS def list_vnfs(self, all_tenants=False, **kwargs): diff --git a/osm_ro/openmanod.cfg b/osm_ro/openmanod.cfg index 2aa319ec..cf3d00c4 100644 --- a/osm_ro/openmanod.cfg +++ b/osm_ro/openmanod.cfg @@ -71,6 +71,8 @@ log_level_db: ERROR #database log levels #log_file_db: /opt/openmano/logs/openmano_db.log #log_level_vim: DEBUG #VIM connection log levels #log_file_vim: /opt/openmano/logs/openmano_vimconn.log +#log_level_wim: DEBUG #WIM connection log levels +#log_file_wim: /opt/openmano/logs/openmano_wimconn.log #log_level_nfvo: DEBUG #Main engine log levels #log_file_nfvo: /opt/openmano/logs/openmano_nfvo.log #log_level_http: DEBUG #Main engine log levels diff --git a/osm_ro/tests/db_helpers.py b/osm_ro/tests/db_helpers.py new file mode 100644 index 00000000..bedf9a5c --- /dev/null +++ b/osm_ro/tests/db_helpers.py @@ -0,0 +1,187 @@ +# -*- coding: utf-8 -*- +## +# Copyright 2018 University of Bristol - High Performance Networks Research +# Group +# All Rights Reserved. +# +# Contributors: Anderson Bravalheri, Dimitrios Gkounis, Abubakar Siddique +# Muqaddas, Navdeep Uniyal, Reza Nejabati and Dimitra Simeonidou +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +# +# For those usages not covered by the Apache License, Version 2.0 please +# contact with: +# +# Neither the name of the University of Bristol nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# This work has been performed in the context of DCMS UK 5G Testbeds +# & Trials Programme and in the framework of the Metro-Haul project - +# funded by the European Commission under Grant number 761727 through the +# Horizon 2020 and 5G-PPP programmes. +## +import hashlib +import shlex +import unittest +from contextlib import contextmanager +from functools import wraps +from hashlib import md5 +from os import environ, pathsep +from subprocess import STDOUT, check_output +from uuid import UUID + +from MySQLdb import connect + +from ..nfvo_db import nfvo_db + +HOST = environ.get('TEST_DB_HOST', 'localhost') +USER = environ.get('TEST_DB_USER', 'mano') +PASSWORD = environ.get('TEST_DB_PASSWORD', 'manopw') +DATABASE = environ.get('TEST_DB_DATABASE', 'mano_db') + + +def uuid(seed): + """Generates strings with a UUID format in a repeatable way""" + return str(UUID(md5(str(seed)).hexdigest())) + + +def sha1(text): + """Generates SHA1 hash code from a text string""" + return hashlib.sha1(text).hexdigest() + + +def run(*args, **kwargs): + """Run a command inside a subprocess, raising an exception when it fails + + Arguments: + *args: you can pass any number of arquments as separated words in the + shell, or just a single string with the entire command + **kwargs: proxied to subprocess.check_output (by default + ``stderr=STDOUT`` and ``universal_newlines=True`` + """ + if len(args) == 1 and isinstance(args[0], str): + args = shlex.split(args[0]) + + opts = dict(stderr=STDOUT, universal_newlines=True) + opts.update(kwargs) + return check_output(args, **opts) + + +# In order to not mess around, enforce user to explicit set the +# test database in a env variable +@unittest.skipUnless( + environ.get('TEST_DB_HOST'), + 'Test database not available. Please set TEST_DB_HOST env var') +class TestCaseWithDatabase(unittest.TestCase): + """Connect to the database and provide methods to facilitate isolating the + database stored inside it between tests. + + In order to avoid connecting, reconnecting, creating tables and destroying + tables all the time, this class manage the database using class-level + fixtures. This reduce the cost of performing these actions but not + guarantees isolation in the DB state between the tests. + To enforce isolation, please call the ``setup_tables`` and + ``empty_database`` directly, or write one single test per class. + """ + + host = HOST + user = USER + password = PASSWORD + database = DATABASE + + @classmethod + def setup_tables(cls): + """Make sure the database is set up and in the right version, with all the + required tables. + """ + dbutils = environ.get('DBUTILS') + + if dbutils: + environ["PATH"] += pathsep + dbutils + + return run('init_mano_db.sh', + '-u', cls.user, + '-p', cls.password, + '-h', cls.host, + '-d', cls.database) + + @classmethod + def empty_database(cls): + """Clear the database, so one test does not interfere with the other""" + # Create a custom connection not attached to the database, so we can + # destroy and recreate the database itself + connection = connect(cls.host, cls.user, cls.password) + cursor = connection.cursor() + cursor.execute( + "DROP DATABASE {};".format( + connection.escape_string(cls.database))) + cursor.execute( + "CREATE DATABASE {};".format( + connection.escape_string(cls.database))) + cursor.close() + connection.close() + + +class TestCaseWithDatabasePerTest(TestCaseWithDatabase): + """Ensure a connection to the database before and + drop tables after each test runs + """ + + def setUp(self): + self.setup_tables() + self.addCleanup(self.empty_database) + + self.maxDiff = None + + self.db = nfvo_db(self.host, self.user, self.password, self.database) + self.db.connect() + + def populate(self, seeds=None, **kwargs): + """Seed the database with initial values""" + if not seeds: + seeds = [] + if not isinstance(seeds, (list, tuple)): + seeds = [seeds] + if kwargs: + seeds.append(kwargs) + self.db.new_rows(seeds) + + def count(self, table): + """Count number of rows in a table""" + return self.db.get_rows( + SELECT='COUNT(*) as count', FROM=table)[0]['count'] + + @contextmanager + def disable_foreign_keys(self): + """Do the test without checking foreign keys""" + try: + cursor = self.db.con.cursor() + cursor.execute('SET FOREIGN_KEY_CHECKS=0;') + yield + finally: + cursor.execute('SET FOREIGN_KEY_CHECKS=1;') + + +def disable_foreign_keys(test): + """Do the test without checking foreign keys. + To be used together in subclasses of TestCaseWithDatabasePerTest + """ + @wraps(test) + def _no_check(self, *args, **kwargs): + with self.disable_foreign_keys(): + result = test(self, *args, **kwargs) + + return result + + return _no_check diff --git a/osm_ro/tests/helpers.py b/osm_ro/tests/helpers.py new file mode 100644 index 00000000..787fbced --- /dev/null +++ b/osm_ro/tests/helpers.py @@ -0,0 +1,121 @@ +# -*- coding: utf-8 -*- +## +# Copyright 2018 University of Bristol - High Performance Networks Research +# Group +# All Rights Reserved. +# +# Contributors: Anderson Bravalheri, Dimitrios Gkounis, Abubakar Siddique +# Muqaddas, Navdeep Uniyal, Reza Nejabati and Dimitra Simeonidou +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +# +# For those usages not covered by the Apache License, Version 2.0 please +# contact with: +# +# Neither the name of the University of Bristol nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# This work has been performed in the context of DCMS UK 5G Testbeds +# & Trials Programme and in the framework of the Metro-Haul project - +# funded by the European Commission under Grant number 761727 through the +# Horizon 2020 and 5G-PPP programmes. +## + +import logging +import unittest +from collections import defaultdict + +from six import StringIO + +from mock import MagicMock, patch + +logger = logging.getLogger() + + +class TestCaseWithLogging(unittest.TestCase): + """Attach a special handler to the root logger, capturing the logs in a + internal buffer (caplog property). + + To retrieve the logs, do:: + + self.caplog.getvalue() + """ + def setUp(self): + super(TestCaseWithLogging, self).setUp() + self.logger = logging.getLogger() + self.caplog = StringIO() + self.log_handler = logging.StreamHandler(self.caplog) + self.logger.addHandler(self.log_handler) + self.logger.setLevel(logging.NOTSET) + + def tearDown(self): + super(TestCaseWithLogging, self).tearDown() + self.log_handler.close() + self.logger.removeHandler(self.log_handler) + + +def mock_imports(modules, preserve=()): + """Given a list of modules, mock everything, unless listed in the preserve + argument. + """ + # Ensure iterable + if isinstance(modules, str): + modules = (modules,) + if isinstance(preserve, str): + preserve = (preserve,) + + # First expand the list, since children modules needs their parent also + # mocked most of the time. + # Example: ['Crypto.PublicKey'] => ['Crypto', 'Crypto.PublicKey'] + all_modules = [] + for name in modules: + parts = name.split('.') + compound_name = [] + for part in parts: + compound_name.append(part) + all_modules.append('.'.join(compound_name)) + + all_modules = set(m for m in all_modules if m not in preserve) + for module in all_modules: + logger.info('Mocking module `%s`', module) + + mocks = {module: MagicMock() for module in all_modules} + + return patch.dict('sys.modules', **mocks) + + +def mock_dict(**kwargs): + """Create a dict that always respond something. + + Arguments: + **kwargs: certain items that should be set in the created object + """ + response = defaultdict(MagicMock) + for k, v in kwargs.items(): + response[k] = v + + return response + + +def mock_object(**kwargs): + """Create an object that always respond something. + + Arguments: + **kwargs: certain attributes that should be set in the created object + """ + response = MagicMock() + for k, v in kwargs.items(): + setattr(response, k, v) + + return response diff --git a/osm_ro/utils.py b/osm_ro/utils.py index 2ae062fd..592abc12 100644 --- a/osm_ro/utils.py +++ b/osm_ro/utils.py @@ -30,8 +30,16 @@ __author__="Alfonso Tierno, Gerardo Garcia" __date__ ="$08-sep-2014 12:21:22$" import datetime +import time import warnings -from jsonschema import validate as js_v, exceptions as js_e +from functools import reduce +from itertools import tee + +from six.moves import filter, filterfalse + +from jsonschema import exceptions as js_e +from jsonschema import validate as js_v + #from bs4 import BeautifulSoup def read_file(file_to_read): @@ -42,7 +50,7 @@ def read_file(file_to_read): f.close() except Exception as e: return (False, str(e)) - + return (True, read_data) def write_file(file_to_write, text): @@ -53,7 +61,7 @@ def write_file(file_to_write, text): f.close() except Exception as e: return (False, str(e)) - + return (True, None) def format_in(http_response, schema): @@ -93,8 +101,22 @@ def remove_extra_items(data, schema): # return text +def delete_nulls(var): + if type(var) is dict: + for k in var.keys(): + if var[k] is None: del var[k] + elif type(var[k]) is dict or type(var[k]) is list or type(var[k]) is tuple: + if delete_nulls(var[k]): del var[k] + if len(var) == 0: return True + elif type(var) is list or type(var) is tuple: + for k in var: + if type(k) is dict: delete_nulls(k) + if len(var) == 0: return True + return False + + def convert_bandwidth(data, reverse=False): - '''Check the field bandwidth recursivelly and when found, it removes units and convert to number + '''Check the field bandwidth recursivelly and when found, it removes units and convert to number It assumes that bandwidth is well formed Attributes: 'data': dictionary bottle.FormsDict variable to be checked. None or empty is consideted valid @@ -127,7 +149,21 @@ def convert_bandwidth(data, reverse=False): if type(k) is dict or type(k) is tuple or type(k) is list: convert_bandwidth(k, reverse) - +def convert_float_timestamp2str(var): + '''Converts timestamps (created_at, modified_at fields) represented as float + to a string with the format '%Y-%m-%dT%H:%i:%s' + It enters recursively in the dict var finding this kind of variables + ''' + if type(var) is dict: + for k,v in var.items(): + if type(v) is float and k in ("created_at", "modified_at"): + var[k] = time.strftime("%Y-%m-%dT%H:%M:%S", time.localtime(v) ) + elif type(v) is dict or type(v) is list or type(v) is tuple: + convert_float_timestamp2str(v) + if len(var) == 0: return True + elif type(var) is list or type(var) is tuple: + for v in var: + convert_float_timestamp2str(v) def convert_datetime2str(var): '''Converts a datetime variable to a string with the format '%Y-%m-%dT%H:%i:%s' @@ -137,7 +173,7 @@ def convert_datetime2str(var): for k,v in var.items(): if type(v) is datetime.datetime: var[k]= v.strftime('%Y-%m-%dT%H:%M:%S') - elif type(v) is dict or type(v) is list or type(v) is tuple: + elif type(v) is dict or type(v) is list or type(v) is tuple: convert_datetime2str(v) if len(var) == 0: return True elif type(var) is list or type(var) is tuple: @@ -145,7 +181,7 @@ def convert_datetime2str(var): convert_datetime2str(v) def convert_str2boolean(data, items): - '''Check recursively the content of data, and if there is an key contained in items, convert value from string to boolean + '''Check recursively the content of data, and if there is an key contained in items, convert value from string to boolean Done recursively Attributes: 'data': dictionary variable to be checked. None or empty is considered valid @@ -207,4 +243,105 @@ def deprecated(message): warnings.simplefilter('default', DeprecationWarning) return func(*args, **kwargs) return deprecated_func - return deprecated_decorator \ No newline at end of file + return deprecated_decorator + + +def truncate(text, max_length=1024): + """Limit huge texts in number of characters""" + text = str(text) + if text and len(text) >= max_length: + return text[:max_length//2-3] + " ... " + text[-max_length//2+3:] + return text + + +def merge_dicts(*dicts, **kwargs): + """Creates a new dict merging N others and keyword arguments. + Right-most dicts take precedence. + Keyword args take precedence. + """ + return reduce( + lambda acc, x: acc.update(x) or acc, + list(dicts) + [kwargs], {}) + + +def remove_none_items(adict): + """Return a similar dict without keys associated to None values""" + return {k: v for k, v in adict.items() if v is not None} + + +def filter_dict_keys(adict, allow): + """Return a similar dict, but just containing the explicitly allowed keys + + Arguments: + adict (dict): Simple python dict data struct + allow (list): Explicits allowed keys + """ + return {k: v for k, v in adict.items() if k in allow} + + +def filter_out_dict_keys(adict, deny): + """Return a similar dict, but not containing the explicitly denied keys + + Arguments: + adict (dict): Simple python dict data struct + deny (list): Explicits denied keys + """ + return {k: v for k, v in adict.items() if k not in deny} + + +def expand_joined_fields(record): + """Given a db query result, explode the fields that contains `.` (join + operations). + + Example + >> expand_joined_fiels({'wim.id': 2}) + # {'wim': {'id': 2}} + """ + result = {} + for field, value in record.items(): + keys = field.split('.') + target = result + target = reduce(lambda target, key: target.setdefault(key, {}), + keys[:-1], result) + target[keys[-1]] = value + + return result + + +def ensure(condition, exception): + """Raise an exception if condition is not met""" + if not condition: + raise exception + + +def partition(predicate, iterable): + """Create two derived iterators from a single one + The first iterator created will loop thought the values where the function + predicate is True, the second one will iterate over the values where it is + false. + """ + iterable1, iterable2 = tee(iterable) + return filter(predicate, iterable2), filterfalse(predicate, iterable1) + + +def pipe(*functions): + """Compose functions of one argument in the opposite order, + So pipe(f, g)(x) = g(f(x)) + """ + return lambda x: reduce(lambda acc, f: f(acc), functions, x) + + +def compose(*functions): + """Compose functions of one argument, + So compose(f, g)(x) = f(g(x)) + """ + return lambda x: reduce(lambda acc, f: f(acc), functions[::-1], x) + + +def safe_get(target, key_path, default=None): + """Given a path of keys (eg.: "key1.key2.key3"), return a nested value in + a nested dict if present, or the default value + """ + keys = key_path.split('.') + target = reduce(lambda acc, key: acc.get(key) or {}, keys[:-1], target) + return target.get(keys[-1], default) diff --git a/osm_ro/vim_thread.py b/osm_ro/vim_thread.py index c981e31f..2713c760 100644 --- a/osm_ro/vim_thread.py +++ b/osm_ro/vim_thread.py @@ -23,7 +23,7 @@ """" This is thread that interacts with a VIM. It processes TASKs sequentially against a single VIM. -The tasks are stored at database in table vim_actions +The tasks are stored at database in table vim_wim_actions The task content is (M: stored at memory, D: stored at database): MD instance_action_id: reference a global action over an instance-scenario: database instance_actions MD task_index: index number of the task. This together with the previous forms a unique key identifier @@ -49,8 +49,8 @@ The task content is (M: stored at memory, D: stored at database): vim_status: VIM status of the element. Stored also at database in the instance_XXX M depends: dict with task_index(from depends_on) to task class M params: same as extra[params] but with the resolved dependencies - M vim_interfaces: similar to extra[interfaces] but with VIM information. Stored at database in the instance_XXX but not at vim_actions - M vim_info: Detailed information of a vm,net from the VIM. Stored at database in the instance_XXX but not at vim_actions + M vim_interfaces: similar to extra[interfaces] but with VIM information. Stored at database in the instance_XXX but not at vim_wim_actions + M vim_info: Detailed information of a vm,net from the VIM. Stored at database in the instance_XXX but not at vim_wim_actions MD error_msg: descriptive text upon an error.Stored also at database instance_XXX MD created_at: task creation time MD modified_at: last task update time. On refresh it contains when this task need to be refreshed @@ -189,7 +189,7 @@ class vim_thread(threading.Thread): while True: # get 200 (database_limit) entries each time with self.db_lock: - vim_actions = self.db.get_rows(FROM="vim_actions", + vim_actions = self.db.get_rows(FROM="vim_wim_actions", WHERE={"datacenter_vim_id": self.datacenter_tenant_id, "item_id>=": old_item_id}, ORDER_BY=("item_id", "item", "created_at",), @@ -393,7 +393,7 @@ class vim_thread(threading.Thread): if task_need_update: with self.db_lock: self.db.update_rows( - 'vim_actions', + 'vim_wim_actions', UPDATE={"extra": yaml.safe_dump(task["extra"], default_flow_style=True, width=256), "error_msg": task.get("error_msg"), "modified_at": now}, WHERE={'instance_action_id': task['instance_action_id'], @@ -463,7 +463,7 @@ class vim_thread(threading.Thread): with self.db_lock: self.db.update_rows('instance_nets', UPDATE=temp_dict, WHERE={"uuid": task["item_id"]}) self.db.update_rows( - 'vim_actions', + 'vim_wim_actions', UPDATE={"extra": yaml.safe_dump(task["extra"], default_flow_style=True, width=256), "error_msg": task.get("error_msg"), "modified_at": now}, WHERE={'instance_action_id': task['instance_action_id'], @@ -644,7 +644,7 @@ class vim_thread(threading.Thread): now = time.time() with self.db_lock: self.db.update_rows( - table="vim_actions", + table="vim_wim_actions", UPDATE={"status": task["status"], "vim_id": task.get("vim_id"), "modified_at": now, "error_msg": task["error_msg"], "extra": yaml.safe_dump(task["extra"], default_flow_style=True, width=256)}, @@ -811,7 +811,7 @@ class vim_thread(threading.Thread): instance_action_id = ins_action_id with self.db_lock: - tasks = self.db.get_rows(FROM="vim_actions", WHERE={"instance_action_id": instance_action_id, + tasks = self.db.get_rows(FROM="vim_wim_actions", WHERE={"instance_action_id": instance_action_id, "task_index": task_index}) if not tasks: return None diff --git a/osm_ro/vimconn_openstack.py b/osm_ro/vimconn_openstack.py index 53a5295a..2b156d74 100644 --- a/osm_ro/vimconn_openstack.py +++ b/osm_ro/vimconn_openstack.py @@ -36,7 +36,7 @@ __author__ = "Alfonso Tierno, Gerardo Garcia, Pablo Montes, xFlow Research, Igor __date__ = "$22-sep-2017 23:59:59$" import vimconn -# import json +import json import logging import netaddr import time @@ -44,6 +44,8 @@ import yaml import random import re import copy +from pprint import pformat +from types import StringTypes from novaclient import client as nClient, exceptions as nvExceptions from keystoneauth1.identity import v2, v3 @@ -77,6 +79,18 @@ supportedClassificationTypes = ['legacy_flow_classifier'] volume_timeout = 600 server_timeout = 600 + +class SafeDumper(yaml.SafeDumper): + def represent_data(self, data): + # Openstack APIs use custom subclasses of dict and YAML safe dumper + # is designed to not handle that (reference issue 142 of pyyaml) + if isinstance(data, dict) and data.__class__ != dict: + # A simple solution is to convert those items back to dicts + data = dict(data.items()) + + return super(SafeDumper, self).represent_data(data) + + class vimconnector(vimconn.vimconnector): def __init__(self, uuid, name, tenant_id, tenant_name, url, url_admin=None, user=None, passwd=None, log_level=None, config={}, persistent_info={}): @@ -158,11 +172,30 @@ class vimconnector(vimconn.vimconnector): vimconn.vimconnector.__setitem__(self, index, value) self.session['reload_client'] = True + def serialize(self, value): + """Serialization of python basic types. + + In the case value is not serializable a message will be logged and a + simple representation of the data that cannot be converted back to + python is returned. + """ + if isinstance(value, StringTypes): + return value + + try: + return yaml.dump(value, Dumper=SafeDumper, + default_flow_style=True, width=256) + except yaml.representer.RepresenterError: + self.logger.debug( + 'The following entity cannot be serialized in YAML:' + '\n\n%s\n\n', pformat(value), exc_info=True) + return str(value) + def _reload_connection(self): '''Called before any operation, it check if credentials has changed Throw keystoneclient.apiclient.exceptions.AuthorizationFailure ''' - #TODO control the timing and possible token timeout, but it seams that python client does this task for us :-) + #TODO control the timing and possible token timeout, but it seams that python client does this task for us :-) if self.session['reload_client']: if self.config.get('APIversion'): self.api_version3 = self.config['APIversion'] == 'v3.3' or self.config['APIversion'] == '3' @@ -562,13 +595,13 @@ class vimconnector(vimconn.vimconnector): net_id: #VIM id of this network status: #Mandatory. Text with one of: # DELETED (not found at vim) - # VIM_ERROR (Cannot connect to VIM, VIM response error, ...) + # VIM_ERROR (Cannot connect to VIM, VIM response error, ...) # OTHER (Vim reported other status not understood) # ERROR (VIM indicates an ERROR status) - # ACTIVE, INACTIVE, DOWN (admin down), + # ACTIVE, INACTIVE, DOWN (admin down), # BUILD (on building process) # - error_msg: #Text with VIM error message, if any. Or the VIM connection ERROR + error_msg: #Text with VIM error message, if any. Or the VIM connection ERROR vim_info: #Text with plain information obtained from vim (yaml.safe_dump) ''' @@ -585,10 +618,9 @@ class vimconnector(vimconn.vimconnector): if net['status'] == "ACTIVE" and not net_vim['admin_state_up']: net['status'] = 'DOWN' - try: - net['vim_info'] = yaml.safe_dump(net_vim, default_flow_style=True, width=256) - except yaml.representer.RepresenterError: - net['vim_info'] = str(net_vim) + + net['vim_info'] = self.serialize(net_vim) + if net_vim.get('fault'): #TODO net['error_msg'] = str(net_vim['fault']) except vimconn.vimconnNotFoundException as e: @@ -721,7 +753,7 @@ class vimconnector(vimconn.vimconnector): # raise vimconn.vimconnException("Passthrough interfaces are not supported for the openstack connector", http_code=vimconn.HTTP_Service_Unavailable) # #TODO, add the key 'pci_passthrough:alias"="