X-Git-Url: https://osm.etsi.org/gitweb/?a=blobdiff_plain;f=RO%2Fosm_ro%2Fwim%2Fpersistence.py;fp=RO%2Fosm_ro%2Fwim%2Fpersistence.py;h=0000000000000000000000000000000000000000;hb=e4d1a3d01fee5d5e234bd3e2d9b26ec0ae4eef44;hp=6fbaf8f70154c9f42ec3b2c60e0db597fd251a60;hpb=ee6a62063244ab492bc4d09cc5fe89388a5d1ea6;p=osm%2FRO.git diff --git a/RO/osm_ro/wim/persistence.py b/RO/osm_ro/wim/persistence.py deleted file mode 100644 index 6fbaf8f7..00000000 --- a/RO/osm_ro/wim/persistence.py +++ /dev/null @@ -1,1011 +0,0 @@ -# -*- coding: utf-8 -*- -## -# Copyright 2018 University of Bristol - High Performance Networks Research -# Group -# All Rights Reserved. -# -# Contributors: Anderson Bravalheri, Dimitrios Gkounis, Abubakar Siddique -# Muqaddas, Navdeep Uniyal, Reza Nejabati and Dimitra Simeonidou -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. -# -# For those usages not covered by the Apache License, Version 2.0 please -# contact with: -# -# Neither the name of the University of Bristol nor the names of its -# contributors may be used to endorse or promote products derived from -# this software without specific prior written permission. -# -# This work has been performed in the context of DCMS UK 5G Testbeds -# & Trials Programme and in the framework of the Metro-Haul project - -# funded by the European Commission under Grant number 761727 through the -# Horizon 2020 and 5G-PPP programmes. -## - -"""This module contains only logic related to managing records in a database -which includes data format normalization, data format validation and etc. -(It works as an extension to `nfvo_db.py` for the WIM feature) - -No domain logic/architectural concern should be present in this file. -""" -import json -import logging -import string -from contextlib import contextmanager -from hashlib import sha1 -from itertools import groupby -from operator import itemgetter -# from sys import exc_info -# from time import time -from uuid import uuid1 as generate_uuid - -import yaml - -from ..utils import ( - check_valid_uuid, - convert_float_timestamp2str, - expand_joined_fields, - filter_dict_keys, - filter_out_dict_keys, - merge_dicts, - remove_none_items -) -from .errors import ( - DbBaseException, - InvalidParameters, - MultipleRecordsFound, - NoRecordFound, - UndefinedUuidOrName, - UndefinedWanMappingType, - UnexpectedDatabaseError, - WimAccountOverwrite, - WimAndTenantAlreadyAttached -) - -_WIM = 'wims AS wim ' - -_WIM_JOIN = ( - _WIM + - ' JOIN wim_nfvo_tenants AS association ' - ' ON association.wim_id=wim.uuid ' - ' JOIN nfvo_tenants AS nfvo_tenant ' - ' ON association.nfvo_tenant_id=nfvo_tenant.uuid ' - ' JOIN wim_accounts AS wim_account ' - ' ON association.wim_account_id=wim_account.uuid ' -) - -_WIM_ACCOUNT_JOIN = ( - 'wim_accounts AS wim_account ' - ' JOIN wim_nfvo_tenants AS association ' - ' ON association.wim_account_id=wim_account.uuid ' - ' JOIN wims AS wim ' - ' ON association.wim_id=wim.uuid ' - ' JOIN nfvo_tenants AS nfvo_tenant ' - ' ON association.nfvo_tenant_id=nfvo_tenant.uuid ' -) - -_DATACENTER_JOIN = ( - 'datacenters AS datacenter ' - ' JOIN tenants_datacenters AS association ' - ' ON association.datacenter_id=datacenter.uuid ' - ' JOIN datacenter_tenants as datacenter_account ' - ' ON association.datacenter_tenant_id=datacenter_account.uuid ' - ' JOIN nfvo_tenants AS nfvo_tenant ' - ' ON association.nfvo_tenant_id=nfvo_tenant.uuid ' -) - -_PORT_MAPPING = 'wim_port_mappings as wim_port_mapping ' - -_PORT_MAPPING_JOIN_WIM = ( - ' JOIN wims as wim ' - ' ON wim_port_mapping.wim_id=wim.uuid ' -) - -_PORT_MAPPING_JOIN_DATACENTER = ( - ' JOIN datacenters as datacenter ' - ' ON wim_port_mapping.datacenter_id=datacenter.uuid ' -) - -_WIM_SELECT = [ - 'wim.{0} as {0}'.format(_field) - for _field in 'uuid name description wim_url type config ' - 'created_at modified_at'.split() -] - -_WIM_ACCOUNT_SELECT = 'uuid name user password config'.split() - -_PORT_MAPPING_SELECT = ('wim_port_mapping.*', ) - -_CONFIDENTIAL_FIELDS = ('password', 'passwd') - -_SERIALIZED_FIELDS = ('config', 'vim_info', 'wim_info', 'conn_info', 'extra', - 'service_mapping_info') - -UNIQUE_PORT_MAPPING_INFO_FIELDS = { - 'dpid-port': ('switch_dpid', 'switch_port') -} -"""Fields that should be unique for each port mapping that relies on -service_mapping_info. - -For example, for port mappings of type 'dpid-port', each combination of -switch_dpid and switch_port should be unique (the same switch cannot -be connected to two different places using the same port) -""" - - -class WimPersistence(object): - """High level interactions with the WIM tables in the database""" - - def __init__(self, db, logger=None): - self.db = db - self.logger = logger or logging.getLogger('openmano.wim.persistence') - - def query(self, - FROM=None, - SELECT=None, - WHERE=None, - ORDER_BY=None, - LIMIT=None, - OFFSET=None, - error_if_none=True, - error_if_multiple=False, - postprocess=None, - hide=_CONFIDENTIAL_FIELDS, - **kwargs): - """Retrieve records from the database. - - Keyword Arguments: - SELECT, FROM, WHERE, LIMIT, ORDER_BY: used to compose the SQL - query. See ``nfvo_db.get_rows``. - OFFSET: only valid when used togheter with LIMIT. - Ignore the OFFSET first results of the query. - error_if_none: by default an error is raised if no record is - found. With this option it is possible to disable this error. - error_if_multiple: by default no error is raised if more then one - record is found. - With this option it is possible to enable this error. - postprocess: function applied to every retrieved record. - This function receives a dict as input and must return it - after modifications. Moreover this function should accept a - second optional parameter ``hide`` indicating - the confidential fiels to be obfuscated. - By default a minimal postprocessing function is applied, - obfuscating confidential fields and converting timestamps. - hide: option proxied to postprocess - - All the remaining keyword arguments will be assumed to be ``name``s or - ``uuid``s to compose the WHERE statement, according to their format. - If the value corresponds to an array, the first element will determine - if it is an name or UUID. - - For example: - - ``wim="abcdef"``` will be turned into ``wim.name="abcdef"``, - - ``datacenter="5286a274-8a1b-4b8d-a667-9c94261ad855"`` - will be turned into - ``datacenter.uuid="5286a274-8a1b-4b8d-a667-9c94261ad855"``. - - ``wim=["5286a274-8a1b-4b8d-a667-9c94261ad855", ...]`` - will be turned into - ``wim.uuid=["5286a274-8a1b-4b8d-a667-9c94261ad855", ...]`` - - Raises: - NoRecordFound: if the query result set is empty - DbBaseException: errors occuring during the execution of the query. - """ - # Defaults: - postprocess = postprocess or _postprocess_record - WHERE = WHERE or {} - - # Find remaining keywords by name or uuid - WHERE.update(_compose_where_from_uuids_or_names(**kwargs)) - WHERE = WHERE or None - # ^ If the where statement is empty, it is better to leave it as None, - # so it can be filtered out at a later stage - LIMIT = ('{:d},{:d}'.format(OFFSET, LIMIT) - if LIMIT and OFFSET else LIMIT) - - query = remove_none_items({ - 'SELECT': SELECT, 'FROM': FROM, 'WHERE': WHERE, - 'LIMIT': LIMIT, 'ORDER_BY': ORDER_BY}) - - records = self.db.get_rows(**query) - - table = FROM.split()[0] - if error_if_none and not records: - raise NoRecordFound(WHERE, table) - - if error_if_multiple and len(records) > 1: - self.logger.error('Multiple records ' - 'FROM %s WHERE %s:\n\n%s\n\n', - FROM, WHERE, json.dumps(records, indent=4)) - raise MultipleRecordsFound(WHERE, table) - - return [ - expand_joined_fields(postprocess(record, hide)) - for record in records - ] - - def query_one(self, *args, **kwargs): - """Similar to ``query``, but ensuring just one result. - ``error_if_multiple`` is enabled by default. - """ - kwargs.setdefault('error_if_multiple', True) - records = self.query(*args, **kwargs) - return records[0] if records else None - - def get_by_uuid(self, table, uuid, **kwargs): - """Retrieve one record from the database based on its uuid - - Arguments: - table (str): table name (to be used in SQL's FROM statement). - uuid (str): unique identifier for record. - - For additional keyword arguments and exceptions see :obj:`~.query` - (``error_if_multiple`` is enabled by default). - """ - if uuid is None: - raise UndefinedUuidOrName(table) - return self.query_one(table, WHERE={'uuid': uuid}, **kwargs) - - def get_by_name_or_uuid(self, table, uuid_or_name, **kwargs): - """Retrieve a record from the database based on a value that can be its - uuid or name. - - Arguments: - table (str): table name (to be used in SQL's FROM statement). - uuid_or_name (str): this value can correspond to either uuid or - name - For additional keyword arguments and exceptions see :obj:`~.query` - (``error_if_multiple`` is enabled by default). - """ - if uuid_or_name is None: - raise UndefinedUuidOrName(table) - - key = 'uuid' if check_valid_uuid(uuid_or_name) else 'name' - return self.query_one(table, WHERE={key: uuid_or_name}, **kwargs) - - def get_wims(self, uuid_or_name=None, tenant=None, **kwargs): - """Retrieve information about one or more WIMs stored in the database - - Arguments: - uuid_or_name (str): uuid or name for WIM - tenant (str): [optional] uuid or name for NFVO tenant - - See :obj:`~.query` for additional keyword arguments. - """ - kwargs.update(wim=uuid_or_name, tenant=tenant) - from_ = _WIM_JOIN if tenant else _WIM - select_ = _WIM_SELECT[:] + (['wim_account.*'] if tenant else []) - - kwargs.setdefault('SELECT', select_) - return self.query(from_, **kwargs) - - def get_wim(self, wim, tenant=None, **kwargs): - """Similar to ``get_wims`` but ensure only one result is returned""" - kwargs.setdefault('error_if_multiple', True) - return self.get_wims(wim, tenant)[0] - - def create_wim(self, wim_descriptor): - """Create a new wim record inside the database and returns its uuid - - Arguments: - wim_descriptor (dict): properties of the record - (usually each field corresponds to a database column, but extra - information can be offloaded to another table or serialized as - JSON/YAML) - Returns: - str: UUID of the created WIM - """ - if "config" in wim_descriptor: - wim_descriptor["config"] = _serialize(wim_descriptor["config"]) - - url = wim_descriptor["wim_url"] - wim_descriptor["wim_url"] = url.strip(string.whitespace + "/") - # ^ This avoid the common problem caused by trailing spaces/slashes in - # the URL (due to CTRL+C/CTRL+V) - - return self.db.new_row( - "wims", wim_descriptor, add_uuid=True, confidential_data=True) - - def update_wim(self, uuid_or_name, wim_descriptor): - """Change an existing WIM record on the database""" - # obtain data, check that only one exist - wim = self.get_by_name_or_uuid('wims', uuid_or_name) - - # edit data - wim_id = wim['uuid'] - where = {'uuid': wim['uuid']} - - # unserialize config, edit and serialize it again - new_config_dict = wim_descriptor.get('config', {}) or {} - config_dict = remove_none_items(merge_dicts( - wim.get('config', {}) or {}, new_config_dict)) - wim_descriptor['config'] = ( - _serialize(config_dict) if config_dict else None) - - self.db.update_rows('wims', wim_descriptor, where) - - return wim_id - - def delete_wim(self, wim): - # get nfvo_tenant info - wim = self.get_by_name_or_uuid('wims', wim) - - self.db.delete_row_by_id('wims', wim['uuid']) - - return wim['uuid'] + ' ' + wim['name'] - - def get_wim_accounts_by(self, wim=None, tenant=None, uuid=None, **kwargs): - """Retrieve WIM account information from the database together - with the related records (wim, nfvo_tenant and wim_nfvo_tenant) - - Arguments: - wim (str): uuid or name for WIM - tenant (str): [optional] uuid or name for NFVO tenant - - See :obj:`~.query` for additional keyword arguments. - """ - kwargs.update(wim=wim, tenant=tenant) - kwargs.setdefault('postprocess', _postprocess_wim_account) - if uuid: - kwargs.setdefault('WHERE', {'wim_account.uuid': uuid}) - return self.query(FROM=_WIM_ACCOUNT_JOIN, **kwargs) - - def get_wim_account_by(self, wim=None, tenant=None, uuid=None, **kwargs): - """Similar to ``get_wim_accounts_by``, but ensuring just one result""" - kwargs.setdefault('error_if_multiple', True) - return self.get_wim_accounts_by(wim, tenant, uuid, **kwargs)[0] - - def get_wim_accounts(self, **kwargs): - """Retrieve all the accounts from the database""" - kwargs.setdefault('postprocess', _postprocess_wim_account) - kwargs.setdefault('WHERE', {"sdn": "false"}) - return self.query(FROM=_WIM_ACCOUNT_JOIN, **kwargs) - - def get_wim_account(self, uuid_or_name, **kwargs): - """Retrieve WIM Account record by UUID or name, - See :obj:`get_by_name_or_uuid` for keyword arguments. - """ - kwargs.setdefault('postprocess', _postprocess_wim_account) - kwargs.setdefault('SELECT', _WIM_ACCOUNT_SELECT) - return self.get_by_name_or_uuid('wim_accounts', uuid_or_name, **kwargs) - - @contextmanager - def _associate(self, wim_id, nfvo_tenant_id): - """Auxiliary method for ``create_wim_account`` - - This method just create a row in the association table - ``wim_nfvo_tenants`` - """ - try: - yield - except DbBaseException as db_exception: - error_msg = str(db_exception) - if all([msg in error_msg - for msg in ("already in use", "'wim_nfvo_tenant'")]): - ex = WimAndTenantAlreadyAttached(wim_id, nfvo_tenant_id) - raise ex from db_exception - raise - - def create_wim_account(self, wim, tenant, properties): - """Associate a wim to a tenant using the ``wim_nfvo_tenants`` table - and create a ``wim_account`` to store credentials and configurations. - - For the sake of simplification, we assume that each NFVO tenant can be - attached to a WIM using only one WIM account. This is automatically - guaranteed via database constraints. - For corner cases, the same WIM can be registered twice using another - name. - - Arguments: - wim (str): name or uuid of the WIM related to the account being - created - tenant (str): name or uuid of the nfvo tenant to which the account - will be created - properties (dict): properties of the account - (eg. user, password, ...) - """ - wim_id = self.get_by_name_or_uuid('wims', wim, SELECT=['uuid'])['uuid'] - tenant = self.get_by_name_or_uuid('nfvo_tenants', tenant, - SELECT=['uuid', 'name']) - account = properties.setdefault('name', tenant['name']) - - wim_account = self.query_one('wim_accounts', - WHERE={'wim_id': wim_id, 'name': account}, - error_if_none=False) - - transaction = [] - used_uuids = [] - - if wim_account is None: - # If a row for the wim account doesn't exist yet, we need to - # create one, otherwise we can just re-use it. - account_id = str(generate_uuid()) - used_uuids.append(account_id) - row = merge_dicts(properties, wim_id=wim_id, uuid=account_id) - transaction.append({'wim_accounts': _preprocess_wim_account(row)}) - else: - account_id = wim_account['uuid'] - properties.pop('config', None) # Config is too complex to compare - diff = {k: v for k, v in properties.items() if v != wim_account[k]} - if diff: - tip = 'Edit the account first, and then attach it to a tenant' - raise WimAccountOverwrite(wim_account, diff, tip) - - transaction.append({ - 'wim_nfvo_tenants': {'nfvo_tenant_id': tenant['uuid'], - 'wim_id': wim_id, - 'wim_account_id': account_id}}) - - with self._associate(wim_id, tenant['uuid']): - self.db.new_rows(transaction, used_uuids, confidential_data=True) - - return account_id - - def update_wim_account(self, uuid, properties, hide=_CONFIDENTIAL_FIELDS): - """Update WIM account record by overwriting fields with new values - - Specially for the field ``config`` this means that a new dict will be - merged to the existing one. - - Attributes: - uuid (str): UUID for the WIM account - properties (dict): fields that should be overwritten - - Returns: - Updated wim_account - """ - wim_account = self.get_by_uuid('wim_accounts', uuid) - safe_fields = 'user password name created'.split() - updates = _preprocess_wim_account( - merge_dicts(wim_account, filter_dict_keys(properties, safe_fields)) - ) - - if properties.get('config'): - old_config = wim_account.get('config') or {} - new_config = merge_dicts(old_config, properties['config']) - updates['config'] = _serialize(new_config) - - num_changes = self.db.update_rows('wim_accounts', UPDATE=updates, - WHERE={'uuid': wim_account['uuid']}) - - if num_changes is None: - raise UnexpectedDatabaseError('Impossible to update wim_account ' - '{name}:{uuid}'.format(*wim_account)) - - return self.get_wim_account(wim_account['uuid'], hide=hide) - - def delete_wim_account(self, uuid): - """Remove WIM account record from the database""" - # Since we have foreign keys configured with ON CASCADE, we can rely - # on the database engine to guarantee consistency, deleting the - # dependant records - return self.db.delete_row_by_id('wim_accounts', uuid) - - def get_datacenters_by(self, datacenter=None, tenant=None, **kwargs): - """Retrieve datacenter information from the database together - with the related records (nfvo_tenant) - - Arguments: - datacenter (str): uuid or name for datacenter - tenant (str): [optional] uuid or name for NFVO tenant - - See :obj:`~.query` for additional keyword arguments. - """ - if tenant: - kwargs.update(datacenter=datacenter, tenant=tenant) - return self.query(_DATACENTER_JOIN, **kwargs) - else: - return [self.get_by_name_or_uuid('datacenters', - datacenter, **kwargs)] - - def get_datacenter_by(self, datacenter=None, tenant=None, **kwargs): - """Similar to ``get_datacenters_by``, but ensuring just one result""" - kwargs.setdefault('error_if_multiple', True) - return self.get_datacenters_by(datacenter, tenant, **kwargs)[0] - - def _create_single_port_mapping(self, properties): - info = properties.setdefault('service_mapping_info', {}) - endpoint_id = properties.get('service_endpoint_id') - - if info.get('mapping_type') and not endpoint_id: - properties['service_endpoint_id'] = ( - self._generate_port_mapping_id(info)) - - properties['service_mapping_info'] = _serialize(info) - - try: - self.db.new_row('wim_port_mappings', properties, - add_uuid=False, confidential_data=True) - except DbBaseException as old_exception: - self.logger.exception(old_exception) - ex = InvalidParameters( - "The mapping must contain the " - "'device_id', 'device_interface_id', and " - "service_mapping_info: " - "('switch_dpid' and 'switch_port') or " - "'service_endpoint_id}'") - raise ex from old_exception - - return properties - - def create_wim_port_mappings(self, wim, port_mappings, tenant=None): - if not isinstance(wim, dict): - wim = self.get_by_name_or_uuid('wims', wim) - - for port_mapping in port_mappings: - port_mapping['wim_name'] = wim['name'] - datacenter = self.get_datacenter_by( - port_mapping['datacenter_name'], tenant) - for pop_wan_port_mapping in port_mapping['pop_wan_mappings']: - element = merge_dicts(pop_wan_port_mapping, { - 'wim_id': wim['uuid'], - 'datacenter_id': datacenter['uuid']}) - self._create_single_port_mapping(element) - - return port_mappings - - def _filter_port_mappings_by_tenant(self, mappings, tenant): - """Make sure all the datacenters and wims listed in the port mapping - belong to an specific tenant - """ - - # NOTE: Theoretically this could be done at SQL level, but given the - # number of tables involved (wim_port_mappings, wim_accounts, - # wims, wim_nfvo_tenants, datacenters, datacenter_tenants, - # tenants_datacents and nfvo_tenants), it would result in a - # extremely complex query. Moreover, the predicate can vary: - # for `get_wim_port_mappings` we can have any combination of - # (wim, datacenter, tenant), not all of them having the 3 values - # so we have combinatorial trouble to write the 'FROM' statement. - - kwargs = {'tenant': tenant, 'error_if_none': False} - # Cache results to speedup things - datacenters = {} - wims = {} - - def _get_datacenter(uuid): - return ( - datacenters.get(uuid) or - datacenters.setdefault( - uuid, self.get_datacenters_by(uuid, **kwargs))) - - def _get_wims(uuid): - return (wims.get(uuid) or - wims.setdefault(uuid, self.get_wims(uuid, **kwargs))) - - return [ - mapping - for mapping in mappings - if (_get_datacenter(mapping['datacenter_id']) and - _get_wims(mapping['wim_id'])) - ] - - def get_wim_port_mappings(self, wim=None, datacenter=None, tenant=None, - **kwargs): - """List all the port mappings, optionally filtering by wim, datacenter - AND/OR tenant - """ - from_ = [_PORT_MAPPING, - _PORT_MAPPING_JOIN_WIM if wim else '', - _PORT_MAPPING_JOIN_DATACENTER if datacenter else ''] - - criteria = ('wim_id', 'datacenter_id') - kwargs.setdefault('error_if_none', False) - mappings = self.query( - ' '.join(from_), - SELECT=_PORT_MAPPING_SELECT, - ORDER_BY=['wim_port_mapping.{}'.format(c) for c in criteria], - wim=wim, datacenter=datacenter, - postprocess=_postprocess_wim_port_mapping, - **kwargs) - - if tenant: - mappings = self._filter_port_mappings_by_tenant(mappings, tenant) - - # We don't have to sort, since we have used 'ORDER_BY' - grouped_mappings = groupby(mappings, key=itemgetter(*criteria)) - - return [ - {'wim_id': key[0], - 'datacenter_id': key[1], - 'pop_wan_mappings': [ - filter_out_dict_keys(mapping, ( - 'id', 'wim_id', 'datacenter_id', - 'created_at', 'modified_at')) - for mapping in group]} - for key, group in grouped_mappings - ] - - def delete_wim_port_mappings(self, wim_id): - self.db.delete_row(FROM='wim_port_mappings', WHERE={"wim_id": wim_id}) - return "port mapping for wim {} deleted.".format(wim_id) - - def update_wim_port_mapping(self, id, properties): - original = self.query_one('wim_port_mappings', WHERE={'id': id}) - - mapping_info = remove_none_items(merge_dicts( - original.get('service_mapping_info') or {}, - properties.get('service_mapping_info') or {})) - - updates = preprocess_record( - merge_dicts(original, remove_none_items(properties), - service_mapping_info=mapping_info)) - - num_changes = self.db.update_rows('wim_port_mappings', - UPDATE=updates, WHERE={'id': id}) - - if num_changes is None: - raise UnexpectedDatabaseError( - 'Impossible to update wim_port_mappings {}:\n{}\n'.format( - id, _serialize(properties)) - ) - - return num_changes - - def get_actions_in_groups(self, wim_account_id, - item_types=('instance_wim_nets',), - group_offset=0, group_limit=150): - """Retrieve actions from the database in groups. - Each group contains all the actions that have the same ``item`` type - and ``item_id``. - - Arguments: - wim_account_id: restrict the search to actions to be performed - using the same account - item_types (list): [optional] filter the actions to the given - item types - group_limit (int): maximum number of groups returned by the - function - group_offset (int): skip the N first groups. Used together with - group_limit for pagination purposes. - - Returns: - List of groups, where each group is a tuple ``(key, actions)``. - In turn, ``key`` is a tuple containing the values of - ``(item, item_id)`` used to create the group and ``actions`` is a - list of ``vim_wim_actions`` records (dicts). - """ - - type_options = set( - '"{}"'.format(self.db.escape_string(t)) for t in item_types) - - items = ('SELECT DISTINCT a.item, a.item_id, a.wim_account_id ' - 'FROM vim_wim_actions AS a ' - 'WHERE a.wim_account_id="{}" AND a.item IN ({}) ' - 'ORDER BY a.item, a.item_id ' - 'LIMIT {:d},{:d}').format( - self.safe_str(wim_account_id), - ','.join(type_options), - group_offset, group_limit) - - join = 'vim_wim_actions NATURAL JOIN ({}) AS items'.format(items) - db_results = self.db.get_rows( - FROM=join, ORDER_BY=('item', 'item_id', 'created_at')) - - results = (_postprocess_action(r) for r in db_results) - criteria = itemgetter('item', 'item_id') - return [(k, list(g)) for k, g in groupby(results, key=criteria)] - - def update_action(self, instance_action_id, task_index, properties): - condition = {'instance_action_id': instance_action_id, - 'task_index': task_index} - try: - action = self.query_one('vim_wim_actions', WHERE=condition) - except Exception: - actions = self.query('vim_wim_actions', WHERE=condition) - self.logger.error('More then one action found:\n%s', - json.dumps(actions, indent=4)) - action = actions[0] - - extra = remove_none_items(merge_dicts( - action.get('extra') or {}, - properties.get('extra') or {})) - - updates = preprocess_record( - merge_dicts(action, properties, extra=extra)) - - num_changes = self.db.update_rows('vim_wim_actions', UPDATE=updates, WHERE=condition) - - if num_changes is None: - raise UnexpectedDatabaseError( - 'Impossible to update vim_wim_actions ' - '{instance_action_id}[{task_index}]'.format(*action)) - - return num_changes - - def get_wan_links(self, uuid=None, **kwargs): - """Retrieve WAN link records from the database - - Keyword Arguments: - uuid, instance_scenario_id, sce_net_id, wim_id, wim_account_id: - attributes that can be used at the WHERE clause - """ - kwargs.setdefault('uuid', uuid) - kwargs.setdefault('error_if_none', False) - - criteria_fields = ('uuid', 'instance_scenario_id', 'sce_net_id', - 'wim_id', 'wim_account_id', 'sdn') - criteria = remove_none_items(filter_dict_keys(kwargs, criteria_fields)) - kwargs = filter_out_dict_keys(kwargs, criteria_fields) - - return self.query('instance_wim_nets', WHERE=criteria, **kwargs) - - def update_wan_link(self, uuid, properties): - wan_link = self.get_by_uuid('instance_wim_nets', uuid) - - wim_info = remove_none_items(merge_dicts( - wan_link.get('wim_info') or {}, - properties.get('wim_info') or {})) - - updates = preprocess_record( - merge_dicts(wan_link, properties, wim_info=wim_info)) - - self.logger.debug({'UPDATE': updates}) - num_changes = self.db.update_rows( - 'instance_wim_nets', UPDATE=updates, - WHERE={'uuid': wan_link['uuid']}) - - if num_changes is None: - raise UnexpectedDatabaseError( - 'Impossible to update instance_wim_nets ' + wan_link['uuid']) - - return num_changes - - def get_instance_nets(self, instance_scenario_id, sce_net_id, **kwargs): - """Retrieve all the instance nets related to the same instance_scenario - and scenario network - """ - return self.query( - 'instance_nets', - WHERE={'instance_scenario_id': instance_scenario_id, - 'sce_net_id': sce_net_id}, - ORDER_BY=kwargs.pop( - 'ORDER_BY', ('instance_scenario_id', 'sce_net_id')), - **kwargs) - - def update_instance_action_counters(self, uuid, failed=None, done=None): - """Atomically increment/decrement number_done and number_failed fields - in the instance action table - """ - changes = remove_none_items({ - 'number_failed': failed and {'INCREMENT': failed}, - 'number_done': done and {'INCREMENT': done} - }) - - if not changes: - return 0 - - return self.db.update_rows('instance_actions', WHERE={'uuid': uuid}, UPDATE=changes) - - def get_only_vm_with_external_net(self, instance_net_id, **kwargs): - """Return an instance VM if that is the only VM connected to an - external network identified by instance_net_id - """ - counting = ('SELECT DISTINCT instance_net_id ' - 'FROM instance_interfaces ' - 'WHERE instance_net_id="{}" AND type="external" ' - 'GROUP BY instance_net_id ' - 'HAVING COUNT(*)=1').format(self.safe_str(instance_net_id)) - - vm_item = ('SELECT DISTINCT instance_vm_id ' - 'FROM instance_interfaces NATURAL JOIN ({}) AS a' - .format(counting)) - - return self.query_one( - 'instance_vms JOIN ({}) as instance_interface ' - 'ON instance_vms.uuid=instance_interface.instance_vm_id' - .format(vm_item), **kwargs) - - def safe_str(self, string): - """Return a SQL safe string""" - return self.db.escape_string(string) - - def reconnect(self): - self.db.reconnect() - - def _generate_port_mapping_id(self, mapping_info): - """Given a port mapping represented by a dict with a 'type' field, - generate a unique string, in a injective way. - """ - mapping_info = mapping_info.copy() # Avoid mutating original object - mapping_type = mapping_info.pop('mapping_type', None) - if not mapping_type: - raise UndefinedWanMappingType(mapping_info) - - unique_fields = UNIQUE_PORT_MAPPING_INFO_FIELDS.get(mapping_type) - - if unique_fields: - mapping_info = filter_dict_keys(mapping_info, unique_fields) - else: - self.logger.warning('Unique fields for WIM port mapping of type ' - '%s not defined. Please add a list of fields ' - 'which combination should be unique in ' - 'UNIQUE_PORT_MAPPING_INFO_FIELDS ' - '(`wim/persistency.py) ', mapping_type) - - repeatable_repr = json.dumps(mapping_info, encoding='utf-8', - sort_keys=True, indent=False) - - return ':'.join([mapping_type, _str2id(repeatable_repr)]) - - -def _serialize(value): - """Serialize an arbitrary value in a consistent way, - so it can be stored in a database inside a text field - """ - return yaml.safe_dump(value, default_flow_style=True, width=256) - - -def _unserialize(text): - """Unserialize text representation into an arbitrary value, - so it can be loaded from the database - """ - return yaml.safe_load(text) - - -def preprocess_record(record): - """Small transformations to be applied to the data that cames from the - user before writing it to the database. By default, filter out timestamps, - and serialize the ``config`` field. - """ - automatic_fields = ['created_at', 'modified_at'] - record = serialize_fields(filter_out_dict_keys(record, automatic_fields)) - - return record - - -def _preprocess_wim_account(wim_account): - """Do the default preprocessing and convert the 'created' field from - boolean to string - """ - wim_account = preprocess_record(wim_account) - - wim_account['sdn'] = False - return wim_account - - -def _postprocess_record(record, hide=_CONFIDENTIAL_FIELDS): - """By default, hide passwords fields, unserialize ``config`` fields, and - convert float timestamps to strings - """ - record = hide_confidential_fields(record, hide) - record = unserialize_fields(record, hide) - - convert_float_timestamp2str(record) - - return record - - -def _postprocess_action(action): - if action.get('extra'): - action['extra'] = _unserialize(action['extra']) - - return action - - -def _postprocess_wim_account(wim_account, hide=_CONFIDENTIAL_FIELDS): - """Do the default postprocessing and convert the 'created' field from - string to boolean - """ - # Fix fields from join - for field in ('type', 'description', 'wim_url'): - if field in wim_account: - wim_account['wim.'+field] = wim_account.pop(field) - - for field in ('id', 'nfvo_tenant_id', 'wim_account_id'): - if field in wim_account: - wim_account['association.'+field] = wim_account.pop(field) - - wim_account = _postprocess_record(wim_account, hide) - - created = wim_account.get('created') - wim_account['created'] = (created is True or created == 'true') - - return wim_account - - -def _postprocess_wim_port_mapping(mapping, hide=_CONFIDENTIAL_FIELDS): - mapping = _postprocess_record(mapping, hide=hide) - mapping_info = mapping.get('service_mapping_info', None) or {} - mapping['service_mapping_info'] = mapping_info - return mapping - - -def hide_confidential_fields(record, fields=_CONFIDENTIAL_FIELDS): - """Obfuscate confidential fields from the input dict. - - Note: - This function performs a SHALLOW operation. - """ - if not(isinstance(record, dict) and fields): - return record - - keys = list(record.keys()) - keys = (k for k in keys for f in fields if k == f or k.endswith('.'+f)) - - return merge_dicts(record, {k: '********' for k in keys if record[k]}) - - -def unserialize_fields(record, hide=_CONFIDENTIAL_FIELDS, - fields=_SERIALIZED_FIELDS): - """Unserialize fields that where stored in the database as a serialized - YAML (or JSON) - """ - keys = list(record.keys()) - keys = (k for k in keys for f in fields if k == f or k.endswith('.'+f)) - - return merge_dicts(record, { - key: hide_confidential_fields(_unserialize(record[key]), hide) - for key in keys if record[key] - }) - - -def serialize_fields(record, fields=_SERIALIZED_FIELDS): - """Serialize fields to be stored in the database as YAML""" - keys = list(record.keys()) - keys = (k for k in keys for f in fields if k == f or k.endswith('.'+f)) - - return merge_dicts(record, { - key: _serialize(record[key]) - for key in keys if record[key] is not None - }) - - -def _decide_name_or_uuid(value): - reference = value - - if isinstance(value, (list, tuple)): - reference = value[0] if value else '' - - return 'uuid' if check_valid_uuid(reference) else 'name' - - -def _compose_where_from_uuids_or_names(**conditions): - """Create a dict containing the right conditions to be used in a database - query. - - This function chooses between ``names`` and ``uuid`` fields based on the - format of the passed string. - If a list is passed, the first element of the list will be used to choose - the name of the field. - If a ``None`` value is passed, ``uuid`` is used. - - Note that this function automatically translates ``tenant`` to - ``nfvo_tenant`` for the sake of brevity. - - Example: - >>> _compose_where_from_uuids_or_names( - wim='abcdef', - tenant=['xyz123', 'def456'] - datacenter='5286a274-8a1b-4b8d-a667-9c94261ad855') - {'wim.name': 'abcdef', - 'nfvo_tenant.name': ['xyz123', 'def456'] - 'datacenter.uuid': '5286a274-8a1b-4b8d-a667-9c94261ad855'} - """ - if 'tenant' in conditions: - conditions['nfvo_tenant'] = conditions.pop('tenant') - - return { - '{}.{}'.format(kind, _decide_name_or_uuid(value)): value - for kind, value in conditions.items() if value - } - - -def _str2id(text): - """Create an ID (following the UUID format) from a piece of arbitrary - text. - - Different texts should generate different IDs, and the same text should - generate the same ID in a repeatable way. - """ - return sha1(text).hexdigest()