1 # -*- coding: utf-8 -*-
3 # Copyright 2018 University of Bristol - High Performance Networks Research
7 # Contributors: Anderson Bravalheri, Dimitrios Gkounis, Abubakar Siddique
8 # Muqaddas, Navdeep Uniyal, Reza Nejabati and Dimitra Simeonidou
10 # Licensed under the Apache License, Version 2.0 (the "License"); you may
11 # not use this file except in compliance with the License. You may obtain
12 # a copy of the License at
14 # http://www.apache.org/licenses/LICENSE-2.0
16 # Unless required by applicable law or agreed to in writing, software
17 # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
18 # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
19 # License for the specific language governing permissions and limitations
22 # For those usages not covered by the Apache License, Version 2.0 please
23 # contact with: <highperformance-networks@bristol.ac.uk>
25 # Neither the name of the University of Bristol nor the names of its
26 # contributors may be used to endorse or promote products derived from
27 # this software without specific prior written permission.
29 # This work has been performed in the context of DCMS UK 5G Testbeds
30 # & Trials Programme and in the framework of the Metro-Haul project -
31 # funded by the European Commission under Grant number 761727 through the
32 # Horizon 2020 and 5G-PPP programmes.
35 """This module contains only logic related to managing records in a database
36 which includes data format normalization, data format validation and etc.
37 (It works as an extension to `nfvo_db.py` for the WIM feature)
39 No domain logic/architectural concern should be present in this file.
43 from contextlib
import contextmanager
44 from hashlib
import sha1
45 from itertools
import groupby
46 from operator
import itemgetter
47 # from sys import exc_info
48 # from time import time
49 from uuid
import uuid1
as generate_uuid
55 convert_float_timestamp2str
,
68 UndefinedWanMappingType
,
69 UnexpectedDatabaseError
,
71 WimAndTenantAlreadyAttached
78 ' JOIN wim_nfvo_tenants AS association '
79 ' ON association.wim_id=wim.uuid '
80 ' JOIN nfvo_tenants AS nfvo_tenant '
81 ' ON association.nfvo_tenant_id=nfvo_tenant.uuid '
82 ' JOIN wim_accounts AS wim_account '
83 ' ON association.wim_account_id=wim_account.uuid '
87 'wim_accounts AS wim_account '
88 ' JOIN wim_nfvo_tenants AS association '
89 ' ON association.wim_account_id=wim_account.uuid '
91 ' ON association.wim_id=wim.uuid '
92 ' JOIN nfvo_tenants AS nfvo_tenant '
93 ' ON association.nfvo_tenant_id=nfvo_tenant.uuid '
97 'datacenters AS datacenter '
98 ' JOIN tenants_datacenters AS association '
99 ' ON association.datacenter_id=datacenter.uuid '
100 ' JOIN datacenter_tenants as datacenter_account '
101 ' ON association.datacenter_tenant_id=datacenter_account.uuid '
102 ' JOIN nfvo_tenants AS nfvo_tenant '
103 ' ON association.nfvo_tenant_id=nfvo_tenant.uuid '
106 _PORT_MAPPING
= 'wim_port_mappings as wim_port_mapping '
108 _PORT_MAPPING_JOIN_WIM
= (
110 ' ON wim_port_mapping.wim_id=wim.uuid '
113 _PORT_MAPPING_JOIN_DATACENTER
= (
114 ' JOIN datacenters as datacenter '
115 ' ON wim_port_mapping.datacenter_id=datacenter.uuid '
119 'wim.{0} as {0}'.format(_field
)
120 for _field
in 'uuid name description wim_url type config '
121 'created_at modified_at'.split()
124 _WIM_ACCOUNT_SELECT
= 'uuid name user password config'.split()
126 _PORT_MAPPING_SELECT
= ('wim_port_mapping.*', )
128 _CONFIDENTIAL_FIELDS
= ('password', 'passwd')
130 _SERIALIZED_FIELDS
= ('config', 'vim_info', 'wim_info', 'conn_info', 'extra',
131 'wan_service_mapping_info')
133 UNIQUE_PORT_MAPPING_INFO_FIELDS
= {
134 'dpid-port': ('wan_switch_dpid', 'wan_switch_port')
136 """Fields that should be unique for each port mapping that relies on
137 wan_service_mapping_info.
139 For example, for port mappings of type 'dpid-port', each combination of
140 wan_switch_dpid and wan_switch_port should be unique (the same switch cannot
141 be connected to two different places using the same port)
145 class WimPersistence(object):
146 """High level interactions with the WIM tables in the database"""
148 def __init__(self
, db
, logger
=None):
150 self
.logger
= logger
or logging
.getLogger('openmano.wim.persistence')
160 error_if_multiple
=False,
162 hide
=_CONFIDENTIAL_FIELDS
,
164 """Retrieve records from the database.
167 SELECT, FROM, WHERE, LIMIT, ORDER_BY: used to compose the SQL
168 query. See ``nfvo_db.get_rows``.
169 OFFSET: only valid when used togheter with LIMIT.
170 Ignore the OFFSET first results of the query.
171 error_if_none: by default an error is raised if no record is
172 found. With this option it is possible to disable this error.
173 error_if_multiple: by default no error is raised if more then one
175 With this option it is possible to enable this error.
176 postprocess: function applied to every retrieved record.
177 This function receives a dict as input and must return it
178 after modifications. Moreover this function should accept a
179 second optional parameter ``hide`` indicating
180 the confidential fiels to be obfuscated.
181 By default a minimal postprocessing function is applied,
182 obfuscating confidential fields and converting timestamps.
183 hide: option proxied to postprocess
185 All the remaining keyword arguments will be assumed to be ``name``s or
186 ``uuid``s to compose the WHERE statement, according to their format.
187 If the value corresponds to an array, the first element will determine
188 if it is an name or UUID.
191 - ``wim="abcdef"``` will be turned into ``wim.name="abcdef"``,
192 - ``datacenter="5286a274-8a1b-4b8d-a667-9c94261ad855"``
194 ``datacenter.uuid="5286a274-8a1b-4b8d-a667-9c94261ad855"``.
195 - ``wim=["5286a274-8a1b-4b8d-a667-9c94261ad855", ...]``
197 ``wim.uuid=["5286a274-8a1b-4b8d-a667-9c94261ad855", ...]``
200 NoRecordFound: if the query result set is empty
201 DbBaseException: errors occuring during the execution of the query.
204 postprocess
= postprocess
or _postprocess_record
207 # Find remaining keywords by name or uuid
208 WHERE
.update(_compose_where_from_uuids_or_names(**kwargs
))
209 WHERE
= WHERE
or None
210 # ^ If the where statement is empty, it is better to leave it as None,
211 # so it can be filtered out at a later stage
212 LIMIT
= ('{:d},{:d}'.format(OFFSET
, LIMIT
)
213 if LIMIT
and OFFSET
else LIMIT
)
215 query
= remove_none_items({
216 'SELECT': SELECT
, 'FROM': FROM
, 'WHERE': WHERE
,
217 'LIMIT': LIMIT
, 'ORDER_BY': ORDER_BY
})
219 records
= self
.db
.get_rows(**query
)
221 table
= FROM
.split()[0]
222 if error_if_none
and not records
:
223 raise NoRecordFound(WHERE
, table
)
225 if error_if_multiple
and len(records
) > 1:
226 self
.logger
.error('Multiple records '
227 'FROM %s WHERE %s:\n\n%s\n\n',
228 FROM
, WHERE
, json
.dumps(records
, indent
=4))
229 raise MultipleRecordsFound(WHERE
, table
)
232 expand_joined_fields(postprocess(record
, hide
))
233 for record
in records
236 def query_one(self
, *args
, **kwargs
):
237 """Similar to ``query``, but ensuring just one result.
238 ``error_if_multiple`` is enabled by default.
240 kwargs
.setdefault('error_if_multiple', True)
241 records
= self
.query(*args
, **kwargs
)
242 return records
[0] if records
else None
244 def get_by_uuid(self
, table
, uuid
, **kwargs
):
245 """Retrieve one record from the database based on its uuid
248 table (str): table name (to be used in SQL's FROM statement).
249 uuid (str): unique identifier for record.
251 For additional keyword arguments and exceptions see :obj:`~.query`
252 (``error_if_multiple`` is enabled by default).
255 raise UndefinedUuidOrName(table
)
256 return self
.query_one(table
, WHERE
={'uuid': uuid
}, **kwargs
)
258 def get_by_name_or_uuid(self
, table
, uuid_or_name
, **kwargs
):
259 """Retrieve a record from the database based on a value that can be its
263 table (str): table name (to be used in SQL's FROM statement).
264 uuid_or_name (str): this value can correspond to either uuid or
266 For additional keyword arguments and exceptions see :obj:`~.query`
267 (``error_if_multiple`` is enabled by default).
269 if uuid_or_name
is None:
270 raise UndefinedUuidOrName(table
)
272 key
= 'uuid' if check_valid_uuid(uuid_or_name
) else 'name'
273 return self
.query_one(table
, WHERE
={key
: uuid_or_name
}, **kwargs
)
275 def get_wims(self
, uuid_or_name
=None, tenant
=None, **kwargs
):
276 """Retrieve information about one or more WIMs stored in the database
279 uuid_or_name (str): uuid or name for WIM
280 tenant (str): [optional] uuid or name for NFVO tenant
282 See :obj:`~.query` for additional keyword arguments.
284 kwargs
.update(wim
=uuid_or_name
, tenant
=tenant
)
285 from_
= _WIM_JOIN
if tenant
else _WIM
286 select_
= _WIM_SELECT
[:] + (['wim_account.*'] if tenant
else [])
288 kwargs
.setdefault('SELECT', select_
)
289 return self
.query(from_
, **kwargs
)
291 def get_wim(self
, wim
, tenant
=None, **kwargs
):
292 """Similar to ``get_wims`` but ensure only one result is returned"""
293 kwargs
.setdefault('error_if_multiple', True)
294 return self
.get_wims(wim
, tenant
)[0]
296 def create_wim(self
, wim_descriptor
):
297 """Create a new wim record inside the database and returns its uuid
300 wim_descriptor (dict): properties of the record
301 (usually each field corresponds to a database column, but extra
302 information can be offloaded to another table or serialized as
305 str: UUID of the created WIM
307 if "config" in wim_descriptor
:
308 wim_descriptor
["config"] = _serialize(wim_descriptor
["config"])
310 return self
.db
.new_row(
311 "wims", wim_descriptor
, add_uuid
=True, confidential_data
=True)
313 def update_wim(self
, uuid_or_name
, wim_descriptor
):
314 """Change an existing WIM record on the database"""
315 # obtain data, check that only one exist
316 wim
= self
.get_by_name_or_uuid('wims', uuid_or_name
)
320 where
= {'uuid': wim
['uuid']}
322 # unserialize config, edit and serialize it again
323 new_config_dict
= wim_descriptor
.get('config', {}) or {}
324 config_dict
= remove_none_items(merge_dicts(
325 wim
.get('config', {}) or {}, new_config_dict
))
326 wim_descriptor
['config'] = (
327 _serialize(config_dict
) if config_dict
else None)
329 self
.db
.update_rows('wims', wim_descriptor
, where
)
333 def delete_wim(self
, wim
):
334 # get nfvo_tenant info
335 wim
= self
.get_by_name_or_uuid('wims', wim
)
337 self
.db
.delete_row_by_id('wims', wim
['uuid'])
339 return wim
['uuid'] + ' ' + wim
['name']
341 def get_wim_accounts_by(self
, wim
=None, tenant
=None, uuid
=None, **kwargs
):
342 """Retrieve WIM account information from the database together
343 with the related records (wim, nfvo_tenant and wim_nfvo_tenant)
346 wim (str): uuid or name for WIM
347 tenant (str): [optional] uuid or name for NFVO tenant
349 See :obj:`~.query` for additional keyword arguments.
351 kwargs
.update(wim
=wim
, tenant
=tenant
)
352 kwargs
.setdefault('postprocess', _postprocess_wim_account
)
354 kwargs
.setdefault('WHERE', {'wim_account.uuid': uuid
})
355 return self
.query(FROM
=_WIM_ACCOUNT_JOIN
, **kwargs
)
357 def get_wim_account_by(self
, wim
=None, tenant
=None, uuid
=None, **kwargs
):
358 """Similar to ``get_wim_accounts_by``, but ensuring just one result"""
359 kwargs
.setdefault('error_if_multiple', True)
360 return self
.get_wim_accounts_by(wim
, tenant
, uuid
, **kwargs
)[0]
362 def get_wim_accounts(self
, **kwargs
):
363 """Retrieve all the accounts from the database"""
364 kwargs
.setdefault('postprocess', _postprocess_wim_account
)
365 kwargs
.setdefault('WHERE', {"sdn": "false"})
366 return self
.query(FROM
=_WIM_ACCOUNT_JOIN
, **kwargs
)
368 def get_wim_account(self
, uuid_or_name
, **kwargs
):
369 """Retrieve WIM Account record by UUID or name,
370 See :obj:`get_by_name_or_uuid` for keyword arguments.
372 kwargs
.setdefault('postprocess', _postprocess_wim_account
)
373 kwargs
.setdefault('SELECT', _WIM_ACCOUNT_SELECT
)
374 return self
.get_by_name_or_uuid('wim_accounts', uuid_or_name
, **kwargs
)
377 def _associate(self
, wim_id
, nfvo_tenant_id
):
378 """Auxiliary method for ``create_wim_account``
380 This method just create a row in the association table
385 except DbBaseException
as db_exception
:
386 error_msg
= str(db_exception
)
387 if all([msg
in error_msg
388 for msg
in ("already in use", "'wim_nfvo_tenant'")]):
389 ex
= WimAndTenantAlreadyAttached(wim_id
, nfvo_tenant_id
)
390 raise ex
from db_exception
393 def create_wim_account(self
, wim
, tenant
, properties
):
394 """Associate a wim to a tenant using the ``wim_nfvo_tenants`` table
395 and create a ``wim_account`` to store credentials and configurations.
397 For the sake of simplification, we assume that each NFVO tenant can be
398 attached to a WIM using only one WIM account. This is automatically
399 guaranteed via database constraints.
400 For corner cases, the same WIM can be registered twice using another
404 wim (str): name or uuid of the WIM related to the account being
406 tenant (str): name or uuid of the nfvo tenant to which the account
408 properties (dict): properties of the account
409 (eg. user, password, ...)
411 wim_id
= self
.get_by_name_or_uuid('wims', wim
, SELECT
=['uuid'])['uuid']
412 tenant
= self
.get_by_name_or_uuid('nfvo_tenants', tenant
,
413 SELECT
=['uuid', 'name'])
414 account
= properties
.setdefault('name', tenant
['name'])
416 wim_account
= self
.query_one('wim_accounts',
417 WHERE
={'wim_id': wim_id
, 'name': account
},
423 if wim_account
is None:
424 # If a row for the wim account doesn't exist yet, we need to
425 # create one, otherwise we can just re-use it.
426 account_id
= str(generate_uuid())
427 used_uuids
.append(account_id
)
428 row
= merge_dicts(properties
, wim_id
=wim_id
, uuid
=account_id
)
429 transaction
.append({'wim_accounts': _preprocess_wim_account(row
)})
431 account_id
= wim_account
['uuid']
432 properties
.pop('config', None) # Config is too complex to compare
433 diff
= {k
: v
for k
, v
in properties
.items() if v
!= wim_account
[k
]}
435 tip
= 'Edit the account first, and then attach it to a tenant'
436 raise WimAccountOverwrite(wim_account
, diff
, tip
)
439 'wim_nfvo_tenants': {'nfvo_tenant_id': tenant
['uuid'],
441 'wim_account_id': account_id
}})
443 with self
._associate
(wim_id
, tenant
['uuid']):
444 self
.db
.new_rows(transaction
, used_uuids
, confidential_data
=True)
448 def update_wim_account(self
, uuid
, properties
, hide
=_CONFIDENTIAL_FIELDS
):
449 """Update WIM account record by overwriting fields with new values
451 Specially for the field ``config`` this means that a new dict will be
452 merged to the existing one.
455 uuid (str): UUID for the WIM account
456 properties (dict): fields that should be overwritten
461 wim_account
= self
.get_by_uuid('wim_accounts', uuid
)
462 safe_fields
= 'user password name created'.split()
463 updates
= _preprocess_wim_account(
464 merge_dicts(wim_account
, filter_dict_keys(properties
, safe_fields
))
467 if properties
.get('config'):
468 old_config
= wim_account
.get('config') or {}
469 new_config
= merge_dicts(old_config
, properties
['config'])
470 updates
['config'] = _serialize(new_config
)
472 num_changes
= self
.db
.update_rows('wim_accounts', UPDATE
=updates
,
473 WHERE
={'uuid': wim_account
['uuid']})
475 if num_changes
is None:
476 raise UnexpectedDatabaseError('Impossible to update wim_account '
477 '{name}:{uuid}'.format(*wim_account
))
479 return self
.get_wim_account(wim_account
['uuid'], hide
=hide
)
481 def delete_wim_account(self
, uuid
):
482 """Remove WIM account record from the database"""
483 # Since we have foreign keys configured with ON CASCADE, we can rely
484 # on the database engine to guarantee consistency, deleting the
486 return self
.db
.delete_row_by_id('wim_accounts', uuid
)
488 def get_datacenters_by(self
, datacenter
=None, tenant
=None, **kwargs
):
489 """Retrieve datacenter information from the database together
490 with the related records (nfvo_tenant)
493 datacenter (str): uuid or name for datacenter
494 tenant (str): [optional] uuid or name for NFVO tenant
496 See :obj:`~.query` for additional keyword arguments.
499 kwargs
.update(datacenter
=datacenter
, tenant
=tenant
)
500 return self
.query(_DATACENTER_JOIN
, **kwargs
)
502 return [self
.get_by_name_or_uuid('datacenters',
503 datacenter
, **kwargs
)]
505 def get_datacenter_by(self
, datacenter
=None, tenant
=None, **kwargs
):
506 """Similar to ``get_datacenters_by``, but ensuring just one result"""
507 kwargs
.setdefault('error_if_multiple', True)
508 return self
.get_datacenters_by(datacenter
, tenant
, **kwargs
)[0]
510 def _create_single_port_mapping(self
, properties
):
511 info
= properties
.setdefault('wan_service_mapping_info', {})
512 endpoint_id
= properties
.get('wan_service_endpoint_id')
514 if info
.get('mapping_type') and not endpoint_id
:
515 properties
['wan_service_endpoint_id'] = (
516 self
._generate
_port
_mapping
_id
(info
))
518 properties
['wan_service_mapping_info'] = _serialize(info
)
521 self
.db
.new_row('wim_port_mappings', properties
,
522 add_uuid
=False, confidential_data
=True)
523 except DbBaseException
as old_exception
:
524 self
.logger
.exception(old_exception
)
525 ex
= InvalidParameters(
526 "The mapping must contain the "
527 "'device_id', 'device_interface_id', and "
528 "wan_service_mapping_info: "
529 "('wan_switch_dpid' and 'wan_switch_port') or "
530 "'wan_service_endpoint_id}'")
531 raise ex
from old_exception
535 def create_wim_port_mappings(self
, wim
, port_mappings
, tenant
=None):
536 if not isinstance(wim
, dict):
537 wim
= self
.get_by_name_or_uuid('wims', wim
)
539 for port_mapping
in port_mappings
:
540 port_mapping
['wim_name'] = wim
['name']
541 datacenter
= self
.get_datacenter_by(
542 port_mapping
['datacenter_name'], tenant
)
543 for pop_wan_port_mapping
in port_mapping
['pop_wan_mappings']:
544 element
= merge_dicts(pop_wan_port_mapping
, {
545 'wim_id': wim
['uuid'],
546 'datacenter_id': datacenter
['uuid']})
547 self
._create
_single
_port
_mapping
(element
)
551 def _filter_port_mappings_by_tenant(self
, mappings
, tenant
):
552 """Make sure all the datacenters and wims listed in the port mapping
553 belong to an specific tenant
556 # NOTE: Theoretically this could be done at SQL level, but given the
557 # number of tables involved (wim_port_mappings, wim_accounts,
558 # wims, wim_nfvo_tenants, datacenters, datacenter_tenants,
559 # tenants_datacents and nfvo_tenants), it would result in a
560 # extremely complex query. Moreover, the predicate can vary:
561 # for `get_wim_port_mappings` we can have any combination of
562 # (wim, datacenter, tenant), not all of them having the 3 values
563 # so we have combinatorial trouble to write the 'FROM' statement.
565 kwargs
= {'tenant': tenant
, 'error_if_none': False}
566 # Cache results to speedup things
570 def _get_datacenter(uuid
):
572 datacenters
.get(uuid
) or
573 datacenters
.setdefault(
574 uuid
, self
.get_datacenters_by(uuid
, **kwargs
)))
577 return (wims
.get(uuid
) or
578 wims
.setdefault(uuid
, self
.get_wims(uuid
, **kwargs
)))
582 for mapping
in mappings
583 if (_get_datacenter(mapping
['datacenter_id']) and
584 _get_wims(mapping
['wim_id']))
587 def get_wim_port_mappings(self
, wim
=None, datacenter
=None, tenant
=None,
589 """List all the port mappings, optionally filtering by wim, datacenter
592 from_
= [_PORT_MAPPING
,
593 _PORT_MAPPING_JOIN_WIM
if wim
else '',
594 _PORT_MAPPING_JOIN_DATACENTER
if datacenter
else '']
596 criteria
= ('wim_id', 'datacenter_id')
597 kwargs
.setdefault('error_if_none', False)
598 mappings
= self
.query(
600 SELECT
=_PORT_MAPPING_SELECT
,
601 ORDER_BY
=['wim_port_mapping.{}'.format(c
) for c
in criteria
],
602 wim
=wim
, datacenter
=datacenter
,
603 postprocess
=_postprocess_wim_port_mapping
,
607 mappings
= self
._filter
_port
_mappings
_by
_tenant
(mappings
, tenant
)
609 # We don't have to sort, since we have used 'ORDER_BY'
610 grouped_mappings
= groupby(mappings
, key
=itemgetter(*criteria
))
614 'datacenter_id': key
[1],
615 'pop_wan_mappings': [
616 filter_out_dict_keys(mapping
, (
617 'id', 'wim_id', 'datacenter_id',
618 'created_at', 'modified_at'))
619 for mapping
in group
]}
620 for key
, group
in grouped_mappings
623 def delete_wim_port_mappings(self
, wim_id
):
624 self
.db
.delete_row(FROM
='wim_port_mappings', WHERE
={"wim_id": wim_id
})
625 return "port mapping for wim {} deleted.".format(wim_id
)
627 def update_wim_port_mapping(self
, id, properties
):
628 original
= self
.query_one('wim_port_mappings', WHERE
={'id': id})
630 mapping_info
= remove_none_items(merge_dicts(
631 original
.get('wan_service_mapping_info') or {},
632 properties
.get('wan_service_mapping_info') or {}))
634 updates
= preprocess_record(
635 merge_dicts(original
, remove_none_items(properties
),
636 wan_service_mapping_info
=mapping_info
))
638 num_changes
= self
.db
.update_rows('wim_port_mappings',
639 UPDATE
=updates
, WHERE
={'id': id})
641 if num_changes
is None:
642 raise UnexpectedDatabaseError(
643 'Impossible to update wim_port_mappings {}:\n{}\n'.format(
644 id, _serialize(properties
))
649 def get_actions_in_groups(self
, wim_account_id
,
650 item_types
=('instance_wim_nets',),
651 group_offset
=0, group_limit
=150):
652 """Retrieve actions from the database in groups.
653 Each group contains all the actions that have the same ``item`` type
657 wim_account_id: restrict the search to actions to be performed
658 using the same account
659 item_types (list): [optional] filter the actions to the given
661 group_limit (int): maximum number of groups returned by the
663 group_offset (int): skip the N first groups. Used together with
664 group_limit for pagination purposes.
667 List of groups, where each group is a tuple ``(key, actions)``.
668 In turn, ``key`` is a tuple containing the values of
669 ``(item, item_id)`` used to create the group and ``actions`` is a
670 list of ``vim_wim_actions`` records (dicts).
674 '"{}"'.format(self
.db
.escape_string(t
)) for t
in item_types
)
676 items
= ('SELECT DISTINCT a.item, a.item_id, a.wim_account_id '
677 'FROM vim_wim_actions AS a '
678 'WHERE a.wim_account_id="{}" AND a.item IN ({}) '
679 'ORDER BY a.item, a.item_id '
680 'LIMIT {:d},{:d}').format(
681 self
.safe_str(wim_account_id
),
682 ','.join(type_options
),
683 group_offset
, group_limit
)
685 join
= 'vim_wim_actions NATURAL JOIN ({}) AS items'.format(items
)
686 db_results
= self
.db
.get_rows(
687 FROM
=join
, ORDER_BY
=('item', 'item_id', 'created_at'))
689 results
= (_postprocess_action(r
) for r
in db_results
)
690 criteria
= itemgetter('item', 'item_id')
691 return [(k
, list(g
)) for k
, g
in groupby(results
, key
=criteria
)]
693 def update_action(self
, instance_action_id
, task_index
, properties
):
694 condition
= {'instance_action_id': instance_action_id
,
695 'task_index': task_index
}
697 action
= self
.query_one('vim_wim_actions', WHERE
=condition
)
699 actions
= self
.query('vim_wim_actions', WHERE
=condition
)
700 self
.logger
.error('More then one action found:\n%s',
701 json
.dumps(actions
, indent
=4))
704 extra
= remove_none_items(merge_dicts(
705 action
.get('extra') or {},
706 properties
.get('extra') or {}))
708 updates
= preprocess_record(
709 merge_dicts(action
, properties
, extra
=extra
))
711 num_changes
= self
.db
.update_rows('vim_wim_actions', UPDATE
=updates
, WHERE
=condition
)
713 if num_changes
is None:
714 raise UnexpectedDatabaseError(
715 'Impossible to update vim_wim_actions '
716 '{instance_action_id}[{task_index}]'.format(*action
))
720 def get_wan_links(self
, uuid
=None, **kwargs
):
721 """Retrieve WAN link records from the database
724 uuid, instance_scenario_id, sce_net_id, wim_id, wim_account_id:
725 attributes that can be used at the WHERE clause
727 kwargs
.setdefault('uuid', uuid
)
728 kwargs
.setdefault('error_if_none', False)
730 criteria_fields
= ('uuid', 'instance_scenario_id', 'sce_net_id',
731 'wim_id', 'wim_account_id', 'sdn')
732 criteria
= remove_none_items(filter_dict_keys(kwargs
, criteria_fields
))
733 kwargs
= filter_out_dict_keys(kwargs
, criteria_fields
)
735 return self
.query('instance_wim_nets', WHERE
=criteria
, **kwargs
)
737 def update_wan_link(self
, uuid
, properties
):
738 wan_link
= self
.get_by_uuid('instance_wim_nets', uuid
)
740 wim_info
= remove_none_items(merge_dicts(
741 wan_link
.get('wim_info') or {},
742 properties
.get('wim_info') or {}))
744 updates
= preprocess_record(
745 merge_dicts(wan_link
, properties
, wim_info
=wim_info
))
747 self
.logger
.debug({'UPDATE': updates
})
748 num_changes
= self
.db
.update_rows(
749 'instance_wim_nets', UPDATE
=updates
,
750 WHERE
={'uuid': wan_link
['uuid']})
752 if num_changes
is None:
753 raise UnexpectedDatabaseError(
754 'Impossible to update instance_wim_nets ' + wan_link
['uuid'])
758 def get_instance_nets(self
, instance_scenario_id
, sce_net_id
, **kwargs
):
759 """Retrieve all the instance nets related to the same instance_scenario
764 WHERE
={'instance_scenario_id': instance_scenario_id
,
765 'sce_net_id': sce_net_id
},
767 'ORDER_BY', ('instance_scenario_id', 'sce_net_id')),
770 def update_instance_action_counters(self
, uuid
, failed
=None, done
=None):
771 """Atomically increment/decrement number_done and number_failed fields
772 in the instance action table
774 changes
= remove_none_items({
775 'number_failed': failed
and {'INCREMENT': failed
},
776 'number_done': done
and {'INCREMENT': done
}
782 return self
.db
.update_rows('instance_actions', WHERE
={'uuid': uuid
}, UPDATE
=changes
)
784 def get_only_vm_with_external_net(self
, instance_net_id
, **kwargs
):
785 """Return an instance VM if that is the only VM connected to an
786 external network identified by instance_net_id
788 counting
= ('SELECT DISTINCT instance_net_id '
789 'FROM instance_interfaces '
790 'WHERE instance_net_id="{}" AND type="external" '
791 'GROUP BY instance_net_id '
792 'HAVING COUNT(*)=1').format(self
.safe_str(instance_net_id
))
794 vm_item
= ('SELECT DISTINCT instance_vm_id '
795 'FROM instance_interfaces NATURAL JOIN ({}) AS a'
798 return self
.query_one(
799 'instance_vms JOIN ({}) as instance_interface '
800 'ON instance_vms.uuid=instance_interface.instance_vm_id'
801 .format(vm_item
), **kwargs
)
803 def safe_str(self
, string
):
804 """Return a SQL safe string"""
805 return self
.db
.escape_string(string
)
810 def _generate_port_mapping_id(self
, mapping_info
):
811 """Given a port mapping represented by a dict with a 'type' field,
812 generate a unique string, in a injective way.
814 mapping_info
= mapping_info
.copy() # Avoid mutating original object
815 mapping_type
= mapping_info
.pop('mapping_type', None)
817 raise UndefinedWanMappingType(mapping_info
)
819 unique_fields
= UNIQUE_PORT_MAPPING_INFO_FIELDS
.get(mapping_type
)
822 mapping_info
= filter_dict_keys(mapping_info
, unique_fields
)
824 self
.logger
.warning('Unique fields for WIM port mapping of type '
825 '%s not defined. Please add a list of fields '
826 'which combination should be unique in '
827 'UNIQUE_PORT_MAPPING_INFO_FIELDS '
828 '(`wim/persistency.py) ', mapping_type
)
830 repeatable_repr
= json
.dumps(mapping_info
, encoding
='utf-8',
831 sort_keys
=True, indent
=False)
833 return ':'.join([mapping_type
, _str2id(repeatable_repr
)])
836 def _serialize(value
):
837 """Serialize an arbitrary value in a consistent way,
838 so it can be stored in a database inside a text field
840 return yaml
.safe_dump(value
, default_flow_style
=True, width
=256)
843 def _unserialize(text
):
844 """Unserialize text representation into an arbitrary value,
845 so it can be loaded from the database
847 return yaml
.safe_load(text
)
850 def preprocess_record(record
):
851 """Small transformations to be applied to the data that cames from the
852 user before writing it to the database. By default, filter out timestamps,
853 and serialize the ``config`` field.
855 automatic_fields
= ['created_at', 'modified_at']
856 record
= serialize_fields(filter_out_dict_keys(record
, automatic_fields
))
861 def _preprocess_wim_account(wim_account
):
862 """Do the default preprocessing and convert the 'created' field from
865 wim_account
= preprocess_record(wim_account
)
867 created
= wim_account
.get('created')
868 wim_account
['created'] = (
869 'true' if created
is True or created
== 'true' else 'false')
874 def _postprocess_record(record
, hide
=_CONFIDENTIAL_FIELDS
):
875 """By default, hide passwords fields, unserialize ``config`` fields, and
876 convert float timestamps to strings
878 record
= hide_confidential_fields(record
, hide
)
879 record
= unserialize_fields(record
, hide
)
881 convert_float_timestamp2str(record
)
886 def _postprocess_action(action
):
887 if action
.get('extra'):
888 action
['extra'] = _unserialize(action
['extra'])
893 def _postprocess_wim_account(wim_account
, hide
=_CONFIDENTIAL_FIELDS
):
894 """Do the default postprocessing and convert the 'created' field from
897 # Fix fields from join
898 for field
in ('type', 'description', 'wim_url'):
899 if field
in wim_account
:
900 wim_account
['wim.'+field
] = wim_account
.pop(field
)
902 for field
in ('id', 'nfvo_tenant_id', 'wim_account_id'):
903 if field
in wim_account
:
904 wim_account
['association.'+field
] = wim_account
.pop(field
)
906 wim_account
= _postprocess_record(wim_account
, hide
)
908 created
= wim_account
.get('created')
909 wim_account
['created'] = (created
is True or created
== 'true')
914 def _postprocess_wim_port_mapping(mapping
, hide
=_CONFIDENTIAL_FIELDS
):
915 mapping
= _postprocess_record(mapping
, hide
=hide
)
916 mapping_info
= mapping
.get('wan_service_mapping_info', None) or {}
917 mapping
['wan_service_mapping_info'] = mapping_info
921 def hide_confidential_fields(record
, fields
=_CONFIDENTIAL_FIELDS
):
922 """Obfuscate confidential fields from the input dict.
925 This function performs a SHALLOW operation.
927 if not(isinstance(record
, dict) and fields
):
930 keys
= list(record
.keys())
931 keys
= (k
for k
in keys
for f
in fields
if k
== f
or k
.endswith('.'+f
))
933 return merge_dicts(record
, {k
: '********' for k
in keys
if record
[k
]})
936 def unserialize_fields(record
, hide
=_CONFIDENTIAL_FIELDS
,
937 fields
=_SERIALIZED_FIELDS
):
938 """Unserialize fields that where stored in the database as a serialized
941 keys
= list(record
.keys())
942 keys
= (k
for k
in keys
for f
in fields
if k
== f
or k
.endswith('.'+f
))
944 return merge_dicts(record
, {
945 key
: hide_confidential_fields(_unserialize(record
[key
]), hide
)
946 for key
in keys
if record
[key
]
950 def serialize_fields(record
, fields
=_SERIALIZED_FIELDS
):
951 """Serialize fields to be stored in the database as YAML"""
952 keys
= list(record
.keys())
953 keys
= (k
for k
in keys
for f
in fields
if k
== f
or k
.endswith('.'+f
))
955 return merge_dicts(record
, {
956 key
: _serialize(record
[key
])
957 for key
in keys
if record
[key
] is not None
961 def _decide_name_or_uuid(value
):
964 if isinstance(value
, (list, tuple)):
965 reference
= value
[0] if value
else ''
967 return 'uuid' if check_valid_uuid(reference
) else 'name'
970 def _compose_where_from_uuids_or_names(**conditions
):
971 """Create a dict containing the right conditions to be used in a database
974 This function chooses between ``names`` and ``uuid`` fields based on the
975 format of the passed string.
976 If a list is passed, the first element of the list will be used to choose
977 the name of the field.
978 If a ``None`` value is passed, ``uuid`` is used.
980 Note that this function automatically translates ``tenant`` to
981 ``nfvo_tenant`` for the sake of brevity.
984 >>> _compose_where_from_uuids_or_names(
986 tenant=['xyz123', 'def456']
987 datacenter='5286a274-8a1b-4b8d-a667-9c94261ad855')
988 {'wim.name': 'abcdef',
989 'nfvo_tenant.name': ['xyz123', 'def456']
990 'datacenter.uuid': '5286a274-8a1b-4b8d-a667-9c94261ad855'}
992 if 'tenant' in conditions
:
993 conditions
['nfvo_tenant'] = conditions
.pop('tenant')
996 '{}.{}'.format(kind
, _decide_name_or_uuid(value
)): value
997 for kind
, value
in conditions
.items() if value
1002 """Create an ID (following the UUID format) from a piece of arbitrary
1005 Different texts should generate different IDs, and the same text should
1006 generate the same ID in a repeatable way.
1008 return sha1(text
).hexdigest()