1 # -*- coding: utf-8 -*-
3 # Copyright 2018 University of Bristol - High Performance Networks Research
7 # Contributors: Anderson Bravalheri, Dimitrios Gkounis, Abubakar Siddique
8 # Muqaddas, Navdeep Uniyal, Reza Nejabati and Dimitra Simeonidou
10 # Licensed under the Apache License, Version 2.0 (the "License"); you may
11 # not use this file except in compliance with the License. You may obtain
12 # a copy of the License at
14 # http://www.apache.org/licenses/LICENSE-2.0
16 # Unless required by applicable law or agreed to in writing, software
17 # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
18 # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
19 # License for the specific language governing permissions and limitations
22 # For those usages not covered by the Apache License, Version 2.0 please
23 # contact with: <highperformance-networks@bristol.ac.uk>
25 # Neither the name of the University of Bristol nor the names of its
26 # contributors may be used to endorse or promote products derived from
27 # this software without specific prior written permission.
29 # This work has been performed in the context of DCMS UK 5G Testbeds
30 # & Trials Programme and in the framework of the Metro-Haul project -
31 # funded by the European Commission under Grant number 761727 through the
32 # Horizon 2020 and 5G-PPP programmes.
35 """This module contains only logic related to managing records in a database
36 which includes data format normalization, data format validation and etc.
37 (It works as an extension to `nfvo_db.py` for the WIM feature)
39 No domain logic/architectural concern should be present in this file.
43 from contextlib
import contextmanager
44 from hashlib
import sha1
45 from itertools
import groupby
46 from operator
import itemgetter
47 from sys
import exc_info
48 # from time import time
49 from uuid
import uuid1
as generate_uuid
55 convert_float_timestamp2str
,
68 UndefinedWanMappingType
,
69 UnexpectedDatabaseError
,
71 WimAndTenantAlreadyAttached
78 ' JOIN wim_nfvo_tenants AS association '
79 ' ON association.wim_id=wim.uuid '
80 ' JOIN nfvo_tenants AS nfvo_tenant '
81 ' ON association.nfvo_tenant_id=nfvo_tenant.uuid '
82 ' JOIN wim_accounts AS wim_account '
83 ' ON association.wim_account_id=wim_account.uuid '
87 'wim_accounts AS wim_account '
88 ' JOIN wim_nfvo_tenants AS association '
89 ' ON association.wim_account_id=wim_account.uuid '
91 ' ON association.wim_id=wim.uuid '
92 ' JOIN nfvo_tenants AS nfvo_tenant '
93 ' ON association.nfvo_tenant_id=nfvo_tenant.uuid '
97 'datacenters AS datacenter '
98 ' JOIN tenants_datacenters AS association '
99 ' ON association.datacenter_id=datacenter.uuid '
100 ' JOIN datacenter_tenants as datacenter_account '
101 ' ON association.datacenter_tenant_id=datacenter_account.uuid '
102 ' JOIN nfvo_tenants AS nfvo_tenant '
103 ' ON association.nfvo_tenant_id=nfvo_tenant.uuid '
106 _PORT_MAPPING
= 'wim_port_mappings as wim_port_mapping '
108 _PORT_MAPPING_JOIN_WIM
= (
110 ' ON wim_port_mapping.wim_id=wim.uuid '
113 _PORT_MAPPING_JOIN_DATACENTER
= (
114 ' JOIN datacenters as datacenter '
115 ' ON wim_port_mapping.datacenter_id=datacenter.uuid '
119 'wim.{0} as {0}'.format(_field
)
120 for _field
in 'uuid name description wim_url type config '
121 'created_at modified_at'.split()
124 _WIM_ACCOUNT_SELECT
= 'uuid name user password config'.split()
126 _PORT_MAPPING_SELECT
= ('wim_port_mapping.*', )
128 _CONFIDENTIAL_FIELDS
= ('password', 'passwd')
130 _SERIALIZED_FIELDS
= ('config', 'vim_info', 'wim_info', 'conn_info', 'extra',
131 'wan_service_mapping_info')
133 UNIQUE_PORT_MAPPING_INFO_FIELDS
= {
134 'dpid-port': ('wan_switch_dpid', 'wan_switch_port')
136 """Fields that should be unique for each port mapping that relies on
137 wan_service_mapping_info.
139 For example, for port mappings of type 'dpid-port', each combination of
140 wan_switch_dpid and wan_switch_port should be unique (the same switch cannot
141 be connected to two different places using the same port)
145 class WimPersistence(object):
146 """High level interactions with the WIM tables in the database"""
148 def __init__(self
, db
, logger
=None):
150 self
.logger
= logger
or logging
.getLogger('openmano.wim.persistence')
160 error_if_multiple
=False,
162 hide
=_CONFIDENTIAL_FIELDS
,
164 """Retrieve records from the database.
167 SELECT, FROM, WHERE, LIMIT, ORDER_BY: used to compose the SQL
168 query. See ``nfvo_db.get_rows``.
169 OFFSET: only valid when used togheter with LIMIT.
170 Ignore the OFFSET first results of the query.
171 error_if_none: by default an error is raised if no record is
172 found. With this option it is possible to disable this error.
173 error_if_multiple: by default no error is raised if more then one
175 With this option it is possible to enable this error.
176 postprocess: function applied to every retrieved record.
177 This function receives a dict as input and must return it
178 after modifications. Moreover this function should accept a
179 second optional parameter ``hide`` indicating
180 the confidential fiels to be obfuscated.
181 By default a minimal postprocessing function is applied,
182 obfuscating confidential fields and converting timestamps.
183 hide: option proxied to postprocess
185 All the remaining keyword arguments will be assumed to be ``name``s or
186 ``uuid``s to compose the WHERE statement, according to their format.
187 If the value corresponds to an array, the first element will determine
188 if it is an name or UUID.
191 - ``wim="abcdef"``` will be turned into ``wim.name="abcdef"``,
192 - ``datacenter="5286a274-8a1b-4b8d-a667-9c94261ad855"``
194 ``datacenter.uuid="5286a274-8a1b-4b8d-a667-9c94261ad855"``.
195 - ``wim=["5286a274-8a1b-4b8d-a667-9c94261ad855", ...]``
197 ``wim.uuid=["5286a274-8a1b-4b8d-a667-9c94261ad855", ...]``
200 NoRecordFound: if the query result set is empty
201 DbBaseException: errors occuring during the execution of the query.
204 postprocess
= postprocess
or _postprocess_record
207 # Find remaining keywords by name or uuid
208 WHERE
.update(_compose_where_from_uuids_or_names(**kwargs
))
209 WHERE
= WHERE
or None
210 # ^ If the where statement is empty, it is better to leave it as None,
211 # so it can be filtered out at a later stage
212 LIMIT
= ('{:d},{:d}'.format(OFFSET
, LIMIT
)
213 if LIMIT
and OFFSET
else LIMIT
)
215 query
= remove_none_items({
216 'SELECT': SELECT
, 'FROM': FROM
, 'WHERE': WHERE
,
217 'LIMIT': LIMIT
, 'ORDER_BY': ORDER_BY
})
219 records
= self
.db
.get_rows(**query
)
221 table
= FROM
.split()[0]
222 if error_if_none
and not records
:
223 raise NoRecordFound(WHERE
, table
)
225 if error_if_multiple
and len(records
) > 1:
226 self
.logger
.error('Multiple records '
227 'FROM %s WHERE %s:\n\n%s\n\n',
228 FROM
, WHERE
, json
.dumps(records
, indent
=4))
229 raise MultipleRecordsFound(WHERE
, table
)
232 expand_joined_fields(postprocess(record
, hide
))
233 for record
in records
236 def query_one(self
, *args
, **kwargs
):
237 """Similar to ``query``, but ensuring just one result.
238 ``error_if_multiple`` is enabled by default.
240 kwargs
.setdefault('error_if_multiple', True)
241 records
= self
.query(*args
, **kwargs
)
242 return records
[0] if records
else None
244 def get_by_uuid(self
, table
, uuid
, **kwargs
):
245 """Retrieve one record from the database based on its uuid
248 table (str): table name (to be used in SQL's FROM statement).
249 uuid (str): unique identifier for record.
251 For additional keyword arguments and exceptions see :obj:`~.query`
252 (``error_if_multiple`` is enabled by default).
255 raise UndefinedUuidOrName(table
)
256 return self
.query_one(table
, WHERE
={'uuid': uuid
}, **kwargs
)
258 def get_by_name_or_uuid(self
, table
, uuid_or_name
, **kwargs
):
259 """Retrieve a record from the database based on a value that can be its
263 table (str): table name (to be used in SQL's FROM statement).
264 uuid_or_name (str): this value can correspond to either uuid or
266 For additional keyword arguments and exceptions see :obj:`~.query`
267 (``error_if_multiple`` is enabled by default).
269 if uuid_or_name
is None:
270 raise UndefinedUuidOrName(table
)
272 key
= 'uuid' if check_valid_uuid(uuid_or_name
) else 'name'
273 return self
.query_one(table
, WHERE
={key
: uuid_or_name
}, **kwargs
)
275 def get_wims(self
, uuid_or_name
=None, tenant
=None, **kwargs
):
276 """Retrieve information about one or more WIMs stored in the database
279 uuid_or_name (str): uuid or name for WIM
280 tenant (str): [optional] uuid or name for NFVO tenant
282 See :obj:`~.query` for additional keyword arguments.
284 kwargs
.update(wim
=uuid_or_name
, tenant
=tenant
)
285 from_
= _WIM_JOIN
if tenant
else _WIM
286 select_
= _WIM_SELECT
[:] + (['wim_account.*'] if tenant
else [])
288 kwargs
.setdefault('SELECT', select_
)
289 return self
.query(from_
, **kwargs
)
291 def get_wim(self
, wim
, tenant
=None, **kwargs
):
292 """Similar to ``get_wims`` but ensure only one result is returned"""
293 kwargs
.setdefault('error_if_multiple', True)
294 return self
.get_wims(wim
, tenant
)[0]
296 def create_wim(self
, wim_descriptor
):
297 """Create a new wim record inside the database and returns its uuid
300 wim_descriptor (dict): properties of the record
301 (usually each field corresponds to a database column, but extra
302 information can be offloaded to another table or serialized as
305 str: UUID of the created WIM
307 if "config" in wim_descriptor
:
308 wim_descriptor
["config"] = _serialize(wim_descriptor
["config"])
310 return self
.db
.new_row(
311 "wims", wim_descriptor
, add_uuid
=True, confidential_data
=True)
313 def update_wim(self
, uuid_or_name
, wim_descriptor
):
314 """Change an existing WIM record on the database"""
315 # obtain data, check that only one exist
316 wim
= self
.get_by_name_or_uuid('wims', uuid_or_name
)
320 where
= {'uuid': wim
['uuid']}
322 # unserialize config, edit and serialize it again
323 new_config_dict
= wim_descriptor
.get('config', {}) or {}
324 config_dict
= remove_none_items(merge_dicts(
325 wim
.get('config', {}) or {}, new_config_dict
))
326 wim_descriptor
['config'] = (
327 _serialize(config_dict
) if config_dict
else None)
329 self
.db
.update_rows('wims', wim_descriptor
, where
)
333 def delete_wim(self
, wim
):
334 # get nfvo_tenant info
335 wim
= self
.get_by_name_or_uuid('wims', wim
)
337 self
.db
.delete_row_by_id('wims', wim
['uuid'])
339 return wim
['uuid'] + ' ' + wim
['name']
341 def get_wim_accounts_by(self
, wim
=None, tenant
=None, uuid
=None, **kwargs
):
342 """Retrieve WIM account information from the database together
343 with the related records (wim, nfvo_tenant and wim_nfvo_tenant)
346 wim (str): uuid or name for WIM
347 tenant (str): [optional] uuid or name for NFVO tenant
349 See :obj:`~.query` for additional keyword arguments.
351 kwargs
.update(wim
=wim
, tenant
=tenant
)
352 kwargs
.setdefault('postprocess', _postprocess_wim_account
)
354 kwargs
.setdefault('WHERE', {'wim_account.uuid': uuid
})
355 return self
.query(FROM
=_WIM_ACCOUNT_JOIN
, **kwargs
)
357 def get_wim_account_by(self
, wim
=None, tenant
=None, uuid
=None, **kwargs
):
358 """Similar to ``get_wim_accounts_by``, but ensuring just one result"""
359 kwargs
.setdefault('error_if_multiple', True)
360 return self
.get_wim_accounts_by(wim
, tenant
, uuid
, **kwargs
)[0]
362 def get_wim_accounts(self
, **kwargs
):
363 """Retrieve all the accounts from the database"""
364 kwargs
.setdefault('postprocess', _postprocess_wim_account
)
365 return self
.query(FROM
=_WIM_ACCOUNT_JOIN
, **kwargs
)
367 def get_wim_account(self
, uuid_or_name
, **kwargs
):
368 """Retrieve WIM Account record by UUID or name,
369 See :obj:`get_by_name_or_uuid` for keyword arguments.
371 kwargs
.setdefault('postprocess', _postprocess_wim_account
)
372 kwargs
.setdefault('SELECT', _WIM_ACCOUNT_SELECT
)
373 return self
.get_by_name_or_uuid('wim_accounts', uuid_or_name
, **kwargs
)
376 def _associate(self
, wim_id
, nfvo_tenant_id
):
377 """Auxiliary method for ``create_wim_account``
379 This method just create a row in the association table
384 except DbBaseException
as db_exception
:
385 error_msg
= str(db_exception
)
386 if all([msg
in error_msg
387 for msg
in ("already in use", "'wim_nfvo_tenant'")]):
388 ex
= WimAndTenantAlreadyAttached(wim_id
, nfvo_tenant_id
)
389 raise ex
from db_exception
392 def create_wim_account(self
, wim
, tenant
, properties
):
393 """Associate a wim to a tenant using the ``wim_nfvo_tenants`` table
394 and create a ``wim_account`` to store credentials and configurations.
396 For the sake of simplification, we assume that each NFVO tenant can be
397 attached to a WIM using only one WIM account. This is automatically
398 guaranteed via database constraints.
399 For corner cases, the same WIM can be registered twice using another
403 wim (str): name or uuid of the WIM related to the account being
405 tenant (str): name or uuid of the nfvo tenant to which the account
407 properties (dict): properties of the account
408 (eg. user, password, ...)
410 wim_id
= self
.get_by_name_or_uuid('wims', wim
, SELECT
=['uuid'])['uuid']
411 tenant
= self
.get_by_name_or_uuid('nfvo_tenants', tenant
,
412 SELECT
=['uuid', 'name'])
413 account
= properties
.setdefault('name', tenant
['name'])
415 wim_account
= self
.query_one('wim_accounts',
416 WHERE
={'wim_id': wim_id
, 'name': account
},
422 if wim_account
is None:
423 # If a row for the wim account doesn't exist yet, we need to
424 # create one, otherwise we can just re-use it.
425 account_id
= str(generate_uuid())
426 used_uuids
.append(account_id
)
427 row
= merge_dicts(properties
, wim_id
=wim_id
, uuid
=account_id
)
428 transaction
.append({'wim_accounts': _preprocess_wim_account(row
)})
430 account_id
= wim_account
['uuid']
431 properties
.pop('config', None) # Config is too complex to compare
432 diff
= {k
: v
for k
, v
in properties
.items() if v
!= wim_account
[k
]}
434 tip
= 'Edit the account first, and then attach it to a tenant'
435 raise WimAccountOverwrite(wim_account
, diff
, tip
)
438 'wim_nfvo_tenants': {'nfvo_tenant_id': tenant
['uuid'],
440 'wim_account_id': account_id
}})
442 with self
._associate
(wim_id
, tenant
['uuid']):
443 self
.db
.new_rows(transaction
, used_uuids
, confidential_data
=True)
447 def update_wim_account(self
, uuid
, properties
, hide
=_CONFIDENTIAL_FIELDS
):
448 """Update WIM account record by overwriting fields with new values
450 Specially for the field ``config`` this means that a new dict will be
451 merged to the existing one.
454 uuid (str): UUID for the WIM account
455 properties (dict): fields that should be overwritten
460 wim_account
= self
.get_by_uuid('wim_accounts', uuid
)
461 safe_fields
= 'user password name created'.split()
462 updates
= _preprocess_wim_account(
463 merge_dicts(wim_account
, filter_dict_keys(properties
, safe_fields
))
466 if properties
.get('config'):
467 old_config
= wim_account
.get('config') or {}
468 new_config
= merge_dicts(old_config
, properties
['config'])
469 updates
['config'] = _serialize(new_config
)
471 num_changes
= self
.db
.update_rows('wim_accounts', UPDATE
=updates
,
472 WHERE
={'uuid': wim_account
['uuid']})
474 if num_changes
is None:
475 raise UnexpectedDatabaseError('Impossible to update wim_account '
476 '{name}:{uuid}'.format(*wim_account
))
478 return self
.get_wim_account(wim_account
['uuid'], hide
=hide
)
480 def delete_wim_account(self
, uuid
):
481 """Remove WIM account record from the database"""
482 # Since we have foreign keys configured with ON CASCADE, we can rely
483 # on the database engine to guarantee consistency, deleting the
485 return self
.db
.delete_row_by_id('wim_accounts', uuid
)
487 def get_datacenters_by(self
, datacenter
=None, tenant
=None, **kwargs
):
488 """Retrieve datacenter information from the database together
489 with the related records (nfvo_tenant)
492 datacenter (str): uuid or name for datacenter
493 tenant (str): [optional] uuid or name for NFVO tenant
495 See :obj:`~.query` for additional keyword arguments.
498 kwargs
.update(datacenter
=datacenter
, tenant
=tenant
)
499 return self
.query(_DATACENTER_JOIN
, **kwargs
)
501 return [self
.get_by_name_or_uuid('datacenters',
502 datacenter
, **kwargs
)]
504 def get_datacenter_by(self
, datacenter
=None, tenant
=None, **kwargs
):
505 """Similar to ``get_datacenters_by``, but ensuring just one result"""
506 kwargs
.setdefault('error_if_multiple', True)
507 return self
.get_datacenters_by(datacenter
, tenant
, **kwargs
)[0]
509 def _create_single_port_mapping(self
, properties
):
510 info
= properties
.setdefault('wan_service_mapping_info', {})
511 endpoint_id
= properties
.get('wan_service_endpoint_id')
513 if info
.get('mapping_type') and not endpoint_id
:
514 properties
['wan_service_endpoint_id'] = (
515 self
._generate
_port
_mapping
_id
(info
))
517 properties
['wan_service_mapping_info'] = _serialize(info
)
520 self
.db
.new_row('wim_port_mappings', properties
,
521 add_uuid
=False, confidential_data
=True)
522 except DbBaseException
as old_exception
:
523 self
.logger
.exception(old_exception
)
524 ex
= InvalidParameters(
525 "The mapping must contain the "
526 "'pop_switch_dpid', 'pop_switch_port', and "
527 "wan_service_mapping_info: "
528 "('wan_switch_dpid' and 'wan_switch_port') or "
529 "'wan_service_endpoint_id}'")
530 raise ex
from old_exception
534 def create_wim_port_mappings(self
, wim
, port_mappings
, tenant
=None):
535 if not isinstance(wim
, dict):
536 wim
= self
.get_by_name_or_uuid('wims', wim
)
538 for port_mapping
in port_mappings
:
539 port_mapping
['wim_name'] = wim
['name']
540 datacenter
= self
.get_datacenter_by(
541 port_mapping
['datacenter_name'], tenant
)
542 for pop_wan_port_mapping
in port_mapping
['pop_wan_mappings']:
543 element
= merge_dicts(pop_wan_port_mapping
, {
544 'wim_id': wim
['uuid'],
545 'datacenter_id': datacenter
['uuid']})
546 self
._create
_single
_port
_mapping
(element
)
550 def _filter_port_mappings_by_tenant(self
, mappings
, tenant
):
551 """Make sure all the datacenters and wims listed in the port mapping
552 belong to an specific tenant
555 # NOTE: Theoretically this could be done at SQL level, but given the
556 # number of tables involved (wim_port_mappings, wim_accounts,
557 # wims, wim_nfvo_tenants, datacenters, datacenter_tenants,
558 # tenants_datacents and nfvo_tenants), it would result in a
559 # extremely complex query. Moreover, the predicate can vary:
560 # for `get_wim_port_mappings` we can have any combination of
561 # (wim, datacenter, tenant), not all of them having the 3 values
562 # so we have combinatorial trouble to write the 'FROM' statement.
564 kwargs
= {'tenant': tenant
, 'error_if_none': False}
565 # Cache results to speedup things
569 def _get_datacenter(uuid
):
571 datacenters
.get(uuid
) or
572 datacenters
.setdefault(
573 uuid
, self
.get_datacenters_by(uuid
, **kwargs
)))
576 return (wims
.get(uuid
) or
577 wims
.setdefault(uuid
, self
.get_wims(uuid
, **kwargs
)))
581 for mapping
in mappings
582 if (_get_datacenter(mapping
['datacenter_id']) and
583 _get_wims(mapping
['wim_id']))
586 def get_wim_port_mappings(self
, wim
=None, datacenter
=None, tenant
=None,
588 """List all the port mappings, optionally filtering by wim, datacenter
591 from_
= [_PORT_MAPPING
,
592 _PORT_MAPPING_JOIN_WIM
if wim
else '',
593 _PORT_MAPPING_JOIN_DATACENTER
if datacenter
else '']
595 criteria
= ('wim_id', 'datacenter_id')
596 kwargs
.setdefault('error_if_none', False)
597 mappings
= self
.query(
599 SELECT
=_PORT_MAPPING_SELECT
,
600 ORDER_BY
=['wim_port_mapping.{}'.format(c
) for c
in criteria
],
601 wim
=wim
, datacenter
=datacenter
,
602 postprocess
=_postprocess_wim_port_mapping
,
606 mappings
= self
._filter
_port
_mappings
_by
_tenant
(mappings
, tenant
)
608 # We don't have to sort, since we have used 'ORDER_BY'
609 grouped_mappings
= groupby(mappings
, key
=itemgetter(*criteria
))
613 'datacenter_id': key
[1],
614 'pop_wan_mappings': [
615 filter_out_dict_keys(mapping
, (
616 'id', 'wim_id', 'datacenter_id',
617 'created_at', 'modified_at'))
618 for mapping
in group
]}
619 for key
, group
in grouped_mappings
622 def delete_wim_port_mappings(self
, wim_id
):
623 self
.db
.delete_row(FROM
='wim_port_mappings', WHERE
={"wim_id": wim_id
})
624 return "port mapping for wim {} deleted.".format(wim_id
)
626 def update_wim_port_mapping(self
, id, properties
):
627 original
= self
.query_one('wim_port_mappings', WHERE
={'id': id})
629 mapping_info
= remove_none_items(merge_dicts(
630 original
.get('wan_service_mapping_info') or {},
631 properties
.get('wan_service_mapping_info') or {}))
633 updates
= preprocess_record(
634 merge_dicts(original
, remove_none_items(properties
),
635 wan_service_mapping_info
=mapping_info
))
637 num_changes
= self
.db
.update_rows('wim_port_mappings',
638 UPDATE
=updates
, WHERE
={'id': id})
640 if num_changes
is None:
641 raise UnexpectedDatabaseError(
642 'Impossible to update wim_port_mappings {}:\n{}\n'.format(
643 id, _serialize(properties
))
648 def get_actions_in_groups(self
, wim_account_id
,
649 item_types
=('instance_wim_nets',),
650 group_offset
=0, group_limit
=150):
651 """Retrieve actions from the database in groups.
652 Each group contains all the actions that have the same ``item`` type
656 wim_account_id: restrict the search to actions to be performed
657 using the same account
658 item_types (list): [optional] filter the actions to the given
660 group_limit (int): maximum number of groups returned by the
662 group_offset (int): skip the N first groups. Used together with
663 group_limit for pagination purposes.
666 List of groups, where each group is a tuple ``(key, actions)``.
667 In turn, ``key`` is a tuple containing the values of
668 ``(item, item_id)`` used to create the group and ``actions`` is a
669 list of ``vim_wim_actions`` records (dicts).
673 '"{}"'.format(self
.db
.escape_string(t
)) for t
in item_types
)
675 items
= ('SELECT DISTINCT a.item, a.item_id, a.wim_account_id '
676 'FROM vim_wim_actions AS a '
677 'WHERE a.wim_account_id="{}" AND a.item IN ({}) '
678 'ORDER BY a.item, a.item_id '
679 'LIMIT {:d},{:d}').format(
680 self
.safe_str(wim_account_id
),
681 ','.join(type_options
),
682 group_offset
, group_limit
)
684 join
= 'vim_wim_actions NATURAL JOIN ({}) AS items'.format(items
)
685 db_results
= self
.db
.get_rows(
686 FROM
=join
, ORDER_BY
=('item', 'item_id', 'created_at'))
688 results
= (_postprocess_action(r
) for r
in db_results
)
689 criteria
= itemgetter('item', 'item_id')
690 return [(k
, list(g
)) for k
, g
in groupby(results
, key
=criteria
)]
692 def update_action(self
, instance_action_id
, task_index
, properties
):
693 condition
= {'instance_action_id': instance_action_id
,
694 'task_index': task_index
}
696 action
= self
.query_one('vim_wim_actions', WHERE
=condition
)
698 actions
= self
.query('vim_wim_actions', WHERE
=condition
)
699 self
.logger
.error('More then one action found:\n%s',
700 json
.dumps(actions
, indent
=4))
703 extra
= remove_none_items(merge_dicts(
704 action
.get('extra') or {},
705 properties
.get('extra') or {}))
707 updates
= preprocess_record(
708 merge_dicts(action
, properties
, extra
=extra
))
710 num_changes
= self
.db
.update_rows('vim_wim_actions', UPDATE
=updates
, WHERE
=condition
)
712 if num_changes
is None:
713 raise UnexpectedDatabaseError(
714 'Impossible to update vim_wim_actions '
715 '{instance_action_id}[{task_index}]'.format(*action
))
719 def get_wan_links(self
, uuid
=None, **kwargs
):
720 """Retrieve WAN link records from the database
723 uuid, instance_scenario_id, sce_net_id, wim_id, wim_account_id:
724 attributes that can be used at the WHERE clause
726 kwargs
.setdefault('uuid', uuid
)
727 kwargs
.setdefault('error_if_none', False)
729 criteria_fields
= ('uuid', 'instance_scenario_id', 'sce_net_id',
730 'wim_id', 'wim_account_id')
731 criteria
= remove_none_items(filter_dict_keys(kwargs
, criteria_fields
))
732 kwargs
= filter_out_dict_keys(kwargs
, criteria_fields
)
734 return self
.query('instance_wim_nets', WHERE
=criteria
, **kwargs
)
736 def update_wan_link(self
, uuid
, properties
):
737 wan_link
= self
.get_by_uuid('instance_wim_nets', uuid
)
739 wim_info
= remove_none_items(merge_dicts(
740 wan_link
.get('wim_info') or {},
741 properties
.get('wim_info') or {}))
743 updates
= preprocess_record(
744 merge_dicts(wan_link
, properties
, wim_info
=wim_info
))
746 self
.logger
.debug({'UPDATE': updates
})
747 num_changes
= self
.db
.update_rows(
748 'instance_wim_nets', UPDATE
=updates
,
749 WHERE
={'uuid': wan_link
['uuid']})
751 if num_changes
is None:
752 raise UnexpectedDatabaseError(
753 'Impossible to update instance_wim_nets ' + wan_link
['uuid'])
757 def get_instance_nets(self
, instance_scenario_id
, sce_net_id
, **kwargs
):
758 """Retrieve all the instance nets related to the same instance_scenario
763 WHERE
={'instance_scenario_id': instance_scenario_id
,
764 'sce_net_id': sce_net_id
},
766 'ORDER_BY', ('instance_scenario_id', 'sce_net_id')),
769 def update_instance_action_counters(self
, uuid
, failed
=None, done
=None):
770 """Atomically increment/decrement number_done and number_failed fields
771 in the instance action table
773 changes
= remove_none_items({
774 'number_failed': failed
and {'INCREMENT': failed
},
775 'number_done': done
and {'INCREMENT': done
}
781 return self
.db
.update_rows('instance_actions', WHERE
={'uuid': uuid
}, UPDATE
=changes
)
783 def get_only_vm_with_external_net(self
, instance_net_id
, **kwargs
):
784 """Return an instance VM if that is the only VM connected to an
785 external network identified by instance_net_id
787 counting
= ('SELECT DISTINCT instance_net_id '
788 'FROM instance_interfaces '
789 'WHERE instance_net_id="{}" AND type="external" '
790 'GROUP BY instance_net_id '
791 'HAVING COUNT(*)=1').format(self
.safe_str(instance_net_id
))
793 vm_item
= ('SELECT DISTINCT instance_vm_id '
794 'FROM instance_interfaces NATURAL JOIN ({}) AS a'
797 return self
.query_one(
798 'instance_vms JOIN ({}) as instance_interface '
799 'ON instance_vms.uuid=instance_interface.instance_vm_id'
800 .format(vm_item
), **kwargs
)
802 def safe_str(self
, string
):
803 """Return a SQL safe string"""
804 return self
.db
.escape_string(string
)
809 def _generate_port_mapping_id(self
, mapping_info
):
810 """Given a port mapping represented by a dict with a 'type' field,
811 generate a unique string, in a injective way.
813 mapping_info
= mapping_info
.copy() # Avoid mutating original object
814 mapping_type
= mapping_info
.pop('mapping_type', None)
816 raise UndefinedWanMappingType(mapping_info
)
818 unique_fields
= UNIQUE_PORT_MAPPING_INFO_FIELDS
.get(mapping_type
)
821 mapping_info
= filter_dict_keys(mapping_info
, unique_fields
)
823 self
.logger
.warning('Unique fields for WIM port mapping of type '
824 '%s not defined. Please add a list of fields '
825 'which combination should be unique in '
826 'UNIQUE_PORT_MAPPING_INFO_FIELDS '
827 '(`wim/persistency.py) ', mapping_type
)
829 repeatable_repr
= json
.dumps(mapping_info
, encoding
='utf-8',
830 sort_keys
=True, indent
=False)
832 return ':'.join([mapping_type
, _str2id(repeatable_repr
)])
835 def _serialize(value
):
836 """Serialize an arbitrary value in a consistent way,
837 so it can be stored in a database inside a text field
839 return yaml
.safe_dump(value
, default_flow_style
=True, width
=256)
842 def _unserialize(text
):
843 """Unserialize text representation into an arbitrary value,
844 so it can be loaded from the database
846 return yaml
.safe_load(text
)
849 def preprocess_record(record
):
850 """Small transformations to be applied to the data that cames from the
851 user before writing it to the database. By default, filter out timestamps,
852 and serialize the ``config`` field.
854 automatic_fields
= ['created_at', 'modified_at']
855 record
= serialize_fields(filter_out_dict_keys(record
, automatic_fields
))
860 def _preprocess_wim_account(wim_account
):
861 """Do the default preprocessing and convert the 'created' field from
864 wim_account
= preprocess_record(wim_account
)
866 created
= wim_account
.get('created')
867 wim_account
['created'] = (
868 'true' if created
is True or created
== 'true' else 'false')
873 def _postprocess_record(record
, hide
=_CONFIDENTIAL_FIELDS
):
874 """By default, hide passwords fields, unserialize ``config`` fields, and
875 convert float timestamps to strings
877 record
= hide_confidential_fields(record
, hide
)
878 record
= unserialize_fields(record
, hide
)
880 convert_float_timestamp2str(record
)
885 def _postprocess_action(action
):
886 if action
.get('extra'):
887 action
['extra'] = _unserialize(action
['extra'])
892 def _postprocess_wim_account(wim_account
, hide
=_CONFIDENTIAL_FIELDS
):
893 """Do the default postprocessing and convert the 'created' field from
896 # Fix fields from join
897 for field
in ('type', 'description', 'wim_url'):
898 if field
in wim_account
:
899 wim_account
['wim.'+field
] = wim_account
.pop(field
)
901 for field
in ('id', 'nfvo_tenant_id', 'wim_account_id'):
902 if field
in wim_account
:
903 wim_account
['association.'+field
] = wim_account
.pop(field
)
905 wim_account
= _postprocess_record(wim_account
, hide
)
907 created
= wim_account
.get('created')
908 wim_account
['created'] = (created
is True or created
== 'true')
913 def _postprocess_wim_port_mapping(mapping
, hide
=_CONFIDENTIAL_FIELDS
):
914 mapping
= _postprocess_record(mapping
, hide
=hide
)
915 mapping_info
= mapping
.get('wan_service_mapping_info', None) or {}
916 mapping
['wan_service_mapping_info'] = mapping_info
920 def hide_confidential_fields(record
, fields
=_CONFIDENTIAL_FIELDS
):
921 """Obfuscate confidential fields from the input dict.
924 This function performs a SHALLOW operation.
926 if not(isinstance(record
, dict) and fields
):
929 keys
= list(record
.keys())
930 keys
= (k
for k
in keys
for f
in fields
if k
== f
or k
.endswith('.'+f
))
932 return merge_dicts(record
, {k
: '********' for k
in keys
if record
[k
]})
935 def unserialize_fields(record
, hide
=_CONFIDENTIAL_FIELDS
,
936 fields
=_SERIALIZED_FIELDS
):
937 """Unserialize fields that where stored in the database as a serialized
940 keys
= list(record
.keys())
941 keys
= (k
for k
in keys
for f
in fields
if k
== f
or k
.endswith('.'+f
))
943 return merge_dicts(record
, {
944 key
: hide_confidential_fields(_unserialize(record
[key
]), hide
)
945 for key
in keys
if record
[key
]
949 def serialize_fields(record
, fields
=_SERIALIZED_FIELDS
):
950 """Serialize fields to be stored in the database as YAML"""
951 keys
= list(record
.keys())
952 keys
= (k
for k
in keys
for f
in fields
if k
== f
or k
.endswith('.'+f
))
954 return merge_dicts(record
, {
955 key
: _serialize(record
[key
])
956 for key
in keys
if record
[key
] is not None
960 def _decide_name_or_uuid(value
):
963 if isinstance(value
, (list, tuple)):
964 reference
= value
[0] if value
else ''
966 return 'uuid' if check_valid_uuid(reference
) else 'name'
969 def _compose_where_from_uuids_or_names(**conditions
):
970 """Create a dict containing the right conditions to be used in a database
973 This function chooses between ``names`` and ``uuid`` fields based on the
974 format of the passed string.
975 If a list is passed, the first element of the list will be used to choose
976 the name of the field.
977 If a ``None`` value is passed, ``uuid`` is used.
979 Note that this function automatically translates ``tenant`` to
980 ``nfvo_tenant`` for the sake of brevity.
983 >>> _compose_where_from_uuids_or_names(
985 tenant=['xyz123', 'def456']
986 datacenter='5286a274-8a1b-4b8d-a667-9c94261ad855')
987 {'wim.name': 'abcdef',
988 'nfvo_tenant.name': ['xyz123', 'def456']
989 'datacenter.uuid': '5286a274-8a1b-4b8d-a667-9c94261ad855'}
991 if 'tenant' in conditions
:
992 conditions
['nfvo_tenant'] = conditions
.pop('tenant')
995 '{}.{}'.format(kind
, _decide_name_or_uuid(value
)): value
996 for kind
, value
in conditions
.items() if value
1001 """Create an ID (following the UUID format) from a piece of arbitrary
1004 Different texts should generate different IDs, and the same text should
1005 generate the same ID in a repeatable way.
1007 return sha1(text
).hexdigest()