1 # -*- coding: utf-8 -*-
3 # Copyright 2018 University of Bristol - High Performance Networks Research
7 # Contributors: Anderson Bravalheri, Dimitrios Gkounis, Abubakar Siddique
8 # Muqaddas, Navdeep Uniyal, Reza Nejabati and Dimitra Simeonidou
10 # Licensed under the Apache License, Version 2.0 (the "License"); you may
11 # not use this file except in compliance with the License. You may obtain
12 # a copy of the License at
14 # http://www.apache.org/licenses/LICENSE-2.0
16 # Unless required by applicable law or agreed to in writing, software
17 # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
18 # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
19 # License for the specific language governing permissions and limitations
22 # For those usages not covered by the Apache License, Version 2.0 please
23 # contact with: <highperformance-networks@bristol.ac.uk>
25 # Neither the name of the University of Bristol nor the names of its
26 # contributors may be used to endorse or promote products derived from
27 # this software without specific prior written permission.
29 # This work has been performed in the context of DCMS UK 5G Testbeds
30 # & Trials Programme and in the framework of the Metro-Haul project -
31 # funded by the European Commission under Grant number 761727 through the
32 # Horizon 2020 and 5G-PPP programmes.
35 """This module contains only logic related to managing records in a database
36 which includes data format normalization, data format validation and etc.
37 (It works as an extension to `nfvo_db.py` for the WIM feature)
39 No domain logic/architectural concern should be present in this file.
43 from contextlib
import contextmanager
44 from hashlib
import sha1
45 from itertools
import groupby
46 from operator
import itemgetter
47 from sys
import exc_info
48 from threading
import Lock
50 from uuid
import uuid1
as generate_uuid
52 from six
import reraise
58 convert_float_timestamp2str
,
71 UndefinedWanMappingType
,
72 UnexpectedDatabaseError
,
74 WimAndTenantAlreadyAttached
81 ' JOIN wim_nfvo_tenants AS association '
82 ' ON association.wim_id=wim.uuid '
83 ' JOIN nfvo_tenants AS nfvo_tenant '
84 ' ON association.nfvo_tenant_id=nfvo_tenant.uuid '
85 ' JOIN wim_accounts AS wim_account '
86 ' ON association.wim_account_id=wim_account.uuid '
90 'wim_accounts AS wim_account '
91 ' JOIN wim_nfvo_tenants AS association '
92 ' ON association.wim_account_id=wim_account.uuid '
94 ' ON association.wim_id=wim.uuid '
95 ' JOIN nfvo_tenants AS nfvo_tenant '
96 ' ON association.nfvo_tenant_id=nfvo_tenant.uuid '
100 'datacenters AS datacenter '
101 ' JOIN tenants_datacenters AS association '
102 ' ON association.datacenter_id=datacenter.uuid '
103 ' JOIN datacenter_tenants as datacenter_account '
104 ' ON association.datacenter_tenant_id=datacenter_account.uuid '
105 ' JOIN nfvo_tenants AS nfvo_tenant '
106 ' ON association.nfvo_tenant_id=nfvo_tenant.uuid '
109 _PORT_MAPPING
= 'wim_port_mappings as wim_port_mapping '
111 _PORT_MAPPING_JOIN_WIM
= (
113 ' ON wim_port_mapping.wim_id=wim.uuid '
116 _PORT_MAPPING_JOIN_DATACENTER
= (
117 ' JOIN datacenters as datacenter '
118 ' ON wim_port_mapping.datacenter_id=datacenter.uuid '
122 'wim.{0} as {0}'.format(_field
)
123 for _field
in 'uuid name description wim_url type config '
124 'created_at modified_at'.split()
127 _WIM_ACCOUNT_SELECT
= 'uuid name user password config'.split()
129 _PORT_MAPPING_SELECT
= ('wim_port_mapping.*', )
131 _CONFIDENTIAL_FIELDS
= ('password', 'passwd')
133 _SERIALIZED_FIELDS
= ('config', 'vim_info', 'wim_info', 'conn_info', 'extra',
134 'wan_service_mapping_info')
136 UNIQUE_PORT_MAPPING_INFO_FIELDS
= {
137 'dpid-port': ('wan_switch_dpid', 'wan_switch_port')
139 """Fields that should be unique for each port mapping that relies on
140 wan_service_mapping_info.
142 For example, for port mappings of type 'dpid-port', each combination of
143 wan_switch_dpid and wan_switch_port should be unique (the same switch cannot
144 be connected to two different places using the same port)
148 class WimPersistence(object):
149 """High level interactions with the WIM tables in the database"""
151 def __init__(self
, db
, logger
=None, lock
=None):
153 self
.logger
= logger
or logging
.getLogger('openmano.wim.persistence')
154 self
.lock
= lock
or Lock()
164 error_if_multiple
=False,
166 hide
=_CONFIDENTIAL_FIELDS
,
168 """Retrieve records from the database.
171 SELECT, FROM, WHERE, LIMIT, ORDER_BY: used to compose the SQL
172 query. See ``nfvo_db.get_rows``.
173 OFFSET: only valid when used togheter with LIMIT.
174 Ignore the OFFSET first results of the query.
175 error_if_none: by default an error is raised if no record is
176 found. With this option it is possible to disable this error.
177 error_if_multiple: by default no error is raised if more then one
179 With this option it is possible to enable this error.
180 postprocess: function applied to every retrieved record.
181 This function receives a dict as input and must return it
182 after modifications. Moreover this function should accept a
183 second optional parameter ``hide`` indicating
184 the confidential fiels to be obfuscated.
185 By default a minimal postprocessing function is applied,
186 obfuscating confidential fields and converting timestamps.
187 hide: option proxied to postprocess
189 All the remaining keyword arguments will be assumed to be ``name``s or
190 ``uuid``s to compose the WHERE statement, according to their format.
191 If the value corresponds to an array, the first element will determine
192 if it is an name or UUID.
195 - ``wim="abcdef"``` will be turned into ``wim.name="abcdef"``,
196 - ``datacenter="5286a274-8a1b-4b8d-a667-9c94261ad855"``
198 ``datacenter.uuid="5286a274-8a1b-4b8d-a667-9c94261ad855"``.
199 - ``wim=["5286a274-8a1b-4b8d-a667-9c94261ad855", ...]``
201 ``wim.uuid=["5286a274-8a1b-4b8d-a667-9c94261ad855", ...]``
204 NoRecordFound: if the query result set is empty
205 DbBaseException: errors occuring during the execution of the query.
208 postprocess
= postprocess
or _postprocess_record
211 # Find remaining keywords by name or uuid
212 WHERE
.update(_compose_where_from_uuids_or_names(**kwargs
))
213 WHERE
= WHERE
or None
214 # ^ If the where statement is empty, it is better to leave it as None,
215 # so it can be filtered out at a later stage
216 LIMIT
= ('{:d},{:d}'.format(OFFSET
, LIMIT
)
217 if LIMIT
and OFFSET
else LIMIT
)
219 query
= remove_none_items({
220 'SELECT': SELECT
, 'FROM': FROM
, 'WHERE': WHERE
,
221 'LIMIT': LIMIT
, 'ORDER_BY': ORDER_BY
})
224 records
= self
.db
.get_rows(**query
)
226 table
= FROM
.split()[0]
227 if error_if_none
and not records
:
228 raise NoRecordFound(WHERE
, table
)
230 if error_if_multiple
and len(records
) > 1:
231 self
.logger
.error('Multiple records '
232 'FROM %s WHERE %s:\n\n%s\n\n',
233 FROM
, WHERE
, json
.dumps(records
, indent
=4))
234 raise MultipleRecordsFound(WHERE
, table
)
237 expand_joined_fields(postprocess(record
, hide
))
238 for record
in records
241 def query_one(self
, *args
, **kwargs
):
242 """Similar to ``query``, but ensuring just one result.
243 ``error_if_multiple`` is enabled by default.
245 kwargs
.setdefault('error_if_multiple', True)
246 records
= self
.query(*args
, **kwargs
)
247 return records
[0] if records
else None
249 def get_by_uuid(self
, table
, uuid
, **kwargs
):
250 """Retrieve one record from the database based on its uuid
253 table (str): table name (to be used in SQL's FROM statement).
254 uuid (str): unique identifier for record.
256 For additional keyword arguments and exceptions see :obj:`~.query`
257 (``error_if_multiple`` is enabled by default).
260 raise UndefinedUuidOrName(table
)
261 return self
.query_one(table
, WHERE
={'uuid': uuid
}, **kwargs
)
263 def get_by_name_or_uuid(self
, table
, uuid_or_name
, **kwargs
):
264 """Retrieve a record from the database based on a value that can be its
268 table (str): table name (to be used in SQL's FROM statement).
269 uuid_or_name (str): this value can correspond to either uuid or
271 For additional keyword arguments and exceptions see :obj:`~.query`
272 (``error_if_multiple`` is enabled by default).
274 if uuid_or_name
is None:
275 raise UndefinedUuidOrName(table
)
277 key
= 'uuid' if check_valid_uuid(uuid_or_name
) else 'name'
278 return self
.query_one(table
, WHERE
={key
: uuid_or_name
}, **kwargs
)
280 def get_wims(self
, uuid_or_name
=None, tenant
=None, **kwargs
):
281 """Retrieve information about one or more WIMs stored in the database
284 uuid_or_name (str): uuid or name for WIM
285 tenant (str): [optional] uuid or name for NFVO tenant
287 See :obj:`~.query` for additional keyword arguments.
289 kwargs
.update(wim
=uuid_or_name
, tenant
=tenant
)
290 from_
= _WIM_JOIN
if tenant
else _WIM
291 select_
= _WIM_SELECT
[:] + (['wim_account.*'] if tenant
else [])
293 kwargs
.setdefault('SELECT', select_
)
294 return self
.query(from_
, **kwargs
)
296 def get_wim(self
, wim
, tenant
=None, **kwargs
):
297 """Similar to ``get_wims`` but ensure only one result is returned"""
298 kwargs
.setdefault('error_if_multiple', True)
299 return self
.get_wims(wim
, tenant
)[0]
301 def create_wim(self
, wim_descriptor
):
302 """Create a new wim record inside the database and returns its uuid
305 wim_descriptor (dict): properties of the record
306 (usually each field corresponds to a database column, but extra
307 information can be offloaded to another table or serialized as
310 str: UUID of the created WIM
312 if "config" in wim_descriptor
:
313 wim_descriptor
["config"] = _serialize(wim_descriptor
["config"])
316 return self
.db
.new_row(
317 "wims", wim_descriptor
, add_uuid
=True, confidential_data
=True)
319 def update_wim(self
, uuid_or_name
, wim_descriptor
):
320 """Change an existing WIM record on the database"""
321 # obtain data, check that only one exist
322 wim
= self
.get_by_name_or_uuid('wims', uuid_or_name
)
326 where
= {'uuid': wim
['uuid']}
328 # unserialize config, edit and serialize it again
329 new_config_dict
= wim_descriptor
.get('config', {}) or {}
330 config_dict
= remove_none_items(merge_dicts(
331 wim
.get('config', {}) or {}, new_config_dict
))
332 wim_descriptor
['config'] = (
333 _serialize(config_dict
) if config_dict
else None)
336 self
.db
.update_rows('wims', wim_descriptor
, where
)
340 def delete_wim(self
, wim
):
341 # get nfvo_tenant info
342 wim
= self
.get_by_name_or_uuid('wims', wim
)
345 self
.db
.delete_row_by_id('wims', wim
['uuid'])
347 return wim
['uuid'] + ' ' + wim
['name']
349 def get_wim_accounts_by(self
, wim
=None, tenant
=None, uuid
=None, **kwargs
):
350 """Retrieve WIM account information from the database together
351 with the related records (wim, nfvo_tenant and wim_nfvo_tenant)
354 wim (str): uuid or name for WIM
355 tenant (str): [optional] uuid or name for NFVO tenant
357 See :obj:`~.query` for additional keyword arguments.
359 kwargs
.update(wim
=wim
, tenant
=tenant
)
360 kwargs
.setdefault('postprocess', _postprocess_wim_account
)
362 kwargs
.setdefault('WHERE', {'wim_account.uuid': uuid
})
363 return self
.query(FROM
=_WIM_ACCOUNT_JOIN
, **kwargs
)
365 def get_wim_account_by(self
, wim
=None, tenant
=None, uuid
=None, **kwargs
):
366 """Similar to ``get_wim_accounts_by``, but ensuring just one result"""
367 kwargs
.setdefault('error_if_multiple', True)
368 return self
.get_wim_accounts_by(wim
, tenant
, uuid
, **kwargs
)[0]
370 def get_wim_accounts(self
, **kwargs
):
371 """Retrieve all the accounts from the database"""
372 kwargs
.setdefault('postprocess', _postprocess_wim_account
)
373 return self
.query(FROM
=_WIM_ACCOUNT_JOIN
, **kwargs
)
375 def get_wim_account(self
, uuid_or_name
, **kwargs
):
376 """Retrieve WIM Account record by UUID or name,
377 See :obj:`get_by_name_or_uuid` for keyword arguments.
379 kwargs
.setdefault('postprocess', _postprocess_wim_account
)
380 kwargs
.setdefault('SELECT', _WIM_ACCOUNT_SELECT
)
381 return self
.get_by_name_or_uuid('wim_accounts', uuid_or_name
, **kwargs
)
384 def _associate(self
, wim_id
, nfvo_tenant_id
):
385 """Auxiliary method for ``create_wim_account``
387 This method just create a row in the association table
393 except DbBaseException
as db_exception
:
394 error_msg
= str(db_exception
)
395 if all([msg
in error_msg
396 for msg
in ("already in use", "'wim_nfvo_tenant'")]):
397 ex
= WimAndTenantAlreadyAttached(wim_id
, nfvo_tenant_id
)
398 reraise(ex
.__class
__, ex
, exc_info()[2])
402 def create_wim_account(self
, wim
, tenant
, properties
):
403 """Associate a wim to a tenant using the ``wim_nfvo_tenants`` table
404 and create a ``wim_account`` to store credentials and configurations.
406 For the sake of simplification, we assume that each NFVO tenant can be
407 attached to a WIM using only one WIM account. This is automatically
408 guaranteed via database constraints.
409 For corner cases, the same WIM can be registered twice using another
413 wim (str): name or uuid of the WIM related to the account being
415 tenant (str): name or uuid of the nfvo tenant to which the account
417 properties (dict): properties of the account
418 (eg. user, password, ...)
420 wim_id
= self
.get_by_name_or_uuid('wims', wim
, SELECT
=['uuid'])['uuid']
421 tenant
= self
.get_by_name_or_uuid('nfvo_tenants', tenant
,
422 SELECT
=['uuid', 'name'])
423 account
= properties
.setdefault('name', tenant
['name'])
425 wim_account
= self
.query_one('wim_accounts',
426 WHERE
={'wim_id': wim_id
, 'name': account
},
432 if wim_account
is None:
433 # If a row for the wim account doesn't exist yet, we need to
434 # create one, otherwise we can just re-use it.
435 account_id
= str(generate_uuid())
436 used_uuids
.append(account_id
)
437 row
= merge_dicts(properties
, wim_id
=wim_id
, uuid
=account_id
)
438 transaction
.append({'wim_accounts': _preprocess_wim_account(row
)})
440 account_id
= wim_account
['uuid']
441 properties
.pop('config', None) # Config is too complex to compare
442 diff
= {k
: v
for k
, v
in properties
.items() if v
!= wim_account
[k
]}
444 tip
= 'Edit the account first, and then attach it to a tenant'
445 raise WimAccountOverwrite(wim_account
, diff
, tip
)
448 'wim_nfvo_tenants': {'nfvo_tenant_id': tenant
['uuid'],
450 'wim_account_id': account_id
}})
452 with self
._associate
(wim_id
, tenant
['uuid']):
453 self
.db
.new_rows(transaction
, used_uuids
, confidential_data
=True)
457 def update_wim_account(self
, uuid
, properties
, hide
=_CONFIDENTIAL_FIELDS
):
458 """Update WIM account record by overwriting fields with new values
460 Specially for the field ``config`` this means that a new dict will be
461 merged to the existing one.
464 uuid (str): UUID for the WIM account
465 properties (dict): fields that should be overwritten
470 wim_account
= self
.get_by_uuid('wim_accounts', uuid
)
471 safe_fields
= 'user password name created'.split()
472 updates
= _preprocess_wim_account(
473 merge_dicts(wim_account
, filter_dict_keys(properties
, safe_fields
))
476 if properties
.get('config'):
477 old_config
= wim_account
.get('config') or {}
478 new_config
= merge_dicts(old_config
, properties
['config'])
479 updates
['config'] = _serialize(new_config
)
482 num_changes
= self
.db
.update_rows(
483 'wim_accounts', UPDATE
=updates
,
484 WHERE
={'uuid': wim_account
['uuid']})
486 if num_changes
is None:
487 raise UnexpectedDatabaseError('Impossible to update wim_account '
488 '{name}:{uuid}'.format(*wim_account
))
490 return self
.get_wim_account(wim_account
['uuid'], hide
=hide
)
492 def delete_wim_account(self
, uuid
):
493 """Remove WIM account record from the database"""
494 # Since we have foreign keys configured with ON CASCADE, we can rely
495 # on the database engine to guarantee consistency, deleting the
498 return self
.db
.delete_row_by_id('wim_accounts', uuid
)
500 def get_datacenters_by(self
, datacenter
=None, tenant
=None, **kwargs
):
501 """Retrieve datacenter information from the database together
502 with the related records (nfvo_tenant)
505 datacenter (str): uuid or name for datacenter
506 tenant (str): [optional] uuid or name for NFVO tenant
508 See :obj:`~.query` for additional keyword arguments.
511 kwargs
.update(datacenter
=datacenter
, tenant
=tenant
)
512 return self
.query(_DATACENTER_JOIN
, **kwargs
)
514 return [self
.get_by_name_or_uuid('datacenters',
515 datacenter
, **kwargs
)]
517 def get_datacenter_by(self
, datacenter
=None, tenant
=None, **kwargs
):
518 """Similar to ``get_datacenters_by``, but ensuring just one result"""
519 kwargs
.setdefault('error_if_multiple', True)
520 return self
.get_datacenters_by(datacenter
, tenant
, **kwargs
)[0]
522 def _create_single_port_mapping(self
, properties
):
523 info
= properties
.setdefault('wan_service_mapping_info', {})
524 endpoint_id
= properties
.get('wan_service_endpoint_id')
526 if info
.get('mapping_type') and not endpoint_id
:
527 properties
['wan_service_endpoint_id'] = (
528 self
._generate
_port
_mapping
_id
(info
))
530 properties
['wan_service_mapping_info'] = _serialize(info
)
534 self
.db
.new_row('wim_port_mappings', properties
,
535 add_uuid
=False, confidential_data
=True)
536 except DbBaseException
as old_exception
:
537 self
.logger
.exception(old_exception
)
538 ex
= InvalidParameters(
539 "The mapping must contain the "
540 "'pop_switch_dpid', 'pop_switch_port', and "
541 "wan_service_mapping_info: "
542 "('wan_switch_dpid' and 'wan_switch_port') or "
543 "'wan_service_endpoint_id}'")
544 reraise(ex
.__class
__, ex
, exc_info()[2])
548 def create_wim_port_mappings(self
, wim
, port_mappings
, tenant
=None):
549 if not isinstance(wim
, dict):
550 wim
= self
.get_by_name_or_uuid('wims', wim
)
552 for port_mapping
in port_mappings
:
553 port_mapping
['wim_name'] = wim
['name']
554 datacenter
= self
.get_datacenter_by(
555 port_mapping
['datacenter_name'], tenant
)
556 for pop_wan_port_mapping
in port_mapping
['pop_wan_mappings']:
557 element
= merge_dicts(pop_wan_port_mapping
, {
558 'wim_id': wim
['uuid'],
559 'datacenter_id': datacenter
['uuid']})
560 self
._create
_single
_port
_mapping
(element
)
564 def _filter_port_mappings_by_tenant(self
, mappings
, tenant
):
565 """Make sure all the datacenters and wims listed in the port mapping
566 belong to an specific tenant
569 # NOTE: Theoretically this could be done at SQL level, but given the
570 # number of tables involved (wim_port_mappings, wim_accounts,
571 # wims, wim_nfvo_tenants, datacenters, datacenter_tenants,
572 # tenants_datacents and nfvo_tenants), it would result in a
573 # extremely complex query. Moreover, the predicate can vary:
574 # for `get_wim_port_mappings` we can have any combination of
575 # (wim, datacenter, tenant), not all of them having the 3 values
576 # so we have combinatorial trouble to write the 'FROM' statement.
578 kwargs
= {'tenant': tenant
, 'error_if_none': False}
579 # Cache results to speedup things
583 def _get_datacenter(uuid
):
585 datacenters
.get(uuid
) or
586 datacenters
.setdefault(
587 uuid
, self
.get_datacenters_by(uuid
, **kwargs
)))
590 return (wims
.get(uuid
) or
591 wims
.setdefault(uuid
, self
.get_wims(uuid
, **kwargs
)))
595 for mapping
in mappings
596 if (_get_datacenter(mapping
['datacenter_id']) and
597 _get_wims(mapping
['wim_id']))
600 def get_wim_port_mappings(self
, wim
=None, datacenter
=None, tenant
=None,
602 """List all the port mappings, optionally filtering by wim, datacenter
605 from_
= [_PORT_MAPPING
,
606 _PORT_MAPPING_JOIN_WIM
if wim
else '',
607 _PORT_MAPPING_JOIN_DATACENTER
if datacenter
else '']
609 criteria
= ('wim_id', 'datacenter_id')
610 kwargs
.setdefault('error_if_none', False)
611 mappings
= self
.query(
613 SELECT
=_PORT_MAPPING_SELECT
,
614 ORDER_BY
=['wim_port_mapping.{}'.format(c
) for c
in criteria
],
615 wim
=wim
, datacenter
=datacenter
,
616 postprocess
=_postprocess_wim_port_mapping
,
620 mappings
= self
._filter
_port
_mappings
_by
_tenant
(mappings
, tenant
)
622 # We don't have to sort, since we have used 'ORDER_BY'
623 grouped_mappings
= groupby(mappings
, key
=itemgetter(*criteria
))
627 'datacenter_id': key
[1],
628 'pop_wan_mappings': [
629 filter_out_dict_keys(mapping
, (
630 'id', 'wim_id', 'datacenter_id',
631 'created_at', 'modified_at'))
632 for mapping
in group
]}
633 for key
, group
in grouped_mappings
636 def delete_wim_port_mappings(self
, wim_id
):
638 self
.db
.delete_row(FROM
='wim_port_mappings',
639 WHERE
={"wim_id": wim_id
})
640 return "port mapping for wim {} deleted.".format(wim_id
)
642 def update_wim_port_mapping(self
, id, properties
):
643 original
= self
.query_one('wim_port_mappings', WHERE
={'id': id})
645 mapping_info
= remove_none_items(merge_dicts(
646 original
.get('wan_service_mapping_info') or {},
647 properties
.get('wan_service_mapping_info') or {}))
649 updates
= preprocess_record(
650 merge_dicts(original
, remove_none_items(properties
),
651 wan_service_mapping_info
=mapping_info
))
654 num_changes
= self
.db
.update_rows(
655 'wim_port_mappings', UPDATE
=updates
, WHERE
={'id': id})
657 if num_changes
is None:
658 raise UnexpectedDatabaseError(
659 'Impossible to update wim_port_mappings %s:\n%s\n',
660 id, _serialize(properties
))
664 def get_actions_in_groups(self
, wim_account_id
,
665 item_types
=('instance_wim_nets',),
666 group_offset
=0, group_limit
=150):
667 """Retrieve actions from the database in groups.
668 Each group contains all the actions that have the same ``item`` type
672 wim_account_id: restrict the search to actions to be performed
673 using the same account
674 item_types (list): [optional] filter the actions to the given
676 group_limit (int): maximum number of groups returned by the
678 group_offset (int): skip the N first groups. Used together with
679 group_limit for pagination purposes.
682 List of groups, where each group is a tuple ``(key, actions)``.
683 In turn, ``key`` is a tuple containing the values of
684 ``(item, item_id)`` used to create the group and ``actions`` is a
685 list of ``vim_wim_actions`` records (dicts).
689 '"{}"'.format(self
.db
.escape_string(t
)) for t
in item_types
)
691 items
= ('SELECT DISTINCT a.item, a.item_id, a.wim_account_id '
692 'FROM vim_wim_actions AS a '
693 'WHERE a.wim_account_id="{}" AND a.item IN ({}) '
694 'ORDER BY a.item, a.item_id '
695 'LIMIT {:d},{:d}').format(
696 self
.safe_str(wim_account_id
),
697 ','.join(type_options
),
698 group_offset
, group_limit
701 join
= 'vim_wim_actions NATURAL JOIN ({}) AS items'.format(items
)
703 db_results
= self
.db
.get_rows(
704 FROM
=join
, ORDER_BY
=('item', 'item_id', 'created_at'))
706 results
= (_postprocess_action(r
) for r
in db_results
)
707 criteria
= itemgetter('item', 'item_id')
708 return [(k
, list(g
)) for k
, g
in groupby(results
, key
=criteria
)]
710 def update_action(self
, instance_action_id
, task_index
, properties
):
711 condition
= {'instance_action_id': instance_action_id
,
712 'task_index': task_index
}
713 action
= self
.query_one('vim_wim_actions', WHERE
=condition
)
715 extra
= remove_none_items(merge_dicts(
716 action
.get('extra') or {},
717 properties
.get('extra') or {}))
719 updates
= preprocess_record(
720 merge_dicts(action
, properties
, extra
=extra
))
723 num_changes
= self
.db
.update_rows('vim_wim_actions',
724 UPDATE
=updates
, WHERE
=condition
)
726 if num_changes
is None:
727 raise UnexpectedDatabaseError(
728 'Impossible to update vim_wim_actions '
729 '{instance_action_id}[{task_index}]'.format(*action
))
733 def get_wan_links(self
, uuid
=None, **kwargs
):
734 """Retrieve WAN link records from the database
737 uuid, instance_scenario_id, sce_net_id, wim_id, wim_account_id:
738 attributes that can be used at the WHERE clause
740 kwargs
.setdefault('uuid', uuid
)
741 kwargs
.setdefault('error_if_none', False)
743 criteria_fields
= ('uuid', 'instance_scenario_id', 'sce_net_id',
744 'wim_id', 'wim_account_id')
745 criteria
= remove_none_items(filter_dict_keys(kwargs
, criteria_fields
))
746 kwargs
= filter_out_dict_keys(kwargs
, criteria_fields
)
748 return self
.query('instance_wim_nets', WHERE
=criteria
, **kwargs
)
750 def update_wan_link(self
, uuid
, properties
):
751 wan_link
= self
.get_by_uuid('instance_wim_nets', uuid
)
753 wim_info
= remove_none_items(merge_dicts(
754 wan_link
.get('wim_info') or {},
755 properties
.get('wim_info') or {}))
757 updates
= preprocess_record(
758 merge_dicts(wan_link
, properties
, wim_info
=wim_info
))
760 self
.logger
.debug({'UPDATE': updates
})
762 num_changes
= self
.db
.update_rows(
763 'instance_wim_nets', UPDATE
=updates
,
764 WHERE
={'uuid': wan_link
['uuid']})
766 if num_changes
is None:
767 raise UnexpectedDatabaseError(
768 'Impossible to update instance_wim_nets ' + wan_link
['uuid'])
772 def get_instance_nets(self
, instance_scenario_id
, sce_net_id
, **kwargs
):
773 """Retrieve all the instance nets related to the same instance_scenario
778 WHERE
={'instance_scenario_id': instance_scenario_id
,
779 'sce_net_id': sce_net_id
},
781 'ORDER_BY', ('instance_scenario_id', 'sce_net_id')),
784 def update_instance_action_counters(self
, uuid
, failed
=None, done
=None):
785 """Atomically increment/decrement number_done and number_failed fields
786 in the instance action table
788 changes
= remove_none_items({
789 'number_failed': failed
and {'INCREMENT': failed
},
790 'number_done': done
and {'INCREMENT': done
}
797 return self
.db
.update_rows('instance_actions',
798 WHERE
={'uuid': uuid
}, UPDATE
=changes
)
800 def get_only_vm_with_external_net(self
, instance_net_id
, **kwargs
):
801 """Return an instance VM if that is the only VM connected to an
802 external network identified by instance_net_id
804 counting
= ('SELECT DISTINCT instance_net_id '
805 'FROM instance_interfaces '
806 'WHERE instance_net_id="{}" AND type="external" '
807 'GROUP BY instance_net_id '
808 'HAVING COUNT(*)=1').format(self
.safe_str(instance_net_id
))
810 vm_item
= ('SELECT DISTINCT instance_vm_id '
811 'FROM instance_interfaces NATURAL JOIN ({}) AS a'
814 return self
.query_one(
815 'instance_vms JOIN ({}) as instance_interface '
816 'ON instance_vms.uuid=instance_interface.instance_vm_id'
817 .format(vm_item
), **kwargs
)
819 def safe_str(self
, string
):
820 """Return a SQL safe string"""
821 return self
.db
.escape_string(string
)
823 def _generate_port_mapping_id(self
, mapping_info
):
824 """Given a port mapping represented by a dict with a 'type' field,
825 generate a unique string, in a injective way.
827 mapping_info
= mapping_info
.copy() # Avoid mutating original object
828 mapping_type
= mapping_info
.pop('mapping_type', None)
830 raise UndefinedWanMappingType(mapping_info
)
832 unique_fields
= UNIQUE_PORT_MAPPING_INFO_FIELDS
.get(mapping_type
)
835 mapping_info
= filter_dict_keys(mapping_info
, unique_fields
)
837 self
.logger
.warning('Unique fields for WIM port mapping of type '
838 '%s not defined. Please add a list of fields '
839 'which combination should be unique in '
840 'UNIQUE_PORT_MAPPING_INFO_FIELDS '
841 '(`wim/persistency.py) ', mapping_type
)
843 repeatable_repr
= json
.dumps(mapping_info
, encoding
='utf-8',
844 sort_keys
=True, indent
=False)
846 return ':'.join([mapping_type
, _str2id(repeatable_repr
)])
849 def _serialize(value
):
850 """Serialize an arbitrary value in a consistent way,
851 so it can be stored in a database inside a text field
853 return yaml
.safe_dump(value
, default_flow_style
=True, width
=256)
856 def _unserialize(text
):
857 """Unserialize text representation into an arbitrary value,
858 so it can be loaded from the database
860 return yaml
.safe_load(text
)
863 def preprocess_record(record
):
864 """Small transformations to be applied to the data that cames from the
865 user before writing it to the database. By default, filter out timestamps,
866 and serialize the ``config`` field.
868 automatic_fields
= ['created_at', 'modified_at']
869 record
= serialize_fields(filter_out_dict_keys(record
, automatic_fields
))
874 def _preprocess_wim_account(wim_account
):
875 """Do the default preprocessing and convert the 'created' field from
878 wim_account
= preprocess_record(wim_account
)
880 created
= wim_account
.get('created')
881 wim_account
['created'] = (
882 'true' if created
is True or created
== 'true' else 'false')
887 def _postprocess_record(record
, hide
=_CONFIDENTIAL_FIELDS
):
888 """By default, hide passwords fields, unserialize ``config`` fields, and
889 convert float timestamps to strings
891 record
= hide_confidential_fields(record
, hide
)
892 record
= unserialize_fields(record
, hide
)
894 convert_float_timestamp2str(record
)
899 def _postprocess_action(action
):
900 if action
.get('extra'):
901 action
['extra'] = _unserialize(action
['extra'])
906 def _postprocess_wim_account(wim_account
, hide
=_CONFIDENTIAL_FIELDS
):
907 """Do the default postprocessing and convert the 'created' field from
910 # Fix fields from join
911 for field
in ('type', 'description', 'wim_url'):
912 if field
in wim_account
:
913 wim_account
['wim.'+field
] = wim_account
.pop(field
)
915 for field
in ('id', 'nfvo_tenant_id', 'wim_account_id'):
916 if field
in wim_account
:
917 wim_account
['association.'+field
] = wim_account
.pop(field
)
919 wim_account
= _postprocess_record(wim_account
, hide
)
921 created
= wim_account
.get('created')
922 wim_account
['created'] = (created
is True or created
== 'true')
927 def _postprocess_wim_port_mapping(mapping
, hide
=_CONFIDENTIAL_FIELDS
):
928 mapping
= _postprocess_record(mapping
, hide
=hide
)
929 mapping_info
= mapping
.get('wan_service_mapping_info', None) or {}
930 mapping
['wan_service_mapping_info'] = mapping_info
934 def hide_confidential_fields(record
, fields
=_CONFIDENTIAL_FIELDS
):
935 """Obfuscate confidential fields from the input dict.
938 This function performs a SHALLOW operation.
940 if not(isinstance(record
, dict) and fields
):
943 keys
= record
.iterkeys()
944 keys
= (k
for k
in keys
for f
in fields
if k
== f
or k
.endswith('.'+f
))
946 return merge_dicts(record
, {k
: '********' for k
in keys
if record
[k
]})
949 def unserialize_fields(record
, hide
=_CONFIDENTIAL_FIELDS
,
950 fields
=_SERIALIZED_FIELDS
):
951 """Unserialize fields that where stored in the database as a serialized
954 keys
= record
.iterkeys()
955 keys
= (k
for k
in keys
for f
in fields
if k
== f
or k
.endswith('.'+f
))
957 return merge_dicts(record
, {
958 key
: hide_confidential_fields(_unserialize(record
[key
]), hide
)
959 for key
in keys
if record
[key
]
963 def serialize_fields(record
, fields
=_SERIALIZED_FIELDS
):
964 """Serialize fields to be stored in the database as YAML"""
965 keys
= record
.iterkeys()
966 keys
= (k
for k
in keys
for f
in fields
if k
== f
or k
.endswith('.'+f
))
968 return merge_dicts(record
, {
969 key
: _serialize(record
[key
])
970 for key
in keys
if record
[key
] is not None
974 def _decide_name_or_uuid(value
):
977 if isinstance(value
, (list, tuple)):
978 reference
= value
[0] if value
else ''
980 return 'uuid' if check_valid_uuid(reference
) else 'name'
983 def _compose_where_from_uuids_or_names(**conditions
):
984 """Create a dict containing the right conditions to be used in a database
987 This function chooses between ``names`` and ``uuid`` fields based on the
988 format of the passed string.
989 If a list is passed, the first element of the list will be used to choose
990 the name of the field.
991 If a ``None`` value is passed, ``uuid`` is used.
993 Note that this function automatically translates ``tenant`` to
994 ``nfvo_tenant`` for the sake of brevity.
997 >>> _compose_where_from_uuids_or_names(
999 tenant=['xyz123', 'def456']
1000 datacenter='5286a274-8a1b-4b8d-a667-9c94261ad855')
1001 {'wim.name': 'abcdef',
1002 'nfvo_tenant.name': ['xyz123', 'def456']
1003 'datacenter.uuid': '5286a274-8a1b-4b8d-a667-9c94261ad855'}
1005 if 'tenant' in conditions
:
1006 conditions
['nfvo_tenant'] = conditions
.pop('tenant')
1009 '{}.{}'.format(kind
, _decide_name_or_uuid(value
)): value
1010 for kind
, value
in conditions
.items() if value
1015 """Create an ID (following the UUID format) from a piece of arbitrary
1018 Different texts should generate different IDs, and the same text should
1019 generate the same ID in a repeatable way.
1021 return sha1(text
).hexdigest()