1 # -*- coding: utf-8 -*-
3 # Copyright 2018 University of Bristol - High Performance Networks Research
7 # Contributors: Anderson Bravalheri, Dimitrios Gkounis, Abubakar Siddique
8 # Muqaddas, Navdeep Uniyal, Reza Nejabati and Dimitra Simeonidou
10 # Licensed under the Apache License, Version 2.0 (the "License"); you may
11 # not use this file except in compliance with the License. You may obtain
12 # a copy of the License at
14 # http://www.apache.org/licenses/LICENSE-2.0
16 # Unless required by applicable law or agreed to in writing, software
17 # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
18 # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
19 # License for the specific language governing permissions and limitations
22 # For those usages not covered by the Apache License, Version 2.0 please
23 # contact with: <highperformance-networks@bristol.ac.uk>
25 # Neither the name of the University of Bristol nor the names of its
26 # contributors may be used to endorse or promote products derived from
27 # this software without specific prior written permission.
29 # This work has been performed in the context of DCMS UK 5G Testbeds
30 # & Trials Programme and in the framework of the Metro-Haul project -
31 # funded by the European Commission under Grant number 761727 through the
32 # Horizon 2020 and 5G-PPP programmes.
35 """This module contains only logic related to managing records in a database
36 which includes data format normalization, data format validation and etc.
37 (It works as an extension to `nfvo_db.py` for the WIM feature)
39 No domain logic/architectural concern should be present in this file.
43 from contextlib
import contextmanager
44 from hashlib
import sha1
45 from itertools
import groupby
46 from operator
import itemgetter
47 from sys
import exc_info
48 from threading
import Lock
50 from uuid
import uuid1
as generate_uuid
52 from six
import reraise
58 convert_float_timestamp2str
,
71 UndefinedWanMappingType
,
72 UnexpectedDatabaseError
,
74 WimAndTenantAlreadyAttached
81 ' JOIN wim_nfvo_tenants AS association '
82 ' ON association.wim_id=wim.uuid '
83 ' JOIN nfvo_tenants AS nfvo_tenant '
84 ' ON association.nfvo_tenant_id=nfvo_tenant.uuid '
85 ' JOIN wim_accounts AS wim_account '
86 ' ON association.wim_account_id=wim_account.uuid '
90 'wim_accounts AS wim_account '
91 ' JOIN wim_nfvo_tenants AS association '
92 ' ON association.wim_account_id=wim_account.uuid '
94 ' ON association.wim_id=wim.uuid '
95 ' JOIN nfvo_tenants AS nfvo_tenant '
96 ' ON association.nfvo_tenant_id=nfvo_tenant.uuid '
100 'datacenters AS datacenter '
101 ' JOIN tenants_datacenters AS association '
102 ' ON association.datacenter_id=datacenter.uuid '
103 ' JOIN datacenter_tenants as datacenter_account '
104 ' ON association.datacenter_tenant_id=datacenter_account.uuid '
105 ' JOIN nfvo_tenants AS nfvo_tenant '
106 ' ON association.nfvo_tenant_id=nfvo_tenant.uuid '
109 _PORT_MAPPING
= 'wim_port_mappings as wim_port_mapping '
111 _PORT_MAPPING_JOIN_WIM
= (
113 ' ON wim_port_mapping.wim_id=wim.uuid '
116 _PORT_MAPPING_JOIN_DATACENTER
= (
117 ' JOIN datacenters as datacenter '
118 ' ON wim_port_mapping.datacenter_id=datacenter.uuid '
122 'wim.{0} as {0}'.format(_field
)
123 for _field
in 'uuid name description wim_url type config '
124 'created_at modified_at'.split()
127 _WIM_ACCOUNT_SELECT
= 'uuid name user password config'.split()
129 _PORT_MAPPING_SELECT
= ('wim_port_mapping.*', )
131 _CONFIDENTIAL_FIELDS
= ('password', 'passwd')
133 _SERIALIZED_FIELDS
= ('config', 'vim_info', 'wim_info', 'conn_info', 'extra',
134 'wan_service_mapping_info')
136 UNIQUE_PORT_MAPPING_INFO_FIELDS
= {
137 'dpid-port': ('wan_switch_dpid', 'wan_switch_port')
139 """Fields that should be unique for each port mapping that relies on
140 wan_service_mapping_info.
142 For example, for port mappings of type 'dpid-port', each combination of
143 wan_switch_dpid and wan_switch_port should be unique (the same switch cannot
144 be connected to two different places using the same port)
148 class WimPersistence(object):
149 """High level interactions with the WIM tables in the database"""
151 def __init__(self
, db
, logger
=None, lock
=None):
153 self
.logger
= logger
or logging
.getLogger('openmano.wim.persistence')
154 self
.lock
= lock
or Lock()
164 error_if_multiple
=False,
166 hide
=_CONFIDENTIAL_FIELDS
,
168 """Retrieve records from the database.
171 SELECT, FROM, WHERE, LIMIT, ORDER_BY: used to compose the SQL
172 query. See ``nfvo_db.get_rows``.
173 OFFSET: only valid when used togheter with LIMIT.
174 Ignore the OFFSET first results of the query.
175 error_if_none: by default an error is raised if no record is
176 found. With this option it is possible to disable this error.
177 error_if_multiple: by default no error is raised if more then one
179 With this option it is possible to enable this error.
180 postprocess: function applied to every retrieved record.
181 This function receives a dict as input and must return it
182 after modifications. Moreover this function should accept a
183 second optional parameter ``hide`` indicating
184 the confidential fiels to be obfuscated.
185 By default a minimal postprocessing function is applied,
186 obfuscating confidential fields and converting timestamps.
187 hide: option proxied to postprocess
189 All the remaining keyword arguments will be assumed to be ``name``s or
190 ``uuid``s to compose the WHERE statement, according to their format.
191 If the value corresponds to an array, the first element will determine
192 if it is an name or UUID.
195 - ``wim="abcdef"``` will be turned into ``wim.name="abcdef"``,
196 - ``datacenter="5286a274-8a1b-4b8d-a667-9c94261ad855"``
198 ``datacenter.uuid="5286a274-8a1b-4b8d-a667-9c94261ad855"``.
199 - ``wim=["5286a274-8a1b-4b8d-a667-9c94261ad855", ...]``
201 ``wim.uuid=["5286a274-8a1b-4b8d-a667-9c94261ad855", ...]``
204 NoRecordFound: if the query result set is empty
205 DbBaseException: errors occuring during the execution of the query.
208 postprocess
= postprocess
or _postprocess_record
211 # Find remaining keywords by name or uuid
212 WHERE
.update(_compose_where_from_uuids_or_names(**kwargs
))
213 WHERE
= WHERE
or None
214 # ^ If the where statement is empty, it is better to leave it as None,
215 # so it can be filtered out at a later stage
216 LIMIT
= ('{:d},{:d}'.format(OFFSET
, LIMIT
)
217 if LIMIT
and OFFSET
else LIMIT
)
219 query
= remove_none_items({
220 'SELECT': SELECT
, 'FROM': FROM
, 'WHERE': WHERE
,
221 'LIMIT': LIMIT
, 'ORDER_BY': ORDER_BY
})
224 records
= self
.db
.get_rows(**query
)
226 table
= FROM
.split()[0]
227 if error_if_none
and not records
:
228 raise NoRecordFound(WHERE
, table
)
230 if error_if_multiple
and len(records
) > 1:
231 self
.logger
.error('Multiple records '
232 'FROM %s WHERE %s:\n\n%s\n\n',
233 FROM
, WHERE
, json
.dumps(records
, indent
=4))
234 raise MultipleRecordsFound(WHERE
, table
)
237 expand_joined_fields(postprocess(record
, hide
))
238 for record
in records
241 def query_one(self
, *args
, **kwargs
):
242 """Similar to ``query``, but ensuring just one result.
243 ``error_if_multiple`` is enabled by default.
245 kwargs
.setdefault('error_if_multiple', True)
246 records
= self
.query(*args
, **kwargs
)
247 return records
[0] if records
else None
249 def get_by_uuid(self
, table
, uuid
, **kwargs
):
250 """Retrieve one record from the database based on its uuid
253 table (str): table name (to be used in SQL's FROM statement).
254 uuid (str): unique identifier for record.
256 For additional keyword arguments and exceptions see :obj:`~.query`
257 (``error_if_multiple`` is enabled by default).
260 raise UndefinedUuidOrName(table
)
261 return self
.query_one(table
, WHERE
={'uuid': uuid
}, **kwargs
)
263 def get_by_name_or_uuid(self
, table
, uuid_or_name
, **kwargs
):
264 """Retrieve a record from the database based on a value that can be its
268 table (str): table name (to be used in SQL's FROM statement).
269 uuid_or_name (str): this value can correspond to either uuid or
271 For additional keyword arguments and exceptions see :obj:`~.query`
272 (``error_if_multiple`` is enabled by default).
274 if uuid_or_name
is None:
275 raise UndefinedUuidOrName(table
)
277 key
= 'uuid' if check_valid_uuid(uuid_or_name
) else 'name'
278 return self
.query_one(table
, WHERE
={key
: uuid_or_name
}, **kwargs
)
280 def get_wims(self
, uuid_or_name
=None, tenant
=None, **kwargs
):
281 """Retrieve information about one or more WIMs stored in the database
284 uuid_or_name (str): uuid or name for WIM
285 tenant (str): [optional] uuid or name for NFVO tenant
287 See :obj:`~.query` for additional keyword arguments.
289 kwargs
.update(wim
=uuid_or_name
, tenant
=tenant
)
290 from_
= _WIM_JOIN
if tenant
else _WIM
291 select_
= _WIM_SELECT
[:] + (['wim_account.*'] if tenant
else [])
293 kwargs
.setdefault('SELECT', select_
)
294 return self
.query(from_
, **kwargs
)
296 def get_wim(self
, wim
, tenant
=None, **kwargs
):
297 """Similar to ``get_wims`` but ensure only one result is returned"""
298 kwargs
.setdefault('error_if_multiple', True)
299 return self
.get_wims(wim
, tenant
)[0]
301 def create_wim(self
, wim_descriptor
):
302 """Create a new wim record inside the database and returns its uuid
305 wim_descriptor (dict): properties of the record
306 (usually each field corresponds to a database column, but extra
307 information can be offloaded to another table or serialized as
310 str: UUID of the created WIM
312 if "config" in wim_descriptor
:
313 wim_descriptor
["config"] = _serialize(wim_descriptor
["config"])
316 return self
.db
.new_row(
317 "wims", wim_descriptor
, add_uuid
=True, confidential_data
=True)
319 def update_wim(self
, uuid_or_name
, wim_descriptor
):
320 """Change an existing WIM record on the database"""
321 # obtain data, check that only one exist
322 wim
= self
.get_by_name_or_uuid('wims', uuid_or_name
)
326 where
= {'uuid': wim
['uuid']}
328 # unserialize config, edit and serialize it again
329 if wim_descriptor
.get('config'):
330 new_config_dict
= wim_descriptor
["config"]
331 config_dict
= remove_none_items(merge_dicts(
332 wim
.get('config') or {}, new_config_dict
))
333 wim_descriptor
['config'] = (
334 _serialize(config_dict
) if config_dict
else None)
337 self
.db
.update_rows('wims', wim_descriptor
, where
)
341 def delete_wim(self
, wim
):
342 # get nfvo_tenant info
343 wim
= self
.get_by_name_or_uuid('wims', wim
)
346 self
.db
.delete_row_by_id('wims', wim
['uuid'])
348 return wim
['uuid'] + ' ' + wim
['name']
350 def get_wim_accounts_by(self
, wim
=None, tenant
=None, uuid
=None, **kwargs
):
351 """Retrieve WIM account information from the database together
352 with the related records (wim, nfvo_tenant and wim_nfvo_tenant)
355 wim (str): uuid or name for WIM
356 tenant (str): [optional] uuid or name for NFVO tenant
358 See :obj:`~.query` for additional keyword arguments.
360 kwargs
.update(wim
=wim
, tenant
=tenant
)
361 kwargs
.setdefault('postprocess', _postprocess_wim_account
)
363 kwargs
.setdefault('WHERE', {'wim_account.uuid': uuid
})
364 return self
.query(FROM
=_WIM_ACCOUNT_JOIN
, **kwargs
)
366 def get_wim_account_by(self
, wim
=None, tenant
=None, **kwargs
):
367 """Similar to ``get_wim_accounts_by``, but ensuring just one result"""
368 kwargs
.setdefault('error_if_multiple', True)
369 return self
.get_wim_accounts_by(wim
, tenant
, **kwargs
)[0]
371 def get_wim_accounts(self
, **kwargs
):
372 """Retrieve all the accounts from the database"""
373 kwargs
.setdefault('postprocess', _postprocess_wim_account
)
374 return self
.query(FROM
=_WIM_ACCOUNT_JOIN
, **kwargs
)
376 def get_wim_account(self
, uuid_or_name
, **kwargs
):
377 """Retrieve WIM Account record by UUID or name,
378 See :obj:`get_by_name_or_uuid` for keyword arguments.
380 kwargs
.setdefault('postprocess', _postprocess_wim_account
)
381 kwargs
.setdefault('SELECT', _WIM_ACCOUNT_SELECT
)
382 return self
.get_by_name_or_uuid('wim_accounts', uuid_or_name
, **kwargs
)
385 def _associate(self
, wim_id
, nfvo_tenant_id
):
386 """Auxiliary method for ``create_wim_account``
388 This method just create a row in the association table
394 except DbBaseException
as db_exception
:
395 error_msg
= str(db_exception
)
396 if all([msg
in error_msg
397 for msg
in ("already in use", "'wim_nfvo_tenant'")]):
398 ex
= WimAndTenantAlreadyAttached(wim_id
, nfvo_tenant_id
)
399 reraise(ex
.__class
__, ex
, exc_info()[2])
403 def create_wim_account(self
, wim
, tenant
, properties
):
404 """Associate a wim to a tenant using the ``wim_nfvo_tenants`` table
405 and create a ``wim_account`` to store credentials and configurations.
407 For the sake of simplification, we assume that each NFVO tenant can be
408 attached to a WIM using only one WIM account. This is automatically
409 guaranteed via database constraints.
410 For corner cases, the same WIM can be registered twice using another
414 wim (str): name or uuid of the WIM related to the account being
416 tenant (str): name or uuid of the nfvo tenant to which the account
418 properties (dict): properties of the account
419 (eg. user, password, ...)
421 wim_id
= self
.get_by_name_or_uuid('wims', wim
, SELECT
=['uuid'])['uuid']
422 tenant
= self
.get_by_name_or_uuid('nfvo_tenants', tenant
,
423 SELECT
=['uuid', 'name'])
424 account
= properties
.setdefault('name', tenant
['name'])
426 wim_account
= self
.query_one('wim_accounts',
427 WHERE
={'wim_id': wim_id
, 'name': account
},
433 if wim_account
is None:
434 # If a row for the wim account doesn't exist yet, we need to
435 # create one, otherwise we can just re-use it.
436 account_id
= str(generate_uuid())
437 used_uuids
.append(account_id
)
438 row
= merge_dicts(properties
, wim_id
=wim_id
, uuid
=account_id
)
439 transaction
.append({'wim_accounts': _preprocess_wim_account(row
)})
441 account_id
= wim_account
['uuid']
442 properties
.pop('config', None) # Config is too complex to compare
443 diff
= {k
: v
for k
, v
in properties
.items() if v
!= wim_account
[k
]}
445 tip
= 'Edit the account first, and then attach it to a tenant'
446 raise WimAccountOverwrite(wim_account
, diff
, tip
)
449 'wim_nfvo_tenants': {'nfvo_tenant_id': tenant
['uuid'],
451 'wim_account_id': account_id
}})
453 with self
._associate
(wim_id
, tenant
['uuid']):
454 self
.db
.new_rows(transaction
, used_uuids
, confidential_data
=True)
458 def update_wim_account(self
, uuid
, properties
, hide
=_CONFIDENTIAL_FIELDS
):
459 """Update WIM account record by overwriting fields with new values
461 Specially for the field ``config`` this means that a new dict will be
462 merged to the existing one.
465 uuid (str): UUID for the WIM account
466 properties (dict): fields that should be overwritten
471 wim_account
= self
.get_by_uuid('wim_accounts', uuid
)
472 safe_fields
= 'user password name created'.split()
473 updates
= _preprocess_wim_account(
474 merge_dicts(wim_account
, filter_dict_keys(properties
, safe_fields
))
477 if properties
.get('config'):
478 old_config
= wim_account
.get('config') or {}
479 new_config
= merge_dicts(old_config
, properties
['config'])
480 updates
['config'] = _serialize(new_config
)
483 num_changes
= self
.db
.update_rows(
484 'wim_accounts', UPDATE
=updates
,
485 WHERE
={'uuid': wim_account
['uuid']})
487 if num_changes
is None:
488 raise UnexpectedDatabaseError('Impossible to update wim_account '
489 '{name}:{uuid}'.format(*wim_account
))
491 return self
.get_wim_account(wim_account
['uuid'], hide
=hide
)
493 def delete_wim_account(self
, uuid
):
494 """Remove WIM account record from the database"""
495 # Since we have foreign keys configured with ON CASCADE, we can rely
496 # on the database engine to guarantee consistency, deleting the
499 return self
.db
.delete_row_by_id('wim_accounts', uuid
)
501 def get_datacenters_by(self
, datacenter
=None, tenant
=None, **kwargs
):
502 """Retrieve datacenter information from the database together
503 with the related records (nfvo_tenant)
506 datacenter (str): uuid or name for datacenter
507 tenant (str): [optional] uuid or name for NFVO tenant
509 See :obj:`~.query` for additional keyword arguments.
511 kwargs
.update(datacenter
=datacenter
, tenant
=tenant
)
512 return self
.query(_DATACENTER_JOIN
, **kwargs
)
514 def get_datacenter_by(self
, datacenter
=None, tenant
=None, **kwargs
):
515 """Similar to ``get_datacenters_by``, but ensuring just one result"""
516 kwargs
.setdefault('error_if_multiple', True)
517 return self
.get_datacenters_by(datacenter
, tenant
, **kwargs
)[0]
519 def _create_single_port_mapping(self
, properties
):
520 info
= properties
.setdefault('wan_service_mapping_info', {})
521 endpoint_id
= properties
.get('wan_service_endpoint_id')
523 if info
.get('mapping_type') and not endpoint_id
:
524 properties
['wan_service_endpoint_id'] = (
525 self
._generate
_port
_mapping
_id
(info
))
527 properties
['wan_service_mapping_info'] = _serialize(info
)
531 self
.db
.new_row('wim_port_mappings', properties
,
532 add_uuid
=False, confidential_data
=True)
533 except DbBaseException
as old_exception
:
534 self
.logger
.exception(old_exception
)
535 ex
= InvalidParameters(
536 "The mapping must contain the "
537 "'pop_switch_dpid', 'pop_switch_port', and "
538 "wan_service_mapping_info: "
539 "('wan_switch_dpid' and 'wan_switch_port') or "
540 "'wan_service_endpoint_id}'")
541 reraise(ex
.__class
__, ex
, exc_info()[2])
545 def create_wim_port_mappings(self
, wim
, port_mappings
, tenant
=None):
546 if not isinstance(wim
, dict):
547 wim
= self
.get_by_name_or_uuid('wims', wim
)
549 for port_mapping
in port_mappings
:
550 port_mapping
['wim_name'] = wim
['name']
551 datacenter
= self
.get_datacenter_by(
552 port_mapping
['datacenter_name'], tenant
)
553 for pop_wan_port_mapping
in port_mapping
['pop_wan_mappings']:
554 element
= merge_dicts(pop_wan_port_mapping
, {
555 'wim_id': wim
['uuid'],
556 'datacenter_id': datacenter
['uuid']})
557 self
._create
_single
_port
_mapping
(element
)
561 def _filter_port_mappings_by_tenant(self
, mappings
, tenant
):
562 """Make sure all the datacenters and wims listed in the port mapping
563 belong to an specific tenant
566 # NOTE: Theoretically this could be done at SQL level, but given the
567 # number of tables involved (wim_port_mappings, wim_accounts,
568 # wims, wim_nfvo_tenants, datacenters, datacenter_tenants,
569 # tenants_datacents and nfvo_tenants), it would result in a
570 # extremely complex query. Moreover, the predicate can vary:
571 # for `get_wim_port_mappings` we can have any combination of
572 # (wim, datacenter, tenant), not all of them having the 3 values
573 # so we have combinatorial trouble to write the 'FROM' statement.
575 kwargs
= {'tenant': tenant
, 'error_if_none': False}
576 # Cache results to speedup things
580 def _get_datacenter(uuid
):
582 datacenters
.get(uuid
) or
583 datacenters
.setdefault(
584 uuid
, self
.get_datacenters_by(uuid
, **kwargs
)))
587 return (wims
.get(uuid
) or
588 wims
.setdefault(uuid
, self
.get_wims(uuid
, **kwargs
)))
592 for mapping
in mappings
593 if (_get_datacenter(mapping
['datacenter_id']) and
594 _get_wims(mapping
['wim_id']))
597 def get_wim_port_mappings(self
, wim
=None, datacenter
=None, tenant
=None,
599 """List all the port mappings, optionally filtering by wim, datacenter
602 from_
= [_PORT_MAPPING
,
603 _PORT_MAPPING_JOIN_WIM
if wim
else '',
604 _PORT_MAPPING_JOIN_DATACENTER
if datacenter
else '']
606 criteria
= ('wim_id', 'datacenter_id')
607 kwargs
.setdefault('error_if_none', False)
608 mappings
= self
.query(
610 SELECT
=_PORT_MAPPING_SELECT
,
611 ORDER_BY
=['wim_port_mapping.{}'.format(c
) for c
in criteria
],
612 wim
=wim
, datacenter
=datacenter
,
613 postprocess
=_postprocess_wim_port_mapping
,
617 mappings
= self
._filter
_port
_mappings
_by
_tenant
(mappings
, tenant
)
619 # We don't have to sort, since we have used 'ORDER_BY'
620 grouped_mappings
= groupby(mappings
, key
=itemgetter(*criteria
))
624 'datacenter_id': key
[1],
625 'wan_pop_port_mappings': [
626 filter_out_dict_keys(mapping
, (
627 'id', 'wim_id', 'datacenter_id',
628 'created_at', 'modified_at'))
629 for mapping
in group
]}
630 for key
, group
in grouped_mappings
633 def delete_wim_port_mappings(self
, wim_id
):
635 self
.db
.delete_row(FROM
='wim_port_mappings',
636 WHERE
={"wim_id": wim_id
})
637 return "port mapping for wim {} deleted.".format(wim_id
)
639 def update_wim_port_mapping(self
, id, properties
):
640 original
= self
.query_one('wim_port_mappings', WHERE
={'id': id})
642 mapping_info
= remove_none_items(merge_dicts(
643 original
.get('wan_service_mapping_info') or {},
644 properties
.get('wan_service_mapping_info') or {}))
646 updates
= preprocess_record(
647 merge_dicts(original
, remove_none_items(properties
),
648 wan_service_mapping_info
=mapping_info
))
651 num_changes
= self
.db
.update_rows(
652 'wim_port_mappings', UPDATE
=updates
, WHERE
={'id': id})
654 if num_changes
is None:
655 raise UnexpectedDatabaseError(
656 'Impossible to update wim_port_mappings %s:\n%s\n',
657 id, _serialize(properties
))
661 def get_actions_in_groups(self
, wim_account_id
,
662 item_types
=('instance_wim_nets',),
663 group_offset
=0, group_limit
=150):
664 """Retrieve actions from the database in groups.
665 Each group contains all the actions that have the same ``item`` type
669 wim_account_id: restrict the search to actions to be performed
670 using the same account
671 item_types (list): [optional] filter the actions to the given
673 group_limit (int): maximum number of groups returned by the
675 group_offset (int): skip the N first groups. Used together with
676 group_limit for pagination purposes.
679 List of groups, where each group is a tuple ``(key, actions)``.
680 In turn, ``key`` is a tuple containing the values of
681 ``(item, item_id)`` used to create the group and ``actions`` is a
682 list of ``vim_wim_actions`` records (dicts).
686 '"{}"'.format(self
.db
.escape_string(t
)) for t
in item_types
)
688 items
= ('SELECT DISTINCT a.item, a.item_id, a.wim_account_id '
689 'FROM vim_wim_actions AS a '
690 'WHERE a.wim_account_id="{}" AND a.item IN ({}) '
691 'ORDER BY a.item, a.item_id '
692 'LIMIT {:d},{:d}').format(
693 self
.safe_str(wim_account_id
),
694 ','.join(type_options
),
695 group_offset
, group_limit
698 join
= 'vim_wim_actions NATURAL JOIN ({}) AS items'.format(items
)
700 db_results
= self
.db
.get_rows(
701 FROM
=join
, ORDER_BY
=('item', 'item_id', 'created_at'))
703 results
= (_postprocess_action(r
) for r
in db_results
)
704 criteria
= itemgetter('item', 'item_id')
705 return [(k
, list(g
)) for k
, g
in groupby(results
, key
=criteria
)]
707 def update_action(self
, instance_action_id
, task_index
, properties
):
708 condition
= {'instance_action_id': instance_action_id
,
709 'task_index': task_index
}
710 action
= self
.query_one('vim_wim_actions', WHERE
=condition
)
712 extra
= remove_none_items(merge_dicts(
713 action
.get('extra') or {},
714 properties
.get('extra') or {}))
716 updates
= preprocess_record(
717 merge_dicts(action
, properties
, extra
=extra
))
720 num_changes
= self
.db
.update_rows('vim_wim_actions',
721 UPDATE
=updates
, WHERE
=condition
)
723 if num_changes
is None:
724 raise UnexpectedDatabaseError(
725 'Impossible to update vim_wim_actions '
726 '{instance_action_id}[{task_index}]'.format(*action
))
730 def get_wan_links(self
, uuid
=None, **kwargs
):
731 """Retrieve WAN link records from the database
734 uuid, instance_scenario_id, sce_net_id, wim_id, wim_account_id:
735 attributes that can be used at the WHERE clause
737 kwargs
.setdefault('uuid', uuid
)
738 kwargs
.setdefault('error_if_none', False)
740 criteria_fields
= ('uuid', 'instance_scenario_id', 'sce_net_id',
741 'wim_id', 'wim_account_id')
742 criteria
= remove_none_items(filter_dict_keys(kwargs
, criteria_fields
))
743 kwargs
= filter_out_dict_keys(kwargs
, criteria_fields
)
745 return self
.query('instance_wim_nets', WHERE
=criteria
, **kwargs
)
747 def update_wan_link(self
, uuid
, properties
):
748 wan_link
= self
.get_by_uuid('instance_wim_nets', uuid
)
750 wim_info
= remove_none_items(merge_dicts(
751 wan_link
.get('wim_info') or {},
752 properties
.get('wim_info') or {}))
754 updates
= preprocess_record(
755 merge_dicts(wan_link
, properties
, wim_info
=wim_info
))
757 self
.logger
.debug({'UPDATE': updates
})
759 num_changes
= self
.db
.update_rows(
760 'instance_wim_nets', UPDATE
=updates
,
761 WHERE
={'uuid': wan_link
['uuid']})
763 if num_changes
is None:
764 raise UnexpectedDatabaseError(
765 'Impossible to update instance_wim_nets ' + wan_link
['uuid'])
769 def get_instance_nets(self
, instance_scenario_id
, sce_net_id
, **kwargs
):
770 """Retrieve all the instance nets related to the same instance_scenario
775 WHERE
={'instance_scenario_id': instance_scenario_id
,
776 'sce_net_id': sce_net_id
},
778 'ORDER_BY', ('instance_scenario_id', 'sce_net_id')),
781 def update_instance_action_counters(self
, uuid
, failed
=None, done
=None):
782 """Atomically increment/decrement number_done and number_failed fields
783 in the instance action table
785 changes
= remove_none_items({
786 'number_failed': failed
and {'INCREMENT': failed
},
787 'number_done': done
and {'INCREMENT': done
}
794 return self
.db
.update_rows('instance_actions',
795 WHERE
={'uuid': uuid
}, UPDATE
=changes
)
797 def get_only_vm_with_external_net(self
, instance_net_id
, **kwargs
):
798 """Return an instance VM if that is the only VM connected to an
799 external network identified by instance_net_id
801 counting
= ('SELECT DISTINCT instance_net_id '
802 'FROM instance_interfaces '
803 'WHERE instance_net_id="{}" AND type="external" '
804 'GROUP BY instance_net_id '
805 'HAVING COUNT(*)=1').format(self
.safe_str(instance_net_id
))
807 vm_item
= ('SELECT DISTINCT instance_vm_id '
808 'FROM instance_interfaces NATURAL JOIN ({}) AS a'
811 return self
.query_one(
812 'instance_vms JOIN ({}) as instance_interface '
813 'ON instance_vms.uuid=instance_interface.instance_vm_id'
814 .format(vm_item
), **kwargs
)
816 def safe_str(self
, string
):
817 """Return a SQL safe string"""
818 return self
.db
.escape_string(string
)
820 def _generate_port_mapping_id(self
, mapping_info
):
821 """Given a port mapping represented by a dict with a 'type' field,
822 generate a unique string, in a injective way.
824 mapping_info
= mapping_info
.copy() # Avoid mutating original object
825 mapping_type
= mapping_info
.pop('mapping_type', None)
827 raise UndefinedWanMappingType(mapping_info
)
829 unique_fields
= UNIQUE_PORT_MAPPING_INFO_FIELDS
.get(mapping_type
)
832 mapping_info
= filter_dict_keys(mapping_info
, unique_fields
)
834 self
.logger
.warning('Unique fields for WIM port mapping of type '
835 '%s not defined. Please add a list of fields '
836 'which combination should be unique in '
837 'UNIQUE_PORT_MAPPING_INFO_FIELDS '
838 '(`wim/persistency.py) ', mapping_type
)
840 repeatable_repr
= json
.dumps(mapping_info
, encoding
='utf-8',
841 sort_keys
=True, indent
=False)
843 return ':'.join([mapping_type
, _str2id(repeatable_repr
)])
846 def _serialize(value
):
847 """Serialize an arbitrary value in a consistent way,
848 so it can be stored in a database inside a text field
850 return yaml
.safe_dump(value
, default_flow_style
=True, width
=256)
853 def _unserialize(text
):
854 """Unserialize text representation into an arbitrary value,
855 so it can be loaded from the database
857 return yaml
.safe_load(text
)
860 def preprocess_record(record
):
861 """Small transformations to be applied to the data that cames from the
862 user before writing it to the database. By default, filter out timestamps,
863 and serialize the ``config`` field.
865 automatic_fields
= ['created_at', 'modified_at']
866 record
= serialize_fields(filter_out_dict_keys(record
, automatic_fields
))
871 def _preprocess_wim_account(wim_account
):
872 """Do the default preprocessing and convert the 'created' field from
875 wim_account
= preprocess_record(wim_account
)
877 created
= wim_account
.get('created')
878 wim_account
['created'] = (
879 'true' if created
is True or created
== 'true' else 'false')
884 def _postprocess_record(record
, hide
=_CONFIDENTIAL_FIELDS
):
885 """By default, hide passwords fields, unserialize ``config`` fields, and
886 convert float timestamps to strings
888 record
= hide_confidential_fields(record
, hide
)
889 record
= unserialize_fields(record
, hide
)
891 convert_float_timestamp2str(record
)
896 def _postprocess_action(action
):
897 if action
.get('extra'):
898 action
['extra'] = _unserialize(action
['extra'])
903 def _postprocess_wim_account(wim_account
, hide
=_CONFIDENTIAL_FIELDS
):
904 """Do the default postprocessing and convert the 'created' field from
907 # Fix fields from join
908 for field
in ('type', 'description', 'wim_url'):
909 if field
in wim_account
:
910 wim_account
['wim.'+field
] = wim_account
.pop(field
)
912 for field
in ('id', 'nfvo_tenant_id', 'wim_account_id'):
913 if field
in wim_account
:
914 wim_account
['association.'+field
] = wim_account
.pop(field
)
916 wim_account
= _postprocess_record(wim_account
, hide
)
918 created
= wim_account
.get('created')
919 wim_account
['created'] = (created
is True or created
== 'true')
924 def _postprocess_wim_port_mapping(mapping
, hide
=_CONFIDENTIAL_FIELDS
):
925 mapping
= _postprocess_record(mapping
, hide
=hide
)
926 mapping_info
= mapping
.get('wan_service_mapping_info', None) or {}
927 mapping
['wan_service_mapping_info'] = mapping_info
931 def hide_confidential_fields(record
, fields
=_CONFIDENTIAL_FIELDS
):
932 """Obfuscate confidential fields from the input dict.
935 This function performs a SHALLOW operation.
937 if not(isinstance(record
, dict) and fields
):
940 keys
= record
.iterkeys()
941 keys
= (k
for k
in keys
for f
in fields
if k
== f
or k
.endswith('.'+f
))
943 return merge_dicts(record
, {k
: '********' for k
in keys
if record
[k
]})
946 def unserialize_fields(record
, hide
=_CONFIDENTIAL_FIELDS
,
947 fields
=_SERIALIZED_FIELDS
):
948 """Unserialize fields that where stored in the database as a serialized
951 keys
= record
.iterkeys()
952 keys
= (k
for k
in keys
for f
in fields
if k
== f
or k
.endswith('.'+f
))
954 return merge_dicts(record
, {
955 key
: hide_confidential_fields(_unserialize(record
[key
]), hide
)
956 for key
in keys
if record
[key
]
960 def serialize_fields(record
, fields
=_SERIALIZED_FIELDS
):
961 """Serialize fields to be stored in the database as YAML"""
962 keys
= record
.iterkeys()
963 keys
= (k
for k
in keys
for f
in fields
if k
== f
or k
.endswith('.'+f
))
965 return merge_dicts(record
, {
966 key
: _serialize(record
[key
])
967 for key
in keys
if record
[key
] is not None
971 def _decide_name_or_uuid(value
):
974 if isinstance(value
, (list, tuple)):
975 reference
= value
[0] if value
else ''
977 return 'uuid' if check_valid_uuid(reference
) else 'name'
980 def _compose_where_from_uuids_or_names(**conditions
):
981 """Create a dict containing the right conditions to be used in a database
984 This function chooses between ``names`` and ``uuid`` fields based on the
985 format of the passed string.
986 If a list is passed, the first element of the list will be used to choose
987 the name of the field.
988 If a ``None`` value is passed, ``uuid`` is used.
990 Note that this function automatically translates ``tenant`` to
991 ``nfvo_tenant`` for the sake of brevity.
994 >>> _compose_where_from_uuids_or_names(
996 tenant=['xyz123', 'def456']
997 datacenter='5286a274-8a1b-4b8d-a667-9c94261ad855')
998 {'wim.name': 'abcdef',
999 'nfvo_tenant.name': ['xyz123', 'def456']
1000 'datacenter.uuid': '5286a274-8a1b-4b8d-a667-9c94261ad855'}
1002 if 'tenant' in conditions
:
1003 conditions
['nfvo_tenant'] = conditions
.pop('tenant')
1006 '{}.{}'.format(kind
, _decide_name_or_uuid(value
)): value
1007 for kind
, value
in conditions
.items() if value
1012 """Create an ID (following the UUID format) from a piece of arbitrary
1015 Different texts should generate different IDs, and the same text should
1016 generate the same ID in a repeatable way.
1018 return sha1(text
).hexdigest()