1 # -*- coding: utf-8 -*-
3 # Copyright 2018 University of Bristol - High Performance Networks Research
7 # Contributors: Anderson Bravalheri, Dimitrios Gkounis, Abubakar Siddique
8 # Muqaddas, Navdeep Uniyal, Reza Nejabati and Dimitra Simeonidou
10 # Licensed under the Apache License, Version 2.0 (the "License"); you may
11 # not use this file except in compliance with the License. You may obtain
12 # a copy of the License at
14 # http://www.apache.org/licenses/LICENSE-2.0
16 # Unless required by applicable law or agreed to in writing, software
17 # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
18 # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
19 # License for the specific language governing permissions and limitations
22 # For those usages not covered by the Apache License, Version 2.0 please
23 # contact with: <highperformance-networks@bristol.ac.uk>
25 # Neither the name of the University of Bristol nor the names of its
26 # contributors may be used to endorse or promote products derived from
27 # this software without specific prior written permission.
29 # This work has been performed in the context of DCMS UK 5G Testbeds
30 # & Trials Programme and in the framework of the Metro-Haul project -
31 # funded by the European Commission under Grant number 761727 through the
32 # Horizon 2020 and 5G-PPP programmes.
35 """This module contains only logic related to managing records in a database
36 which includes data format normalization, data format validation and etc.
37 (It works as an extension to `nfvo_db.py` for the WIM feature)
39 No domain logic/architectural concern should be present in this file.
43 from contextlib
import contextmanager
44 from hashlib
import sha1
45 from itertools
import groupby
46 from operator
import itemgetter
47 from sys
import exc_info
48 # from time import time
49 from uuid
import uuid1
as generate_uuid
51 from six
import reraise
57 convert_float_timestamp2str
,
70 UndefinedWanMappingType
,
71 UnexpectedDatabaseError
,
73 WimAndTenantAlreadyAttached
80 ' JOIN wim_nfvo_tenants AS association '
81 ' ON association.wim_id=wim.uuid '
82 ' JOIN nfvo_tenants AS nfvo_tenant '
83 ' ON association.nfvo_tenant_id=nfvo_tenant.uuid '
84 ' JOIN wim_accounts AS wim_account '
85 ' ON association.wim_account_id=wim_account.uuid '
89 'wim_accounts AS wim_account '
90 ' JOIN wim_nfvo_tenants AS association '
91 ' ON association.wim_account_id=wim_account.uuid '
93 ' ON association.wim_id=wim.uuid '
94 ' JOIN nfvo_tenants AS nfvo_tenant '
95 ' ON association.nfvo_tenant_id=nfvo_tenant.uuid '
99 'datacenters AS datacenter '
100 ' JOIN tenants_datacenters AS association '
101 ' ON association.datacenter_id=datacenter.uuid '
102 ' JOIN datacenter_tenants as datacenter_account '
103 ' ON association.datacenter_tenant_id=datacenter_account.uuid '
104 ' JOIN nfvo_tenants AS nfvo_tenant '
105 ' ON association.nfvo_tenant_id=nfvo_tenant.uuid '
108 _PORT_MAPPING
= 'wim_port_mappings as wim_port_mapping '
110 _PORT_MAPPING_JOIN_WIM
= (
112 ' ON wim_port_mapping.wim_id=wim.uuid '
115 _PORT_MAPPING_JOIN_DATACENTER
= (
116 ' JOIN datacenters as datacenter '
117 ' ON wim_port_mapping.datacenter_id=datacenter.uuid '
121 'wim.{0} as {0}'.format(_field
)
122 for _field
in 'uuid name description wim_url type config '
123 'created_at modified_at'.split()
126 _WIM_ACCOUNT_SELECT
= 'uuid name user password config'.split()
128 _PORT_MAPPING_SELECT
= ('wim_port_mapping.*', )
130 _CONFIDENTIAL_FIELDS
= ('password', 'passwd')
132 _SERIALIZED_FIELDS
= ('config', 'vim_info', 'wim_info', 'conn_info', 'extra',
133 'wan_service_mapping_info')
135 UNIQUE_PORT_MAPPING_INFO_FIELDS
= {
136 'dpid-port': ('wan_switch_dpid', 'wan_switch_port')
138 """Fields that should be unique for each port mapping that relies on
139 wan_service_mapping_info.
141 For example, for port mappings of type 'dpid-port', each combination of
142 wan_switch_dpid and wan_switch_port should be unique (the same switch cannot
143 be connected to two different places using the same port)
147 class WimPersistence(object):
148 """High level interactions with the WIM tables in the database"""
150 def __init__(self
, db
, logger
=None):
152 self
.logger
= logger
or logging
.getLogger('openmano.wim.persistence')
162 error_if_multiple
=False,
164 hide
=_CONFIDENTIAL_FIELDS
,
166 """Retrieve records from the database.
169 SELECT, FROM, WHERE, LIMIT, ORDER_BY: used to compose the SQL
170 query. See ``nfvo_db.get_rows``.
171 OFFSET: only valid when used togheter with LIMIT.
172 Ignore the OFFSET first results of the query.
173 error_if_none: by default an error is raised if no record is
174 found. With this option it is possible to disable this error.
175 error_if_multiple: by default no error is raised if more then one
177 With this option it is possible to enable this error.
178 postprocess: function applied to every retrieved record.
179 This function receives a dict as input and must return it
180 after modifications. Moreover this function should accept a
181 second optional parameter ``hide`` indicating
182 the confidential fiels to be obfuscated.
183 By default a minimal postprocessing function is applied,
184 obfuscating confidential fields and converting timestamps.
185 hide: option proxied to postprocess
187 All the remaining keyword arguments will be assumed to be ``name``s or
188 ``uuid``s to compose the WHERE statement, according to their format.
189 If the value corresponds to an array, the first element will determine
190 if it is an name or UUID.
193 - ``wim="abcdef"``` will be turned into ``wim.name="abcdef"``,
194 - ``datacenter="5286a274-8a1b-4b8d-a667-9c94261ad855"``
196 ``datacenter.uuid="5286a274-8a1b-4b8d-a667-9c94261ad855"``.
197 - ``wim=["5286a274-8a1b-4b8d-a667-9c94261ad855", ...]``
199 ``wim.uuid=["5286a274-8a1b-4b8d-a667-9c94261ad855", ...]``
202 NoRecordFound: if the query result set is empty
203 DbBaseException: errors occuring during the execution of the query.
206 postprocess
= postprocess
or _postprocess_record
209 # Find remaining keywords by name or uuid
210 WHERE
.update(_compose_where_from_uuids_or_names(**kwargs
))
211 WHERE
= WHERE
or None
212 # ^ If the where statement is empty, it is better to leave it as None,
213 # so it can be filtered out at a later stage
214 LIMIT
= ('{:d},{:d}'.format(OFFSET
, LIMIT
)
215 if LIMIT
and OFFSET
else LIMIT
)
217 query
= remove_none_items({
218 'SELECT': SELECT
, 'FROM': FROM
, 'WHERE': WHERE
,
219 'LIMIT': LIMIT
, 'ORDER_BY': ORDER_BY
})
221 records
= self
.db
.get_rows(**query
)
223 table
= FROM
.split()[0]
224 if error_if_none
and not records
:
225 raise NoRecordFound(WHERE
, table
)
227 if error_if_multiple
and len(records
) > 1:
228 self
.logger
.error('Multiple records '
229 'FROM %s WHERE %s:\n\n%s\n\n',
230 FROM
, WHERE
, json
.dumps(records
, indent
=4))
231 raise MultipleRecordsFound(WHERE
, table
)
234 expand_joined_fields(postprocess(record
, hide
))
235 for record
in records
238 def query_one(self
, *args
, **kwargs
):
239 """Similar to ``query``, but ensuring just one result.
240 ``error_if_multiple`` is enabled by default.
242 kwargs
.setdefault('error_if_multiple', True)
243 records
= self
.query(*args
, **kwargs
)
244 return records
[0] if records
else None
246 def get_by_uuid(self
, table
, uuid
, **kwargs
):
247 """Retrieve one record from the database based on its uuid
250 table (str): table name (to be used in SQL's FROM statement).
251 uuid (str): unique identifier for record.
253 For additional keyword arguments and exceptions see :obj:`~.query`
254 (``error_if_multiple`` is enabled by default).
257 raise UndefinedUuidOrName(table
)
258 return self
.query_one(table
, WHERE
={'uuid': uuid
}, **kwargs
)
260 def get_by_name_or_uuid(self
, table
, uuid_or_name
, **kwargs
):
261 """Retrieve a record from the database based on a value that can be its
265 table (str): table name (to be used in SQL's FROM statement).
266 uuid_or_name (str): this value can correspond to either uuid or
268 For additional keyword arguments and exceptions see :obj:`~.query`
269 (``error_if_multiple`` is enabled by default).
271 if uuid_or_name
is None:
272 raise UndefinedUuidOrName(table
)
274 key
= 'uuid' if check_valid_uuid(uuid_or_name
) else 'name'
275 return self
.query_one(table
, WHERE
={key
: uuid_or_name
}, **kwargs
)
277 def get_wims(self
, uuid_or_name
=None, tenant
=None, **kwargs
):
278 """Retrieve information about one or more WIMs stored in the database
281 uuid_or_name (str): uuid or name for WIM
282 tenant (str): [optional] uuid or name for NFVO tenant
284 See :obj:`~.query` for additional keyword arguments.
286 kwargs
.update(wim
=uuid_or_name
, tenant
=tenant
)
287 from_
= _WIM_JOIN
if tenant
else _WIM
288 select_
= _WIM_SELECT
[:] + (['wim_account.*'] if tenant
else [])
290 kwargs
.setdefault('SELECT', select_
)
291 return self
.query(from_
, **kwargs
)
293 def get_wim(self
, wim
, tenant
=None, **kwargs
):
294 """Similar to ``get_wims`` but ensure only one result is returned"""
295 kwargs
.setdefault('error_if_multiple', True)
296 return self
.get_wims(wim
, tenant
)[0]
298 def create_wim(self
, wim_descriptor
):
299 """Create a new wim record inside the database and returns its uuid
302 wim_descriptor (dict): properties of the record
303 (usually each field corresponds to a database column, but extra
304 information can be offloaded to another table or serialized as
307 str: UUID of the created WIM
309 if "config" in wim_descriptor
:
310 wim_descriptor
["config"] = _serialize(wim_descriptor
["config"])
312 return self
.db
.new_row(
313 "wims", wim_descriptor
, add_uuid
=True, confidential_data
=True)
315 def update_wim(self
, uuid_or_name
, wim_descriptor
):
316 """Change an existing WIM record on the database"""
317 # obtain data, check that only one exist
318 wim
= self
.get_by_name_or_uuid('wims', uuid_or_name
)
322 where
= {'uuid': wim
['uuid']}
324 # unserialize config, edit and serialize it again
325 new_config_dict
= wim_descriptor
.get('config', {}) or {}
326 config_dict
= remove_none_items(merge_dicts(
327 wim
.get('config', {}) or {}, new_config_dict
))
328 wim_descriptor
['config'] = (
329 _serialize(config_dict
) if config_dict
else None)
331 self
.db
.update_rows('wims', wim_descriptor
, where
)
335 def delete_wim(self
, wim
):
336 # get nfvo_tenant info
337 wim
= self
.get_by_name_or_uuid('wims', wim
)
339 self
.db
.delete_row_by_id('wims', wim
['uuid'])
341 return wim
['uuid'] + ' ' + wim
['name']
343 def get_wim_accounts_by(self
, wim
=None, tenant
=None, uuid
=None, **kwargs
):
344 """Retrieve WIM account information from the database together
345 with the related records (wim, nfvo_tenant and wim_nfvo_tenant)
348 wim (str): uuid or name for WIM
349 tenant (str): [optional] uuid or name for NFVO tenant
351 See :obj:`~.query` for additional keyword arguments.
353 kwargs
.update(wim
=wim
, tenant
=tenant
)
354 kwargs
.setdefault('postprocess', _postprocess_wim_account
)
356 kwargs
.setdefault('WHERE', {'wim_account.uuid': uuid
})
357 return self
.query(FROM
=_WIM_ACCOUNT_JOIN
, **kwargs
)
359 def get_wim_account_by(self
, wim
=None, tenant
=None, uuid
=None, **kwargs
):
360 """Similar to ``get_wim_accounts_by``, but ensuring just one result"""
361 kwargs
.setdefault('error_if_multiple', True)
362 return self
.get_wim_accounts_by(wim
, tenant
, uuid
, **kwargs
)[0]
364 def get_wim_accounts(self
, **kwargs
):
365 """Retrieve all the accounts from the database"""
366 kwargs
.setdefault('postprocess', _postprocess_wim_account
)
367 return self
.query(FROM
=_WIM_ACCOUNT_JOIN
, **kwargs
)
369 def get_wim_account(self
, uuid_or_name
, **kwargs
):
370 """Retrieve WIM Account record by UUID or name,
371 See :obj:`get_by_name_or_uuid` for keyword arguments.
373 kwargs
.setdefault('postprocess', _postprocess_wim_account
)
374 kwargs
.setdefault('SELECT', _WIM_ACCOUNT_SELECT
)
375 return self
.get_by_name_or_uuid('wim_accounts', uuid_or_name
, **kwargs
)
378 def _associate(self
, wim_id
, nfvo_tenant_id
):
379 """Auxiliary method for ``create_wim_account``
381 This method just create a row in the association table
386 except DbBaseException
as db_exception
:
387 error_msg
= str(db_exception
)
388 if all([msg
in error_msg
389 for msg
in ("already in use", "'wim_nfvo_tenant'")]):
390 ex
= WimAndTenantAlreadyAttached(wim_id
, nfvo_tenant_id
)
391 reraise(ex
.__class
__, ex
, exc_info()[2])
395 def create_wim_account(self
, wim
, tenant
, properties
):
396 """Associate a wim to a tenant using the ``wim_nfvo_tenants`` table
397 and create a ``wim_account`` to store credentials and configurations.
399 For the sake of simplification, we assume that each NFVO tenant can be
400 attached to a WIM using only one WIM account. This is automatically
401 guaranteed via database constraints.
402 For corner cases, the same WIM can be registered twice using another
406 wim (str): name or uuid of the WIM related to the account being
408 tenant (str): name or uuid of the nfvo tenant to which the account
410 properties (dict): properties of the account
411 (eg. user, password, ...)
413 wim_id
= self
.get_by_name_or_uuid('wims', wim
, SELECT
=['uuid'])['uuid']
414 tenant
= self
.get_by_name_or_uuid('nfvo_tenants', tenant
,
415 SELECT
=['uuid', 'name'])
416 account
= properties
.setdefault('name', tenant
['name'])
418 wim_account
= self
.query_one('wim_accounts',
419 WHERE
={'wim_id': wim_id
, 'name': account
},
425 if wim_account
is None:
426 # If a row for the wim account doesn't exist yet, we need to
427 # create one, otherwise we can just re-use it.
428 account_id
= str(generate_uuid())
429 used_uuids
.append(account_id
)
430 row
= merge_dicts(properties
, wim_id
=wim_id
, uuid
=account_id
)
431 transaction
.append({'wim_accounts': _preprocess_wim_account(row
)})
433 account_id
= wim_account
['uuid']
434 properties
.pop('config', None) # Config is too complex to compare
435 diff
= {k
: v
for k
, v
in properties
.items() if v
!= wim_account
[k
]}
437 tip
= 'Edit the account first, and then attach it to a tenant'
438 raise WimAccountOverwrite(wim_account
, diff
, tip
)
441 'wim_nfvo_tenants': {'nfvo_tenant_id': tenant
['uuid'],
443 'wim_account_id': account_id
}})
445 with self
._associate
(wim_id
, tenant
['uuid']):
446 self
.db
.new_rows(transaction
, used_uuids
, confidential_data
=True)
450 def update_wim_account(self
, uuid
, properties
, hide
=_CONFIDENTIAL_FIELDS
):
451 """Update WIM account record by overwriting fields with new values
453 Specially for the field ``config`` this means that a new dict will be
454 merged to the existing one.
457 uuid (str): UUID for the WIM account
458 properties (dict): fields that should be overwritten
463 wim_account
= self
.get_by_uuid('wim_accounts', uuid
)
464 safe_fields
= 'user password name created'.split()
465 updates
= _preprocess_wim_account(
466 merge_dicts(wim_account
, filter_dict_keys(properties
, safe_fields
))
469 if properties
.get('config'):
470 old_config
= wim_account
.get('config') or {}
471 new_config
= merge_dicts(old_config
, properties
['config'])
472 updates
['config'] = _serialize(new_config
)
474 num_changes
= self
.db
.update_rows('wim_accounts', UPDATE
=updates
,
475 WHERE
={'uuid': wim_account
['uuid']})
477 if num_changes
is None:
478 raise UnexpectedDatabaseError('Impossible to update wim_account '
479 '{name}:{uuid}'.format(*wim_account
))
481 return self
.get_wim_account(wim_account
['uuid'], hide
=hide
)
483 def delete_wim_account(self
, uuid
):
484 """Remove WIM account record from the database"""
485 # Since we have foreign keys configured with ON CASCADE, we can rely
486 # on the database engine to guarantee consistency, deleting the
488 return self
.db
.delete_row_by_id('wim_accounts', uuid
)
490 def get_datacenters_by(self
, datacenter
=None, tenant
=None, **kwargs
):
491 """Retrieve datacenter information from the database together
492 with the related records (nfvo_tenant)
495 datacenter (str): uuid or name for datacenter
496 tenant (str): [optional] uuid or name for NFVO tenant
498 See :obj:`~.query` for additional keyword arguments.
501 kwargs
.update(datacenter
=datacenter
, tenant
=tenant
)
502 return self
.query(_DATACENTER_JOIN
, **kwargs
)
504 return [self
.get_by_name_or_uuid('datacenters',
505 datacenter
, **kwargs
)]
507 def get_datacenter_by(self
, datacenter
=None, tenant
=None, **kwargs
):
508 """Similar to ``get_datacenters_by``, but ensuring just one result"""
509 kwargs
.setdefault('error_if_multiple', True)
510 return self
.get_datacenters_by(datacenter
, tenant
, **kwargs
)[0]
512 def _create_single_port_mapping(self
, properties
):
513 info
= properties
.setdefault('wan_service_mapping_info', {})
514 endpoint_id
= properties
.get('wan_service_endpoint_id')
516 if info
.get('mapping_type') and not endpoint_id
:
517 properties
['wan_service_endpoint_id'] = (
518 self
._generate
_port
_mapping
_id
(info
))
520 properties
['wan_service_mapping_info'] = _serialize(info
)
523 self
.db
.new_row('wim_port_mappings', properties
,
524 add_uuid
=False, confidential_data
=True)
525 except DbBaseException
as old_exception
:
526 self
.logger
.exception(old_exception
)
527 ex
= InvalidParameters(
528 "The mapping must contain the "
529 "'pop_switch_dpid', 'pop_switch_port', and "
530 "wan_service_mapping_info: "
531 "('wan_switch_dpid' and 'wan_switch_port') or "
532 "'wan_service_endpoint_id}'")
533 reraise(ex
.__class
__, ex
, exc_info()[2])
537 def create_wim_port_mappings(self
, wim
, port_mappings
, tenant
=None):
538 if not isinstance(wim
, dict):
539 wim
= self
.get_by_name_or_uuid('wims', wim
)
541 for port_mapping
in port_mappings
:
542 port_mapping
['wim_name'] = wim
['name']
543 datacenter
= self
.get_datacenter_by(
544 port_mapping
['datacenter_name'], tenant
)
545 for pop_wan_port_mapping
in port_mapping
['pop_wan_mappings']:
546 element
= merge_dicts(pop_wan_port_mapping
, {
547 'wim_id': wim
['uuid'],
548 'datacenter_id': datacenter
['uuid']})
549 self
._create
_single
_port
_mapping
(element
)
553 def _filter_port_mappings_by_tenant(self
, mappings
, tenant
):
554 """Make sure all the datacenters and wims listed in the port mapping
555 belong to an specific tenant
558 # NOTE: Theoretically this could be done at SQL level, but given the
559 # number of tables involved (wim_port_mappings, wim_accounts,
560 # wims, wim_nfvo_tenants, datacenters, datacenter_tenants,
561 # tenants_datacents and nfvo_tenants), it would result in a
562 # extremely complex query. Moreover, the predicate can vary:
563 # for `get_wim_port_mappings` we can have any combination of
564 # (wim, datacenter, tenant), not all of them having the 3 values
565 # so we have combinatorial trouble to write the 'FROM' statement.
567 kwargs
= {'tenant': tenant
, 'error_if_none': False}
568 # Cache results to speedup things
572 def _get_datacenter(uuid
):
574 datacenters
.get(uuid
) or
575 datacenters
.setdefault(
576 uuid
, self
.get_datacenters_by(uuid
, **kwargs
)))
579 return (wims
.get(uuid
) or
580 wims
.setdefault(uuid
, self
.get_wims(uuid
, **kwargs
)))
584 for mapping
in mappings
585 if (_get_datacenter(mapping
['datacenter_id']) and
586 _get_wims(mapping
['wim_id']))
589 def get_wim_port_mappings(self
, wim
=None, datacenter
=None, tenant
=None,
591 """List all the port mappings, optionally filtering by wim, datacenter
594 from_
= [_PORT_MAPPING
,
595 _PORT_MAPPING_JOIN_WIM
if wim
else '',
596 _PORT_MAPPING_JOIN_DATACENTER
if datacenter
else '']
598 criteria
= ('wim_id', 'datacenter_id')
599 kwargs
.setdefault('error_if_none', False)
600 mappings
= self
.query(
602 SELECT
=_PORT_MAPPING_SELECT
,
603 ORDER_BY
=['wim_port_mapping.{}'.format(c
) for c
in criteria
],
604 wim
=wim
, datacenter
=datacenter
,
605 postprocess
=_postprocess_wim_port_mapping
,
609 mappings
= self
._filter
_port
_mappings
_by
_tenant
(mappings
, tenant
)
611 # We don't have to sort, since we have used 'ORDER_BY'
612 grouped_mappings
= groupby(mappings
, key
=itemgetter(*criteria
))
616 'datacenter_id': key
[1],
617 'pop_wan_mappings': [
618 filter_out_dict_keys(mapping
, (
619 'id', 'wim_id', 'datacenter_id',
620 'created_at', 'modified_at'))
621 for mapping
in group
]}
622 for key
, group
in grouped_mappings
625 def delete_wim_port_mappings(self
, wim_id
):
626 self
.db
.delete_row(FROM
='wim_port_mappings', WHERE
={"wim_id": wim_id
})
627 return "port mapping for wim {} deleted.".format(wim_id
)
629 def update_wim_port_mapping(self
, id, properties
):
630 original
= self
.query_one('wim_port_mappings', WHERE
={'id': id})
632 mapping_info
= remove_none_items(merge_dicts(
633 original
.get('wan_service_mapping_info') or {},
634 properties
.get('wan_service_mapping_info') or {}))
636 updates
= preprocess_record(
637 merge_dicts(original
, remove_none_items(properties
),
638 wan_service_mapping_info
=mapping_info
))
640 num_changes
= self
.db
.update_rows('wim_port_mappings',
641 UPDATE
=updates
, WHERE
={'id': id})
643 if num_changes
is None:
644 raise UnexpectedDatabaseError(
645 'Impossible to update wim_port_mappings %s:\n%s\n',
646 id, _serialize(properties
))
650 def get_actions_in_groups(self
, wim_account_id
,
651 item_types
=('instance_wim_nets',),
652 group_offset
=0, group_limit
=150):
653 """Retrieve actions from the database in groups.
654 Each group contains all the actions that have the same ``item`` type
658 wim_account_id: restrict the search to actions to be performed
659 using the same account
660 item_types (list): [optional] filter the actions to the given
662 group_limit (int): maximum number of groups returned by the
664 group_offset (int): skip the N first groups. Used together with
665 group_limit for pagination purposes.
668 List of groups, where each group is a tuple ``(key, actions)``.
669 In turn, ``key`` is a tuple containing the values of
670 ``(item, item_id)`` used to create the group and ``actions`` is a
671 list of ``vim_wim_actions`` records (dicts).
675 '"{}"'.format(self
.db
.escape_string(t
)) for t
in item_types
)
677 items
= ('SELECT DISTINCT a.item, a.item_id, a.wim_account_id '
678 'FROM vim_wim_actions AS a '
679 'WHERE a.wim_account_id="{}" AND a.item IN ({}) '
680 'ORDER BY a.item, a.item_id '
681 'LIMIT {:d},{:d}').format(
682 self
.safe_str(wim_account_id
),
683 ','.join(type_options
),
684 group_offset
, group_limit
)
686 join
= 'vim_wim_actions NATURAL JOIN ({}) AS items'.format(items
)
687 db_results
= self
.db
.get_rows(
688 FROM
=join
, ORDER_BY
=('item', 'item_id', 'created_at'))
690 results
= (_postprocess_action(r
) for r
in db_results
)
691 criteria
= itemgetter('item', 'item_id')
692 return [(k
, list(g
)) for k
, g
in groupby(results
, key
=criteria
)]
694 def update_action(self
, instance_action_id
, task_index
, properties
):
695 condition
= {'instance_action_id': instance_action_id
,
696 'task_index': task_index
}
698 action
= self
.query_one('vim_wim_actions', WHERE
=condition
)
700 actions
= self
.query('vim_wim_actions', WHERE
=condition
)
701 self
.logger
.error('More then one action found:\n%s',
702 json
.dumps(actions
, indent
=4))
705 extra
= remove_none_items(merge_dicts(
706 action
.get('extra') or {},
707 properties
.get('extra') or {}))
709 updates
= preprocess_record(
710 merge_dicts(action
, properties
, extra
=extra
))
712 num_changes
= self
.db
.update_rows('vim_wim_actions', UPDATE
=updates
, WHERE
=condition
)
714 if num_changes
is None:
715 raise UnexpectedDatabaseError(
716 'Impossible to update vim_wim_actions '
717 '{instance_action_id}[{task_index}]'.format(*action
))
721 def get_wan_links(self
, uuid
=None, **kwargs
):
722 """Retrieve WAN link records from the database
725 uuid, instance_scenario_id, sce_net_id, wim_id, wim_account_id:
726 attributes that can be used at the WHERE clause
728 kwargs
.setdefault('uuid', uuid
)
729 kwargs
.setdefault('error_if_none', False)
731 criteria_fields
= ('uuid', 'instance_scenario_id', 'sce_net_id',
732 'wim_id', 'wim_account_id')
733 criteria
= remove_none_items(filter_dict_keys(kwargs
, criteria_fields
))
734 kwargs
= filter_out_dict_keys(kwargs
, criteria_fields
)
736 return self
.query('instance_wim_nets', WHERE
=criteria
, **kwargs
)
738 def update_wan_link(self
, uuid
, properties
):
739 wan_link
= self
.get_by_uuid('instance_wim_nets', uuid
)
741 wim_info
= remove_none_items(merge_dicts(
742 wan_link
.get('wim_info') or {},
743 properties
.get('wim_info') or {}))
745 updates
= preprocess_record(
746 merge_dicts(wan_link
, properties
, wim_info
=wim_info
))
748 self
.logger
.debug({'UPDATE': updates
})
749 num_changes
= self
.db
.update_rows(
750 'instance_wim_nets', UPDATE
=updates
,
751 WHERE
={'uuid': wan_link
['uuid']})
753 if num_changes
is None:
754 raise UnexpectedDatabaseError(
755 'Impossible to update instance_wim_nets ' + wan_link
['uuid'])
759 def get_instance_nets(self
, instance_scenario_id
, sce_net_id
, **kwargs
):
760 """Retrieve all the instance nets related to the same instance_scenario
765 WHERE
={'instance_scenario_id': instance_scenario_id
,
766 'sce_net_id': sce_net_id
},
768 'ORDER_BY', ('instance_scenario_id', 'sce_net_id')),
771 def update_instance_action_counters(self
, uuid
, failed
=None, done
=None):
772 """Atomically increment/decrement number_done and number_failed fields
773 in the instance action table
775 changes
= remove_none_items({
776 'number_failed': failed
and {'INCREMENT': failed
},
777 'number_done': done
and {'INCREMENT': done
}
783 return self
.db
.update_rows('instance_actions', WHERE
={'uuid': uuid
}, UPDATE
=changes
)
785 def get_only_vm_with_external_net(self
, instance_net_id
, **kwargs
):
786 """Return an instance VM if that is the only VM connected to an
787 external network identified by instance_net_id
789 counting
= ('SELECT DISTINCT instance_net_id '
790 'FROM instance_interfaces '
791 'WHERE instance_net_id="{}" AND type="external" '
792 'GROUP BY instance_net_id '
793 'HAVING COUNT(*)=1').format(self
.safe_str(instance_net_id
))
795 vm_item
= ('SELECT DISTINCT instance_vm_id '
796 'FROM instance_interfaces NATURAL JOIN ({}) AS a'
799 return self
.query_one(
800 'instance_vms JOIN ({}) as instance_interface '
801 'ON instance_vms.uuid=instance_interface.instance_vm_id'
802 .format(vm_item
), **kwargs
)
804 def safe_str(self
, string
):
805 """Return a SQL safe string"""
806 return self
.db
.escape_string(string
)
811 def _generate_port_mapping_id(self
, mapping_info
):
812 """Given a port mapping represented by a dict with a 'type' field,
813 generate a unique string, in a injective way.
815 mapping_info
= mapping_info
.copy() # Avoid mutating original object
816 mapping_type
= mapping_info
.pop('mapping_type', None)
818 raise UndefinedWanMappingType(mapping_info
)
820 unique_fields
= UNIQUE_PORT_MAPPING_INFO_FIELDS
.get(mapping_type
)
823 mapping_info
= filter_dict_keys(mapping_info
, unique_fields
)
825 self
.logger
.warning('Unique fields for WIM port mapping of type '
826 '%s not defined. Please add a list of fields '
827 'which combination should be unique in '
828 'UNIQUE_PORT_MAPPING_INFO_FIELDS '
829 '(`wim/persistency.py) ', mapping_type
)
831 repeatable_repr
= json
.dumps(mapping_info
, encoding
='utf-8',
832 sort_keys
=True, indent
=False)
834 return ':'.join([mapping_type
, _str2id(repeatable_repr
)])
837 def _serialize(value
):
838 """Serialize an arbitrary value in a consistent way,
839 so it can be stored in a database inside a text field
841 return yaml
.safe_dump(value
, default_flow_style
=True, width
=256)
844 def _unserialize(text
):
845 """Unserialize text representation into an arbitrary value,
846 so it can be loaded from the database
848 return yaml
.safe_load(text
)
851 def preprocess_record(record
):
852 """Small transformations to be applied to the data that cames from the
853 user before writing it to the database. By default, filter out timestamps,
854 and serialize the ``config`` field.
856 automatic_fields
= ['created_at', 'modified_at']
857 record
= serialize_fields(filter_out_dict_keys(record
, automatic_fields
))
862 def _preprocess_wim_account(wim_account
):
863 """Do the default preprocessing and convert the 'created' field from
866 wim_account
= preprocess_record(wim_account
)
868 created
= wim_account
.get('created')
869 wim_account
['created'] = (
870 'true' if created
is True or created
== 'true' else 'false')
875 def _postprocess_record(record
, hide
=_CONFIDENTIAL_FIELDS
):
876 """By default, hide passwords fields, unserialize ``config`` fields, and
877 convert float timestamps to strings
879 record
= hide_confidential_fields(record
, hide
)
880 record
= unserialize_fields(record
, hide
)
882 convert_float_timestamp2str(record
)
887 def _postprocess_action(action
):
888 if action
.get('extra'):
889 action
['extra'] = _unserialize(action
['extra'])
894 def _postprocess_wim_account(wim_account
, hide
=_CONFIDENTIAL_FIELDS
):
895 """Do the default postprocessing and convert the 'created' field from
898 # Fix fields from join
899 for field
in ('type', 'description', 'wim_url'):
900 if field
in wim_account
:
901 wim_account
['wim.'+field
] = wim_account
.pop(field
)
903 for field
in ('id', 'nfvo_tenant_id', 'wim_account_id'):
904 if field
in wim_account
:
905 wim_account
['association.'+field
] = wim_account
.pop(field
)
907 wim_account
= _postprocess_record(wim_account
, hide
)
909 created
= wim_account
.get('created')
910 wim_account
['created'] = (created
is True or created
== 'true')
915 def _postprocess_wim_port_mapping(mapping
, hide
=_CONFIDENTIAL_FIELDS
):
916 mapping
= _postprocess_record(mapping
, hide
=hide
)
917 mapping_info
= mapping
.get('wan_service_mapping_info', None) or {}
918 mapping
['wan_service_mapping_info'] = mapping_info
922 def hide_confidential_fields(record
, fields
=_CONFIDENTIAL_FIELDS
):
923 """Obfuscate confidential fields from the input dict.
926 This function performs a SHALLOW operation.
928 if not(isinstance(record
, dict) and fields
):
931 keys
= record
.iterkeys()
932 keys
= (k
for k
in keys
for f
in fields
if k
== f
or k
.endswith('.'+f
))
934 return merge_dicts(record
, {k
: '********' for k
in keys
if record
[k
]})
937 def unserialize_fields(record
, hide
=_CONFIDENTIAL_FIELDS
,
938 fields
=_SERIALIZED_FIELDS
):
939 """Unserialize fields that where stored in the database as a serialized
942 keys
= record
.iterkeys()
943 keys
= (k
for k
in keys
for f
in fields
if k
== f
or k
.endswith('.'+f
))
945 return merge_dicts(record
, {
946 key
: hide_confidential_fields(_unserialize(record
[key
]), hide
)
947 for key
in keys
if record
[key
]
951 def serialize_fields(record
, fields
=_SERIALIZED_FIELDS
):
952 """Serialize fields to be stored in the database as YAML"""
953 keys
= record
.iterkeys()
954 keys
= (k
for k
in keys
for f
in fields
if k
== f
or k
.endswith('.'+f
))
956 return merge_dicts(record
, {
957 key
: _serialize(record
[key
])
958 for key
in keys
if record
[key
] is not None
962 def _decide_name_or_uuid(value
):
965 if isinstance(value
, (list, tuple)):
966 reference
= value
[0] if value
else ''
968 return 'uuid' if check_valid_uuid(reference
) else 'name'
971 def _compose_where_from_uuids_or_names(**conditions
):
972 """Create a dict containing the right conditions to be used in a database
975 This function chooses between ``names`` and ``uuid`` fields based on the
976 format of the passed string.
977 If a list is passed, the first element of the list will be used to choose
978 the name of the field.
979 If a ``None`` value is passed, ``uuid`` is used.
981 Note that this function automatically translates ``tenant`` to
982 ``nfvo_tenant`` for the sake of brevity.
985 >>> _compose_where_from_uuids_or_names(
987 tenant=['xyz123', 'def456']
988 datacenter='5286a274-8a1b-4b8d-a667-9c94261ad855')
989 {'wim.name': 'abcdef',
990 'nfvo_tenant.name': ['xyz123', 'def456']
991 'datacenter.uuid': '5286a274-8a1b-4b8d-a667-9c94261ad855'}
993 if 'tenant' in conditions
:
994 conditions
['nfvo_tenant'] = conditions
.pop('tenant')
997 '{}.{}'.format(kind
, _decide_name_or_uuid(value
)): value
998 for kind
, value
in conditions
.items() if value
1003 """Create an ID (following the UUID format) from a piece of arbitrary
1006 Different texts should generate different IDs, and the same text should
1007 generate the same ID in a repeatable way.
1009 return sha1(text
).hexdigest()