1 # -*- coding: utf-8 -*-
3 # Copyright 2018 University of Bristol - High Performance Networks Research
7 # Contributors: Anderson Bravalheri, Dimitrios Gkounis, Abubakar Siddique
8 # Muqaddas, Navdeep Uniyal, Reza Nejabati and Dimitra Simeonidou
10 # Licensed under the Apache License, Version 2.0 (the "License"); you may
11 # not use this file except in compliance with the License. You may obtain
12 # a copy of the License at
14 # http://www.apache.org/licenses/LICENSE-2.0
16 # Unless required by applicable law or agreed to in writing, software
17 # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
18 # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
19 # License for the specific language governing permissions and limitations
22 # For those usages not covered by the Apache License, Version 2.0 please
23 # contact with: <highperformance-networks@bristol.ac.uk>
25 # Neither the name of the University of Bristol nor the names of its
26 # contributors may be used to endorse or promote products derived from
27 # this software without specific prior written permission.
29 # This work has been performed in the context of DCMS UK 5G Testbeds
30 # & Trials Programme and in the framework of the Metro-Haul project -
31 # funded by the European Commission under Grant number 761727 through the
32 # Horizon 2020 and 5G-PPP programmes.
35 """This module contains only logic related to managing records in a database
36 which includes data format normalization, data format validation and etc.
37 (It works as an extension to `nfvo_db.py` for the WIM feature)
39 No domain logic/architectural concern should be present in this file.
43 from contextlib
import contextmanager
44 from hashlib
import sha1
45 from itertools
import groupby
46 from operator
import itemgetter
47 from sys
import exc_info
49 from uuid
import uuid1
as generate_uuid
51 from six
import reraise
57 convert_float_timestamp2str
,
70 UndefinedWanMappingType
,
71 UnexpectedDatabaseError
,
73 WimAndTenantAlreadyAttached
80 ' JOIN wim_nfvo_tenants AS association '
81 ' ON association.wim_id=wim.uuid '
82 ' JOIN nfvo_tenants AS nfvo_tenant '
83 ' ON association.nfvo_tenant_id=nfvo_tenant.uuid '
84 ' JOIN wim_accounts AS wim_account '
85 ' ON association.wim_account_id=wim_account.uuid '
89 'wim_accounts AS wim_account '
90 ' JOIN wim_nfvo_tenants AS association '
91 ' ON association.wim_account_id=wim_account.uuid '
93 ' ON association.wim_id=wim.uuid '
94 ' JOIN nfvo_tenants AS nfvo_tenant '
95 ' ON association.nfvo_tenant_id=nfvo_tenant.uuid '
99 'datacenters AS datacenter '
100 ' JOIN tenants_datacenters AS association '
101 ' ON association.datacenter_id=datacenter.uuid '
102 ' JOIN datacenter_tenants as datacenter_account '
103 ' ON association.datacenter_tenant_id=datacenter_account.uuid '
104 ' JOIN nfvo_tenants AS nfvo_tenant '
105 ' ON association.nfvo_tenant_id=nfvo_tenant.uuid '
108 _PORT_MAPPING
= 'wim_port_mappings as wim_port_mapping '
110 _PORT_MAPPING_JOIN_WIM
= (
112 ' ON wim_port_mapping.wim_id=wim.uuid '
115 _PORT_MAPPING_JOIN_DATACENTER
= (
116 ' JOIN datacenters as datacenter '
117 ' ON wim_port_mapping.datacenter_id=datacenter.uuid '
121 'wim.{0} as {0}'.format(_field
)
122 for _field
in 'uuid name description wim_url type config '
123 'created_at modified_at'.split()
126 _WIM_ACCOUNT_SELECT
= 'uuid name user password config'.split()
128 _PORT_MAPPING_SELECT
= ('wim_port_mapping.*', )
130 _CONFIDENTIAL_FIELDS
= ('password', 'passwd')
132 _SERIALIZED_FIELDS
= ('config', 'vim_info', 'wim_info', 'conn_info', 'extra',
133 'wan_service_mapping_info')
135 UNIQUE_PORT_MAPPING_INFO_FIELDS
= {
136 'dpid-port': ('wan_switch_dpid', 'wan_switch_port')
138 """Fields that should be unique for each port mapping that relies on
139 wan_service_mapping_info.
141 For example, for port mappings of type 'dpid-port', each combination of
142 wan_switch_dpid and wan_switch_port should be unique (the same switch cannot
143 be connected to two different places using the same port)
147 class WimPersistence(object):
148 """High level interactions with the WIM tables in the database"""
150 def __init__(self
, db
, logger
=None):
152 self
.logger
= logger
or logging
.getLogger('openmano.wim.persistence')
162 error_if_multiple
=False,
164 hide
=_CONFIDENTIAL_FIELDS
,
166 """Retrieve records from the database.
169 SELECT, FROM, WHERE, LIMIT, ORDER_BY: used to compose the SQL
170 query. See ``nfvo_db.get_rows``.
171 OFFSET: only valid when used togheter with LIMIT.
172 Ignore the OFFSET first results of the query.
173 error_if_none: by default an error is raised if no record is
174 found. With this option it is possible to disable this error.
175 error_if_multiple: by default no error is raised if more then one
177 With this option it is possible to enable this error.
178 postprocess: function applied to every retrieved record.
179 This function receives a dict as input and must return it
180 after modifications. Moreover this function should accept a
181 second optional parameter ``hide`` indicating
182 the confidential fiels to be obfuscated.
183 By default a minimal postprocessing function is applied,
184 obfuscating confidential fields and converting timestamps.
185 hide: option proxied to postprocess
187 All the remaining keyword arguments will be assumed to be ``name``s or
188 ``uuid``s to compose the WHERE statement, according to their format.
189 If the value corresponds to an array, the first element will determine
190 if it is an name or UUID.
193 - ``wim="abcdef"``` will be turned into ``wim.name="abcdef"``,
194 - ``datacenter="5286a274-8a1b-4b8d-a667-9c94261ad855"``
196 ``datacenter.uuid="5286a274-8a1b-4b8d-a667-9c94261ad855"``.
197 - ``wim=["5286a274-8a1b-4b8d-a667-9c94261ad855", ...]``
199 ``wim.uuid=["5286a274-8a1b-4b8d-a667-9c94261ad855", ...]``
202 NoRecordFound: if the query result set is empty
203 DbBaseException: errors occuring during the execution of the query.
206 postprocess
= postprocess
or _postprocess_record
209 # Find remaining keywords by name or uuid
210 WHERE
.update(_compose_where_from_uuids_or_names(**kwargs
))
211 WHERE
= WHERE
or None
212 # ^ If the where statement is empty, it is better to leave it as None,
213 # so it can be filtered out at a later stage
214 LIMIT
= ('{:d},{:d}'.format(OFFSET
, LIMIT
)
215 if LIMIT
and OFFSET
else LIMIT
)
217 query
= remove_none_items({
218 'SELECT': SELECT
, 'FROM': FROM
, 'WHERE': WHERE
,
219 'LIMIT': LIMIT
, 'ORDER_BY': ORDER_BY
})
221 records
= self
.db
.get_rows(**query
)
223 table
= FROM
.split()[0]
224 if error_if_none
and not records
:
225 raise NoRecordFound(WHERE
, table
)
227 if error_if_multiple
and len(records
) > 1:
228 self
.logger
.error('Multiple records '
229 'FROM %s WHERE %s:\n\n%s\n\n',
230 FROM
, WHERE
, json
.dumps(records
, indent
=4))
231 raise MultipleRecordsFound(WHERE
, table
)
234 expand_joined_fields(postprocess(record
, hide
))
235 for record
in records
238 def query_one(self
, *args
, **kwargs
):
239 """Similar to ``query``, but ensuring just one result.
240 ``error_if_multiple`` is enabled by default.
242 kwargs
.setdefault('error_if_multiple', True)
243 records
= self
.query(*args
, **kwargs
)
244 return records
[0] if records
else None
246 def get_by_uuid(self
, table
, uuid
, **kwargs
):
247 """Retrieve one record from the database based on its uuid
250 table (str): table name (to be used in SQL's FROM statement).
251 uuid (str): unique identifier for record.
253 For additional keyword arguments and exceptions see :obj:`~.query`
254 (``error_if_multiple`` is enabled by default).
257 raise UndefinedUuidOrName(table
)
258 return self
.query_one(table
, WHERE
={'uuid': uuid
}, **kwargs
)
260 def get_by_name_or_uuid(self
, table
, uuid_or_name
, **kwargs
):
261 """Retrieve a record from the database based on a value that can be its
265 table (str): table name (to be used in SQL's FROM statement).
266 uuid_or_name (str): this value can correspond to either uuid or
268 For additional keyword arguments and exceptions see :obj:`~.query`
269 (``error_if_multiple`` is enabled by default).
271 if uuid_or_name
is None:
272 raise UndefinedUuidOrName(table
)
274 key
= 'uuid' if check_valid_uuid(uuid_or_name
) else 'name'
275 return self
.query_one(table
, WHERE
={key
: uuid_or_name
}, **kwargs
)
277 def get_wims(self
, uuid_or_name
=None, tenant
=None, **kwargs
):
278 """Retrieve information about one or more WIMs stored in the database
281 uuid_or_name (str): uuid or name for WIM
282 tenant (str): [optional] uuid or name for NFVO tenant
284 See :obj:`~.query` for additional keyword arguments.
286 kwargs
.update(wim
=uuid_or_name
, tenant
=tenant
)
287 from_
= _WIM_JOIN
if tenant
else _WIM
288 select_
= _WIM_SELECT
[:] + (['wim_account.*'] if tenant
else [])
290 kwargs
.setdefault('SELECT', select_
)
291 return self
.query(from_
, **kwargs
)
293 def get_wim(self
, wim
, tenant
=None, **kwargs
):
294 """Similar to ``get_wims`` but ensure only one result is returned"""
295 kwargs
.setdefault('error_if_multiple', True)
296 return self
.get_wims(wim
, tenant
)[0]
298 def create_wim(self
, wim_descriptor
):
299 """Create a new wim record inside the database and returns its uuid
302 wim_descriptor (dict): properties of the record
303 (usually each field corresponds to a database column, but extra
304 information can be offloaded to another table or serialized as
307 str: UUID of the created WIM
309 if "config" in wim_descriptor
:
310 wim_descriptor
["config"] = _serialize(wim_descriptor
["config"])
312 return self
.db
.new_row(
313 "wims", wim_descriptor
, add_uuid
=True, confidential_data
=True)
315 def update_wim(self
, uuid_or_name
, wim_descriptor
):
316 """Change an existing WIM record on the database"""
317 # obtain data, check that only one exist
318 wim
= self
.get_by_name_or_uuid('wims', uuid_or_name
)
322 where
= {'uuid': wim
['uuid']}
324 # unserialize config, edit and serialize it again
325 new_config_dict
= wim_descriptor
.get('config', {}) or {}
326 config_dict
= remove_none_items(merge_dicts(
327 wim
.get('config', {}) or {}, new_config_dict
))
328 wim_descriptor
['config'] = (
329 _serialize(config_dict
) if config_dict
else None)
331 self
.db
.update_rows('wims', wim_descriptor
, where
)
335 def delete_wim(self
, wim
):
336 # get nfvo_tenant info
337 wim
= self
.get_by_name_or_uuid('wims', wim
)
339 self
.db
.delete_row_by_id('wims', wim
['uuid'])
341 return wim
['uuid'] + ' ' + wim
['name']
343 def get_wim_accounts_by(self
, wim
=None, tenant
=None, uuid
=None, **kwargs
):
344 """Retrieve WIM account information from the database together
345 with the related records (wim, nfvo_tenant and wim_nfvo_tenant)
348 wim (str): uuid or name for WIM
349 tenant (str): [optional] uuid or name for NFVO tenant
351 See :obj:`~.query` for additional keyword arguments.
353 kwargs
.update(wim
=wim
, tenant
=tenant
)
354 kwargs
.setdefault('postprocess', _postprocess_wim_account
)
356 kwargs
.setdefault('WHERE', {'wim_account.uuid': uuid
})
357 return self
.query(FROM
=_WIM_ACCOUNT_JOIN
, **kwargs
)
359 def get_wim_account_by(self
, wim
=None, tenant
=None, uuid
=None, **kwargs
):
360 """Similar to ``get_wim_accounts_by``, but ensuring just one result"""
361 kwargs
.setdefault('error_if_multiple', True)
362 return self
.get_wim_accounts_by(wim
, tenant
, uuid
, **kwargs
)[0]
364 def get_wim_accounts(self
, **kwargs
):
365 """Retrieve all the accounts from the database"""
366 kwargs
.setdefault('postprocess', _postprocess_wim_account
)
367 return self
.query(FROM
=_WIM_ACCOUNT_JOIN
, **kwargs
)
369 def get_wim_account(self
, uuid_or_name
, **kwargs
):
370 """Retrieve WIM Account record by UUID or name,
371 See :obj:`get_by_name_or_uuid` for keyword arguments.
373 kwargs
.setdefault('postprocess', _postprocess_wim_account
)
374 kwargs
.setdefault('SELECT', _WIM_ACCOUNT_SELECT
)
375 return self
.get_by_name_or_uuid('wim_accounts', uuid_or_name
, **kwargs
)
378 def _associate(self
, wim_id
, nfvo_tenant_id
):
379 """Auxiliary method for ``create_wim_account``
381 This method just create a row in the association table
386 except DbBaseException
as db_exception
:
387 error_msg
= str(db_exception
)
388 if all([msg
in error_msg
389 for msg
in ("already in use", "'wim_nfvo_tenant'")]):
390 ex
= WimAndTenantAlreadyAttached(wim_id
, nfvo_tenant_id
)
391 reraise(ex
.__class
__, ex
, exc_info()[2])
395 def create_wim_account(self
, wim
, tenant
, properties
):
396 """Associate a wim to a tenant using the ``wim_nfvo_tenants`` table
397 and create a ``wim_account`` to store credentials and configurations.
399 For the sake of simplification, we assume that each NFVO tenant can be
400 attached to a WIM using only one WIM account. This is automatically
401 guaranteed via database constraints.
402 For corner cases, the same WIM can be registered twice using another
406 wim (str): name or uuid of the WIM related to the account being
408 tenant (str): name or uuid of the nfvo tenant to which the account
410 properties (dict): properties of the account
411 (eg. user, password, ...)
413 wim_id
= self
.get_by_name_or_uuid('wims', wim
, SELECT
=['uuid'])['uuid']
414 tenant
= self
.get_by_name_or_uuid('nfvo_tenants', tenant
,
415 SELECT
=['uuid', 'name'])
416 account
= properties
.setdefault('name', tenant
['name'])
418 wim_account
= self
.query_one('wim_accounts',
419 WHERE
={'wim_id': wim_id
, 'name': account
},
425 if wim_account
is None:
426 # If a row for the wim account doesn't exist yet, we need to
427 # create one, otherwise we can just re-use it.
428 account_id
= str(generate_uuid())
429 used_uuids
.append(account_id
)
430 row
= merge_dicts(properties
, wim_id
=wim_id
, uuid
=account_id
)
431 transaction
.append({'wim_accounts': _preprocess_wim_account(row
)})
433 account_id
= wim_account
['uuid']
434 properties
.pop('config', None) # Config is too complex to compare
435 diff
= {k
: v
for k
, v
in properties
.items() if v
!= wim_account
[k
]}
437 tip
= 'Edit the account first, and then attach it to a tenant'
438 raise WimAccountOverwrite(wim_account
, diff
, tip
)
441 'wim_nfvo_tenants': {'nfvo_tenant_id': tenant
['uuid'],
443 'wim_account_id': account_id
}})
445 with self
._associate
(wim_id
, tenant
['uuid']):
446 self
.db
.new_rows(transaction
, used_uuids
, confidential_data
=True)
450 def update_wim_account(self
, uuid
, properties
, hide
=_CONFIDENTIAL_FIELDS
):
451 """Update WIM account record by overwriting fields with new values
453 Specially for the field ``config`` this means that a new dict will be
454 merged to the existing one.
457 uuid (str): UUID for the WIM account
458 properties (dict): fields that should be overwritten
463 wim_account
= self
.get_by_uuid('wim_accounts', uuid
)
464 safe_fields
= 'user password name created'.split()
465 updates
= _preprocess_wim_account(
466 merge_dicts(wim_account
, filter_dict_keys(properties
, safe_fields
))
469 if properties
.get('config'):
470 old_config
= wim_account
.get('config') or {}
471 new_config
= merge_dicts(old_config
, properties
['config'])
472 updates
['config'] = _serialize(new_config
)
474 num_changes
= self
.db
.update_rows('wim_accounts', UPDATE
=updates
,
475 WHERE
={'uuid': wim_account
['uuid']})
477 if num_changes
is None:
478 raise UnexpectedDatabaseError('Impossible to update wim_account '
479 '{name}:{uuid}'.format(*wim_account
))
481 return self
.get_wim_account(wim_account
['uuid'], hide
=hide
)
483 def delete_wim_account(self
, uuid
):
484 """Remove WIM account record from the database"""
485 # Since we have foreign keys configured with ON CASCADE, we can rely
486 # on the database engine to guarantee consistency, deleting the
488 return self
.db
.delete_row_by_id('wim_accounts', uuid
)
490 def get_datacenters_by(self
, datacenter
=None, tenant
=None, **kwargs
):
491 """Retrieve datacenter information from the database together
492 with the related records (nfvo_tenant)
495 datacenter (str): uuid or name for datacenter
496 tenant (str): [optional] uuid or name for NFVO tenant
498 See :obj:`~.query` for additional keyword arguments.
501 kwargs
.update(datacenter
=datacenter
, tenant
=tenant
)
502 return self
.query(_DATACENTER_JOIN
, **kwargs
)
504 return [self
.get_by_name_or_uuid('datacenters',
505 datacenter
, **kwargs
)]
507 def get_datacenter_by(self
, datacenter
=None, tenant
=None, **kwargs
):
508 """Similar to ``get_datacenters_by``, but ensuring just one result"""
509 kwargs
.setdefault('error_if_multiple', True)
510 return self
.get_datacenters_by(datacenter
, tenant
, **kwargs
)[0]
512 def _create_single_port_mapping(self
, properties
):
513 info
= properties
.setdefault('wan_service_mapping_info', {})
514 endpoint_id
= properties
.get('wan_service_endpoint_id')
516 if info
.get('mapping_type') and not endpoint_id
:
517 properties
['wan_service_endpoint_id'] = (
518 self
._generate
_port
_mapping
_id
(info
))
520 properties
['wan_service_mapping_info'] = _serialize(info
)
523 self
.db
.new_row('wim_port_mappings', properties
,
524 add_uuid
=False, confidential_data
=True)
525 except DbBaseException
as old_exception
:
526 self
.logger
.exception(old_exception
)
527 ex
= InvalidParameters(
528 "The mapping must contain the "
529 "'pop_switch_dpid', 'pop_switch_port', and "
530 "wan_service_mapping_info: "
531 "('wan_switch_dpid' and 'wan_switch_port') or "
532 "'wan_service_endpoint_id}'")
533 reraise(ex
.__class
__, ex
, exc_info()[2])
537 def create_wim_port_mappings(self
, wim
, port_mappings
, tenant
=None):
538 if not isinstance(wim
, dict):
539 wim
= self
.get_by_name_or_uuid('wims', wim
)
541 for port_mapping
in port_mappings
:
542 port_mapping
['wim_name'] = wim
['name']
543 datacenter
= self
.get_datacenter_by(
544 port_mapping
['datacenter_name'], tenant
)
545 for pop_wan_port_mapping
in port_mapping
['pop_wan_mappings']:
546 element
= merge_dicts(pop_wan_port_mapping
, {
547 'wim_id': wim
['uuid'],
548 'datacenter_id': datacenter
['uuid']})
549 self
._create
_single
_port
_mapping
(element
)
553 def _filter_port_mappings_by_tenant(self
, mappings
, tenant
):
554 """Make sure all the datacenters and wims listed in the port mapping
555 belong to an specific tenant
558 # NOTE: Theoretically this could be done at SQL level, but given the
559 # number of tables involved (wim_port_mappings, wim_accounts,
560 # wims, wim_nfvo_tenants, datacenters, datacenter_tenants,
561 # tenants_datacents and nfvo_tenants), it would result in a
562 # extremely complex query. Moreover, the predicate can vary:
563 # for `get_wim_port_mappings` we can have any combination of
564 # (wim, datacenter, tenant), not all of them having the 3 values
565 # so we have combinatorial trouble to write the 'FROM' statement.
567 kwargs
= {'tenant': tenant
, 'error_if_none': False}
568 # Cache results to speedup things
572 def _get_datacenter(uuid
):
574 datacenters
.get(uuid
) or
575 datacenters
.setdefault(
576 uuid
, self
.get_datacenters_by(uuid
, **kwargs
)))
579 return (wims
.get(uuid
) or
580 wims
.setdefault(uuid
, self
.get_wims(uuid
, **kwargs
)))
584 for mapping
in mappings
585 if (_get_datacenter(mapping
['datacenter_id']) and
586 _get_wims(mapping
['wim_id']))
589 def get_wim_port_mappings(self
, wim
=None, datacenter
=None, tenant
=None,
591 """List all the port mappings, optionally filtering by wim, datacenter
594 from_
= [_PORT_MAPPING
,
595 _PORT_MAPPING_JOIN_WIM
if wim
else '',
596 _PORT_MAPPING_JOIN_DATACENTER
if datacenter
else '']
598 criteria
= ('wim_id', 'datacenter_id')
599 kwargs
.setdefault('error_if_none', False)
600 mappings
= self
.query(
602 SELECT
=_PORT_MAPPING_SELECT
,
603 ORDER_BY
=['wim_port_mapping.{}'.format(c
) for c
in criteria
],
604 wim
=wim
, datacenter
=datacenter
,
605 postprocess
=_postprocess_wim_port_mapping
,
609 mappings
= self
._filter
_port
_mappings
_by
_tenant
(mappings
, tenant
)
611 # We don't have to sort, since we have used 'ORDER_BY'
612 grouped_mappings
= groupby(mappings
, key
=itemgetter(*criteria
))
616 'datacenter_id': key
[1],
617 'pop_wan_mappings': [
618 filter_out_dict_keys(mapping
, (
619 'id', 'wim_id', 'datacenter_id',
620 'created_at', 'modified_at'))
621 for mapping
in group
]}
622 for key
, group
in grouped_mappings
625 def delete_wim_port_mappings(self
, wim_id
):
626 self
.db
.delete_row(FROM
='wim_port_mappings', WHERE
={"wim_id": wim_id
})
627 return "port mapping for wim {} deleted.".format(wim_id
)
629 def update_wim_port_mapping(self
, id, properties
):
630 original
= self
.query_one('wim_port_mappings', WHERE
={'id': id})
632 mapping_info
= remove_none_items(merge_dicts(
633 original
.get('wan_service_mapping_info') or {},
634 properties
.get('wan_service_mapping_info') or {}))
636 updates
= preprocess_record(
637 merge_dicts(original
, remove_none_items(properties
),
638 wan_service_mapping_info
=mapping_info
))
640 num_changes
= self
.db
.update_rows('wim_port_mappings',
641 UPDATE
=updates
, WHERE
={'id': id})
643 if num_changes
is None:
644 raise UnexpectedDatabaseError(
645 'Impossible to update wim_port_mappings %s:\n%s\n',
646 id, _serialize(properties
))
650 def get_actions_in_groups(self
, wim_account_id
,
651 item_types
=('instance_wim_nets',),
652 group_offset
=0, group_limit
=150):
653 """Retrieve actions from the database in groups.
654 Each group contains all the actions that have the same ``item`` type
658 wim_account_id: restrict the search to actions to be performed
659 using the same account
660 item_types (list): [optional] filter the actions to the given
662 group_limit (int): maximum number of groups returned by the
664 group_offset (int): skip the N first groups. Used together with
665 group_limit for pagination purposes.
668 List of groups, where each group is a tuple ``(key, actions)``.
669 In turn, ``key`` is a tuple containing the values of
670 ``(item, item_id)`` used to create the group and ``actions`` is a
671 list of ``vim_wim_actions`` records (dicts).
675 '"{}"'.format(self
.db
.escape_string(t
)) for t
in item_types
)
677 items
= ('SELECT DISTINCT a.item, a.item_id, a.wim_account_id '
678 'FROM vim_wim_actions AS a '
679 'WHERE a.wim_account_id="{}" AND a.item IN ({}) '
680 'ORDER BY a.item, a.item_id '
681 'LIMIT {:d},{:d}').format(
682 self
.safe_str(wim_account_id
),
683 ','.join(type_options
),
684 group_offset
, group_limit
687 join
= 'vim_wim_actions NATURAL JOIN ({}) AS items'.format(items
)
688 db_results
= self
.db
.get_rows(
689 FROM
=join
, ORDER_BY
=('item', 'item_id', 'created_at'))
691 results
= (_postprocess_action(r
) for r
in db_results
)
692 criteria
= itemgetter('item', 'item_id')
693 return [(k
, list(g
)) for k
, g
in groupby(results
, key
=criteria
)]
695 def update_action(self
, instance_action_id
, task_index
, properties
):
696 condition
= {'instance_action_id': instance_action_id
,
697 'task_index': task_index
}
699 action
= self
.query_one('vim_wim_actions', WHERE
=condition
)
701 actions
= self
.query('vim_wim_actions', WHERE
=condition
)
702 self
.logger
.error('More then one action found:\n%s',
703 json
.dumps(actions
, indent
=4))
706 extra
= remove_none_items(merge_dicts(
707 action
.get('extra') or {},
708 properties
.get('extra') or {}))
710 updates
= preprocess_record(
711 merge_dicts(action
, properties
, extra
=extra
))
713 num_changes
= self
.db
.update_rows('vim_wim_actions',
714 UPDATE
=updates
, WHERE
=condition
)
716 if num_changes
is None:
717 raise UnexpectedDatabaseError(
718 'Impossible to update vim_wim_actions '
719 '{instance_action_id}[{task_index}]'.format(*action
))
723 def get_wan_links(self
, uuid
=None, **kwargs
):
724 """Retrieve WAN link records from the database
727 uuid, instance_scenario_id, sce_net_id, wim_id, wim_account_id:
728 attributes that can be used at the WHERE clause
730 kwargs
.setdefault('uuid', uuid
)
731 kwargs
.setdefault('error_if_none', False)
733 criteria_fields
= ('uuid', 'instance_scenario_id', 'sce_net_id',
734 'wim_id', 'wim_account_id')
735 criteria
= remove_none_items(filter_dict_keys(kwargs
, criteria_fields
))
736 kwargs
= filter_out_dict_keys(kwargs
, criteria_fields
)
738 return self
.query('instance_wim_nets', WHERE
=criteria
, **kwargs
)
740 def update_wan_link(self
, uuid
, properties
):
741 wan_link
= self
.get_by_uuid('instance_wim_nets', uuid
)
743 wim_info
= remove_none_items(merge_dicts(
744 wan_link
.get('wim_info') or {},
745 properties
.get('wim_info') or {}))
747 updates
= preprocess_record(
748 merge_dicts(wan_link
, properties
, wim_info
=wim_info
))
750 self
.logger
.debug({'UPDATE': updates
})
751 num_changes
= self
.db
.update_rows(
752 'instance_wim_nets', UPDATE
=updates
,
753 WHERE
={'uuid': wan_link
['uuid']})
755 if num_changes
is None:
756 raise UnexpectedDatabaseError(
757 'Impossible to update instance_wim_nets ' + wan_link
['uuid'])
761 def get_instance_nets(self
, instance_scenario_id
, sce_net_id
, **kwargs
):
762 """Retrieve all the instance nets related to the same instance_scenario
767 WHERE
={'instance_scenario_id': instance_scenario_id
,
768 'sce_net_id': sce_net_id
},
770 'ORDER_BY', ('instance_scenario_id', 'sce_net_id')),
773 def update_instance_action_counters(self
, uuid
, failed
=None, done
=None):
774 """Atomically increment/decrement number_done and number_failed fields
775 in the instance action table
777 changes
= remove_none_items({
778 'number_failed': failed
and {'INCREMENT': failed
},
779 'number_done': done
and {'INCREMENT': done
}
785 return self
.db
.update_rows('instance_actions',
786 WHERE
={'uuid': uuid
}, UPDATE
=changes
)
788 def get_only_vm_with_external_net(self
, instance_net_id
, **kwargs
):
789 """Return an instance VM if that is the only VM connected to an
790 external network identified by instance_net_id
792 counting
= ('SELECT DISTINCT instance_net_id '
793 'FROM instance_interfaces '
794 'WHERE instance_net_id="{}" AND type="external" '
795 'GROUP BY instance_net_id '
796 'HAVING COUNT(*)=1').format(self
.safe_str(instance_net_id
))
798 vm_item
= ('SELECT DISTINCT instance_vm_id '
799 'FROM instance_interfaces NATURAL JOIN ({}) AS a'
802 return self
.query_one(
803 'instance_vms JOIN ({}) as instance_interface '
804 'ON instance_vms.uuid=instance_interface.instance_vm_id'
805 .format(vm_item
), **kwargs
)
807 def safe_str(self
, string
):
808 """Return a SQL safe string"""
809 return self
.db
.escape_string(string
)
814 def _generate_port_mapping_id(self
, mapping_info
):
815 """Given a port mapping represented by a dict with a 'type' field,
816 generate a unique string, in a injective way.
818 mapping_info
= mapping_info
.copy() # Avoid mutating original object
819 mapping_type
= mapping_info
.pop('mapping_type', None)
821 raise UndefinedWanMappingType(mapping_info
)
823 unique_fields
= UNIQUE_PORT_MAPPING_INFO_FIELDS
.get(mapping_type
)
826 mapping_info
= filter_dict_keys(mapping_info
, unique_fields
)
828 self
.logger
.warning('Unique fields for WIM port mapping of type '
829 '%s not defined. Please add a list of fields '
830 'which combination should be unique in '
831 'UNIQUE_PORT_MAPPING_INFO_FIELDS '
832 '(`wim/persistency.py) ', mapping_type
)
834 repeatable_repr
= json
.dumps(mapping_info
, encoding
='utf-8',
835 sort_keys
=True, indent
=False)
837 return ':'.join([mapping_type
, _str2id(repeatable_repr
)])
840 def _serialize(value
):
841 """Serialize an arbitrary value in a consistent way,
842 so it can be stored in a database inside a text field
844 return yaml
.safe_dump(value
, default_flow_style
=True, width
=256)
847 def _unserialize(text
):
848 """Unserialize text representation into an arbitrary value,
849 so it can be loaded from the database
851 return yaml
.safe_load(text
)
854 def preprocess_record(record
):
855 """Small transformations to be applied to the data that cames from the
856 user before writing it to the database. By default, filter out timestamps,
857 and serialize the ``config`` field.
859 automatic_fields
= ['created_at', 'modified_at']
860 record
= serialize_fields(filter_out_dict_keys(record
, automatic_fields
))
865 def _preprocess_wim_account(wim_account
):
866 """Do the default preprocessing and convert the 'created' field from
869 wim_account
= preprocess_record(wim_account
)
871 created
= wim_account
.get('created')
872 wim_account
['created'] = (
873 'true' if created
is True or created
== 'true' else 'false')
878 def _postprocess_record(record
, hide
=_CONFIDENTIAL_FIELDS
):
879 """By default, hide passwords fields, unserialize ``config`` fields, and
880 convert float timestamps to strings
882 record
= hide_confidential_fields(record
, hide
)
883 record
= unserialize_fields(record
, hide
)
885 convert_float_timestamp2str(record
)
890 def _postprocess_action(action
):
891 if action
.get('extra'):
892 action
['extra'] = _unserialize(action
['extra'])
897 def _postprocess_wim_account(wim_account
, hide
=_CONFIDENTIAL_FIELDS
):
898 """Do the default postprocessing and convert the 'created' field from
901 # Fix fields from join
902 for field
in ('type', 'description', 'wim_url'):
903 if field
in wim_account
:
904 wim_account
['wim.'+field
] = wim_account
.pop(field
)
906 for field
in ('id', 'nfvo_tenant_id', 'wim_account_id'):
907 if field
in wim_account
:
908 wim_account
['association.'+field
] = wim_account
.pop(field
)
910 wim_account
= _postprocess_record(wim_account
, hide
)
912 created
= wim_account
.get('created')
913 wim_account
['created'] = (created
is True or created
== 'true')
918 def _postprocess_wim_port_mapping(mapping
, hide
=_CONFIDENTIAL_FIELDS
):
919 mapping
= _postprocess_record(mapping
, hide
=hide
)
920 mapping_info
= mapping
.get('wan_service_mapping_info', None) or {}
921 mapping
['wan_service_mapping_info'] = mapping_info
925 def hide_confidential_fields(record
, fields
=_CONFIDENTIAL_FIELDS
):
926 """Obfuscate confidential fields from the input dict.
929 This function performs a SHALLOW operation.
931 if not(isinstance(record
, dict) and fields
):
934 keys
= record
.iterkeys()
935 keys
= (k
for k
in keys
for f
in fields
if k
== f
or k
.endswith('.'+f
))
937 return merge_dicts(record
, {k
: '********' for k
in keys
if record
[k
]})
940 def unserialize_fields(record
, hide
=_CONFIDENTIAL_FIELDS
,
941 fields
=_SERIALIZED_FIELDS
):
942 """Unserialize fields that where stored in the database as a serialized
945 keys
= record
.iterkeys()
946 keys
= (k
for k
in keys
for f
in fields
if k
== f
or k
.endswith('.'+f
))
948 return merge_dicts(record
, {
949 key
: hide_confidential_fields(_unserialize(record
[key
]), hide
)
950 for key
in keys
if record
[key
]
954 def serialize_fields(record
, fields
=_SERIALIZED_FIELDS
):
955 """Serialize fields to be stored in the database as YAML"""
956 keys
= record
.iterkeys()
957 keys
= (k
for k
in keys
for f
in fields
if k
== f
or k
.endswith('.'+f
))
959 return merge_dicts(record
, {
960 key
: _serialize(record
[key
])
961 for key
in keys
if record
[key
] is not None
965 def _decide_name_or_uuid(value
):
968 if isinstance(value
, (list, tuple)):
969 reference
= value
[0] if value
else ''
971 return 'uuid' if check_valid_uuid(reference
) else 'name'
974 def _compose_where_from_uuids_or_names(**conditions
):
975 """Create a dict containing the right conditions to be used in a database
978 This function chooses between ``names`` and ``uuid`` fields based on the
979 format of the passed string.
980 If a list is passed, the first element of the list will be used to choose
981 the name of the field.
982 If a ``None`` value is passed, ``uuid`` is used.
984 Note that this function automatically translates ``tenant`` to
985 ``nfvo_tenant`` for the sake of brevity.
988 >>> _compose_where_from_uuids_or_names(
990 tenant=['xyz123', 'def456']
991 datacenter='5286a274-8a1b-4b8d-a667-9c94261ad855')
992 {'wim.name': 'abcdef',
993 'nfvo_tenant.name': ['xyz123', 'def456']
994 'datacenter.uuid': '5286a274-8a1b-4b8d-a667-9c94261ad855'}
996 if 'tenant' in conditions
:
997 conditions
['nfvo_tenant'] = conditions
.pop('tenant')
1000 '{}.{}'.format(kind
, _decide_name_or_uuid(value
)): value
1001 for kind
, value
in conditions
.items() if value
1006 """Create an ID (following the UUID format) from a piece of arbitrary
1009 Different texts should generate different IDs, and the same text should
1010 generate the same ID in a repeatable way.
1012 return sha1(text
).hexdigest()