1 # -*- coding: utf-8 -*-
3 # Copyright 2018 University of Bristol - High Performance Networks Research
7 # Contributors: Anderson Bravalheri, Dimitrios Gkounis, Abubakar Siddique
8 # Muqaddas, Navdeep Uniyal, Reza Nejabati and Dimitra Simeonidou
10 # Licensed under the Apache License, Version 2.0 (the "License"); you may
11 # not use this file except in compliance with the License. You may obtain
12 # a copy of the License at
14 # http://www.apache.org/licenses/LICENSE-2.0
16 # Unless required by applicable law or agreed to in writing, software
17 # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
18 # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
19 # License for the specific language governing permissions and limitations
22 # For those usages not covered by the Apache License, Version 2.0 please
23 # contact with: <highperformance-networks@bristol.ac.uk>
25 # Neither the name of the University of Bristol nor the names of its
26 # contributors may be used to endorse or promote products derived from
27 # this software without specific prior written permission.
29 # This work has been performed in the context of DCMS UK 5G Testbeds
30 # & Trials Programme and in the framework of the Metro-Haul project -
31 # funded by the European Commission under Grant number 761727 through the
32 # Horizon 2020 and 5G-PPP programmes.
35 """This module contains only logic related to managing records in a database
36 which includes data format normalization, data format validation and etc.
37 (It works as an extension to `nfvo_db.py` for the WIM feature)
39 No domain logic/architectural concern should be present in this file.
43 from contextlib
import contextmanager
44 from hashlib
import sha1
45 from itertools
import groupby
46 from operator
import itemgetter
47 from sys
import exc_info
48 # from time import time
49 from uuid
import uuid1
as generate_uuid
51 from six
import reraise
57 convert_float_timestamp2str
,
70 UndefinedWanMappingType
,
71 UnexpectedDatabaseError
,
73 WimAndTenantAlreadyAttached
80 ' JOIN wim_nfvo_tenants AS association '
81 ' ON association.wim_id=wim.uuid '
82 ' JOIN nfvo_tenants AS nfvo_tenant '
83 ' ON association.nfvo_tenant_id=nfvo_tenant.uuid '
84 ' JOIN wim_accounts AS wim_account '
85 ' ON association.wim_account_id=wim_account.uuid '
89 'wim_accounts AS wim_account '
90 ' JOIN wim_nfvo_tenants AS association '
91 ' ON association.wim_account_id=wim_account.uuid '
93 ' ON association.wim_id=wim.uuid '
94 ' JOIN nfvo_tenants AS nfvo_tenant '
95 ' ON association.nfvo_tenant_id=nfvo_tenant.uuid '
99 'datacenters AS datacenter '
100 ' JOIN tenants_datacenters AS association '
101 ' ON association.datacenter_id=datacenter.uuid '
102 ' JOIN datacenter_tenants as datacenter_account '
103 ' ON association.datacenter_tenant_id=datacenter_account.uuid '
104 ' JOIN nfvo_tenants AS nfvo_tenant '
105 ' ON association.nfvo_tenant_id=nfvo_tenant.uuid '
108 _PORT_MAPPING
= 'wim_port_mappings as wim_port_mapping '
110 _PORT_MAPPING_JOIN_WIM
= (
112 ' ON wim_port_mapping.wim_id=wim.uuid '
115 _PORT_MAPPING_JOIN_DATACENTER
= (
116 ' JOIN datacenters as datacenter '
117 ' ON wim_port_mapping.datacenter_id=datacenter.uuid '
121 'wim.{0} as {0}'.format(_field
)
122 for _field
in 'uuid name description wim_url type config '
123 'created_at modified_at'.split()
126 _WIM_ACCOUNT_SELECT
= 'uuid name user password config'.split()
128 _PORT_MAPPING_SELECT
= ('wim_port_mapping.*', )
130 _CONFIDENTIAL_FIELDS
= ('password', 'passwd')
132 _SERIALIZED_FIELDS
= ('config', 'vim_info', 'wim_info', 'conn_info', 'extra',
133 'wan_service_mapping_info')
135 UNIQUE_PORT_MAPPING_INFO_FIELDS
= {
136 'dpid-port': ('wan_switch_dpid', 'wan_switch_port')
138 """Fields that should be unique for each port mapping that relies on
139 wan_service_mapping_info.
141 For example, for port mappings of type 'dpid-port', each combination of
142 wan_switch_dpid and wan_switch_port should be unique (the same switch cannot
143 be connected to two different places using the same port)
147 class WimPersistence(object):
148 """High level interactions with the WIM tables in the database"""
150 def __init__(self
, db
, logger
=None):
152 self
.logger
= logger
or logging
.getLogger('openmano.wim.persistence')
162 error_if_multiple
=False,
164 hide
=_CONFIDENTIAL_FIELDS
,
166 """Retrieve records from the database.
169 SELECT, FROM, WHERE, LIMIT, ORDER_BY: used to compose the SQL
170 query. See ``nfvo_db.get_rows``.
171 OFFSET: only valid when used togheter with LIMIT.
172 Ignore the OFFSET first results of the query.
173 error_if_none: by default an error is raised if no record is
174 found. With this option it is possible to disable this error.
175 error_if_multiple: by default no error is raised if more then one
177 With this option it is possible to enable this error.
178 postprocess: function applied to every retrieved record.
179 This function receives a dict as input and must return it
180 after modifications. Moreover this function should accept a
181 second optional parameter ``hide`` indicating
182 the confidential fiels to be obfuscated.
183 By default a minimal postprocessing function is applied,
184 obfuscating confidential fields and converting timestamps.
185 hide: option proxied to postprocess
187 All the remaining keyword arguments will be assumed to be ``name``s or
188 ``uuid``s to compose the WHERE statement, according to their format.
189 If the value corresponds to an array, the first element will determine
190 if it is an name or UUID.
193 - ``wim="abcdef"``` will be turned into ``wim.name="abcdef"``,
194 - ``datacenter="5286a274-8a1b-4b8d-a667-9c94261ad855"``
196 ``datacenter.uuid="5286a274-8a1b-4b8d-a667-9c94261ad855"``.
197 - ``wim=["5286a274-8a1b-4b8d-a667-9c94261ad855", ...]``
199 ``wim.uuid=["5286a274-8a1b-4b8d-a667-9c94261ad855", ...]``
202 NoRecordFound: if the query result set is empty
203 DbBaseException: errors occuring during the execution of the query.
206 postprocess
= postprocess
or _postprocess_record
209 # Find remaining keywords by name or uuid
210 WHERE
.update(_compose_where_from_uuids_or_names(**kwargs
))
211 WHERE
= WHERE
or None
212 # ^ If the where statement is empty, it is better to leave it as None,
213 # so it can be filtered out at a later stage
214 LIMIT
= ('{:d},{:d}'.format(OFFSET
, LIMIT
)
215 if LIMIT
and OFFSET
else LIMIT
)
217 query
= remove_none_items({
218 'SELECT': SELECT
, 'FROM': FROM
, 'WHERE': WHERE
,
219 'LIMIT': LIMIT
, 'ORDER_BY': ORDER_BY
})
221 records
= self
.db
.get_rows(**query
)
223 table
= FROM
.split()[0]
224 if error_if_none
and not records
:
225 raise NoRecordFound(WHERE
, table
)
227 if error_if_multiple
and len(records
) > 1:
228 self
.logger
.error('Multiple records '
229 'FROM %s WHERE %s:\n\n%s\n\n',
230 FROM
, WHERE
, json
.dumps(records
, indent
=4))
231 raise MultipleRecordsFound(WHERE
, table
)
234 expand_joined_fields(postprocess(record
, hide
))
235 for record
in records
238 def query_one(self
, *args
, **kwargs
):
239 """Similar to ``query``, but ensuring just one result.
240 ``error_if_multiple`` is enabled by default.
242 kwargs
.setdefault('error_if_multiple', True)
243 records
= self
.query(*args
, **kwargs
)
244 return records
[0] if records
else None
246 def get_by_uuid(self
, table
, uuid
, **kwargs
):
247 """Retrieve one record from the database based on its uuid
250 table (str): table name (to be used in SQL's FROM statement).
251 uuid (str): unique identifier for record.
253 For additional keyword arguments and exceptions see :obj:`~.query`
254 (``error_if_multiple`` is enabled by default).
257 raise UndefinedUuidOrName(table
)
258 return self
.query_one(table
, WHERE
={'uuid': uuid
}, **kwargs
)
260 def get_by_name_or_uuid(self
, table
, uuid_or_name
, **kwargs
):
261 """Retrieve a record from the database based on a value that can be its
265 table (str): table name (to be used in SQL's FROM statement).
266 uuid_or_name (str): this value can correspond to either uuid or
268 For additional keyword arguments and exceptions see :obj:`~.query`
269 (``error_if_multiple`` is enabled by default).
271 if uuid_or_name
is None:
272 raise UndefinedUuidOrName(table
)
274 key
= 'uuid' if check_valid_uuid(uuid_or_name
) else 'name'
275 return self
.query_one(table
, WHERE
={key
: uuid_or_name
}, **kwargs
)
277 def get_wims(self
, uuid_or_name
=None, tenant
=None, **kwargs
):
278 """Retrieve information about one or more WIMs stored in the database
281 uuid_or_name (str): uuid or name for WIM
282 tenant (str): [optional] uuid or name for NFVO tenant
284 See :obj:`~.query` for additional keyword arguments.
286 kwargs
.update(wim
=uuid_or_name
, tenant
=tenant
)
287 from_
= _WIM_JOIN
if tenant
else _WIM
288 select_
= _WIM_SELECT
[:] + (['wim_account.*'] if tenant
else [])
290 kwargs
.setdefault('SELECT', select_
)
291 return self
.query(from_
, **kwargs
)
293 def get_wim(self
, wim
, tenant
=None, **kwargs
):
294 """Similar to ``get_wims`` but ensure only one result is returned"""
295 kwargs
.setdefault('error_if_multiple', True)
296 return self
.get_wims(wim
, tenant
)[0]
298 def create_wim(self
, wim_descriptor
):
299 """Create a new wim record inside the database and returns its uuid
302 wim_descriptor (dict): properties of the record
303 (usually each field corresponds to a database column, but extra
304 information can be offloaded to another table or serialized as
307 str: UUID of the created WIM
309 if "config" in wim_descriptor
:
310 wim_descriptor
["config"] = _serialize(wim_descriptor
["config"])
312 return self
.db
.new_row(
313 "wims", wim_descriptor
, add_uuid
=True, confidential_data
=True)
315 def update_wim(self
, uuid_or_name
, wim_descriptor
):
316 """Change an existing WIM record on the database"""
317 # obtain data, check that only one exist
318 wim
= self
.get_by_name_or_uuid('wims', uuid_or_name
)
322 where
= {'uuid': wim
['uuid']}
324 # unserialize config, edit and serialize it again
325 new_config_dict
= wim_descriptor
.get('config', {}) or {}
326 config_dict
= remove_none_items(merge_dicts(
327 wim
.get('config', {}) or {}, new_config_dict
))
328 wim_descriptor
['config'] = (
329 _serialize(config_dict
) if config_dict
else None)
331 self
.db
.update_rows('wims', wim_descriptor
, where
)
335 def delete_wim(self
, wim
):
336 # get nfvo_tenant info
337 wim
= self
.get_by_name_or_uuid('wims', wim
)
339 self
.db
.delete_row_by_id('wims', wim
['uuid'])
341 return wim
['uuid'] + ' ' + wim
['name']
343 def get_wim_accounts_by(self
, wim
=None, tenant
=None, uuid
=None, **kwargs
):
344 """Retrieve WIM account information from the database together
345 with the related records (wim, nfvo_tenant and wim_nfvo_tenant)
348 wim (str): uuid or name for WIM
349 tenant (str): [optional] uuid or name for NFVO tenant
351 See :obj:`~.query` for additional keyword arguments.
353 kwargs
.update(wim
=wim
, tenant
=tenant
)
354 kwargs
.setdefault('postprocess', _postprocess_wim_account
)
356 kwargs
.setdefault('WHERE', {'wim_account.uuid': uuid
})
357 return self
.query(FROM
=_WIM_ACCOUNT_JOIN
, **kwargs
)
359 def get_wim_account_by(self
, wim
=None, tenant
=None, uuid
=None, **kwargs
):
360 """Similar to ``get_wim_accounts_by``, but ensuring just one result"""
361 kwargs
.setdefault('error_if_multiple', True)
362 return self
.get_wim_accounts_by(wim
, tenant
, uuid
, **kwargs
)[0]
364 def get_wim_accounts(self
, **kwargs
):
365 """Retrieve all the accounts from the database"""
366 kwargs
.setdefault('postprocess', _postprocess_wim_account
)
367 return self
.query(FROM
=_WIM_ACCOUNT_JOIN
, **kwargs
)
369 def get_wim_account(self
, uuid_or_name
, **kwargs
):
370 """Retrieve WIM Account record by UUID or name,
371 See :obj:`get_by_name_or_uuid` for keyword arguments.
373 kwargs
.setdefault('postprocess', _postprocess_wim_account
)
374 kwargs
.setdefault('SELECT', _WIM_ACCOUNT_SELECT
)
375 return self
.get_by_name_or_uuid('wim_accounts', uuid_or_name
, **kwargs
)
378 def _associate(self
, wim_id
, nfvo_tenant_id
):
379 """Auxiliary method for ``create_wim_account``
381 This method just create a row in the association table
386 except DbBaseException
as db_exception
:
387 error_msg
= str(db_exception
)
388 if all([msg
in error_msg
389 for msg
in ("already in use", "'wim_nfvo_tenant'")]):
390 ex
= WimAndTenantAlreadyAttached(wim_id
, nfvo_tenant_id
)
391 reraise(ex
.__class
__, ex
, exc_info()[2])
395 def create_wim_account(self
, wim
, tenant
, properties
):
396 """Associate a wim to a tenant using the ``wim_nfvo_tenants`` table
397 and create a ``wim_account`` to store credentials and configurations.
399 For the sake of simplification, we assume that each NFVO tenant can be
400 attached to a WIM using only one WIM account. This is automatically
401 guaranteed via database constraints.
402 For corner cases, the same WIM can be registered twice using another
406 wim (str): name or uuid of the WIM related to the account being
408 tenant (str): name or uuid of the nfvo tenant to which the account
410 properties (dict): properties of the account
411 (eg. user, password, ...)
413 wim_id
= self
.get_by_name_or_uuid('wims', wim
, SELECT
=['uuid'])['uuid']
414 tenant
= self
.get_by_name_or_uuid('nfvo_tenants', tenant
,
415 SELECT
=['uuid', 'name'])
416 account
= properties
.setdefault('name', tenant
['name'])
418 wim_account
= self
.query_one('wim_accounts',
419 WHERE
={'wim_id': wim_id
, 'name': account
},
425 if wim_account
is None:
426 # If a row for the wim account doesn't exist yet, we need to
427 # create one, otherwise we can just re-use it.
428 account_id
= str(generate_uuid())
429 used_uuids
.append(account_id
)
430 row
= merge_dicts(properties
, wim_id
=wim_id
, uuid
=account_id
)
431 transaction
.append({'wim_accounts': _preprocess_wim_account(row
)})
433 account_id
= wim_account
['uuid']
434 properties
.pop('config', None) # Config is too complex to compare
435 diff
= {k
: v
for k
, v
in properties
.items() if v
!= wim_account
[k
]}
437 tip
= 'Edit the account first, and then attach it to a tenant'
438 raise WimAccountOverwrite(wim_account
, diff
, tip
)
441 'wim_nfvo_tenants': {'nfvo_tenant_id': tenant
['uuid'],
443 'wim_account_id': account_id
}})
445 with self
._associate
(wim_id
, tenant
['uuid']):
446 self
.db
.new_rows(transaction
, used_uuids
, confidential_data
=True)
450 def update_wim_account(self
, uuid
, properties
, hide
=_CONFIDENTIAL_FIELDS
):
451 """Update WIM account record by overwriting fields with new values
453 Specially for the field ``config`` this means that a new dict will be
454 merged to the existing one.
457 uuid (str): UUID for the WIM account
458 properties (dict): fields that should be overwritten
463 wim_account
= self
.get_by_uuid('wim_accounts', uuid
)
464 safe_fields
= 'user password name created'.split()
465 updates
= _preprocess_wim_account(
466 merge_dicts(wim_account
, filter_dict_keys(properties
, safe_fields
))
469 if properties
.get('config'):
470 old_config
= wim_account
.get('config') or {}
471 new_config
= merge_dicts(old_config
, properties
['config'])
472 updates
['config'] = _serialize(new_config
)
474 num_changes
= self
.db
.update_rows('wim_accounts', UPDATE
=updates
,
475 WHERE
={'uuid': wim_account
['uuid']})
477 if num_changes
is None:
478 raise UnexpectedDatabaseError('Impossible to update wim_account '
479 '{name}:{uuid}'.format(*wim_account
))
481 return self
.get_wim_account(wim_account
['uuid'], hide
=hide
)
483 def delete_wim_account(self
, uuid
):
484 """Remove WIM account record from the database"""
485 # Since we have foreign keys configured with ON CASCADE, we can rely
486 # on the database engine to guarantee consistency, deleting the
488 return self
.db
.delete_row_by_id('wim_accounts', uuid
)
490 def get_datacenters_by(self
, datacenter
=None, tenant
=None, **kwargs
):
491 """Retrieve datacenter information from the database together
492 with the related records (nfvo_tenant)
495 datacenter (str): uuid or name for datacenter
496 tenant (str): [optional] uuid or name for NFVO tenant
498 See :obj:`~.query` for additional keyword arguments.
501 kwargs
.update(datacenter
=datacenter
, tenant
=tenant
)
502 return self
.query(_DATACENTER_JOIN
, **kwargs
)
504 return [self
.get_by_name_or_uuid('datacenters',
505 datacenter
, **kwargs
)]
507 def get_datacenter_by(self
, datacenter
=None, tenant
=None, **kwargs
):
508 """Similar to ``get_datacenters_by``, but ensuring just one result"""
509 kwargs
.setdefault('error_if_multiple', True)
510 return self
.get_datacenters_by(datacenter
, tenant
, **kwargs
)[0]
512 def _create_single_port_mapping(self
, properties
):
513 info
= properties
.setdefault('wan_service_mapping_info', {})
514 endpoint_id
= properties
.get('wan_service_endpoint_id')
516 if info
.get('mapping_type') and not endpoint_id
:
517 properties
['wan_service_endpoint_id'] = (
518 self
._generate
_port
_mapping
_id
(info
))
520 properties
['wan_service_mapping_info'] = _serialize(info
)
523 self
.db
.new_row('wim_port_mappings', properties
,
524 add_uuid
=False, confidential_data
=True)
525 except DbBaseException
as old_exception
:
526 self
.logger
.exception(old_exception
)
527 ex
= InvalidParameters(
528 "The mapping must contain the "
529 "'pop_switch_dpid', 'pop_switch_port', and "
530 "wan_service_mapping_info: "
531 "('wan_switch_dpid' and 'wan_switch_port') or "
532 "'wan_service_endpoint_id}'")
533 reraise(ex
.__class
__, ex
, exc_info()[2])
537 def create_wim_port_mappings(self
, wim
, port_mappings
, tenant
=None):
538 if not isinstance(wim
, dict):
539 wim
= self
.get_by_name_or_uuid('wims', wim
)
541 for port_mapping
in port_mappings
:
542 port_mapping
['wim_name'] = wim
['name']
543 datacenter
= self
.get_datacenter_by(
544 port_mapping
['datacenter_name'], tenant
)
545 for pop_wan_port_mapping
in port_mapping
['pop_wan_mappings']:
546 element
= merge_dicts(pop_wan_port_mapping
, {
547 'wim_id': wim
['uuid'],
548 'datacenter_id': datacenter
['uuid']})
549 self
._create
_single
_port
_mapping
(element
)
553 def _filter_port_mappings_by_tenant(self
, mappings
, tenant
):
554 """Make sure all the datacenters and wims listed in the port mapping
555 belong to an specific tenant
558 # NOTE: Theoretically this could be done at SQL level, but given the
559 # number of tables involved (wim_port_mappings, wim_accounts,
560 # wims, wim_nfvo_tenants, datacenters, datacenter_tenants,
561 # tenants_datacents and nfvo_tenants), it would result in a
562 # extremely complex query. Moreover, the predicate can vary:
563 # for `get_wim_port_mappings` we can have any combination of
564 # (wim, datacenter, tenant), not all of them having the 3 values
565 # so we have combinatorial trouble to write the 'FROM' statement.
567 kwargs
= {'tenant': tenant
, 'error_if_none': False}
568 # Cache results to speedup things
572 def _get_datacenter(uuid
):
574 datacenters
.get(uuid
) or
575 datacenters
.setdefault(
576 uuid
, self
.get_datacenters_by(uuid
, **kwargs
)))
579 return (wims
.get(uuid
) or
580 wims
.setdefault(uuid
, self
.get_wims(uuid
, **kwargs
)))
584 for mapping
in mappings
585 if (_get_datacenter(mapping
['datacenter_id']) and
586 _get_wims(mapping
['wim_id']))
589 def get_wim_port_mappings(self
, wim
=None, datacenter
=None, tenant
=None,
591 """List all the port mappings, optionally filtering by wim, datacenter
594 from_
= [_PORT_MAPPING
,
595 _PORT_MAPPING_JOIN_WIM
if wim
else '',
596 _PORT_MAPPING_JOIN_DATACENTER
if datacenter
else '']
598 criteria
= ('wim_id', 'datacenter_id')
599 kwargs
.setdefault('error_if_none', False)
600 mappings
= self
.query(
602 SELECT
=_PORT_MAPPING_SELECT
,
603 ORDER_BY
=['wim_port_mapping.{}'.format(c
) for c
in criteria
],
604 wim
=wim
, datacenter
=datacenter
,
605 postprocess
=_postprocess_wim_port_mapping
,
609 mappings
= self
._filter
_port
_mappings
_by
_tenant
(mappings
, tenant
)
611 # We don't have to sort, since we have used 'ORDER_BY'
612 grouped_mappings
= groupby(mappings
, key
=itemgetter(*criteria
))
616 'datacenter_id': key
[1],
617 'pop_wan_mappings': [
618 filter_out_dict_keys(mapping
, (
619 'id', 'wim_id', 'datacenter_id',
620 'created_at', 'modified_at'))
621 for mapping
in group
]}
622 for key
, group
in grouped_mappings
625 def delete_wim_port_mappings(self
, wim_id
):
626 self
.db
.delete_row(FROM
='wim_port_mappings', WHERE
={"wim_id": wim_id
})
627 return "port mapping for wim {} deleted.".format(wim_id
)
629 def update_wim_port_mapping(self
, id, properties
):
630 original
= self
.query_one('wim_port_mappings', WHERE
={'id': id})
632 mapping_info
= remove_none_items(merge_dicts(
633 original
.get('wan_service_mapping_info') or {},
634 properties
.get('wan_service_mapping_info') or {}))
636 updates
= preprocess_record(
637 merge_dicts(original
, remove_none_items(properties
),
638 wan_service_mapping_info
=mapping_info
))
640 num_changes
= self
.db
.update_rows('wim_port_mappings',
641 UPDATE
=updates
, WHERE
={'id': id})
643 if num_changes
is None:
644 raise UnexpectedDatabaseError(
645 'Impossible to update wim_port_mappings {}:\n{}\n'.format(
646 id, _serialize(properties
))
651 def get_actions_in_groups(self
, wim_account_id
,
652 item_types
=('instance_wim_nets',),
653 group_offset
=0, group_limit
=150):
654 """Retrieve actions from the database in groups.
655 Each group contains all the actions that have the same ``item`` type
659 wim_account_id: restrict the search to actions to be performed
660 using the same account
661 item_types (list): [optional] filter the actions to the given
663 group_limit (int): maximum number of groups returned by the
665 group_offset (int): skip the N first groups. Used together with
666 group_limit for pagination purposes.
669 List of groups, where each group is a tuple ``(key, actions)``.
670 In turn, ``key`` is a tuple containing the values of
671 ``(item, item_id)`` used to create the group and ``actions`` is a
672 list of ``vim_wim_actions`` records (dicts).
676 '"{}"'.format(self
.db
.escape_string(t
)) for t
in item_types
)
678 items
= ('SELECT DISTINCT a.item, a.item_id, a.wim_account_id '
679 'FROM vim_wim_actions AS a '
680 'WHERE a.wim_account_id="{}" AND a.item IN ({}) '
681 'ORDER BY a.item, a.item_id '
682 'LIMIT {:d},{:d}').format(
683 self
.safe_str(wim_account_id
),
684 ','.join(type_options
),
685 group_offset
, group_limit
)
687 join
= 'vim_wim_actions NATURAL JOIN ({}) AS items'.format(items
)
688 db_results
= self
.db
.get_rows(
689 FROM
=join
, ORDER_BY
=('item', 'item_id', 'created_at'))
691 results
= (_postprocess_action(r
) for r
in db_results
)
692 criteria
= itemgetter('item', 'item_id')
693 return [(k
, list(g
)) for k
, g
in groupby(results
, key
=criteria
)]
695 def update_action(self
, instance_action_id
, task_index
, properties
):
696 condition
= {'instance_action_id': instance_action_id
,
697 'task_index': task_index
}
699 action
= self
.query_one('vim_wim_actions', WHERE
=condition
)
701 actions
= self
.query('vim_wim_actions', WHERE
=condition
)
702 self
.logger
.error('More then one action found:\n%s',
703 json
.dumps(actions
, indent
=4))
706 extra
= remove_none_items(merge_dicts(
707 action
.get('extra') or {},
708 properties
.get('extra') or {}))
710 updates
= preprocess_record(
711 merge_dicts(action
, properties
, extra
=extra
))
713 num_changes
= self
.db
.update_rows('vim_wim_actions', UPDATE
=updates
, WHERE
=condition
)
715 if num_changes
is None:
716 raise UnexpectedDatabaseError(
717 'Impossible to update vim_wim_actions '
718 '{instance_action_id}[{task_index}]'.format(*action
))
722 def get_wan_links(self
, uuid
=None, **kwargs
):
723 """Retrieve WAN link records from the database
726 uuid, instance_scenario_id, sce_net_id, wim_id, wim_account_id:
727 attributes that can be used at the WHERE clause
729 kwargs
.setdefault('uuid', uuid
)
730 kwargs
.setdefault('error_if_none', False)
732 criteria_fields
= ('uuid', 'instance_scenario_id', 'sce_net_id',
733 'wim_id', 'wim_account_id')
734 criteria
= remove_none_items(filter_dict_keys(kwargs
, criteria_fields
))
735 kwargs
= filter_out_dict_keys(kwargs
, criteria_fields
)
737 return self
.query('instance_wim_nets', WHERE
=criteria
, **kwargs
)
739 def update_wan_link(self
, uuid
, properties
):
740 wan_link
= self
.get_by_uuid('instance_wim_nets', uuid
)
742 wim_info
= remove_none_items(merge_dicts(
743 wan_link
.get('wim_info') or {},
744 properties
.get('wim_info') or {}))
746 updates
= preprocess_record(
747 merge_dicts(wan_link
, properties
, wim_info
=wim_info
))
749 self
.logger
.debug({'UPDATE': updates
})
750 num_changes
= self
.db
.update_rows(
751 'instance_wim_nets', UPDATE
=updates
,
752 WHERE
={'uuid': wan_link
['uuid']})
754 if num_changes
is None:
755 raise UnexpectedDatabaseError(
756 'Impossible to update instance_wim_nets ' + wan_link
['uuid'])
760 def get_instance_nets(self
, instance_scenario_id
, sce_net_id
, **kwargs
):
761 """Retrieve all the instance nets related to the same instance_scenario
766 WHERE
={'instance_scenario_id': instance_scenario_id
,
767 'sce_net_id': sce_net_id
},
769 'ORDER_BY', ('instance_scenario_id', 'sce_net_id')),
772 def update_instance_action_counters(self
, uuid
, failed
=None, done
=None):
773 """Atomically increment/decrement number_done and number_failed fields
774 in the instance action table
776 changes
= remove_none_items({
777 'number_failed': failed
and {'INCREMENT': failed
},
778 'number_done': done
and {'INCREMENT': done
}
784 return self
.db
.update_rows('instance_actions', WHERE
={'uuid': uuid
}, UPDATE
=changes
)
786 def get_only_vm_with_external_net(self
, instance_net_id
, **kwargs
):
787 """Return an instance VM if that is the only VM connected to an
788 external network identified by instance_net_id
790 counting
= ('SELECT DISTINCT instance_net_id '
791 'FROM instance_interfaces '
792 'WHERE instance_net_id="{}" AND type="external" '
793 'GROUP BY instance_net_id '
794 'HAVING COUNT(*)=1').format(self
.safe_str(instance_net_id
))
796 vm_item
= ('SELECT DISTINCT instance_vm_id '
797 'FROM instance_interfaces NATURAL JOIN ({}) AS a'
800 return self
.query_one(
801 'instance_vms JOIN ({}) as instance_interface '
802 'ON instance_vms.uuid=instance_interface.instance_vm_id'
803 .format(vm_item
), **kwargs
)
805 def safe_str(self
, string
):
806 """Return a SQL safe string"""
807 return self
.db
.escape_string(string
)
812 def _generate_port_mapping_id(self
, mapping_info
):
813 """Given a port mapping represented by a dict with a 'type' field,
814 generate a unique string, in a injective way.
816 mapping_info
= mapping_info
.copy() # Avoid mutating original object
817 mapping_type
= mapping_info
.pop('mapping_type', None)
819 raise UndefinedWanMappingType(mapping_info
)
821 unique_fields
= UNIQUE_PORT_MAPPING_INFO_FIELDS
.get(mapping_type
)
824 mapping_info
= filter_dict_keys(mapping_info
, unique_fields
)
826 self
.logger
.warning('Unique fields for WIM port mapping of type '
827 '%s not defined. Please add a list of fields '
828 'which combination should be unique in '
829 'UNIQUE_PORT_MAPPING_INFO_FIELDS '
830 '(`wim/persistency.py) ', mapping_type
)
832 repeatable_repr
= json
.dumps(mapping_info
, encoding
='utf-8',
833 sort_keys
=True, indent
=False)
835 return ':'.join([mapping_type
, _str2id(repeatable_repr
)])
838 def _serialize(value
):
839 """Serialize an arbitrary value in a consistent way,
840 so it can be stored in a database inside a text field
842 return yaml
.safe_dump(value
, default_flow_style
=True, width
=256)
845 def _unserialize(text
):
846 """Unserialize text representation into an arbitrary value,
847 so it can be loaded from the database
849 return yaml
.safe_load(text
)
852 def preprocess_record(record
):
853 """Small transformations to be applied to the data that cames from the
854 user before writing it to the database. By default, filter out timestamps,
855 and serialize the ``config`` field.
857 automatic_fields
= ['created_at', 'modified_at']
858 record
= serialize_fields(filter_out_dict_keys(record
, automatic_fields
))
863 def _preprocess_wim_account(wim_account
):
864 """Do the default preprocessing and convert the 'created' field from
867 wim_account
= preprocess_record(wim_account
)
869 created
= wim_account
.get('created')
870 wim_account
['created'] = (
871 'true' if created
is True or created
== 'true' else 'false')
876 def _postprocess_record(record
, hide
=_CONFIDENTIAL_FIELDS
):
877 """By default, hide passwords fields, unserialize ``config`` fields, and
878 convert float timestamps to strings
880 record
= hide_confidential_fields(record
, hide
)
881 record
= unserialize_fields(record
, hide
)
883 convert_float_timestamp2str(record
)
888 def _postprocess_action(action
):
889 if action
.get('extra'):
890 action
['extra'] = _unserialize(action
['extra'])
895 def _postprocess_wim_account(wim_account
, hide
=_CONFIDENTIAL_FIELDS
):
896 """Do the default postprocessing and convert the 'created' field from
899 # Fix fields from join
900 for field
in ('type', 'description', 'wim_url'):
901 if field
in wim_account
:
902 wim_account
['wim.'+field
] = wim_account
.pop(field
)
904 for field
in ('id', 'nfvo_tenant_id', 'wim_account_id'):
905 if field
in wim_account
:
906 wim_account
['association.'+field
] = wim_account
.pop(field
)
908 wim_account
= _postprocess_record(wim_account
, hide
)
910 created
= wim_account
.get('created')
911 wim_account
['created'] = (created
is True or created
== 'true')
916 def _postprocess_wim_port_mapping(mapping
, hide
=_CONFIDENTIAL_FIELDS
):
917 mapping
= _postprocess_record(mapping
, hide
=hide
)
918 mapping_info
= mapping
.get('wan_service_mapping_info', None) or {}
919 mapping
['wan_service_mapping_info'] = mapping_info
923 def hide_confidential_fields(record
, fields
=_CONFIDENTIAL_FIELDS
):
924 """Obfuscate confidential fields from the input dict.
927 This function performs a SHALLOW operation.
929 if not(isinstance(record
, dict) and fields
):
932 keys
= record
.iterkeys()
933 keys
= (k
for k
in keys
for f
in fields
if k
== f
or k
.endswith('.'+f
))
935 return merge_dicts(record
, {k
: '********' for k
in keys
if record
[k
]})
938 def unserialize_fields(record
, hide
=_CONFIDENTIAL_FIELDS
,
939 fields
=_SERIALIZED_FIELDS
):
940 """Unserialize fields that where stored in the database as a serialized
943 keys
= record
.iterkeys()
944 keys
= (k
for k
in keys
for f
in fields
if k
== f
or k
.endswith('.'+f
))
946 return merge_dicts(record
, {
947 key
: hide_confidential_fields(_unserialize(record
[key
]), hide
)
948 for key
in keys
if record
[key
]
952 def serialize_fields(record
, fields
=_SERIALIZED_FIELDS
):
953 """Serialize fields to be stored in the database as YAML"""
954 keys
= record
.iterkeys()
955 keys
= (k
for k
in keys
for f
in fields
if k
== f
or k
.endswith('.'+f
))
957 return merge_dicts(record
, {
958 key
: _serialize(record
[key
])
959 for key
in keys
if record
[key
] is not None
963 def _decide_name_or_uuid(value
):
966 if isinstance(value
, (list, tuple)):
967 reference
= value
[0] if value
else ''
969 return 'uuid' if check_valid_uuid(reference
) else 'name'
972 def _compose_where_from_uuids_or_names(**conditions
):
973 """Create a dict containing the right conditions to be used in a database
976 This function chooses between ``names`` and ``uuid`` fields based on the
977 format of the passed string.
978 If a list is passed, the first element of the list will be used to choose
979 the name of the field.
980 If a ``None`` value is passed, ``uuid`` is used.
982 Note that this function automatically translates ``tenant`` to
983 ``nfvo_tenant`` for the sake of brevity.
986 >>> _compose_where_from_uuids_or_names(
988 tenant=['xyz123', 'def456']
989 datacenter='5286a274-8a1b-4b8d-a667-9c94261ad855')
990 {'wim.name': 'abcdef',
991 'nfvo_tenant.name': ['xyz123', 'def456']
992 'datacenter.uuid': '5286a274-8a1b-4b8d-a667-9c94261ad855'}
994 if 'tenant' in conditions
:
995 conditions
['nfvo_tenant'] = conditions
.pop('tenant')
998 '{}.{}'.format(kind
, _decide_name_or_uuid(value
)): value
999 for kind
, value
in conditions
.items() if value
1004 """Create an ID (following the UUID format) from a piece of arbitrary
1007 Different texts should generate different IDs, and the same text should
1008 generate the same ID in a repeatable way.
1010 return sha1(text
).hexdigest()