1 # -*- coding: utf-8 -*-
3 # Copyright 2018 University of Bristol - High Performance Networks Research
7 # Contributors: Anderson Bravalheri, Dimitrios Gkounis, Abubakar Siddique
8 # Muqaddas, Navdeep Uniyal, Reza Nejabati and Dimitra Simeonidou
10 # Licensed under the Apache License, Version 2.0 (the "License"); you may
11 # not use this file except in compliance with the License. You may obtain
12 # a copy of the License at
14 # http://www.apache.org/licenses/LICENSE-2.0
16 # Unless required by applicable law or agreed to in writing, software
17 # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
18 # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
19 # License for the specific language governing permissions and limitations
22 # For those usages not covered by the Apache License, Version 2.0 please
23 # contact with: <highperformance-networks@bristol.ac.uk>
25 # Neither the name of the University of Bristol nor the names of its
26 # contributors may be used to endorse or promote products derived from
27 # this software without specific prior written permission.
29 # This work has been performed in the context of DCMS UK 5G Testbeds
30 # & Trials Programme and in the framework of the Metro-Haul project -
31 # funded by the European Commission under Grant number 761727 through the
32 # Horizon 2020 and 5G-PPP programmes.
35 """This module contains only logic related to managing records in a database
36 which includes data format normalization, data format validation and etc.
37 (It works as an extension to `nfvo_db.py` for the WIM feature)
39 No domain logic/architectural concern should be present in this file.
44 from contextlib
import contextmanager
45 from hashlib
import sha1
46 from itertools
import groupby
47 from operator
import itemgetter
48 # from sys import exc_info
49 # from time import time
50 from uuid
import uuid1
as generate_uuid
56 convert_float_timestamp2str
,
69 UndefinedWanMappingType
,
70 UnexpectedDatabaseError
,
72 WimAndTenantAlreadyAttached
79 ' JOIN wim_nfvo_tenants AS association '
80 ' ON association.wim_id=wim.uuid '
81 ' JOIN nfvo_tenants AS nfvo_tenant '
82 ' ON association.nfvo_tenant_id=nfvo_tenant.uuid '
83 ' JOIN wim_accounts AS wim_account '
84 ' ON association.wim_account_id=wim_account.uuid '
88 'wim_accounts AS wim_account '
89 ' JOIN wim_nfvo_tenants AS association '
90 ' ON association.wim_account_id=wim_account.uuid '
92 ' ON association.wim_id=wim.uuid '
93 ' JOIN nfvo_tenants AS nfvo_tenant '
94 ' ON association.nfvo_tenant_id=nfvo_tenant.uuid '
98 'datacenters AS datacenter '
99 ' JOIN tenants_datacenters AS association '
100 ' ON association.datacenter_id=datacenter.uuid '
101 ' JOIN datacenter_tenants as datacenter_account '
102 ' ON association.datacenter_tenant_id=datacenter_account.uuid '
103 ' JOIN nfvo_tenants AS nfvo_tenant '
104 ' ON association.nfvo_tenant_id=nfvo_tenant.uuid '
107 _PORT_MAPPING
= 'wim_port_mappings as wim_port_mapping '
109 _PORT_MAPPING_JOIN_WIM
= (
111 ' ON wim_port_mapping.wim_id=wim.uuid '
114 _PORT_MAPPING_JOIN_DATACENTER
= (
115 ' JOIN datacenters as datacenter '
116 ' ON wim_port_mapping.datacenter_id=datacenter.uuid '
120 'wim.{0} as {0}'.format(_field
)
121 for _field
in 'uuid name description wim_url type config '
122 'created_at modified_at'.split()
125 _WIM_ACCOUNT_SELECT
= 'uuid name user password config'.split()
127 _PORT_MAPPING_SELECT
= ('wim_port_mapping.*', )
129 _CONFIDENTIAL_FIELDS
= ('password', 'passwd')
131 _SERIALIZED_FIELDS
= ('config', 'vim_info', 'wim_info', 'conn_info', 'extra',
132 'service_mapping_info')
134 UNIQUE_PORT_MAPPING_INFO_FIELDS
= {
135 'dpid-port': ('switch_dpid', 'switch_port')
137 """Fields that should be unique for each port mapping that relies on
138 service_mapping_info.
140 For example, for port mappings of type 'dpid-port', each combination of
141 switch_dpid and switch_port should be unique (the same switch cannot
142 be connected to two different places using the same port)
146 class WimPersistence(object):
147 """High level interactions with the WIM tables in the database"""
149 def __init__(self
, db
, logger
=None):
151 self
.logger
= logger
or logging
.getLogger('openmano.wim.persistence')
161 error_if_multiple
=False,
163 hide
=_CONFIDENTIAL_FIELDS
,
165 """Retrieve records from the database.
168 SELECT, FROM, WHERE, LIMIT, ORDER_BY: used to compose the SQL
169 query. See ``nfvo_db.get_rows``.
170 OFFSET: only valid when used togheter with LIMIT.
171 Ignore the OFFSET first results of the query.
172 error_if_none: by default an error is raised if no record is
173 found. With this option it is possible to disable this error.
174 error_if_multiple: by default no error is raised if more then one
176 With this option it is possible to enable this error.
177 postprocess: function applied to every retrieved record.
178 This function receives a dict as input and must return it
179 after modifications. Moreover this function should accept a
180 second optional parameter ``hide`` indicating
181 the confidential fiels to be obfuscated.
182 By default a minimal postprocessing function is applied,
183 obfuscating confidential fields and converting timestamps.
184 hide: option proxied to postprocess
186 All the remaining keyword arguments will be assumed to be ``name``s or
187 ``uuid``s to compose the WHERE statement, according to their format.
188 If the value corresponds to an array, the first element will determine
189 if it is an name or UUID.
192 - ``wim="abcdef"``` will be turned into ``wim.name="abcdef"``,
193 - ``datacenter="5286a274-8a1b-4b8d-a667-9c94261ad855"``
195 ``datacenter.uuid="5286a274-8a1b-4b8d-a667-9c94261ad855"``.
196 - ``wim=["5286a274-8a1b-4b8d-a667-9c94261ad855", ...]``
198 ``wim.uuid=["5286a274-8a1b-4b8d-a667-9c94261ad855", ...]``
201 NoRecordFound: if the query result set is empty
202 DbBaseException: errors occuring during the execution of the query.
205 postprocess
= postprocess
or _postprocess_record
208 # Find remaining keywords by name or uuid
209 WHERE
.update(_compose_where_from_uuids_or_names(**kwargs
))
210 WHERE
= WHERE
or None
211 # ^ If the where statement is empty, it is better to leave it as None,
212 # so it can be filtered out at a later stage
213 LIMIT
= ('{:d},{:d}'.format(OFFSET
, LIMIT
)
214 if LIMIT
and OFFSET
else LIMIT
)
216 query
= remove_none_items({
217 'SELECT': SELECT
, 'FROM': FROM
, 'WHERE': WHERE
,
218 'LIMIT': LIMIT
, 'ORDER_BY': ORDER_BY
})
220 records
= self
.db
.get_rows(**query
)
222 table
= FROM
.split()[0]
223 if error_if_none
and not records
:
224 raise NoRecordFound(WHERE
, table
)
226 if error_if_multiple
and len(records
) > 1:
227 self
.logger
.error('Multiple records '
228 'FROM %s WHERE %s:\n\n%s\n\n',
229 FROM
, WHERE
, json
.dumps(records
, indent
=4))
230 raise MultipleRecordsFound(WHERE
, table
)
233 expand_joined_fields(postprocess(record
, hide
))
234 for record
in records
237 def query_one(self
, *args
, **kwargs
):
238 """Similar to ``query``, but ensuring just one result.
239 ``error_if_multiple`` is enabled by default.
241 kwargs
.setdefault('error_if_multiple', True)
242 records
= self
.query(*args
, **kwargs
)
243 return records
[0] if records
else None
245 def get_by_uuid(self
, table
, uuid
, **kwargs
):
246 """Retrieve one record from the database based on its uuid
249 table (str): table name (to be used in SQL's FROM statement).
250 uuid (str): unique identifier for record.
252 For additional keyword arguments and exceptions see :obj:`~.query`
253 (``error_if_multiple`` is enabled by default).
256 raise UndefinedUuidOrName(table
)
257 return self
.query_one(table
, WHERE
={'uuid': uuid
}, **kwargs
)
259 def get_by_name_or_uuid(self
, table
, uuid_or_name
, **kwargs
):
260 """Retrieve a record from the database based on a value that can be its
264 table (str): table name (to be used in SQL's FROM statement).
265 uuid_or_name (str): this value can correspond to either uuid or
267 For additional keyword arguments and exceptions see :obj:`~.query`
268 (``error_if_multiple`` is enabled by default).
270 if uuid_or_name
is None:
271 raise UndefinedUuidOrName(table
)
273 key
= 'uuid' if check_valid_uuid(uuid_or_name
) else 'name'
274 return self
.query_one(table
, WHERE
={key
: uuid_or_name
}, **kwargs
)
276 def get_wims(self
, uuid_or_name
=None, tenant
=None, **kwargs
):
277 """Retrieve information about one or more WIMs stored in the database
280 uuid_or_name (str): uuid or name for WIM
281 tenant (str): [optional] uuid or name for NFVO tenant
283 See :obj:`~.query` for additional keyword arguments.
285 kwargs
.update(wim
=uuid_or_name
, tenant
=tenant
)
286 from_
= _WIM_JOIN
if tenant
else _WIM
287 select_
= _WIM_SELECT
[:] + (['wim_account.*'] if tenant
else [])
289 kwargs
.setdefault('SELECT', select_
)
290 return self
.query(from_
, **kwargs
)
292 def get_wim(self
, wim
, tenant
=None, **kwargs
):
293 """Similar to ``get_wims`` but ensure only one result is returned"""
294 kwargs
.setdefault('error_if_multiple', True)
295 return self
.get_wims(wim
, tenant
)[0]
297 def create_wim(self
, wim_descriptor
):
298 """Create a new wim record inside the database and returns its uuid
301 wim_descriptor (dict): properties of the record
302 (usually each field corresponds to a database column, but extra
303 information can be offloaded to another table or serialized as
306 str: UUID of the created WIM
308 if "config" in wim_descriptor
:
309 wim_descriptor
["config"] = _serialize(wim_descriptor
["config"])
311 url
= wim_descriptor
["wim_url"]
312 wim_descriptor
["wim_url"] = url
.strip(string
.whitespace
+ "/")
313 # ^ This avoid the common problem caused by trailing spaces/slashes in
314 # the URL (due to CTRL+C/CTRL+V)
316 return self
.db
.new_row(
317 "wims", wim_descriptor
, add_uuid
=True, confidential_data
=True)
319 def update_wim(self
, uuid_or_name
, wim_descriptor
):
320 """Change an existing WIM record on the database"""
321 # obtain data, check that only one exist
322 wim
= self
.get_by_name_or_uuid('wims', uuid_or_name
)
326 where
= {'uuid': wim
['uuid']}
328 # unserialize config, edit and serialize it again
329 new_config_dict
= wim_descriptor
.get('config', {}) or {}
330 config_dict
= remove_none_items(merge_dicts(
331 wim
.get('config', {}) or {}, new_config_dict
))
332 wim_descriptor
['config'] = (
333 _serialize(config_dict
) if config_dict
else None)
335 self
.db
.update_rows('wims', wim_descriptor
, where
)
339 def delete_wim(self
, wim
):
340 # get nfvo_tenant info
341 wim
= self
.get_by_name_or_uuid('wims', wim
)
343 self
.db
.delete_row_by_id('wims', wim
['uuid'])
345 return wim
['uuid'] + ' ' + wim
['name']
347 def get_wim_accounts_by(self
, wim
=None, tenant
=None, uuid
=None, **kwargs
):
348 """Retrieve WIM account information from the database together
349 with the related records (wim, nfvo_tenant and wim_nfvo_tenant)
352 wim (str): uuid or name for WIM
353 tenant (str): [optional] uuid or name for NFVO tenant
355 See :obj:`~.query` for additional keyword arguments.
357 kwargs
.update(wim
=wim
, tenant
=tenant
)
358 kwargs
.setdefault('postprocess', _postprocess_wim_account
)
360 kwargs
.setdefault('WHERE', {'wim_account.uuid': uuid
})
361 return self
.query(FROM
=_WIM_ACCOUNT_JOIN
, **kwargs
)
363 def get_wim_account_by(self
, wim
=None, tenant
=None, uuid
=None, **kwargs
):
364 """Similar to ``get_wim_accounts_by``, but ensuring just one result"""
365 kwargs
.setdefault('error_if_multiple', True)
366 return self
.get_wim_accounts_by(wim
, tenant
, uuid
, **kwargs
)[0]
368 def get_wim_accounts(self
, **kwargs
):
369 """Retrieve all the accounts from the database"""
370 kwargs
.setdefault('postprocess', _postprocess_wim_account
)
371 kwargs
.setdefault('WHERE', {"sdn": "false"})
372 return self
.query(FROM
=_WIM_ACCOUNT_JOIN
, **kwargs
)
374 def get_wim_account(self
, uuid_or_name
, **kwargs
):
375 """Retrieve WIM Account record by UUID or name,
376 See :obj:`get_by_name_or_uuid` for keyword arguments.
378 kwargs
.setdefault('postprocess', _postprocess_wim_account
)
379 kwargs
.setdefault('SELECT', _WIM_ACCOUNT_SELECT
)
380 return self
.get_by_name_or_uuid('wim_accounts', uuid_or_name
, **kwargs
)
383 def _associate(self
, wim_id
, nfvo_tenant_id
):
384 """Auxiliary method for ``create_wim_account``
386 This method just create a row in the association table
391 except DbBaseException
as db_exception
:
392 error_msg
= str(db_exception
)
393 if all([msg
in error_msg
394 for msg
in ("already in use", "'wim_nfvo_tenant'")]):
395 ex
= WimAndTenantAlreadyAttached(wim_id
, nfvo_tenant_id
)
396 raise ex
from db_exception
399 def create_wim_account(self
, wim
, tenant
, properties
):
400 """Associate a wim to a tenant using the ``wim_nfvo_tenants`` table
401 and create a ``wim_account`` to store credentials and configurations.
403 For the sake of simplification, we assume that each NFVO tenant can be
404 attached to a WIM using only one WIM account. This is automatically
405 guaranteed via database constraints.
406 For corner cases, the same WIM can be registered twice using another
410 wim (str): name or uuid of the WIM related to the account being
412 tenant (str): name or uuid of the nfvo tenant to which the account
414 properties (dict): properties of the account
415 (eg. user, password, ...)
417 wim_id
= self
.get_by_name_or_uuid('wims', wim
, SELECT
=['uuid'])['uuid']
418 tenant
= self
.get_by_name_or_uuid('nfvo_tenants', tenant
,
419 SELECT
=['uuid', 'name'])
420 account
= properties
.setdefault('name', tenant
['name'])
422 wim_account
= self
.query_one('wim_accounts',
423 WHERE
={'wim_id': wim_id
, 'name': account
},
429 if wim_account
is None:
430 # If a row for the wim account doesn't exist yet, we need to
431 # create one, otherwise we can just re-use it.
432 account_id
= str(generate_uuid())
433 used_uuids
.append(account_id
)
434 row
= merge_dicts(properties
, wim_id
=wim_id
, uuid
=account_id
)
435 transaction
.append({'wim_accounts': _preprocess_wim_account(row
)})
437 account_id
= wim_account
['uuid']
438 properties
.pop('config', None) # Config is too complex to compare
439 diff
= {k
: v
for k
, v
in properties
.items() if v
!= wim_account
[k
]}
441 tip
= 'Edit the account first, and then attach it to a tenant'
442 raise WimAccountOverwrite(wim_account
, diff
, tip
)
445 'wim_nfvo_tenants': {'nfvo_tenant_id': tenant
['uuid'],
447 'wim_account_id': account_id
}})
449 with self
._associate
(wim_id
, tenant
['uuid']):
450 self
.db
.new_rows(transaction
, used_uuids
, confidential_data
=True)
454 def update_wim_account(self
, uuid
, properties
, hide
=_CONFIDENTIAL_FIELDS
):
455 """Update WIM account record by overwriting fields with new values
457 Specially for the field ``config`` this means that a new dict will be
458 merged to the existing one.
461 uuid (str): UUID for the WIM account
462 properties (dict): fields that should be overwritten
467 wim_account
= self
.get_by_uuid('wim_accounts', uuid
)
468 safe_fields
= 'user password name created'.split()
469 updates
= _preprocess_wim_account(
470 merge_dicts(wim_account
, filter_dict_keys(properties
, safe_fields
))
473 if properties
.get('config'):
474 old_config
= wim_account
.get('config') or {}
475 new_config
= merge_dicts(old_config
, properties
['config'])
476 updates
['config'] = _serialize(new_config
)
478 num_changes
= self
.db
.update_rows('wim_accounts', UPDATE
=updates
,
479 WHERE
={'uuid': wim_account
['uuid']})
481 if num_changes
is None:
482 raise UnexpectedDatabaseError('Impossible to update wim_account '
483 '{name}:{uuid}'.format(*wim_account
))
485 return self
.get_wim_account(wim_account
['uuid'], hide
=hide
)
487 def delete_wim_account(self
, uuid
):
488 """Remove WIM account record from the database"""
489 # Since we have foreign keys configured with ON CASCADE, we can rely
490 # on the database engine to guarantee consistency, deleting the
492 return self
.db
.delete_row_by_id('wim_accounts', uuid
)
494 def get_datacenters_by(self
, datacenter
=None, tenant
=None, **kwargs
):
495 """Retrieve datacenter information from the database together
496 with the related records (nfvo_tenant)
499 datacenter (str): uuid or name for datacenter
500 tenant (str): [optional] uuid or name for NFVO tenant
502 See :obj:`~.query` for additional keyword arguments.
505 kwargs
.update(datacenter
=datacenter
, tenant
=tenant
)
506 return self
.query(_DATACENTER_JOIN
, **kwargs
)
508 return [self
.get_by_name_or_uuid('datacenters',
509 datacenter
, **kwargs
)]
511 def get_datacenter_by(self
, datacenter
=None, tenant
=None, **kwargs
):
512 """Similar to ``get_datacenters_by``, but ensuring just one result"""
513 kwargs
.setdefault('error_if_multiple', True)
514 return self
.get_datacenters_by(datacenter
, tenant
, **kwargs
)[0]
516 def _create_single_port_mapping(self
, properties
):
517 info
= properties
.setdefault('service_mapping_info', {})
518 endpoint_id
= properties
.get('service_endpoint_id')
520 if info
.get('mapping_type') and not endpoint_id
:
521 properties
['service_endpoint_id'] = (
522 self
._generate
_port
_mapping
_id
(info
))
524 properties
['service_mapping_info'] = _serialize(info
)
527 self
.db
.new_row('wim_port_mappings', properties
,
528 add_uuid
=False, confidential_data
=True)
529 except DbBaseException
as old_exception
:
530 self
.logger
.exception(old_exception
)
531 ex
= InvalidParameters(
532 "The mapping must contain the "
533 "'device_id', 'device_interface_id', and "
534 "service_mapping_info: "
535 "('switch_dpid' and 'switch_port') or "
536 "'service_endpoint_id}'")
537 raise ex
from old_exception
541 def create_wim_port_mappings(self
, wim
, port_mappings
, tenant
=None):
542 if not isinstance(wim
, dict):
543 wim
= self
.get_by_name_or_uuid('wims', wim
)
545 for port_mapping
in port_mappings
:
546 port_mapping
['wim_name'] = wim
['name']
547 datacenter
= self
.get_datacenter_by(
548 port_mapping
['datacenter_name'], tenant
)
549 for pop_wan_port_mapping
in port_mapping
['pop_wan_mappings']:
550 element
= merge_dicts(pop_wan_port_mapping
, {
551 'wim_id': wim
['uuid'],
552 'datacenter_id': datacenter
['uuid']})
553 self
._create
_single
_port
_mapping
(element
)
557 def _filter_port_mappings_by_tenant(self
, mappings
, tenant
):
558 """Make sure all the datacenters and wims listed in the port mapping
559 belong to an specific tenant
562 # NOTE: Theoretically this could be done at SQL level, but given the
563 # number of tables involved (wim_port_mappings, wim_accounts,
564 # wims, wim_nfvo_tenants, datacenters, datacenter_tenants,
565 # tenants_datacents and nfvo_tenants), it would result in a
566 # extremely complex query. Moreover, the predicate can vary:
567 # for `get_wim_port_mappings` we can have any combination of
568 # (wim, datacenter, tenant), not all of them having the 3 values
569 # so we have combinatorial trouble to write the 'FROM' statement.
571 kwargs
= {'tenant': tenant
, 'error_if_none': False}
572 # Cache results to speedup things
576 def _get_datacenter(uuid
):
578 datacenters
.get(uuid
) or
579 datacenters
.setdefault(
580 uuid
, self
.get_datacenters_by(uuid
, **kwargs
)))
583 return (wims
.get(uuid
) or
584 wims
.setdefault(uuid
, self
.get_wims(uuid
, **kwargs
)))
588 for mapping
in mappings
589 if (_get_datacenter(mapping
['datacenter_id']) and
590 _get_wims(mapping
['wim_id']))
593 def get_wim_port_mappings(self
, wim
=None, datacenter
=None, tenant
=None,
595 """List all the port mappings, optionally filtering by wim, datacenter
598 from_
= [_PORT_MAPPING
,
599 _PORT_MAPPING_JOIN_WIM
if wim
else '',
600 _PORT_MAPPING_JOIN_DATACENTER
if datacenter
else '']
602 criteria
= ('wim_id', 'datacenter_id')
603 kwargs
.setdefault('error_if_none', False)
604 mappings
= self
.query(
606 SELECT
=_PORT_MAPPING_SELECT
,
607 ORDER_BY
=['wim_port_mapping.{}'.format(c
) for c
in criteria
],
608 wim
=wim
, datacenter
=datacenter
,
609 postprocess
=_postprocess_wim_port_mapping
,
613 mappings
= self
._filter
_port
_mappings
_by
_tenant
(mappings
, tenant
)
615 # We don't have to sort, since we have used 'ORDER_BY'
616 grouped_mappings
= groupby(mappings
, key
=itemgetter(*criteria
))
620 'datacenter_id': key
[1],
621 'pop_wan_mappings': [
622 filter_out_dict_keys(mapping
, (
623 'id', 'wim_id', 'datacenter_id',
624 'created_at', 'modified_at'))
625 for mapping
in group
]}
626 for key
, group
in grouped_mappings
629 def delete_wim_port_mappings(self
, wim_id
):
630 self
.db
.delete_row(FROM
='wim_port_mappings', WHERE
={"wim_id": wim_id
})
631 return "port mapping for wim {} deleted.".format(wim_id
)
633 def update_wim_port_mapping(self
, id, properties
):
634 original
= self
.query_one('wim_port_mappings', WHERE
={'id': id})
636 mapping_info
= remove_none_items(merge_dicts(
637 original
.get('service_mapping_info') or {},
638 properties
.get('service_mapping_info') or {}))
640 updates
= preprocess_record(
641 merge_dicts(original
, remove_none_items(properties
),
642 service_mapping_info
=mapping_info
))
644 num_changes
= self
.db
.update_rows('wim_port_mappings',
645 UPDATE
=updates
, WHERE
={'id': id})
647 if num_changes
is None:
648 raise UnexpectedDatabaseError(
649 'Impossible to update wim_port_mappings {}:\n{}\n'.format(
650 id, _serialize(properties
))
655 def get_actions_in_groups(self
, wim_account_id
,
656 item_types
=('instance_wim_nets',),
657 group_offset
=0, group_limit
=150):
658 """Retrieve actions from the database in groups.
659 Each group contains all the actions that have the same ``item`` type
663 wim_account_id: restrict the search to actions to be performed
664 using the same account
665 item_types (list): [optional] filter the actions to the given
667 group_limit (int): maximum number of groups returned by the
669 group_offset (int): skip the N first groups. Used together with
670 group_limit for pagination purposes.
673 List of groups, where each group is a tuple ``(key, actions)``.
674 In turn, ``key`` is a tuple containing the values of
675 ``(item, item_id)`` used to create the group and ``actions`` is a
676 list of ``vim_wim_actions`` records (dicts).
680 '"{}"'.format(self
.db
.escape_string(t
)) for t
in item_types
)
682 items
= ('SELECT DISTINCT a.item, a.item_id, a.wim_account_id '
683 'FROM vim_wim_actions AS a '
684 'WHERE a.wim_account_id="{}" AND a.item IN ({}) '
685 'ORDER BY a.item, a.item_id '
686 'LIMIT {:d},{:d}').format(
687 self
.safe_str(wim_account_id
),
688 ','.join(type_options
),
689 group_offset
, group_limit
)
691 join
= 'vim_wim_actions NATURAL JOIN ({}) AS items'.format(items
)
692 db_results
= self
.db
.get_rows(
693 FROM
=join
, ORDER_BY
=('item', 'item_id', 'created_at'))
695 results
= (_postprocess_action(r
) for r
in db_results
)
696 criteria
= itemgetter('item', 'item_id')
697 return [(k
, list(g
)) for k
, g
in groupby(results
, key
=criteria
)]
699 def update_action(self
, instance_action_id
, task_index
, properties
):
700 condition
= {'instance_action_id': instance_action_id
,
701 'task_index': task_index
}
703 action
= self
.query_one('vim_wim_actions', WHERE
=condition
)
705 actions
= self
.query('vim_wim_actions', WHERE
=condition
)
706 self
.logger
.error('More then one action found:\n%s',
707 json
.dumps(actions
, indent
=4))
710 extra
= remove_none_items(merge_dicts(
711 action
.get('extra') or {},
712 properties
.get('extra') or {}))
714 updates
= preprocess_record(
715 merge_dicts(action
, properties
, extra
=extra
))
717 num_changes
= self
.db
.update_rows('vim_wim_actions', UPDATE
=updates
, WHERE
=condition
)
719 if num_changes
is None:
720 raise UnexpectedDatabaseError(
721 'Impossible to update vim_wim_actions '
722 '{instance_action_id}[{task_index}]'.format(*action
))
726 def get_wan_links(self
, uuid
=None, **kwargs
):
727 """Retrieve WAN link records from the database
730 uuid, instance_scenario_id, sce_net_id, wim_id, wim_account_id:
731 attributes that can be used at the WHERE clause
733 kwargs
.setdefault('uuid', uuid
)
734 kwargs
.setdefault('error_if_none', False)
736 criteria_fields
= ('uuid', 'instance_scenario_id', 'sce_net_id',
737 'wim_id', 'wim_account_id', 'sdn')
738 criteria
= remove_none_items(filter_dict_keys(kwargs
, criteria_fields
))
739 kwargs
= filter_out_dict_keys(kwargs
, criteria_fields
)
741 return self
.query('instance_wim_nets', WHERE
=criteria
, **kwargs
)
743 def update_wan_link(self
, uuid
, properties
):
744 wan_link
= self
.get_by_uuid('instance_wim_nets', uuid
)
746 wim_info
= remove_none_items(merge_dicts(
747 wan_link
.get('wim_info') or {},
748 properties
.get('wim_info') or {}))
750 updates
= preprocess_record(
751 merge_dicts(wan_link
, properties
, wim_info
=wim_info
))
753 self
.logger
.debug({'UPDATE': updates
})
754 num_changes
= self
.db
.update_rows(
755 'instance_wim_nets', UPDATE
=updates
,
756 WHERE
={'uuid': wan_link
['uuid']})
758 if num_changes
is None:
759 raise UnexpectedDatabaseError(
760 'Impossible to update instance_wim_nets ' + wan_link
['uuid'])
764 def get_instance_nets(self
, instance_scenario_id
, sce_net_id
, **kwargs
):
765 """Retrieve all the instance nets related to the same instance_scenario
770 WHERE
={'instance_scenario_id': instance_scenario_id
,
771 'sce_net_id': sce_net_id
},
773 'ORDER_BY', ('instance_scenario_id', 'sce_net_id')),
776 def update_instance_action_counters(self
, uuid
, failed
=None, done
=None):
777 """Atomically increment/decrement number_done and number_failed fields
778 in the instance action table
780 changes
= remove_none_items({
781 'number_failed': failed
and {'INCREMENT': failed
},
782 'number_done': done
and {'INCREMENT': done
}
788 return self
.db
.update_rows('instance_actions', WHERE
={'uuid': uuid
}, UPDATE
=changes
)
790 def get_only_vm_with_external_net(self
, instance_net_id
, **kwargs
):
791 """Return an instance VM if that is the only VM connected to an
792 external network identified by instance_net_id
794 counting
= ('SELECT DISTINCT instance_net_id '
795 'FROM instance_interfaces '
796 'WHERE instance_net_id="{}" AND type="external" '
797 'GROUP BY instance_net_id '
798 'HAVING COUNT(*)=1').format(self
.safe_str(instance_net_id
))
800 vm_item
= ('SELECT DISTINCT instance_vm_id '
801 'FROM instance_interfaces NATURAL JOIN ({}) AS a'
804 return self
.query_one(
805 'instance_vms JOIN ({}) as instance_interface '
806 'ON instance_vms.uuid=instance_interface.instance_vm_id'
807 .format(vm_item
), **kwargs
)
809 def safe_str(self
, string
):
810 """Return a SQL safe string"""
811 return self
.db
.escape_string(string
)
816 def _generate_port_mapping_id(self
, mapping_info
):
817 """Given a port mapping represented by a dict with a 'type' field,
818 generate a unique string, in a injective way.
820 mapping_info
= mapping_info
.copy() # Avoid mutating original object
821 mapping_type
= mapping_info
.pop('mapping_type', None)
823 raise UndefinedWanMappingType(mapping_info
)
825 unique_fields
= UNIQUE_PORT_MAPPING_INFO_FIELDS
.get(mapping_type
)
828 mapping_info
= filter_dict_keys(mapping_info
, unique_fields
)
830 self
.logger
.warning('Unique fields for WIM port mapping of type '
831 '%s not defined. Please add a list of fields '
832 'which combination should be unique in '
833 'UNIQUE_PORT_MAPPING_INFO_FIELDS '
834 '(`wim/persistency.py) ', mapping_type
)
836 repeatable_repr
= json
.dumps(mapping_info
, encoding
='utf-8',
837 sort_keys
=True, indent
=False)
839 return ':'.join([mapping_type
, _str2id(repeatable_repr
)])
842 def _serialize(value
):
843 """Serialize an arbitrary value in a consistent way,
844 so it can be stored in a database inside a text field
846 return yaml
.safe_dump(value
, default_flow_style
=True, width
=256)
849 def _unserialize(text
):
850 """Unserialize text representation into an arbitrary value,
851 so it can be loaded from the database
853 return yaml
.safe_load(text
)
856 def preprocess_record(record
):
857 """Small transformations to be applied to the data that cames from the
858 user before writing it to the database. By default, filter out timestamps,
859 and serialize the ``config`` field.
861 automatic_fields
= ['created_at', 'modified_at']
862 record
= serialize_fields(filter_out_dict_keys(record
, automatic_fields
))
867 def _preprocess_wim_account(wim_account
):
868 """Do the default preprocessing and convert the 'created' field from
871 wim_account
= preprocess_record(wim_account
)
873 wim_account
['sdn'] = False
877 def _postprocess_record(record
, hide
=_CONFIDENTIAL_FIELDS
):
878 """By default, hide passwords fields, unserialize ``config`` fields, and
879 convert float timestamps to strings
881 record
= hide_confidential_fields(record
, hide
)
882 record
= unserialize_fields(record
, hide
)
884 convert_float_timestamp2str(record
)
889 def _postprocess_action(action
):
890 if action
.get('extra'):
891 action
['extra'] = _unserialize(action
['extra'])
896 def _postprocess_wim_account(wim_account
, hide
=_CONFIDENTIAL_FIELDS
):
897 """Do the default postprocessing and convert the 'created' field from
900 # Fix fields from join
901 for field
in ('type', 'description', 'wim_url'):
902 if field
in wim_account
:
903 wim_account
['wim.'+field
] = wim_account
.pop(field
)
905 for field
in ('id', 'nfvo_tenant_id', 'wim_account_id'):
906 if field
in wim_account
:
907 wim_account
['association.'+field
] = wim_account
.pop(field
)
909 wim_account
= _postprocess_record(wim_account
, hide
)
911 created
= wim_account
.get('created')
912 wim_account
['created'] = (created
is True or created
== 'true')
917 def _postprocess_wim_port_mapping(mapping
, hide
=_CONFIDENTIAL_FIELDS
):
918 mapping
= _postprocess_record(mapping
, hide
=hide
)
919 mapping_info
= mapping
.get('service_mapping_info', None) or {}
920 mapping
['service_mapping_info'] = mapping_info
924 def hide_confidential_fields(record
, fields
=_CONFIDENTIAL_FIELDS
):
925 """Obfuscate confidential fields from the input dict.
928 This function performs a SHALLOW operation.
930 if not(isinstance(record
, dict) and fields
):
933 keys
= list(record
.keys())
934 keys
= (k
for k
in keys
for f
in fields
if k
== f
or k
.endswith('.'+f
))
936 return merge_dicts(record
, {k
: '********' for k
in keys
if record
[k
]})
939 def unserialize_fields(record
, hide
=_CONFIDENTIAL_FIELDS
,
940 fields
=_SERIALIZED_FIELDS
):
941 """Unserialize fields that where stored in the database as a serialized
944 keys
= list(record
.keys())
945 keys
= (k
for k
in keys
for f
in fields
if k
== f
or k
.endswith('.'+f
))
947 return merge_dicts(record
, {
948 key
: hide_confidential_fields(_unserialize(record
[key
]), hide
)
949 for key
in keys
if record
[key
]
953 def serialize_fields(record
, fields
=_SERIALIZED_FIELDS
):
954 """Serialize fields to be stored in the database as YAML"""
955 keys
= list(record
.keys())
956 keys
= (k
for k
in keys
for f
in fields
if k
== f
or k
.endswith('.'+f
))
958 return merge_dicts(record
, {
959 key
: _serialize(record
[key
])
960 for key
in keys
if record
[key
] is not None
964 def _decide_name_or_uuid(value
):
967 if isinstance(value
, (list, tuple)):
968 reference
= value
[0] if value
else ''
970 return 'uuid' if check_valid_uuid(reference
) else 'name'
973 def _compose_where_from_uuids_or_names(**conditions
):
974 """Create a dict containing the right conditions to be used in a database
977 This function chooses between ``names`` and ``uuid`` fields based on the
978 format of the passed string.
979 If a list is passed, the first element of the list will be used to choose
980 the name of the field.
981 If a ``None`` value is passed, ``uuid`` is used.
983 Note that this function automatically translates ``tenant`` to
984 ``nfvo_tenant`` for the sake of brevity.
987 >>> _compose_where_from_uuids_or_names(
989 tenant=['xyz123', 'def456']
990 datacenter='5286a274-8a1b-4b8d-a667-9c94261ad855')
991 {'wim.name': 'abcdef',
992 'nfvo_tenant.name': ['xyz123', 'def456']
993 'datacenter.uuid': '5286a274-8a1b-4b8d-a667-9c94261ad855'}
995 if 'tenant' in conditions
:
996 conditions
['nfvo_tenant'] = conditions
.pop('tenant')
999 '{}.{}'.format(kind
, _decide_name_or_uuid(value
)): value
1000 for kind
, value
in conditions
.items() if value
1005 """Create an ID (following the UUID format) from a piece of arbitrary
1008 Different texts should generate different IDs, and the same text should
1009 generate the same ID in a repeatable way.
1011 return sha1(text
).hexdigest()