2 # Copyright 2019 Telefonica Investigacion y Desarrollo, S.A.U.
3 # This file is part of OSM
6 # Licensed under the Apache License, Version 2.0 (the "License");
7 # you may not use this file except in compliance with the License.
8 # You may obtain a copy of the License at
10 # http://www.apache.org/licenses/LICENSE-2.0
12 # Unless required by applicable law or agreed to in writing, software
13 # distributed under the License is distributed on an "AS IS" BASIS,
14 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
16 # See the License for the specific language governing permissions and
17 # limitations under the License.
19 # For those usages not covered by the Apache License, Version 2.0 please
20 # contact with: nfvlabs@tid.es
31 from juju
.action
import Action
32 from juju
.application
import Application
33 from juju
.client
import client
34 from juju
.controller
import Controller
35 from juju
.errors
import JujuAPIError
36 from juju
.machine
import Machine
37 from juju
.model
import Model
38 from n2vc
.exceptions
import (
39 N2VCBadArgumentsException
,
41 N2VCConnectionException
,
42 N2VCExecutionException
,
43 N2VCInvalidCertificate
,
46 JujuK8sProxycharmNotSupported
,
48 from n2vc
.juju_observer
import JujuModelObserver
49 from n2vc
.n2vc_conn
import N2VCConnector
50 from n2vc
.n2vc_conn
import obj_to_dict
, obj_to_yaml
51 from n2vc
.provisioner
import AsyncSSHProvisioner
52 from n2vc
.libjuju
import Libjuju
55 class N2VCJujuConnector(N2VCConnector
):
58 ####################################################################################
59 ################################### P U B L I C ####################################
60 ####################################################################################
63 BUILT_IN_CLOUDS
= ["localhost", "microk8s"]
71 url
: str = "127.0.0.1:17070",
72 username
: str = "admin",
73 vca_config
: dict = None,
76 """Initialize juju N2VC connector
79 # parent class constructor
80 N2VCConnector
.__init
__(
88 vca_config
=vca_config
,
89 on_update_db
=on_update_db
,
92 # silence websocket traffic log
93 logging
.getLogger("websockets.protocol").setLevel(logging
.INFO
)
94 logging
.getLogger("juju.client.connection").setLevel(logging
.WARN
)
95 logging
.getLogger("model").setLevel(logging
.WARN
)
97 self
.log
.info("Initializing N2VC juju connector...")
100 ##############################################################
102 ##############################################################
107 raise N2VCBadArgumentsException("Argument url is mandatory", ["url"])
108 url_parts
= url
.split(":")
109 if len(url_parts
) != 2:
110 raise N2VCBadArgumentsException(
111 "Argument url: bad format (localhost:port) -> {}".format(url
), ["url"]
113 self
.hostname
= url_parts
[0]
115 self
.port
= int(url_parts
[1])
117 raise N2VCBadArgumentsException(
118 "url port must be a number -> {}".format(url
), ["url"]
123 raise N2VCBadArgumentsException(
124 "Argument username is mandatory", ["username"]
128 if vca_config
is None:
129 raise N2VCBadArgumentsException(
130 "Argument vca_config is mandatory", ["vca_config"]
133 if "secret" in vca_config
:
134 self
.secret
= vca_config
["secret"]
136 raise N2VCBadArgumentsException(
137 "Argument vca_config.secret is mandatory", ["vca_config.secret"]
140 # pubkey of juju client in osm machine: ~/.local/share/juju/ssh/juju_id_rsa.pub
141 # if exists, it will be written in lcm container: _create_juju_public_key()
142 if "public_key" in vca_config
:
143 self
.public_key
= vca_config
["public_key"]
145 self
.public_key
= None
147 # TODO: Verify ca_cert is valid before using. VCA will crash
148 # if the ca_cert isn't formatted correctly.
149 def base64_to_cacert(b64string
):
150 """Convert the base64-encoded string containing the VCA CACERT.
156 cacert
= base64
.b64decode(b64string
).decode("utf-8")
158 cacert
= re
.sub(r
"\\n", r
"\n", cacert
,)
159 except binascii
.Error
as e
:
160 self
.log
.debug("Caught binascii.Error: {}".format(e
))
161 raise N2VCInvalidCertificate(message
="Invalid CA Certificate")
165 self
.ca_cert
= vca_config
.get("ca_cert")
167 self
.ca_cert
= base64_to_cacert(vca_config
["ca_cert"])
169 if "api_proxy" in vca_config
:
170 self
.api_proxy
= vca_config
["api_proxy"]
172 "api_proxy for native charms configured: {}".format(self
.api_proxy
)
176 "api_proxy is not configured. Support for native charms is disabled"
179 if "enable_os_upgrade" in vca_config
:
180 self
.enable_os_upgrade
= vca_config
["enable_os_upgrade"]
182 self
.enable_os_upgrade
= True
184 if "apt_mirror" in vca_config
:
185 self
.apt_mirror
= vca_config
["apt_mirror"]
187 self
.apt_mirror
= None
189 self
.cloud
= vca_config
.get('cloud')
190 self
.k8s_cloud
= None
191 if "k8s_cloud" in vca_config
:
192 self
.k8s_cloud
= vca_config
.get("k8s_cloud")
193 self
.log
.debug('Arguments have been checked')
196 self
.controller
= None # it will be filled when connect to juju
197 self
.juju_models
= {} # model objects for every model_name
198 self
.juju_observers
= {} # model observers for every model_name
200 False # while connecting to juju (to avoid duplicate connections)
202 self
._authenticated
= (
203 False # it will be True when juju connection be stablished
205 self
._creating
_model
= False # True during model creation
206 self
.libjuju
= Libjuju(
208 api_proxy
=self
.api_proxy
,
209 enable_os_upgrade
=self
.enable_os_upgrade
,
210 apt_mirror
=self
.apt_mirror
,
211 username
=self
.username
,
212 password
=self
.secret
,
220 # create juju pub key file in lcm container at
221 # ./local/share/juju/ssh/juju_id_rsa.pub
222 self
._create
_juju
_public
_key
()
224 self
.log
.info("N2VC juju connector initialized")
226 async def get_status(self
, namespace
: str, yaml_format
: bool = True):
228 # self.log.info('Getting NS status. namespace: {}'.format(namespace))
230 _nsi_id
, ns_id
, _vnf_id
, _vdu_id
, _vdu_count
= self
._get
_namespace
_components
(
233 # model name is ns_id
235 if model_name
is None:
236 msg
= "Namespace {} not valid".format(namespace
)
238 raise N2VCBadArgumentsException(msg
, ["namespace"])
241 models
= await self
.libjuju
.list_models(contains
=ns_id
)
244 status
[m
] = self
.libjuju
.get_model_status(m
)
247 return obj_to_yaml(status
)
249 return obj_to_dict(status
)
251 async def create_execution_environment(
255 reuse_ee_id
: str = None,
256 progress_timeout
: float = None,
257 total_timeout
: float = None,
261 "Creating execution environment. namespace: {}, reuse_ee_id: {}".format(
262 namespace
, reuse_ee_id
268 model_name
, application_name
, machine_id
= self
._get
_ee
_id
_components
(
278 ) = self
._get
_namespace
_components
(namespace
=namespace
)
279 # model name is ns_id
282 application_name
= self
._get
_application
_name
(namespace
=namespace
)
285 "model name: {}, application name: {}, machine_id: {}".format(
286 model_name
, application_name
, machine_id
290 # create or reuse a new juju machine
292 if not await self
.libjuju
.model_exists(model_name
):
293 await self
.libjuju
.add_model(model_name
, cloud_name
=self
.cloud
)
294 machine
, new
= await self
.libjuju
.create_machine(
295 model_name
=model_name
,
296 machine_id
=machine_id
,
298 progress_timeout
=progress_timeout
,
299 total_timeout
=total_timeout
,
301 # id for the execution environment
302 ee_id
= N2VCJujuConnector
._build
_ee
_id
(
303 model_name
=model_name
,
304 application_name
=application_name
,
305 machine_id
=str(machine
.entity_id
),
307 self
.log
.debug("ee_id: {}".format(ee_id
))
310 # write ee_id in database
311 self
._write
_ee
_id
_db
(db_dict
=db_dict
, ee_id
=ee_id
)
313 except Exception as e
:
314 message
= "Error creating machine on juju: {}".format(e
)
315 self
.log
.error(message
)
316 raise N2VCException(message
=message
)
318 # new machine credentials
320 "hostname": machine
.dns_name
,
324 "Execution environment created. ee_id: {}, credentials: {}".format(
329 return ee_id
, credentials
331 async def register_execution_environment(
336 progress_timeout
: float = None,
337 total_timeout
: float = None,
341 "Registering execution environment. namespace={}, credentials={}".format(
342 namespace
, credentials
346 if credentials
is None:
347 raise N2VCBadArgumentsException(
348 message
="credentials are mandatory", bad_args
=["credentials"]
350 if credentials
.get("hostname"):
351 hostname
= credentials
["hostname"]
353 raise N2VCBadArgumentsException(
354 message
="hostname is mandatory", bad_args
=["credentials.hostname"]
356 if credentials
.get("username"):
357 username
= credentials
["username"]
359 raise N2VCBadArgumentsException(
360 message
="username is mandatory", bad_args
=["credentials.username"]
362 if "private_key_path" in credentials
:
363 private_key_path
= credentials
["private_key_path"]
365 # if not passed as argument, use generated private key path
366 private_key_path
= self
.private_key_path
368 _nsi_id
, ns_id
, _vnf_id
, _vdu_id
, _vdu_count
= self
._get
_namespace
_components
(
375 application_name
= self
._get
_application
_name
(namespace
=namespace
)
377 # register machine on juju
379 if not self
.api_proxy
:
380 msg
= "Cannot provision machine: api_proxy is not defined"
381 self
.log
.error(msg
=msg
)
382 raise N2VCException(message
=msg
)
383 if not await self
.libjuju
.model_exists(model_name
):
384 await self
.libjuju
.add_model(model_name
, cloud_name
=self
.cloud
)
385 machine_id
= await self
.libjuju
.provision_machine(
386 model_name
=model_name
,
389 private_key_path
=private_key_path
,
391 progress_timeout
=progress_timeout
,
392 total_timeout
=total_timeout
,
394 except Exception as e
:
395 self
.log
.error("Error registering machine: {}".format(e
))
397 message
="Error registering machine on juju: {}".format(e
)
400 self
.log
.info("Machine registered: {}".format(machine_id
))
402 # id for the execution environment
403 ee_id
= N2VCJujuConnector
._build
_ee
_id
(
404 model_name
=model_name
,
405 application_name
=application_name
,
406 machine_id
=str(machine_id
),
409 self
.log
.info("Execution environment registered. ee_id: {}".format(ee_id
))
413 async def install_configuration_sw(
418 progress_timeout
: float = None,
419 total_timeout
: float = None,
426 "Installing configuration sw on ee_id: {}, "
427 "artifact path: {}, db_dict: {}"
428 ).format(ee_id
, artifact_path
, db_dict
)
432 if ee_id
is None or len(ee_id
) == 0:
433 raise N2VCBadArgumentsException(
434 message
="ee_id is mandatory", bad_args
=["ee_id"]
436 if artifact_path
is None or len(artifact_path
) == 0:
437 raise N2VCBadArgumentsException(
438 message
="artifact_path is mandatory", bad_args
=["artifact_path"]
441 raise N2VCBadArgumentsException(
442 message
="db_dict is mandatory", bad_args
=["db_dict"]
450 ) = N2VCJujuConnector
._get
_ee
_id
_components
(ee_id
=ee_id
)
452 "model: {}, application: {}, machine: {}".format(
453 model_name
, application_name
, machine_id
457 raise N2VCBadArgumentsException(
458 message
="ee_id={} is not a valid execution environment id".format(
464 # remove // in charm path
465 while artifact_path
.find("//") >= 0:
466 artifact_path
= artifact_path
.replace("//", "/")
469 if not self
.fs
.file_exists(artifact_path
, mode
="dir"):
470 msg
= "artifact path does not exist: {}".format(artifact_path
)
471 raise N2VCBadArgumentsException(message
=msg
, bad_args
=["artifact_path"])
473 if artifact_path
.startswith("/"):
474 full_path
= self
.fs
.path
+ artifact_path
476 full_path
= self
.fs
.path
+ "/" + artifact_path
479 await self
.libjuju
.deploy_charm(
480 model_name
=model_name
,
481 application_name
=application_name
,
483 machine_id
=machine_id
,
485 progress_timeout
=progress_timeout
,
486 total_timeout
=total_timeout
,
490 except Exception as e
:
492 message
="Error desploying charm into ee={} : {}".format(ee_id
, e
)
495 self
.log
.info("Configuration sw installed")
497 async def install_k8s_proxy_charm(
503 progress_timeout
: float = None,
504 total_timeout
: float = None,
508 Install a k8s proxy charm
510 :param charm_name: Name of the charm being deployed
511 :param namespace: collection of all the uuids related to the charm.
512 :param str artifact_path: where to locate the artifacts (parent folder) using
514 the final artifact path will be a combination of this artifact_path and
515 additional string from the config_dict (e.g. charm name)
516 :param dict db_dict: where to write into database when the status changes.
517 It contains a dict with
518 {collection: <str>, filter: {}, path: <str>},
519 e.g. {collection: "nsrs", filter:
520 {_id: <nsd-id>, path: "_admin.deployed.VCA.3"}
521 :param float progress_timeout:
522 :param float total_timeout:
523 :param config: Dictionary with additional configuration
525 :returns ee_id: execution environment id.
527 self
.log
.info('Installing k8s proxy charm: {}, artifact path: {}, db_dict: {}'
528 .format(charm_name
, artifact_path
, db_dict
))
530 if not self
.k8s_cloud
:
531 raise JujuK8sProxycharmNotSupported("There is not k8s_cloud available")
533 if artifact_path
is None or len(artifact_path
) == 0:
534 raise N2VCBadArgumentsException(
535 message
="artifact_path is mandatory", bad_args
=["artifact_path"]
538 raise N2VCBadArgumentsException(message
='db_dict is mandatory', bad_args
=['db_dict'])
540 # remove // in charm path
541 while artifact_path
.find('//') >= 0:
542 artifact_path
= artifact_path
.replace('//', '/')
545 if not self
.fs
.file_exists(artifact_path
, mode
="dir"):
546 msg
= 'artifact path does not exist: {}'.format(artifact_path
)
547 raise N2VCBadArgumentsException(message
=msg
, bad_args
=['artifact_path'])
549 if artifact_path
.startswith('/'):
550 full_path
= self
.fs
.path
+ artifact_path
552 full_path
= self
.fs
.path
+ '/' + artifact_path
554 _
, ns_id
, _
, _
, _
= self
._get
_namespace
_components
(namespace
=namespace
)
555 model_name
= '{}-k8s'.format(ns_id
)
557 await self
.libjuju
.add_model(model_name
, self
.k8s_cloud
)
558 application_name
= self
._get
_application
_name
(namespace
)
561 await self
.libjuju
.deploy_charm(
562 model_name
=model_name
,
563 application_name
=application_name
,
567 progress_timeout
=progress_timeout
,
568 total_timeout
=total_timeout
,
571 except Exception as e
:
572 raise N2VCException(message
='Error deploying charm: {}'.format(e
))
574 self
.log
.info('K8s proxy charm installed')
575 ee_id
= N2VCJujuConnector
._build
_ee
_id
(
576 model_name
=model_name
,
577 application_name
=application_name
,
582 async def get_ee_ssh_public__key(
586 progress_timeout
: float = None,
587 total_timeout
: float = None,
592 "Generating priv/pub key pair and get pub key on ee_id: {}, db_dict: {}"
593 ).format(ee_id
, db_dict
)
597 if ee_id
is None or len(ee_id
) == 0:
598 raise N2VCBadArgumentsException(
599 message
="ee_id is mandatory", bad_args
=["ee_id"]
602 raise N2VCBadArgumentsException(
603 message
="db_dict is mandatory", bad_args
=["db_dict"]
611 ) = N2VCJujuConnector
._get
_ee
_id
_components
(ee_id
=ee_id
)
613 "model: {}, application: {}, machine: {}".format(
614 model_name
, application_name
, machine_id
618 raise N2VCBadArgumentsException(
619 message
="ee_id={} is not a valid execution environment id".format(
625 # try to execute ssh layer primitives (if exist):
631 application_name
= N2VCJujuConnector
._format
_app
_name
(application_name
)
633 # execute action: generate-ssh-key
635 output
, _status
= await self
.libjuju
.execute_action(
636 model_name
=model_name
,
637 application_name
=application_name
,
638 action_name
="generate-ssh-key",
640 progress_timeout
=progress_timeout
,
641 total_timeout
=total_timeout
,
643 except Exception as e
:
645 "Skipping exception while executing action generate-ssh-key: {}".format(
650 # execute action: get-ssh-public-key
652 output
, _status
= await self
.libjuju
.execute_action(
653 model_name
=model_name
,
654 application_name
=application_name
,
655 action_name
="get-ssh-public-key",
657 progress_timeout
=progress_timeout
,
658 total_timeout
=total_timeout
,
660 except Exception as e
:
661 msg
= "Cannot execute action get-ssh-public-key: {}\n".format(e
)
663 raise N2VCExecutionException(e
, primitive_name
="get-ssh-public-key")
665 # return public key if exists
666 return output
["pubkey"] if "pubkey" in output
else output
668 async def add_relation(
669 self
, ee_id_1
: str, ee_id_2
: str, endpoint_1
: str, endpoint_2
: str
673 "adding new relation between {} and {}, endpoints: {}, {}".format(
674 ee_id_1
, ee_id_2
, endpoint_1
, endpoint_2
680 message
= "EE 1 is mandatory"
681 self
.log
.error(message
)
682 raise N2VCBadArgumentsException(message
=message
, bad_args
=["ee_id_1"])
684 message
= "EE 2 is mandatory"
685 self
.log
.error(message
)
686 raise N2VCBadArgumentsException(message
=message
, bad_args
=["ee_id_2"])
688 message
= "endpoint 1 is mandatory"
689 self
.log
.error(message
)
690 raise N2VCBadArgumentsException(message
=message
, bad_args
=["endpoint_1"])
692 message
= "endpoint 2 is mandatory"
693 self
.log
.error(message
)
694 raise N2VCBadArgumentsException(message
=message
, bad_args
=["endpoint_2"])
696 # get the model, the applications and the machines from the ee_id's
697 model_1
, app_1
, _machine_1
= self
._get
_ee
_id
_components
(ee_id_1
)
698 model_2
, app_2
, _machine_2
= self
._get
_ee
_id
_components
(ee_id_2
)
700 # model must be the same
701 if model_1
!= model_2
:
702 message
= "EE models are not the same: {} vs {}".format(ee_id_1
, ee_id_2
)
703 self
.log
.error(message
)
704 raise N2VCBadArgumentsException(
705 message
=message
, bad_args
=["ee_id_1", "ee_id_2"]
708 # add juju relations between two applications
710 await self
.libjuju
.add_relation(
712 application_name_1
=app_1
,
713 application_name_2
=app_2
,
714 relation_1
=endpoint_1
,
715 relation_2
=endpoint_2
,
717 except Exception as e
:
718 message
= "Error adding relation between {} and {}: {}".format(
721 self
.log
.error(message
)
722 raise N2VCException(message
=message
)
724 async def remove_relation(self
):
726 self
.log
.info("Method not implemented yet")
727 raise MethodNotImplemented()
729 async def deregister_execution_environments(self
):
730 self
.log
.info("Method not implemented yet")
731 raise MethodNotImplemented()
733 async def delete_namespace(
734 self
, namespace
: str, db_dict
: dict = None, total_timeout
: float = None
736 self
.log
.info("Deleting namespace={}".format(namespace
))
739 if namespace
is None:
740 raise N2VCBadArgumentsException(
741 message
="namespace is mandatory", bad_args
=["namespace"]
744 _nsi_id
, ns_id
, _vnf_id
, _vdu_id
, _vdu_count
= self
._get
_namespace
_components
(
747 if ns_id
is not None:
749 models
= await self
.libjuju
.list_models(contains
=ns_id
)
751 await self
.libjuju
.destroy_model(
752 model_name
=model
, total_timeout
=total_timeout
754 except Exception as e
:
756 message
="Error deleting namespace {} : {}".format(namespace
, e
)
759 raise N2VCBadArgumentsException(
760 message
="only ns_id is permitted to delete yet", bad_args
=["namespace"]
763 self
.log
.info("Namespace {} deleted".format(namespace
))
765 async def delete_execution_environment(
766 self
, ee_id
: str, db_dict
: dict = None, total_timeout
: float = None
768 self
.log
.info("Deleting execution environment ee_id={}".format(ee_id
))
772 raise N2VCBadArgumentsException(
773 message
="ee_id is mandatory", bad_args
=["ee_id"]
776 model_name
, application_name
, _machine_id
= self
._get
_ee
_id
_components
(
780 # destroy the application
782 await self
.libjuju
.destroy_model(
783 model_name
=model_name
, total_timeout
=total_timeout
785 except Exception as e
:
788 "Error deleting execution environment {} (application {}) : {}"
789 ).format(ee_id
, application_name
, e
)
792 # destroy the machine
794 # await self._juju_destroy_machine(
795 # model_name=model_name,
796 # machine_id=machine_id,
797 # total_timeout=total_timeout
799 # except Exception as e:
800 # raise N2VCException(
801 # message='Error deleting execution environment {} (machine {}) : {}'
802 # .format(ee_id, machine_id, e))
804 self
.log
.info("Execution environment {} deleted".format(ee_id
))
806 async def exec_primitive(
811 db_dict
: dict = None,
812 progress_timeout
: float = None,
813 total_timeout
: float = None,
817 "Executing primitive: {} on ee: {}, params: {}".format(
818 primitive_name
, ee_id
, params_dict
823 if ee_id
is None or len(ee_id
) == 0:
824 raise N2VCBadArgumentsException(
825 message
="ee_id is mandatory", bad_args
=["ee_id"]
827 if primitive_name
is None or len(primitive_name
) == 0:
828 raise N2VCBadArgumentsException(
829 message
="action_name is mandatory", bad_args
=["action_name"]
831 if params_dict
is None:
839 ) = N2VCJujuConnector
._get
_ee
_id
_components
(ee_id
=ee_id
)
841 raise N2VCBadArgumentsException(
842 message
="ee_id={} is not a valid execution environment id".format(
848 if primitive_name
== "config":
849 # Special case: config primitive
851 await self
.libjuju
.configure_application(
852 model_name
=model_name
,
853 application_name
=application_name
,
856 actions
= await self
.libjuju
.get_actions(
857 application_name
=application_name
, model_name
=model_name
,
860 "Application {} has these actions: {}".format(
861 application_name
, actions
864 if "verify-ssh-credentials" in actions
:
865 # execute verify-credentials
868 for _
in range(num_retries
):
870 self
.log
.debug("Executing action verify-ssh-credentials...")
871 output
, ok
= await self
.libjuju
.execute_action(
872 model_name
=model_name
,
873 application_name
=application_name
,
874 action_name
="verify-ssh-credentials",
876 progress_timeout
=progress_timeout
,
877 total_timeout
=total_timeout
,
882 "Error executing verify-ssh-credentials: {}. Retrying..."
884 await asyncio
.sleep(retry_timeout
)
887 self
.log
.debug("Result: {}, output: {}".format(ok
, output
))
889 except asyncio
.CancelledError
:
893 "Error executing verify-ssh-credentials after {} retries. ".format(
898 msg
= "Action verify-ssh-credentials does not exist in application {}".format(
901 self
.log
.debug(msg
=msg
)
902 except Exception as e
:
903 self
.log
.error("Error configuring juju application: {}".format(e
))
904 raise N2VCExecutionException(
905 message
="Error configuring application into ee={} : {}".format(
908 primitive_name
=primitive_name
,
913 output
, status
= await self
.libjuju
.execute_action(
914 model_name
=model_name
,
915 application_name
=application_name
,
916 action_name
=primitive_name
,
918 progress_timeout
=progress_timeout
,
919 total_timeout
=total_timeout
,
922 if status
== "completed":
925 raise Exception("status is not completed: {}".format(status
))
926 except Exception as e
:
928 "Error executing primitive {}: {}".format(primitive_name
, e
)
930 raise N2VCExecutionException(
931 message
="Error executing primitive {} into ee={} : {}".format(
932 primitive_name
, ee_id
, e
934 primitive_name
=primitive_name
,
937 async def disconnect(self
):
938 self
.log
.info("closing juju N2VC...")
940 await self
.libjuju
.disconnect()
941 except Exception as e
:
942 raise N2VCConnectionException(
943 message
="Error disconnecting controller: {}".format(e
), url
=self
.url
947 ####################################################################################
948 ################################### P R I V A T E ##################################
949 ####################################################################################
952 def _write_ee_id_db(self
, db_dict
: dict, ee_id
: str):
954 # write ee_id to database: _admin.deployed.VCA.x
956 the_table
= db_dict
["collection"]
957 the_filter
= db_dict
["filter"]
958 the_path
= db_dict
["path"]
959 if not the_path
[-1] == ".":
960 the_path
= the_path
+ "."
961 update_dict
= {the_path
+ "ee_id": ee_id
}
962 # self.log.debug('Writing ee_id to database: {}'.format(the_path))
966 update_dict
=update_dict
,
969 except asyncio
.CancelledError
:
971 except Exception as e
:
972 self
.log
.error("Error writing ee_id to database: {}".format(e
))
975 def _build_ee_id(model_name
: str, application_name
: str, machine_id
: str):
977 Build an execution environment id form model, application and machine
979 :param application_name:
983 # id for the execution environment
984 return "{}.{}.{}".format(model_name
, application_name
, machine_id
)
987 def _get_ee_id_components(ee_id
: str) -> (str, str, str):
989 Get model, application and machine components from an execution environment id
991 :return: model_name, application_name, machine_id
995 return None, None, None
997 # split components of id
998 parts
= ee_id
.split(".")
999 model_name
= parts
[0]
1000 application_name
= parts
[1]
1001 machine_id
= parts
[2]
1002 return model_name
, application_name
, machine_id
1004 def _get_application_name(self
, namespace
: str) -> str:
1006 Build application name from namespace
1008 :return: app-vnf-<vnf id>-vdu-<vdu-id>-cnt-<vdu-count>
1011 # TODO: Enforce the Juju 50-character application limit
1013 # split namespace components
1014 _
, _
, vnf_id
, vdu_id
, vdu_count
= self
._get
_namespace
_components
(
1018 if vnf_id
is None or len(vnf_id
) == 0:
1021 # Shorten the vnf_id to its last twelve characters
1022 vnf_id
= "vnf-" + vnf_id
[-12:]
1024 if vdu_id
is None or len(vdu_id
) == 0:
1027 # Shorten the vdu_id to its last twelve characters
1028 vdu_id
= "-vdu-" + vdu_id
[-12:]
1030 if vdu_count
is None or len(vdu_count
) == 0:
1033 vdu_count
= "-cnt-" + vdu_count
1035 application_name
= "app-{}{}{}".format(vnf_id
, vdu_id
, vdu_count
)
1037 return N2VCJujuConnector
._format
_app
_name
(application_name
)
1039 async def _juju_create_machine(
1042 application_name
: str,
1043 machine_id
: str = None,
1044 db_dict
: dict = None,
1045 progress_timeout
: float = None,
1046 total_timeout
: float = None,
1050 "creating machine in model: {}, existing machine id: {}".format(
1051 model_name
, machine_id
1055 # get juju model and observer (create model if needed)
1056 model
= await self
._juju
_get
_model
(model_name
=model_name
)
1057 observer
= self
.juju_observers
[model_name
]
1059 # find machine id in model
1061 if machine_id
is not None:
1062 self
.log
.debug("Finding existing machine id {} in model".format(machine_id
))
1063 # get juju existing machines in the model
1064 existing_machines
= await model
.get_machines()
1065 if machine_id
in existing_machines
:
1067 "Machine id {} found in model (reusing it)".format(machine_id
)
1069 machine
= model
.machines
[machine_id
]
1072 self
.log
.debug("Creating a new machine in juju...")
1073 # machine does not exist, create it and wait for it
1074 machine
= await model
.add_machine(
1075 spec
=None, constraints
=None, disks
=None, series
="xenial"
1078 # register machine with observer
1079 observer
.register_machine(machine
=machine
, db_dict
=db_dict
)
1081 # id for the execution environment
1082 ee_id
= N2VCJujuConnector
._build
_ee
_id
(
1083 model_name
=model_name
,
1084 application_name
=application_name
,
1085 machine_id
=str(machine
.entity_id
),
1088 # write ee_id in database
1089 self
._write
_ee
_id
_db
(db_dict
=db_dict
, ee_id
=ee_id
)
1091 # wait for machine creation
1092 await observer
.wait_for_machine(
1093 machine_id
=str(machine
.entity_id
),
1094 progress_timeout
=progress_timeout
,
1095 total_timeout
=total_timeout
,
1100 self
.log
.debug("Reusing old machine pending")
1102 # register machine with observer
1103 observer
.register_machine(machine
=machine
, db_dict
=db_dict
)
1105 # machine does exist, but it is in creation process (pending), wait for
1106 # create finalisation
1107 await observer
.wait_for_machine(
1108 machine_id
=machine
.entity_id
,
1109 progress_timeout
=progress_timeout
,
1110 total_timeout
=total_timeout
,
1113 self
.log
.debug("Machine ready at " + str(machine
.dns_name
))
1116 async def _juju_provision_machine(
1121 private_key_path
: str,
1122 db_dict
: dict = None,
1123 progress_timeout
: float = None,
1124 total_timeout
: float = None,
1127 if not self
.api_proxy
:
1128 msg
= "Cannot provision machine: api_proxy is not defined"
1129 self
.log
.error(msg
=msg
)
1130 raise N2VCException(message
=msg
)
1133 "provisioning machine. model: {}, hostname: {}, username: {}".format(
1134 model_name
, hostname
, username
1138 if not self
._authenticated
:
1139 await self
._juju
_login
()
1141 # get juju model and observer
1142 model
= await self
._juju
_get
_model
(model_name
=model_name
)
1143 observer
= self
.juju_observers
[model_name
]
1145 # TODO check if machine is already provisioned
1146 machine_list
= await model
.get_machines()
1148 provisioner
= AsyncSSHProvisioner(
1151 private_key_path
=private_key_path
,
1157 params
= await provisioner
.provision_machine()
1158 except Exception as ex
:
1159 msg
= "Exception provisioning machine: {}".format(ex
)
1161 raise N2VCException(message
=msg
)
1163 params
.jobs
= ["JobHostUnits"]
1165 connection
= model
.connection()
1167 # Submit the request.
1168 self
.log
.debug("Adding machine to model")
1169 client_facade
= client
.ClientFacade
.from_connection(connection
)
1170 results
= await client_facade
.AddMachines(params
=[params
])
1171 error
= results
.machines
[0].error
1173 msg
= "Error adding machine: {}".format(error
.message
)
1174 self
.log
.error(msg
=msg
)
1175 raise ValueError(msg
)
1177 machine_id
= results
.machines
[0].machine
1179 # Need to run this after AddMachines has been called,
1180 # as we need the machine_id
1181 self
.log
.debug("Installing Juju agent into machine {}".format(machine_id
))
1182 asyncio
.ensure_future(
1183 provisioner
.install_agent(
1184 connection
=connection
,
1186 machine_id
=machine_id
,
1191 # wait for machine in model (now, machine is not yet in model, so we must
1195 machine_list
= await model
.get_machines()
1196 if machine_id
in machine_list
:
1197 self
.log
.debug("Machine {} found in model!".format(machine_id
))
1198 machine
= model
.machines
.get(machine_id
)
1200 await asyncio
.sleep(2)
1203 msg
= "Machine {} not found in model".format(machine_id
)
1204 self
.log
.error(msg
=msg
)
1205 raise Exception(msg
)
1207 # register machine with observer
1208 observer
.register_machine(machine
=machine
, db_dict
=db_dict
)
1210 # wait for machine creation
1211 self
.log
.debug("waiting for provision finishes... {}".format(machine_id
))
1212 await observer
.wait_for_machine(
1213 machine_id
=machine_id
,
1214 progress_timeout
=progress_timeout
,
1215 total_timeout
=total_timeout
,
1218 self
.log
.debug("Machine provisioned {}".format(machine_id
))
1222 async def _juju_deploy_charm(
1225 application_name
: str,
1229 progress_timeout
: float = None,
1230 total_timeout
: float = None,
1231 config
: dict = None,
1232 ) -> (Application
, int):
1234 # get juju model and observer
1235 model
= await self
._juju
_get
_model
(model_name
=model_name
)
1236 observer
= self
.juju_observers
[model_name
]
1238 # check if application already exists
1240 if application_name
in model
.applications
:
1241 application
= model
.applications
[application_name
]
1243 if application
is None:
1245 # application does not exist, create it and wait for it
1247 "deploying application {} to machine {}, model {}".format(
1248 application_name
, machine_id
, model_name
1251 self
.log
.debug("charm: {}".format(charm_path
))
1252 machine
= model
.machines
[machine_id
]
1254 application
= await model
.deploy(
1255 entity_url
=charm_path
,
1256 application_name
=application_name
,
1259 series
=machine
.series
,
1264 # register application with observer
1265 observer
.register_application(application
=application
, db_dict
=db_dict
)
1268 "waiting for application deployed... {}".format(application
.entity_id
)
1270 retries
= await observer
.wait_for_application(
1271 application_id
=application
.entity_id
,
1272 progress_timeout
=progress_timeout
,
1273 total_timeout
=total_timeout
,
1275 self
.log
.debug("application deployed")
1279 # register application with observer
1280 observer
.register_application(application
=application
, db_dict
=db_dict
)
1282 # application already exists, but not finalised
1283 self
.log
.debug("application already exists, waiting for deployed...")
1284 retries
= await observer
.wait_for_application(
1285 application_id
=application
.entity_id
,
1286 progress_timeout
=progress_timeout
,
1287 total_timeout
=total_timeout
,
1289 self
.log
.debug("application deployed")
1291 return application
, retries
1293 async def _juju_execute_action(
1296 application_name
: str,
1299 progress_timeout
: float = None,
1300 total_timeout
: float = None,
1304 # get juju model and observer
1305 model
= await self
._juju
_get
_model
(model_name
=model_name
)
1306 observer
= self
.juju_observers
[model_name
]
1308 application
= await self
._juju
_get
_application
(
1309 model_name
=model_name
, application_name
=application_name
1313 for u
in application
.units
:
1314 if await u
.is_leader_from_status():
1316 if unit
is not None:
1317 actions
= await application
.get_actions()
1318 if action_name
in actions
:
1320 'executing action "{}" using params: {}'.format(action_name
, kwargs
)
1322 action
= await unit
.run_action(action_name
, **kwargs
)
1324 # register action with observer
1325 observer
.register_action(action
=action
, db_dict
=db_dict
)
1327 await observer
.wait_for_action(
1328 action_id
=action
.entity_id
,
1329 progress_timeout
=progress_timeout
,
1330 total_timeout
=total_timeout
,
1332 self
.log
.debug("action completed with status: {}".format(action
.status
))
1333 output
= await model
.get_action_output(action_uuid
=action
.entity_id
)
1334 status
= await model
.get_action_status(uuid_or_prefix
=action
.entity_id
)
1335 if action
.entity_id
in status
:
1336 status
= status
[action
.entity_id
]
1339 return output
, status
1341 raise N2VCExecutionException(
1342 message
="Cannot execute action on charm", primitive_name
=action_name
1345 async def _juju_configure_application(
1348 application_name
: str,
1351 progress_timeout
: float = None,
1352 total_timeout
: float = None,
1355 # get the application
1356 application
= await self
._juju
_get
_application
(
1357 model_name
=model_name
, application_name
=application_name
1361 "configuring the application {} -> {}".format(application_name
, config
)
1363 res
= await application
.set_config(config
)
1365 "application {} configured. res={}".format(application_name
, res
)
1368 # Verify the config is set
1369 new_conf
= await application
.get_config()
1371 value
= new_conf
[key
]["value"]
1372 self
.log
.debug(" {} = {}".format(key
, value
))
1373 if config
[key
] != value
:
1374 raise N2VCException(
1375 message
="key {} is not configured correctly {} != {}".format(
1376 key
, config
[key
], new_conf
[key
]
1380 # check if 'verify-ssh-credentials' action exists
1381 # unit = application.units[0]
1382 actions
= await application
.get_actions()
1383 if "verify-ssh-credentials" not in actions
:
1385 "Action verify-ssh-credentials does not exist in application {}"
1386 ).format(application_name
)
1387 self
.log
.debug(msg
=msg
)
1390 # execute verify-credentials
1392 retry_timeout
= 15.0
1393 for _
in range(num_retries
):
1395 self
.log
.debug("Executing action verify-ssh-credentials...")
1396 output
, ok
= await self
._juju
_execute
_action
(
1397 model_name
=model_name
,
1398 application_name
=application_name
,
1399 action_name
="verify-ssh-credentials",
1401 progress_timeout
=progress_timeout
,
1402 total_timeout
=total_timeout
,
1404 self
.log
.debug("Result: {}, output: {}".format(ok
, output
))
1406 except asyncio
.CancelledError
:
1408 except Exception as e
:
1410 "Error executing verify-ssh-credentials: {}. Retrying...".format(e
)
1412 await asyncio
.sleep(retry_timeout
)
1415 "Error executing verify-ssh-credentials after {} retries. ".format(
1421 async def _juju_get_application(self
, model_name
: str, application_name
: str):
1422 """Get the deployed application."""
1424 model
= await self
._juju
_get
_model
(model_name
=model_name
)
1426 application_name
= N2VCJujuConnector
._format
_app
_name
(application_name
)
1428 if model
.applications
and application_name
in model
.applications
:
1429 return model
.applications
[application_name
]
1431 raise N2VCException(
1432 message
="Cannot get application {} from model {}".format(
1433 application_name
, model_name
1437 async def _juju_get_model(self
, model_name
: str) -> Model
:
1438 """ Get a model object from juju controller
1439 If the model does not exits, it creates it.
1441 :param str model_name: name of the model
1442 :returns Model: model obtained from juju controller or Exception
1446 model_name
= N2VCJujuConnector
._format
_model
_name
(model_name
)
1448 if model_name
in self
.juju_models
:
1449 return self
.juju_models
[model_name
]
1451 if self
._creating
_model
:
1452 self
.log
.debug("Another coroutine is creating a model. Wait...")
1453 while self
._creating
_model
:
1454 # another coroutine is creating a model, wait
1455 await asyncio
.sleep(0.1)
1456 # retry (perhaps another coroutine has created the model meanwhile)
1457 if model_name
in self
.juju_models
:
1458 return self
.juju_models
[model_name
]
1461 self
._creating
_model
= True
1463 # get juju model names from juju
1464 model_list
= await self
.controller
.list_models()
1465 if model_name
not in model_list
:
1467 "Model {} does not exist. Creating new model...".format(model_name
)
1469 config_dict
= {"authorized-keys": self
.public_key
}
1471 config_dict
["apt-mirror"] = self
.apt_mirror
1472 if not self
.enable_os_upgrade
:
1473 config_dict
["enable-os-refresh-update"] = False
1474 config_dict
["enable-os-upgrade"] = False
1475 if self
.cloud
in self
.BUILT_IN_CLOUDS
:
1476 model
= await self
.controller
.add_model(
1477 model_name
=model_name
,
1479 cloud_name
=self
.cloud
,
1482 model
= await self
.controller
.add_model(
1483 model_name
=model_name
,
1485 cloud_name
=self
.cloud
,
1486 credential_name
=self
.cloud
,
1488 self
.log
.info("New model created, name={}".format(model_name
))
1491 "Model already exists in juju. Getting model {}".format(model_name
)
1493 model
= await self
.controller
.get_model(model_name
)
1494 self
.log
.debug("Existing model in juju, name={}".format(model_name
))
1496 self
.juju_models
[model_name
] = model
1497 self
.juju_observers
[model_name
] = JujuModelObserver(n2vc
=self
, model
=model
)
1500 except Exception as e
:
1501 msg
= "Cannot get model {}. Exception: {}".format(model_name
, e
)
1503 raise N2VCException(msg
)
1505 self
._creating
_model
= False
1507 async def _juju_add_relation(
1510 application_name_1
: str,
1511 application_name_2
: str,
1516 # get juju model and observer
1517 model
= await self
._juju
_get
_model
(model_name
=model_name
)
1519 r1
= "{}:{}".format(application_name_1
, relation_1
)
1520 r2
= "{}:{}".format(application_name_2
, relation_2
)
1522 self
.log
.debug("adding relation: {} -> {}".format(r1
, r2
))
1524 await model
.add_relation(relation1
=r1
, relation2
=r2
)
1525 except JujuAPIError
as e
:
1526 # If one of the applications in the relationship doesn't exist, or the
1527 # relation has already been added,
1528 # let the operation fail silently.
1529 if "not found" in e
.message
:
1531 if "already exists" in e
.message
:
1533 # another execption, raise it
1536 async def _juju_destroy_application(self
, model_name
: str, application_name
: str):
1539 "Destroying application {} in model {}".format(application_name
, model_name
)
1542 # get juju model and observer
1543 model
= await self
._juju
_get
_model
(model_name
=model_name
)
1544 observer
= self
.juju_observers
[model_name
]
1546 application
= model
.applications
.get(application_name
)
1548 observer
.unregister_application(application_name
)
1549 await application
.destroy()
1551 self
.log
.debug("Application not found: {}".format(application_name
))
1553 async def _juju_destroy_machine(
1554 self
, model_name
: str, machine_id
: str, total_timeout
: float = None
1558 "Destroying machine {} in model {}".format(machine_id
, model_name
)
1561 if total_timeout
is None:
1562 total_timeout
= 3600
1564 # get juju model and observer
1565 model
= await self
._juju
_get
_model
(model_name
=model_name
)
1566 observer
= self
.juju_observers
[model_name
]
1568 machines
= await model
.get_machines()
1569 if machine_id
in machines
:
1570 machine
= model
.machines
[machine_id
]
1571 observer
.unregister_machine(machine_id
)
1572 # TODO: change this by machine.is_manual when this is upstreamed:
1573 # https://github.com/juju/python-libjuju/pull/396
1574 if "instance-id" in machine
.safe_data
and machine
.safe_data
[
1576 ].startswith("manual:"):
1577 self
.log
.debug("machine.destroy(force=True) started.")
1578 await machine
.destroy(force
=True)
1579 self
.log
.debug("machine.destroy(force=True) passed.")
1581 end
= time
.time() + total_timeout
1582 # wait for machine removal
1583 machines
= await model
.get_machines()
1584 while machine_id
in machines
and time
.time() < end
:
1586 "Waiting for machine {} is destroyed".format(machine_id
)
1588 await asyncio
.sleep(0.5)
1589 machines
= await model
.get_machines()
1590 self
.log
.debug("Machine destroyed: {}".format(machine_id
))
1592 self
.log
.debug("Machine not found: {}".format(machine_id
))
1594 async def _juju_destroy_model(self
, model_name
: str, total_timeout
: float = None):
1596 self
.log
.debug("Destroying model {}".format(model_name
))
1598 if total_timeout
is None:
1599 total_timeout
= 3600
1600 end
= time
.time() + total_timeout
1602 model
= await self
._juju
_get
_model
(model_name
=model_name
)
1605 raise N2VCNotFound(message
="Model {} does not exist".format(model_name
))
1607 uuid
= model
.info
.uuid
1609 # destroy applications
1610 for application_name
in model
.applications
:
1612 await self
._juju
_destroy
_application
(
1613 model_name
=model_name
, application_name
=application_name
1615 except Exception as e
:
1617 "Error destroying application {} in model {}: {}".format(
1618 application_name
, model_name
, e
1623 machines
= await model
.get_machines()
1624 for machine_id
in machines
:
1626 await self
._juju
_destroy
_machine
(
1627 model_name
=model_name
, machine_id
=machine_id
1629 except asyncio
.CancelledError
:
1632 # ignore exceptions destroying machine
1635 await self
._juju
_disconnect
_model
(model_name
=model_name
)
1637 self
.log
.debug("destroying model {}...".format(model_name
))
1638 await self
.controller
.destroy_model(uuid
)
1639 # self.log.debug('model destroy requested {}'.format(model_name))
1641 # wait for model is completely destroyed
1642 self
.log
.debug("Waiting for model {} to be destroyed...".format(model_name
))
1644 while time
.time() < end
:
1646 # await self.controller.get_model(uuid)
1647 models
= await self
.controller
.list_models()
1648 if model_name
not in models
:
1650 "The model {} ({}) was destroyed".format(model_name
, uuid
)
1653 except asyncio
.CancelledError
:
1655 except Exception as e
:
1657 await asyncio
.sleep(5)
1658 raise N2VCException(
1659 "Timeout waiting for model {} to be destroyed {}".format(
1660 model_name
, last_exception
1664 async def _juju_login(self
):
1665 """Connect to juju controller
1669 # if already authenticated, exit function
1670 if self
._authenticated
:
1673 # if connecting, wait for finish
1674 # another task could be trying to connect in parallel
1675 while self
._connecting
:
1676 await asyncio
.sleep(0.1)
1678 # double check after other task has finished
1679 if self
._authenticated
:
1683 self
._connecting
= True
1685 "connecting to juju controller: {} {}:{}{}".format(
1688 self
.secret
[:8] + "...",
1689 " with ca_cert" if self
.ca_cert
else "",
1693 # Create controller object
1694 self
.controller
= Controller(loop
=self
.loop
)
1695 # Connect to controller
1696 await self
.controller
.connect(
1698 username
=self
.username
,
1699 password
=self
.secret
,
1700 cacert
=self
.ca_cert
,
1702 self
._authenticated
= True
1703 self
.log
.info("juju controller connected")
1704 except Exception as e
:
1705 message
= "Exception connecting to juju: {}".format(e
)
1706 self
.log
.error(message
)
1707 raise N2VCConnectionException(message
=message
, url
=self
.url
)
1709 self
._connecting
= False
1711 async def _juju_logout(self
):
1712 """Logout of the Juju controller."""
1713 if not self
._authenticated
:
1716 # disconnect all models
1717 for model_name
in self
.juju_models
:
1719 await self
._juju
_disconnect
_model
(model_name
)
1720 except Exception as e
:
1722 "Error disconnecting model {} : {}".format(model_name
, e
)
1724 # continue with next model...
1726 self
.log
.info("Disconnecting controller")
1728 await self
.controller
.disconnect()
1729 except Exception as e
:
1730 raise N2VCConnectionException(
1731 message
="Error disconnecting controller: {}".format(e
), url
=self
.url
1734 self
.controller
= None
1735 self
._authenticated
= False
1736 self
.log
.info("disconnected")
1738 async def _juju_disconnect_model(self
, model_name
: str):
1739 self
.log
.debug("Disconnecting model {}".format(model_name
))
1740 if model_name
in self
.juju_models
:
1741 await self
.juju_models
[model_name
].disconnect()
1742 self
.juju_models
[model_name
] = None
1743 self
.juju_observers
[model_name
] = None
1745 self
.warning("Cannot disconnect model: {}".format(model_name
))
1747 def _create_juju_public_key(self
):
1748 """Recreate the Juju public key on lcm container, if needed
1749 Certain libjuju commands expect to be run from the same machine as Juju
1750 is bootstrapped to. This method will write the public key to disk in
1751 that location: ~/.local/share/juju/ssh/juju_id_rsa.pub
1754 # Make sure that we have a public key before writing to disk
1755 if self
.public_key
is None or len(self
.public_key
) == 0:
1756 if "OSMLCM_VCA_PUBKEY" in os
.environ
:
1757 self
.public_key
= os
.getenv("OSMLCM_VCA_PUBKEY", "")
1758 if len(self
.public_key
) == 0:
1763 pk_path
= "{}/.local/share/juju/ssh".format(os
.path
.expanduser("~"))
1764 file_path
= "{}/juju_id_rsa.pub".format(pk_path
)
1766 "writing juju public key to file:\n{}\npublic key: {}".format(
1767 file_path
, self
.public_key
1770 if not os
.path
.exists(pk_path
):
1771 # create path and write file
1772 os
.makedirs(pk_path
)
1773 with
open(file_path
, "w") as f
:
1774 self
.log
.debug("Creating juju public key file: {}".format(file_path
))
1775 f
.write(self
.public_key
)
1777 self
.log
.debug("juju public key file already exists: {}".format(file_path
))
1780 def _format_model_name(name
: str) -> str:
1781 """Format the name of the model.
1783 Model names may only contain lowercase letters, digits and hyphens
1786 return name
.replace("_", "-").replace(" ", "-").lower()
1789 def _format_app_name(name
: str) -> str:
1790 """Format the name of the application (in order to assure valid application name).
1792 Application names have restrictions (run juju deploy --help):
1793 - contains lowercase letters 'a'-'z'
1794 - contains numbers '0'-'9'
1795 - contains hyphens '-'
1796 - starts with a lowercase letter
1797 - not two or more consecutive hyphens
1798 - after a hyphen, not a group with all numbers
1801 def all_numbers(s
: str) -> bool:
1807 new_name
= name
.replace("_", "-")
1808 new_name
= new_name
.replace(" ", "-")
1809 new_name
= new_name
.lower()
1810 while new_name
.find("--") >= 0:
1811 new_name
= new_name
.replace("--", "-")
1812 groups
= new_name
.split("-")
1814 # find 'all numbers' groups and prefix them with a letter
1816 for i
in range(len(groups
)):
1818 if all_numbers(group
):
1824 if app_name
[0].isdigit():
1825 app_name
= "z" + app_name