2 # Copyright 2019 Telefonica Investigacion y Desarrollo, S.A.U.
3 # This file is part of OSM
6 # Licensed under the Apache License, Version 2.0 (the "License");
7 # you may not use this file except in compliance with the License.
8 # You may obtain a copy of the License at
10 # http://www.apache.org/licenses/LICENSE-2.0
12 # Unless required by applicable law or agreed to in writing, software
13 # distributed under the License is distributed on an "AS IS" BASIS,
14 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
16 # See the License for the specific language governing permissions and
17 # limitations under the License.
19 # For those usages not covered by the Apache License, Version 2.0 please
20 # contact with: nfvlabs@tid.es
31 from juju
.action
import Action
32 from juju
.application
import Application
33 from juju
.client
import client
34 from juju
.controller
import Controller
35 from juju
.errors
import JujuAPIError
36 from juju
.machine
import Machine
37 from juju
.model
import Model
38 from n2vc
.exceptions
import (
39 N2VCBadArgumentsException
,
41 N2VCConnectionException
,
42 N2VCExecutionException
,
43 N2VCInvalidCertificate
,
46 JujuK8sProxycharmNotSupported
,
48 from n2vc
.juju_observer
import JujuModelObserver
49 from n2vc
.n2vc_conn
import N2VCConnector
50 from n2vc
.n2vc_conn
import obj_to_dict
, obj_to_yaml
51 from n2vc
.provisioner
import AsyncSSHProvisioner
52 from n2vc
.libjuju
import Libjuju
55 class N2VCJujuConnector(N2VCConnector
):
58 ####################################################################################
59 ################################### P U B L I C ####################################
60 ####################################################################################
63 BUILT_IN_CLOUDS
= ["localhost", "microk8s"]
71 url
: str = "127.0.0.1:17070",
72 username
: str = "admin",
73 vca_config
: dict = None,
76 """Initialize juju N2VC connector
79 # parent class constructor
80 N2VCConnector
.__init
__(
88 vca_config
=vca_config
,
89 on_update_db
=on_update_db
,
92 # silence websocket traffic log
93 logging
.getLogger("websockets.protocol").setLevel(logging
.INFO
)
94 logging
.getLogger("juju.client.connection").setLevel(logging
.WARN
)
95 logging
.getLogger("model").setLevel(logging
.WARN
)
97 self
.log
.info("Initializing N2VC juju connector...")
100 ##############################################################
102 ##############################################################
107 raise N2VCBadArgumentsException("Argument url is mandatory", ["url"])
108 url_parts
= url
.split(":")
109 if len(url_parts
) != 2:
110 raise N2VCBadArgumentsException(
111 "Argument url: bad format (localhost:port) -> {}".format(url
), ["url"]
113 self
.hostname
= url_parts
[0]
115 self
.port
= int(url_parts
[1])
117 raise N2VCBadArgumentsException(
118 "url port must be a number -> {}".format(url
), ["url"]
123 raise N2VCBadArgumentsException(
124 "Argument username is mandatory", ["username"]
128 if vca_config
is None:
129 raise N2VCBadArgumentsException(
130 "Argument vca_config is mandatory", ["vca_config"]
133 if "secret" in vca_config
:
134 self
.secret
= vca_config
["secret"]
136 raise N2VCBadArgumentsException(
137 "Argument vca_config.secret is mandatory", ["vca_config.secret"]
140 # pubkey of juju client in osm machine: ~/.local/share/juju/ssh/juju_id_rsa.pub
141 # if exists, it will be written in lcm container: _create_juju_public_key()
142 if "public_key" in vca_config
:
143 self
.public_key
= vca_config
["public_key"]
145 self
.public_key
= None
147 # TODO: Verify ca_cert is valid before using. VCA will crash
148 # if the ca_cert isn't formatted correctly.
149 def base64_to_cacert(b64string
):
150 """Convert the base64-encoded string containing the VCA CACERT.
156 cacert
= base64
.b64decode(b64string
).decode("utf-8")
158 cacert
= re
.sub(r
"\\n", r
"\n", cacert
,)
159 except binascii
.Error
as e
:
160 self
.log
.debug("Caught binascii.Error: {}".format(e
))
161 raise N2VCInvalidCertificate(message
="Invalid CA Certificate")
165 self
.ca_cert
= vca_config
.get("ca_cert")
167 self
.ca_cert
= base64_to_cacert(vca_config
["ca_cert"])
169 if "api_proxy" in vca_config
and vca_config
["api_proxy"] != "":
170 self
.api_proxy
= vca_config
["api_proxy"]
172 "api_proxy for native charms configured: {}".format(self
.api_proxy
)
176 "api_proxy is not configured"
178 self
.api_proxy
= None
180 if "enable_os_upgrade" in vca_config
:
181 self
.enable_os_upgrade
= vca_config
["enable_os_upgrade"]
183 self
.enable_os_upgrade
= True
185 if "apt_mirror" in vca_config
:
186 self
.apt_mirror
= vca_config
["apt_mirror"]
188 self
.apt_mirror
= None
190 self
.cloud
= vca_config
.get('cloud')
191 self
.k8s_cloud
= None
192 if "k8s_cloud" in vca_config
:
193 self
.k8s_cloud
= vca_config
.get("k8s_cloud")
194 self
.log
.debug('Arguments have been checked')
197 self
.controller
= None # it will be filled when connect to juju
198 self
.juju_models
= {} # model objects for every model_name
199 self
.juju_observers
= {} # model observers for every model_name
201 False # while connecting to juju (to avoid duplicate connections)
203 self
._authenticated
= (
204 False # it will be True when juju connection be stablished
206 self
._creating
_model
= False # True during model creation
207 self
.libjuju
= Libjuju(
209 api_proxy
=self
.api_proxy
,
210 enable_os_upgrade
=self
.enable_os_upgrade
,
211 apt_mirror
=self
.apt_mirror
,
212 username
=self
.username
,
213 password
=self
.secret
,
221 # create juju pub key file in lcm container at
222 # ./local/share/juju/ssh/juju_id_rsa.pub
223 self
._create
_juju
_public
_key
()
225 self
.log
.info("N2VC juju connector initialized")
227 async def get_status(self
, namespace
: str, yaml_format
: bool = True):
229 # self.log.info('Getting NS status. namespace: {}'.format(namespace))
231 _nsi_id
, ns_id
, _vnf_id
, _vdu_id
, _vdu_count
= self
._get
_namespace
_components
(
234 # model name is ns_id
236 if model_name
is None:
237 msg
= "Namespace {} not valid".format(namespace
)
239 raise N2VCBadArgumentsException(msg
, ["namespace"])
242 models
= await self
.libjuju
.list_models(contains
=ns_id
)
245 status
[m
] = await self
.libjuju
.get_model_status(m
)
248 return obj_to_yaml(status
)
250 return obj_to_dict(status
)
252 async def create_execution_environment(
256 reuse_ee_id
: str = None,
257 progress_timeout
: float = None,
258 total_timeout
: float = None,
262 "Creating execution environment. namespace: {}, reuse_ee_id: {}".format(
263 namespace
, reuse_ee_id
269 model_name
, application_name
, machine_id
= self
._get
_ee
_id
_components
(
279 ) = self
._get
_namespace
_components
(namespace
=namespace
)
280 # model name is ns_id
283 application_name
= self
._get
_application
_name
(namespace
=namespace
)
286 "model name: {}, application name: {}, machine_id: {}".format(
287 model_name
, application_name
, machine_id
291 # create or reuse a new juju machine
293 if not await self
.libjuju
.model_exists(model_name
):
294 await self
.libjuju
.add_model(model_name
, cloud_name
=self
.cloud
)
295 machine
, new
= await self
.libjuju
.create_machine(
296 model_name
=model_name
,
297 machine_id
=machine_id
,
299 progress_timeout
=progress_timeout
,
300 total_timeout
=total_timeout
,
302 # id for the execution environment
303 ee_id
= N2VCJujuConnector
._build
_ee
_id
(
304 model_name
=model_name
,
305 application_name
=application_name
,
306 machine_id
=str(machine
.entity_id
),
308 self
.log
.debug("ee_id: {}".format(ee_id
))
311 # write ee_id in database
312 self
._write
_ee
_id
_db
(db_dict
=db_dict
, ee_id
=ee_id
)
314 except Exception as e
:
315 message
= "Error creating machine on juju: {}".format(e
)
316 self
.log
.error(message
)
317 raise N2VCException(message
=message
)
319 # new machine credentials
321 "hostname": machine
.dns_name
,
325 "Execution environment created. ee_id: {}, credentials: {}".format(
330 return ee_id
, credentials
332 async def register_execution_environment(
337 progress_timeout
: float = None,
338 total_timeout
: float = None,
342 "Registering execution environment. namespace={}, credentials={}".format(
343 namespace
, credentials
347 if credentials
is None:
348 raise N2VCBadArgumentsException(
349 message
="credentials are mandatory", bad_args
=["credentials"]
351 if credentials
.get("hostname"):
352 hostname
= credentials
["hostname"]
354 raise N2VCBadArgumentsException(
355 message
="hostname is mandatory", bad_args
=["credentials.hostname"]
357 if credentials
.get("username"):
358 username
= credentials
["username"]
360 raise N2VCBadArgumentsException(
361 message
="username is mandatory", bad_args
=["credentials.username"]
363 if "private_key_path" in credentials
:
364 private_key_path
= credentials
["private_key_path"]
366 # if not passed as argument, use generated private key path
367 private_key_path
= self
.private_key_path
369 _nsi_id
, ns_id
, _vnf_id
, _vdu_id
, _vdu_count
= self
._get
_namespace
_components
(
376 application_name
= self
._get
_application
_name
(namespace
=namespace
)
378 # register machine on juju
380 if not await self
.libjuju
.model_exists(model_name
):
381 await self
.libjuju
.add_model(model_name
, cloud_name
=self
.cloud
)
382 machine_id
= await self
.libjuju
.provision_machine(
383 model_name
=model_name
,
386 private_key_path
=private_key_path
,
388 progress_timeout
=progress_timeout
,
389 total_timeout
=total_timeout
,
391 except Exception as e
:
392 self
.log
.error("Error registering machine: {}".format(e
))
394 message
="Error registering machine on juju: {}".format(e
)
397 self
.log
.info("Machine registered: {}".format(machine_id
))
399 # id for the execution environment
400 ee_id
= N2VCJujuConnector
._build
_ee
_id
(
401 model_name
=model_name
,
402 application_name
=application_name
,
403 machine_id
=str(machine_id
),
406 self
.log
.info("Execution environment registered. ee_id: {}".format(ee_id
))
410 async def install_configuration_sw(
415 progress_timeout
: float = None,
416 total_timeout
: float = None,
423 "Installing configuration sw on ee_id: {}, "
424 "artifact path: {}, db_dict: {}"
425 ).format(ee_id
, artifact_path
, db_dict
)
429 if ee_id
is None or len(ee_id
) == 0:
430 raise N2VCBadArgumentsException(
431 message
="ee_id is mandatory", bad_args
=["ee_id"]
433 if artifact_path
is None or len(artifact_path
) == 0:
434 raise N2VCBadArgumentsException(
435 message
="artifact_path is mandatory", bad_args
=["artifact_path"]
438 raise N2VCBadArgumentsException(
439 message
="db_dict is mandatory", bad_args
=["db_dict"]
447 ) = N2VCJujuConnector
._get
_ee
_id
_components
(ee_id
=ee_id
)
449 "model: {}, application: {}, machine: {}".format(
450 model_name
, application_name
, machine_id
454 raise N2VCBadArgumentsException(
455 message
="ee_id={} is not a valid execution environment id".format(
461 # remove // in charm path
462 while artifact_path
.find("//") >= 0:
463 artifact_path
= artifact_path
.replace("//", "/")
466 if not self
.fs
.file_exists(artifact_path
, mode
="dir"):
467 msg
= "artifact path does not exist: {}".format(artifact_path
)
468 raise N2VCBadArgumentsException(message
=msg
, bad_args
=["artifact_path"])
470 if artifact_path
.startswith("/"):
471 full_path
= self
.fs
.path
+ artifact_path
473 full_path
= self
.fs
.path
+ "/" + artifact_path
476 await self
.libjuju
.deploy_charm(
477 model_name
=model_name
,
478 application_name
=application_name
,
480 machine_id
=machine_id
,
482 progress_timeout
=progress_timeout
,
483 total_timeout
=total_timeout
,
487 except Exception as e
:
489 message
="Error desploying charm into ee={} : {}".format(ee_id
, e
)
492 self
.log
.info("Configuration sw installed")
494 async def install_k8s_proxy_charm(
500 progress_timeout
: float = None,
501 total_timeout
: float = None,
505 Install a k8s proxy charm
507 :param charm_name: Name of the charm being deployed
508 :param namespace: collection of all the uuids related to the charm.
509 :param str artifact_path: where to locate the artifacts (parent folder) using
511 the final artifact path will be a combination of this artifact_path and
512 additional string from the config_dict (e.g. charm name)
513 :param dict db_dict: where to write into database when the status changes.
514 It contains a dict with
515 {collection: <str>, filter: {}, path: <str>},
516 e.g. {collection: "nsrs", filter:
517 {_id: <nsd-id>, path: "_admin.deployed.VCA.3"}
518 :param float progress_timeout:
519 :param float total_timeout:
520 :param config: Dictionary with additional configuration
522 :returns ee_id: execution environment id.
524 self
.log
.info('Installing k8s proxy charm: {}, artifact path: {}, db_dict: {}'
525 .format(charm_name
, artifact_path
, db_dict
))
527 if not self
.k8s_cloud
:
528 raise JujuK8sProxycharmNotSupported("There is not k8s_cloud available")
530 if artifact_path
is None or len(artifact_path
) == 0:
531 raise N2VCBadArgumentsException(
532 message
="artifact_path is mandatory", bad_args
=["artifact_path"]
535 raise N2VCBadArgumentsException(message
='db_dict is mandatory', bad_args
=['db_dict'])
537 # remove // in charm path
538 while artifact_path
.find('//') >= 0:
539 artifact_path
= artifact_path
.replace('//', '/')
542 if not self
.fs
.file_exists(artifact_path
, mode
="dir"):
543 msg
= 'artifact path does not exist: {}'.format(artifact_path
)
544 raise N2VCBadArgumentsException(message
=msg
, bad_args
=['artifact_path'])
546 if artifact_path
.startswith('/'):
547 full_path
= self
.fs
.path
+ artifact_path
549 full_path
= self
.fs
.path
+ '/' + artifact_path
551 _
, ns_id
, _
, _
, _
= self
._get
_namespace
_components
(namespace
=namespace
)
552 model_name
= '{}-k8s'.format(ns_id
)
554 await self
.libjuju
.add_model(model_name
, self
.k8s_cloud
)
555 application_name
= self
._get
_application
_name
(namespace
)
558 await self
.libjuju
.deploy_charm(
559 model_name
=model_name
,
560 application_name
=application_name
,
564 progress_timeout
=progress_timeout
,
565 total_timeout
=total_timeout
,
568 except Exception as e
:
569 raise N2VCException(message
='Error deploying charm: {}'.format(e
))
571 self
.log
.info('K8s proxy charm installed')
572 ee_id
= N2VCJujuConnector
._build
_ee
_id
(
573 model_name
=model_name
,
574 application_name
=application_name
,
578 self
._write
_ee
_id
_db
(db_dict
=db_dict
, ee_id
=ee_id
)
582 async def get_ee_ssh_public__key(
586 progress_timeout
: float = None,
587 total_timeout
: float = None,
592 "Generating priv/pub key pair and get pub key on ee_id: {}, db_dict: {}"
593 ).format(ee_id
, db_dict
)
597 if ee_id
is None or len(ee_id
) == 0:
598 raise N2VCBadArgumentsException(
599 message
="ee_id is mandatory", bad_args
=["ee_id"]
602 raise N2VCBadArgumentsException(
603 message
="db_dict is mandatory", bad_args
=["db_dict"]
611 ) = N2VCJujuConnector
._get
_ee
_id
_components
(ee_id
=ee_id
)
613 "model: {}, application: {}, machine: {}".format(
614 model_name
, application_name
, machine_id
618 raise N2VCBadArgumentsException(
619 message
="ee_id={} is not a valid execution environment id".format(
625 # try to execute ssh layer primitives (if exist):
631 application_name
= N2VCJujuConnector
._format
_app
_name
(application_name
)
633 # execute action: generate-ssh-key
635 output
, _status
= await self
.libjuju
.execute_action(
636 model_name
=model_name
,
637 application_name
=application_name
,
638 action_name
="generate-ssh-key",
640 progress_timeout
=progress_timeout
,
641 total_timeout
=total_timeout
,
643 except Exception as e
:
645 "Skipping exception while executing action generate-ssh-key: {}".format(
650 # execute action: get-ssh-public-key
652 output
, _status
= await self
.libjuju
.execute_action(
653 model_name
=model_name
,
654 application_name
=application_name
,
655 action_name
="get-ssh-public-key",
657 progress_timeout
=progress_timeout
,
658 total_timeout
=total_timeout
,
660 except Exception as e
:
661 msg
= "Cannot execute action get-ssh-public-key: {}\n".format(e
)
663 raise N2VCExecutionException(e
, primitive_name
="get-ssh-public-key")
665 # return public key if exists
666 return output
["pubkey"] if "pubkey" in output
else output
668 async def add_relation(
669 self
, ee_id_1
: str, ee_id_2
: str, endpoint_1
: str, endpoint_2
: str
673 "adding new relation between {} and {}, endpoints: {}, {}".format(
674 ee_id_1
, ee_id_2
, endpoint_1
, endpoint_2
680 message
= "EE 1 is mandatory"
681 self
.log
.error(message
)
682 raise N2VCBadArgumentsException(message
=message
, bad_args
=["ee_id_1"])
684 message
= "EE 2 is mandatory"
685 self
.log
.error(message
)
686 raise N2VCBadArgumentsException(message
=message
, bad_args
=["ee_id_2"])
688 message
= "endpoint 1 is mandatory"
689 self
.log
.error(message
)
690 raise N2VCBadArgumentsException(message
=message
, bad_args
=["endpoint_1"])
692 message
= "endpoint 2 is mandatory"
693 self
.log
.error(message
)
694 raise N2VCBadArgumentsException(message
=message
, bad_args
=["endpoint_2"])
696 # get the model, the applications and the machines from the ee_id's
697 model_1
, app_1
, _machine_1
= self
._get
_ee
_id
_components
(ee_id_1
)
698 model_2
, app_2
, _machine_2
= self
._get
_ee
_id
_components
(ee_id_2
)
700 # model must be the same
701 if model_1
!= model_2
:
702 message
= "EE models are not the same: {} vs {}".format(ee_id_1
, ee_id_2
)
703 self
.log
.error(message
)
704 raise N2VCBadArgumentsException(
705 message
=message
, bad_args
=["ee_id_1", "ee_id_2"]
708 # add juju relations between two applications
710 await self
.libjuju
.add_relation(
712 endpoint_1
="{}:{}".format(app_1
, endpoint_1
),
713 endpoint_2
="{}:{}".format(app_2
, endpoint_2
),
715 except Exception as e
:
716 message
= "Error adding relation between {} and {}: {}".format(
719 self
.log
.error(message
)
720 raise N2VCException(message
=message
)
722 async def remove_relation(self
):
724 self
.log
.info("Method not implemented yet")
725 raise MethodNotImplemented()
727 async def deregister_execution_environments(self
):
728 self
.log
.info("Method not implemented yet")
729 raise MethodNotImplemented()
731 async def delete_namespace(
732 self
, namespace
: str, db_dict
: dict = None, total_timeout
: float = None
734 self
.log
.info("Deleting namespace={}".format(namespace
))
737 if namespace
is None:
738 raise N2VCBadArgumentsException(
739 message
="namespace is mandatory", bad_args
=["namespace"]
742 _nsi_id
, ns_id
, _vnf_id
, _vdu_id
, _vdu_count
= self
._get
_namespace
_components
(
745 if ns_id
is not None:
747 models
= await self
.libjuju
.list_models(contains
=ns_id
)
749 await self
.libjuju
.destroy_model(
750 model_name
=model
, total_timeout
=total_timeout
752 except Exception as e
:
754 message
="Error deleting namespace {} : {}".format(namespace
, e
)
757 raise N2VCBadArgumentsException(
758 message
="only ns_id is permitted to delete yet", bad_args
=["namespace"]
761 self
.log
.info("Namespace {} deleted".format(namespace
))
763 async def delete_execution_environment(
764 self
, ee_id
: str, db_dict
: dict = None, total_timeout
: float = None
766 self
.log
.info("Deleting execution environment ee_id={}".format(ee_id
))
770 raise N2VCBadArgumentsException(
771 message
="ee_id is mandatory", bad_args
=["ee_id"]
774 model_name
, application_name
, _machine_id
= self
._get
_ee
_id
_components
(
778 # destroy the application
780 await self
.libjuju
.destroy_model(
781 model_name
=model_name
, total_timeout
=total_timeout
783 except Exception as e
:
786 "Error deleting execution environment {} (application {}) : {}"
787 ).format(ee_id
, application_name
, e
)
790 # destroy the machine
792 # await self._juju_destroy_machine(
793 # model_name=model_name,
794 # machine_id=machine_id,
795 # total_timeout=total_timeout
797 # except Exception as e:
798 # raise N2VCException(
799 # message='Error deleting execution environment {} (machine {}) : {}'
800 # .format(ee_id, machine_id, e))
802 self
.log
.info("Execution environment {} deleted".format(ee_id
))
804 async def exec_primitive(
809 db_dict
: dict = None,
810 progress_timeout
: float = None,
811 total_timeout
: float = None,
815 "Executing primitive: {} on ee: {}, params: {}".format(
816 primitive_name
, ee_id
, params_dict
821 if ee_id
is None or len(ee_id
) == 0:
822 raise N2VCBadArgumentsException(
823 message
="ee_id is mandatory", bad_args
=["ee_id"]
825 if primitive_name
is None or len(primitive_name
) == 0:
826 raise N2VCBadArgumentsException(
827 message
="action_name is mandatory", bad_args
=["action_name"]
829 if params_dict
is None:
837 ) = N2VCJujuConnector
._get
_ee
_id
_components
(ee_id
=ee_id
)
839 raise N2VCBadArgumentsException(
840 message
="ee_id={} is not a valid execution environment id".format(
846 if primitive_name
== "config":
847 # Special case: config primitive
849 await self
.libjuju
.configure_application(
850 model_name
=model_name
,
851 application_name
=application_name
,
854 actions
= await self
.libjuju
.get_actions(
855 application_name
=application_name
, model_name
=model_name
,
858 "Application {} has these actions: {}".format(
859 application_name
, actions
862 if "verify-ssh-credentials" in actions
:
863 # execute verify-credentials
866 for _
in range(num_retries
):
868 self
.log
.debug("Executing action verify-ssh-credentials...")
869 output
, ok
= await self
.libjuju
.execute_action(
870 model_name
=model_name
,
871 application_name
=application_name
,
872 action_name
="verify-ssh-credentials",
874 progress_timeout
=progress_timeout
,
875 total_timeout
=total_timeout
,
880 "Error executing verify-ssh-credentials: {}. Retrying..."
882 await asyncio
.sleep(retry_timeout
)
885 self
.log
.debug("Result: {}, output: {}".format(ok
, output
))
887 except asyncio
.CancelledError
:
891 "Error executing verify-ssh-credentials after {} retries. ".format(
896 msg
= "Action verify-ssh-credentials does not exist in application {}".format(
899 self
.log
.debug(msg
=msg
)
900 except Exception as e
:
901 self
.log
.error("Error configuring juju application: {}".format(e
))
902 raise N2VCExecutionException(
903 message
="Error configuring application into ee={} : {}".format(
906 primitive_name
=primitive_name
,
911 output
, status
= await self
.libjuju
.execute_action(
912 model_name
=model_name
,
913 application_name
=application_name
,
914 action_name
=primitive_name
,
916 progress_timeout
=progress_timeout
,
917 total_timeout
=total_timeout
,
920 if status
== "completed":
923 raise Exception("status is not completed: {}".format(status
))
924 except Exception as e
:
926 "Error executing primitive {}: {}".format(primitive_name
, e
)
928 raise N2VCExecutionException(
929 message
="Error executing primitive {} into ee={} : {}".format(
930 primitive_name
, ee_id
, e
932 primitive_name
=primitive_name
,
935 async def disconnect(self
):
936 self
.log
.info("closing juju N2VC...")
938 await self
.libjuju
.disconnect()
939 except Exception as e
:
940 raise N2VCConnectionException(
941 message
="Error disconnecting controller: {}".format(e
), url
=self
.url
945 ####################################################################################
946 ################################### P R I V A T E ##################################
947 ####################################################################################
950 def _write_ee_id_db(self
, db_dict
: dict, ee_id
: str):
952 # write ee_id to database: _admin.deployed.VCA.x
954 the_table
= db_dict
["collection"]
955 the_filter
= db_dict
["filter"]
956 the_path
= db_dict
["path"]
957 if not the_path
[-1] == ".":
958 the_path
= the_path
+ "."
959 update_dict
= {the_path
+ "ee_id": ee_id
}
960 # self.log.debug('Writing ee_id to database: {}'.format(the_path))
964 update_dict
=update_dict
,
967 except asyncio
.CancelledError
:
969 except Exception as e
:
970 self
.log
.error("Error writing ee_id to database: {}".format(e
))
973 def _build_ee_id(model_name
: str, application_name
: str, machine_id
: str):
975 Build an execution environment id form model, application and machine
977 :param application_name:
981 # id for the execution environment
982 return "{}.{}.{}".format(model_name
, application_name
, machine_id
)
985 def _get_ee_id_components(ee_id
: str) -> (str, str, str):
987 Get model, application and machine components from an execution environment id
989 :return: model_name, application_name, machine_id
993 return None, None, None
995 # split components of id
996 parts
= ee_id
.split(".")
997 model_name
= parts
[0]
998 application_name
= parts
[1]
999 machine_id
= parts
[2]
1000 return model_name
, application_name
, machine_id
1002 def _get_application_name(self
, namespace
: str) -> str:
1004 Build application name from namespace
1006 :return: app-vnf-<vnf id>-vdu-<vdu-id>-cnt-<vdu-count>
1009 # TODO: Enforce the Juju 50-character application limit
1011 # split namespace components
1012 _
, _
, vnf_id
, vdu_id
, vdu_count
= self
._get
_namespace
_components
(
1016 if vnf_id
is None or len(vnf_id
) == 0:
1019 # Shorten the vnf_id to its last twelve characters
1020 vnf_id
= "vnf-" + vnf_id
[-12:]
1022 if vdu_id
is None or len(vdu_id
) == 0:
1025 # Shorten the vdu_id to its last twelve characters
1026 vdu_id
= "-vdu-" + vdu_id
[-12:]
1028 if vdu_count
is None or len(vdu_count
) == 0:
1031 vdu_count
= "-cnt-" + vdu_count
1033 application_name
= "app-{}{}{}".format(vnf_id
, vdu_id
, vdu_count
)
1035 return N2VCJujuConnector
._format
_app
_name
(application_name
)
1037 async def _juju_create_machine(
1040 application_name
: str,
1041 machine_id
: str = None,
1042 db_dict
: dict = None,
1043 progress_timeout
: float = None,
1044 total_timeout
: float = None,
1048 "creating machine in model: {}, existing machine id: {}".format(
1049 model_name
, machine_id
1053 # get juju model and observer (create model if needed)
1054 model
= await self
._juju
_get
_model
(model_name
=model_name
)
1055 observer
= self
.juju_observers
[model_name
]
1057 # find machine id in model
1059 if machine_id
is not None:
1060 self
.log
.debug("Finding existing machine id {} in model".format(machine_id
))
1061 # get juju existing machines in the model
1062 existing_machines
= await model
.get_machines()
1063 if machine_id
in existing_machines
:
1065 "Machine id {} found in model (reusing it)".format(machine_id
)
1067 machine
= model
.machines
[machine_id
]
1070 self
.log
.debug("Creating a new machine in juju...")
1071 # machine does not exist, create it and wait for it
1072 machine
= await model
.add_machine(
1073 spec
=None, constraints
=None, disks
=None, series
="xenial"
1076 # register machine with observer
1077 observer
.register_machine(machine
=machine
, db_dict
=db_dict
)
1079 # id for the execution environment
1080 ee_id
= N2VCJujuConnector
._build
_ee
_id
(
1081 model_name
=model_name
,
1082 application_name
=application_name
,
1083 machine_id
=str(machine
.entity_id
),
1086 # write ee_id in database
1087 self
._write
_ee
_id
_db
(db_dict
=db_dict
, ee_id
=ee_id
)
1089 # wait for machine creation
1090 await observer
.wait_for_machine(
1091 machine_id
=str(machine
.entity_id
),
1092 progress_timeout
=progress_timeout
,
1093 total_timeout
=total_timeout
,
1098 self
.log
.debug("Reusing old machine pending")
1100 # register machine with observer
1101 observer
.register_machine(machine
=machine
, db_dict
=db_dict
)
1103 # machine does exist, but it is in creation process (pending), wait for
1104 # create finalisation
1105 await observer
.wait_for_machine(
1106 machine_id
=machine
.entity_id
,
1107 progress_timeout
=progress_timeout
,
1108 total_timeout
=total_timeout
,
1111 self
.log
.debug("Machine ready at " + str(machine
.dns_name
))
1114 async def _juju_provision_machine(
1119 private_key_path
: str,
1120 db_dict
: dict = None,
1121 progress_timeout
: float = None,
1122 total_timeout
: float = None,
1125 if not self
.api_proxy
:
1126 msg
= "Cannot provision machine: api_proxy is not defined"
1127 self
.log
.error(msg
=msg
)
1128 raise N2VCException(message
=msg
)
1131 "provisioning machine. model: {}, hostname: {}, username: {}".format(
1132 model_name
, hostname
, username
1136 if not self
._authenticated
:
1137 await self
._juju
_login
()
1139 # get juju model and observer
1140 model
= await self
._juju
_get
_model
(model_name
=model_name
)
1141 observer
= self
.juju_observers
[model_name
]
1143 # TODO check if machine is already provisioned
1144 machine_list
= await model
.get_machines()
1146 provisioner
= AsyncSSHProvisioner(
1149 private_key_path
=private_key_path
,
1155 params
= await provisioner
.provision_machine()
1156 except Exception as ex
:
1157 msg
= "Exception provisioning machine: {}".format(ex
)
1159 raise N2VCException(message
=msg
)
1161 params
.jobs
= ["JobHostUnits"]
1163 connection
= model
.connection()
1165 # Submit the request.
1166 self
.log
.debug("Adding machine to model")
1167 client_facade
= client
.ClientFacade
.from_connection(connection
)
1168 results
= await client_facade
.AddMachines(params
=[params
])
1169 error
= results
.machines
[0].error
1171 msg
= "Error adding machine: {}".format(error
.message
)
1172 self
.log
.error(msg
=msg
)
1173 raise ValueError(msg
)
1175 machine_id
= results
.machines
[0].machine
1177 # Need to run this after AddMachines has been called,
1178 # as we need the machine_id
1179 self
.log
.debug("Installing Juju agent into machine {}".format(machine_id
))
1180 asyncio
.ensure_future(
1181 provisioner
.install_agent(
1182 connection
=connection
,
1184 machine_id
=machine_id
,
1185 proxy
=self
.api_proxy
,
1189 # wait for machine in model (now, machine is not yet in model, so we must
1193 machine_list
= await model
.get_machines()
1194 if machine_id
in machine_list
:
1195 self
.log
.debug("Machine {} found in model!".format(machine_id
))
1196 machine
= model
.machines
.get(machine_id
)
1198 await asyncio
.sleep(2)
1201 msg
= "Machine {} not found in model".format(machine_id
)
1202 self
.log
.error(msg
=msg
)
1203 raise Exception(msg
)
1205 # register machine with observer
1206 observer
.register_machine(machine
=machine
, db_dict
=db_dict
)
1208 # wait for machine creation
1209 self
.log
.debug("waiting for provision finishes... {}".format(machine_id
))
1210 await observer
.wait_for_machine(
1211 machine_id
=machine_id
,
1212 progress_timeout
=progress_timeout
,
1213 total_timeout
=total_timeout
,
1216 self
.log
.debug("Machine provisioned {}".format(machine_id
))
1220 async def _juju_deploy_charm(
1223 application_name
: str,
1227 progress_timeout
: float = None,
1228 total_timeout
: float = None,
1229 config
: dict = None,
1230 ) -> (Application
, int):
1232 # get juju model and observer
1233 model
= await self
._juju
_get
_model
(model_name
=model_name
)
1234 observer
= self
.juju_observers
[model_name
]
1236 # check if application already exists
1238 if application_name
in model
.applications
:
1239 application
= model
.applications
[application_name
]
1241 if application
is None:
1243 # application does not exist, create it and wait for it
1245 "deploying application {} to machine {}, model {}".format(
1246 application_name
, machine_id
, model_name
1249 self
.log
.debug("charm: {}".format(charm_path
))
1250 machine
= model
.machines
[machine_id
]
1252 application
= await model
.deploy(
1253 entity_url
=charm_path
,
1254 application_name
=application_name
,
1257 series
=machine
.series
,
1262 # register application with observer
1263 observer
.register_application(application
=application
, db_dict
=db_dict
)
1266 "waiting for application deployed... {}".format(application
.entity_id
)
1268 retries
= await observer
.wait_for_application(
1269 application_id
=application
.entity_id
,
1270 progress_timeout
=progress_timeout
,
1271 total_timeout
=total_timeout
,
1273 self
.log
.debug("application deployed")
1277 # register application with observer
1278 observer
.register_application(application
=application
, db_dict
=db_dict
)
1280 # application already exists, but not finalised
1281 self
.log
.debug("application already exists, waiting for deployed...")
1282 retries
= await observer
.wait_for_application(
1283 application_id
=application
.entity_id
,
1284 progress_timeout
=progress_timeout
,
1285 total_timeout
=total_timeout
,
1287 self
.log
.debug("application deployed")
1289 return application
, retries
1291 async def _juju_execute_action(
1294 application_name
: str,
1297 progress_timeout
: float = None,
1298 total_timeout
: float = None,
1302 # get juju model and observer
1303 model
= await self
._juju
_get
_model
(model_name
=model_name
)
1304 observer
= self
.juju_observers
[model_name
]
1306 application
= await self
._juju
_get
_application
(
1307 model_name
=model_name
, application_name
=application_name
1311 for u
in application
.units
:
1312 if await u
.is_leader_from_status():
1314 if unit
is not None:
1315 actions
= await application
.get_actions()
1316 if action_name
in actions
:
1318 'executing action "{}" using params: {}'.format(action_name
, kwargs
)
1320 action
= await unit
.run_action(action_name
, **kwargs
)
1322 # register action with observer
1323 observer
.register_action(action
=action
, db_dict
=db_dict
)
1325 await observer
.wait_for_action(
1326 action_id
=action
.entity_id
,
1327 progress_timeout
=progress_timeout
,
1328 total_timeout
=total_timeout
,
1330 self
.log
.debug("action completed with status: {}".format(action
.status
))
1331 output
= await model
.get_action_output(action_uuid
=action
.entity_id
)
1332 status
= await model
.get_action_status(uuid_or_prefix
=action
.entity_id
)
1333 if action
.entity_id
in status
:
1334 status
= status
[action
.entity_id
]
1337 return output
, status
1339 raise N2VCExecutionException(
1340 message
="Cannot execute action on charm", primitive_name
=action_name
1343 async def _juju_configure_application(
1346 application_name
: str,
1349 progress_timeout
: float = None,
1350 total_timeout
: float = None,
1353 # get the application
1354 application
= await self
._juju
_get
_application
(
1355 model_name
=model_name
, application_name
=application_name
1359 "configuring the application {} -> {}".format(application_name
, config
)
1361 res
= await application
.set_config(config
)
1363 "application {} configured. res={}".format(application_name
, res
)
1366 # Verify the config is set
1367 new_conf
= await application
.get_config()
1369 value
= new_conf
[key
]["value"]
1370 self
.log
.debug(" {} = {}".format(key
, value
))
1371 if config
[key
] != value
:
1372 raise N2VCException(
1373 message
="key {} is not configured correctly {} != {}".format(
1374 key
, config
[key
], new_conf
[key
]
1378 # check if 'verify-ssh-credentials' action exists
1379 # unit = application.units[0]
1380 actions
= await application
.get_actions()
1381 if "verify-ssh-credentials" not in actions
:
1383 "Action verify-ssh-credentials does not exist in application {}"
1384 ).format(application_name
)
1385 self
.log
.debug(msg
=msg
)
1388 # execute verify-credentials
1390 retry_timeout
= 15.0
1391 for _
in range(num_retries
):
1393 self
.log
.debug("Executing action verify-ssh-credentials...")
1394 output
, ok
= await self
._juju
_execute
_action
(
1395 model_name
=model_name
,
1396 application_name
=application_name
,
1397 action_name
="verify-ssh-credentials",
1399 progress_timeout
=progress_timeout
,
1400 total_timeout
=total_timeout
,
1402 self
.log
.debug("Result: {}, output: {}".format(ok
, output
))
1404 except asyncio
.CancelledError
:
1406 except Exception as e
:
1408 "Error executing verify-ssh-credentials: {}. Retrying...".format(e
)
1410 await asyncio
.sleep(retry_timeout
)
1413 "Error executing verify-ssh-credentials after {} retries. ".format(
1419 async def _juju_get_application(self
, model_name
: str, application_name
: str):
1420 """Get the deployed application."""
1422 model
= await self
._juju
_get
_model
(model_name
=model_name
)
1424 application_name
= N2VCJujuConnector
._format
_app
_name
(application_name
)
1426 if model
.applications
and application_name
in model
.applications
:
1427 return model
.applications
[application_name
]
1429 raise N2VCException(
1430 message
="Cannot get application {} from model {}".format(
1431 application_name
, model_name
1435 async def _juju_get_model(self
, model_name
: str) -> Model
:
1436 """ Get a model object from juju controller
1437 If the model does not exits, it creates it.
1439 :param str model_name: name of the model
1440 :returns Model: model obtained from juju controller or Exception
1444 model_name
= N2VCJujuConnector
._format
_model
_name
(model_name
)
1446 if model_name
in self
.juju_models
:
1447 return self
.juju_models
[model_name
]
1449 if self
._creating
_model
:
1450 self
.log
.debug("Another coroutine is creating a model. Wait...")
1451 while self
._creating
_model
:
1452 # another coroutine is creating a model, wait
1453 await asyncio
.sleep(0.1)
1454 # retry (perhaps another coroutine has created the model meanwhile)
1455 if model_name
in self
.juju_models
:
1456 return self
.juju_models
[model_name
]
1459 self
._creating
_model
= True
1461 # get juju model names from juju
1462 model_list
= await self
.controller
.list_models()
1463 if model_name
not in model_list
:
1465 "Model {} does not exist. Creating new model...".format(model_name
)
1467 config_dict
= {"authorized-keys": self
.public_key
}
1469 config_dict
["apt-mirror"] = self
.apt_mirror
1470 if not self
.enable_os_upgrade
:
1471 config_dict
["enable-os-refresh-update"] = False
1472 config_dict
["enable-os-upgrade"] = False
1473 if self
.cloud
in self
.BUILT_IN_CLOUDS
:
1474 model
= await self
.controller
.add_model(
1475 model_name
=model_name
,
1477 cloud_name
=self
.cloud
,
1480 model
= await self
.controller
.add_model(
1481 model_name
=model_name
,
1483 cloud_name
=self
.cloud
,
1484 credential_name
=self
.cloud
,
1486 self
.log
.info("New model created, name={}".format(model_name
))
1489 "Model already exists in juju. Getting model {}".format(model_name
)
1491 model
= await self
.controller
.get_model(model_name
)
1492 self
.log
.debug("Existing model in juju, name={}".format(model_name
))
1494 self
.juju_models
[model_name
] = model
1495 self
.juju_observers
[model_name
] = JujuModelObserver(n2vc
=self
, model
=model
)
1498 except Exception as e
:
1499 msg
= "Cannot get model {}. Exception: {}".format(model_name
, e
)
1501 raise N2VCException(msg
)
1503 self
._creating
_model
= False
1505 async def _juju_add_relation(
1508 application_name_1
: str,
1509 application_name_2
: str,
1514 # get juju model and observer
1515 model
= await self
._juju
_get
_model
(model_name
=model_name
)
1517 r1
= "{}:{}".format(application_name_1
, relation_1
)
1518 r2
= "{}:{}".format(application_name_2
, relation_2
)
1520 self
.log
.debug("adding relation: {} -> {}".format(r1
, r2
))
1522 await model
.add_relation(relation1
=r1
, relation2
=r2
)
1523 except JujuAPIError
as e
:
1524 # If one of the applications in the relationship doesn't exist, or the
1525 # relation has already been added,
1526 # let the operation fail silently.
1527 if "not found" in e
.message
:
1529 if "already exists" in e
.message
:
1531 # another execption, raise it
1534 async def _juju_destroy_application(self
, model_name
: str, application_name
: str):
1537 "Destroying application {} in model {}".format(application_name
, model_name
)
1540 # get juju model and observer
1541 model
= await self
._juju
_get
_model
(model_name
=model_name
)
1542 observer
= self
.juju_observers
[model_name
]
1544 application
= model
.applications
.get(application_name
)
1546 observer
.unregister_application(application_name
)
1547 await application
.destroy()
1549 self
.log
.debug("Application not found: {}".format(application_name
))
1551 async def _juju_destroy_machine(
1552 self
, model_name
: str, machine_id
: str, total_timeout
: float = None
1556 "Destroying machine {} in model {}".format(machine_id
, model_name
)
1559 if total_timeout
is None:
1560 total_timeout
= 3600
1562 # get juju model and observer
1563 model
= await self
._juju
_get
_model
(model_name
=model_name
)
1564 observer
= self
.juju_observers
[model_name
]
1566 machines
= await model
.get_machines()
1567 if machine_id
in machines
:
1568 machine
= model
.machines
[machine_id
]
1569 observer
.unregister_machine(machine_id
)
1570 # TODO: change this by machine.is_manual when this is upstreamed:
1571 # https://github.com/juju/python-libjuju/pull/396
1572 if "instance-id" in machine
.safe_data
and machine
.safe_data
[
1574 ].startswith("manual:"):
1575 self
.log
.debug("machine.destroy(force=True) started.")
1576 await machine
.destroy(force
=True)
1577 self
.log
.debug("machine.destroy(force=True) passed.")
1579 end
= time
.time() + total_timeout
1580 # wait for machine removal
1581 machines
= await model
.get_machines()
1582 while machine_id
in machines
and time
.time() < end
:
1584 "Waiting for machine {} is destroyed".format(machine_id
)
1586 await asyncio
.sleep(0.5)
1587 machines
= await model
.get_machines()
1588 self
.log
.debug("Machine destroyed: {}".format(machine_id
))
1590 self
.log
.debug("Machine not found: {}".format(machine_id
))
1592 async def _juju_destroy_model(self
, model_name
: str, total_timeout
: float = None):
1594 self
.log
.debug("Destroying model {}".format(model_name
))
1596 if total_timeout
is None:
1597 total_timeout
= 3600
1598 end
= time
.time() + total_timeout
1600 model
= await self
._juju
_get
_model
(model_name
=model_name
)
1603 raise N2VCNotFound(message
="Model {} does not exist".format(model_name
))
1605 uuid
= model
.info
.uuid
1607 # destroy applications
1608 for application_name
in model
.applications
:
1610 await self
._juju
_destroy
_application
(
1611 model_name
=model_name
, application_name
=application_name
1613 except Exception as e
:
1615 "Error destroying application {} in model {}: {}".format(
1616 application_name
, model_name
, e
1621 machines
= await model
.get_machines()
1622 for machine_id
in machines
:
1624 await self
._juju
_destroy
_machine
(
1625 model_name
=model_name
, machine_id
=machine_id
1627 except asyncio
.CancelledError
:
1630 # ignore exceptions destroying machine
1633 await self
._juju
_disconnect
_model
(model_name
=model_name
)
1635 self
.log
.debug("destroying model {}...".format(model_name
))
1636 await self
.controller
.destroy_model(uuid
)
1637 # self.log.debug('model destroy requested {}'.format(model_name))
1639 # wait for model is completely destroyed
1640 self
.log
.debug("Waiting for model {} to be destroyed...".format(model_name
))
1642 while time
.time() < end
:
1644 # await self.controller.get_model(uuid)
1645 models
= await self
.controller
.list_models()
1646 if model_name
not in models
:
1648 "The model {} ({}) was destroyed".format(model_name
, uuid
)
1651 except asyncio
.CancelledError
:
1653 except Exception as e
:
1655 await asyncio
.sleep(5)
1656 raise N2VCException(
1657 "Timeout waiting for model {} to be destroyed {}".format(
1658 model_name
, last_exception
1662 async def _juju_login(self
):
1663 """Connect to juju controller
1667 # if already authenticated, exit function
1668 if self
._authenticated
:
1671 # if connecting, wait for finish
1672 # another task could be trying to connect in parallel
1673 while self
._connecting
:
1674 await asyncio
.sleep(0.1)
1676 # double check after other task has finished
1677 if self
._authenticated
:
1681 self
._connecting
= True
1683 "connecting to juju controller: {} {}:{}{}".format(
1686 self
.secret
[:8] + "...",
1687 " with ca_cert" if self
.ca_cert
else "",
1691 # Create controller object
1692 self
.controller
= Controller(loop
=self
.loop
)
1693 # Connect to controller
1694 await self
.controller
.connect(
1696 username
=self
.username
,
1697 password
=self
.secret
,
1698 cacert
=self
.ca_cert
,
1700 self
._authenticated
= True
1701 self
.log
.info("juju controller connected")
1702 except Exception as e
:
1703 message
= "Exception connecting to juju: {}".format(e
)
1704 self
.log
.error(message
)
1705 raise N2VCConnectionException(message
=message
, url
=self
.url
)
1707 self
._connecting
= False
1709 async def _juju_logout(self
):
1710 """Logout of the Juju controller."""
1711 if not self
._authenticated
:
1714 # disconnect all models
1715 for model_name
in self
.juju_models
:
1717 await self
._juju
_disconnect
_model
(model_name
)
1718 except Exception as e
:
1720 "Error disconnecting model {} : {}".format(model_name
, e
)
1722 # continue with next model...
1724 self
.log
.info("Disconnecting controller")
1726 await self
.controller
.disconnect()
1727 except Exception as e
:
1728 raise N2VCConnectionException(
1729 message
="Error disconnecting controller: {}".format(e
), url
=self
.url
1732 self
.controller
= None
1733 self
._authenticated
= False
1734 self
.log
.info("disconnected")
1736 async def _juju_disconnect_model(self
, model_name
: str):
1737 self
.log
.debug("Disconnecting model {}".format(model_name
))
1738 if model_name
in self
.juju_models
:
1739 await self
.juju_models
[model_name
].disconnect()
1740 self
.juju_models
[model_name
] = None
1741 self
.juju_observers
[model_name
] = None
1743 self
.warning("Cannot disconnect model: {}".format(model_name
))
1745 def _create_juju_public_key(self
):
1746 """Recreate the Juju public key on lcm container, if needed
1747 Certain libjuju commands expect to be run from the same machine as Juju
1748 is bootstrapped to. This method will write the public key to disk in
1749 that location: ~/.local/share/juju/ssh/juju_id_rsa.pub
1752 # Make sure that we have a public key before writing to disk
1753 if self
.public_key
is None or len(self
.public_key
) == 0:
1754 if "OSMLCM_VCA_PUBKEY" in os
.environ
:
1755 self
.public_key
= os
.getenv("OSMLCM_VCA_PUBKEY", "")
1756 if len(self
.public_key
) == 0:
1761 pk_path
= "{}/.local/share/juju/ssh".format(os
.path
.expanduser("~"))
1762 file_path
= "{}/juju_id_rsa.pub".format(pk_path
)
1764 "writing juju public key to file:\n{}\npublic key: {}".format(
1765 file_path
, self
.public_key
1768 if not os
.path
.exists(pk_path
):
1769 # create path and write file
1770 os
.makedirs(pk_path
)
1771 with
open(file_path
, "w") as f
:
1772 self
.log
.debug("Creating juju public key file: {}".format(file_path
))
1773 f
.write(self
.public_key
)
1775 self
.log
.debug("juju public key file already exists: {}".format(file_path
))
1778 def _format_model_name(name
: str) -> str:
1779 """Format the name of the model.
1781 Model names may only contain lowercase letters, digits and hyphens
1784 return name
.replace("_", "-").replace(" ", "-").lower()
1787 def _format_app_name(name
: str) -> str:
1788 """Format the name of the application (in order to assure valid application name).
1790 Application names have restrictions (run juju deploy --help):
1791 - contains lowercase letters 'a'-'z'
1792 - contains numbers '0'-'9'
1793 - contains hyphens '-'
1794 - starts with a lowercase letter
1795 - not two or more consecutive hyphens
1796 - after a hyphen, not a group with all numbers
1799 def all_numbers(s
: str) -> bool:
1805 new_name
= name
.replace("_", "-")
1806 new_name
= new_name
.replace(" ", "-")
1807 new_name
= new_name
.lower()
1808 while new_name
.find("--") >= 0:
1809 new_name
= new_name
.replace("--", "-")
1810 groups
= new_name
.split("-")
1812 # find 'all numbers' groups and prefix them with a letter
1814 for i
in range(len(groups
)):
1816 if all_numbers(group
):
1822 if app_name
[0].isdigit():
1823 app_name
= "z" + app_name