2 # Copyright 2019 Telefonica Investigacion y Desarrollo, S.A.U.
3 # This file is part of OSM
6 # Licensed under the Apache License, Version 2.0 (the "License");
7 # you may not use this file except in compliance with the License.
8 # You may obtain a copy of the License at
10 # http://www.apache.org/licenses/LICENSE-2.0
12 # Unless required by applicable law or agreed to in writing, software
13 # distributed under the License is distributed on an "AS IS" BASIS,
14 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
16 # See the License for the specific language governing permissions and
17 # limitations under the License.
19 # For those usages not covered by the Apache License, Version 2.0 please
20 # contact with: nfvlabs@tid.es
31 from juju
.action
import Action
32 from juju
.application
import Application
33 from juju
.client
import client
34 from juju
.controller
import Controller
35 from juju
.errors
import JujuAPIError
36 from juju
.machine
import Machine
37 from juju
.model
import Model
38 from n2vc
.exceptions
import (
39 N2VCBadArgumentsException
,
41 N2VCConnectionException
,
42 N2VCExecutionException
,
43 N2VCInvalidCertificate
,
46 JujuK8sProxycharmNotSupported
,
48 from n2vc
.juju_observer
import JujuModelObserver
49 from n2vc
.n2vc_conn
import N2VCConnector
50 from n2vc
.n2vc_conn
import obj_to_dict
, obj_to_yaml
51 from n2vc
.provisioner
import AsyncSSHProvisioner
52 from n2vc
.libjuju
import Libjuju
55 class N2VCJujuConnector(N2VCConnector
):
58 ####################################################################################
59 ################################### P U B L I C ####################################
60 ####################################################################################
63 BUILT_IN_CLOUDS
= ["localhost", "microk8s"]
71 url
: str = "127.0.0.1:17070",
72 username
: str = "admin",
73 vca_config
: dict = None,
76 """Initialize juju N2VC connector
79 # parent class constructor
80 N2VCConnector
.__init
__(
88 vca_config
=vca_config
,
89 on_update_db
=on_update_db
,
92 # silence websocket traffic log
93 logging
.getLogger("websockets.protocol").setLevel(logging
.INFO
)
94 logging
.getLogger("juju.client.connection").setLevel(logging
.WARN
)
95 logging
.getLogger("model").setLevel(logging
.WARN
)
97 self
.log
.info("Initializing N2VC juju connector...")
100 ##############################################################
102 ##############################################################
107 raise N2VCBadArgumentsException("Argument url is mandatory", ["url"])
108 url_parts
= url
.split(":")
109 if len(url_parts
) != 2:
110 raise N2VCBadArgumentsException(
111 "Argument url: bad format (localhost:port) -> {}".format(url
), ["url"]
113 self
.hostname
= url_parts
[0]
115 self
.port
= int(url_parts
[1])
117 raise N2VCBadArgumentsException(
118 "url port must be a number -> {}".format(url
), ["url"]
123 raise N2VCBadArgumentsException(
124 "Argument username is mandatory", ["username"]
128 if vca_config
is None:
129 raise N2VCBadArgumentsException(
130 "Argument vca_config is mandatory", ["vca_config"]
133 if "secret" in vca_config
:
134 self
.secret
= vca_config
["secret"]
136 raise N2VCBadArgumentsException(
137 "Argument vca_config.secret is mandatory", ["vca_config.secret"]
140 # pubkey of juju client in osm machine: ~/.local/share/juju/ssh/juju_id_rsa.pub
141 # if exists, it will be written in lcm container: _create_juju_public_key()
142 if "public_key" in vca_config
:
143 self
.public_key
= vca_config
["public_key"]
145 self
.public_key
= None
147 # TODO: Verify ca_cert is valid before using. VCA will crash
148 # if the ca_cert isn't formatted correctly.
149 def base64_to_cacert(b64string
):
150 """Convert the base64-encoded string containing the VCA CACERT.
156 cacert
= base64
.b64decode(b64string
).decode("utf-8")
158 cacert
= re
.sub(r
"\\n", r
"\n", cacert
,)
159 except binascii
.Error
as e
:
160 self
.log
.debug("Caught binascii.Error: {}".format(e
))
161 raise N2VCInvalidCertificate(message
="Invalid CA Certificate")
165 self
.ca_cert
= vca_config
.get("ca_cert")
167 self
.ca_cert
= base64_to_cacert(vca_config
["ca_cert"])
169 if "api_proxy" in vca_config
and vca_config
["api_proxy"] != "":
170 self
.api_proxy
= vca_config
["api_proxy"]
172 "api_proxy for native charms configured: {}".format(self
.api_proxy
)
176 "api_proxy is not configured"
178 self
.api_proxy
= None
180 if "enable_os_upgrade" in vca_config
:
181 self
.enable_os_upgrade
= vca_config
["enable_os_upgrade"]
183 self
.enable_os_upgrade
= True
185 if "apt_mirror" in vca_config
:
186 self
.apt_mirror
= vca_config
["apt_mirror"]
188 self
.apt_mirror
= None
190 self
.cloud
= vca_config
.get('cloud')
191 self
.k8s_cloud
= None
192 if "k8s_cloud" in vca_config
:
193 self
.k8s_cloud
= vca_config
.get("k8s_cloud")
194 self
.log
.debug('Arguments have been checked')
197 self
.controller
= None # it will be filled when connect to juju
198 self
.juju_models
= {} # model objects for every model_name
199 self
.juju_observers
= {} # model observers for every model_name
201 False # while connecting to juju (to avoid duplicate connections)
203 self
._authenticated
= (
204 False # it will be True when juju connection be stablished
206 self
._creating
_model
= False # True during model creation
207 self
.libjuju
= Libjuju(
209 api_proxy
=self
.api_proxy
,
210 enable_os_upgrade
=self
.enable_os_upgrade
,
211 apt_mirror
=self
.apt_mirror
,
212 username
=self
.username
,
213 password
=self
.secret
,
221 # create juju pub key file in lcm container at
222 # ./local/share/juju/ssh/juju_id_rsa.pub
223 self
._create
_juju
_public
_key
()
225 self
.log
.info("N2VC juju connector initialized")
227 async def get_status(self
, namespace
: str, yaml_format
: bool = True):
229 # self.log.info('Getting NS status. namespace: {}'.format(namespace))
231 _nsi_id
, ns_id
, _vnf_id
, _vdu_id
, _vdu_count
= self
._get
_namespace
_components
(
234 # model name is ns_id
236 if model_name
is None:
237 msg
= "Namespace {} not valid".format(namespace
)
239 raise N2VCBadArgumentsException(msg
, ["namespace"])
242 models
= await self
.libjuju
.list_models(contains
=ns_id
)
245 status
[m
] = await self
.libjuju
.get_model_status(m
)
248 return obj_to_yaml(status
)
250 return obj_to_dict(status
)
252 async def create_execution_environment(
256 reuse_ee_id
: str = None,
257 progress_timeout
: float = None,
258 total_timeout
: float = None,
262 "Creating execution environment. namespace: {}, reuse_ee_id: {}".format(
263 namespace
, reuse_ee_id
269 model_name
, application_name
, machine_id
= self
._get
_ee
_id
_components
(
279 ) = self
._get
_namespace
_components
(namespace
=namespace
)
280 # model name is ns_id
283 application_name
= self
._get
_application
_name
(namespace
=namespace
)
286 "model name: {}, application name: {}, machine_id: {}".format(
287 model_name
, application_name
, machine_id
291 # create or reuse a new juju machine
293 if not await self
.libjuju
.model_exists(model_name
):
294 await self
.libjuju
.add_model(model_name
, cloud_name
=self
.cloud
)
295 machine
, new
= await self
.libjuju
.create_machine(
296 model_name
=model_name
,
297 machine_id
=machine_id
,
299 progress_timeout
=progress_timeout
,
300 total_timeout
=total_timeout
,
302 # id for the execution environment
303 ee_id
= N2VCJujuConnector
._build
_ee
_id
(
304 model_name
=model_name
,
305 application_name
=application_name
,
306 machine_id
=str(machine
.entity_id
),
308 self
.log
.debug("ee_id: {}".format(ee_id
))
311 # write ee_id in database
312 self
._write
_ee
_id
_db
(db_dict
=db_dict
, ee_id
=ee_id
)
314 except Exception as e
:
315 message
= "Error creating machine on juju: {}".format(e
)
316 self
.log
.error(message
)
317 raise N2VCException(message
=message
)
319 # new machine credentials
321 "hostname": machine
.dns_name
,
325 "Execution environment created. ee_id: {}, credentials: {}".format(
330 return ee_id
, credentials
332 async def register_execution_environment(
337 progress_timeout
: float = None,
338 total_timeout
: float = None,
342 "Registering execution environment. namespace={}, credentials={}".format(
343 namespace
, credentials
347 if credentials
is None:
348 raise N2VCBadArgumentsException(
349 message
="credentials are mandatory", bad_args
=["credentials"]
351 if credentials
.get("hostname"):
352 hostname
= credentials
["hostname"]
354 raise N2VCBadArgumentsException(
355 message
="hostname is mandatory", bad_args
=["credentials.hostname"]
357 if credentials
.get("username"):
358 username
= credentials
["username"]
360 raise N2VCBadArgumentsException(
361 message
="username is mandatory", bad_args
=["credentials.username"]
363 if "private_key_path" in credentials
:
364 private_key_path
= credentials
["private_key_path"]
366 # if not passed as argument, use generated private key path
367 private_key_path
= self
.private_key_path
369 _nsi_id
, ns_id
, _vnf_id
, _vdu_id
, _vdu_count
= self
._get
_namespace
_components
(
376 application_name
= self
._get
_application
_name
(namespace
=namespace
)
378 # register machine on juju
380 if not await self
.libjuju
.model_exists(model_name
):
381 await self
.libjuju
.add_model(model_name
, cloud_name
=self
.cloud
)
382 machine_id
= await self
.libjuju
.provision_machine(
383 model_name
=model_name
,
386 private_key_path
=private_key_path
,
388 progress_timeout
=progress_timeout
,
389 total_timeout
=total_timeout
,
391 except Exception as e
:
392 self
.log
.error("Error registering machine: {}".format(e
))
394 message
="Error registering machine on juju: {}".format(e
)
397 self
.log
.info("Machine registered: {}".format(machine_id
))
399 # id for the execution environment
400 ee_id
= N2VCJujuConnector
._build
_ee
_id
(
401 model_name
=model_name
,
402 application_name
=application_name
,
403 machine_id
=str(machine_id
),
406 self
.log
.info("Execution environment registered. ee_id: {}".format(ee_id
))
410 async def install_configuration_sw(
415 progress_timeout
: float = None,
416 total_timeout
: float = None,
423 "Installing configuration sw on ee_id: {}, "
424 "artifact path: {}, db_dict: {}"
425 ).format(ee_id
, artifact_path
, db_dict
)
429 if ee_id
is None or len(ee_id
) == 0:
430 raise N2VCBadArgumentsException(
431 message
="ee_id is mandatory", bad_args
=["ee_id"]
433 if artifact_path
is None or len(artifact_path
) == 0:
434 raise N2VCBadArgumentsException(
435 message
="artifact_path is mandatory", bad_args
=["artifact_path"]
438 raise N2VCBadArgumentsException(
439 message
="db_dict is mandatory", bad_args
=["db_dict"]
447 ) = N2VCJujuConnector
._get
_ee
_id
_components
(ee_id
=ee_id
)
449 "model: {}, application: {}, machine: {}".format(
450 model_name
, application_name
, machine_id
454 raise N2VCBadArgumentsException(
455 message
="ee_id={} is not a valid execution environment id".format(
461 # remove // in charm path
462 while artifact_path
.find("//") >= 0:
463 artifact_path
= artifact_path
.replace("//", "/")
466 if not self
.fs
.file_exists(artifact_path
, mode
="dir"):
467 msg
= "artifact path does not exist: {}".format(artifact_path
)
468 raise N2VCBadArgumentsException(message
=msg
, bad_args
=["artifact_path"])
470 if artifact_path
.startswith("/"):
471 full_path
= self
.fs
.path
+ artifact_path
473 full_path
= self
.fs
.path
+ "/" + artifact_path
476 await self
.libjuju
.deploy_charm(
477 model_name
=model_name
,
478 application_name
=application_name
,
480 machine_id
=machine_id
,
482 progress_timeout
=progress_timeout
,
483 total_timeout
=total_timeout
,
487 except Exception as e
:
489 message
="Error desploying charm into ee={} : {}".format(ee_id
, e
)
492 self
.log
.info("Configuration sw installed")
494 async def install_k8s_proxy_charm(
500 progress_timeout
: float = None,
501 total_timeout
: float = None,
505 Install a k8s proxy charm
507 :param charm_name: Name of the charm being deployed
508 :param namespace: collection of all the uuids related to the charm.
509 :param str artifact_path: where to locate the artifacts (parent folder) using
511 the final artifact path will be a combination of this artifact_path and
512 additional string from the config_dict (e.g. charm name)
513 :param dict db_dict: where to write into database when the status changes.
514 It contains a dict with
515 {collection: <str>, filter: {}, path: <str>},
516 e.g. {collection: "nsrs", filter:
517 {_id: <nsd-id>, path: "_admin.deployed.VCA.3"}
518 :param float progress_timeout:
519 :param float total_timeout:
520 :param config: Dictionary with additional configuration
522 :returns ee_id: execution environment id.
524 self
.log
.info('Installing k8s proxy charm: {}, artifact path: {}, db_dict: {}'
525 .format(charm_name
, artifact_path
, db_dict
))
527 if not self
.k8s_cloud
:
528 raise JujuK8sProxycharmNotSupported("There is not k8s_cloud available")
530 if artifact_path
is None or len(artifact_path
) == 0:
531 raise N2VCBadArgumentsException(
532 message
="artifact_path is mandatory", bad_args
=["artifact_path"]
535 raise N2VCBadArgumentsException(message
='db_dict is mandatory', bad_args
=['db_dict'])
537 # remove // in charm path
538 while artifact_path
.find('//') >= 0:
539 artifact_path
= artifact_path
.replace('//', '/')
542 if not self
.fs
.file_exists(artifact_path
, mode
="dir"):
543 msg
= 'artifact path does not exist: {}'.format(artifact_path
)
544 raise N2VCBadArgumentsException(message
=msg
, bad_args
=['artifact_path'])
546 if artifact_path
.startswith('/'):
547 full_path
= self
.fs
.path
+ artifact_path
549 full_path
= self
.fs
.path
+ '/' + artifact_path
551 _
, ns_id
, _
, _
, _
= self
._get
_namespace
_components
(namespace
=namespace
)
552 model_name
= '{}-k8s'.format(ns_id
)
554 await self
.libjuju
.add_model(model_name
, self
.k8s_cloud
)
555 application_name
= self
._get
_application
_name
(namespace
)
558 await self
.libjuju
.deploy_charm(
559 model_name
=model_name
,
560 application_name
=application_name
,
564 progress_timeout
=progress_timeout
,
565 total_timeout
=total_timeout
,
568 except Exception as e
:
569 raise N2VCException(message
='Error deploying charm: {}'.format(e
))
571 self
.log
.info('K8s proxy charm installed')
572 ee_id
= N2VCJujuConnector
._build
_ee
_id
(
573 model_name
=model_name
,
574 application_name
=application_name
,
578 self
._write
_ee
_id
_db
(db_dict
=db_dict
, ee_id
=ee_id
)
582 async def get_ee_ssh_public__key(
586 progress_timeout
: float = None,
587 total_timeout
: float = None,
592 "Generating priv/pub key pair and get pub key on ee_id: {}, db_dict: {}"
593 ).format(ee_id
, db_dict
)
597 if ee_id
is None or len(ee_id
) == 0:
598 raise N2VCBadArgumentsException(
599 message
="ee_id is mandatory", bad_args
=["ee_id"]
602 raise N2VCBadArgumentsException(
603 message
="db_dict is mandatory", bad_args
=["db_dict"]
611 ) = N2VCJujuConnector
._get
_ee
_id
_components
(ee_id
=ee_id
)
613 "model: {}, application: {}, machine: {}".format(
614 model_name
, application_name
, machine_id
618 raise N2VCBadArgumentsException(
619 message
="ee_id={} is not a valid execution environment id".format(
625 # try to execute ssh layer primitives (if exist):
631 application_name
= N2VCJujuConnector
._format
_app
_name
(application_name
)
633 # execute action: generate-ssh-key
635 output
, _status
= await self
.libjuju
.execute_action(
636 model_name
=model_name
,
637 application_name
=application_name
,
638 action_name
="generate-ssh-key",
640 progress_timeout
=progress_timeout
,
641 total_timeout
=total_timeout
,
643 except Exception as e
:
645 "Skipping exception while executing action generate-ssh-key: {}".format(
650 # execute action: get-ssh-public-key
652 output
, _status
= await self
.libjuju
.execute_action(
653 model_name
=model_name
,
654 application_name
=application_name
,
655 action_name
="get-ssh-public-key",
657 progress_timeout
=progress_timeout
,
658 total_timeout
=total_timeout
,
660 except Exception as e
:
661 msg
= "Cannot execute action get-ssh-public-key: {}\n".format(e
)
663 raise N2VCExecutionException(e
, primitive_name
="get-ssh-public-key")
665 # return public key if exists
666 return output
["pubkey"] if "pubkey" in output
else output
668 async def add_relation(
669 self
, ee_id_1
: str, ee_id_2
: str, endpoint_1
: str, endpoint_2
: str
673 "adding new relation between {} and {}, endpoints: {}, {}".format(
674 ee_id_1
, ee_id_2
, endpoint_1
, endpoint_2
680 message
= "EE 1 is mandatory"
681 self
.log
.error(message
)
682 raise N2VCBadArgumentsException(message
=message
, bad_args
=["ee_id_1"])
684 message
= "EE 2 is mandatory"
685 self
.log
.error(message
)
686 raise N2VCBadArgumentsException(message
=message
, bad_args
=["ee_id_2"])
688 message
= "endpoint 1 is mandatory"
689 self
.log
.error(message
)
690 raise N2VCBadArgumentsException(message
=message
, bad_args
=["endpoint_1"])
692 message
= "endpoint 2 is mandatory"
693 self
.log
.error(message
)
694 raise N2VCBadArgumentsException(message
=message
, bad_args
=["endpoint_2"])
696 # get the model, the applications and the machines from the ee_id's
697 model_1
, app_1
, _machine_1
= self
._get
_ee
_id
_components
(ee_id_1
)
698 model_2
, app_2
, _machine_2
= self
._get
_ee
_id
_components
(ee_id_2
)
700 # model must be the same
701 if model_1
!= model_2
:
702 message
= "EE models are not the same: {} vs {}".format(ee_id_1
, ee_id_2
)
703 self
.log
.error(message
)
704 raise N2VCBadArgumentsException(
705 message
=message
, bad_args
=["ee_id_1", "ee_id_2"]
708 # add juju relations between two applications
710 await self
.libjuju
.add_relation(
712 application_name_1
=app_1
,
713 application_name_2
=app_2
,
714 relation_1
=endpoint_1
,
715 relation_2
=endpoint_2
,
717 except Exception as e
:
718 message
= "Error adding relation between {} and {}: {}".format(
721 self
.log
.error(message
)
722 raise N2VCException(message
=message
)
724 async def remove_relation(self
):
726 self
.log
.info("Method not implemented yet")
727 raise MethodNotImplemented()
729 async def deregister_execution_environments(self
):
730 self
.log
.info("Method not implemented yet")
731 raise MethodNotImplemented()
733 async def delete_namespace(
734 self
, namespace
: str, db_dict
: dict = None, total_timeout
: float = None
736 self
.log
.info("Deleting namespace={}".format(namespace
))
739 if namespace
is None:
740 raise N2VCBadArgumentsException(
741 message
="namespace is mandatory", bad_args
=["namespace"]
744 _nsi_id
, ns_id
, _vnf_id
, _vdu_id
, _vdu_count
= self
._get
_namespace
_components
(
747 if ns_id
is not None:
749 models
= await self
.libjuju
.list_models(contains
=ns_id
)
751 await self
.libjuju
.destroy_model(
752 model_name
=model
, total_timeout
=total_timeout
754 except Exception as e
:
756 message
="Error deleting namespace {} : {}".format(namespace
, e
)
759 raise N2VCBadArgumentsException(
760 message
="only ns_id is permitted to delete yet", bad_args
=["namespace"]
763 self
.log
.info("Namespace {} deleted".format(namespace
))
765 async def delete_execution_environment(
766 self
, ee_id
: str, db_dict
: dict = None, total_timeout
: float = None
768 self
.log
.info("Deleting execution environment ee_id={}".format(ee_id
))
772 raise N2VCBadArgumentsException(
773 message
="ee_id is mandatory", bad_args
=["ee_id"]
776 model_name
, application_name
, _machine_id
= self
._get
_ee
_id
_components
(
780 # destroy the application
782 await self
.libjuju
.destroy_model(
783 model_name
=model_name
, total_timeout
=total_timeout
785 except Exception as e
:
788 "Error deleting execution environment {} (application {}) : {}"
789 ).format(ee_id
, application_name
, e
)
792 # destroy the machine
794 # await self._juju_destroy_machine(
795 # model_name=model_name,
796 # machine_id=machine_id,
797 # total_timeout=total_timeout
799 # except Exception as e:
800 # raise N2VCException(
801 # message='Error deleting execution environment {} (machine {}) : {}'
802 # .format(ee_id, machine_id, e))
804 self
.log
.info("Execution environment {} deleted".format(ee_id
))
806 async def exec_primitive(
811 db_dict
: dict = None,
812 progress_timeout
: float = None,
813 total_timeout
: float = None,
817 "Executing primitive: {} on ee: {}, params: {}".format(
818 primitive_name
, ee_id
, params_dict
823 if ee_id
is None or len(ee_id
) == 0:
824 raise N2VCBadArgumentsException(
825 message
="ee_id is mandatory", bad_args
=["ee_id"]
827 if primitive_name
is None or len(primitive_name
) == 0:
828 raise N2VCBadArgumentsException(
829 message
="action_name is mandatory", bad_args
=["action_name"]
831 if params_dict
is None:
839 ) = N2VCJujuConnector
._get
_ee
_id
_components
(ee_id
=ee_id
)
841 raise N2VCBadArgumentsException(
842 message
="ee_id={} is not a valid execution environment id".format(
848 if primitive_name
== "config":
849 # Special case: config primitive
851 await self
.libjuju
.configure_application(
852 model_name
=model_name
,
853 application_name
=application_name
,
856 actions
= await self
.libjuju
.get_actions(
857 application_name
=application_name
, model_name
=model_name
,
860 "Application {} has these actions: {}".format(
861 application_name
, actions
864 if "verify-ssh-credentials" in actions
:
865 # execute verify-credentials
868 for _
in range(num_retries
):
870 self
.log
.debug("Executing action verify-ssh-credentials...")
871 output
, ok
= await self
.libjuju
.execute_action(
872 model_name
=model_name
,
873 application_name
=application_name
,
874 action_name
="verify-ssh-credentials",
876 progress_timeout
=progress_timeout
,
877 total_timeout
=total_timeout
,
882 "Error executing verify-ssh-credentials: {}. Retrying..."
884 await asyncio
.sleep(retry_timeout
)
887 self
.log
.debug("Result: {}, output: {}".format(ok
, output
))
889 except asyncio
.CancelledError
:
893 "Error executing verify-ssh-credentials after {} retries. ".format(
898 msg
= "Action verify-ssh-credentials does not exist in application {}".format(
901 self
.log
.debug(msg
=msg
)
902 except Exception as e
:
903 self
.log
.error("Error configuring juju application: {}".format(e
))
904 raise N2VCExecutionException(
905 message
="Error configuring application into ee={} : {}".format(
908 primitive_name
=primitive_name
,
913 output
, status
= await self
.libjuju
.execute_action(
914 model_name
=model_name
,
915 application_name
=application_name
,
916 action_name
=primitive_name
,
918 progress_timeout
=progress_timeout
,
919 total_timeout
=total_timeout
,
922 if status
== "completed":
925 raise Exception("status is not completed: {}".format(status
))
926 except Exception as e
:
928 "Error executing primitive {}: {}".format(primitive_name
, e
)
930 raise N2VCExecutionException(
931 message
="Error executing primitive {} into ee={} : {}".format(
932 primitive_name
, ee_id
, e
934 primitive_name
=primitive_name
,
937 async def disconnect(self
):
938 self
.log
.info("closing juju N2VC...")
940 await self
.libjuju
.disconnect()
941 except Exception as e
:
942 raise N2VCConnectionException(
943 message
="Error disconnecting controller: {}".format(e
), url
=self
.url
947 ####################################################################################
948 ################################### P R I V A T E ##################################
949 ####################################################################################
952 def _write_ee_id_db(self
, db_dict
: dict, ee_id
: str):
954 # write ee_id to database: _admin.deployed.VCA.x
956 the_table
= db_dict
["collection"]
957 the_filter
= db_dict
["filter"]
958 the_path
= db_dict
["path"]
959 if not the_path
[-1] == ".":
960 the_path
= the_path
+ "."
961 update_dict
= {the_path
+ "ee_id": ee_id
}
962 # self.log.debug('Writing ee_id to database: {}'.format(the_path))
966 update_dict
=update_dict
,
969 except asyncio
.CancelledError
:
971 except Exception as e
:
972 self
.log
.error("Error writing ee_id to database: {}".format(e
))
975 def _build_ee_id(model_name
: str, application_name
: str, machine_id
: str):
977 Build an execution environment id form model, application and machine
979 :param application_name:
983 # id for the execution environment
984 return "{}.{}.{}".format(model_name
, application_name
, machine_id
)
987 def _get_ee_id_components(ee_id
: str) -> (str, str, str):
989 Get model, application and machine components from an execution environment id
991 :return: model_name, application_name, machine_id
995 return None, None, None
997 # split components of id
998 parts
= ee_id
.split(".")
999 model_name
= parts
[0]
1000 application_name
= parts
[1]
1001 machine_id
= parts
[2]
1002 return model_name
, application_name
, machine_id
1004 def _get_application_name(self
, namespace
: str) -> str:
1006 Build application name from namespace
1008 :return: app-vnf-<vnf id>-vdu-<vdu-id>-cnt-<vdu-count>
1011 # TODO: Enforce the Juju 50-character application limit
1013 # split namespace components
1014 _
, _
, vnf_id
, vdu_id
, vdu_count
= self
._get
_namespace
_components
(
1018 if vnf_id
is None or len(vnf_id
) == 0:
1021 # Shorten the vnf_id to its last twelve characters
1022 vnf_id
= "vnf-" + vnf_id
[-12:]
1024 if vdu_id
is None or len(vdu_id
) == 0:
1027 # Shorten the vdu_id to its last twelve characters
1028 vdu_id
= "-vdu-" + vdu_id
[-12:]
1030 if vdu_count
is None or len(vdu_count
) == 0:
1033 vdu_count
= "-cnt-" + vdu_count
1035 application_name
= "app-{}{}{}".format(vnf_id
, vdu_id
, vdu_count
)
1037 return N2VCJujuConnector
._format
_app
_name
(application_name
)
1039 async def _juju_create_machine(
1042 application_name
: str,
1043 machine_id
: str = None,
1044 db_dict
: dict = None,
1045 progress_timeout
: float = None,
1046 total_timeout
: float = None,
1050 "creating machine in model: {}, existing machine id: {}".format(
1051 model_name
, machine_id
1055 # get juju model and observer (create model if needed)
1056 model
= await self
._juju
_get
_model
(model_name
=model_name
)
1057 observer
= self
.juju_observers
[model_name
]
1059 # find machine id in model
1061 if machine_id
is not None:
1062 self
.log
.debug("Finding existing machine id {} in model".format(machine_id
))
1063 # get juju existing machines in the model
1064 existing_machines
= await model
.get_machines()
1065 if machine_id
in existing_machines
:
1067 "Machine id {} found in model (reusing it)".format(machine_id
)
1069 machine
= model
.machines
[machine_id
]
1072 self
.log
.debug("Creating a new machine in juju...")
1073 # machine does not exist, create it and wait for it
1074 machine
= await model
.add_machine(
1075 spec
=None, constraints
=None, disks
=None, series
="xenial"
1078 # register machine with observer
1079 observer
.register_machine(machine
=machine
, db_dict
=db_dict
)
1081 # id for the execution environment
1082 ee_id
= N2VCJujuConnector
._build
_ee
_id
(
1083 model_name
=model_name
,
1084 application_name
=application_name
,
1085 machine_id
=str(machine
.entity_id
),
1088 # write ee_id in database
1089 self
._write
_ee
_id
_db
(db_dict
=db_dict
, ee_id
=ee_id
)
1091 # wait for machine creation
1092 await observer
.wait_for_machine(
1093 machine_id
=str(machine
.entity_id
),
1094 progress_timeout
=progress_timeout
,
1095 total_timeout
=total_timeout
,
1100 self
.log
.debug("Reusing old machine pending")
1102 # register machine with observer
1103 observer
.register_machine(machine
=machine
, db_dict
=db_dict
)
1105 # machine does exist, but it is in creation process (pending), wait for
1106 # create finalisation
1107 await observer
.wait_for_machine(
1108 machine_id
=machine
.entity_id
,
1109 progress_timeout
=progress_timeout
,
1110 total_timeout
=total_timeout
,
1113 self
.log
.debug("Machine ready at " + str(machine
.dns_name
))
1116 async def _juju_provision_machine(
1121 private_key_path
: str,
1122 db_dict
: dict = None,
1123 progress_timeout
: float = None,
1124 total_timeout
: float = None,
1127 if not self
.api_proxy
:
1128 msg
= "Cannot provision machine: api_proxy is not defined"
1129 self
.log
.error(msg
=msg
)
1130 raise N2VCException(message
=msg
)
1133 "provisioning machine. model: {}, hostname: {}, username: {}".format(
1134 model_name
, hostname
, username
1138 if not self
._authenticated
:
1139 await self
._juju
_login
()
1141 # get juju model and observer
1142 model
= await self
._juju
_get
_model
(model_name
=model_name
)
1143 observer
= self
.juju_observers
[model_name
]
1145 # TODO check if machine is already provisioned
1146 machine_list
= await model
.get_machines()
1148 provisioner
= AsyncSSHProvisioner(
1151 private_key_path
=private_key_path
,
1157 params
= await provisioner
.provision_machine()
1158 except Exception as ex
:
1159 msg
= "Exception provisioning machine: {}".format(ex
)
1161 raise N2VCException(message
=msg
)
1163 params
.jobs
= ["JobHostUnits"]
1165 connection
= model
.connection()
1167 # Submit the request.
1168 self
.log
.debug("Adding machine to model")
1169 client_facade
= client
.ClientFacade
.from_connection(connection
)
1170 results
= await client_facade
.AddMachines(params
=[params
])
1171 error
= results
.machines
[0].error
1173 msg
= "Error adding machine: {}".format(error
.message
)
1174 self
.log
.error(msg
=msg
)
1175 raise ValueError(msg
)
1177 machine_id
= results
.machines
[0].machine
1179 # Need to run this after AddMachines has been called,
1180 # as we need the machine_id
1181 self
.log
.debug("Installing Juju agent into machine {}".format(machine_id
))
1182 asyncio
.ensure_future(
1183 provisioner
.install_agent(
1184 connection
=connection
,
1186 machine_id
=machine_id
,
1187 proxy
=self
.api_proxy
,
1191 # wait for machine in model (now, machine is not yet in model, so we must
1195 machine_list
= await model
.get_machines()
1196 if machine_id
in machine_list
:
1197 self
.log
.debug("Machine {} found in model!".format(machine_id
))
1198 machine
= model
.machines
.get(machine_id
)
1200 await asyncio
.sleep(2)
1203 msg
= "Machine {} not found in model".format(machine_id
)
1204 self
.log
.error(msg
=msg
)
1205 raise Exception(msg
)
1207 # register machine with observer
1208 observer
.register_machine(machine
=machine
, db_dict
=db_dict
)
1210 # wait for machine creation
1211 self
.log
.debug("waiting for provision finishes... {}".format(machine_id
))
1212 await observer
.wait_for_machine(
1213 machine_id
=machine_id
,
1214 progress_timeout
=progress_timeout
,
1215 total_timeout
=total_timeout
,
1218 self
.log
.debug("Machine provisioned {}".format(machine_id
))
1222 async def _juju_deploy_charm(
1225 application_name
: str,
1229 progress_timeout
: float = None,
1230 total_timeout
: float = None,
1231 config
: dict = None,
1232 ) -> (Application
, int):
1234 # get juju model and observer
1235 model
= await self
._juju
_get
_model
(model_name
=model_name
)
1236 observer
= self
.juju_observers
[model_name
]
1238 # check if application already exists
1240 if application_name
in model
.applications
:
1241 application
= model
.applications
[application_name
]
1243 if application
is None:
1245 # application does not exist, create it and wait for it
1247 "deploying application {} to machine {}, model {}".format(
1248 application_name
, machine_id
, model_name
1251 self
.log
.debug("charm: {}".format(charm_path
))
1252 machine
= model
.machines
[machine_id
]
1254 application
= await model
.deploy(
1255 entity_url
=charm_path
,
1256 application_name
=application_name
,
1259 series
=machine
.series
,
1264 # register application with observer
1265 observer
.register_application(application
=application
, db_dict
=db_dict
)
1268 "waiting for application deployed... {}".format(application
.entity_id
)
1270 retries
= await observer
.wait_for_application(
1271 application_id
=application
.entity_id
,
1272 progress_timeout
=progress_timeout
,
1273 total_timeout
=total_timeout
,
1275 self
.log
.debug("application deployed")
1279 # register application with observer
1280 observer
.register_application(application
=application
, db_dict
=db_dict
)
1282 # application already exists, but not finalised
1283 self
.log
.debug("application already exists, waiting for deployed...")
1284 retries
= await observer
.wait_for_application(
1285 application_id
=application
.entity_id
,
1286 progress_timeout
=progress_timeout
,
1287 total_timeout
=total_timeout
,
1289 self
.log
.debug("application deployed")
1291 return application
, retries
1293 async def _juju_execute_action(
1296 application_name
: str,
1299 progress_timeout
: float = None,
1300 total_timeout
: float = None,
1304 # get juju model and observer
1305 model
= await self
._juju
_get
_model
(model_name
=model_name
)
1306 observer
= self
.juju_observers
[model_name
]
1308 application
= await self
._juju
_get
_application
(
1309 model_name
=model_name
, application_name
=application_name
1313 for u
in application
.units
:
1314 if await u
.is_leader_from_status():
1316 if unit
is not None:
1317 actions
= await application
.get_actions()
1318 if action_name
in actions
:
1320 'executing action "{}" using params: {}'.format(action_name
, kwargs
)
1322 action
= await unit
.run_action(action_name
, **kwargs
)
1324 # register action with observer
1325 observer
.register_action(action
=action
, db_dict
=db_dict
)
1327 await observer
.wait_for_action(
1328 action_id
=action
.entity_id
,
1329 progress_timeout
=progress_timeout
,
1330 total_timeout
=total_timeout
,
1332 self
.log
.debug("action completed with status: {}".format(action
.status
))
1333 output
= await model
.get_action_output(action_uuid
=action
.entity_id
)
1334 status
= await model
.get_action_status(uuid_or_prefix
=action
.entity_id
)
1335 if action
.entity_id
in status
:
1336 status
= status
[action
.entity_id
]
1339 return output
, status
1341 raise N2VCExecutionException(
1342 message
="Cannot execute action on charm", primitive_name
=action_name
1345 async def _juju_configure_application(
1348 application_name
: str,
1351 progress_timeout
: float = None,
1352 total_timeout
: float = None,
1355 # get the application
1356 application
= await self
._juju
_get
_application
(
1357 model_name
=model_name
, application_name
=application_name
1361 "configuring the application {} -> {}".format(application_name
, config
)
1363 res
= await application
.set_config(config
)
1365 "application {} configured. res={}".format(application_name
, res
)
1368 # Verify the config is set
1369 new_conf
= await application
.get_config()
1371 value
= new_conf
[key
]["value"]
1372 self
.log
.debug(" {} = {}".format(key
, value
))
1373 if config
[key
] != value
:
1374 raise N2VCException(
1375 message
="key {} is not configured correctly {} != {}".format(
1376 key
, config
[key
], new_conf
[key
]
1380 # check if 'verify-ssh-credentials' action exists
1381 # unit = application.units[0]
1382 actions
= await application
.get_actions()
1383 if "verify-ssh-credentials" not in actions
:
1385 "Action verify-ssh-credentials does not exist in application {}"
1386 ).format(application_name
)
1387 self
.log
.debug(msg
=msg
)
1390 # execute verify-credentials
1392 retry_timeout
= 15.0
1393 for _
in range(num_retries
):
1395 self
.log
.debug("Executing action verify-ssh-credentials...")
1396 output
, ok
= await self
._juju
_execute
_action
(
1397 model_name
=model_name
,
1398 application_name
=application_name
,
1399 action_name
="verify-ssh-credentials",
1401 progress_timeout
=progress_timeout
,
1402 total_timeout
=total_timeout
,
1404 self
.log
.debug("Result: {}, output: {}".format(ok
, output
))
1406 except asyncio
.CancelledError
:
1408 except Exception as e
:
1410 "Error executing verify-ssh-credentials: {}. Retrying...".format(e
)
1412 await asyncio
.sleep(retry_timeout
)
1415 "Error executing verify-ssh-credentials after {} retries. ".format(
1421 async def _juju_get_application(self
, model_name
: str, application_name
: str):
1422 """Get the deployed application."""
1424 model
= await self
._juju
_get
_model
(model_name
=model_name
)
1426 application_name
= N2VCJujuConnector
._format
_app
_name
(application_name
)
1428 if model
.applications
and application_name
in model
.applications
:
1429 return model
.applications
[application_name
]
1431 raise N2VCException(
1432 message
="Cannot get application {} from model {}".format(
1433 application_name
, model_name
1437 async def _juju_get_model(self
, model_name
: str) -> Model
:
1438 """ Get a model object from juju controller
1439 If the model does not exits, it creates it.
1441 :param str model_name: name of the model
1442 :returns Model: model obtained from juju controller or Exception
1446 model_name
= N2VCJujuConnector
._format
_model
_name
(model_name
)
1448 if model_name
in self
.juju_models
:
1449 return self
.juju_models
[model_name
]
1451 if self
._creating
_model
:
1452 self
.log
.debug("Another coroutine is creating a model. Wait...")
1453 while self
._creating
_model
:
1454 # another coroutine is creating a model, wait
1455 await asyncio
.sleep(0.1)
1456 # retry (perhaps another coroutine has created the model meanwhile)
1457 if model_name
in self
.juju_models
:
1458 return self
.juju_models
[model_name
]
1461 self
._creating
_model
= True
1463 # get juju model names from juju
1464 model_list
= await self
.controller
.list_models()
1465 if model_name
not in model_list
:
1467 "Model {} does not exist. Creating new model...".format(model_name
)
1469 config_dict
= {"authorized-keys": self
.public_key
}
1471 config_dict
["apt-mirror"] = self
.apt_mirror
1472 if not self
.enable_os_upgrade
:
1473 config_dict
["enable-os-refresh-update"] = False
1474 config_dict
["enable-os-upgrade"] = False
1475 if self
.cloud
in self
.BUILT_IN_CLOUDS
:
1476 model
= await self
.controller
.add_model(
1477 model_name
=model_name
,
1479 cloud_name
=self
.cloud
,
1482 model
= await self
.controller
.add_model(
1483 model_name
=model_name
,
1485 cloud_name
=self
.cloud
,
1486 credential_name
=self
.cloud
,
1488 self
.log
.info("New model created, name={}".format(model_name
))
1491 "Model already exists in juju. Getting model {}".format(model_name
)
1493 model
= await self
.controller
.get_model(model_name
)
1494 self
.log
.debug("Existing model in juju, name={}".format(model_name
))
1496 self
.juju_models
[model_name
] = model
1497 self
.juju_observers
[model_name
] = JujuModelObserver(n2vc
=self
, model
=model
)
1500 except Exception as e
:
1501 msg
= "Cannot get model {}. Exception: {}".format(model_name
, e
)
1503 raise N2VCException(msg
)
1505 self
._creating
_model
= False
1507 async def _juju_add_relation(
1510 application_name_1
: str,
1511 application_name_2
: str,
1516 # get juju model and observer
1517 model
= await self
._juju
_get
_model
(model_name
=model_name
)
1519 r1
= "{}:{}".format(application_name_1
, relation_1
)
1520 r2
= "{}:{}".format(application_name_2
, relation_2
)
1522 self
.log
.debug("adding relation: {} -> {}".format(r1
, r2
))
1524 await model
.add_relation(relation1
=r1
, relation2
=r2
)
1525 except JujuAPIError
as e
:
1526 # If one of the applications in the relationship doesn't exist, or the
1527 # relation has already been added,
1528 # let the operation fail silently.
1529 if "not found" in e
.message
:
1531 if "already exists" in e
.message
:
1533 # another execption, raise it
1536 async def _juju_destroy_application(self
, model_name
: str, application_name
: str):
1539 "Destroying application {} in model {}".format(application_name
, model_name
)
1542 # get juju model and observer
1543 model
= await self
._juju
_get
_model
(model_name
=model_name
)
1544 observer
= self
.juju_observers
[model_name
]
1546 application
= model
.applications
.get(application_name
)
1548 observer
.unregister_application(application_name
)
1549 await application
.destroy()
1551 self
.log
.debug("Application not found: {}".format(application_name
))
1553 async def _juju_destroy_machine(
1554 self
, model_name
: str, machine_id
: str, total_timeout
: float = None
1558 "Destroying machine {} in model {}".format(machine_id
, model_name
)
1561 if total_timeout
is None:
1562 total_timeout
= 3600
1564 # get juju model and observer
1565 model
= await self
._juju
_get
_model
(model_name
=model_name
)
1566 observer
= self
.juju_observers
[model_name
]
1568 machines
= await model
.get_machines()
1569 if machine_id
in machines
:
1570 machine
= model
.machines
[machine_id
]
1571 observer
.unregister_machine(machine_id
)
1572 # TODO: change this by machine.is_manual when this is upstreamed:
1573 # https://github.com/juju/python-libjuju/pull/396
1574 if "instance-id" in machine
.safe_data
and machine
.safe_data
[
1576 ].startswith("manual:"):
1577 self
.log
.debug("machine.destroy(force=True) started.")
1578 await machine
.destroy(force
=True)
1579 self
.log
.debug("machine.destroy(force=True) passed.")
1581 end
= time
.time() + total_timeout
1582 # wait for machine removal
1583 machines
= await model
.get_machines()
1584 while machine_id
in machines
and time
.time() < end
:
1586 "Waiting for machine {} is destroyed".format(machine_id
)
1588 await asyncio
.sleep(0.5)
1589 machines
= await model
.get_machines()
1590 self
.log
.debug("Machine destroyed: {}".format(machine_id
))
1592 self
.log
.debug("Machine not found: {}".format(machine_id
))
1594 async def _juju_destroy_model(self
, model_name
: str, total_timeout
: float = None):
1596 self
.log
.debug("Destroying model {}".format(model_name
))
1598 if total_timeout
is None:
1599 total_timeout
= 3600
1600 end
= time
.time() + total_timeout
1602 model
= await self
._juju
_get
_model
(model_name
=model_name
)
1605 raise N2VCNotFound(message
="Model {} does not exist".format(model_name
))
1607 uuid
= model
.info
.uuid
1609 # destroy applications
1610 for application_name
in model
.applications
:
1612 await self
._juju
_destroy
_application
(
1613 model_name
=model_name
, application_name
=application_name
1615 except Exception as e
:
1617 "Error destroying application {} in model {}: {}".format(
1618 application_name
, model_name
, e
1623 machines
= await model
.get_machines()
1624 for machine_id
in machines
:
1626 await self
._juju
_destroy
_machine
(
1627 model_name
=model_name
, machine_id
=machine_id
1629 except asyncio
.CancelledError
:
1632 # ignore exceptions destroying machine
1635 await self
._juju
_disconnect
_model
(model_name
=model_name
)
1637 self
.log
.debug("destroying model {}...".format(model_name
))
1638 await self
.controller
.destroy_model(uuid
)
1639 # self.log.debug('model destroy requested {}'.format(model_name))
1641 # wait for model is completely destroyed
1642 self
.log
.debug("Waiting for model {} to be destroyed...".format(model_name
))
1644 while time
.time() < end
:
1646 # await self.controller.get_model(uuid)
1647 models
= await self
.controller
.list_models()
1648 if model_name
not in models
:
1650 "The model {} ({}) was destroyed".format(model_name
, uuid
)
1653 except asyncio
.CancelledError
:
1655 except Exception as e
:
1657 await asyncio
.sleep(5)
1658 raise N2VCException(
1659 "Timeout waiting for model {} to be destroyed {}".format(
1660 model_name
, last_exception
1664 async def _juju_login(self
):
1665 """Connect to juju controller
1669 # if already authenticated, exit function
1670 if self
._authenticated
:
1673 # if connecting, wait for finish
1674 # another task could be trying to connect in parallel
1675 while self
._connecting
:
1676 await asyncio
.sleep(0.1)
1678 # double check after other task has finished
1679 if self
._authenticated
:
1683 self
._connecting
= True
1685 "connecting to juju controller: {} {}:{}{}".format(
1688 self
.secret
[:8] + "...",
1689 " with ca_cert" if self
.ca_cert
else "",
1693 # Create controller object
1694 self
.controller
= Controller(loop
=self
.loop
)
1695 # Connect to controller
1696 await self
.controller
.connect(
1698 username
=self
.username
,
1699 password
=self
.secret
,
1700 cacert
=self
.ca_cert
,
1702 self
._authenticated
= True
1703 self
.log
.info("juju controller connected")
1704 except Exception as e
:
1705 message
= "Exception connecting to juju: {}".format(e
)
1706 self
.log
.error(message
)
1707 raise N2VCConnectionException(message
=message
, url
=self
.url
)
1709 self
._connecting
= False
1711 async def _juju_logout(self
):
1712 """Logout of the Juju controller."""
1713 if not self
._authenticated
:
1716 # disconnect all models
1717 for model_name
in self
.juju_models
:
1719 await self
._juju
_disconnect
_model
(model_name
)
1720 except Exception as e
:
1722 "Error disconnecting model {} : {}".format(model_name
, e
)
1724 # continue with next model...
1726 self
.log
.info("Disconnecting controller")
1728 await self
.controller
.disconnect()
1729 except Exception as e
:
1730 raise N2VCConnectionException(
1731 message
="Error disconnecting controller: {}".format(e
), url
=self
.url
1734 self
.controller
= None
1735 self
._authenticated
= False
1736 self
.log
.info("disconnected")
1738 async def _juju_disconnect_model(self
, model_name
: str):
1739 self
.log
.debug("Disconnecting model {}".format(model_name
))
1740 if model_name
in self
.juju_models
:
1741 await self
.juju_models
[model_name
].disconnect()
1742 self
.juju_models
[model_name
] = None
1743 self
.juju_observers
[model_name
] = None
1745 self
.warning("Cannot disconnect model: {}".format(model_name
))
1747 def _create_juju_public_key(self
):
1748 """Recreate the Juju public key on lcm container, if needed
1749 Certain libjuju commands expect to be run from the same machine as Juju
1750 is bootstrapped to. This method will write the public key to disk in
1751 that location: ~/.local/share/juju/ssh/juju_id_rsa.pub
1754 # Make sure that we have a public key before writing to disk
1755 if self
.public_key
is None or len(self
.public_key
) == 0:
1756 if "OSMLCM_VCA_PUBKEY" in os
.environ
:
1757 self
.public_key
= os
.getenv("OSMLCM_VCA_PUBKEY", "")
1758 if len(self
.public_key
) == 0:
1763 pk_path
= "{}/.local/share/juju/ssh".format(os
.path
.expanduser("~"))
1764 file_path
= "{}/juju_id_rsa.pub".format(pk_path
)
1766 "writing juju public key to file:\n{}\npublic key: {}".format(
1767 file_path
, self
.public_key
1770 if not os
.path
.exists(pk_path
):
1771 # create path and write file
1772 os
.makedirs(pk_path
)
1773 with
open(file_path
, "w") as f
:
1774 self
.log
.debug("Creating juju public key file: {}".format(file_path
))
1775 f
.write(self
.public_key
)
1777 self
.log
.debug("juju public key file already exists: {}".format(file_path
))
1780 def _format_model_name(name
: str) -> str:
1781 """Format the name of the model.
1783 Model names may only contain lowercase letters, digits and hyphens
1786 return name
.replace("_", "-").replace(" ", "-").lower()
1789 def _format_app_name(name
: str) -> str:
1790 """Format the name of the application (in order to assure valid application name).
1792 Application names have restrictions (run juju deploy --help):
1793 - contains lowercase letters 'a'-'z'
1794 - contains numbers '0'-'9'
1795 - contains hyphens '-'
1796 - starts with a lowercase letter
1797 - not two or more consecutive hyphens
1798 - after a hyphen, not a group with all numbers
1801 def all_numbers(s
: str) -> bool:
1807 new_name
= name
.replace("_", "-")
1808 new_name
= new_name
.replace(" ", "-")
1809 new_name
= new_name
.lower()
1810 while new_name
.find("--") >= 0:
1811 new_name
= new_name
.replace("--", "-")
1812 groups
= new_name
.split("-")
1814 # find 'all numbers' groups and prefix them with a letter
1816 for i
in range(len(groups
)):
1818 if all_numbers(group
):
1824 if app_name
[0].isdigit():
1825 app_name
= "z" + app_name