1 # Copyright 2019 Canonical Ltd.
3 # Licensed under the Apache License, Version 2.0 (the "License");
4 # you may not use this file except in compliance with the License.
5 # You may obtain a copy of the License at
7 # http://www.apache.org/licenses/LICENSE-2.0
9 # Unless required by applicable law or agreed to in writing, software
10 # distributed under the License is distributed on an "AS IS" BASIS,
11 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 # See the License for the specific language governing permissions and
13 # limitations under the License.
23 from n2vc
.config
import EnvironConfig
24 from n2vc
.exceptions
import K8sException
25 from n2vc
.k8s_conn
import K8sConnector
26 from n2vc
.kubectl
import Kubectl
, CORE_CLIENT
, RBAC_CLIENT
27 from .exceptions
import MethodNotImplemented
28 from n2vc
.libjuju
import Libjuju
29 from n2vc
.utils
import obj_to_dict
, obj_to_yaml
30 from n2vc
.store
import MotorStore
31 from n2vc
.vca
.cloud
import Cloud
32 from n2vc
.vca
.connection
import get_connection
33 from kubernetes
.client
.models
import (
43 from typing
import Dict
45 SERVICE_ACCOUNT_TOKEN_KEY
= "token"
46 SERVICE_ACCOUNT_ROOT_CA_KEY
= "ca.crt"
47 RBAC_LABEL_KEY_NAME
= "rbac-id"
49 ADMIN_NAMESPACE
= "kube-system"
50 RBAC_STACK_PREFIX
= "juju-credential"
53 def generate_rbac_id():
54 return binascii
.hexlify(os
.urandom(4)).decode()
57 class K8sJujuConnector(K8sConnector
):
64 kubectl_command
: str = "/usr/bin/kubectl",
65 juju_command
: str = "/usr/bin/juju",
71 :param fs: file system for kubernetes and helm configuration
72 :param db: Database object
73 :param kubectl_command: path to kubectl executable
74 :param helm_command: path to helm executable
76 :param: loop: Asyncio loop
80 K8sConnector
.__init
__(
84 on_update_db
=on_update_db
,
88 self
.loop
= loop
or asyncio
.get_event_loop()
89 self
.log
.debug("Initializing K8S Juju connector")
91 db_uri
= EnvironConfig(prefixes
=["OSMLCM_", "OSMMON_"]).get("database_uri")
92 self
._store
= MotorStore(db_uri
)
93 self
.loading_libjuju
= asyncio
.Lock(loop
=self
.loop
)
95 self
.log
.debug("K8S Juju connector initialized")
96 # TODO: Remove these commented lines:
97 # self.authenticated = False
99 # self.juju_secret = ""
106 namespace
: str = "kube-system",
107 reuse_cluster_uuid
: str = None,
111 It prepares a given K8s cluster environment to run Juju bundles.
113 :param k8s_creds: credentials to access a given K8s cluster, i.e. a valid
115 :param namespace: optional namespace to be used for juju. By default,
116 'kube-system' will be used
117 :param reuse_cluster_uuid: existing cluster uuid for reuse
118 :param: kwargs: Additional parameters
121 :return: uuid of the K8s cluster and True if connector has installed some
122 software in the cluster
123 (on error, an exception will be raised)
125 libjuju
= await self
._get
_libjuju
(kwargs
.get("vca_id"))
127 cluster_uuid
= reuse_cluster_uuid
or str(uuid
.uuid4())
129 kubecfg
= tempfile
.NamedTemporaryFile()
130 with
open(kubecfg
.name
, "w") as kubecfg_file
:
131 kubecfg_file
.write(k8s_creds
)
132 kubectl
= Kubectl(config_file
=kubecfg
.name
)
134 # CREATING RESOURCES IN K8S
135 rbac_id
= generate_rbac_id()
136 metadata_name
= "{}-{}".format(RBAC_STACK_PREFIX
, rbac_id
)
137 labels
= {RBAC_STACK_PREFIX
: rbac_id
}
139 # Create cleanup dictionary to clean up created resources
140 # if it fails in the middle of the process
143 self
._create
_cluster
_role
(
150 "delete": self
._delete
_cluster
_role
,
151 "args": (kubectl
, metadata_name
),
155 self
._create
_service
_account
(
162 "delete": self
._delete
_service
_account
,
163 "args": (kubectl
, metadata_name
),
167 self
._create
_cluster
_role
_binding
(
174 "delete": self
._delete
_service
_account
,
175 "args": (kubectl
, metadata_name
),
178 token
, client_cert_data
= await self
._get
_secret
_data
(
183 default_storage_class
= kubectl
.get_default_storage_class()
184 await libjuju
.add_k8s(
188 client_cert_data
=client_cert_data
,
189 configuration
=kubectl
.configuration
,
190 storage_class
=default_storage_class
,
191 credential_name
=self
._get
_credential
_name
(cluster_uuid
),
193 return cluster_uuid
, True
194 except Exception as e
:
195 self
.log
.error("Error initializing k8scluster: {}".format(e
))
196 if len(cleanup_data
) > 0:
197 self
.log
.debug("Cleaning up created resources in k8s cluster...")
198 for item
in cleanup_data
:
199 delete_function
= item
["delete"]
200 delete_args
= item
["args"]
201 delete_function(*delete_args
)
202 self
.log
.debug("Cleanup finished")
205 """Repo Management"""
211 _type
: str = "charm",
213 raise MethodNotImplemented()
215 async def repo_list(self
):
216 raise MethodNotImplemented()
218 async def repo_remove(
222 raise MethodNotImplemented()
224 async def synchronize_repos(self
, cluster_uuid
: str, name
: str):
226 Returns None as currently add_repo is not implemented
236 uninstall_sw
: bool = False,
241 Resets the Kubernetes cluster by removing the model that represents it.
243 :param cluster_uuid str: The UUID of the cluster to reset
244 :param force: Force reset
245 :param uninstall_sw: Boolean to uninstall sw
246 :param: kwargs: Additional parameters
249 :return: Returns True if successful or raises an exception.
253 self
.log
.debug("[reset] Removing k8s cloud")
254 libjuju
= await self
._get
_libjuju
(kwargs
.get("vca_id"))
256 cloud
= Cloud(cluster_uuid
, self
._get
_credential
_name
(cluster_uuid
))
258 cloud_creds
= await libjuju
.get_cloud_credentials(cloud
)
260 await libjuju
.remove_cloud(cluster_uuid
)
262 kubecfg
= self
.get_credentials(cluster_uuid
=cluster_uuid
)
264 kubecfg_file
= tempfile
.NamedTemporaryFile()
265 with
open(kubecfg_file
.name
, "w") as f
:
267 kubectl
= Kubectl(config_file
=kubecfg_file
.name
)
270 self
._delete
_cluster
_role
_binding
,
271 self
._delete
_service
_account
,
272 self
._delete
_cluster
_role
,
275 credential_attrs
= cloud_creds
[0].result
["attrs"]
276 if RBAC_LABEL_KEY_NAME
in credential_attrs
:
277 rbac_id
= credential_attrs
[RBAC_LABEL_KEY_NAME
]
278 metadata_name
= "{}-{}".format(RBAC_STACK_PREFIX
, rbac_id
)
279 delete_args
= (kubectl
, metadata_name
)
280 for delete_func
in delete_functions
:
282 delete_func(*delete_args
)
283 except Exception as e
:
284 self
.log
.warning("Cannot remove resource in K8s {}".format(e
))
286 except Exception as e
:
287 self
.log
.debug("Caught exception during reset: {}".format(e
))
299 timeout
: float = 1800,
301 db_dict
: dict = None,
302 kdu_name
: str = None,
303 namespace
: str = None,
308 :param cluster_uuid str: The UUID of the cluster to install to
309 :param kdu_model str: The name or path of a bundle to install
310 :param kdu_instance: Kdu instance name
311 :param atomic bool: If set, waits until the model is active and resets
312 the cluster on failure.
313 :param timeout int: The time, in seconds, to wait for the install
315 :param params dict: Key-value pairs of instantiation parameters
316 :param kdu_name: Name of the KDU instance to be installed
317 :param namespace: K8s namespace to use for the KDU instance
318 :param kwargs: Additional parameters
321 :return: If successful, returns ?
323 libjuju
= await self
._get
_libjuju
(kwargs
.get("vca_id"))
327 raise K8sException("db_dict must be set")
329 raise K8sException("bundle must be set")
331 if bundle
.startswith("cs:"):
333 elif bundle
.startswith("http"):
337 new_workdir
= kdu_model
.strip(kdu_model
.split("/")[-1])
338 os
.chdir(new_workdir
)
339 bundle
= "local:{}".format(kdu_model
)
341 self
.log
.debug("Checking for model named {}".format(kdu_instance
))
343 # Create the new model
344 self
.log
.debug("Adding model: {}".format(kdu_instance
))
345 cloud
= Cloud(cluster_uuid
, self
._get
_credential
_name
(cluster_uuid
))
346 await libjuju
.add_model(kdu_instance
, cloud
)
349 # TODO: Instantiation parameters
352 "Juju bundle that models the KDU, in any of the following ways:
353 - <juju-repo>/<juju-bundle>
354 - <juju-bundle folder under k8s_models folder in the package>
355 - <juju-bundle tgz file (w/ or w/o extension) under k8s_models folder
357 - <URL_where_to_fetch_juju_bundle>
360 previous_workdir
= os
.getcwd()
361 except FileNotFoundError
:
362 previous_workdir
= "/app/storage"
364 self
.log
.debug("[install] deploying {}".format(bundle
))
365 await libjuju
.deploy(
366 bundle
, model_name
=kdu_instance
, wait
=atomic
, timeout
=timeout
368 os
.chdir(previous_workdir
)
369 if self
.on_update_db
:
370 await self
.on_update_db(
373 filter=db_dict
["filter"],
374 vca_id
=kwargs
.get("vca_id")
378 async def instances_list(self
, cluster_uuid
: str) -> list:
380 returns a list of deployed releases in a cluster
382 :param cluster_uuid: the cluster
391 kdu_model
: str = None,
396 :param cluster_uuid str: The UUID of the cluster to upgrade
397 :param kdu_instance str: The unique name of the KDU instance
398 :param kdu_model str: The name or path of the bundle to upgrade to
399 :param params dict: Key-value pairs of instantiation parameters
401 :return: If successful, reference to the new revision number of the
405 # TODO: Loop through the bundle and upgrade each charm individually
408 The API doesn't have a concept of bundle upgrades, because there are
409 many possible changes: charm revision, disk, number of units, etc.
411 As such, we are only supporting a limited subset of upgrades. We'll
412 upgrade the charm revision but leave storage and scale untouched.
414 Scale changes should happen through OSM constructs, and changes to
415 storage would require a redeployment of the service, at least in this
418 raise MethodNotImplemented()
430 :param cluster_uuid str: The UUID of the cluster to rollback
431 :param kdu_instance str: The unique name of the KDU instance
432 :param revision int: The revision to revert to. If omitted, rolls back
433 the previous upgrade.
435 :return: If successful, returns the revision of active KDU instance,
436 or raises an exception
438 raise MethodNotImplemented()
448 """Uninstall a KDU instance
450 :param cluster_uuid str: The UUID of the cluster
451 :param kdu_instance str: The unique name of the KDU instance
452 :param kwargs: Additional parameters
455 :return: Returns True if successful, or raises an exception
458 self
.log
.debug("[uninstall] Destroying model")
459 libjuju
= await self
._get
_libjuju
(kwargs
.get("vca_id"))
461 await libjuju
.destroy_model(kdu_instance
, total_timeout
=3600)
463 # self.log.debug("[uninstall] Model destroyed and disconnecting")
464 # await controller.disconnect()
467 # TODO: Remove these commented lines
468 # if not self.authenticated:
469 # self.log.debug("[uninstall] Connecting to controller")
470 # await self.login(cluster_uuid)
472 async def exec_primitive(
474 cluster_uuid
: str = None,
475 kdu_instance
: str = None,
476 primitive_name
: str = None,
477 timeout
: float = 300,
479 db_dict
: dict = None,
482 """Exec primitive (Juju action)
484 :param cluster_uuid str: The UUID of the cluster
485 :param kdu_instance str: The unique name of the KDU instance
486 :param primitive_name: Name of action that will be executed
487 :param timeout: Timeout for action execution
488 :param params: Dictionary of all the parameters needed for the action
489 :param db_dict: Dictionary for any additional data
490 :param kwargs: Additional parameters
493 :return: Returns the output of the action
495 libjuju
= await self
._get
_libjuju
(kwargs
.get("vca_id"))
497 if not params
or "application-name" not in params
:
499 "Missing application-name argument, \
500 argument needed for K8s actions"
504 "[exec_primitive] Getting model "
505 "kdu_instance: {}".format(kdu_instance
)
507 application_name
= params
["application-name"]
508 actions
= await libjuju
.get_actions(application_name
, kdu_instance
)
509 if primitive_name
not in actions
:
510 raise K8sException("Primitive {} not found".format(primitive_name
))
511 output
, status
= await libjuju
.execute_action(
512 application_name
, kdu_instance
, primitive_name
, **params
515 if status
!= "completed":
517 "status is not completed: {} output: {}".format(status
, output
)
519 if self
.on_update_db
:
520 await self
.on_update_db(cluster_uuid
, kdu_instance
, filter=db_dict
["filter"])
524 except Exception as e
:
525 error_msg
= "Error executing primitive {}: {}".format(primitive_name
, e
)
526 self
.log
.error(error_msg
)
527 raise K8sException(message
=error_msg
)
531 async def inspect_kdu(
537 Inspects a bundle and returns a dictionary of config parameters and
538 their default values.
540 :param kdu_model str: The name or path of the bundle to inspect.
542 :return: If successful, returns a dictionary of available parameters
543 and their default values.
547 if not os
.path
.exists(kdu_model
):
548 raise K8sException("file {} not found".format(kdu_model
))
550 with
open(kdu_model
, "r") as f
:
551 bundle
= yaml
.safe_load(f
.read())
555 'description': 'Test bundle',
556 'bundle': 'kubernetes',
559 'charm': 'cs:~charmed-osm/mariadb-k8s-20',
562 'password': 'manopw',
563 'root_password': 'osm4u',
566 'series': 'kubernetes'
571 # TODO: This should be returned in an agreed-upon format
572 kdu
= bundle
["applications"]
582 If available, returns the README of the bundle.
584 :param kdu_model str: The name or path of a bundle
586 :return: If found, returns the contents of the README.
590 files
= ["README", "README.txt", "README.md"]
591 path
= os
.path
.dirname(kdu_model
)
592 for file in os
.listdir(path
):
594 with
open(file, "r") as f
:
600 async def status_kdu(
604 complete_status
: bool = False,
605 yaml_format
: bool = False,
608 """Get the status of the KDU
610 Get the current status of the KDU instance.
612 :param cluster_uuid str: The UUID of the cluster
613 :param kdu_instance str: The unique id of the KDU instance
614 :param complete_status: To get the complete_status of the KDU
615 :param yaml_format: To get the status in proper format for NSR record
616 :param: kwargs: Additional parameters
619 :return: Returns a dictionary containing namespace, state, resources,
620 and deployment_time and returns complete_status if complete_status is True
622 libjuju
= await self
._get
_libjuju
(kwargs
.get("vca_id"))
625 model_status
= await libjuju
.get_model_status(kdu_instance
)
627 if not complete_status
:
628 for name
in model_status
.applications
:
629 application
= model_status
.applications
[name
]
630 status
[name
] = {"status": application
["status"]["status"]}
633 return obj_to_yaml(model_status
)
635 return obj_to_dict(model_status
)
639 async def update_vca_status(self
, vcastatus
: dict, kdu_instance
: str, **kwargs
):
641 Add all configs, actions, executed actions of all applications in a model to vcastatus dict
643 :param vcastatus dict: dict containing vcastatus
644 :param kdu_instance str: The unique id of the KDU instance
645 :param: kwargs: Additional parameters
650 libjuju
= await self
._get
_libjuju
(kwargs
.get("vca_id"))
652 for model_name
in vcastatus
:
653 # Adding executed actions
654 vcastatus
[model_name
]["executedActions"] = \
655 await libjuju
.get_executed_actions(kdu_instance
)
657 for application
in vcastatus
[model_name
]["applications"]:
658 # Adding application actions
659 vcastatus
[model_name
]["applications"][application
]["actions"] = \
660 await libjuju
.get_actions(application
, kdu_instance
)
661 # Adding application configs
662 vcastatus
[model_name
]["applications"][application
]["configs"] = \
663 await libjuju
.get_application_configs(kdu_instance
, application
)
665 except Exception as e
:
666 self
.log
.debug("Error in updating vca status: {}".format(str(e
)))
668 async def get_services(
669 self
, cluster_uuid
: str, kdu_instance
: str, namespace
: str
671 """Return a list of services of a kdu_instance"""
673 credentials
= self
.get_credentials(cluster_uuid
=cluster_uuid
)
675 kubecfg
= tempfile
.NamedTemporaryFile()
676 with
open(kubecfg
.name
, "w") as kubecfg_file
:
677 kubecfg_file
.write(credentials
)
678 kubectl
= Kubectl(config_file
=kubecfg
.name
)
680 return kubectl
.get_services(
681 field_selector
="metadata.namespace={}".format(kdu_instance
)
684 async def get_service(
685 self
, cluster_uuid
: str, service_name
: str, namespace
: str
687 """Return data for a specific service inside a namespace"""
689 credentials
= self
.get_credentials(cluster_uuid
=cluster_uuid
)
691 kubecfg
= tempfile
.NamedTemporaryFile()
692 with
open(kubecfg
.name
, "w") as kubecfg_file
:
693 kubecfg_file
.write(credentials
)
694 kubectl
= Kubectl(config_file
=kubecfg
.name
)
696 return kubectl
.get_services(
697 field_selector
="metadata.name={},metadata.namespace={}".format(
698 service_name
, namespace
702 def get_credentials(self
, cluster_uuid
: str) -> str:
704 Get Cluster Kubeconfig
706 k8scluster
= self
.db
.get_one(
707 "k8sclusters", q_filter
={"_id": cluster_uuid
}, fail_on_empty
=False
710 self
.db
.encrypt_decrypt_fields(
711 k8scluster
.get("credentials"),
713 ["password", "secret"],
714 schema_version
=k8scluster
["schema_version"],
715 salt
=k8scluster
["_id"],
718 return yaml
.safe_dump(k8scluster
.get("credentials"))
720 def _get_credential_name(self
, cluster_uuid
: str) -> str:
722 Get credential name for a k8s cloud
724 We cannot use the cluster_uuid for the credential name directly,
725 because it cannot start with a number, it must start with a letter.
726 Therefore, the k8s cloud credential name will be "cred-" followed
729 :param: cluster_uuid: Cluster UUID of the kubernetes cloud (=cloud_name)
731 :return: Name to use for the credential name.
733 return "cred-{}".format(cluster_uuid
)
739 """Get the namespace UUID
740 Gets the namespace's unique name
742 :param cluster_uuid str: The UUID of the cluster
743 :returns: The namespace UUID, or raises an exception
747 def _create_cluster_role(
751 labels
: Dict
[str, str],
753 cluster_roles
= kubectl
.clients
[RBAC_CLIENT
].list_cluster_role(
754 field_selector
="metadata.name={}".format(name
)
757 if len(cluster_roles
.items
) > 0:
759 "Cluster role with metadata.name={} already exists".format(name
)
762 metadata
= V1ObjectMeta(name
=name
, labels
=labels
, namespace
=ADMIN_NAMESPACE
)
764 cluster_role
= V1ClusterRole(
767 V1PolicyRule(api_groups
=["*"], resources
=["*"], verbs
=["*"]),
768 V1PolicyRule(non_resource_ur_ls
=["*"], verbs
=["*"]),
772 kubectl
.clients
[RBAC_CLIENT
].create_cluster_role(cluster_role
)
774 def _delete_cluster_role(self
, kubectl
: Kubectl
, name
: str):
775 kubectl
.clients
[RBAC_CLIENT
].delete_cluster_role(name
)
777 def _create_service_account(
781 labels
: Dict
[str, str],
783 service_accounts
= kubectl
.clients
[CORE_CLIENT
].list_namespaced_service_account(
784 ADMIN_NAMESPACE
, field_selector
="metadata.name={}".format(name
)
786 if len(service_accounts
.items
) > 0:
788 "Service account with metadata.name={} already exists".format(name
)
791 metadata
= V1ObjectMeta(name
=name
, labels
=labels
, namespace
=ADMIN_NAMESPACE
)
792 service_account
= V1ServiceAccount(metadata
=metadata
)
794 kubectl
.clients
[CORE_CLIENT
].create_namespaced_service_account(
795 ADMIN_NAMESPACE
, service_account
798 def _delete_service_account(self
, kubectl
: Kubectl
, name
: str):
799 kubectl
.clients
[CORE_CLIENT
].delete_namespaced_service_account(
800 name
, ADMIN_NAMESPACE
803 def _create_cluster_role_binding(
807 labels
: Dict
[str, str],
809 role_bindings
= kubectl
.clients
[RBAC_CLIENT
].list_cluster_role_binding(
810 field_selector
="metadata.name={}".format(name
)
812 if len(role_bindings
.items
) > 0:
813 raise Exception("Generated rbac id already exists")
815 role_binding
= V1ClusterRoleBinding(
816 metadata
=V1ObjectMeta(name
=name
, labels
=labels
),
817 role_ref
=V1RoleRef(kind
="ClusterRole", name
=name
, api_group
=""),
819 V1Subject(kind
="ServiceAccount", name
=name
, namespace
=ADMIN_NAMESPACE
)
822 kubectl
.clients
[RBAC_CLIENT
].create_cluster_role_binding(role_binding
)
824 def _delete_cluster_role_binding(self
, kubectl
: Kubectl
, name
: str):
825 kubectl
.clients
[RBAC_CLIENT
].delete_cluster_role_binding(name
)
827 async def _get_secret_data(self
, kubectl
: Kubectl
, name
: str) -> (str, str):
828 v1_core
= kubectl
.clients
[CORE_CLIENT
]
834 service_accounts
= v1_core
.list_namespaced_service_account(
835 ADMIN_NAMESPACE
, field_selector
="metadata.name={}".format(name
)
837 if len(service_accounts
.items
) == 0:
839 "Service account not found with metadata.name={}".format(name
)
841 service_account
= service_accounts
.items
[0]
842 if service_account
.secrets
and len(service_account
.secrets
) > 0:
843 secret_name
= service_account
.secrets
[0].name
844 if secret_name
is not None or not retries_limit
:
848 "Failed getting the secret from service account {}".format(name
)
850 secret
= v1_core
.list_namespaced_secret(
852 field_selector
="metadata.name={}".format(secret_name
),
855 token
= secret
.data
[SERVICE_ACCOUNT_TOKEN_KEY
]
856 client_certificate_data
= secret
.data
[SERVICE_ACCOUNT_ROOT_CA_KEY
]
859 base64
.b64decode(token
).decode("utf-8"),
860 base64
.b64decode(client_certificate_data
).decode("utf-8"),
864 def generate_kdu_instance_name(**kwargs
):
865 db_dict
= kwargs
.get("db_dict")
866 kdu_name
= kwargs
.get("kdu_name", None)
868 kdu_instance
= "{}-{}".format(kdu_name
, db_dict
["filter"]["_id"])
870 kdu_instance
= db_dict
["filter"]["_id"]
873 async def _get_libjuju(self
, vca_id
: str = None) -> Libjuju
:
877 :param: vca_id: VCA ID
878 If None, get a libjuju object with a Connection to the default VCA
879 Else, geta libjuju object with a Connection to the specified VCA
882 while self
.loading_libjuju
.locked():
883 await asyncio
.sleep(0.1)
885 async with self
.loading_libjuju
:
886 vca_connection
= await get_connection(self
._store
)
887 self
.libjuju
= Libjuju(vca_connection
, loop
=self
.loop
, log
=self
.log
)
890 vca_connection
= await get_connection(self
._store
, vca_id
)