blob: 3130216001ff83feb03b94dfa813236087a4b4de [file] [log] [blame]
# Copyright 2019 Canonical Ltd.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import asyncio
import os
import uuid
import yaml
import tempfile
import binascii
import base64
from n2vc.config import ModelConfig
from n2vc.exceptions import K8sException, N2VCBadArgumentsException
from n2vc.k8s_conn import K8sConnector
from n2vc.kubectl import Kubectl, CORE_CLIENT, RBAC_CLIENT
from .exceptions import MethodNotImplemented
from n2vc.utils import base64_to_cacert
from n2vc.libjuju import Libjuju
from n2vc.utils import obj_to_dict, obj_to_yaml
from kubernetes.client.models import (
V1ClusterRole,
V1ObjectMeta,
V1PolicyRule,
V1ServiceAccount,
V1ClusterRoleBinding,
V1RoleRef,
V1Subject,
)
from typing import Dict
SERVICE_ACCOUNT_TOKEN_KEY = "token"
SERVICE_ACCOUNT_ROOT_CA_KEY = "ca.crt"
RBAC_LABEL_KEY_NAME = "rbac-id"
ADMIN_NAMESPACE = "kube-system"
RBAC_STACK_PREFIX = "juju-credential"
def generate_rbac_id():
return binascii.hexlify(os.urandom(4)).decode()
class K8sJujuConnector(K8sConnector):
def __init__(
self,
fs: object,
db: object,
kubectl_command: str = "/usr/bin/kubectl",
juju_command: str = "/usr/bin/juju",
log: object = None,
loop: object = None,
on_update_db=None,
vca_config: dict = None,
):
"""
:param fs: file system for kubernetes and helm configuration
:param db: Database object
:param kubectl_command: path to kubectl executable
:param helm_command: path to helm executable
:param log: logger
:param: loop: Asyncio loop
"""
# parent class
K8sConnector.__init__(
self,
db,
log=log,
on_update_db=on_update_db,
)
self.fs = fs
self.loop = loop or asyncio.get_event_loop()
self.log.debug("Initializing K8S Juju connector")
required_vca_config = [
"host",
"user",
"secret",
"ca_cert",
]
if not vca_config or not all(k in vca_config for k in required_vca_config):
raise N2VCBadArgumentsException(
message="Missing arguments in vca_config: {}".format(vca_config),
bad_args=required_vca_config,
)
port = vca_config["port"] if "port" in vca_config else 17070
url = "{}:{}".format(vca_config["host"], port)
model_config = ModelConfig(vca_config)
username = vca_config["user"]
secret = vca_config["secret"]
ca_cert = base64_to_cacert(vca_config["ca_cert"])
self.libjuju = Libjuju(
endpoint=url,
api_proxy=None, # Not needed for k8s charms
model_config=model_config,
username=username,
password=secret,
cacert=ca_cert,
loop=self.loop,
log=self.log,
db=self.db,
)
self.log.debug("K8S Juju connector initialized")
# TODO: Remove these commented lines:
# self.authenticated = False
# self.models = {}
# self.juju_secret = ""
"""Initialization"""
async def init_env(
self,
k8s_creds: str,
namespace: str = "kube-system",
reuse_cluster_uuid: str = None,
) -> (str, bool):
"""
It prepares a given K8s cluster environment to run Juju bundles.
:param k8s_creds: credentials to access a given K8s cluster, i.e. a valid
'.kube/config'
:param namespace: optional namespace to be used for juju. By default,
'kube-system' will be used
:param reuse_cluster_uuid: existing cluster uuid for reuse
:return: uuid of the K8s cluster and True if connector has installed some
software in the cluster
(on error, an exception will be raised)
"""
cluster_uuid = reuse_cluster_uuid or str(uuid.uuid4())
kubecfg = tempfile.NamedTemporaryFile()
with open(kubecfg.name, "w") as kubecfg_file:
kubecfg_file.write(k8s_creds)
kubectl = Kubectl(config_file=kubecfg.name)
# CREATING RESOURCES IN K8S
rbac_id = generate_rbac_id()
metadata_name = "{}-{}".format(RBAC_STACK_PREFIX, rbac_id)
labels = {RBAC_STACK_PREFIX: rbac_id}
# Create cleanup dictionary to clean up created resources
# if it fails in the middle of the process
cleanup_data = []
try:
self._create_cluster_role(
kubectl,
name=metadata_name,
labels=labels,
)
cleanup_data.append(
{
"delete": self._delete_cluster_role,
"args": (kubectl, metadata_name),
}
)
self._create_service_account(
kubectl,
name=metadata_name,
labels=labels,
)
cleanup_data.append(
{
"delete": self._delete_service_account,
"args": (kubectl, metadata_name),
}
)
self._create_cluster_role_binding(
kubectl,
name=metadata_name,
labels=labels,
)
cleanup_data.append(
{
"delete": self._delete_service_account,
"args": (kubectl, metadata_name),
}
)
token, client_cert_data = await self._get_secret_data(
kubectl,
metadata_name,
)
default_storage_class = kubectl.get_default_storage_class()
await self.libjuju.add_k8s(
name=cluster_uuid,
rbac_id=rbac_id,
token=token,
client_cert_data=client_cert_data,
configuration=kubectl.configuration,
storage_class=default_storage_class,
credential_name=self._get_credential_name(cluster_uuid),
)
return cluster_uuid, True
except Exception as e:
self.log.error("Error initializing k8scluster: {}".format(e))
if len(cleanup_data) > 0:
self.log.debug("Cleaning up created resources in k8s cluster...")
for item in cleanup_data:
delete_function = item["delete"]
delete_args = item["args"]
delete_function(*delete_args)
self.log.debug("Cleanup finished")
raise e
"""Repo Management"""
async def repo_add(
self,
name: str,
url: str,
_type: str = "charm",
):
raise MethodNotImplemented()
async def repo_list(self):
raise MethodNotImplemented()
async def repo_remove(
self,
name: str,
):
raise MethodNotImplemented()
async def synchronize_repos(self, cluster_uuid: str, name: str):
"""
Returns None as currently add_repo is not implemented
"""
return None
"""Reset"""
async def reset(
self, cluster_uuid: str, force: bool = False, uninstall_sw: bool = False
) -> bool:
"""Reset a cluster
Resets the Kubernetes cluster by removing the model that represents it.
:param cluster_uuid str: The UUID of the cluster to reset
:return: Returns True if successful or raises an exception.
"""
try:
self.log.debug("[reset] Removing k8s cloud")
cloud_creds = await self.libjuju.get_cloud_credentials(
cluster_uuid,
self._get_credential_name(cluster_uuid),
)
await self.libjuju.remove_cloud(cluster_uuid)
kubecfg = self.get_credentials(cluster_uuid=cluster_uuid)
kubecfg_file = tempfile.NamedTemporaryFile()
with open(kubecfg_file.name, "w") as f:
f.write(kubecfg)
kubectl = Kubectl(config_file=kubecfg_file.name)
delete_functions = [
self._delete_cluster_role_binding,
self._delete_service_account,
self._delete_cluster_role,
]
credential_attrs = cloud_creds[0].result["attrs"]
if RBAC_LABEL_KEY_NAME in credential_attrs:
rbac_id = credential_attrs[RBAC_LABEL_KEY_NAME]
metadata_name = "{}-{}".format(RBAC_STACK_PREFIX, rbac_id)
delete_args = (kubectl, metadata_name)
for delete_func in delete_functions:
try:
delete_func(*delete_args)
except Exception as e:
self.log.warning("Cannot remove resource in K8s {}".format(e))
except Exception as e:
self.log.debug("Caught exception during reset: {}".format(e))
raise e
return True
"""Deployment"""
async def install(
self,
cluster_uuid: str,
kdu_model: str,
kdu_instance: str,
atomic: bool = True,
timeout: float = 1800,
params: dict = None,
db_dict: dict = None,
kdu_name: str = None,
namespace: str = None,
) -> bool:
"""Install a bundle
:param cluster_uuid str: The UUID of the cluster to install to
:param kdu_model str: The name or path of a bundle to install
:param kdu_instance: Kdu instance name
:param atomic bool: If set, waits until the model is active and resets
the cluster on failure.
:param timeout int: The time, in seconds, to wait for the install
to finish
:param params dict: Key-value pairs of instantiation parameters
:param kdu_name: Name of the KDU instance to be installed
:param namespace: K8s namespace to use for the KDU instance
:return: If successful, returns ?
"""
bundle = kdu_model
if not db_dict:
raise K8sException("db_dict must be set")
if not bundle:
raise K8sException("bundle must be set")
if bundle.startswith("cs:"):
pass
elif bundle.startswith("http"):
# Download the file
pass
else:
new_workdir = kdu_model.strip(kdu_model.split("/")[-1])
os.chdir(new_workdir)
bundle = "local:{}".format(kdu_model)
self.log.debug("Checking for model named {}".format(kdu_instance))
# Create the new model
self.log.debug("Adding model: {}".format(kdu_instance))
await self.libjuju.add_model(
model_name=kdu_instance,
cloud_name=cluster_uuid,
credential_name=self._get_credential_name(cluster_uuid),
)
# if model:
# TODO: Instantiation parameters
"""
"Juju bundle that models the KDU, in any of the following ways:
- <juju-repo>/<juju-bundle>
- <juju-bundle folder under k8s_models folder in the package>
- <juju-bundle tgz file (w/ or w/o extension) under k8s_models folder
in the package>
- <URL_where_to_fetch_juju_bundle>
"""
try:
previous_workdir = os.getcwd()
except FileNotFoundError:
previous_workdir = "/app/storage"
self.log.debug("[install] deploying {}".format(bundle))
await self.libjuju.deploy(
bundle, model_name=kdu_instance, wait=atomic, timeout=timeout
)
os.chdir(previous_workdir)
if self.on_update_db:
await self.on_update_db(cluster_uuid, kdu_instance, filter=db_dict["filter"])
return True
async def instances_list(self, cluster_uuid: str) -> list:
"""
returns a list of deployed releases in a cluster
:param cluster_uuid: the cluster
:return:
"""
return []
async def upgrade(
self,
cluster_uuid: str,
kdu_instance: str,
kdu_model: str = None,
params: dict = None,
) -> str:
"""Upgrade a model
:param cluster_uuid str: The UUID of the cluster to upgrade
:param kdu_instance str: The unique name of the KDU instance
:param kdu_model str: The name or path of the bundle to upgrade to
:param params dict: Key-value pairs of instantiation parameters
:return: If successful, reference to the new revision number of the
KDU instance.
"""
# TODO: Loop through the bundle and upgrade each charm individually
"""
The API doesn't have a concept of bundle upgrades, because there are
many possible changes: charm revision, disk, number of units, etc.
As such, we are only supporting a limited subset of upgrades. We'll
upgrade the charm revision but leave storage and scale untouched.
Scale changes should happen through OSM constructs, and changes to
storage would require a redeployment of the service, at least in this
initial release.
"""
raise MethodNotImplemented()
"""Rollback"""
async def rollback(
self,
cluster_uuid: str,
kdu_instance: str,
revision: int = 0,
) -> str:
"""Rollback a model
:param cluster_uuid str: The UUID of the cluster to rollback
:param kdu_instance str: The unique name of the KDU instance
:param revision int: The revision to revert to. If omitted, rolls back
the previous upgrade.
:return: If successful, returns the revision of active KDU instance,
or raises an exception
"""
raise MethodNotImplemented()
"""Deletion"""
async def uninstall(self, cluster_uuid: str, kdu_instance: str) -> bool:
"""Uninstall a KDU instance
:param cluster_uuid str: The UUID of the cluster
:param kdu_instance str: The unique name of the KDU instance
:return: Returns True if successful, or raises an exception
"""
self.log.debug("[uninstall] Destroying model")
await self.libjuju.destroy_model(kdu_instance, total_timeout=3600)
# self.log.debug("[uninstall] Model destroyed and disconnecting")
# await controller.disconnect()
return True
# TODO: Remove these commented lines
# if not self.authenticated:
# self.log.debug("[uninstall] Connecting to controller")
# await self.login(cluster_uuid)
async def exec_primitive(
self,
cluster_uuid: str = None,
kdu_instance: str = None,
primitive_name: str = None,
timeout: float = 300,
params: dict = None,
db_dict: dict = None,
) -> str:
"""Exec primitive (Juju action)
:param cluster_uuid str: The UUID of the cluster
:param kdu_instance str: The unique name of the KDU instance
:param primitive_name: Name of action that will be executed
:param timeout: Timeout for action execution
:param params: Dictionary of all the parameters needed for the action
:db_dict: Dictionary for any additional data
:return: Returns the output of the action
"""
if not params or "application-name" not in params:
raise K8sException(
"Missing application-name argument, \
argument needed for K8s actions"
)
try:
self.log.debug(
"[exec_primitive] Getting model "
"kdu_instance: {}".format(kdu_instance)
)
application_name = params["application-name"]
actions = await self.libjuju.get_actions(application_name, kdu_instance)
if primitive_name not in actions:
raise K8sException("Primitive {} not found".format(primitive_name))
output, status = await self.libjuju.execute_action(
application_name, kdu_instance, primitive_name, **params
)
if status != "completed":
raise K8sException(
"status is not completed: {} output: {}".format(status, output)
)
if self.on_update_db:
await self.on_update_db(cluster_uuid, kdu_instance, filter=db_dict["filter"])
return output
except Exception as e:
error_msg = "Error executing primitive {}: {}".format(primitive_name, e)
self.log.error(error_msg)
raise K8sException(message=error_msg)
"""Introspection"""
async def inspect_kdu(
self,
kdu_model: str,
) -> dict:
"""Inspect a KDU
Inspects a bundle and returns a dictionary of config parameters and
their default values.
:param kdu_model str: The name or path of the bundle to inspect.
:return: If successful, returns a dictionary of available parameters
and their default values.
"""
kdu = {}
if not os.path.exists(kdu_model):
raise K8sException("file {} not found".format(kdu_model))
with open(kdu_model, "r") as f:
bundle = yaml.safe_load(f.read())
"""
{
'description': 'Test bundle',
'bundle': 'kubernetes',
'applications': {
'mariadb-k8s': {
'charm': 'cs:~charmed-osm/mariadb-k8s-20',
'scale': 1,
'options': {
'password': 'manopw',
'root_password': 'osm4u',
'user': 'mano'
},
'series': 'kubernetes'
}
}
}
"""
# TODO: This should be returned in an agreed-upon format
kdu = bundle["applications"]
return kdu
async def help_kdu(
self,
kdu_model: str,
) -> str:
"""View the README
If available, returns the README of the bundle.
:param kdu_model str: The name or path of a bundle
:return: If found, returns the contents of the README.
"""
readme = None
files = ["README", "README.txt", "README.md"]
path = os.path.dirname(kdu_model)
for file in os.listdir(path):
if file in files:
with open(file, "r") as f:
readme = f.read()
break
return readme
async def status_kdu(
self,
cluster_uuid: str,
kdu_instance: str,
complete_status: bool = False,
yaml_format: bool = False
) -> dict:
"""Get the status of the KDU
Get the current status of the KDU instance.
:param cluster_uuid str: The UUID of the cluster
:param kdu_instance str: The unique id of the KDU instance
:param complete_status: To get the complete_status of the KDU
:param yaml_format: To get the status in proper format for NSR record
:return: Returns a dictionary containing namespace, state, resources,
and deployment_time and returns complete_status if complete_status is True
"""
status = {}
model_status = await self.libjuju.get_model_status(kdu_instance)
if not complete_status:
for name in model_status.applications:
application = model_status.applications[name]
status[name] = {"status": application["status"]["status"]}
else:
if yaml_format:
return obj_to_yaml(model_status)
else:
return obj_to_dict(model_status)
return status
async def update_vca_status(self, vcastatus: dict, kdu_instance: str):
"""
Add all configs, actions, executed actions of all applications in a model to vcastatus dict
:param vcastatus dict: dict containing vcastatus
:param kdu_instance str: The unique id of the KDU instance
:return: None
"""
try:
for model_name in vcastatus:
# Adding executed actions
vcastatus[model_name]["executedActions"] = \
await self.libjuju.get_executed_actions(kdu_instance)
for application in vcastatus[model_name]["applications"]:
# Adding application actions
vcastatus[model_name]["applications"][application]["actions"] = \
await self.libjuju.get_actions(application, kdu_instance)
# Adding application configs
vcastatus[model_name]["applications"][application]["configs"] = \
await self.libjuju.get_application_configs(kdu_instance, application)
except Exception as e:
self.log.debug("Error in updating vca status: {}".format(str(e)))
async def get_services(
self, cluster_uuid: str, kdu_instance: str, namespace: str
) -> list:
"""Return a list of services of a kdu_instance"""
credentials = self.get_credentials(cluster_uuid=cluster_uuid)
kubecfg = tempfile.NamedTemporaryFile()
with open(kubecfg.name, "w") as kubecfg_file:
kubecfg_file.write(credentials)
kubectl = Kubectl(config_file=kubecfg.name)
return kubectl.get_services(
field_selector="metadata.namespace={}".format(kdu_instance)
)
async def get_service(
self, cluster_uuid: str, service_name: str, namespace: str
) -> object:
"""Return data for a specific service inside a namespace"""
credentials = self.get_credentials(cluster_uuid=cluster_uuid)
kubecfg = tempfile.NamedTemporaryFile()
with open(kubecfg.name, "w") as kubecfg_file:
kubecfg_file.write(credentials)
kubectl = Kubectl(config_file=kubecfg.name)
return kubectl.get_services(
field_selector="metadata.name={},metadata.namespace={}".format(
service_name, namespace
)
)[0]
def get_credentials(self, cluster_uuid: str) -> str:
"""
Get Cluster Kubeconfig
"""
k8scluster = self.db.get_one(
"k8sclusters", q_filter={"_id": cluster_uuid}, fail_on_empty=False
)
self.db.encrypt_decrypt_fields(
k8scluster.get("credentials"),
"decrypt",
["password", "secret"],
schema_version=k8scluster["schema_version"],
salt=k8scluster["_id"],
)
return yaml.safe_dump(k8scluster.get("credentials"))
def _get_credential_name(self, cluster_uuid: str) -> str:
"""
Get credential name for a k8s cloud
We cannot use the cluster_uuid for the credential name directly,
because it cannot start with a number, it must start with a letter.
Therefore, the k8s cloud credential name will be "cred-" followed
by the cluster uuid.
:param: cluster_uuid: Cluster UUID of the kubernetes cloud (=cloud_name)
:return: Name to use for the credential name.
"""
return "cred-{}".format(cluster_uuid)
def get_namespace(
self,
cluster_uuid: str,
) -> str:
"""Get the namespace UUID
Gets the namespace's unique name
:param cluster_uuid str: The UUID of the cluster
:returns: The namespace UUID, or raises an exception
"""
pass
def _create_cluster_role(
self,
kubectl: Kubectl,
name: str,
labels: Dict[str, str],
):
cluster_roles = kubectl.clients[RBAC_CLIENT].list_cluster_role(
field_selector="metadata.name={}".format(name)
)
if len(cluster_roles.items) > 0:
raise Exception(
"Cluster role with metadata.name={} already exists".format(name)
)
metadata = V1ObjectMeta(name=name, labels=labels, namespace=ADMIN_NAMESPACE)
# Cluster role
cluster_role = V1ClusterRole(
metadata=metadata,
rules=[
V1PolicyRule(api_groups=["*"], resources=["*"], verbs=["*"]),
V1PolicyRule(non_resource_ur_ls=["*"], verbs=["*"]),
],
)
kubectl.clients[RBAC_CLIENT].create_cluster_role(cluster_role)
def _delete_cluster_role(self, kubectl: Kubectl, name: str):
kubectl.clients[RBAC_CLIENT].delete_cluster_role(name)
def _create_service_account(
self,
kubectl: Kubectl,
name: str,
labels: Dict[str, str],
):
service_accounts = kubectl.clients[CORE_CLIENT].list_namespaced_service_account(
ADMIN_NAMESPACE, field_selector="metadata.name={}".format(name)
)
if len(service_accounts.items) > 0:
raise Exception(
"Service account with metadata.name={} already exists".format(name)
)
metadata = V1ObjectMeta(name=name, labels=labels, namespace=ADMIN_NAMESPACE)
service_account = V1ServiceAccount(metadata=metadata)
kubectl.clients[CORE_CLIENT].create_namespaced_service_account(
ADMIN_NAMESPACE, service_account
)
def _delete_service_account(self, kubectl: Kubectl, name: str):
kubectl.clients[CORE_CLIENT].delete_namespaced_service_account(
name, ADMIN_NAMESPACE
)
def _create_cluster_role_binding(
self,
kubectl: Kubectl,
name: str,
labels: Dict[str, str],
):
role_bindings = kubectl.clients[RBAC_CLIENT].list_cluster_role_binding(
field_selector="metadata.name={}".format(name)
)
if len(role_bindings.items) > 0:
raise Exception("Generated rbac id already exists")
role_binding = V1ClusterRoleBinding(
metadata=V1ObjectMeta(name=name, labels=labels),
role_ref=V1RoleRef(kind="ClusterRole", name=name, api_group=""),
subjects=[
V1Subject(kind="ServiceAccount", name=name, namespace=ADMIN_NAMESPACE)
],
)
kubectl.clients[RBAC_CLIENT].create_cluster_role_binding(role_binding)
def _delete_cluster_role_binding(self, kubectl: Kubectl, name: str):
kubectl.clients[RBAC_CLIENT].delete_cluster_role_binding(name)
async def _get_secret_data(self, kubectl: Kubectl, name: str) -> (str, str):
v1_core = kubectl.clients[CORE_CLIENT]
retries_limit = 10
secret_name = None
while True:
retries_limit -= 1
service_accounts = v1_core.list_namespaced_service_account(
ADMIN_NAMESPACE, field_selector="metadata.name={}".format(name)
)
if len(service_accounts.items) == 0:
raise Exception(
"Service account not found with metadata.name={}".format(name)
)
service_account = service_accounts.items[0]
if service_account.secrets and len(service_account.secrets) > 0:
secret_name = service_account.secrets[0].name
if secret_name is not None or not retries_limit:
break
if not secret_name:
raise Exception(
"Failed getting the secret from service account {}".format(name)
)
secret = v1_core.list_namespaced_secret(
ADMIN_NAMESPACE,
field_selector="metadata.name={}".format(secret_name),
).items[0]
token = secret.data[SERVICE_ACCOUNT_TOKEN_KEY]
client_certificate_data = secret.data[SERVICE_ACCOUNT_ROOT_CA_KEY]
return (
base64.b64decode(token).decode("utf-8"),
base64.b64decode(client_certificate_data).decode("utf-8"),
)
@staticmethod
def generate_kdu_instance_name(**kwargs):
db_dict = kwargs.get("db_dict")
kdu_name = kwargs.get("kdu_name", None)
if kdu_name:
kdu_instance = "{}-{}".format(kdu_name, db_dict["filter"]["_id"])
else:
kdu_instance = db_dict["filter"]["_id"]
return kdu_instance