From 5e541439938d7d593d398dadfd23c3a23c53a971 Mon Sep 17 00:00:00 2001 From: garciadeblas Date: Wed, 11 Jun 2025 11:13:45 +0200 Subject: [PATCH] Feature 11049: Cluster management with CAPI in Openstack-based clouds Change-Id: Ie5af0867a97834594b63ddd6151f6423d188ed2c Signed-off-by: garciadeblas --- .../capi/cni/calico/templates/calico.yaml | 62 ++++++ .../manifests/base/openstack.yaml | 179 ++++++++++++++++++ .../templates/capi-cluster-postinstall.yaml | 64 +++++++ .../templates/capi-cluster-secret.yaml | 74 ++++++++ .../templates/capi-cluster.yaml | 87 +++++++++ .../capi/openstack-kubeadm/templates/cni.yaml | 64 +++++++ .../capi/templates/capi-controllers.yaml | 31 +++ .../add-operators-and-crds.sh | 4 + 8 files changed, 565 insertions(+) create mode 100644 installers/flux/templates/sw-catalogs/cloud-resources/capi/cni/calico/templates/calico.yaml create mode 100644 installers/flux/templates/sw-catalogs/cloud-resources/capi/openstack-kubeadm/manifests/base/openstack.yaml create mode 100644 installers/flux/templates/sw-catalogs/cloud-resources/capi/openstack-kubeadm/templates/capi-cluster-postinstall.yaml create mode 100644 installers/flux/templates/sw-catalogs/cloud-resources/capi/openstack-kubeadm/templates/capi-cluster-secret.yaml create mode 100644 installers/flux/templates/sw-catalogs/cloud-resources/capi/openstack-kubeadm/templates/capi-cluster.yaml create mode 100644 installers/flux/templates/sw-catalogs/cloud-resources/capi/openstack-kubeadm/templates/cni.yaml create mode 100644 installers/flux/templates/sw-catalogs/infra-controllers/capi/templates/capi-controllers.yaml diff --git a/installers/flux/templates/sw-catalogs/cloud-resources/capi/cni/calico/templates/calico.yaml b/installers/flux/templates/sw-catalogs/cloud-resources/capi/cni/calico/templates/calico.yaml new file mode 100644 index 00000000..ce84af6f --- /dev/null +++ b/installers/flux/templates/sw-catalogs/cloud-resources/capi/cni/calico/templates/calico.yaml @@ -0,0 +1,62 @@ +####################################################################################### +# Copyright ETSI Contributors and Others. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. +####################################################################################### + +--- +# TEMPLATE_PARAMETERS: +# =================== +# +# CLUSTER_KUSTOMIZATION_NAME: Name of the cluster in the management cluster (e.g., for `Kustomization`s). +# - Alternatively, it can be patched at: +# .metadata.name +# .metadata.labels.cluster +# .spec.commonMetadata.labels.cluster +# .spec.postBuild.substitute.cluster_resource_name +# +# PARAMETERS TO PATCH: +# =================== +# +# .spec.postBuild.substitute.cluster_name: Name of the cluster in the target cloud. It may differ from `CLUSTER_KUSTOMIZATION_NAME` since naming restrictions are often different from K8s resource naming restrictions (e.g., hyphens vs. underscores). + +# Cluster resource +apiVersion: kustomize.toolkit.fluxcd.io/v1 +kind: Kustomization +metadata: + name: ${CLUSTER_KUSTOMIZATION_NAME}-cni + namespace: managed-resources + labels: + cluster: ${CLUSTER_KUSTOMIZATION_NAME} +spec: + commonMetadata: + labels: + cluster: ${CLUSTER_KUSTOMIZATION_NAME} + interval: 1h + retryInterval: 1m + timeout: 5m + dependsOn: + - name: ${CLUSTER_KUSTOMIZATION_NAME} + sourceRef: + kind: GitRepository + name: sw-catalogs + namespace: flux-system + path: ./cloud-resources/capi/cni/calico/manifests + prune: true + # force: true + wait: true + # Input parameters + postBuild: + substitute: + cluster_name: "myclustername" diff --git a/installers/flux/templates/sw-catalogs/cloud-resources/capi/openstack-kubeadm/manifests/base/openstack.yaml b/installers/flux/templates/sw-catalogs/cloud-resources/capi/openstack-kubeadm/manifests/base/openstack.yaml new file mode 100644 index 00000000..b505ec68 --- /dev/null +++ b/installers/flux/templates/sw-catalogs/cloud-resources/capi/openstack-kubeadm/manifests/base/openstack.yaml @@ -0,0 +1,179 @@ +####################################################################################### +# Copyright ETSI Contributors and Others. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. +####################################################################################### + +--- +apiVersion: bootstrap.cluster.x-k8s.io/v1beta1 +kind: KubeadmConfigTemplate +metadata: + name: ${cluster_name}-md-0 + namespace: ${namespace} +spec: + template: + spec: + files: [] + joinConfiguration: + nodeRegistration: + kubeletExtraArgs: + cloud-provider: external + provider-id: openstack:///'{{ instance_id }}' + name: '{{ local_hostname }}' +--- +apiVersion: cluster.x-k8s.io/v1beta1 +kind: Cluster +metadata: + name: ${cluster_name} + namespace: ${namespace} +spec: + clusterNetwork: + pods: + cidrBlocks: + - 192.168.0.0/16 + serviceDomain: cluster.local + controlPlaneRef: + apiVersion: controlplane.cluster.x-k8s.io/v1beta1 + kind: KubeadmControlPlane + name: ${cluster_name}-control-plane + infrastructureRef: + apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 + kind: OpenStackCluster + name: ${cluster_name} +--- +apiVersion: cluster.x-k8s.io/v1beta1 +kind: MachineDeployment +metadata: + name: ${cluster_name}-md-0 + namespace: ${namespace} +spec: + clusterName: ${cluster_name} + replicas: ${worker_machine_count} + selector: + matchLabels: null + template: + spec: + bootstrap: + configRef: + apiVersion: bootstrap.cluster.x-k8s.io/v1beta1 + kind: KubeadmConfigTemplate + name: ${cluster_name}-md-0 + clusterName: ${cluster_name} + failureDomain: ${openstack_failure_domain} + infrastructureRef: + apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 + kind: OpenStackMachineTemplate + name: ${cluster_name}-md-0 + version: ${kubernetes_version} +--- +apiVersion: controlplane.cluster.x-k8s.io/v1beta1 +kind: KubeadmControlPlane +metadata: + name: ${cluster_name}-control-plane + namespace: ${namespace} +spec: + kubeadmConfigSpec: + clusterConfiguration: + apiServer: + extraArgs: + cloud-provider: external + controllerManager: + extraArgs: + cloud-provider: external + files: [] + initConfiguration: + nodeRegistration: + kubeletExtraArgs: + cloud-provider: external + provider-id: openstack:///'{{ instance_id }}' + name: '{{ local_hostname }}' + joinConfiguration: + nodeRegistration: + kubeletExtraArgs: + cloud-provider: external + provider-id: openstack:///'{{ instance_id }}' + name: '{{ local_hostname }}' + machineTemplate: + infrastructureRef: + apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 + kind: OpenStackMachineTemplate + name: ${cluster_name}-control-plane + replicas: "${control_plane_machine_count}" + version: ${kubernetes_version} +--- +apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 +kind: OpenStackCluster +metadata: + name: ${cluster_name} + namespace: ${namespace} +spec: + apiServerLoadBalancer: + enabled: false + externalNetwork: + id: ${openstack_external_network_id} + identityRef: + cloudName: ${openstack_cloud} + name: ${openstack_cloud}-capo-config + managedSecurityGroups: + allNodesSecurityGroupRules: + - description: Created by cluster-api-provider-openstack - BGP (calico) + direction: ingress + etherType: IPv4 + name: BGP (Calico) + portRangeMax: 179 + portRangeMin: 179 + protocol: tcp + remoteManagedGroups: + - controlplane + - worker + - description: Created by cluster-api-provider-openstack - IP-in-IP (calico) + direction: ingress + etherType: IPv4 + name: IP-in-IP (calico) + protocol: "4" + remoteManagedGroups: + - controlplane + - worker + managedSubnets: + - cidr: 10.6.0.0/24 + dnsNameservers: + - ${openstack_dns_nameservers} +--- +apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 +kind: OpenStackMachineTemplate +metadata: + name: ${cluster_name}-control-plane + namespace: ${namespace} +spec: + template: + spec: + flavor: ${openstack_control_plane_machine_flavor} + image: + filter: + name: ${openstack_control_plane_image_name} + sshKeyName: ${openstack_ssh_key_name} +--- +apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 +kind: OpenStackMachineTemplate +metadata: + name: ${cluster_name}-md-0 + namespace: ${namespace} +spec: + template: + spec: + flavor: ${openstack_node_machine_flavor} + image: + filter: + name: ${openstack_worker_image_name} + sshKeyName: ${openstack_ssh_key_name} diff --git a/installers/flux/templates/sw-catalogs/cloud-resources/capi/openstack-kubeadm/templates/capi-cluster-postinstall.yaml b/installers/flux/templates/sw-catalogs/cloud-resources/capi/openstack-kubeadm/templates/capi-cluster-postinstall.yaml new file mode 100644 index 00000000..6ae6dd07 --- /dev/null +++ b/installers/flux/templates/sw-catalogs/cloud-resources/capi/openstack-kubeadm/templates/capi-cluster-postinstall.yaml @@ -0,0 +1,64 @@ +####################################################################################### +# Copyright ETSI Contributors and Others. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. +####################################################################################### + +--- +# TEMPLATE_PARAMETERS: +# =================== +# +# CLUSTER_KUSTOMIZATION_NAME: Name of the cluster in the management cluster (e.g., for `Kustomization`s). +# - Alternatively, it can be patched at: +# .metadata.name +# .metadata.labels.cluster +# .spec.commonMetadata.labels.cluster +# .spec.postBuild.substitute.cluster_resource_name +# +# PARAMETERS TO PATCH: +# =================== +# +# .spec.postBuild.substitute.cluster_name: Name of the cluster in the target cloud. It may differ from `CLUSTER_KUSTOMIZATION_NAME` since naming restrictions are often different from K8s resource naming restrictions (e.g., hyphens vs. underscores). + +# Cluster resource +apiVersion: kustomize.toolkit.fluxcd.io/v1 +kind: Kustomization +metadata: + name: ${CLUSTER_KUSTOMIZATION_NAME}-postinstall + namespace: managed-resources + labels: + cluster: ${CLUSTER_KUSTOMIZATION_NAME} +spec: + commonMetadata: + labels: + cluster: ${CLUSTER_KUSTOMIZATION_NAME} + interval: 1h + retryInterval: 1m + timeout: 5m + sourceRef: + kind: GitRepository + name: sw-catalogs + namespace: flux-system + path: ./cloud-resources/capi/openstack-kubeadm/manifests/post-install + kubeConfig: + secretRef: + name: ${CLUSTER_KUBECONFIG_SECRET_NAME} + key: ${CLUSTER_KUBECONFIG_SECRET_KEY} + prune: true + # force: true + wait: true + # Input parameters + postBuild: + substitute: + cluster_name: "myclustername" diff --git a/installers/flux/templates/sw-catalogs/cloud-resources/capi/openstack-kubeadm/templates/capi-cluster-secret.yaml b/installers/flux/templates/sw-catalogs/cloud-resources/capi/openstack-kubeadm/templates/capi-cluster-secret.yaml new file mode 100644 index 00000000..4dad7134 --- /dev/null +++ b/installers/flux/templates/sw-catalogs/cloud-resources/capi/openstack-kubeadm/templates/capi-cluster-secret.yaml @@ -0,0 +1,74 @@ +####################################################################################### +# Copyright ETSI Contributors and Others. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. +####################################################################################### + +--- +# TEMPLATE PARAMETERS: +# =================== +# +# CLUSTER_KUSTOMIZATION_NAME: Name of the cluster in the management cluster (i.e., the `Kustomization`). +# CLUSTER_NAME: Name of the cluster in the target cloud. It may differ from `CLUSTER_KUSTOMIZATION_NAME` since naming restrictions are often different from K8s resource naming restrictions (e.g., hyphens vs. underscores). +# CLUSTER_AGE_SECRET_NAME: Name of the secret in the management cluster that keeps the private key for age/sops in the remote cluster. + +# Creates remote `managed-resources.cloud-conf-toml` secret +apiVersion: kustomize.toolkit.fluxcd.io/v1beta2 +kind: Kustomization +metadata: + name: ${CLUSTER_KUSTOMIZATION_NAME}-capo-config-toml + namespace: managed-resources + labels: + cluster: ${CLUSTER_KUSTOMIZATION_NAME} +spec: + # interval: 1h + interval: 5m + retryInterval: 1m + timeout: 5m + prune: true + # wait: true + force: true + sourceRef: + kind: GitRepository + name: sw-catalogs + namespace: flux-system + path: ./cloud-resources/flux-remote-bootstrap/bootstrap/manifests/secret + kubeConfig: + secretRef: + name: ${CLUSTER_KUBECONFIG_SECRET_NAME} + key: ${CLUSTER_KUBECONFIG_SECRET_KEY} + patches: + - patch: |- + apiVersion: v1 + kind: Secret + metadata: + name: ${secret_name} + namespace: ${secret_namespace} + stringData: + cloud.conf: |- + [Global] + auth-url=${os_auth_url} + region=${os_region_name} + username=${os_username} + password=${os_password} + tenant-id=${os_project_id} + domain-id=${os_project_domain_id} + # Inputs: + postBuild: + substitute: + secret_name: cloud-config + secret_namespace: kube-system + substituteFrom: + - kind: Secret + name: ${OPENSTACK_CLOUD_NAME}-capo-config-toml diff --git a/installers/flux/templates/sw-catalogs/cloud-resources/capi/openstack-kubeadm/templates/capi-cluster.yaml b/installers/flux/templates/sw-catalogs/cloud-resources/capi/openstack-kubeadm/templates/capi-cluster.yaml new file mode 100644 index 00000000..5c1074aa --- /dev/null +++ b/installers/flux/templates/sw-catalogs/cloud-resources/capi/openstack-kubeadm/templates/capi-cluster.yaml @@ -0,0 +1,87 @@ +####################################################################################### +# Copyright ETSI Contributors and Others. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. +####################################################################################### + +--- +# TEMPLATE_PARAMETERS: +# =================== +# +# CLUSTER_KUSTOMIZATION_NAME: Name of the Kustomization in the management cluster representing the AKS cluster. +# - Alternatively, it can be patched at: +# .metadata.name +# .metadata.labels.cluster +# .spec.commonMetadata.labels.cluster +# +# PARAMETERS TO PATCH: +# =================== +# +# .spec.postBuild.substitute.cluster_name: Name of the cluster in the target cloud. +# .spec.postBuild.substitute.control_plane_machine_count: Number of control plane nodes. +# .spec.postBuild.substitute.kubernetes_version: Kubernetes version. +# .spec.postBuild.substitute.namespace: Namespace to be deployed to. +# .spec.postBuild.substitute.worker_machine_count: Number of worker nodes. +# .spec.postBuild.substitute.openstack_cloud: Target cluster region. +# .spec.postBuild.substitute.openstack_cloud_conf: Name of the cloud conf secret. +# .spec.postBuild.substitute.openstack_control_plane_machine_flavor: VM size of control plane node. +# .spec.postBuild.substitute.openstack_dns_nameservers: Nameserver to use in the workload cluster. +# .spec.postBuild.substitute.openstack_external_network_id: ID of the external network. +# .spec.postBuild.substitute.openstack_failure_domain: OpenStack nova name +# .spec.postBuild.substitute.openstack_control_plane_image_name: Base image of the control plane VM +# .spec.postBuild.substitute.openstack_worker_image_name: Base image of the worker VM +# .spec.postBuild.substitute.openstack_node_machine_flavor: VM size of worker node. +# .spec.postBuild.substitute.openstack_ssh_key_name: Name of the SSH key to use with OpenStack. + +apiVersion: kustomize.toolkit.fluxcd.io/v1 +kind: Kustomization +metadata: + name: ${CLUSTER_KUSTOMIZATION_NAME} + namespace: managed-resources + labels: + cluster: ${CLUSTER_KUSTOMIZATION_NAME} +spec: + commonMetadata: + labels: + cluster: ${CLUSTER_KUSTOMIZATION_NAME} + interval: 1h + retryInterval: 1m + timeout: 5m + sourceRef: + kind: GitRepository + name: sw-catalogs + namespace: flux-system + path: ./cloud-resources/capi/openstack-kubeadm/manifests/base + prune: true + wait: true + # force: true + # Input parameters + postBuild: + substitute: + cluster_name: "myopenstackcluster01" + cni: "cni-plugin" + control_plane_machine_count: "1" + kubernetes_version: "v1.30.1" + namespace: "capi-system" + worker_machine_count: "0" + openstack_cloud: "default" + openstack_cloud_conf: "1" + openstack_control_plane_machine_flavor: "" + openstack_dns_nameservers: "" + openstack_external_network_id: "" + openstack_failure_domain: "" + openstack_worker_image_name: "" + openstack_control_plane_image_name: "" + openstack_node_machine_flavor: "" + openstack_ssh_key_name: "" diff --git a/installers/flux/templates/sw-catalogs/cloud-resources/capi/openstack-kubeadm/templates/cni.yaml b/installers/flux/templates/sw-catalogs/cloud-resources/capi/openstack-kubeadm/templates/cni.yaml new file mode 100644 index 00000000..36e4e71b --- /dev/null +++ b/installers/flux/templates/sw-catalogs/cloud-resources/capi/openstack-kubeadm/templates/cni.yaml @@ -0,0 +1,64 @@ +####################################################################################### +# Copyright ETSI Contributors and Others. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. +####################################################################################### + +--- +# TEMPLATE_PARAMETERS: +# =================== +# +# CLUSTER_KUSTOMIZATION_NAME: Name of the cluster in the management cluster (e.g., for `Kustomization`s). +# - Alternatively, it can be patched at: +# .metadata.name +# .metadata.labels.cluster +# .spec.commonMetadata.labels.cluster +# .spec.postBuild.substitute.cluster_resource_name +# +# PARAMETERS TO PATCH: +# =================== +# +# .spec.postBuild.substitute.cluster_name: Name of the cluster in the target cloud. It may differ from `CLUSTER_KUSTOMIZATION_NAME` since naming restrictions are often different from K8s resource naming restrictions (e.g., hyphens vs. underscores). + +# Cluster resource +apiVersion: kustomize.toolkit.fluxcd.io/v1 +kind: Kustomization +metadata: + name: ${CLUSTER_KUSTOMIZATION_NAME}-cni + namespace: managed-resources + labels: + cluster: ${CLUSTER_KUSTOMIZATION_NAME} +spec: + commonMetadata: + labels: + cluster: ${CLUSTER_KUSTOMIZATION_NAME} + interval: 1h + retryInterval: 1m + timeout: 5m + sourceRef: + kind: GitRepository + name: sw-catalogs + namespace: flux-system + path: ./cloud-resources/capi/cni/${CNI}/manifests + kubeConfig: + secretRef: + name: ${CLUSTER_KUBECONFIG_SECRET_NAME} + key: ${CLUSTER_KUBECONFIG_SECRET_KEY} + prune: true + # force: true + wait: true + # Input parameters + postBuild: + substitute: + cluster_name: "myclustername" diff --git a/installers/flux/templates/sw-catalogs/infra-controllers/capi/templates/capi-controllers.yaml b/installers/flux/templates/sw-catalogs/infra-controllers/capi/templates/capi-controllers.yaml new file mode 100644 index 00000000..865d56e7 --- /dev/null +++ b/installers/flux/templates/sw-catalogs/infra-controllers/capi/templates/capi-controllers.yaml @@ -0,0 +1,31 @@ +####################################################################################### +# Copyright ETSI Contributors and Others. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. +####################################################################################### + +--- +apiVersion: kustomize.toolkit.fluxcd.io/v1 +kind: Kustomization +metadata: + name: cluster-api + namespace: flux-system +spec: + interval: 1h0m0s + path: ./infra-controllers/capi/manifests + prune: true + sourceRef: + kind: GitRepository + name: sw-catalogs + namespace: flux-system diff --git a/installers/mgmt-operators-and-crds/add-operators-and-crds.sh b/installers/mgmt-operators-and-crds/add-operators-and-crds.sh index 2ee36aab..d2f46faf 100755 --- a/installers/mgmt-operators-and-crds/add-operators-and-crds.sh +++ b/installers/mgmt-operators-and-crds/add-operators-and-crds.sh @@ -50,3 +50,7 @@ cp "${PACKAGE}/templates"/* "${ADDON_CTRL_DIR}/" # Add the Argo WorkFlows controller PACKAGE="${SW_CATALOGS_REPO_DIR}/infra-controllers/argo-workflows" cp "${PACKAGE}/templates"/* "${ADDON_CTRL_DIR}/" + +# Add the CAPI controller and providers +PACKAGE="${SW_CATALOGS_REPO_DIR}/infra-controllers/capi" +cp "${PACKAGE}/templates"/* "${ADDON_CTRL_DIR}/" -- 2.25.1