From 7b53d2629d0c181e026e4eaa1b271b12f36c85f5 Mon Sep 17 00:00:00 2001 From: garciadeblas Date: Fri, 4 Nov 2022 23:18:48 +0100 Subject: [PATCH] Feature 10967 New option ngsa to install monitoring pipeline architecture This change covers the installation of Apache Airflow and Prometheus Pushgateway as an experimental option in the installer. Changes are the following: - `installers/full_install_osm.sh`: - The installer includes a new option "--ng-sa" to install Airflow and Prometheus Pushgateway - When the option is used, the script will call `installers/install_ng_sa.sh` in turn. - `installers/install_ng_sa.sh`: - This script will install Airflow and Pushgateway in the OSM Kubernetes cluster in osm namespace using the helm charts from the respective communities. - `installers/helm/values/airflow/values.yaml`: - File with the values to be used for the installation of Airflow helm chart. - `docker/Airflow/Dockerfile`: - Dockerfile used to build the Airflow image, incorporating the DAG Python files, requirements and internal Python libraries used by DAGs from `osm_ngsa.deb`. Change-Id: I04cb60b25a9a32e42d4a97fac2d1f6abf868b1f7 Signed-off-by: garciadeblas --- docker/Airflow/Dockerfile | 37 +++++++++ installers/full_install_osm.sh | 82 +++++++++++------- installers/helm/values/airflow-values.yaml | 31 +++++++ installers/install_ngsa.sh | 97 ++++++++++++++++++++++ installers/install_osm.sh | 6 +- 5 files changed, 216 insertions(+), 37 deletions(-) create mode 100644 docker/Airflow/Dockerfile create mode 100644 installers/helm/values/airflow-values.yaml create mode 100755 installers/install_ngsa.sh diff --git a/docker/Airflow/Dockerfile b/docker/Airflow/Dockerfile new file mode 100644 index 00000000..b350a09b --- /dev/null +++ b/docker/Airflow/Dockerfile @@ -0,0 +1,37 @@ +####################################################################################### +# Copyright ETSI Contributors and Others. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. +####################################################################################### + +FROM apache/airflow:2.3.0-python3.8 +USER root +RUN DEBIAN_FRONTEND=noninteractive apt-get --yes update && \ + DEBIAN_FRONTEND=noninteractive apt-get --yes install \ + gcc git python3 python3-dev python3-venv python3-pip \ + python3-setuptools wget && \ + python3 -m pip install -U pip build + +ARG PYTHON3_OSM_COMMON_URL +ARG PYTHON3_OSM_NGSA_URL +RUN curl $PYTHON3_OSM_COMMON_URL -o osm_common.deb +RUN dpkg -i ./osm_common.deb +# RUN curl $PYTHON3_OSM_NGSA_URL -o osm_ngsa.deb +# RUN dpkg -i ./osm_ngsa.deb + +RUN pip3 install \ + -r /usr/lib/python3/dist-packages/osm_common/requirements.txt +# -r /usr/lib/python3/dist-packages/osm_ngsa/requirements.txt + +USER airflow diff --git a/installers/full_install_osm.sh b/installers/full_install_osm.sh index 84249a5d..50b2c031 100755 --- a/installers/full_install_osm.sh +++ b/installers/full_install_osm.sh @@ -36,6 +36,7 @@ function usage(){ echo -e " -P use VCA/juju public key file" echo -e " -A use VCA/juju API proxy" echo -e " --pla: install the PLA module for placement support" + echo -e " --ng-sa: install Airflow and Pushgateway to get VNF and NS status (experimental)" echo -e " -m : install OSM but only rebuild or pull the specified docker images (NG-UI, NBI, LCM, RO, MON, POL, PLA, KAFKA, MONGO, PROMETHEUS, PROMETHEUS-CADVISOR, KEYSTONE-DB, NONE)" echo -e " -o : ONLY (un)installs one of the addons (k8s_monitor)" echo -e " -O : Install OSM to an OpenStack infrastructure. is required. If a is used, the clouds.yaml file should be under ~/.config/openstack/ or /etc/openstack/" @@ -310,27 +311,6 @@ function generate_k8s_manifest_files() { [ -z "${DEBUG_INSTALL}" ] || DEBUG beginning of function #Kubernetes resources sudo cp -bR ${OSM_DEVOPS}/installers/docker/osm_pods $OSM_DOCKER_WORK_DIR - sudo rm -f ${OSM_DOCKER_WORK_DIR}/osm_pods/ng-prometheus.yaml - [ -z "${DEBUG_INSTALL}" ] || DEBUG end of function -} - -function generate_prometheus_grafana_files() { - #this only works with docker swarm - [ -z "${DEBUG_INSTALL}" ] || DEBUG beginning of function - # Prometheus files - sudo mkdir -p $OSM_DOCKER_WORK_DIR/prometheus - sudo cp -b ${OSM_DEVOPS}/installers/docker/prometheus/prometheus.yml $OSM_DOCKER_WORK_DIR/prometheus/prometheus.yml - - # Grafana files - sudo mkdir -p $OSM_DOCKER_WORK_DIR/grafana - sudo cp -b ${OSM_DEVOPS}/installers/docker/grafana/dashboards-osm.yml $OSM_DOCKER_WORK_DIR/grafana/dashboards-osm.yml - sudo cp -b ${OSM_DEVOPS}/installers/docker/grafana/datasource-prometheus.yml $OSM_DOCKER_WORK_DIR/grafana/datasource-prometheus.yml - sudo cp -b ${OSM_DEVOPS}/installers/docker/grafana/osm-sample-dashboard.json $OSM_DOCKER_WORK_DIR/grafana/osm-sample-dashboard.json - sudo cp -b ${OSM_DEVOPS}/installers/docker/grafana/osm-system-dashboard.json $OSM_DOCKER_WORK_DIR/grafana/osm-system-dashboard.json - - # Prometheus Exporters files - sudo mkdir -p $OSM_DOCKER_WORK_DIR/prometheus_exporters - sudo cp -b ${OSM_DEVOPS}/installers/docker/prometheus_exporters/node_exporter.service $OSM_DOCKER_WORK_DIR/prometheus_exporters/node_exporter.service [ -z "${DEBUG_INSTALL}" ] || DEBUG end of function } @@ -345,6 +325,9 @@ function generate_docker_env_files() { sudo cp $OSM_DOCKER_WORK_DIR/pol.env{,~} sudo cp $OSM_DOCKER_WORK_DIR/ro-db.env{,~} sudo cp $OSM_DOCKER_WORK_DIR/ro.env{,~} + if [ -n "${INSTALL_NGSA}" ]; then + sudo cp $OSM_DOCKER_WORK_DIR/ngsa.env{,~} + fi echo "Generating docker env files" # LCM @@ -465,12 +448,16 @@ function generate_docker_env_files() { sudo sed -i "s|OSMMON_VCA_CACERT.*|OSMMON_VCA_CACERT=${OSM_VCA_CACERT}|g" $OSM_DOCKER_WORK_DIR/mon.env fi - # POL if [ ! -f $OSM_DOCKER_WORK_DIR/pol.env ]; then echo "OSMPOL_SQL_DATABASE_URI=mysql://root:${MYSQL_ROOT_PASSWORD}@mysql:3306/pol" | sudo tee -a $OSM_DOCKER_WORK_DIR/pol.env fi + # NG-SA + if [ -n "${INSTALL_NGSA}" ] && [ ! -f $OSM_DOCKER_WORK_DIR/ngsa.env ]; then + echo "OSMMON_DATABASE_COMMONKEY=${OSM_DATABASE_COMMONKEY}" | sudo tee -a $OSM_DOCKER_WORK_DIR/ngsa.env + fi + echo "Finished generation of docker env files" [ -z "${DEBUG_INSTALL}" ] || DEBUG end of function } @@ -486,6 +473,9 @@ function kube_secrets(){ kubectl create secret generic ro-secret -n $OSM_STACK_NAME --from-env-file=$OSM_DOCKER_WORK_DIR/ro.env kubectl create secret generic keystone-secret -n $OSM_STACK_NAME --from-env-file=$OSM_DOCKER_WORK_DIR/keystone.env kubectl create secret generic pol-secret -n $OSM_STACK_NAME --from-env-file=$OSM_DOCKER_WORK_DIR/pol.env + if [ -n "${INSTALL_NGSA}" ]; then + kubectl create secret generic ngsa-secret -n $OSM_STACK_NAME --from-env-file=$OSM_DOCKER_WORK_DIR/ngsa.env + fi [ -z "${DEBUG_INSTALL}" ] || DEBUG end of function } @@ -506,13 +496,18 @@ function deploy_charmed_services() { function deploy_osm_pla_service() { [ -z "${DEBUG_INSTALL}" ] || DEBUG beginning of function - # corresponding to namespace_vol - sudo sed -i "s#path: /var/lib/osm#path: $OSM_NAMESPACE_VOL#g" $OSM_DOCKER_WORK_DIR/osm_pla/pla.yaml # corresponding to deploy_osm_services kubectl apply -n $OSM_STACK_NAME -f $OSM_DOCKER_WORK_DIR/osm_pla [ -z "${DEBUG_INSTALL}" ] || DEBUG end of function } +function install_osm_ngsa_service() { + [ -z "${DEBUG_INSTALL}" ] || DEBUG beginning of function + $OSM_DEVOPS/installers/install_ngsa.sh -d ${OSM_HELM_WORK_DIR} -D ${OSM_DEVOPS} ${DEBUG_INSTALL} || \ + FATAL_TRACK install_osm_ngsa_service "install_ngsa.sh failed" + [ -z "${DEBUG_INSTALL}" ] || DEBUG end of function +} + function parse_yaml() { [ -z "${DEBUG_INSTALL}" ] || DEBUG beginning of function TAG=$1 @@ -521,19 +516,23 @@ function parse_yaml() { for module in $services; do if [ "$module" == "pla" ]; then if [ -n "$INSTALL_PLA" ]; then - echo "Updating K8s manifest file from opensourcemano\/${module}:.* to ${DOCKER_REGISTRY_URL}${DOCKER_USER}\/${module}:${TAG}" + echo "Updating K8s manifest file from opensourcemano\/pla:.* to ${DOCKER_REGISTRY_URL}${DOCKER_USER}\/pla:${TAG}" sudo sed -i "s#opensourcemano/pla:.*#${DOCKER_REGISTRY_URL}${DOCKER_USER}/pla:${TAG}#g" ${OSM_DOCKER_WORK_DIR}/osm_pla/pla.yaml fi else - echo "Updating K8s manifest file from opensourcemano\/${module}:.* to ${DOCKER_REGISTRY_URL}${DOCKER_USER}\/${module}:${TAG}" - sudo sed -i "s#opensourcemano/${module}:.*#${DOCKER_REGISTRY_URL}${DOCKER_USER}/${module}:${TAG}#g" ${OSM_K8S_WORK_DIR}/${module}.yaml + image=${module} + if [ "$module" == "ng-prometheus" ]; then + image="prometheus" + fi + echo "Updating K8s manifest file from opensourcemano\/${image}:.* to ${DOCKER_REGISTRY_URL}${DOCKER_USER}\/${image}:${TAG}" + sudo sed -i "s#opensourcemano/${image}:.*#${DOCKER_REGISTRY_URL}${DOCKER_USER}/${image}:${TAG}#g" ${OSM_K8S_WORK_DIR}/${module}.yaml fi done [ -z "${DEBUG_INSTALL}" ] || DEBUG end of function } function update_manifest_files() { - osm_services="nbi lcm ro pol mon ng-ui keystone pla prometheus" + osm_services="nbi lcm ro pol mon ng-ui keystone pla prometheus ng-prometheus" list_of_services="" for module in $osm_services; do module_upper="${module^^}" @@ -547,14 +546,23 @@ function update_manifest_files() { if [ -n "$MODULE_DOCKER_TAG" ]; then parse_yaml $MODULE_DOCKER_TAG $list_of_services_to_rebuild fi + # The manifest for prometheus is prometheus.yaml or ng-prometheus.yaml, depending on the installation option + if [ -n "$INSTALL_NGSA" ]; then + sudo rm -f ${OSM_K8S_WORK_DIR}/prometheus.yaml + else + sudo rm -f ${OSM_K8S_WORK_DIR}/ng-prometheus.yaml + fi [ -z "${DEBUG_INSTALL}" ] || DEBUG end of function } function namespace_vol() { [ -z "${DEBUG_INSTALL}" ] || DEBUG beginning of function - osm_services="nbi lcm ro pol mon kafka mysql prometheus" + # List of services with a volume mounted in path /var/lib/osm + osm_services="mysql" for osm in $osm_services; do - sudo sed -i "s#path: /var/lib/osm#path: $OSM_NAMESPACE_VOL#g" $OSM_K8S_WORK_DIR/$osm.yaml + if [ -f "$OSM_K8S_WORK_DIR/$osm.yaml" ] ; then + sudo sed -i "s#path: /var/lib/osm#path: $OSM_NAMESPACE_VOL#g" $OSM_K8S_WORK_DIR/$osm.yaml + fi done [ -z "${DEBUG_INSTALL}" ] || DEBUG end of function } @@ -685,7 +693,7 @@ function install_osm() { track deploy_osm namespace_vol_ok deploy_osm_services track deploy_osm deploy_osm_services_k8s_ok - if [ -n "$INSTALL_PLA"]; then + if [ -n "$INSTALL_PLA" ]; then # optional PLA install deploy_osm_pla_service track deploy_osm deploy_osm_pla_ok @@ -695,6 +703,11 @@ function install_osm() { install_k8s_monitoring track deploy_osm install_k8s_monitoring_ok fi + if [ -n "$INSTALL_NGSA" ]; then + # optional PLA install + install_osm_ngsa_service + track deploy_osm install_osm_ngsa_ok + fi [ -z "$INSTALL_NOHOSTCLIENT" ] && install_osmclient track osmclient osmclient_ok @@ -792,6 +805,7 @@ function dump_vars(){ echo "INSTALL_K8S_MONITOR=$INSTALL_K8S_MONITOR" echo "INSTALL_LIGHTWEIGHT=$INSTALL_LIGHTWEIGHT" echo "INSTALL_LXD=$INSTALL_LXD" + echo "INSTALL_NGSA=$INSTALL_NGSA" echo "INSTALL_NODOCKER=$INSTALL_NODOCKER" echo "INSTALL_NOJUJU=$INSTALL_NOJUJU" echo "INSTALL_NOLXD=$INSTALL_NOLXD" @@ -809,6 +823,7 @@ function dump_vars(){ echo "OSM_DEVOPS=$OSM_DEVOPS" echo "OSM_DOCKER_TAG=$OSM_DOCKER_TAG" echo "OSM_DOCKER_WORK_DIR=$OSM_DOCKER_WORK_DIR" + echo "OSM_HELM_WORK_DIR=$OSM_HELM_WORK_DIR" echo "OSM_K8S_WORK_DIR=$OSM_K8S_WORK_DIR" echo "OSM_STACK_NAME=$OSM_STACK_NAME" echo "OSM_VCA_HOST=$OSM_VCA_HOST" @@ -861,6 +876,7 @@ RELEASE="ReleaseTEN" REPOSITORY="stable" INSTALL_VIMEMU="" INSTALL_PLA="" +INSTALL_NGSA="" LXD_REPOSITORY_BASE="https://osm-download.etsi.org/repository/osm/lxd" LXD_REPOSITORY_PATH="" INSTALL_LIGHTWEIGHT="y" @@ -892,8 +908,9 @@ DOCKER_NOBUILD="" REPOSITORY_KEY="OSM%20ETSI%20Release%20Key.gpg" REPOSITORY_BASE="https://osm-download.etsi.org/repository/osm/debian" OSM_WORK_DIR="/etc/osm" -OSM_DOCKER_WORK_DIR="/etc/osm/docker" +OSM_DOCKER_WORK_DIR="${OSM_WORK_DIR}/docker" OSM_K8S_WORK_DIR="${OSM_DOCKER_WORK_DIR}/osm_pods" +OSM_HELM_WORK_DIR="${OSM_WORK_DIR}/helm" OSM_HOST_VOL="/var/lib/osm" OSM_NAMESPACE_VOL="${OSM_HOST_VOL}/${OSM_STACK_NAME}" OSM_DOCKER_TAG=latest @@ -1057,6 +1074,7 @@ while getopts ":a:b:r:n:k:u:R:D:o:O:m:N:H:S:s:t:U:P:A:l:L:K:d:p:T:f:F:-: hy" o; [ "${OPTARG}" == "tag" ] && continue [ "${OPTARG}" == "registry" ] && continue [ "${OPTARG}" == "pla" ] && INSTALL_PLA="y" && continue + [ "${OPTARG}" == "ng-sa" ] && INSTALL_NGSA="y" && continue [ "${OPTARG}" == "volume" ] && OPENSTACK_ATTACH_VOLUME="true" && continue [ "${OPTARG}" == "nocachelxdimages" ] && continue [ "${OPTARG}" == "cachelxdimages" ] && INSTALL_CACHELXDIMAGES="--cachelxdimages" && continue diff --git a/installers/helm/values/airflow-values.yaml b/installers/helm/values/airflow-values.yaml new file mode 100644 index 00000000..0a370972 --- /dev/null +++ b/installers/helm/values/airflow-values.yaml @@ -0,0 +1,31 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +--- +defaultAirflowRepository: gerardogarcia/airflow +defaultAirflowTag: "0.3" +webserverSecretKeySecretName: airflow-webserver-secret +webserver: + service: + type: NodePort + ports: + - name: airflow-ui + port: "{{ .Values.ports.airflowUI }}" + targetPort: "{{ .Values.ports.airflowUI }}" +extraEnvFrom: | + - secretRef: + name: ngsa-secret + diff --git a/installers/install_ngsa.sh b/installers/install_ngsa.sh new file mode 100755 index 00000000..6ddf2986 --- /dev/null +++ b/installers/install_ngsa.sh @@ -0,0 +1,97 @@ +#!/bin/bash +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +set +eux + +# Helm chart 1.6.0 correspondes to Airflow 2.3.0 +AIRFLOW_HELM_VERSION=1.6.0 +PROMPUSHGW_HELM_VERSION=1.18.2 + +# Install Airflow helm chart +function install_airflow() { + [ -z "${DEBUG_INSTALL}" ] || DEBUG beginning of function + # copy airflow-values.yaml to the destination folder + sudo mkdir -p ${OSM_HELM_WORK_DIR} + sudo cp ${OSM_DEVOPS}/installers/helm/values/airflow-values.yaml ${OSM_HELM_WORK_DIR} + if ! helm -n osm status airflow 2> /dev/null ; then + # if it does not exist, create secrets and install + kubectl -n osm create secret generic airflow-webserver-secret --from-literal="webserver-secret-key=$(python3 -c 'import secrets; print(secrets.token_hex(16))')" + helm repo add apache-airflow https://airflow.apache.org + helm repo update + helm -n osm install airflow apache-airflow/airflow -f ${OSM_HELM_WORK_DIR}/airflow-values.yaml --version ${AIRFLOW_HELM_VERSION} + else + # if it exists, upgrade + helm repo update + helm -n osm upgrade airflow apache-airflow/airflow -f ${OSM_HELM_WORK_DIR}/airflow-values.yaml --version ${AIRFLOW_HELM_VERSION} + fi + [ -z "${DEBUG_INSTALL}" ] || DEBUG end of function +} + +# Install Prometheus Pushgateway helm chart +function install_prometheus_pushgateway() { + [ -z "${DEBUG_INSTALL}" ] || DEBUG end of function + if ! helm -n osm status pushgateway 2> /dev/null ; then + # if it does not exist, install + helm repo add prometheus-community https://prometheus-community.github.io/helm-charts + helm repo update + helm -n osm install pushgateway prometheus-community/prometheus-pushgateway --version ${PROMPUSHGW_HELM_VERSION} + else + # if it exists, upgrade + helm repo update + helm -n osm upgrade pushgateway prometheus-community/prometheus-pushgateway --version ${PROMPUSHGW_HELM_VERSION} + fi + [ -z "${DEBUG_INSTALL}" ] || DEBUG end of function +} + +# main +while getopts ":D:d:-: " o; do + case "${o}" in + D) + OSM_DEVOPS="${OPTARG}" + ;; + d) + OSM_HELM_WORK_DIR="${OPTARG}" + ;; + -) + [ "${OPTARG}" == "debug" ] && DEBUG_INSTALL="y" && continue + echo -e "Invalid option: '--$OPTARG'\n" >&2 + exit 1 + ;; + :) + echo "Option -$OPTARG requires an argument" >&2 + exit 1 + ;; + \?) + echo -e "Invalid option: '-$OPTARG'\n" >&2 + exit 1 + ;; + *) + exit 1 + ;; + esac +done + +source $OSM_DEVOPS/common/logging +source $OSM_DEVOPS/common/track + +echo "DEBUG_INSTALL=$DEBUG_INSTALL" +echo "OSM_DEVOPS=$OSM_DEVOPS" +echo "OSM_HELM_WORK_DIR=$OSM_HELM_WORK_DIR" + +install_airflow +track deploy_osm airflow_ok +install_prometheus_pushgateway +track deploy_osm pushgateway_ok + diff --git a/installers/install_osm.sh b/installers/install_osm.sh index 7199c3b7..11ce0bec 100755 --- a/installers/install_osm.sh +++ b/installers/install_osm.sh @@ -38,15 +38,12 @@ function usage(){ echo -e " -H use specific juju host controller IP" echo -e " -S use VCA/juju secret key" echo -e " -P use VCA/juju public key file" - echo -e " -C use VCA/juju CA certificate file" echo -e " -A use VCA/juju API proxy" echo -e " --pla: install the PLA module for placement support" + echo -e " --ng-sa: install Airflow and Pushgateway to get VNF and NS status (experimental)" echo -e " -m : install OSM but only rebuild or pull the specified docker images (NG-UI, NBI, LCM, RO, MON, POL, PLA, KAFKA, MONGO, PROMETHEUS, PROMETHEUS-CADVISOR, KEYSTONE-DB, NONE)" echo -e " -o : ONLY (un)installs one of the addons (k8s_monitor)" echo -e " -O : Install OSM to an OpenStack infrastructure. is required. If a is used, the clouds.yaml file should be under ~/.config/openstack/ or /etc/openstack/" - echo -e " -m : install OSM but only rebuild the specified docker images (LW-UI, NBI, LCM, RO, MON, POL, KAFKA, MONGO, PROMETHEUS, PROMETHEUS-CADVISOR, KEYSTONE-DB, PLA, NONE)" - echo -e " -o : ONLY (un)installs one of the addons (vimemu, elk_stack, k8s_monitor)" - echo -e " -O : Install OSM to an OpenStack infrastructure. is required. If a is used, the clouds.yaml file should be under ~/.config/openstack/ or /etc/openstack/" echo -e " -N : Public network name required to setup OSM to OpenStack" echo -e " -f : Public SSH key to use to deploy OSM to OpenStack" echo -e " -F : Cloud-Init userdata file to deploy OSM to OpenStack" @@ -87,7 +84,6 @@ function usage(){ echo -e " [--ha]: Installs High Availability bundle. (--charmed option)" echo -e " [--tag]: Docker image tag. (--charmed option)" echo -e " [--registry]: Docker registry with optional credentials as user:pass@hostname:port (--charmed option)" - } add_repo() { -- 2.17.1