Ansible installer update

  * Moved Ansible to version 2.10
  * Python Virtual env
  * New option to use a custom SSH key
  * Use of cloud-init file supported
  * Increased the number of options to be passed to the remote installer

Change-Id: I70131f1a24707656c9390445df91d17abab95763
Signed-off-by: Antonio Marsico <antonio.marsico@bt.com>

installers/full_install_osm.sh

@@ -41,8 +41,10 @@ function usage(){
     echo -e "     --pla:          install the PLA module for placement support"
     echo -e "     -m <MODULE>:    install OSM but only rebuild or pull the specified docker images (LW-UI, NG-UI, NBI, LCM, RO, MON, POL, PLA, KAFKA, MONGO, PROMETHEUS, PROMETHEUS-CADVISOR, KEYSTONE-DB, NONE)"
     echo -e "     -o <ADDON>:     ONLY (un)installs one of the addons (vimemu, elk_stack, k8s_monitor)"
-    echo -e "     -O <openrc file/cloud name>: Install OSM to an OpenStack infrastructure. <openrc file/cloud name> is required. If a <cloud name> is used, the clouds.yaml file should be under ~/.config/openstack/ or /etc/openstack/"
+    echo -e "     -O <openrc file path/cloud name>: Install OSM to an OpenStack infrastructure. <openrc file/cloud name> is required. If a <cloud name> is used, the clouds.yaml file should be under ~/.config/openstack/ or /etc/openstack/"
     echo -e "     -N <openstack public network name/ID>: Public network name required to setup OSM to OpenStack"
+    echo -e "     -f <path to SSH public key>: Public SSH key to use to deploy OSM to OpenStack"
+    echo -e "     -F <path to cloud-init file>: Cloud-Init userdata file to deploy OSM to OpenStack"
     echo -e "     -D <devops path> use local devops installation path"
     echo -e "     -w <work dir>   Location to store runtime installation"
     echo -e "     -t <docker tag> specify osm docker tag (default is latest)"
@@ -244,6 +246,12 @@ EONG
         [ -z "$CONTROLLER_NAME" ] && sg lxd -c "juju kill-controller -t 0 -y $OSM_STACK_NAME"
     fi
     remove_crontab_job
+
+    # Cleanup Openstack installer venv
+    if [ -d "$OPENSTACK_PYTHON_VENV" ]; then
+        rm -r $OPENSTACK_PYTHON_VENV
+    fi
+
     [ -z "$INSTALL_NOHOSTCLIENT" ] && uninstall_osmclient
     echo "Some docker images will be kept in case they are used by other docker stacks"
     echo "To remove them, just run 'docker image prune' in a terminal"
@@ -1417,23 +1425,45 @@ function install_to_openstack() {
     $WORKDIR_SUDO apt install -y python3-pip
     $WORKDIR_SUDO -H LC_ALL=C python3 -m pip install -U pip
 
+    # Create a venv to avoid conflicts with the host installation
+    python3 -m venv $OPENSTACK_PYTHON_VENV
+
+    source $OPENSTACK_PYTHON_VENV/bin/activate
+
     # Install Ansible, OpenStack client and SDK
-    $WORKDIR_SUDO -H LC_ALL=C python3 -m pip install -U python-openstackclient "openstacksdk<1" "ansible>=2.9,<3"
+    python -m pip install -U wheel
+    python -m pip install -U python-openstackclient "openstacksdk>=0.12.0,<1" "ansible>=2.10,<2.11"
+
+    # Install the Openstack cloud module (ansible>=2.10)
+    ansible-galaxy collection install openstack.cloud
 
     export ANSIBLE_CONFIG="$OSM_DEVOPS/installers/openstack/ansible.cfg"
 
     OSM_INSTALLER_ARGS="${REPO_ARGS[@]}"
 
+    ANSIBLE_VARS="external_network_name=$2 setup_volume=$3 server_name=$OPENSTACK_VM_NAME"
+
+    if [ -n "$OPENSTACK_SSH_KEY_FILE" ]; then
+        ANSIBLE_VARS+=" key_file=$OPENSTACK_SSH_KEY_FILE"
+    fi
+
+    if [ -n "$OPENSTACK_USERDATA_FILE" ]; then
+        ANSIBLE_VARS+=" userdata_file=$OPENSTACK_USERDATA_FILE"
+    fi
+
     # Execute the Ansible playbook based on openrc or clouds.yaml
     if [ -e "$1" ]; then
         . $1
-        ansible-playbook -e external_network_name=$2 -e installer_args="\"$OSM_INSTALLER_ARGS\"" \
-        -e setup_volume=$3 $OSM_DEVOPS/installers/openstack/site.yml
+        ansible-playbook -e installer_args="\"$OSM_INSTALLER_ARGS\"" -e "$ANSIBLE_VARS" \
+        $OSM_DEVOPS/installers/openstack/site.yml
     else
-        ansible-playbook -e external_network_name=$2 -e installer_args="\"$OSM_INSTALLER_ARGS\"" \
-        -e setup_volume=$3 -e cloud_name=$1 $OSM_DEVOPS/installers/openstack/site.yml
+        ansible-playbook -e installer_args="\"$OSM_INSTALLER_ARGS\"" -e "$ANSIBLE_VARS" \
+        -e cloud_name=$1 $OSM_DEVOPS/installers/openstack/site.yml
     fi
 
+    # Exit from venv
+    deactivate
+
     return 0
 }
 
@@ -1499,6 +1529,9 @@ function dump_vars(){
     echo "OPENSTACK_PUBLIC_NET_NAME=$OPENSTACK_PUBLIC_NET_NAME"
     echo "OPENSTACK_OPENRC_FILE_OR_CLOUD=$OPENSTACK_OPENRC_FILE_OR_CLOUD"
     echo "OPENSTACK_ATTACH_VOLUME=$OPENSTACK_ATTACH_VOLUME"
+    echo "OPENSTACK_SSH_KEY_FILE"="$OPENSTACK_SSH_KEY_FILE"
+    echo "OPENSTACK_USERDATA_FILE"="$OPENSTACK_USERDATA_FILE"
+    echo "OPENSTACK_VM_NAME"="$OPENSTACK_VM_NAME"
     echo "INSTALL_K8S_MONITOR=$INSTALL_K8S_MONITOR"
     echo "TO_REBUILD=$TO_REBUILD"
     echo "INSTALL_NOLXD=$INSTALL_NOLXD"
@@ -1570,6 +1603,10 @@ INSTALL_TO_OPENSTACK=""
 OPENSTACK_OPENRC_FILE_OR_CLOUD=""
 OPENSTACK_PUBLIC_NET_NAME=""
 OPENSTACK_ATTACH_VOLUME="false"
+OPENSTACK_SSH_KEY_FILE=""
+OPENSTACK_USERDATA_FILE=""
+OPENSTACK_VM_NAME="server-osm"
+OPENSTACK_PYTHON_VENV="$HOME/.virtual-envs/osm"
 INSTALL_ONLY=""
 INSTALL_ELK=""
 TO_REBUILD=""
@@ -1617,7 +1654,7 @@ DOCKER_REGISTRY_URL=
 DOCKER_PROXY_URL=
 MODULE_DOCKER_TAG=
 
-while getopts ":b:r:c:n:k:u:R:D:o:O:m:N:H:S:s:w:t:U:P:A:l:L:K:d:p:T:-: hy" o; do
+while getopts ":b:r:c:n:k:u:R:D:o:O:m:N:H:S:s:w:t:U:P:A:l:L:K:d:p:T:f:F:-: hy" o; do
     case "${o}" in
         b)
             COMMIT_ID=${OPTARG}
@@ -1628,13 +1665,13 @@ while getopts ":b:r:c:n:k:u:R:D:o:O:m:N:H:S:s:w:t:U:P:A:l:L:K:d:p:T:-: hy" o; do
             REPO_ARGS+=(-r "$REPOSITORY")
             ;;
         c)
-            [ "${OPTARG}" == "swarm" ] && KUBERNETES="" && continue
+            [ "${OPTARG}" == "swarm" ] && KUBERNETES="" && REPO_ARGS+=(-c "${OPTARG}") && continue
             [ "${OPTARG}" == "k8s" ] && KUBERNETES="y" && continue
             echo -e "Invalid argument for -i : ' $OPTARG'\n" >&2
             usage && exit 1
             ;;
         n)
-            [ "${OPTARG}" == "lwui" ] && NGUI="" && continue
+            [ "${OPTARG}" == "lwui" ] && NGUI="" && REPO_ARGS+=(-n "${OPTARG}") && continue
             [ "${OPTARG}" == "ngui" ] && continue
             echo -e "Invalid argument for -n : ' $OPTARG'\n" >&2
             usage && exit 1
@@ -1669,6 +1706,12 @@ while getopts ":b:r:c:n:k:u:R:D:o:O:m:N:H:S:s:w:t:U:P:A:l:L:K:d:p:T:-: hy" o; do
                 usage && exit 1
             fi
             ;;
+        f)
+            OPENSTACK_SSH_KEY_FILE="${OPTARG}"
+            ;;
+        F)
+            OPENSTACK_USERDATA_FILE="${OPTARG}"
+            ;;
         N)
             OPENSTACK_PUBLIC_NET_NAME="${OPTARG}"
             ;;

installers/install_osm.sh

@@ -114,8 +114,11 @@ if [ $? -eq 0 ]; then
 fi
 }
 
-while getopts ":b:r:c:n:k:u:R:l:L:K:p:D:o:O:m:N:H:S:s:w:t:U:P:A:d:p:-: hy" o; do
+while getopts ":b:r:c:n:k:u:R:l:L:K:p:D:o:O:m:N:H:S:s:w:t:U:P:A:d:p:f:F:-: hy" o; do
     case "${o}" in
+        D)
+            DEVOPS_PATH="${OPTARG}"
+            ;;
         r)
             REPOSITORY="${OPTARG}"
             ;;
@@ -151,4 +154,4 @@ clean_old_repo
 add_repo "deb [arch=amd64] $REPOSITORY_BASE/$RELEASE $REPOSITORY devops"
 sudo DEBIAN_FRONTEND=noninteractive apt-get -q update
 sudo DEBIAN_FRONTEND=noninteractive apt-get install osm-devops
-/usr/share/osm-devops/installers/full_install_osm.sh -R $RELEASE -r $REPOSITORY -u $REPOSITORY_BASE -D /usr/share/osm-devops -t $DOCKER_TAG "$@"
+$DEVOPS_PATH/installers/full_install_osm.sh -R $RELEASE -r $REPOSITORY -u $REPOSITORY_BASE -D $DEVOPS_PATH -t $DOCKER_TAG "$@"

installers/openstack/README.md

@@ -23,7 +23,7 @@ This short guide explains how to use directly the Ansible playbook to install OS
 ## Prerequisites
 The ansible playbook requires `ansible` and `openstacksdk` to be executed. `python-openstackclient` is not mandatory but highly recommended. They are part of Python pip and can be installed as follows:
 
-`$ sudo -H pip install python-openstackclient "openstacksdk<1" "ansible>=2.9,<3"`
+`$ sudo -H pip install python-openstackclient "openstacksdk<1" "ansible>=2.10,<2.11"`
 
 ## Execute the playbook
 

installers/openstack/group_vars/all.yml

@@ -17,7 +17,7 @@
 
 osm_installer_path: /usr/share/osm-devops/installers/install_osm.sh
 
-prefix: server
+server_name: server-osm
 
 cloud_name:
 
@@ -27,7 +27,13 @@ external_network_name:
 
 installer_args:
 
-os_key_name: ansible-key
+userdata_file:
+
+local_userdata:
+
+os_key_name: osm-ansible-key
+
+key_file:
 
 os_flavor:
   name: medium-4-cpu-8-gb-40-disk
@@ -36,7 +42,7 @@ os_flavor:
   disk: 40
 
 servers:
-  - name: osm
+  - name: '1'
     user: ubuntu
     image: ubuntu1804
     image_url: https://cloud-images.ubuntu.com/bionic/current/bionic-server-cloudimg-amd64.img

installers/openstack/roles/create_instances/tasks/main.yml

@@ -14,15 +14,20 @@
 #   limitations under the License.
 #   Author: Antonio Marsico (antonio.marsico@bt.com)
 
+- set_fact:
+    local_userdata: "{{ lookup('file', userdata_file) }}"
+  when: userdata_file != none
+
 - name: Launch OSM instances
-  os_server:
+  openstack.cloud.server:
     cloud: "{{ cloud_name }}"
-    name: "{{ prefix }}-{{ item.name }}"
+    name: "{{ server_name }}-{{ item.name }}"
     state: present
     key_name: "{{ item.key }}"
     nics: "{{ item.nics }}"
     image: "{{ item.image }}"
     flavor: "{{ item.flavor }}"
+    userdata: "{{ local_userdata }}"
     security_groups:
       - default
       - "{{external_network_name}}_access"
@@ -31,14 +36,16 @@
   when: setup_volume|bool == False
 
 - name: Launch OSM instances with a volume
-  os_server:
+  openstack.cloud.server:
     cloud: "{{ cloud_name }}"
-    name: "{{ prefix }}-{{ item.name }}"
+    name: "{{ server_name }}-{{ item.name }}"
     state: present
     key_name: "{{ item.key }}"
     nics: "{{ item.nics }}"
     image: "{{ item.image }}"
     flavor: "{{ item.flavor }}"
+    volume_size: "{{ os_flavor.disk }}"
+    userdata: "{{ local_userdata }}"
     boot_from_volume: yes
     terminate_volume: yes
     security_groups:
@@ -59,3 +66,13 @@
     ansible_private_key_file: "~/.ssh/{{ os_key_name }}"
     ansible_user: "{{ item['item']['user'] }}"
   with_items: "{{ os_hosts.results }}"
+  when: key_file == none
+
+- name: Add OSM host to the local Ansible inventory
+  add_host:
+    name: "{{ item.openstack.accessIPv4 }}"
+    groups: "{{ item['item']['meta']['group'] }}"
+    ansible_private_key_file: "{{ key_file | regex_replace('.pub') }}"
+    ansible_user: "{{ item['item']['user'] }}"
+  with_items: "{{ os_hosts.results }}"
+  when: key_file != none

installers/openstack/roles/osm_installation/tasks/main.yml

@@ -32,13 +32,6 @@
   ignore_errors: yes
   register: curl_result
 
-- name: Pre-configure iptables persistent
-  debconf: name=iptables-persistent question={{ item }} vtype=boolean value=true
-  become: yes
-  with_items:
-  - iptables-persistent/autosave_v4
-  - iptables-persistent/autosave_v6
-
 - debug:
     var: installer_args
     verbosity: 2

installers/openstack/roles/setup_openstack/tasks/main.yml

@@ -15,7 +15,7 @@
 #   Author: Antonio Marsico (antonio.marsico@bt.com)
 
 - name: Looking for the OpenStack external network
-  os_networks_info:
+  openstack.cloud.networks_info:
     cloud: "{{ cloud_name }}"
     filters:
       name: "{{ external_network_name }}"
@@ -27,7 +27,7 @@
     verbosity: 2
 
 - name: Gather information about previously created subnets
-  os_subnets_info:
+  openstack.cloud.subnets_info:
     cloud: "{{ cloud_name }}"
     name: "{{ openstack_external_networks.openstack_networks[0].subnets[0] }}"
   register: subnet_info
@@ -41,7 +41,7 @@
     cidr: "{{ subnet_info.openstack_subnets[0].cidr }}"
 
 - name: Creating a new openstack flavor
-  os_nova_flavor:
+  openstack.cloud.compute_flavor:
     cloud: "{{ cloud_name }}"
     state: present
     name: "{{os_flavor.name}}"
@@ -50,7 +50,7 @@
     disk: "{{os_flavor.disk}}"
 
 - name: Gather information about OpenStack images
-  os_image_info:
+  openstack.cloud.image_info:
     cloud: "{{ cloud_name }}"
     image: "{{ item.image }}"
   with_items: "{{ servers }}"
@@ -74,7 +74,7 @@
   when: item.1.openstack_image == none and item.0.image == item.1.item.image
 
 - name: Creating images
-  os_image:
+  openstack.cloud.image:
     cloud: "{{ cloud_name }}"
     name: "{{ item.0.image }}"
     container_format: bare
@@ -91,31 +91,50 @@
   with_items: "{{ servers }}"
 
 - name: Creating a security group
-  os_security_group:
+  openstack.cloud.security_group:
     cloud: "{{ cloud_name }}"
     state: present
     name: "{{external_network_name}}_access"
     description: Security group for LAN external access
 
 - name: Creating ICMP rule
-  os_security_group_rule:
+  openstack.cloud.security_group_rule:
     cloud: "{{ cloud_name }}"
     security_group: "{{external_network_name}}_access"
     protocol: icmp
     remote_ip_prefix: 0.0.0.0/0
 
 - name: Creating TCP access rule
-  os_security_group_rule:
+  openstack.cloud.security_group_rule:
     cloud: "{{ cloud_name }}"
     security_group: "{{external_network_name}}_access"
     protocol: tcp
     remote_ip_prefix: "{{ cidr }}"
 
+- name: Allow HTTP from anywhere
+  openstack.cloud.security_group_rule:
+    cloud: "{{ cloud_name }}"
+    security_group: "{{external_network_name}}_access"
+    protocol: tcp
+    port_range_min: 80
+    port_range_max: 80
+    remote_ip_prefix: 0.0.0.0/0
+
+- name: Allow SSH from anywhere
+  openstack.cloud.security_group_rule:
+    cloud: "{{ cloud_name }}"
+    security_group: "{{external_network_name}}_access"
+    protocol: tcp
+    port_range_min: 22
+    port_range_max: 22
+    remote_ip_prefix: 0.0.0.0/0
+
 - name: SSH key check
-  os_keypair:
+  openstack.cloud.keypair:
     cloud: "{{ cloud_name }}"
     state: present
     name: "{{ os_key_name }}"
+    public_key_file: "{{ key_file }}"
   register: keypair
 
 - debug:
@@ -128,7 +147,7 @@
     content: "{{ keypair.key.public_key }}"
     dest: "~/.ssh/{{ keypair.key.name }}.pub"
     mode: '600'
-  when: keypair.key.public_key is not none
+  when: keypair.key.public_key != none and key_file == none
 
 - name: Creating the new ansible private key
   local_action:
@@ -136,4 +155,4 @@
     content: "{{ keypair.key.private_key }}"
     dest: "~/.ssh/{{ keypair.key.name }}"
     mode: '600'
-  when: keypair.key.private_key is not none
\ No newline at end of file
+  when: keypair.key.private_key != none and key_file == none
\ No newline at end of file