From 15b76cb2c888af2de4c5eb3028447c5ac2e42d98 Mon Sep 17 00:00:00 2001 From: Philip Joseph Date: Fri, 8 Jul 2016 01:42:08 +0530 Subject: [PATCH] New Feature Request: Enhance descriptor package creation script --- Makefile | 6 +- src/generate_descriptor_pkg.sh | 737 +++++++++++++++++++++++++++++++-- 2 files changed, 709 insertions(+), 34 deletions(-) diff --git a/Makefile b/Makefile index 5993ff5d..6a5a4d8a 100644 --- a/Makefile +++ b/Makefile @@ -66,15 +66,15 @@ $(NSD_BUILD_DIR)/%: $(NSD_SRC_DIR)/% src/gen_nsd_pkg.sh $< $@ $(BUILD_DIR)/nsd_pkgs/%.tar.gz: $(NSD_BUILD_DIR)/% - src/generate_descriptor_pkg.sh $(BUILD_DIR)/nsd_pkgs $< + src/generate_descriptor_pkg.sh -d $(BUILD_DIR)/nsd_pkgs $< $(VNFD_BUILD_DIR)/ims_allin1_2p_vnf/charms/clearwater-aio-proxy: $(VNFD_BUILD_DIR)/ims_allin1_2p_vnf $(BUILD_DIR)/clearwater-juju # Copy the IMS Charm into the IMS vnf package directory before packaging cp -rf $(BUILD_DIR)/clearwater-juju/charms/trusty/clearwater-aio-proxy $(VNFD_BUILD_DIR)/ims_allin1_2p_vnf/charms $(VNFD_BUILD_DIR)/6wind_vnf/charms/vpe-router: $(VNFD_BUILD_DIR)/6wind_vnf $(BUILD_DIR)/juju-charms - # Copy the IMS Charm into the IMS vnf package directory before packaging + # Copy the PE Charm into the PE vnf package directory before packaging cp -rf $(BUILD_DIR)/juju-charms/vpe-router $(VNFD_BUILD_DIR)/6wind_vnf/charms $(BUILD_DIR)/vnfd_pkgs/%.tar.gz: $(VNFD_BUILD_DIR)/% $(VNFD_BUILD_DIR)/ims_allin1_2p_vnf/charms/clearwater-aio-proxy $(VNFD_BUILD_DIR)/6wind_vnf/charms/vpe-router - src/generate_descriptor_pkg.sh $(BUILD_DIR)/vnfd_pkgs $< + src/generate_descriptor_pkg.sh -d $(BUILD_DIR)/vnfd_pkgs $< diff --git a/src/generate_descriptor_pkg.sh b/src/generate_descriptor_pkg.sh index a91637d2..19fa12a0 100755 --- a/src/generate_descriptor_pkg.sh +++ b/src/generate_descriptor_pkg.sh @@ -1,42 +1,717 @@ #!/bin/bash -# -# Copyright 2016 RIFT.IO Inc -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -# Author(s): Austin Cormier -# Creation Date: 2016/05/23 -# + +############################################################################ +# Copyright 2016 RIFT.io Inc # +# # +# Licensed under the Apache License, Version 2.0 (the "License"); # +# you may not use this file except in compliance with the License. # +# You may obtain a copy of the License at # +# # +# http://www.apache.org/licenses/LICENSE-2.0 # +# # +# Unless required by applicable law or agreed to in writing, software # +# distributed under the License is distributed on an "AS IS" BASIS, # +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # +# See the License for the specific language governing permissions and # +# limitations under the License. # +############################################################################ + # # This shell script is used to create a descriptor package # The main functions of this script include: # - Generate checksums.txt file # - Generate a tar.gz file +# This script can be used to create the required folders for +# a descriptor package and a template descriptor + +# Usage: generate_descriptor_pkg.sh + +# Descriptor names should be +# - (nsd|vnfd).(yaml|yml|json|xml) +# - *_(nsd|vnfd).(yaml|yml|json|xml) +# - *_(nsd|vnfd)_*.(yaml|yml|json|xml) +# - (nsd|vnfd)/*.(yaml|yml|json|xml) +# + +SCRIPTNAME=`basename $0` + +# From https://osm.etsi.org/wikipub/index.php/Release_0_Data_Model_Details +# Supported folders for VNFD +# cloud_init - Rel 4.3, not yet part of OSM +VNFD_FOLDERS=(images scripts icons charms cloud_init) + +# Supported folders for NSD +# OSM document specifies (ns|vnf)-config folder, while Rel 4.3 +# is using (ns|vnf)_config. +NSD_FOLDERS=(scripts icons ns_config vnf_config) + +# Other files allowed in the descriptor base directory +ALLOWED_FILES=(README) + +DESC_TYPES=(vnfd nsd) +DESC_EXTN=(yml yaml json xml) +CHKSUM='checksums.txt' + +VERBOSE=false +DRY_RUN=false +CREATE=false +RM="--remove-files" +DEBUG=false + +function usage() { + cat <] [-N] [-c] [base-directory] + + -h|--help : show this message + + -t|--package-type : Descriptor package type + is NSD or VNFD. Script will try to + determine the type if not provided. + Default is vnfd for create-folders. + + -d|--destination-dir : Directory to create the + archived file. + + -N|--no-remove-files : Do not remove the package files after creating + archive + + -c|--create-folder : Create folder with the structure for the + package type using the base-dir and package-dir + and a descriptor template + + -v| --verbose : Generate logs + + -n| --dry-run : Validate the package dir + + base-dir : Directory where the archive file or folders are created, + if destination directory is not specified. + Default is current directory + + package-dir : The descriptor name and directory +EOF +} + +function write_vnfd_tmpl() { + name=$(basename $1) + desc_file="${name}_vnfd.yaml" + + cat >$desc_file < tag with correct values +# Update or remove tags +vnfd:vnfd-catalog: + vnfd: + - id: ${name} + name: ${name} + short-name: ${name} + description: ${name} + + # Place the logo as png in icons directory and provide the name here + logo: + + # This is an optional section and can be removed + vnf-configuration: + config-attributes: + config-delay: '0' + config-priority: '1' + + # Specify the config method + # Remove this section if the VNF does not use a config method + juju: + charm: + + # config-primitive specifies the config and actions for the VNF/charm + # This is an optional section + config-primitive: + # If there are no config to be set, remove this section + - name: config + # There can be multiple parameter sections + # The name and type should be same as VNF/charm specification + parameter: + - name: + data-type: STRING + mandatory: + default-value: + + # If there are no actions specified at VNF level, remove this section + # Multiple action sections can be specified + # The name should be the name of the action + - name: + # There can be multiple parameter sections + # The name and type should be same as charm specification + parameter: + - name: + data-type: STRING + mandatory: + + # Any config or action that need to be executed when VNF comes up + # This is an optional section + initial-config-primitive: + - name: config + # Any config to be set when VNF comes up + # For charm normally the VNF management IP need to be set + # in one of the config parameters + # Sepcifying the value a will cause the MANO + # to replace this with the management ip of VNF when it is + # instantiated + parameter: + - name: + value: + seq: '1' + + mgmt-interface: + # This should be id of one of the VDUs specified for this VNF + vdu-id: ${name}-VM + port: + + connection-point: + # Specify the connection points from VDU + # Can specify multiple connection points here + - name: eth0 + type: VPORT + - name: eth1 + type: VPORT + + # Atleast one VDU need to be specified + vdu: + - id: ${name}-VM + name: ${name}-VM + description: ${name}-VM + + # Image including the full path + image: + + # Flavour of the VM to be instantiated for the VDU + vm-flavor: + memory-mb: '4096' + storage-gb: '10' + vcpu-count: '2' + + external-interface: + # Specify the external interfaces + # There can be multiple interfaces defined + - name: eth0 + virtual-interface: + bandwidth: '0' + type: VIRTIO + vpci: 0000:00:0a.0 + vnfd-connection-point-ref: eth0 + - name: eth1 + virtual-interface: + bandwidth: '0' + type: OM-MGMT + vpci: 0000:00:0b.0 + vnfd-connection-point-ref: eth1 + + # Mgmt vpci, match with the specification in the + # external interface for mgmt + mgmt-vpci: 0000:00:0a.0 + + # Specify EPA parameters + # This section is optional + guest-epa: + cpu-pinning-policy: DEDICATED + cpu-thread-pinning-policy: PREFER + mempage-size: LARGE + numa-node-policy: + mem-policy: STRICT + node: + - id: '0' + paired-threads: + num-paired-threads: '1' + node-cnt: '1' + +EOF + + if [ $VERBOSE == true ]; then + echo "INFO: Created $desc_file" + fi +} + +function write_nsd_tmpl() { + name=$(basename $1) + desc_file="${name}_nsd.yaml" + + cat >$desc_file < tag with correct values +# Update or remove tags +nsd:nsd-catalog: + nsd: + - id: ${name} + name: ${name} + short-name: ${name} + description: ${name} + + # Place the logo as png in icons directory and provide the name here + logo: + + # Specify the VNFDs that are part of this NSD + constituent-vnfd: + # The member-vnf-index needs to be unique, starting from 1 + # vnfd-id-ref is the id of the VNFD + # Multiple constituent VNFDs can be specified + - member-vnf-index: + vnfd-id-ref: + + vld: + # Management network for the VNFs + - id: management + name: management + provider-network: + overlay-type: VLAN + physical-network: + type: ELAN + vnfd-connection-point-ref: + # Specify all the constituent VNFs + # member-vnf-index-ref - entry from constituent vnf + # vnfd-id-ref - VNFD id + # vnfd-connection-point-ref - connection point name in the VNFD + - member-vnf-index-ref: '1' + vnfd-connection-point-ref: + vnfd-id-ref: + - member-vnf-index-ref: '2' + vnfd-connection-point-ref: + vnfd-id-ref: + + # Addtional netowrk can be specified for data, etc + - id: + name: + type: ELAN + provider-network: + overlay-type: VLAN + physical-network: + vnfd-connection-point-ref: + - member-vnf-index-ref: + vnfd-connection-point-ref: + vnfd-id-ref: + + # NS level configuration primitive + # This section is optional + config-primitive: + - name: + # List of parameters, optional + parameter: + - name: + data-type: string + default-value: + mandatory: + + # List of parameters grouped by name + # optional section + parameter-group: + - mandatory: + name: + parameter: + - name: + data-type: integer + default-value: + hidden: + mandatory: + + # Script based NS level configuration + user-defined-script: +EOF + + if [ $VERBOSE == true ]; then + echo "INFO: Created $desc_file" + fi +} + +function write_nsd_config_tmpl() { + name=$(basename $1) + cfg_file="ns_config/$name.yaml" + + cat >$cfg_file < +if [ $? != 0 ] ; then + echo "ERROR: Failed parsing options ($?)." >&2 + usage + exit 1 +fi -mkdir -p $1 -mkdir -p $2 +echo "$OPTS" +eval set -- "$OPTS" -dest_dir=$(cd $1 && pwd) -pkg_dir=$(cd $2 && pwd) +cur_dir=`pwd` -echo $(pwd) -cd ${pkg_dir} -rm -rf checksums.txt -find * -type f | - while read file; do - md5sum $file >> checksums.txt +# Check if the array contains a specific value +# Taken from +# http://stackoverflow.com/questions/3685970/check-if-an-array-contains-a-value +function contains() { + local n=$# + local value=${!n} + for ((i=1;i < $#;i++)); do + if [ "${!i}" == "${value}" ]; then + echo "y" + return 0 + fi done -cd .. -tar -zcvf ${dest_dir}/$(basename $pkg_dir).tar.gz $(basename $pkg_dir) + echo "n" + return 1 +} + +function check_type() { + type=$1 + if [ $(contains "${DESC_TYPES[@]}" $type) == "y" ]; then + TYPE=$type + else + echo "ERROR: Unknown descriptor type $type!" >&2 + exit 1 + fi +} + +function get_expr(){ + # First argument is to specify if this is a negative match or match + # Rest are filename expressions without extension + # + + local regex=" " + local n=$1 + local neg="${1}" + for ((i=2;i <= $#;i++)); do + if [ $i -eq 2 ]; then + if [ $neg == true ]; then + subexpr='! -name' + else + subexpr='-name' + fi + else + if [ $neg == true ]; then + subexpr=' -a ! -name' + else + subexpr=' -o -name' + fi + fi + + for extn in ${DESC_EXTN[$@]}; do + regex="$regex $subexpr ${!i}.$extn" + done + done + + if [ $VERBOSE == true ]; then + echo "INFO: Generate expression: $expr" + fi + + echo "$expr" +} + +function get_expr(){ + # First argument is to specify if this is a negative match or not + # Rest are filename expressions without extension + + local regex=" " + local n=$# + local neg="${1}" + local first="true" + for ((i=2;i <= $#;i++)); do + for extn in "${DESC_EXTN[@]}"; do + if [ $first == true ]; then + if [ $neg == true ]; then + subexpr='! -name' + else + subexpr='-name' + fi + first=false + else + if [ $neg == true ]; then + subexpr=' -a ! -name' + else + subexpr=' -o -name' + fi + fi + + regex="$regex $subexpr ${!i}.$extn" + done + done + + echo "$regex" +} + +while true; do + case "$1" in + -v | --verbose ) VERBOSE=true; shift ;; + -h | --help ) usage; exit 0; shift ;; + -n | --dry-run ) DRY_RUN=true; shift ;; + -t | --package-type ) check_type "$2"; shift; shift ;; + -d | --destination-dir ) DEST_DIR=$2; shift; shift;; + -c | --create-folder ) CREATE=true; shift;; + -N | --no-remove-files ) RM=''; shift;; + --debug ) DEBUG=true; shift;; + -- ) shift; break ;; + * ) break ;; + esac +done + +if [ $DEBUG == true ]; then + echo "INFO: Debugging ON" + set -x +fi + +if [ $VERBOSE == true ]; then + echo "INFO: Descriptor type: $TYPE" +fi + +# Dry run is to validate existing descriptor folders +if [ $DRY_RUN == true ] && [ $CREATE == true ]; then + echo "ERROR: Option dry-run with create-folders not supported!" >&2 + exit 1 +fi + +if [ $# -gt 1 ]; then + BASE_DIR=$1 + PKG=$(basename $2) +else + BASE_DIR=$(dirname $1) + PKG=$(basename $1) +fi + +if [ $VERBOSE == true ]; then + echo "INFO: Using base dir: $BASE_DIR" +fi + +if [ $VERBOSE == true ]; then + echo "INFO: Using package: $PKG" +fi + +if [ -z "$PKG" ]; then + echo "ERROR: Need to specify the package-dir" >&2 + usage >&2 + exit 1 +fi + +cd $BASE_DIR +if [ $? -ne 0 ]; then + echo "ERROR: Unable to change to base directory $BASE_DIR!" >&2 + exit 1 +fi + +# Get full base dir path +BASE_DIR=`pwd` +cd $cur_dir + +if [ -z $DEST_DIR ]; then + DEST_DIR=$BASE_DIR # Default to base directory +fi + +mkdir -p $DEST_DIR + +cd $DEST_DIR +if [ $? -ne 0 ]; then + echo "ERROR: Not able to access destination directory $DEST_DIR!" >&2 + exit 1 +fi + +# Get the full destination dir path +DEST_DIR=`pwd` +cd $cur_dir + +dir=${BASE_DIR}/${PKG} + +function add_chksum() { + if [ $VERBOSE == true ]; then + echo "INFO: Add file $1 to $CHKSUM" + fi + + md5sum $1 >> $CHKSUM +} + +if [ $CREATE == false ]; then + if [ ! -d $dir ]; then + echo "INFO: Package folder $dir not found!" >&2 + exit 1 + fi + + cd $dir + if [ $? -ne 0 ]; then + rc=$? + echo "ERROR: changing directory to $dir ($rc)" >&2 + exit $rc + fi + + # Remove checksum file, if present + rm -f $CHKSUM + + # Check if the descriptor file is present + if [ -z $TYPE ]; then + # Desc type not specified, look for the desc file and guess the type + # Required for backward compatibility + for ty in ${DESC_TYPES[@]}; do + re=$(get_expr false "$ty" "*_$ty" "*_${ty}_*") + desc=$(find * -maxdepth 0 -type f $re 2>/dev/null) + + if [ -z $desc ] || [ ${#desc[@]} -eq 0 ]; then + # Check the vnfd|nsd folder + if [ ! -d $ty ]; then + continue + fi + re=$(get_expr false "*") + desc=$(find $ty/* -maxdepth 0 -type f $re 2>/dev/null) + if [ -z $desc ] || [ ${#desc[@]} -eq 0 ]; then + continue + elif [ ${#desc[@]} -gt 1 ]; then + echo "ERROR: Found multiple descriptor files: ${desc[@]}" >&2 + exit 1 + fi + # Descriptor sub directory + desc_sub_dir=$ty + fi + + TYPE=$ty + if [ $TYPE == 'nsd' ]; then + folders=("${NSD_FOLDERS[@]}") + else + folders=("${VNFD_FOLDERS[@]}") + fi + + if [ $VERBOSE == true ]; then + echo "INFO: Determined descriptor is of type $TYPE" + fi + break + done + + if [ -z $TYPE ]; then + echo "ERROR: Unable to determine the descriptor type!" >&2 + exit 1 + fi + else + if [ $TYPE == 'nsd' ]; then + folders=("${NSD_FOLDERS[@]}") + else + folders=("${VNFD_FOLDERS[@]}") + fi + + # Check for descriptor of type provided on command line + re=$(get_expr false "$TYPE" "*_${TYPE}" "*_${TYPE}_*") + desc=$(find * -maxdepth 0 -type f $re 2>/dev/null) + + if [ -z $desc ] || [ ${#desc[@]} -eq 0 ]; then + # Check if it is under vnfd/nsd subdirectory + # Backward compatibility support + re=$(get_expr false "*") + desc=$(find $TYPE/* -maxdepth 0 -type f $re 2>/dev/null) + if [ -z $desc ] || [ ${#desc[@]} -eq 0 ]; then + echo "ERROR: Did not find descriptor file of type $TYPE" \ + " in $dir" >&2 + exit 1 + fi + desc_sub_dir=$ty + fi + fi + + if [ ${#desc[@]} -gt 1 ]; then + echo "ERROR: Found multiple files of type $TYPE in $dir: $desc" >&2 + exit 1 + fi + + descriptor=${desc[0]} + + # Check if there are files not supported + files=$(find * -maxdepth 0 -type f ! -name $descriptor 2>/dev/null) + + for f in ${files[@]}; do + if [ $(contains "${ALLOWED_FILES[@]}" $f) == "n" ]; then + echo "WARN: Unsupported file $f found" + fi + done + + if [ $VERBOSE == true ]; then + echo "INFO: Found descriptor package: ${desc_sub_dir} ${descriptor}" + fi + + if [ $DRY_RUN == false ]; then + add_chksum ${descriptor} + fi + + # Check the folders are supported ones + dirs=$( find * -maxdepth 0 -type d ) + + for d in ${dirs[@]}; do + if [ $(contains "${folders[@]}" $d) == "y" ]; then + if [ $DRY_RUN == false ]; then + find $d/* -type f 2>/dev/null| + while read file; do + add_chksum $file + done + fi + elif [ -z $desc_sub_dir ] || [ $d != $desc_sub_dir ]; then + echo "WARN: $d is not part of standard folders " \ + "for descriptor type $TYPE in $PKG" + fi + done + + if [ $VERBOSE == true ]; then + echo "INFO: Creating archive for $PKG" + fi + + cd $BASE_DIR + if [ $DRY_RUN == false ]; then + tar zcvf "$DEST_DIR/$PKG.tar.gz" "${PKG}" ${RM} + if [ $? -ne 0 ]; then + rc=$? + echo "ERROR: creating archive for $PKG ($rc)" >&2 + exit $rc + fi + fi +else + # Create the folders for the descriptor + if [ $VERBOSE == true ]; then + echo "INFO: Creating folders for $PKG in $dir" + fi + + # Create, default to VNFD if no type is defined + if [ -z $TYPE ]; then + TYPE=vnfd + echo "WARNING: Defaulting to descriptor type $TYPE" + fi + + mkdir -p $dir && cd $dir + if [ $? -ne 0 ]; then + rc=$? + echo "ERROR: creating directory $dir ($rc)" >&2 + exit $rc + fi + + if [ $TYPE == 'nsd' ]; then + folders=("${NSD_FOLDERS[@]}") + else + folders=("${VNFD_FOLDERS[@]}") + fi + + for d in ${folders[@]}; do + mkdir -p $dir/$d + if [ $? -ne 0 ]; then + rc=$? + echo "ERROR: creating directory $dir/$d ($rc)" >&2 + exit $rc + fi + if [ $VERBOSE == true ]; then + echo "Created folder $d in $dir" + fi + done + + if [ $VERBOSE == true ]; then + echo "INFO: Created folders for in $dir" + fi + + # Write a descriptor template file + if [ $TYPE == 'vnfd' ]; then + write_vnfd_tmpl $dir + else + write_nsd_tmpl $dir + # write_nsd_config_tmpl $dir + fi + +fi + +cd $cur_dir -- 2.25.1