Merge changes I1d0ee897,Iee1ed292

* changes:
  VCA CA certificate as env var for LCM
  full_install_osm.sh: escaping spaces in OSMLCM_VCA_PUBKEY

diff --git a/descriptor-packages/tools/generate_descriptor_pkg.sh b/descriptor-packages/tools/generate_descriptor_pkg.sh
index 5ee4d2a..bff8785 100755 (executable)
--- a/descriptor-packages/tools/generate_descriptor_pkg.sh
+++ b/descriptor-packages/tools/generate_descriptor_pkg.sh
@@ -790,6 +790,13 @@ if [ $CREATE == false ]; then
     fi
 
     cd $BASE_DIR
+    # Mrityunjay Yadav: Validate descriptor
+    python /usr/share/osm-devops/descriptor-packages/tools/validate_descriptor.py $DEST_DIR/$PKG/$descriptor
+    if [ $? -ne 0 ]; then
+        rc=$?
+        echo "ERROR: validating descriptor for $PKG ($rc)" >&2
+        exit $rc
+    fi
     if [ $DRY_RUN == false ]; then
         if [ $VERBOSE == true ]; then
             tar zcvf "$DEST_DIR/$PKG.tar.gz" "${PKG}" ${RM}

diff --git a/descriptor-packages/tools/validate_descriptor.py b/descriptor-packages/tools/validate_descriptor.py
index 6ae9836..1c67195 100755 (executable)
--- a/descriptor-packages/tools/validate_descriptor.py
+++ b/descriptor-packages/tools/validate_descriptor.py
@@ -22,6 +22,7 @@ import json
 import yaml
 import sys
 import getopt
+import os
 
 """
 Tests the format of OSM VNFD and NSD descriptors
@@ -46,6 +47,7 @@ def usage():
     print("      -i|--input FILE: (same as param FILE) descriptor file to be upgraded")
     return
 
+
 def remove_prefix(desc, prefix):
     """
     Recursively removes prefix from keys
@@ -68,7 +70,33 @@ def remove_prefix(desc, prefix):
             if isinstance(desc, (list, tuple, dict)):
                 remove_prefix(i, prefix)
 
-if __name__=="__main__":
+
+# Mrityunjay Yadav: Function to verify charm included in VNF Package
+def validate_charm(charm, desc_file):
+    """
+    Verify charm included in VNF Package and raised error if invalid
+    :param charm: vnf-configuration/vdu-configuration
+    :param desc_file: descriptor file
+    :return: None
+    """
+    check_list = ['layer.yaml', 'metadata.yaml', 'actions.yaml', 'actions', 'hooks']
+    charm_name = charm['juju']['charm']
+    charm_dir = os.path.join(os.path.abspath(os.path.dirname(desc_file)), 'charms', charm_name)
+
+    config_primitive = charm.get('config-primitive', [])
+    initial_config_primitive = charm.get('initial-config-primitive', [])
+
+    if charm.get('metrics'):
+        check_list.append('metrics.yaml')
+
+    if os.path.exists(charm_dir):
+        if not all(item in os.listdir(charm_dir) for item in check_list):
+            raise KeyError("Invalid charm {}".format(charm_name))
+    else:
+        raise KeyError("Provided charm:{} does not exist in descriptor.".format(charm_name))
+
+
+if __name__ == "__main__":
     error_position = []
     format_output_yaml = True
     input_file_name = None
@@ -133,10 +161,17 @@ if __name__=="__main__":
                         if interface.get("virtual-interface", {}).get("type") == "OM-MGMT":
                             raise KeyError(
                                 "Wrong 'Virtual-interface type': Deprecated 'OM-MGMT' value. Please, use 'PARAVIRT' instead")
+                    # Mrityunjay yadav: Verify charm if included in vdu
+                    if vdu.get("vdu-configuration", False):
+                        validate_charm(vdu["vdu-configuration"], input_file_name)
                 if vnfd.get("mgmt-interface"):
                     mgmt_iface = True
                     if vnfd["mgmt-interface"].get("vdu-id"):
                         raise KeyError("'mgmt-iface': Deprecated 'vdu-id' field. Please, use 'cp' field instead")
+                # Mrityunjay yadav: Verify charm if included in vnf
+                if vnfd.get("vnf-configuration", False):
+                    validate_charm(vnfd["vnf-configuration"], input_file_name)
+
             if not mgmt_iface:
                 raise KeyError("'mgmt-iface' is a mandatory field and it is not defined")
             myvnfd = vnfd_catalog.vnfd()

diff --git a/docker/RO/Dockerfile b/docker/RO/Dockerfile
index e5c29d1..4240d26 100644 (file)
--- a/docker/RO/Dockerfile
+++ b/docker/RO/Dockerfile
@@ -20,9 +20,6 @@ ARG IM_VERSION
 
 RUN apt-get update && apt-get install -y python-osm-ro${RO_VERSION} python-osm-im${IM_VERSION}
 
-RUN mkdir -p /bin/RO
-
-COPY scripts/ /bin/RO
 
 VOLUME /var/log/osm
 
@@ -51,4 +48,5 @@ ENV RO_DB_OVIM_NAME=mano_vim_db
 
 HEALTHCHECK --start-period=130s --interval=10s --timeout=5s --retries=12 \
   CMD curl --silent --fail localhost:9090/openmano/tenants || exit 1
-CMD /bin/RO/start.sh
+CMD /usr/bin/RO-start.sh
+

diff --git a/docker/light-ui/Dockerfile b/docker/light-ui/Dockerfile
index 2558834..0d317ab 100644 (file)
--- a/docker/light-ui/Dockerfile
+++ b/docker/light-ui/Dockerfile
@@ -1,7 +1,7 @@
 FROM ubuntu:16.04
 
 RUN apt-get update && apt-get install -y npm git python-pip nginx supervisor curl software-properties-common libmysqlclient-dev mysql-client
-RUN npm install -g bower
+#RUN npm install -g bower
 RUN ln -s /usr/bin/nodejs /usr/bin/node
 RUN echo "daemon off;" >> /etc/nginx/nginx.conf
 
@@ -18,7 +18,8 @@ RUN apt-get install -y osm-lightui${LWUI_VERSION} && \
     rm -rf /var/lib/apt/lists/*
 
 WORKDIR /usr/share/osm-lightui
-RUN bower install --allow-root
+#RUN bower install --allow-root
+RUN npm install
 RUN pip install -r requirements.txt
 RUN pip install uwsgi
 RUN cp /usr/share/osm-lightui/nginx-app.conf /etc/nginx/sites-available/default

diff --git a/docker/osmclient/Dockerfile b/docker/osmclient/Dockerfile
index d544a6e..a791b04 100644 (file)
--- a/docker/osmclient/Dockerfile
+++ b/docker/osmclient/Dockerfile
@@ -7,7 +7,7 @@ RUN apt-get update && apt-get -y install curl software-properties-common
 RUN apt-get update && apt-get -y install python \
     libcurl4-gnutls-dev libgnutls-dev iputils-ping python-pip \
     python-openstackclient wget
-RUN pip install python-magic pytest
+RUN pip install python-magic pyangbind pytest==4.6.3
 
 ARG REPOSITORY_BASE=http://osm-download.etsi.org/repository/osm/debian
 ARG RELEASE=ReleaseFOUR-daily
@@ -15,12 +15,13 @@ ARG REPOSITORY_KEY=OSM%20ETSI%20Release%20Key.gpg
 ARG REPOSITORY=testing
 
 RUN curl ${REPOSITORY_BASE}/${RELEASE}/${REPOSITORY_KEY} | apt-key add -
-RUN add-apt-repository -y "deb ${REPOSITORY_BASE}/${RELEASE} ${REPOSITORY} devops osmclient" && apt update
+RUN add-apt-repository -y "deb ${REPOSITORY_BASE}/${RELEASE} ${REPOSITORY} devops osmclient IM" && apt update
 
 ARG OSMCLIENT_VERSION
 ARG DEVOPS_VERSION
+ARG IM_VERSION
 
-RUN apt-get update && apt-get -y install osm-devops${DEVOPS_VERSION} python-osmclient${OSMCLIENT_VERSION}
+RUN apt-get update && apt-get -y install osm-devops${DEVOPS_VERSION} python-osmclient${OSMCLIENT_VERSION} python-osm-im${IM_VERSION}
 
 ENV OSM_SOL005=True
 ENV OSM_HOSTNAME=nbi:9999

diff --git a/installers/docker/docker-compose.yaml b/installers/docker/docker-compose.yaml
index d8866e5..621664a 100644 (file)
--- a/installers/docker/docker-compose.yaml
+++ b/installers/docker/docker-compose.yaml
@@ -60,7 +60,7 @@ services:
     networks:
       - netOSM
   keystone:
-    image: ${DOCKER_USER:-opensourcemano}/keystone:${TAG:-5}
+    image: ${DOCKER_USER:-opensourcemano}/keystone:${TAG:-6}
     networks:
       - netOSM
     environment:
@@ -70,7 +70,7 @@ services:
     ports:
       - "${OSM_KEYSTONE_PORTS:-5000:5000}"
   nbi:
-    image: ${DOCKER_USER:-opensourcemano}/nbi:${TAG:-5}
+    image: ${DOCKER_USER:-opensourcemano}/nbi:${TAG:-6}
     networks:
       - netOSM
     volumes:
@@ -86,7 +86,7 @@ services:
     #  - kafka
     #  - mongo
   lcm:
-    image: ${DOCKER_USER:-opensourcemano}/lcm:${TAG:-5}
+    image: ${DOCKER_USER:-opensourcemano}/lcm:${TAG:-6}
     networks:
       - netOSM
     volumes:
@@ -114,7 +114,7 @@ services:
 #    ports:
 #      - "3306:3306"
   ro:
-    image: ${DOCKER_USER:-opensourcemano}/ro:${TAG:-5}
+    image: ${DOCKER_USER:-opensourcemano}/ro:${TAG:-6}
     networks:
       - netOSM
     environment:
@@ -128,7 +128,7 @@ services:
     ports:
       - "${OSM_RO_PORTS:-9090:9090}"
   mon:
-    image: ${DOCKER_USER:-opensourcemano}/mon:${TAG:-5}
+    image: ${DOCKER_USER:-opensourcemano}/mon:${TAG:-6}
     networks:
       - netOSM
     volumes:
@@ -148,7 +148,7 @@ services:
             max-file: 5
             max-size: 10m
   pol:
-    image: ${DOCKER_USER:-opensourcemano}/pol:${TAG:-5}
+    image: ${DOCKER_USER:-opensourcemano}/pol:${TAG:-6}
     networks:
       - netOSM
     volumes:
@@ -167,7 +167,7 @@ services:
         max-file: 5
         max-size: 10m
   light-ui:
-    image: ${DOCKER_USER:-opensourcemano}/light-ui:${TAG:-5}
+    image: ${DOCKER_USER:-opensourcemano}/light-ui:${TAG:-6}
     networks:
       - netOSM
     environment:

diff --git a/installers/full_install_osm.sh b/installers/full_install_osm.sh
index 89a0ba6..ffeb960 100755 (executable)
--- a/installers/full_install_osm.sh
+++ b/installers/full_install_osm.sh
@@ -181,7 +181,7 @@ EONG
         remove_network $OSM_STACK_NAME
         echo "Removing $OSM_DOCKER_WORK_DIR"
         $WORKDIR_SUDO rm -rf $OSM_DOCKER_WORK_DIR
-        sg lxd -c "juju destroy-controller --yes $OSM_STACK_NAME"
+        sg lxd -c "juju destroy-controller --destroy-all-models --yes $OSM_STACK_NAME"
     fi
     echo "Some docker images will be kept in case they are used by other docker stacks"
     echo "To remove them, just run 'docker image prune' in a terminal"
@@ -1165,7 +1165,7 @@ SHOWOPTS=""
 COMMIT_ID=""
 ASSUME_YES=""
 INSTALL_FROM_SOURCE=""
-RELEASE="ReleaseFIVE"
+RELEASE="ReleaseSIX"
 REPOSITORY="stable"
 INSTALL_VIMEMU=""
 INSTALL_FROM_LXDIMAGES=""

diff --git a/jenkins/ci-pipelines/ci_stage_1.groovy b/jenkins/ci-pipelines/ci_stage_1.groovy
index d74de59..d3c4fa1 100644 (file)
--- a/jenkins/ci-pipelines/ci_stage_1.groovy
+++ b/jenkins/ci-pipelines/ci_stage_1.groovy
@@ -15,6 +15,7 @@
  *   under the License.
  */
 
+stage_3_merge_result = ''
 def Get_MDG(project) {
     // split the project.
     def values = project.split('/')
@@ -88,11 +89,22 @@ node("${params.NODE}") {
 
         println("TEST_INSTALL = ${params.TEST_INSTALL}, downstream job: ${downstream_job_name}")
 
-        stage_2_result = build job: "${downstream_job_name}", parameters: downstream_params, propagate: true
-        if (stage_2_result.getResult() != 'SUCCESS') {
-            project = stage_2_result.getProjectName()
-            build = stage_2_result.getNumber()
-            error("${project} build ${build} failed")
+        stage_3_merge_result = build job: "${downstream_job_name}", parameters: downstream_params, propagate: true
+        if (stage_3_merge_result.getResult() != 'SUCCESS') {
+            project = stage_3_merge_result.getProjectName()
+            build = stage_3_merge_result.getNumber()
+            // Jayant if the build fails the below error will cause the pipeline to terminate. 
+                       // error("${project} build ${build} failed")
+        }
+    }
+       stage('Send Email') {
+        if((stage_3_merge_result.getResult() != 'SUCCESS') && (${env.JOB_NAME} == 'daily-stage_4')){
+            emailext (
+                subject: "[OSM-Jenkins] Job: ${env.JOB_NAME} Build: ${env.BUILD_NUMBER} Result: ${stage_3_merge_result.getResult()}",
+                body: """ Check console output at "${env.BUILD_URL}"  """,
+                to: 'OSM_MDL@list.etsi.org',
+                recipientProviders: [culprits()]
+            )
         }
     }
 }

diff --git a/jenkins/ci-pipelines/ci_stage_3.groovy b/jenkins/ci-pipelines/ci_stage_3.groovy
index 7e9de04..1a4c009 100644 (file)
--- a/jenkins/ci-pipelines/ci_stage_3.groovy
+++ b/jenkins/ci-pipelines/ci_stage_3.groovy
@@ -32,7 +32,7 @@ properties([
         string(defaultValue: 'artifactory-osm', description: '', name: 'ARTIFACTORY_SERVER'),
         string(defaultValue: 'osm-stage_4', description: '', name: 'DOWNSTREAM_STAGE_NAME'),
         string(defaultValue: 'releasesix-daily', description: '', name: 'DOCKER_TAG'),
-        booleanParam(defaultValue: false, description: '', name: 'SAVE_CONTAINER_ON_FAIL'),
+        booleanParam(defaultValue: true, description: '', name: 'SAVE_CONTAINER_ON_FAIL'),
         booleanParam(defaultValue: false, description: '', name: 'SAVE_CONTAINER_ON_PASS'),
         booleanParam(defaultValue: true, description: '', name: 'SAVE_ARTIFACTS_ON_SMOKE_SUCCESS'),
         booleanParam(defaultValue: true, description: '', name: 'DO_STAGE_4'),
@@ -41,7 +41,7 @@ properties([
         booleanParam(defaultValue: true, description: '', name: 'DO_SMOKE'),
         booleanParam(defaultValue: true, description: '', name: 'DO_DOCKERPUSH'),
         booleanParam(defaultValue: false, description: '', name: 'SAVE_ARTIFACTS_OVERRIDE'),
-        string(defaultValue: '/home/jenkins/hive/openstack-whitestack.rc', description: '', name: 'HIVE_VIM_1'),
+        string(defaultValue: '/home/jenkins/hive/openstack-etsi.rc', description: '', name: 'HIVE_VIM_1'),
     ])
 ])
 
@@ -229,7 +229,9 @@ node("${params.NODE}") {
                     {
                         repo_base_url = "-u ${params.REPOSITORY_BASE}"
                     }
-             
+                                       if ( params.DO_STAGE_4 ) {
+                        sh "docker stack list |  awk '{ print \$1 }'| xargs docker stack rm"
+                                       }
                     sh """
                         export PATH=$PATH:/snap/bin
                         installers/full_install_osm.sh -y -s ${container_name} --test --nolxd --nodocker --nojuju --nohostports --nohostclient \
@@ -296,19 +298,22 @@ node("${params.NODE}") {
             currentBuild.result = 'FAILURE'
         }
         finally {
-            sh "docker stop ${http_server_name}"
-            sh "docker rm ${http_server_name}"
+
 
             if ( params.DO_INSTALL ) {
                 if (error) {
                     if ( !params.SAVE_CONTAINER_ON_FAIL ) {
                         uninstall_osm container_name
+                        sh "docker stop ${http_server_name}"
+                        sh "docker rm ${http_server_name}"
                     }
                     throw error 
                 }
                 else {
                     if ( !params.SAVE_CONTAINER_ON_PASS ) {
                         uninstall_osm container_name
+                        sh "docker stop ${http_server_name}"
+                        sh "docker rm ${http_server_name}"
                     }
                 }
             }

diff --git a/tools/license_scan.sh b/tools/license_scan.sh
index 47106e0..f7a4ab7 100755 (executable)
--- a/tools/license_scan.sh
+++ b/tools/license_scan.sh
@@ -15,6 +15,10 @@
 #
 #!/bin/sh
 
+# Change log:
+# 1. Bug 722 : Jayant Madavi : JM00553988@techmahindra.com : Enhancement to use new fossology server. Had to change the variable name at #    couple of places, while scanning the variable name was adding curl as a license.
+# 2.
+ 
 echo GERRIT BRANCH is $GERRIT_BRANCH
 dpkg -l wget &>/dev/null ||sudo apt-get install -y wget
 dpkg -l curl &>/dev/null ||sudo apt-get install -y curl
@@ -32,21 +36,21 @@ RE="FATAL: your file did not get passed through"
 for file in $(git diff --name-only origin/$GERRIT_BRANCH); do
     if [ -f $file ]; then
         if [ -s $file ]; then
-            license=$(wget -qO - --post-file $file https://osm.etsi.org/fossology/?mod=agent_nomos_once |sed "s/^[ \t]*//;s/[ \t]*$//")
-            result=$(echo $license | grep "$RE")
+            licnse=$(curl -s -X POST  -H 'Accept: text' -H 'Cache-Control: no-cache' -H 'Connection: keep-alive'  -H 'Content-Type: multipart/form-data'  -H 'cache-control: no-cache'  -F "file_input=@\"$file\""  -F 'showheader=1' https://fossology-osm.etsi.org/?mod=agent_nomos_once |grep "A one shot license analysis shows the following license(s) in file"|sed -n 's:.*<strong>\(.*\)</strong>.*:\1:p' |xargs)
+            result=$(echo $licnse | grep "$RE")
             if [ -n "$result" ]; then
                 # possibly we have exceeded the post rate
                 sleep 10
-                license=$(wget -qO - --post-file $file https://osm.etsi.org/fossology/?mod=agent_nomos_once |sed "s/^[ \t]*//;s/[ \t]*$//")
+                licnse=$(curl -s -X POST  -H 'Accept: text' -H 'Cache-Control: no-cache' -H 'Connection: keep-alive'  -H 'Content-Type: multipart/form-data'  -H 'cache-control: no-cache'  -F "file_input=@\"$file\""  -F 'showheader=1' https://fossology-osm.etsi.org/?mod=agent_nomos_once |grep "A one shot license analysis shows the following license(s) in file"|sed -n 's:.*<strong>\(.*\)</strong>.*:\1:p' |xargs)
             fi
         else
-            license="No_license_found"
+            licnse="No_license_found"
         fi
     else
-        license="DELETED"
+        licnse="DELETED"
     fi
-    echo "$file $license"
-    case "$license" in
+    echo "$file $licnse"
+    case "$licnse" in
         "Apache-2.0")
             apache=$((apache + 1))
             ;;