From: madavi Date: Tue, 23 Jul 2019 10:03:18 +0000 (+0200) Subject: Merge changes I1d0ee897,Iee1ed292 X-Git-Tag: v7.0.0rc1~89 X-Git-Url: https://osm.etsi.org/gitweb/?a=commitdiff_plain;h=a1b5bdbd86de48f3bfbf590e6148a311851cde39;hp=0c2f62b7d926648b50f9f99e8c88224c1821b89f;p=osm%2Fdevops.git Merge changes I1d0ee897,Iee1ed292 * changes: VCA CA certificate as env var for LCM full_install_osm.sh: escaping spaces in OSMLCM_VCA_PUBKEY --- diff --git a/descriptor-packages/tools/generate_descriptor_pkg.sh b/descriptor-packages/tools/generate_descriptor_pkg.sh index 5ee4d2a9..bff87854 100755 --- a/descriptor-packages/tools/generate_descriptor_pkg.sh +++ b/descriptor-packages/tools/generate_descriptor_pkg.sh @@ -790,6 +790,13 @@ if [ $CREATE == false ]; then fi cd $BASE_DIR + # Mrityunjay Yadav: Validate descriptor + python /usr/share/osm-devops/descriptor-packages/tools/validate_descriptor.py $DEST_DIR/$PKG/$descriptor + if [ $? -ne 0 ]; then + rc=$? + echo "ERROR: validating descriptor for $PKG ($rc)" >&2 + exit $rc + fi if [ $DRY_RUN == false ]; then if [ $VERBOSE == true ]; then tar zcvf "$DEST_DIR/$PKG.tar.gz" "${PKG}" ${RM} diff --git a/descriptor-packages/tools/validate_descriptor.py b/descriptor-packages/tools/validate_descriptor.py index 6ae9836f..1c67195f 100755 --- a/descriptor-packages/tools/validate_descriptor.py +++ b/descriptor-packages/tools/validate_descriptor.py @@ -22,6 +22,7 @@ import json import yaml import sys import getopt +import os """ Tests the format of OSM VNFD and NSD descriptors @@ -46,6 +47,7 @@ def usage(): print(" -i|--input FILE: (same as param FILE) descriptor file to be upgraded") return + def remove_prefix(desc, prefix): """ Recursively removes prefix from keys @@ -68,7 +70,33 @@ def remove_prefix(desc, prefix): if isinstance(desc, (list, tuple, dict)): remove_prefix(i, prefix) -if __name__=="__main__": + +# Mrityunjay Yadav: Function to verify charm included in VNF Package +def validate_charm(charm, desc_file): + """ + Verify charm included in VNF Package and raised error if invalid + :param charm: vnf-configuration/vdu-configuration + :param desc_file: descriptor file + :return: None + """ + check_list = ['layer.yaml', 'metadata.yaml', 'actions.yaml', 'actions', 'hooks'] + charm_name = charm['juju']['charm'] + charm_dir = os.path.join(os.path.abspath(os.path.dirname(desc_file)), 'charms', charm_name) + + config_primitive = charm.get('config-primitive', []) + initial_config_primitive = charm.get('initial-config-primitive', []) + + if charm.get('metrics'): + check_list.append('metrics.yaml') + + if os.path.exists(charm_dir): + if not all(item in os.listdir(charm_dir) for item in check_list): + raise KeyError("Invalid charm {}".format(charm_name)) + else: + raise KeyError("Provided charm:{} does not exist in descriptor.".format(charm_name)) + + +if __name__ == "__main__": error_position = [] format_output_yaml = True input_file_name = None @@ -133,10 +161,17 @@ if __name__=="__main__": if interface.get("virtual-interface", {}).get("type") == "OM-MGMT": raise KeyError( "Wrong 'Virtual-interface type': Deprecated 'OM-MGMT' value. Please, use 'PARAVIRT' instead") + # Mrityunjay yadav: Verify charm if included in vdu + if vdu.get("vdu-configuration", False): + validate_charm(vdu["vdu-configuration"], input_file_name) if vnfd.get("mgmt-interface"): mgmt_iface = True if vnfd["mgmt-interface"].get("vdu-id"): raise KeyError("'mgmt-iface': Deprecated 'vdu-id' field. Please, use 'cp' field instead") + # Mrityunjay yadav: Verify charm if included in vnf + if vnfd.get("vnf-configuration", False): + validate_charm(vnfd["vnf-configuration"], input_file_name) + if not mgmt_iface: raise KeyError("'mgmt-iface' is a mandatory field and it is not defined") myvnfd = vnfd_catalog.vnfd() diff --git a/docker/RO/Dockerfile b/docker/RO/Dockerfile index e5c29d18..4240d26d 100644 --- a/docker/RO/Dockerfile +++ b/docker/RO/Dockerfile @@ -20,9 +20,6 @@ ARG IM_VERSION RUN apt-get update && apt-get install -y python-osm-ro${RO_VERSION} python-osm-im${IM_VERSION} -RUN mkdir -p /bin/RO - -COPY scripts/ /bin/RO VOLUME /var/log/osm @@ -51,4 +48,5 @@ ENV RO_DB_OVIM_NAME=mano_vim_db HEALTHCHECK --start-period=130s --interval=10s --timeout=5s --retries=12 \ CMD curl --silent --fail localhost:9090/openmano/tenants || exit 1 -CMD /bin/RO/start.sh +CMD /usr/bin/RO-start.sh + diff --git a/docker/light-ui/Dockerfile b/docker/light-ui/Dockerfile index 2558834e..0d317abd 100644 --- a/docker/light-ui/Dockerfile +++ b/docker/light-ui/Dockerfile @@ -1,7 +1,7 @@ FROM ubuntu:16.04 RUN apt-get update && apt-get install -y npm git python-pip nginx supervisor curl software-properties-common libmysqlclient-dev mysql-client -RUN npm install -g bower +#RUN npm install -g bower RUN ln -s /usr/bin/nodejs /usr/bin/node RUN echo "daemon off;" >> /etc/nginx/nginx.conf @@ -18,7 +18,8 @@ RUN apt-get install -y osm-lightui${LWUI_VERSION} && \ rm -rf /var/lib/apt/lists/* WORKDIR /usr/share/osm-lightui -RUN bower install --allow-root +#RUN bower install --allow-root +RUN npm install RUN pip install -r requirements.txt RUN pip install uwsgi RUN cp /usr/share/osm-lightui/nginx-app.conf /etc/nginx/sites-available/default diff --git a/docker/osmclient/Dockerfile b/docker/osmclient/Dockerfile index d544a6ee..a791b049 100644 --- a/docker/osmclient/Dockerfile +++ b/docker/osmclient/Dockerfile @@ -7,7 +7,7 @@ RUN apt-get update && apt-get -y install curl software-properties-common RUN apt-get update && apt-get -y install python \ libcurl4-gnutls-dev libgnutls-dev iputils-ping python-pip \ python-openstackclient wget -RUN pip install python-magic pytest +RUN pip install python-magic pyangbind pytest==4.6.3 ARG REPOSITORY_BASE=http://osm-download.etsi.org/repository/osm/debian ARG RELEASE=ReleaseFOUR-daily @@ -15,12 +15,13 @@ ARG REPOSITORY_KEY=OSM%20ETSI%20Release%20Key.gpg ARG REPOSITORY=testing RUN curl ${REPOSITORY_BASE}/${RELEASE}/${REPOSITORY_KEY} | apt-key add - -RUN add-apt-repository -y "deb ${REPOSITORY_BASE}/${RELEASE} ${REPOSITORY} devops osmclient" && apt update +RUN add-apt-repository -y "deb ${REPOSITORY_BASE}/${RELEASE} ${REPOSITORY} devops osmclient IM" && apt update ARG OSMCLIENT_VERSION ARG DEVOPS_VERSION +ARG IM_VERSION -RUN apt-get update && apt-get -y install osm-devops${DEVOPS_VERSION} python-osmclient${OSMCLIENT_VERSION} +RUN apt-get update && apt-get -y install osm-devops${DEVOPS_VERSION} python-osmclient${OSMCLIENT_VERSION} python-osm-im${IM_VERSION} ENV OSM_SOL005=True ENV OSM_HOSTNAME=nbi:9999 diff --git a/installers/docker/docker-compose.yaml b/installers/docker/docker-compose.yaml index d8866e5b..621664ac 100644 --- a/installers/docker/docker-compose.yaml +++ b/installers/docker/docker-compose.yaml @@ -60,7 +60,7 @@ services: networks: - netOSM keystone: - image: ${DOCKER_USER:-opensourcemano}/keystone:${TAG:-5} + image: ${DOCKER_USER:-opensourcemano}/keystone:${TAG:-6} networks: - netOSM environment: @@ -70,7 +70,7 @@ services: ports: - "${OSM_KEYSTONE_PORTS:-5000:5000}" nbi: - image: ${DOCKER_USER:-opensourcemano}/nbi:${TAG:-5} + image: ${DOCKER_USER:-opensourcemano}/nbi:${TAG:-6} networks: - netOSM volumes: @@ -86,7 +86,7 @@ services: # - kafka # - mongo lcm: - image: ${DOCKER_USER:-opensourcemano}/lcm:${TAG:-5} + image: ${DOCKER_USER:-opensourcemano}/lcm:${TAG:-6} networks: - netOSM volumes: @@ -114,7 +114,7 @@ services: # ports: # - "3306:3306" ro: - image: ${DOCKER_USER:-opensourcemano}/ro:${TAG:-5} + image: ${DOCKER_USER:-opensourcemano}/ro:${TAG:-6} networks: - netOSM environment: @@ -128,7 +128,7 @@ services: ports: - "${OSM_RO_PORTS:-9090:9090}" mon: - image: ${DOCKER_USER:-opensourcemano}/mon:${TAG:-5} + image: ${DOCKER_USER:-opensourcemano}/mon:${TAG:-6} networks: - netOSM volumes: @@ -148,7 +148,7 @@ services: max-file: 5 max-size: 10m pol: - image: ${DOCKER_USER:-opensourcemano}/pol:${TAG:-5} + image: ${DOCKER_USER:-opensourcemano}/pol:${TAG:-6} networks: - netOSM volumes: @@ -167,7 +167,7 @@ services: max-file: 5 max-size: 10m light-ui: - image: ${DOCKER_USER:-opensourcemano}/light-ui:${TAG:-5} + image: ${DOCKER_USER:-opensourcemano}/light-ui:${TAG:-6} networks: - netOSM environment: diff --git a/installers/full_install_osm.sh b/installers/full_install_osm.sh index 89a0ba66..ffeb9605 100755 --- a/installers/full_install_osm.sh +++ b/installers/full_install_osm.sh @@ -181,7 +181,7 @@ EONG remove_network $OSM_STACK_NAME echo "Removing $OSM_DOCKER_WORK_DIR" $WORKDIR_SUDO rm -rf $OSM_DOCKER_WORK_DIR - sg lxd -c "juju destroy-controller --yes $OSM_STACK_NAME" + sg lxd -c "juju destroy-controller --destroy-all-models --yes $OSM_STACK_NAME" fi echo "Some docker images will be kept in case they are used by other docker stacks" echo "To remove them, just run 'docker image prune' in a terminal" @@ -1165,7 +1165,7 @@ SHOWOPTS="" COMMIT_ID="" ASSUME_YES="" INSTALL_FROM_SOURCE="" -RELEASE="ReleaseFIVE" +RELEASE="ReleaseSIX" REPOSITORY="stable" INSTALL_VIMEMU="" INSTALL_FROM_LXDIMAGES="" diff --git a/jenkins/ci-pipelines/ci_stage_1.groovy b/jenkins/ci-pipelines/ci_stage_1.groovy index d74de591..d3c4fa1b 100644 --- a/jenkins/ci-pipelines/ci_stage_1.groovy +++ b/jenkins/ci-pipelines/ci_stage_1.groovy @@ -15,6 +15,7 @@ * under the License. */ +stage_3_merge_result = '' def Get_MDG(project) { // split the project. def values = project.split('/') @@ -88,11 +89,22 @@ node("${params.NODE}") { println("TEST_INSTALL = ${params.TEST_INSTALL}, downstream job: ${downstream_job_name}") - stage_2_result = build job: "${downstream_job_name}", parameters: downstream_params, propagate: true - if (stage_2_result.getResult() != 'SUCCESS') { - project = stage_2_result.getProjectName() - build = stage_2_result.getNumber() - error("${project} build ${build} failed") + stage_3_merge_result = build job: "${downstream_job_name}", parameters: downstream_params, propagate: true + if (stage_3_merge_result.getResult() != 'SUCCESS') { + project = stage_3_merge_result.getProjectName() + build = stage_3_merge_result.getNumber() + // Jayant if the build fails the below error will cause the pipeline to terminate. + // error("${project} build ${build} failed") + } + } + stage('Send Email') { + if((stage_3_merge_result.getResult() != 'SUCCESS') && (${env.JOB_NAME} == 'daily-stage_4')){ + emailext ( + subject: "[OSM-Jenkins] Job: ${env.JOB_NAME} Build: ${env.BUILD_NUMBER} Result: ${stage_3_merge_result.getResult()}", + body: """ Check console output at "${env.BUILD_URL}" """, + to: 'OSM_MDL@list.etsi.org', + recipientProviders: [culprits()] + ) } } } diff --git a/jenkins/ci-pipelines/ci_stage_3.groovy b/jenkins/ci-pipelines/ci_stage_3.groovy index 7e9de044..1a4c0091 100644 --- a/jenkins/ci-pipelines/ci_stage_3.groovy +++ b/jenkins/ci-pipelines/ci_stage_3.groovy @@ -32,7 +32,7 @@ properties([ string(defaultValue: 'artifactory-osm', description: '', name: 'ARTIFACTORY_SERVER'), string(defaultValue: 'osm-stage_4', description: '', name: 'DOWNSTREAM_STAGE_NAME'), string(defaultValue: 'releasesix-daily', description: '', name: 'DOCKER_TAG'), - booleanParam(defaultValue: false, description: '', name: 'SAVE_CONTAINER_ON_FAIL'), + booleanParam(defaultValue: true, description: '', name: 'SAVE_CONTAINER_ON_FAIL'), booleanParam(defaultValue: false, description: '', name: 'SAVE_CONTAINER_ON_PASS'), booleanParam(defaultValue: true, description: '', name: 'SAVE_ARTIFACTS_ON_SMOKE_SUCCESS'), booleanParam(defaultValue: true, description: '', name: 'DO_STAGE_4'), @@ -41,7 +41,7 @@ properties([ booleanParam(defaultValue: true, description: '', name: 'DO_SMOKE'), booleanParam(defaultValue: true, description: '', name: 'DO_DOCKERPUSH'), booleanParam(defaultValue: false, description: '', name: 'SAVE_ARTIFACTS_OVERRIDE'), - string(defaultValue: '/home/jenkins/hive/openstack-whitestack.rc', description: '', name: 'HIVE_VIM_1'), + string(defaultValue: '/home/jenkins/hive/openstack-etsi.rc', description: '', name: 'HIVE_VIM_1'), ]) ]) @@ -229,7 +229,9 @@ node("${params.NODE}") { { repo_base_url = "-u ${params.REPOSITORY_BASE}" } - + if ( params.DO_STAGE_4 ) { + sh "docker stack list | awk '{ print \$1 }'| xargs docker stack rm" + } sh """ export PATH=$PATH:/snap/bin installers/full_install_osm.sh -y -s ${container_name} --test --nolxd --nodocker --nojuju --nohostports --nohostclient \ @@ -296,19 +298,22 @@ node("${params.NODE}") { currentBuild.result = 'FAILURE' } finally { - sh "docker stop ${http_server_name}" - sh "docker rm ${http_server_name}" + if ( params.DO_INSTALL ) { if (error) { if ( !params.SAVE_CONTAINER_ON_FAIL ) { uninstall_osm container_name + sh "docker stop ${http_server_name}" + sh "docker rm ${http_server_name}" } throw error } else { if ( !params.SAVE_CONTAINER_ON_PASS ) { uninstall_osm container_name + sh "docker stop ${http_server_name}" + sh "docker rm ${http_server_name}" } } } diff --git a/tools/license_scan.sh b/tools/license_scan.sh index 47106e0e..f7a4ab7e 100755 --- a/tools/license_scan.sh +++ b/tools/license_scan.sh @@ -15,6 +15,10 @@ # #!/bin/sh +# Change log: +# 1. Bug 722 : Jayant Madavi : JM00553988@techmahindra.com : Enhancement to use new fossology server. Had to change the variable name at # couple of places, while scanning the variable name was adding curl as a license. +# 2. + echo GERRIT BRANCH is $GERRIT_BRANCH dpkg -l wget &>/dev/null ||sudo apt-get install -y wget dpkg -l curl &>/dev/null ||sudo apt-get install -y curl @@ -32,21 +36,21 @@ RE="FATAL: your file did not get passed through" for file in $(git diff --name-only origin/$GERRIT_BRANCH); do if [ -f $file ]; then if [ -s $file ]; then - license=$(wget -qO - --post-file $file https://osm.etsi.org/fossology/?mod=agent_nomos_once |sed "s/^[ \t]*//;s/[ \t]*$//") - result=$(echo $license | grep "$RE") + licnse=$(curl -s -X POST -H 'Accept: text' -H 'Cache-Control: no-cache' -H 'Connection: keep-alive' -H 'Content-Type: multipart/form-data' -H 'cache-control: no-cache' -F "file_input=@\"$file\"" -F 'showheader=1' https://fossology-osm.etsi.org/?mod=agent_nomos_once |grep "A one shot license analysis shows the following license(s) in file"|sed -n 's:.*\(.*\).*:\1:p' |xargs) + result=$(echo $licnse | grep "$RE") if [ -n "$result" ]; then # possibly we have exceeded the post rate sleep 10 - license=$(wget -qO - --post-file $file https://osm.etsi.org/fossology/?mod=agent_nomos_once |sed "s/^[ \t]*//;s/[ \t]*$//") + licnse=$(curl -s -X POST -H 'Accept: text' -H 'Cache-Control: no-cache' -H 'Connection: keep-alive' -H 'Content-Type: multipart/form-data' -H 'cache-control: no-cache' -F "file_input=@\"$file\"" -F 'showheader=1' https://fossology-osm.etsi.org/?mod=agent_nomos_once |grep "A one shot license analysis shows the following license(s) in file"|sed -n 's:.*\(.*\).*:\1:p' |xargs) fi else - license="No_license_found" + licnse="No_license_found" fi else - license="DELETED" + licnse="DELETED" fi - echo "$file $license" - case "$license" in + echo "$file $licnse" + case "$licnse" in "Apache-2.0") apache=$((apache + 1)) ;;