From e570b10455cf0ca49f1597e91282ce75c06688ea Mon Sep 17 00:00:00 2001 From: Frank Bryden Date: Mon, 7 Sep 2020 08:45:54 +0000 Subject: [PATCH] PoC of SOL005 Robot conformance tests into OSM's CICD Change-Id: If8373ae545331ae9137c1c8fc3628cae5c56406d Signed-off-by: Frank Bryden --- README_tst010_robot_cicd.md | 50 +++++++++++ conformance-tests/osm_client.py | 61 +++++++++++++ conformance-tests/run_conformance_tests.py | 89 +++++++++++++++++++ .../test-lists/NSDManagement-API.txt | 42 +++++++++ .../test-lists/NSFaultManagement-API.txt | 16 ++++ .../test-lists/NSLifecycleManagement-API.txt | 16 ++++ .../NSPerformanceManagement-API.txt | 16 ++++ .../test-lists/VNFPackageManagement-API.txt | 16 ++++ docker/Dockerfile | 3 + .../resources/basic_12-ns_primitives_data.py | 4 +- robot-systest/run_test.sh | 20 ++++- 11 files changed, 330 insertions(+), 3 deletions(-) create mode 100644 README_tst010_robot_cicd.md create mode 100644 conformance-tests/osm_client.py create mode 100644 conformance-tests/run_conformance_tests.py create mode 100644 conformance-tests/test-lists/NSDManagement-API.txt create mode 100644 conformance-tests/test-lists/NSFaultManagement-API.txt create mode 100644 conformance-tests/test-lists/NSLifecycleManagement-API.txt create mode 100644 conformance-tests/test-lists/NSPerformanceManagement-API.txt create mode 100644 conformance-tests/test-lists/VNFPackageManagement-API.txt diff --git a/README_tst010_robot_cicd.md b/README_tst010_robot_cicd.md new file mode 100644 index 0000000..ff07e5d --- /dev/null +++ b/README_tst010_robot_cicd.md @@ -0,0 +1,50 @@ + + +# Guide to SOL005 integration into OSM's existing CI/CD pipeline +## Add tests to test selection files +### Locate the correct test selection file + +The test selection files are located in `/conformance-tests/test-lists/`. Each API has its own test selection file of the same name (i.e. NSDManagement-API.txt for NSDManagement-API). + +### Test selection files +Each test selection file has the following format +```git l +-t [Test name] +--variable [varName:value] +``` +For example +``` +-t PATCH NSD Content - Method not implemented +-t DELETE NSD Content - Method not implemented +--variable nsdInfoId:$NSD_INFO_ID +--variable nsdInfoIdPlain:$NSD_INFO_ID +``` +For values which need to be created dynamically (like resource IDs), use environment variables (see below). +## Add resource management code +### Resource creation +The test management code resides in `/conformance-tests/run_conformance_tests.py`. +Using the osm client, create the relevant resources and assign the required values to environment variables (using the `os.environ` dictionary). + +### Env variable subbing +The code then replaces the environment variables referred to in the test selection files using the `envsubst` unix command). + +### Resource clearing +Don't forget to clear all created resources at the end of the testing! + +## Reports +The reports are located in `/conformance-tests/reports/{API_NAME}/*`. \ No newline at end of file diff --git a/conformance-tests/osm_client.py b/conformance-tests/osm_client.py new file mode 100644 index 0000000..70831e4 --- /dev/null +++ b/conformance-tests/osm_client.py @@ -0,0 +1,61 @@ +# Copyright 2020 ETSI OSM +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License + +from subprocess import run, PIPE +import requests + +class OSMException(Exception): + """A base class for MyProject exceptions.""" + +class ResourceException(OSMException): + def __init__(self, *args, **kwargs): + super().__init__(*args) + self.message = kwargs.get('message') + +class OSM: + def __init__(self, osm_hostname): + self.osm_hostname = osm_hostname + + def run_command(self, command): + p = run(["osm {}".format(command)], shell=True, stdout=PIPE, stderr=PIPE) + if p.returncode != 0: + print(p.stdout) + print(p.stderr) + raise ResourceException(message=p.stdout) + else: + return self.clean_output(p.stdout) + + def get_auth_token(self): + r = requests.post("https://{}:9999/osm/admin/v1/tokens".format(self.osm_hostname), verify=False, headers={"Accept":"application/json"}, json={ + "username": "admin", + "password": "admin", + "project": "admin" + }) + return r.json()["id"] + + def clean_output(self, output): + return output.decode("utf-8").strip() + + def create_nsd(self, filename): + return self.run_command("nsd-create {}".format(filename)) + + def create_vnfd(self, filename): + return self.run_command("vnfd-create {}".format(filename)) + + def delete_nsd(self, id): + return self.run_command("nsd-delete {}".format(id)) + + def delete_vnfd(self, id): + return self.run_command("vnfd-delete {}".format(id)) \ No newline at end of file diff --git a/conformance-tests/run_conformance_tests.py b/conformance-tests/run_conformance_tests.py new file mode 100644 index 0000000..02a8221 --- /dev/null +++ b/conformance-tests/run_conformance_tests.py @@ -0,0 +1,89 @@ +# Copyright 2020 ETSI OSM +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os +from shutil import copy +from subprocess import run +from osm_client import * + +packages_folder = os.environ["PACKAGES_FOLDER"] +conformance_folder = "{}/{}".format(os.environ["ROBOT_DEVOPS_FOLDER"], "conformance-tests") +test_lists_folder = "{}/{}".format(conformance_folder, "test-lists") +sol005_folder = "{}/repo/SOL005".format(conformance_folder) +osm_hostname = os.environ["OSM_HOSTNAME"] + + +def get_package_path(package_name): + return "{}/{}".format(packages_folder, package_name) + + +def get_suite_path(suite_name): + return "{}/repo/SOL005/{}".format(conformance_folder, suite_name) + + +def run_test_suite(suite_dir, suite_name, arg_file): + print("robot -d {}/repo/{}/reports --argumentfile {} .".format(conformance_folder, suite_name, arg_file)) + run(["robot --loglevel=TRACE -d {}/reports/{} --argumentfile {} .".format(conformance_folder, + suite_name, arg_file)], cwd=suite_dir, shell=True) + + +def sub_env_vars(file_dir, file_name): + run(["envsubst < {0} > {0}".format(file_name)], cwd=file_dir, shell=True) + + +# RESOURCE CREATION HAPPENS BELOW +nsd_id, vnfd_id = "", "" + +osm = OSM(osm_hostname) + +os.environ["AUTH_TOKEN"] = osm.get_auth_token() + +try: + vnfd_id = osm.create_vnfd(get_package_path("hackfest_basic_vnf/hackfest_basic_vnfd.yaml")) + nsd_id = osm.create_nsd(get_package_path("hackfest_basic_ns/hackfest_basic_nsd.yaml")) +except ResourceException as e: + print(e.message) + +print("VNFD: {}\nNSD: {}".format(vnfd_id, nsd_id)) +# Apply relevant env variables (required for test vars) +os.environ["NSD_INFO_ID"] = nsd_id + +# RESOURCE CREATION HAPPENS ABOVE + +# Copy test selection files over to relevant directories +(_, _, filenames) = next(os.walk(test_lists_folder)) +for f in filenames: + if f.endswith(".txt"): + # Apply ENV Variables + sub_env_vars(test_lists_folder, f) + + # Then copy to appropriate directory + print("Copying {} to {}".format(f, get_suite_path(f.split(".")[0]))) + copy("{}/{}".format(test_lists_folder, f), get_suite_path(f.split(".")[0])) + + +# Run the robot tests +(_, directories, _) = next(os.walk(sol005_folder)) +for d in directories: + run_test_suite("{}/{}".format(sol005_folder, d), d, d + ".txt") + +# We then need to clear the created resources +try: + osm.delete_nsd(nsd_id) + osm.delete_vnfd(vnfd_id) +except ResourceException as e: + print("Deletion failed: {}".format(e.message)) + +print("Cleared resources") diff --git a/conformance-tests/test-lists/NSDManagement-API.txt b/conformance-tests/test-lists/NSDManagement-API.txt new file mode 100644 index 0000000..ca2a280 --- /dev/null +++ b/conformance-tests/test-lists/NSDManagement-API.txt @@ -0,0 +1,42 @@ +# Copyright 2020 ETSI OSM +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License + +-t GET Individual Network Service Descriptor Information with invalid resource identifier +-t Disable Individual Network Service Descriptor +-t Enable Individual Network Service Descriptor +-t Enable Individual Network Service Descriptor with conflict due to operational state ENABLED +# -t DELETE Individual Network Service Descriptor in operational state ENABLED +-t POST Individual Network Service Descriptor - Method not implemented +-t PUT Individual Network Service Descriptor - Method not implemented +-t Get single file NSD Content in Plain Format +-t Get NSD Content in Zip Format +-t Get single file NSD Content in Plain or Zip Format +-t Get multi file NSD Content in Plain or Zip Format +-t Get multi file NSD Content in Plain Format +-t Get NSD Content with invalid resource identifier +-t Get NSD Content with conflict due to onboarding state +-t GET NSD Content with Range Request and NFVO not supporting Range Requests +-t Upload NSD Content as Zip file in synchronous mode +-t Upload NSD Content as plain text file in synchronous mode +-t POST NSD Content - Method not implemented +-t PATCH NSD Content - Method not implemented +-t DELETE NSD Content - Method not implemented +--variable nsdInfoId:$NSD_INFO_ID +--variable nsdInfoIdPlain:$NSD_INFO_ID + +--variable NFVO_HOST:$OSM_HOSTNAME +--variable NFVO_PORT:9999 +--variable apiRoot:/osm/ +--variable AUTHORIZATION:Bearer $AUTH_TOKEN \ No newline at end of file diff --git a/conformance-tests/test-lists/NSFaultManagement-API.txt b/conformance-tests/test-lists/NSFaultManagement-API.txt new file mode 100644 index 0000000..07849cf --- /dev/null +++ b/conformance-tests/test-lists/NSFaultManagement-API.txt @@ -0,0 +1,16 @@ +# Copyright 2020 ETSI OSM +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License + +-t NS Fault Alarm Notification \ No newline at end of file diff --git a/conformance-tests/test-lists/NSLifecycleManagement-API.txt b/conformance-tests/test-lists/NSLifecycleManagement-API.txt new file mode 100644 index 0000000..e813b80 --- /dev/null +++ b/conformance-tests/test-lists/NSLifecycleManagement-API.txt @@ -0,0 +1,16 @@ +# Copyright 2020 ETSI OSM +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License + +-t POST Cancel operation task \ No newline at end of file diff --git a/conformance-tests/test-lists/NSPerformanceManagement-API.txt b/conformance-tests/test-lists/NSPerformanceManagement-API.txt new file mode 100644 index 0000000..0073969 --- /dev/null +++ b/conformance-tests/test-lists/NSPerformanceManagement-API.txt @@ -0,0 +1,16 @@ +# Copyright 2020 ETSI OSM +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License + +-t Get Individual Performance Report \ No newline at end of file diff --git a/conformance-tests/test-lists/VNFPackageManagement-API.txt b/conformance-tests/test-lists/VNFPackageManagement-API.txt new file mode 100644 index 0000000..50a59af --- /dev/null +++ b/conformance-tests/test-lists/VNFPackageManagement-API.txt @@ -0,0 +1,16 @@ +# Copyright 2020 ETSI OSM +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License + +-t GET Individual VNF Package Artifact \ No newline at end of file diff --git a/docker/Dockerfile b/docker/Dockerfile index b13f2ea..97b8222 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -35,6 +35,9 @@ RUN git clone https://osm.etsi.org/gitlab/vnf-onboarding/osm-packages.git --recu COPY robot-systest /robot-systest COPY charm.sh /usr/sbin/charm +# Copy conformance tests data +COPY conformance-tests /robot-systest/conformance-tests + # Folder where Robot tests are stored ENV ROBOT_DEVOPS_FOLDER=/robot-systest diff --git a/robot-systest/resources/basic_12-ns_primitives_data.py b/robot-systest/resources/basic_12-ns_primitives_data.py index 128235c..34d09f4 100644 --- a/robot-systest/resources/basic_12-ns_primitives_data.py +++ b/robot-systest/resources/basic_12-ns_primitives_data.py @@ -19,13 +19,13 @@ home = str(Path.home()) # NS and VNF descriptor package files vnfd_pkg1 = 'charm-packages/nscharm_policy_vnf' vnfd_pkg2 = 'charm-packages/nscharm_user_vnf' -nsd_pkg = 'charm-packages/nscharm_ns' +nsd_pkg = 'charm-packages/native_charm_ns' # NSD and VNFD names in OSM vnfd_name1 = 'nscharm-policy-vnf' vnfd_name2 = 'nscharm-user-vnf' nsd_name = 'nscharm-ns' # NS Descriptor file -nsd_file = 'nscharm_nsd.yaml' +nsd_file = 'native_charm_nsd.yaml' # NS instance name ns_name = 'test_nscharm' # SSH keys to be used diff --git a/robot-systest/run_test.sh b/robot-systest/run_test.sh index a279ff0..863cb44 100755 --- a/robot-systest/run_test.sh +++ b/robot-systest/run_test.sh @@ -32,6 +32,12 @@ download_packages(){ git checkout ${PACKAGES}) } +download_tst010(){ + # Fetch conformance tests + git clone --single-branch --branch ${NFV_TESTS_BRANCH} https://forge.etsi.org/rep/nfv/api-tests.git /robot-systest/conformance-tests/repo + python3 -m pip install -r /robot-systest/conformance-tests/repo/requirements.txt +} + create_vim(){ attempts=3 @@ -47,7 +53,7 @@ create_vim(){ ((i++)) if [[ $i -eq 5 ]]; then echo "VIM stuck in PROCESSING after 50 seconds" - osm vim-delete ${VIM_TARGET} + osm vim-delete --force ${VIM_TARGET} sleep 5 break fi @@ -68,6 +74,7 @@ create_vim(){ PARAMS="" +RUN_CONFORMANCE_TESTS=false while (( "$#" )); do case "$1" in @@ -87,6 +94,11 @@ while (( "$#" )); do create_vim shift 1 ;; + -T) + NFV_TESTS_BRANCH=$2 && download_tst010 + RUN_CONFORMANCE_TESTS=true + shift 1 + ;; -h|--help) echo "OSM TESTS TOOL @@ -104,6 +116,7 @@ Options: -o [OPTIONAL]: It is used to specify a particular osmclient version. Default: latest -p [OPTIONAL]: OSM packages repository branch. Default: master -t [OPTIONAL]: Robot tests tags. [sanity, regression, particular_test]. Default: sanity + -T [OPTIONAL]: Run SOL005 Robot conformance tests -c To create a VIM for the tests Volumes: @@ -131,6 +144,11 @@ if [[ -n "$BRANCH_NAME" ]]; then OSMCLIENT=$BRANCH_NAME && install_osmclient fi + +if [ "$RUN_CONFORMANCE_TESTS" = true ] ; then + python3 ${ROBOT_DEVOPS_FOLDER}/conformance-tests/run_conformance_tests.py +fi + if [[ -z "${TEST}" ]]; then printf "Test not provided. \nRunning default test: sanity\n" TEST="sanity" -- 2.17.1