From 70de2aff30e8a45b50d01d8dd8845e0a8d060fe4 Mon Sep 17 00:00:00 2001 From: beierlm Date: Tue, 23 Mar 2021 16:04:34 -0400 Subject: [PATCH] Feature 10296 Pip Standardization Cleanup of Dockerfile Moved requirements from devops/docker/tests/Dockerfile into this module so they are visible. Added pylint and safety hooks. Added place for unit tests, but none are written yet, so does not run them. Change-Id: I75ce40c558bd9bc8975269dca681ba84678e6de6 Signed-off-by: beierlm --- .gitignore | 3 + Dockerfile | 22 +++++-- build-debpkg.sh | 2 +- requirements-dev.in | 17 +++++ requirements-dev.txt | 32 ++++++++++ requirements-dist.in | 17 +++++ requirements-dist.txt | 23 +++++++ requirements-test.in | 24 +++++++ requirements-test.txt | 26 ++++++++ requirements.in | 22 +++++++ requirements.txt | 48 ++++++++++++++ robot-systest/lib/ns_lib.robot | 21 +++++- robot-systest/resources/__init__.py | 17 +++++ setup.py | 45 +++++++++++++ tox.ini | 99 +++++++++++++++++++++++++++++ 15 files changed, 408 insertions(+), 10 deletions(-) create mode 100644 requirements-dev.in create mode 100644 requirements-dev.txt create mode 100644 requirements-dist.in create mode 100644 requirements-dist.txt create mode 100644 requirements-test.in create mode 100644 requirements-test.txt create mode 100644 requirements.in create mode 100644 requirements.txt create mode 100644 robot-systest/resources/__init__.py create mode 100644 setup.py create mode 100644 tox.ini diff --git a/.gitignore b/.gitignore index 3384736..9b3d525 100644 --- a/.gitignore +++ b/.gitignore @@ -19,6 +19,8 @@ #IDEs .idea .vscode +.project +.pydevproject #Generated folders when building the deb package locally .tox @@ -36,3 +38,4 @@ dist local reports +envfile.rc diff --git a/Dockerfile b/Dockerfile index 65c2c78..b949260 100644 --- a/Dockerfile +++ b/Dockerfile @@ -11,13 +11,21 @@ # See the License for the specific language governing permissions and # limitations under the License. -# This Dockerfile is intented for devops and deb package generation +######################################################################################## +# This Dockerfile is intented for devops testing and deb package generation +# +# To run stage 2 locally: +# +# docker build -t stage2 . +# docker run -ti -v `pwd`:/work -w /work --entrypoint /bin/bash stage2 +# devops-stages/stage-test.sh +# devops-stages/stage-build.sh # -# Use Dockerfile.local for running osm/NBI in a docker container from source -# Use Dockerfile.fromdeb for running osm/NBI in a docker container from last stable package - -FROM ubuntu:16.04 +FROM ubuntu:18.04 -RUN apt-get update && apt-get -y install git make libcurl4-gnutls-dev \ - libgnutls-dev debhelper apt-utils dh-make +RUN DEBIAN_FRONTEND=noninteractive apt-get update && \ + DEBIAN_FRONTEND=noninteractive apt-get -y install \ + debhelper \ + dh-make \ + git diff --git a/build-debpkg.sh b/build-debpkg.sh index 61cf51b..bd8955a 100755 --- a/build-debpkg.sh +++ b/build-debpkg.sh @@ -16,7 +16,7 @@ PKG_DIRECTORIES="robot-systest" -PKG_FILES="CONTRIBUTING.md LICENSE README.md charm.sh" +PKG_FILES="CONTRIBUTING.md LICENSE README.md charm.sh requirements.txt" MDG_NAME=tests DEB_INSTALL=debian/osm-${MDG_NAME}.install export DEBEMAIL="gerardo.garciadeblas@telefonica.com" diff --git a/requirements-dev.in b/requirements-dev.in new file mode 100644 index 0000000..3ae8b5f --- /dev/null +++ b/requirements-dev.in @@ -0,0 +1,17 @@ +# Copyright ETSI Contributors and Others. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +git+https://osm.etsi.org/gerrit/osm/common.git@v9.0#egg=osm-common +-r https://osm.etsi.org/gitweb/?p=osm/common.git;a=blob_plain;f=requirements.txt;hb=v9.0 diff --git a/requirements-dev.txt b/requirements-dev.txt new file mode 100644 index 0000000..816639a --- /dev/null +++ b/requirements-dev.txt @@ -0,0 +1,32 @@ +aiokafka==0.7.0 + # via -r https://osm.etsi.org/gitweb/?p=osm/common.git;a=blob_plain;f=requirements.txt;hb=v9.0 +dataclasses==0.6 + # via -r https://osm.etsi.org/gitweb/?p=osm/common.git;a=blob_plain;f=requirements.txt;hb=v9.0 +kafka-python==2.0.2 + # via + # -r https://osm.etsi.org/gitweb/?p=osm/common.git;a=blob_plain;f=requirements.txt;hb=v9.0 + # aiokafka +git+https://osm.etsi.org/gerrit/osm/common.git@v9.0#egg=osm-common + # via -r requirements-dev.in +pycrypto==2.6.1 + # via -r https://osm.etsi.org/gitweb/?p=osm/common.git;a=blob_plain;f=requirements.txt;hb=v9.0 +pymongo==3.11.3 + # via -r https://osm.etsi.org/gitweb/?p=osm/common.git;a=blob_plain;f=requirements.txt;hb=v9.0 +pyyaml==5.4.1 + # via -r https://osm.etsi.org/gitweb/?p=osm/common.git;a=blob_plain;f=requirements.txt;hb=v9.0 +####################################################################################### +# Copyright ETSI Contributors and Others. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. +####################################################################################### diff --git a/requirements-dist.in b/requirements-dist.in new file mode 100644 index 0000000..11f0a2a --- /dev/null +++ b/requirements-dist.in @@ -0,0 +1,17 @@ +# Copyright ETSI Contributors and Others. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +stdeb +setuptools-version-command \ No newline at end of file diff --git a/requirements-dist.txt b/requirements-dist.txt new file mode 100644 index 0000000..7393626 --- /dev/null +++ b/requirements-dist.txt @@ -0,0 +1,23 @@ +setuptools-version-command==2.2 + # via -r requirements-dist.in +stdeb==0.10.0 + # via -r requirements-dist.in + +# The following packages are considered to be unsafe in a requirements file: +# setuptools +####################################################################################### +# Copyright ETSI Contributors and Others. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. +####################################################################################### diff --git a/requirements-test.in b/requirements-test.in new file mode 100644 index 0000000..623556d --- /dev/null +++ b/requirements-test.in @@ -0,0 +1,24 @@ +# Copyright 2017 Intel Research and Development Ireland Limited +# ************************************************************* + +# This file is part of OSM Monitoring module +# All Rights Reserved to Intel Corporation + +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at + +# http://www.apache.org/licenses/LICENSE-2.0 + +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +# For those usages not covered by the Apache License, Version 2.0 please +# contact: helena.mcgough@intel.com or adrian.hoban@intel.com +## +coverage +mock +nose2 diff --git a/requirements-test.txt b/requirements-test.txt new file mode 100644 index 0000000..d0c984f --- /dev/null +++ b/requirements-test.txt @@ -0,0 +1,26 @@ +coverage==5.5 + # via + # -r requirements-test.in + # nose2 +mock==4.0.3 + # via -r requirements-test.in +nose2==0.10.0 + # via -r requirements-test.in +six==1.15.0 + # via nose2 +####################################################################################### +# Copyright ETSI Contributors and Others. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. +####################################################################################### diff --git a/requirements.in b/requirements.in new file mode 100644 index 0000000..8ad3006 --- /dev/null +++ b/requirements.in @@ -0,0 +1,22 @@ +# Copyright ETSI Contributors and Others. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +aiokafka +peewee==3.8.* +jsonschema==2.6.* +pyyaml +pymysql==0.9.* +peewee-migrate==1.1.* +requests==2.* diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..0fbd162 --- /dev/null +++ b/requirements.txt @@ -0,0 +1,48 @@ +aiokafka==0.7.0 + # via -r requirements.in +cached-property==1.5.2 + # via peewee-migrate +certifi==2020.12.5 + # via requests +chardet==4.0.0 + # via requests +click==7.1.2 + # via peewee-migrate +idna==2.10 + # via requests +jsonschema==2.6.0 + # via -r requirements.in +kafka-python==2.0.2 + # via aiokafka +mock==4.0.3 + # via peewee-migrate +peewee-migrate==1.1.6 + # via -r requirements.in +peewee==3.8.2 + # via + # -r requirements.in + # peewee-migrate +pymysql==0.9.3 + # via -r requirements.in +pyyaml==5.4.1 + # via -r requirements.in +requests==2.25.1 + # via -r requirements.in +urllib3==1.26.4 + # via requests +####################################################################################### +# Copyright ETSI Contributors and Others. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. +####################################################################################### diff --git a/robot-systest/lib/ns_lib.robot b/robot-systest/lib/ns_lib.robot index 065a562..501bee2 100644 --- a/robot-systest/lib/ns_lib.robot +++ b/robot-systest/lib/ns_lib.robot @@ -117,6 +117,8 @@ Get Vnf Ip List Check For Ns Instance To Configured [arguments] ${ns_name} + ${rc} ${stdout}= Run and Return RC and Output openstack server list + log ${stdout} ${rc} ${stdout}= run and return rc and output osm ns-list --filter name="${ns_name}" log ${stdout} Should Be Equal As Integers ${rc} ${success_return_code} @@ -125,6 +127,8 @@ Check For Ns Instance To Configured Check For NS Instance For Failure [Arguments] ${ns_name} + ${rc} ${stdout}= Run and Return RC and Output openstack server list + log ${stdout} ${rc} ${stdout}= Run and Return RC and Output osm ns-list --filter name="${ns_name}" log ${stdout} Should Be Equal As Integers ${rc} ${success_return_code} @@ -133,6 +137,8 @@ Check For NS Instance For Failure Check For NS Instance To Be Deleted [Arguments] ${ns} + ${rc} ${stdout}= Run and Return RC and Output openstack server list + log ${stdout} ${rc} ${stdout}= Run and Return RC and Output osm ns-list | awk '{print $2}' | grep ${ns} Should Not Be Equal As Strings ${stdout} ${ns} @@ -233,7 +239,7 @@ Check For NS Operation Completed [Arguments] ${ns_operation_id} - ${rc} ${stdout}= Run and Return RC and Output osm ns-op-show ${ns_operation_id} --literal | yq r - operationState + ${rc} ${stdout}= Run and Return RC and Output osm ns-op-show ${ns_operation_id} --literal | yq .operationState | tr -d \\" log ${stdout} Should Be Equal As Integers ${rc} ${success_return_code} Should Contain ${stdout} COMPLETED msg=Timeout waiting for ns-action with id ${ns_operation_id} values=False @@ -255,8 +261,19 @@ Get Vnf Vdur Names [Arguments] ${vnf_id} - ${rc} ${stdout}= Run and Return RC and Output osm vnf-show ${vnf_id} --literal | yq r - vdur.*.name + ${rc} ${stdout}= Run and Return RC and Output osm vnf-show ${vnf_id} --literal | yq .vdur[].name | tr -d \\" Should Be Equal As Integers ${rc} ${success_return_code} msg=${stdout} values=False @{vdur} = Split String ${stdout} [Return] @{vdur} + +Get Vnf Kdu Replica Count + [Documentation] Return the number of KDU replicas (empty if none) of a VNF instance. + + [Arguments] ${vnf_id} ${kdu_name} + + ${rc} ${stdout}= Run and Return RC and Output osm vnf-show ${vnf_id} --kdu ${kdu_name} | yq .config.replicaCount | tr -d \\" + Should Be Equal As Integers ${rc} ${success_return_code} msg=${stdout} values=False + ${return} = Set Variable If '${stdout}' == 'null' ${EMPTY} ${stdout} + [Return] ${return} + diff --git a/robot-systest/resources/__init__.py b/robot-systest/resources/__init__.py new file mode 100644 index 0000000..d10c534 --- /dev/null +++ b/robot-systest/resources/__init__.py @@ -0,0 +1,17 @@ +####################################################################################### +# Copyright ETSI Contributors and Others. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. +####################################################################################### + diff --git a/setup.py b/setup.py new file mode 100644 index 0000000..2049a93 --- /dev/null +++ b/setup.py @@ -0,0 +1,45 @@ +####################################################################################### +# Copyright ETSI Contributors and Others. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. +####################################################################################### + +from setuptools import setup + +_name = 'tests' +_version_command = ('git describe --match v* --tags --long --dirty', 'pep440-git-full') +_author = "OSM Support" +_author_email = 'osmsupport@etsi.org' +_description = 'OSM Tests' +_maintainer = 'OSM Support' +_maintainer_email = 'osmsupport@etsi.org' +_license = 'Apache 2.0' +_url = 'https://osm.etsi.org/gitweb/?p=osm/tests.git;a=tree' + +setup( + name=_name, + version_command=_version_command, + description=_description, + long_description=open('README.md', encoding='utf-8').read(), + author=_author, + author_email=_author_email, + maintainer=_maintainer, + maintainer_email=_maintainer_email, + url=_url, + license=_license, + packages=[_name], + package_dir={_name: 'robot-systest'}, + include_package_data=True, + setup_requires=['setuptools-version-command'] +) diff --git a/tox.ini b/tox.ini new file mode 100644 index 0000000..4f67210 --- /dev/null +++ b/tox.ini @@ -0,0 +1,99 @@ +####################################################################################### +# Copyright ETSI Contributors and Others. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. +####################################################################################### + +[tox] +envlist = black, cover, flake8, pylint, safety + +[tox:jenkins] +toxworkdir = /tmp/.tox + +[testenv] +usedevelop = True +basepython = python3 +setenv = VIRTUAL_ENV={envdir} + PYTHONDONTWRITEBYTECODE = 1 +deps = -r{toxinidir}/requirements.txt + +####################################################################################### +[testenv:black] +deps = black +skip_install = true +commands = + - black --check --diff robot-systest --fast + + +####################################################################################### +[testenv:cover] +commands = + sh -c 'echo No unit tests' +whitelist_externals = sh + + +####################################################################################### +[testenv:flake8] +deps = flake8 +commands = + flake8 robot-systest + + +####################################################################################### +[testenv:pylint] +deps = {[testenv]deps} + -r{toxinidir}/requirements-dev.txt + pylint +commands = + pylint -E robot-systest/resources + + +####################################################################################### +[testenv:safety] +setenv = + LC_ALL=C.UTF-8 + LANG=C.UTF-8 +deps = {[testenv]deps} + safety +commands = + - safety check --full-report + + +####################################################################################### +[testenv:pip-compile] +deps = pip-tools==5.5.0 +commands = + - sh -c 'for file in requirements*.in ; do pip-compile -rU --no-header $file ;\ + out=`echo $file | sed "s/.in/.txt/"` ; \ + head -16 tox.ini >> $out ;\ + done' +whitelist_externals = sh + + +####################################################################################### +[flake8] +ignore = + W291, + W293, + E123, + E125, + E226, + E241 +exclude = + .git, + __pycache__, + .tox, +max-line-length = 120 +show-source = True +builtins = _ -- 2.25.1