From 4335f9962fec0516628b5d0b8337fc977e55669c Mon Sep 17 00:00:00 2001 From: beierlm Date: Wed, 24 Mar 2021 09:26:06 -0400 Subject: [PATCH] Feature 10296 Pip Standardization Creates standard template for tox.ini Introduces 'safety' for listing upstream CVEs Pins all versions of all upstream modules Change-Id: I20f3b8b1c386aa03ebbec37ab74bc106451328d1 Signed-off-by: beierlm --- .gitignore | 4 + Dockerfile | 48 ++++-- MANIFEST.in | 2 +- debian/python3-osmclient.postinst | 4 +- devops-stages/stage-build.sh | 2 +- devops-stages/stage-test.sh | 2 +- osmclient/scripts/osm.py | 10 +- osmclient/sol005/client.py | 4 +- osmclient/sol005/vim.py | 2 +- osmclient/sol005/vnfd.py | 3 +- osmclient/v1/vim.py | 2 +- requirements-dev.in | 16 ++ requirements-dev.txt | 18 +++ .../py3dist-overrides => requirements-dist.in | 6 +- requirements-dist.txt | 23 +++ requirements-test.in | 21 +++ requirements-test.txt | 26 +++ test-requirements.txt => requirements.in | 11 +- requirements.txt | 68 +++++--- setup.py | 19 +-- stdeb.cfg | 3 - tox.ini | 148 ++++++++++++++---- 22 files changed, 335 insertions(+), 107 deletions(-) create mode 100644 requirements-dev.in create mode 100644 requirements-dev.txt rename debian/py3dist-overrides => requirements-dist.in (87%) create mode 100644 requirements-dist.txt create mode 100755 requirements-test.in create mode 100644 requirements-test.txt rename test-requirements.txt => requirements.in (88%) diff --git a/.gitignore b/.gitignore index b4c9ff5..da10ebb 100644 --- a/.gitignore +++ b/.gitignore @@ -32,3 +32,7 @@ stage/ snap/.snapcraft/ .vscode build/ +.project +.pydevproject +.settings/ + diff --git a/Dockerfile b/Dockerfile index 8c16461..99a85a0 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,19 +1,41 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at # -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at +# http://www.apache.org/licenses/LICENSE-2.0 # -# http://www.apache.org/licenses/LICENSE-2.0 +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. +######################################################################################## +# This Dockerfile is intented for devops testing and deb package generation # -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. +# To run stage 2 locally: # +# docker build -t stage2 . +# docker run -ti -v `pwd`:/work -w /work --entrypoint /bin/bash stage2 +# devops-stages/stage-test.sh +# devops-stages/stage-build.sh +# + FROM ubuntu:18.04 -RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get -y install git \ - make python3 python3-pip libcurl4-openssl-dev libssl-dev tox python3-dev \ - debhelper python3-setuptools python3-all python-all python-pip \ - apt-utils wget +RUN DEBIAN_FRONTEND=noninteractive apt-get update && \ + DEBIAN_FRONTEND=noninteractive apt-get -y install \ + debhelper \ + git \ + python3 \ + python3-all \ + python3-dev \ + python3-setuptools + +RUN python3 -m easy_install pip==21.0.1 +RUN pip3 install tox==3.22.0 + +RUN DEBIAN_FRONTEND=noninteractive apt-get -y install \ + libcurl4-openssl-dev \ + libssl-dev \ + wget diff --git a/MANIFEST.in b/MANIFEST.in index 4deac18..bff0dde 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -14,4 +14,4 @@ # License for the specific language governing permissions and limitations # under the License. -include osmclient/templates/*.yaml.j2 \ No newline at end of file +include osmclient/templates/*.yaml.j2 osmclient/requirements.txt \ No newline at end of file diff --git a/debian/python3-osmclient.postinst b/debian/python3-osmclient.postinst index 75ec83f..f78b5cb 100644 --- a/debian/python3-osmclient.postinst +++ b/debian/python3-osmclient.postinst @@ -15,8 +15,7 @@ ## echo "POST INSTALL OSMCLIENT" -#Install pyangbind, required for python3-osm-im -python3 -m pip install pyangbind verboselogs + #configure autocomplete for osmclient [ -z "$SUDO_USER" ] && SUDO_USER="$USER" su $SUDO_USER -c 'mkdir -p $HOME/.bash_completion.d' @@ -27,4 +26,3 @@ then echo " inserting .bash_completion.d/osm-complete.sh execution at .bashrc" su $SUDO_USER -c 'echo ". ${HOME}/.bash_completion.d/osm-complete.sh" >> ${HOME}/.bashrc' fi - diff --git a/devops-stages/stage-build.sh b/devops-stages/stage-build.sh index f00cd3e..164b385 100755 --- a/devops-stages/stage-build.sh +++ b/devops-stages/stage-build.sh @@ -14,4 +14,4 @@ # limitations under the License. rm -rf deb_dist osmclient-*.tar.gz -tox -e build +tox -e dist diff --git a/devops-stages/stage-test.sh b/devops-stages/stage-test.sh index 0177ba3..a3f8e18 100755 --- a/devops-stages/stage-test.sh +++ b/devops-stages/stage-test.sh @@ -11,5 +11,5 @@ # implied. # See the License for the specific language governing permissions and # limitations under the License. -tox --recreate +tox diff --git a/osmclient/scripts/osm.py b/osmclient/scripts/osm.py index 603d373..4af0dad 100755 --- a/osmclient/scripts/osm.py +++ b/osmclient/scripts/osm.py @@ -384,7 +384,7 @@ def ns_list(ctx, filter, long): project_list = ctx.obj.project.list() try: vim_list = ctx.obj.vim.list() - except: + except Exception: vim_list = [] else: table = PrettyTable( @@ -3756,7 +3756,8 @@ def sdnc_show(ctx, name): @click.option( "--k8s-nets", prompt=True, - help='list of VIM networks, in JSON inline format, where the cluster is accessible via L3 routing, e.g. "{(k8s_net1:vim_network1) [,(k8s_net2:vim_network2) ...]}"', + help='''list of VIM networks, in JSON inline format, where the cluster is + accessible via L3 routing, e.g. "{(k8s_net1:vim_network1) [,(k8s_net2:vim_network2) ...]}"''', ) @click.option("--description", default=None, help="human readable description") @click.option( @@ -3812,7 +3813,8 @@ def k8scluster_add( @click.option("--vim", help="VIM target, the VIM where the cluster resides") @click.option( "--k8s-nets", - help='list of VIM networks, in JSON inline format, where the cluster is accessible via L3 routing, e.g. "{(k8s_net1:vim_network1) [,(k8s_net2:vim_network2) ...]}"', + help='''list of VIM networks, in JSON inline format, where the cluster is accessible + via L3 routing, e.g. "{(k8s_net1:vim_network1) [,(k8s_net2:vim_network2) ...]}"''', ) @click.option("--description", help="human readable description") @click.option( @@ -3920,7 +3922,7 @@ def k8scluster_list(ctx, filter, literal, long): ) try: vim_list = ctx.obj.vim.list() - except: + except Exception: vim_list = [] for cluster in resp: logger.debug("Cluster details: {}".format(yaml.safe_dump(cluster))) diff --git a/osmclient/sol005/client.py b/osmclient/sol005/client.py index 495c425..36062bf 100644 --- a/osmclient/sol005/client.py +++ b/osmclient/sol005/client.py @@ -149,9 +149,9 @@ class Client(object): def set_default_params(self, **kwargs): host = kwargs.pop("host", None) - if host != None: + if host is not None: self._host = host port = kwargs.pop("port", None) - if port != None: + if port is not None: self._so_port = port self._http_client.set_query_admin(**kwargs) diff --git a/osmclient/sol005/vim.py b/osmclient/sol005/vim.py index f2a11e6..e5bf399 100644 --- a/osmclient/sol005/vim.py +++ b/osmclient/sol005/vim.py @@ -75,7 +75,7 @@ class Vim(object): self._logger.debug("") self._client.get_token() if "vim-type" not in vim_access: - #'openstack' not in vim_access['vim-type']): + # 'openstack' not in vim_access['vim-type']): raise Exception("vim type not provided") vim_account = {} diff --git a/osmclient/sol005/vnfd.py b/osmclient/sol005/vnfd.py index 4542d56..39f4c59 100644 --- a/osmclient/sol005/vnfd.py +++ b/osmclient/sol005/vnfd.py @@ -282,7 +282,8 @@ class Vnfd(object): ): special_override_string = ( "{}vdu.{}.int-cpd.{}.virtual-network-interface-" - "requirement.{}.virtual-interface.type=PARAVIRT;".format( + "requirement.{}.virtual-interface.type=" + "PARAVIRT;".format( special_override_string, vdu_number, cpd_number, diff --git a/osmclient/v1/vim.py b/osmclient/v1/vim.py index 35266aa..1e8b604 100644 --- a/osmclient/v1/vim.py +++ b/osmclient/v1/vim.py @@ -60,7 +60,7 @@ class Vim(object): # currently assumes vim_acc if "vim-type" not in vim_access: - #'openstack' not in vim_access['vim-type']): + # 'openstack' not in vim_access['vim-type']): raise Exception("vim type not provided") vim_account["datacenter"]["name"] = name diff --git a/requirements-dev.in b/requirements-dev.in new file mode 100644 index 0000000..620d427 --- /dev/null +++ b/requirements-dev.in @@ -0,0 +1,16 @@ +# Copyright ETSI Contributors and Others. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +git+https://osm.etsi.org/gerrit/osm/IM.git@master#egg=osm-im diff --git a/requirements-dev.txt b/requirements-dev.txt new file mode 100644 index 0000000..cd3f5bb --- /dev/null +++ b/requirements-dev.txt @@ -0,0 +1,18 @@ +git+https://osm.etsi.org/gerrit/osm/IM.git@master#egg=osm-im + # via -r requirements-dev.in +####################################################################################### +# Copyright ETSI Contributors and Others. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. +####################################################################################### diff --git a/debian/py3dist-overrides b/requirements-dist.in similarity index 87% rename from debian/py3dist-overrides rename to requirements-dist.in index e6059cd..11f0a2a 100644 --- a/debian/py3dist-overrides +++ b/requirements-dist.in @@ -1,4 +1,5 @@ -# -*- coding: utf-8 -*- +# Copyright ETSI Contributors and Others. +# # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at @@ -12,4 +13,5 @@ # See the License for the specific language governing permissions and # limitations under the License. -python-magic +stdeb +setuptools-version-command \ No newline at end of file diff --git a/requirements-dist.txt b/requirements-dist.txt new file mode 100644 index 0000000..7393626 --- /dev/null +++ b/requirements-dist.txt @@ -0,0 +1,23 @@ +setuptools-version-command==2.2 + # via -r requirements-dist.in +stdeb==0.10.0 + # via -r requirements-dist.in + +# The following packages are considered to be unsafe in a requirements file: +# setuptools +####################################################################################### +# Copyright ETSI Contributors and Others. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. +####################################################################################### diff --git a/requirements-test.in b/requirements-test.in new file mode 100755 index 0000000..9a0542d --- /dev/null +++ b/requirements-test.in @@ -0,0 +1,21 @@ +# Copyright 2020 ArctosLabs Scandinavia AB +# ************************************************************* + +# This file is part of OSM Placement module +# All Rights Reserved to ArctosLabs Scandinavia AB + +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at + +# http://www.apache.org/licenses/LICENSE-2.0 + +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +coverage +mock +nose2 diff --git a/requirements-test.txt b/requirements-test.txt new file mode 100644 index 0000000..d0c984f --- /dev/null +++ b/requirements-test.txt @@ -0,0 +1,26 @@ +coverage==5.5 + # via + # -r requirements-test.in + # nose2 +mock==4.0.3 + # via -r requirements-test.in +nose2==0.10.0 + # via -r requirements-test.in +six==1.15.0 + # via nose2 +####################################################################################### +# Copyright ETSI Contributors and Others. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. +####################################################################################### diff --git a/test-requirements.txt b/requirements.in similarity index 88% rename from test-requirements.txt rename to requirements.in index e944b57..44fea8b 100644 --- a/test-requirements.txt +++ b/requirements.in @@ -14,5 +14,12 @@ # License for the specific language governing permissions and limitations # under the License. -nose -mock +Click +jinja2 +packaging +prettytable +pycurl +python-magic +pyyaml +requests +verboselogs diff --git a/requirements.txt b/requirements.txt index f3cb782..ac1825e 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,26 +1,48 @@ -# Copyright 2019 ETSI OSM +certifi==2020.12.5 + # via requests +chardet==4.0.0 + # via requests +click==7.1.2 + # via -r requirements.in +idna==2.10 + # via requests +jinja2==2.11.3 + # via -r requirements.in +markupsafe==1.1.1 + # via jinja2 +packaging==20.9 + # via -r requirements.in +prettytable==2.1.0 + # via -r requirements.in +pycurl==7.43.0.6 + # via -r requirements.in +pyparsing==2.4.7 + # via packaging +python-magic==0.4.22 + # via -r requirements.in +pyyaml==5.4.1 + # via -r requirements.in +requests==2.25.1 + # via -r requirements.in +urllib3==1.26.4 + # via requests +verboselogs==1.7 + # via -r requirements.in +wcwidth==0.2.5 + # via prettytable +####################################################################################### +# Copyright ETSI Contributors and Others. # -# All Rights Reserved. +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at # -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at +# http://www.apache.org/licenses/LICENSE-2.0 # -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -Click -prettytable<=1.0.1 -PyYAML -pycurl -python-magic -jinja2 -verboselogs -packaging -requests -git+https://osm.etsi.org/gerrit/osm/IM.git#egg=osm-im +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. +####################################################################################### diff --git a/setup.py b/setup.py index ae4850c..ff87842 100644 --- a/setup.py +++ b/setup.py @@ -28,21 +28,10 @@ setup( maintainer="Gerardo Garcia", maintainer_email="gerardo.garciadeblas@telefonica.com", description=_description, - license="Apache 2", - install_requires=[ - "Click", - "prettytable", - "pyyaml", - "pycurl", - "python-magic", - "jinja2", - "osm-im", - "verboselogs", - "packaging", - "requests", - ], - setup_requires=["setuptools-version-command"], - test_suite="nose.collector", + license='Apache 2', + + setup_requires=['setuptools-version-command'], + test_suite='nose.collector', entry_points={ "console_scripts": [ "osm = osmclient.scripts.osm:cli", diff --git a/stdeb.cfg b/stdeb.cfg index 9629697..d96d374 100644 --- a/stdeb.cfg +++ b/stdeb.cfg @@ -16,6 +16,3 @@ X-Python3-Version : >= 3.5 Suite: xenial Build-Depends: dh-python Maintainer: Gerardo Garcia -Depends: python-setuptools, python-pycurl, python-click, python-prettytable, python-yaml, python-osm-im -Depends3: python3-pycurl, python3-click, python3-prettytable, python3-yaml, python3-osm-im - diff --git a/tox.ini b/tox.ini index 608dc8d..083443b 100644 --- a/tox.ini +++ b/tox.ini @@ -1,45 +1,125 @@ -# Copyright 2019 ETSI OSM +####################################################################################### +# Copyright ETSI Contributors and Others. # -# All Rights Reserved. +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at # -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. +# http://www.apache.org/licenses/LICENSE-2.0 # +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. +####################################################################################### + [tox] -envlist = py36,flakes -toxworkdir={toxinidir}/.tox +envlist = black, cover, flake8, pylint, safety -[testenv] -deps = -r{toxinidir}/requirements.txt - -r{toxinidir}/test-requirements.txt -install_command = python3 -m pip install -U {opts} {packages} -commands=nosetests +[tox:jenkins] +toxworkdir = /tmp/.tox -[testenv:flakes] +[testenv] +usedevelop = True basepython = python3 +setenv = VIRTUAL_ENV={envdir} + PYTHONDONTWRITEBYTECODE = 1 +deps = -r{toxinidir}/requirements.txt +parallel_show_output = true + +####################################################################################### +[testenv:black] +deps = black +skip_install = true +commands = + - black --check --diff osmclient/ + + +####################################################################################### +[testenv:cover] +deps = {[testenv]deps} + -r{toxinidir}/requirements-dev.txt + -r{toxinidir}/requirements-test.txt +commands = + sh -c 'rm -f nosetests.xml' + coverage erase + nose2 -C --coverage osmclient + coverage report --omit='*tests*' + coverage html -d ./cover --omit='*tests*' + coverage xml -o coverage.xml --omit=*tests* +whitelist_externals = sh + + +####################################################################################### +[testenv:flake8] deps = flake8 - pyflakes - -r{toxinidir}/requirements.txt - -r{toxinidir}/test-requirements.txt -install_command = python3 -m pip install -U {opts} {packages} commands = - flake8 setup.py - pyflakes osmclient + flake8 osmclient/ setup.py -[testenv:build] -basepython = python3 -deps = stdeb - setuptools-version-command - -r{toxinidir}/requirements.txt -install_command = python3 -m pip install -U {opts} {packages} -commands = python3 setup.py --command-packages=stdeb.command bdist_deb + +####################################################################################### +[testenv:pylint] +deps = {[testenv]deps} + -r{toxinidir}/requirements-dev.txt + -r{toxinidir}/requirements-test.txt + pylint +commands = + - pylint -E osmclient + + +####################################################################################### +[testenv:safety] +setenv = + LC_ALL=C.UTF-8 + LANG=C.UTF-8 +deps = {[testenv]deps} + safety +commands = + - safety check --full-report + + +####################################################################################### +[testenv:pip-compile] +deps = pip-tools==5.5.0 +commands = + - sh -c 'for file in requirements*.in ; do pip-compile -rU --no-header $file ;\ + out=`echo $file | sed "s/.in/.txt/"` ; \ + head -16 tox.ini >> $out ;\ + done' +whitelist_externals = sh + + +####################################################################################### +[testenv:dist] +deps = {[testenv]deps} + -r{toxinidir}/requirements-dist.txt + +# In the commands, we copy the requirements.txt to be presented as a source file (.py) +# so it gets included in the .deb package for others to consume +commands = + sh -c 'cp requirements.txt osmclient/requirements.txt' + python3 setup.py --command-packages=stdeb.command sdist_dsc + sh -c 'cd deb_dist/osmclient*/ && dpkg-buildpackage -rfakeroot -uc -us' + sh -c 'rm osmclient/requirements.txt' +whitelist_externals = sh + +####################################################################################### +[flake8] +ignore = + W291, + W293, + W503, + E123, + E125, + E226, + E241 +exclude = + .git, + __pycache__, + .tox, +max-line-length = 128 +show-source = True +builtins = _ -- 2.17.1