Feature 10296 Pip Standardization
Creates standard template for tox.ini
Introduces 'safety' for listing upstream CVEs
Pins all versions of all upstream modules
Change-Id: I20f3b8b1c386aa03ebbec37ab74bc106451328d1
Signed-off-by: beierlm <mark.beierl@canonical.com>
diff --git a/.gitignore b/.gitignore
index b4c9ff5..da10ebb 100644
--- a/.gitignore
+++ b/.gitignore
@@ -32,3 +32,7 @@
snap/.snapcraft/
.vscode
build/
+.project
+.pydevproject
+.settings/
+
diff --git a/Dockerfile b/Dockerfile
index 8c16461..99a85a0 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -1,19 +1,41 @@
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
#
-# Licensed under the Apache License, Version 2.0 (the "License"); you may
-# not use this file except in compliance with the License. You may obtain
-# a copy of the License at
+# http://www.apache.org/licenses/LICENSE-2.0
#
-# http://www.apache.org/licenses/LICENSE-2.0
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+# implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+########################################################################################
+# This Dockerfile is intented for devops testing and deb package generation
#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
-# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
-# License for the specific language governing permissions and limitations
-# under the License.
+# To run stage 2 locally:
#
+# docker build -t stage2 .
+# docker run -ti -v `pwd`:/work -w /work --entrypoint /bin/bash stage2
+# devops-stages/stage-test.sh
+# devops-stages/stage-build.sh
+#
+
FROM ubuntu:18.04
-RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get -y install git \
- make python3 python3-pip libcurl4-openssl-dev libssl-dev tox python3-dev \
- debhelper python3-setuptools python3-all python-all python-pip \
- apt-utils wget
+RUN DEBIAN_FRONTEND=noninteractive apt-get update && \
+ DEBIAN_FRONTEND=noninteractive apt-get -y install \
+ debhelper \
+ git \
+ python3 \
+ python3-all \
+ python3-dev \
+ python3-setuptools
+
+RUN python3 -m easy_install pip==21.0.1
+RUN pip3 install tox==3.22.0
+
+RUN DEBIAN_FRONTEND=noninteractive apt-get -y install \
+ libcurl4-openssl-dev \
+ libssl-dev \
+ wget
diff --git a/MANIFEST.in b/MANIFEST.in
index 4deac18..bff0dde 100644
--- a/MANIFEST.in
+++ b/MANIFEST.in
@@ -14,4 +14,4 @@
# License for the specific language governing permissions and limitations
# under the License.
-include osmclient/templates/*.yaml.j2
\ No newline at end of file
+include osmclient/templates/*.yaml.j2 osmclient/requirements.txt
\ No newline at end of file
diff --git a/debian/python3-osmclient.postinst b/debian/python3-osmclient.postinst
index 75ec83f..f78b5cb 100644
--- a/debian/python3-osmclient.postinst
+++ b/debian/python3-osmclient.postinst
@@ -15,8 +15,7 @@
##
echo "POST INSTALL OSMCLIENT"
-#Install pyangbind, required for python3-osm-im
-python3 -m pip install pyangbind verboselogs
+
#configure autocomplete for osmclient
[ -z "$SUDO_USER" ] && SUDO_USER="$USER"
su $SUDO_USER -c 'mkdir -p $HOME/.bash_completion.d'
@@ -27,4 +26,3 @@
echo " inserting .bash_completion.d/osm-complete.sh execution at .bashrc"
su $SUDO_USER -c 'echo ". ${HOME}/.bash_completion.d/osm-complete.sh" >> ${HOME}/.bashrc'
fi
-
diff --git a/devops-stages/stage-build.sh b/devops-stages/stage-build.sh
index f00cd3e..164b385 100755
--- a/devops-stages/stage-build.sh
+++ b/devops-stages/stage-build.sh
@@ -14,4 +14,4 @@
# limitations under the License.
rm -rf deb_dist osmclient-*.tar.gz
-tox -e build
+tox -e dist
diff --git a/devops-stages/stage-test.sh b/devops-stages/stage-test.sh
index 0177ba3..a3f8e18 100755
--- a/devops-stages/stage-test.sh
+++ b/devops-stages/stage-test.sh
@@ -11,5 +11,5 @@
# implied.
# See the License for the specific language governing permissions and
# limitations under the License.
-tox --recreate
+tox
diff --git a/osmclient/scripts/osm.py b/osmclient/scripts/osm.py
index 603d373..4af0dad 100755
--- a/osmclient/scripts/osm.py
+++ b/osmclient/scripts/osm.py
@@ -384,7 +384,7 @@
project_list = ctx.obj.project.list()
try:
vim_list = ctx.obj.vim.list()
- except:
+ except Exception:
vim_list = []
else:
table = PrettyTable(
@@ -3756,7 +3756,8 @@
@click.option(
"--k8s-nets",
prompt=True,
- help='list of VIM networks, in JSON inline format, where the cluster is accessible via L3 routing, e.g. "{(k8s_net1:vim_network1) [,(k8s_net2:vim_network2) ...]}"',
+ help='''list of VIM networks, in JSON inline format, where the cluster is
+ accessible via L3 routing, e.g. "{(k8s_net1:vim_network1) [,(k8s_net2:vim_network2) ...]}"''',
)
@click.option("--description", default=None, help="human readable description")
@click.option(
@@ -3812,7 +3813,8 @@
@click.option("--vim", help="VIM target, the VIM where the cluster resides")
@click.option(
"--k8s-nets",
- help='list of VIM networks, in JSON inline format, where the cluster is accessible via L3 routing, e.g. "{(k8s_net1:vim_network1) [,(k8s_net2:vim_network2) ...]}"',
+ help='''list of VIM networks, in JSON inline format, where the cluster is accessible
+ via L3 routing, e.g. "{(k8s_net1:vim_network1) [,(k8s_net2:vim_network2) ...]}"''',
)
@click.option("--description", help="human readable description")
@click.option(
@@ -3920,7 +3922,7 @@
)
try:
vim_list = ctx.obj.vim.list()
- except:
+ except Exception:
vim_list = []
for cluster in resp:
logger.debug("Cluster details: {}".format(yaml.safe_dump(cluster)))
diff --git a/osmclient/sol005/client.py b/osmclient/sol005/client.py
index 495c425..36062bf 100644
--- a/osmclient/sol005/client.py
+++ b/osmclient/sol005/client.py
@@ -149,9 +149,9 @@
def set_default_params(self, **kwargs):
host = kwargs.pop("host", None)
- if host != None:
+ if host is not None:
self._host = host
port = kwargs.pop("port", None)
- if port != None:
+ if port is not None:
self._so_port = port
self._http_client.set_query_admin(**kwargs)
diff --git a/osmclient/sol005/vim.py b/osmclient/sol005/vim.py
index f2a11e6..e5bf399 100644
--- a/osmclient/sol005/vim.py
+++ b/osmclient/sol005/vim.py
@@ -75,7 +75,7 @@
self._logger.debug("")
self._client.get_token()
if "vim-type" not in vim_access:
- #'openstack' not in vim_access['vim-type']):
+ # 'openstack' not in vim_access['vim-type']):
raise Exception("vim type not provided")
vim_account = {}
diff --git a/osmclient/sol005/vnfd.py b/osmclient/sol005/vnfd.py
index 4542d56..39f4c59 100644
--- a/osmclient/sol005/vnfd.py
+++ b/osmclient/sol005/vnfd.py
@@ -282,7 +282,8 @@
):
special_override_string = (
"{}vdu.{}.int-cpd.{}.virtual-network-interface-"
- "requirement.{}.virtual-interface.type=PARAVIRT;".format(
+ "requirement.{}.virtual-interface.type="
+ "PARAVIRT;".format(
special_override_string,
vdu_number,
cpd_number,
diff --git a/osmclient/v1/vim.py b/osmclient/v1/vim.py
index 35266aa..1e8b604 100644
--- a/osmclient/v1/vim.py
+++ b/osmclient/v1/vim.py
@@ -60,7 +60,7 @@
# currently assumes vim_acc
if "vim-type" not in vim_access:
- #'openstack' not in vim_access['vim-type']):
+ # 'openstack' not in vim_access['vim-type']):
raise Exception("vim type not provided")
vim_account["datacenter"]["name"] = name
diff --git a/debian/py3dist-overrides b/requirements-dev.in
similarity index 83%
copy from debian/py3dist-overrides
copy to requirements-dev.in
index e6059cd..620d427 100644
--- a/debian/py3dist-overrides
+++ b/requirements-dev.in
@@ -1,4 +1,5 @@
-# -*- coding: utf-8 -*-
+# Copyright ETSI Contributors and Others.
+#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
@@ -12,4 +13,4 @@
# See the License for the specific language governing permissions and
# limitations under the License.
-python-magic
+git+https://osm.etsi.org/gerrit/osm/IM.git@master#egg=osm-im
diff --git a/debian/py3dist-overrides b/requirements-dev.txt
similarity index 62%
copy from debian/py3dist-overrides
copy to requirements-dev.txt
index e6059cd..cd3f5bb 100644
--- a/debian/py3dist-overrides
+++ b/requirements-dev.txt
@@ -1,4 +1,8 @@
-# -*- coding: utf-8 -*-
+git+https://osm.etsi.org/gerrit/osm/IM.git@master#egg=osm-im
+ # via -r requirements-dev.in
+#######################################################################################
+# Copyright ETSI Contributors and Others.
+#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
@@ -11,5 +15,4 @@
# implied.
# See the License for the specific language governing permissions and
# limitations under the License.
-
-python-magic
+#######################################################################################
diff --git a/debian/py3dist-overrides b/requirements-dist.in
similarity index 86%
rename from debian/py3dist-overrides
rename to requirements-dist.in
index e6059cd..11f0a2a 100644
--- a/debian/py3dist-overrides
+++ b/requirements-dist.in
@@ -1,4 +1,5 @@
-# -*- coding: utf-8 -*-
+# Copyright ETSI Contributors and Others.
+#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
@@ -12,4 +13,5 @@
# See the License for the specific language governing permissions and
# limitations under the License.
-python-magic
+stdeb
+setuptools-version-command
\ No newline at end of file
diff --git a/requirements-dist.txt b/requirements-dist.txt
new file mode 100644
index 0000000..7393626
--- /dev/null
+++ b/requirements-dist.txt
@@ -0,0 +1,23 @@
+setuptools-version-command==2.2
+ # via -r requirements-dist.in
+stdeb==0.10.0
+ # via -r requirements-dist.in
+
+# The following packages are considered to be unsafe in a requirements file:
+# setuptools
+#######################################################################################
+# Copyright ETSI Contributors and Others.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+# implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#######################################################################################
diff --git a/requirements-test.in b/requirements-test.in
new file mode 100755
index 0000000..9a0542d
--- /dev/null
+++ b/requirements-test.in
@@ -0,0 +1,21 @@
+# Copyright 2020 ArctosLabs Scandinavia AB
+# *************************************************************
+
+# This file is part of OSM Placement module
+# All Rights Reserved to ArctosLabs Scandinavia AB
+
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+
+# http://www.apache.org/licenses/LICENSE-2.0
+
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+coverage
+mock
+nose2
diff --git a/requirements-test.txt b/requirements-test.txt
new file mode 100644
index 0000000..d0c984f
--- /dev/null
+++ b/requirements-test.txt
@@ -0,0 +1,26 @@
+coverage==5.5
+ # via
+ # -r requirements-test.in
+ # nose2
+mock==4.0.3
+ # via -r requirements-test.in
+nose2==0.10.0
+ # via -r requirements-test.in
+six==1.15.0
+ # via nose2
+#######################################################################################
+# Copyright ETSI Contributors and Others.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+# implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#######################################################################################
diff --git a/test-requirements.txt b/requirements.in
similarity index 87%
rename from test-requirements.txt
rename to requirements.in
index e944b57..44fea8b 100644
--- a/test-requirements.txt
+++ b/requirements.in
@@ -14,5 +14,12 @@
# License for the specific language governing permissions and limitations
# under the License.
-nose
-mock
+Click
+jinja2
+packaging
+prettytable
+pycurl
+python-magic
+pyyaml
+requests
+verboselogs
diff --git a/requirements.txt b/requirements.txt
index f3cb782..ac1825e 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -1,26 +1,48 @@
-# Copyright 2019 ETSI OSM
+certifi==2020.12.5
+ # via requests
+chardet==4.0.0
+ # via requests
+click==7.1.2
+ # via -r requirements.in
+idna==2.10
+ # via requests
+jinja2==2.11.3
+ # via -r requirements.in
+markupsafe==1.1.1
+ # via jinja2
+packaging==20.9
+ # via -r requirements.in
+prettytable==2.1.0
+ # via -r requirements.in
+pycurl==7.43.0.6
+ # via -r requirements.in
+pyparsing==2.4.7
+ # via packaging
+python-magic==0.4.22
+ # via -r requirements.in
+pyyaml==5.4.1
+ # via -r requirements.in
+requests==2.25.1
+ # via -r requirements.in
+urllib3==1.26.4
+ # via requests
+verboselogs==1.7
+ # via -r requirements.in
+wcwidth==0.2.5
+ # via prettytable
+#######################################################################################
+# Copyright ETSI Contributors and Others.
#
-# All Rights Reserved.
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
#
-# Licensed under the Apache License, Version 2.0 (the "License"); you may
-# not use this file except in compliance with the License. You may obtain
-# a copy of the License at
+# http://www.apache.org/licenses/LICENSE-2.0
#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
-# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
-# License for the specific language governing permissions and limitations
-# under the License.
-
-Click
-prettytable<=1.0.1
-PyYAML
-pycurl
-python-magic
-jinja2
-verboselogs
-packaging
-requests
-git+https://osm.etsi.org/gerrit/osm/IM.git#egg=osm-im
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+# implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#######################################################################################
diff --git a/setup.py b/setup.py
index ae4850c..ff87842 100644
--- a/setup.py
+++ b/setup.py
@@ -28,21 +28,10 @@
maintainer="Gerardo Garcia",
maintainer_email="gerardo.garciadeblas@telefonica.com",
description=_description,
- license="Apache 2",
- install_requires=[
- "Click",
- "prettytable",
- "pyyaml",
- "pycurl",
- "python-magic",
- "jinja2",
- "osm-im",
- "verboselogs",
- "packaging",
- "requests",
- ],
- setup_requires=["setuptools-version-command"],
- test_suite="nose.collector",
+ license='Apache 2',
+
+ setup_requires=['setuptools-version-command'],
+ test_suite='nose.collector',
entry_points={
"console_scripts": [
"osm = osmclient.scripts.osm:cli",
diff --git a/stdeb.cfg b/stdeb.cfg
index 9629697..d96d374 100644
--- a/stdeb.cfg
+++ b/stdeb.cfg
@@ -16,6 +16,3 @@
Suite: xenial
Build-Depends: dh-python
Maintainer: Gerardo Garcia <gerardo.garciadeblas@telefonica.com>
-Depends: python-setuptools, python-pycurl, python-click, python-prettytable, python-yaml, python-osm-im
-Depends3: python3-pycurl, python3-click, python3-prettytable, python3-yaml, python3-osm-im
-
diff --git a/tox.ini b/tox.ini
index 608dc8d..083443b 100644
--- a/tox.ini
+++ b/tox.ini
@@ -1,45 +1,125 @@
-# Copyright 2019 ETSI OSM
+#######################################################################################
+# Copyright ETSI Contributors and Others.
#
-# All Rights Reserved.
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
#
-# Licensed under the Apache License, Version 2.0 (the "License"); you may
-# not use this file except in compliance with the License. You may obtain
-# a copy of the License at
+# http://www.apache.org/licenses/LICENSE-2.0
#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
-# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
-# License for the specific language governing permissions and limitations
-# under the License.
-#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+# implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#######################################################################################
+
[tox]
-envlist = py36,flakes
-toxworkdir={toxinidir}/.tox
+envlist = black, cover, flake8, pylint, safety
+
+[tox:jenkins]
+toxworkdir = /tmp/.tox
[testenv]
-deps = -r{toxinidir}/requirements.txt
- -r{toxinidir}/test-requirements.txt
-install_command = python3 -m pip install -U {opts} {packages}
-commands=nosetests
-
-[testenv:flakes]
+usedevelop = True
basepython = python3
-deps = flake8
- pyflakes
- -r{toxinidir}/requirements.txt
- -r{toxinidir}/test-requirements.txt
-install_command = python3 -m pip install -U {opts} {packages}
+setenv = VIRTUAL_ENV={envdir}
+ PYTHONDONTWRITEBYTECODE = 1
+deps = -r{toxinidir}/requirements.txt
+parallel_show_output = true
+
+#######################################################################################
+[testenv:black]
+deps = black
+skip_install = true
commands =
- flake8 setup.py
- pyflakes osmclient
+ - black --check --diff osmclient/
-[testenv:build]
-basepython = python3
-deps = stdeb
- setuptools-version-command
- -r{toxinidir}/requirements.txt
-install_command = python3 -m pip install -U {opts} {packages}
-commands = python3 setup.py --command-packages=stdeb.command bdist_deb
+
+#######################################################################################
+[testenv:cover]
+deps = {[testenv]deps}
+ -r{toxinidir}/requirements-dev.txt
+ -r{toxinidir}/requirements-test.txt
+commands =
+ sh -c 'rm -f nosetests.xml'
+ coverage erase
+ nose2 -C --coverage osmclient
+ coverage report --omit='*tests*'
+ coverage html -d ./cover --omit='*tests*'
+ coverage xml -o coverage.xml --omit=*tests*
+whitelist_externals = sh
+
+
+#######################################################################################
+[testenv:flake8]
+deps = flake8
+commands =
+ flake8 osmclient/ setup.py
+
+
+#######################################################################################
+[testenv:pylint]
+deps = {[testenv]deps}
+ -r{toxinidir}/requirements-dev.txt
+ -r{toxinidir}/requirements-test.txt
+ pylint
+commands =
+ - pylint -E osmclient
+
+
+#######################################################################################
+[testenv:safety]
+setenv =
+ LC_ALL=C.UTF-8
+ LANG=C.UTF-8
+deps = {[testenv]deps}
+ safety
+commands =
+ - safety check --full-report
+
+
+#######################################################################################
+[testenv:pip-compile]
+deps = pip-tools==5.5.0
+commands =
+ - sh -c 'for file in requirements*.in ; do pip-compile -rU --no-header $file ;\
+ out=`echo $file | sed "s/.in/.txt/"` ; \
+ head -16 tox.ini >> $out ;\
+ done'
+whitelist_externals = sh
+
+
+#######################################################################################
+[testenv:dist]
+deps = {[testenv]deps}
+ -r{toxinidir}/requirements-dist.txt
+
+# In the commands, we copy the requirements.txt to be presented as a source file (.py)
+# so it gets included in the .deb package for others to consume
+commands =
+ sh -c 'cp requirements.txt osmclient/requirements.txt'
+ python3 setup.py --command-packages=stdeb.command sdist_dsc
+ sh -c 'cd deb_dist/osmclient*/ && dpkg-buildpackage -rfakeroot -uc -us'
+ sh -c 'rm osmclient/requirements.txt'
+whitelist_externals = sh
+
+#######################################################################################
+[flake8]
+ignore =
+ W291,
+ W293,
+ W503,
+ E123,
+ E125,
+ E226,
+ E241
+exclude =
+ .git,
+ __pycache__,
+ .tox,
+max-line-length = 128
+show-source = True
+builtins = _