From 7c6840b23b5d6d8bc14eb7e76d1262c4ccde3097 Mon Sep 17 00:00:00 2001 From: David Garcia Date: Fri, 12 Feb 2021 17:19:13 +0100 Subject: [PATCH] Feature 10296 Pip Standardization Creates standard template for tox.ini Introduces 'safety' for listing upstream CVEs Pins all versions of all upstream modules Removes Makefile step and places all build logic into tox.ini Change-Id: Ic14321bd5bddb322db08ac3e25202cc64cac2dff Signed-off-by: David Garcia --- .gitignore | 4 +- Dockerfile | 39 +++-- Makefile => MANIFEST.in | 20 +-- devops-stages/stage-build.sh | 5 +- devops-stages/stage-test.sh | 4 +- requirements-dev.in | 15 ++ requirements-dev.txt | 30 ++++ requirements-dist.in | 16 ++ requirements-dist.txt | 23 +++ test-requirements.txt => requirements-test.in | 9 +- requirements-test.txt | 50 ++++++ requirements.in | 17 ++ requirements.txt | 132 +++++++++++++-- setup.py | 42 ++--- stdeb.cfg | 19 ++- tox.ini | 150 ++++++++++++------ 16 files changed, 451 insertions(+), 124 deletions(-) rename Makefile => MANIFEST.in (60%) create mode 100644 requirements-dev.in create mode 100644 requirements-dev.txt create mode 100644 requirements-dist.in create mode 100644 requirements-dist.txt rename test-requirements.txt => requirements-test.in (90%) create mode 100644 requirements-test.txt create mode 100644 requirements.in diff --git a/.gitignore b/.gitignore index ae1f284..191b24e 100644 --- a/.gitignore +++ b/.gitignore @@ -20,6 +20,7 @@ tests/charms/deps dist/ .cache/ .local/ +.eggs N2VC.egg-info/ .coverage cover @@ -34,4 +35,5 @@ bin/ lib/ lib64 pyvenv.cfg -share/ \ No newline at end of file +share/ +venv/ diff --git a/Dockerfile b/Dockerfile index 88aae5d..3fad212 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,19 +1,36 @@ -# Copyright 2019 Canonical Ltd. -# # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # -# http://www.apache.org/licenses/LICENSE-2.0 +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. +######################################################################################## +# This Dockerfile is intented for devops testing and deb package generation +# +# To run stage 2 locally: +# +# docker build -t stage2 . +# docker run -ti -v `pwd`:/work -w /work --entrypoint /bin/bash stage2 +# devops-stages/stage-test.sh +# devops-stages/stage-build.sh # -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. FROM ubuntu:18.04 -RUN apt-get update && apt-get -y install git make python python3 \ - libcurl4-gnutls-dev libgnutls28-dev tox python3-dev python3-pip \ - debhelper python3-setuptools python-all python3-all apt-utils +RUN DEBIAN_FRONTEND=noninteractive apt-get update && \ + DEBIAN_FRONTEND=noninteractive apt-get -y install \ + debhelper \ + git \ + python3 \ + python3-all \ + python3-dev \ + python3-setuptools + +RUN python3 -m easy_install pip==21.0.1 +RUN pip3 install tox==3.22.0 diff --git a/Makefile b/MANIFEST.in similarity index 60% rename from Makefile rename to MANIFEST.in index a334b5a..f1be81c 100644 --- a/Makefile +++ b/MANIFEST.in @@ -1,4 +1,4 @@ -# Copyright 2019 Canonical Ltd. +# Copyright 2021 Canonical Ltd. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -12,17 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -clean: - find . -name __pycache__ -type d -exec rm -r {} + - find . -name *.pyc -delete - rm -rf .tox - rm -rf tests/charms/builds/* - lxc list test- --format=json|jq '.[]["name"]'| xargs lxc delete --force || true -.tox: - tox -r --notest -test: lint - tox -lint: - tox -e lint -package: - python3 setup.py --command-packages=stdeb.command bdist_deb +include README.rst +recursive-include n2vc *.sh *.txt +recursive-include devops-stages * +recursive-exclude n2vc/tests * \ No newline at end of file diff --git a/devops-stages/stage-build.sh b/devops-stages/stage-build.sh index 8f7f155..06184b6 100755 --- a/devops-stages/stage-build.sh +++ b/devops-stages/stage-build.sh @@ -13,5 +13,6 @@ # limitations under the License. #!/bin/sh -rm -rf deb_dist -tox -e build --recreate +rm -rf dist deb_dist N2VC-*.tar.gz N2VC.egg-info .eggs .tox + +tox -e dist \ No newline at end of file diff --git a/devops-stages/stage-test.sh b/devops-stages/stage-test.sh index 41875be..50f588a 100755 --- a/devops-stages/stage-test.sh +++ b/devops-stages/stage-test.sh @@ -12,5 +12,5 @@ # See the License for the specific language governing permissions and # limitations under the License. -#!/bin/sh -tox --recreate \ No newline at end of file +OUTPUT=$(TOX_PARALLEL_NO_SPINNER=1 tox --parallel=auto) +printf "$OUTPUT" diff --git a/requirements-dev.in b/requirements-dev.in new file mode 100644 index 0000000..abc0642 --- /dev/null +++ b/requirements-dev.in @@ -0,0 +1,15 @@ +# Copyright 2021 Canonical Ltd. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +git+https://osm.etsi.org/gerrit/osm/common.git@master#egg=osm-common diff --git a/requirements-dev.txt b/requirements-dev.txt new file mode 100644 index 0000000..485e45e --- /dev/null +++ b/requirements-dev.txt @@ -0,0 +1,30 @@ +aiokafka==0.7.0 + # via osm-common +dataclasses==0.6 + # via osm-common +kafka-python==2.0.2 + # via aiokafka +git+https://osm.etsi.org/gerrit/osm/common.git@master#egg=osm-common + # via -r requirements-dev.in +pycrypto==2.6.1 + # via osm-common +pymongo==3.11.3 + # via osm-common +pyyaml==5.4.1 + # via osm-common +####################################################################################### +# Copyright ETSI Contributors and Others. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. +####################################################################################### diff --git a/requirements-dist.in b/requirements-dist.in new file mode 100644 index 0000000..2b3c735 --- /dev/null +++ b/requirements-dist.in @@ -0,0 +1,16 @@ +# Copyright 2021 Canonical Ltd. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +stdeb +setuptools-version-command diff --git a/requirements-dist.txt b/requirements-dist.txt new file mode 100644 index 0000000..7393626 --- /dev/null +++ b/requirements-dist.txt @@ -0,0 +1,23 @@ +setuptools-version-command==2.2 + # via -r requirements-dist.in +stdeb==0.10.0 + # via -r requirements-dist.in + +# The following packages are considered to be unsafe in a requirements file: +# setuptools +####################################################################################### +# Copyright ETSI Contributors and Others. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. +####################################################################################### diff --git a/test-requirements.txt b/requirements-test.in similarity index 90% rename from test-requirements.txt rename to requirements-test.in index 45ed6dc..08df82d 100644 --- a/test-requirements.txt +++ b/requirements-test.in @@ -1,4 +1,4 @@ -# Copyright 2020 Canonical Ltd. +# Copyright 2021 Canonical Ltd. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -12,8 +12,9 @@ # See the License for the specific language governing permissions and # limitations under the License. -flake8<3.0 +asynctest +coverage +flake8 mock +nose2 requests-mock -coverage==4.5.3 -asynctest diff --git a/requirements-test.txt b/requirements-test.txt new file mode 100644 index 0000000..7e3a14d --- /dev/null +++ b/requirements-test.txt @@ -0,0 +1,50 @@ +asynctest==0.13.0 + # via -r requirements-test.in +certifi==2020.12.5 + # via requests +chardet==4.0.0 + # via requests +coverage==5.5 + # via + # -r requirements-test.in + # nose2 +flake8==3.9.0 + # via -r requirements-test.in +idna==2.10 + # via requests +mccabe==0.6.1 + # via flake8 +mock==4.0.3 + # via -r requirements-test.in +nose2==0.10.0 + # via -r requirements-test.in +pycodestyle==2.7.0 + # via flake8 +pyflakes==2.3.1 + # via flake8 +requests-mock==1.8.0 + # via -r requirements-test.in +requests==2.25.1 + # via requests-mock +six==1.15.0 + # via + # nose2 + # requests-mock +urllib3==1.26.4 + # via requests +####################################################################################### +# Copyright ETSI Contributors and Others. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. +####################################################################################### diff --git a/requirements.in b/requirements.in new file mode 100644 index 0000000..ea82d48 --- /dev/null +++ b/requirements.in @@ -0,0 +1,17 @@ +# Copyright 2021 Canonical Ltd. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +juju==2.8.4 +kubernetes==10.0.1 +pyasn1 \ No newline at end of file diff --git a/requirements.txt b/requirements.txt index 6e94e0a..8ff8c08 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,19 +1,125 @@ -# Copyright 2020 Canonical Ltd. +bcrypt==3.2.0 + # via paramiko +cachetools==4.2.1 + # via google-auth +certifi==2020.12.5 + # via + # kubernetes + # requests +cffi==1.14.5 + # via + # bcrypt + # cryptography + # pynacl +chardet==4.0.0 + # via requests +cryptography==3.4.7 + # via paramiko +google-auth==1.28.0 + # via kubernetes +idna==2.10 + # via requests +juju==2.8.4 + # via -r requirements.in +jujubundlelib==0.5.6 + # via theblues +kubernetes==10.0.1 + # via -r requirements.in +macaroonbakery==1.3.1 + # via + # juju + # theblues +mypy-extensions==0.4.3 + # via typing-inspect +oauthlib==3.1.0 + # via requests-oauthlib +paramiko==2.7.2 + # via juju +protobuf==3.15.6 + # via macaroonbakery +pyasn1-modules==0.2.8 + # via google-auth +pyasn1==0.4.8 + # via + # -r requirements.in + # juju + # pyasn1-modules + # rsa +pycparser==2.20 + # via cffi +pymacaroons==0.13.0 + # via macaroonbakery +pynacl==1.4.0 + # via + # macaroonbakery + # paramiko + # pymacaroons +pyrfc3339==1.1 + # via + # juju + # macaroonbakery +python-dateutil==2.8.1 + # via kubernetes +pytz==2021.1 + # via pyrfc3339 +pyyaml==5.4.1 + # via + # juju + # jujubundlelib + # kubernetes +requests-oauthlib==1.3.0 + # via kubernetes +requests==2.25.1 + # via + # kubernetes + # macaroonbakery + # requests-oauthlib + # theblues +rsa==4.7.2 + # via google-auth +six==1.15.0 + # via + # bcrypt + # google-auth + # kubernetes + # macaroonbakery + # protobuf + # pymacaroons + # pynacl + # python-dateutil + # websocket-client +theblues==0.5.2 + # via juju +toposort==1.6 + # via juju +typing-extensions==3.7.4.3 + # via typing-inspect +typing-inspect==0.6.0 + # via juju +urllib3==1.26.4 + # via + # kubernetes + # requests +websocket-client==0.58.0 + # via kubernetes +websockets==7.0 + # via juju + +# The following packages are considered to be unsafe in a requirements file: +# setuptools +####################################################################################### +# Copyright ETSI Contributors and Others. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # -# http://www.apache.org/licenses/LICENSE-2.0 +# http://www.apache.org/licenses/LICENSE-2.0 # -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -git+https://osm.etsi.org/gerrit/osm/common.git#egg=osm-common -juju==2.8.4 -pyasn1>=0.4.4 -kubernetes==10.0.1 -chardet==3.0.4 \ No newline at end of file +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. +####################################################################################### diff --git a/setup.py b/setup.py index 7a4fe87..ec1dccb 100644 --- a/setup.py +++ b/setup.py @@ -14,27 +14,29 @@ from setuptools import setup, find_packages -_description = 'OSM library implementing common interface towards VCA module' +_name = "N2VC" +_version_command = ("git describe --match v* --tags --long --dirty", "pep440-git-full") +_author = "OSM Support" +_author_email = "osmsupport@etsi.org" +_description = "OSM library implementing common interface towards VCA module" +_maintainer = "OSM Support" +_maintainer_email = "osmsupport@etsi.org" +_license = "Apache 2.0" +_url = "https://osm.etsi.org/gitweb/?p=osm/N2VC.git;a=tree" setup( - name='N2VC', - version_command=('git describe --match v* --tags --long --dirty', - 'pep440-git-full'), - packages=find_packages( - exclude=["*.tests", "*.tests.*", "tests.*", "tests"]), - install_requires=[ - 'juju==2.8.4', - 'pyasn1>=0.4.4', - 'kubernetes==10.0.1' - ], - include_package_data=True, - maintainer='Adam Israel', - maintainer_email='adam.israel@canonical.com', + name=_name, + version_command=_version_command, description=_description, - url='', - license='Apache 2', - entry_points={ - 'console_scripts': [ - ], - }, + long_description=open("README.md", encoding="utf-8").read(), + author=_author, + author_email=_author_email, + maintainer=_maintainer, + maintainer_email=_maintainer_email, + url=_url, + license=_license, + packages=find_packages(exclude=["*.tests", "*.tests.*", "tests.*", "tests"]), + include_package_data=True, + entry_points={"console_scripts": []}, + setup_requires=['setuptools-version-command'] ) diff --git a/stdeb.cfg b/stdeb.cfg index c07507b..a5053ee 100644 --- a/stdeb.cfg +++ b/stdeb.cfg @@ -1,5 +1,16 @@ +# Copyright 2021 Canonical Ltd. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + [DEFAULT] -Suite: xenial -XS-Python-Version: >= 3.5 -Maintainer: Adam Israel -Depends: python3-pip +X-Python3-Version : >= 3.5 \ No newline at end of file diff --git a/tox.ini b/tox.ini index 4d64762..21fc186 100644 --- a/tox.ini +++ b/tox.ini @@ -1,77 +1,123 @@ -# Copyright 2019 Canonical Ltd. +####################################################################################### +# Copyright ETSI Contributors and Others. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # -# http://www.apache.org/licenses/LICENSE-2.0 +# http://www.apache.org/licenses/LICENSE-2.0 # -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. +####################################################################################### [tox] -envlist = cover, flake8, pylint, pylint-tests -skipsdist=True +envlist = black, cover, flake8, pylint, safety + +[tox:jenkins] +toxworkdir = /tmp/.tox [testenv] +usedevelop = True +basepython = python3 +setenv = VIRTUAL_ENV={envdir} + PYTHONDONTWRITEBYTECODE = 1 +deps = -r{toxinidir}/requirements.txt + + +####################################################################################### +[testenv:black] +deps = black +skip_install = true +commands = + - black --check --diff n2vc/ + +####################################################################################### [testenv:cover] -basepython = python3 -deps = - nose2 - -rrequirements.txt - -rtest-requirements.txt +deps = {[testenv]deps} + -r{toxinidir}/requirements-dev.txt + -r{toxinidir}/requirements-test.txt +commands = + sh -c 'rm -f nosetests.xml' + coverage erase + nose2 -C --coverage n2vc + coverage report --omit='*tests*' + coverage html -d ./cover --omit='*tests*' + coverage xml -o coverage.xml --omit=*tests* +whitelist_externals = sh + + +####################################################################################### +[testenv:flake8] +deps = flake8 commands = - coverage erase - nose2 -C --coverage n2vc --plugin nose2.plugins.junitxml -s n2vc - coverage report --omit='*n2vc/tests*' - coverage html -d ./cover --omit='*n2vc/tests*' - coverage xml -o coverage.xml --omit='*n2vc/tests*' + flake8 n2vc/ setup.py + +####################################################################################### [testenv:pylint] -basepython = python3 -deps = - pylint==2.6.0 - -rrequirements.txt +deps = {[testenv]deps} + -r{toxinidir}/requirements-dev.txt + -r{toxinidir}/requirements-test.txt + pylint commands = - pylint -E n2vc --ignore=tests + pylint -E n2vc -[testenv:pylint-tests] -basepython = python3 -deps = - pylint==2.6.0 - -rrequirements.txt - -rtest-requirements.txt + +####################################################################################### +[testenv:safety] +setenv = + LC_ALL=C.UTF-8 + LANG=C.UTF-8 +deps = {[testenv]deps} + safety commands = - pylint -E n2vc.tests + - safety check --full-report -[testenv:black] -basepython = python3 -deps = - black + +####################################################################################### +[testenv:pip-compile] +deps = pip-tools==5.5.0 commands = - black --check --diff n2vc + - sh -c 'for file in requirements*.in ; do pip-compile -rU --no-header $file ;\ + out=`echo $file | sed "s/.in/.txt/"` ; \ + head -16 tox.ini >> $out ;\ + done' +whitelist_externals = sh -[testenv:flake8] -basepython = python3 -deps = - flake8 - -rrequirements.txt + +####################################################################################### +[testenv:dist] +deps = {[testenv]deps} + -r{toxinidir}/requirements-dist.txt + +# In the commands, we copy the requirements.txt to be presented as a source file (.py) +# so it gets included in the .deb package for others to consume commands = - flake8 n2vc + sh -c 'cp requirements.txt n2vc/requirements.txt' + python3 setup.py --command-packages=stdeb.command sdist_dsc + sh -c 'cd deb_dist/n2vc*/ && dpkg-buildpackage -rfakeroot -uc -us' + sh -c 'rm n2vc/requirements.txt' +whitelist_externals = sh +####################################################################################### [flake8] -# W503 is invalid PEP-8 -max-line-length = 100 +ignore = + W291, + W293, + E123, + E125, + E226, + E241 +exclude = + .git, + __pycache__, + .tox, +max-line-length = 120 show-source = True -ignore = W503,E203 -exclude=.venv,.git,.tox,dist,doc,*lib/python*,*egg,build,devops_stages/*,.rst - -[testenv:build] -deps = - stdeb - setuptools-version-command -commands = python3 setup.py --command-packages=stdeb.command bdist_deb +builtins = _ -- 2.25.1