From: beierlm Date: Wed, 24 Mar 2021 13:26:06 +0000 (-0400) Subject: Feature 10296 Pip Standardization X-Git-Tag: v9.1.2 X-Git-Url: https://osm.etsi.org/gitweb/?a=commitdiff_plain;h=e1276ee1e385c97a007a4efd917124e3e2022d6f;p=osm%2Fosmclient.git Feature 10296 Pip Standardization Creates standard template for tox.ini Introduces 'safety' for listing upstream CVEs Pins all versions of all upstream modules Change-Id: I20f3b8b1c386aa03ebbec37ab74bc106451328d1 Signed-off-by: beierlm --- diff --git a/.gitignore b/.gitignore index b4c9ff5..da10ebb 100644 --- a/.gitignore +++ b/.gitignore @@ -32,3 +32,7 @@ stage/ snap/.snapcraft/ .vscode build/ +.project +.pydevproject +.settings/ + diff --git a/Dockerfile b/Dockerfile index 8c16461..99a85a0 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,19 +1,41 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at # -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at +# http://www.apache.org/licenses/LICENSE-2.0 # -# http://www.apache.org/licenses/LICENSE-2.0 +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. +######################################################################################## +# This Dockerfile is intented for devops testing and deb package generation # -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. +# To run stage 2 locally: # +# docker build -t stage2 . +# docker run -ti -v `pwd`:/work -w /work --entrypoint /bin/bash stage2 +# devops-stages/stage-test.sh +# devops-stages/stage-build.sh +# + FROM ubuntu:18.04 -RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get -y install git \ - make python3 python3-pip libcurl4-openssl-dev libssl-dev tox python3-dev \ - debhelper python3-setuptools python3-all python-all python-pip \ - apt-utils wget +RUN DEBIAN_FRONTEND=noninteractive apt-get update && \ + DEBIAN_FRONTEND=noninteractive apt-get -y install \ + debhelper \ + git \ + python3 \ + python3-all \ + python3-dev \ + python3-setuptools + +RUN python3 -m easy_install pip==21.0.1 +RUN pip3 install tox==3.22.0 + +RUN DEBIAN_FRONTEND=noninteractive apt-get -y install \ + libcurl4-openssl-dev \ + libssl-dev \ + wget diff --git a/MANIFEST.in b/MANIFEST.in index 4deac18..bff0dde 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -14,4 +14,4 @@ # License for the specific language governing permissions and limitations # under the License. -include osmclient/templates/*.yaml.j2 \ No newline at end of file +include osmclient/templates/*.yaml.j2 osmclient/requirements.txt \ No newline at end of file diff --git a/debian/py3dist-overrides b/debian/py3dist-overrides deleted file mode 100644 index e6059cd..0000000 --- a/debian/py3dist-overrides +++ /dev/null @@ -1,15 +0,0 @@ -# -*- coding: utf-8 -*- -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or -# implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -python-magic diff --git a/debian/python3-osmclient.postinst b/debian/python3-osmclient.postinst index 75ec83f..f78b5cb 100644 --- a/debian/python3-osmclient.postinst +++ b/debian/python3-osmclient.postinst @@ -15,8 +15,7 @@ ## echo "POST INSTALL OSMCLIENT" -#Install pyangbind, required for python3-osm-im -python3 -m pip install pyangbind verboselogs + #configure autocomplete for osmclient [ -z "$SUDO_USER" ] && SUDO_USER="$USER" su $SUDO_USER -c 'mkdir -p $HOME/.bash_completion.d' @@ -27,4 +26,3 @@ then echo " inserting .bash_completion.d/osm-complete.sh execution at .bashrc" su $SUDO_USER -c 'echo ". ${HOME}/.bash_completion.d/osm-complete.sh" >> ${HOME}/.bashrc' fi - diff --git a/devops-stages/stage-build.sh b/devops-stages/stage-build.sh index f00cd3e..164b385 100755 --- a/devops-stages/stage-build.sh +++ b/devops-stages/stage-build.sh @@ -14,4 +14,4 @@ # limitations under the License. rm -rf deb_dist osmclient-*.tar.gz -tox -e build +tox -e dist diff --git a/devops-stages/stage-test.sh b/devops-stages/stage-test.sh index 0177ba3..a3f8e18 100755 --- a/devops-stages/stage-test.sh +++ b/devops-stages/stage-test.sh @@ -11,5 +11,5 @@ # implied. # See the License for the specific language governing permissions and # limitations under the License. -tox --recreate +tox diff --git a/requirements-dev.in b/requirements-dev.in new file mode 100644 index 0000000..04d9320 --- /dev/null +++ b/requirements-dev.in @@ -0,0 +1,17 @@ +# Copyright ETSI Contributors and Others. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +git+https://osm.etsi.org/gerrit/osm/IM.git@v9.0#egg=osm-im +-r https://osm.etsi.org/gitweb/?p=osm/IM.git;a=blob_plain;f=requirements.txt;hb=v9.0 diff --git a/requirements-dev.txt b/requirements-dev.txt new file mode 100644 index 0000000..6c5d895 --- /dev/null +++ b/requirements-dev.txt @@ -0,0 +1,47 @@ +bitarray==1.8.1 + # via + # -r https://osm.etsi.org/gitweb/?p=osm/IM.git;a=blob_plain;f=requirements.txt;hb=v9.0 + # pyangbind +enum34==1.1.10 + # via + # -r https://osm.etsi.org/gitweb/?p=osm/IM.git;a=blob_plain;f=requirements.txt;hb=v9.0 + # pyangbind +lxml==4.6.3 + # via + # -r https://osm.etsi.org/gitweb/?p=osm/IM.git;a=blob_plain;f=requirements.txt;hb=v9.0 + # pyang + # pyangbind +git+https://osm.etsi.org/gerrit/osm/IM.git@v9.0#egg=osm-im + # via -r requirements-dev.in +pyang==2.4.0 + # via + # -r https://osm.etsi.org/gitweb/?p=osm/IM.git;a=blob_plain;f=requirements.txt;hb=v9.0 + # pyangbind +pyangbind==0.8.1 + # via -r https://osm.etsi.org/gitweb/?p=osm/IM.git;a=blob_plain;f=requirements.txt;hb=v9.0 +pyyaml==5.4.1 + # via -r https://osm.etsi.org/gitweb/?p=osm/IM.git;a=blob_plain;f=requirements.txt;hb=v9.0 +regex==2021.3.17 + # via + # -r https://osm.etsi.org/gitweb/?p=osm/IM.git;a=blob_plain;f=requirements.txt;hb=v9.0 + # pyangbind +six==1.15.0 + # via + # -r https://osm.etsi.org/gitweb/?p=osm/IM.git;a=blob_plain;f=requirements.txt;hb=v9.0 + # pyangbind +####################################################################################### +# Copyright ETSI Contributors and Others. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. +####################################################################################### diff --git a/requirements-dist.in b/requirements-dist.in new file mode 100644 index 0000000..11f0a2a --- /dev/null +++ b/requirements-dist.in @@ -0,0 +1,17 @@ +# Copyright ETSI Contributors and Others. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +stdeb +setuptools-version-command \ No newline at end of file diff --git a/requirements-dist.txt b/requirements-dist.txt new file mode 100644 index 0000000..7393626 --- /dev/null +++ b/requirements-dist.txt @@ -0,0 +1,23 @@ +setuptools-version-command==2.2 + # via -r requirements-dist.in +stdeb==0.10.0 + # via -r requirements-dist.in + +# The following packages are considered to be unsafe in a requirements file: +# setuptools +####################################################################################### +# Copyright ETSI Contributors and Others. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. +####################################################################################### diff --git a/requirements-test.in b/requirements-test.in new file mode 100755 index 0000000..9a0542d --- /dev/null +++ b/requirements-test.in @@ -0,0 +1,21 @@ +# Copyright 2020 ArctosLabs Scandinavia AB +# ************************************************************* + +# This file is part of OSM Placement module +# All Rights Reserved to ArctosLabs Scandinavia AB + +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at + +# http://www.apache.org/licenses/LICENSE-2.0 + +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +coverage +mock +nose2 diff --git a/requirements-test.txt b/requirements-test.txt new file mode 100644 index 0000000..d0c984f --- /dev/null +++ b/requirements-test.txt @@ -0,0 +1,26 @@ +coverage==5.5 + # via + # -r requirements-test.in + # nose2 +mock==4.0.3 + # via -r requirements-test.in +nose2==0.10.0 + # via -r requirements-test.in +six==1.15.0 + # via nose2 +####################################################################################### +# Copyright ETSI Contributors and Others. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. +####################################################################################### diff --git a/requirements.in b/requirements.in new file mode 100644 index 0000000..44fea8b --- /dev/null +++ b/requirements.in @@ -0,0 +1,25 @@ +# Copyright 2019 ETSI OSM +# +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +Click +jinja2 +packaging +prettytable +pycurl +python-magic +pyyaml +requests +verboselogs diff --git a/requirements.txt b/requirements.txt index 0aa10a7..ac1825e 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,26 +1,48 @@ -# Copyright 2019 ETSI OSM +certifi==2020.12.5 + # via requests +chardet==4.0.0 + # via requests +click==7.1.2 + # via -r requirements.in +idna==2.10 + # via requests +jinja2==2.11.3 + # via -r requirements.in +markupsafe==1.1.1 + # via jinja2 +packaging==20.9 + # via -r requirements.in +prettytable==2.1.0 + # via -r requirements.in +pycurl==7.43.0.6 + # via -r requirements.in +pyparsing==2.4.7 + # via packaging +python-magic==0.4.22 + # via -r requirements.in +pyyaml==5.4.1 + # via -r requirements.in +requests==2.25.1 + # via -r requirements.in +urllib3==1.26.4 + # via requests +verboselogs==1.7 + # via -r requirements.in +wcwidth==0.2.5 + # via prettytable +####################################################################################### +# Copyright ETSI Contributors and Others. # -# All Rights Reserved. +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at # -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at +# http://www.apache.org/licenses/LICENSE-2.0 # -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -Click -prettytable<=1.0.1 -PyYAML -pycurl -python-magic -jinja2 -verboselogs -packaging -requests -git+https://osm.etsi.org/gerrit/osm/IM.git@v9.0#egg=osm-im +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. +####################################################################################### diff --git a/setup.py b/setup.py index 43ce3ee..e2be9f4 100644 --- a/setup.py +++ b/setup.py @@ -27,10 +27,6 @@ setup( maintainer_email='gerardo.garciadeblas@telefonica.com', description=_description, license='Apache 2', - install_requires=[ - 'Click', 'prettytable', 'pyyaml', 'pycurl', 'python-magic', - 'jinja2', 'osm-im', 'verboselogs', 'packaging', 'requests', - ], setup_requires=['setuptools-version-command'], test_suite='nose.collector', entry_points={ diff --git a/stdeb.cfg b/stdeb.cfg index 9629697..d96d374 100644 --- a/stdeb.cfg +++ b/stdeb.cfg @@ -16,6 +16,3 @@ X-Python3-Version : >= 3.5 Suite: xenial Build-Depends: dh-python Maintainer: Gerardo Garcia -Depends: python-setuptools, python-pycurl, python-click, python-prettytable, python-yaml, python-osm-im -Depends3: python3-pycurl, python3-click, python3-prettytable, python3-yaml, python3-osm-im - diff --git a/test-requirements.txt b/test-requirements.txt deleted file mode 100644 index e944b57..0000000 --- a/test-requirements.txt +++ /dev/null @@ -1,18 +0,0 @@ -# Copyright 2019 ETSI OSM -# -# All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -nose -mock diff --git a/tox.ini b/tox.ini index 608dc8d..b6b5e4d 100644 --- a/tox.ini +++ b/tox.ini @@ -1,45 +1,125 @@ -# Copyright 2019 ETSI OSM +####################################################################################### +# Copyright ETSI Contributors and Others. # -# All Rights Reserved. +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at # -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. +# http://www.apache.org/licenses/LICENSE-2.0 # +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. +####################################################################################### + [tox] -envlist = py36,flakes -toxworkdir={toxinidir}/.tox +envlist = black, cover, flake8, pylint, safety -[testenv] -deps = -r{toxinidir}/requirements.txt - -r{toxinidir}/test-requirements.txt -install_command = python3 -m pip install -U {opts} {packages} -commands=nosetests +[tox:jenkins] +toxworkdir = /tmp/.tox -[testenv:flakes] +[testenv] +usedevelop = True basepython = python3 +setenv = VIRTUAL_ENV={envdir} + PYTHONDONTWRITEBYTECODE = 1 +deps = -r{toxinidir}/requirements.txt +parallel_show_output = true + +####################################################################################### +[testenv:black] +deps = black +skip_install = true +commands = + - black --check --diff osmclient/ + + +####################################################################################### +[testenv:cover] +deps = {[testenv]deps} + -r{toxinidir}/requirements-dev.txt + -r{toxinidir}/requirements-test.txt +commands = + sh -c 'rm -f nosetests.xml' + coverage erase + nose2 -C --coverage osmclient + coverage report --omit='*tests*' + coverage html -d ./cover --omit='*tests*' + coverage xml -o coverage.xml --omit=*tests* +whitelist_externals = sh + + +####################################################################################### +[testenv:flake8] deps = flake8 - pyflakes - -r{toxinidir}/requirements.txt - -r{toxinidir}/test-requirements.txt -install_command = python3 -m pip install -U {opts} {packages} commands = - flake8 setup.py - pyflakes osmclient + - flake8 osmclient/ setup.py -[testenv:build] -basepython = python3 -deps = stdeb - setuptools-version-command - -r{toxinidir}/requirements.txt -install_command = python3 -m pip install -U {opts} {packages} -commands = python3 setup.py --command-packages=stdeb.command bdist_deb + +####################################################################################### +[testenv:pylint] +deps = {[testenv]deps} + -r{toxinidir}/requirements-dev.txt + -r{toxinidir}/requirements-test.txt + pylint +commands = + - pylint -E osmclient + + +####################################################################################### +[testenv:safety] +setenv = + LC_ALL=C.UTF-8 + LANG=C.UTF-8 +deps = {[testenv]deps} + safety +commands = + - safety check --full-report + + +####################################################################################### +[testenv:pip-compile] +deps = pip-tools==5.5.0 +commands = + - sh -c 'for file in requirements*.in ; do pip-compile -rU --no-header $file ;\ + out=`echo $file | sed "s/.in/.txt/"` ; \ + head -16 tox.ini >> $out ;\ + done' +whitelist_externals = sh + + +####################################################################################### +[testenv:dist] +deps = {[testenv]deps} + -r{toxinidir}/requirements-dist.txt + +# In the commands, we copy the requirements.txt to be presented as a source file (.py) +# so it gets included in the .deb package for others to consume +commands = + sh -c 'cp requirements.txt osmclient/requirements.txt' + python3 setup.py --command-packages=stdeb.command sdist_dsc + sh -c 'cd deb_dist/osmclient*/ && dpkg-buildpackage -rfakeroot -uc -us' + sh -c 'rm osmclient/requirements.txt' +whitelist_externals = sh + +####################################################################################### +[flake8] +ignore = + W291, + W293, + W503, + E123, + E125, + E226, + E241 +exclude = + .git, + __pycache__, + .tox, +max-line-length = 128 +show-source = True +builtins = _