Bug 1846: LTS Support

Updates base to Ubuntu 20.04
Updates python to 3.8
Updates pip dependencies to more recent versions
Adds apt cache to stage 2 dockerfile

Fixes bug 1846

Change-Id: I589199979818b0fe5892df1d40849da8f6e06ded
Signed-off-by: beierlm <mark.beierl@canonical.com>

diff --git a/Dockerfile b/Dockerfile
index 1a95764..026d471 100644 (file)
--- a/Dockerfile
+++ b/Dockerfile
@@ -21,19 +21,26 @@
 #   devops-stages/stage-build.sh
 #
 
-FROM ubuntu:18.04
+FROM ubuntu:20.04
+
+ARG APT_PROXY
+RUN if [ ! -z $APT_PROXY ] ; then \
+    echo "Acquire::http::Proxy \"$APT_PROXY\";" > /etc/apt/apt.conf.d/proxy.conf ;\
+    echo "Acquire::https::Proxy \"$APT_PROXY\";" >> /etc/apt/apt.conf.d/proxy.conf ;\
+    fi
 
 RUN DEBIAN_FRONTEND=noninteractive apt-get update && \
     DEBIAN_FRONTEND=noninteractive apt-get -y install \
         debhelper \
+        dh-python \
         git \
-        python3 \
+        python3.8 \
         python3-all \
-        python3-dev \
+        python3.8-dev \
         python3-setuptools
 
-RUN python3 -m easy_install pip==21.0.1
-RUN pip3 install tox==3.22.0
+RUN python3 -m easy_install pip==21.3.1
+RUN pip install tox==3.24.5
 
 RUN DEBIAN_FRONTEND=noninteractive apt-get -y install \
     libcurl4-openssl-dev \
@@ -41,4 +48,4 @@ RUN DEBIAN_FRONTEND=noninteractive apt-get -y install \
     wget
 
 ENV LC_ALL C.UTF-8
-ENV LANG C.UTF-8
\ No newline at end of file
+ENV LANG C.UTF-8

diff --git a/requirements-dev.txt b/requirements-dev.txt
index 27c3446..9cd1b87 100644 (file)
--- a/requirements-dev.txt
+++ b/requirements-dev.txt
@@ -1,4 +1,20 @@
-bitarray==1.8.1
+#######################################################################################
+# Copyright ETSI Contributors and Others.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+# implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#######################################################################################
+bitarray==2.3.5
     # via
     #   -r https://osm.etsi.org/gitweb/?p=osm/IM.git;a=blob_plain;f=requirements.txt;hb=v10.0
     #   pyangbind
@@ -6,14 +22,14 @@ enum34==1.1.10
     # via
     #   -r https://osm.etsi.org/gitweb/?p=osm/IM.git;a=blob_plain;f=requirements.txt;hb=v10.0
     #   pyangbind
-lxml==4.6.3
+lxml==4.7.1
     # via
     #   -r https://osm.etsi.org/gitweb/?p=osm/IM.git;a=blob_plain;f=requirements.txt;hb=v10.0
     #   pyang
     #   pyangbind
-git+https://osm.etsi.org/gerrit/osm/IM.git@v10.0#egg=osm-im
+osm-im @ git+https://osm.etsi.org/gerrit/osm/IM.git@master
     # via -r requirements-dev.in
-pyang==2.4.0
+pyang==2.5.2
     # via
     #   -r https://osm.etsi.org/gitweb/?p=osm/IM.git;a=blob_plain;f=requirements.txt;hb=v10.0
     #   pyangbind
@@ -21,27 +37,11 @@ pyangbind==0.8.1
     # via -r https://osm.etsi.org/gitweb/?p=osm/IM.git;a=blob_plain;f=requirements.txt;hb=v10.0
 pyyaml==5.4.1
     # via -r https://osm.etsi.org/gitweb/?p=osm/IM.git;a=blob_plain;f=requirements.txt;hb=v10.0
-regex==2021.3.17
+regex==2021.11.10
     # via
     #   -r https://osm.etsi.org/gitweb/?p=osm/IM.git;a=blob_plain;f=requirements.txt;hb=v10.0
     #   pyangbind
-six==1.15.0
+six==1.16.0
     # via
     #   -r https://osm.etsi.org/gitweb/?p=osm/IM.git;a=blob_plain;f=requirements.txt;hb=v10.0
     #   pyangbind
-#######################################################################################
-# Copyright ETSI Contributors and Others.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#    http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
-# implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#######################################################################################

diff --git a/requirements-dist.in b/requirements-dist.in
index 11f0a2a..4f8784f 100644 (file)
--- a/requirements-dist.in
+++ b/requirements-dist.in
@@ -14,4 +14,5 @@
 # limitations under the License.
 
 stdeb
-setuptools-version-command
\ No newline at end of file
+setuptools-version-command
+setuptools<60
\ No newline at end of file

diff --git a/requirements-dist.txt b/requirements-dist.txt
index 7393626..8192b48 100644 (file)
--- a/requirements-dist.txt
+++ b/requirements-dist.txt
@@ -1,10 +1,3 @@
-setuptools-version-command==2.2
-    # via -r requirements-dist.in
-stdeb==0.10.0
-    # via -r requirements-dist.in
-
-# The following packages are considered to be unsafe in a requirements file:
-# setuptools
 #######################################################################################
 # Copyright ETSI Contributors and Others.
 #
@@ -21,3 +14,13 @@ stdeb==0.10.0
 # See the License for the specific language governing permissions and
 # limitations under the License.
 #######################################################################################
+setuptools-version-command==99.9
+    # via -r requirements-dist.in
+stdeb==0.10.0
+    # via -r requirements-dist.in
+
+# The following packages are considered to be unsafe in a requirements file:
+setuptools==59.8.0
+    # via
+    #   -r requirements-dist.in
+    #   setuptools-version-command

diff --git a/requirements-test.txt b/requirements-test.txt
index d0c984f..7589d84 100644 (file)
--- a/requirements-test.txt
+++ b/requirements-test.txt
@@ -1,13 +1,3 @@
-coverage==5.5
-    # via
-    #   -r requirements-test.in
-    #   nose2
-mock==4.0.3
-    # via -r requirements-test.in
-nose2==0.10.0
-    # via -r requirements-test.in
-six==1.15.0
-    # via nose2
 #######################################################################################
 # Copyright ETSI Contributors and Others.
 #
@@ -24,3 +14,13 @@ six==1.15.0
 # See the License for the specific language governing permissions and
 # limitations under the License.
 #######################################################################################
+coverage==6.2
+    # via
+    #   -r requirements-test.in
+    #   nose2
+mock==4.0.3
+    # via -r requirements-test.in
+nose2==0.10.0
+    # via -r requirements-test.in
+six==1.16.0
+    # via nose2

diff --git a/requirements.in b/requirements.in
index 44fea8b..e73bacc 100644 (file)
--- a/requirements.in
+++ b/requirements.in
@@ -20,6 +20,6 @@ packaging
 prettytable
 pycurl
 python-magic
-pyyaml
+pyyaml==5.4.1
 requests
 verboselogs

diff --git a/requirements.txt b/requirements.txt
index ac1825e..049c791 100644 (file)
--- a/requirements.txt
+++ b/requirements.txt
@@ -1,48 +1,48 @@
-certifi==2020.12.5
+#######################################################################################
+# Copyright ETSI Contributors and Others.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+# implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#######################################################################################
+certifi==2021.10.8
     # via requests
-chardet==4.0.0
+charset-normalizer==2.0.10
     # via requests
-click==7.1.2
+click==8.0.3
     # via -r requirements.in
-idna==2.10
+idna==3.3
     # via requests
-jinja2==2.11.3
+jinja2==3.0.3
     # via -r requirements.in
-markupsafe==1.1.1
+markupsafe==2.0.1
     # via jinja2
-packaging==20.9
+packaging==21.3
     # via -r requirements.in
-prettytable==2.1.0
+prettytable==3.0.0
     # via -r requirements.in
-pycurl==7.43.0.6
+pycurl==7.44.1
     # via -r requirements.in
-pyparsing==2.4.7
+pyparsing==3.0.6
     # via packaging
-python-magic==0.4.22
+python-magic==0.4.24
     # via -r requirements.in
 pyyaml==5.4.1
     # via -r requirements.in
-requests==2.25.1
+requests==2.27.1
     # via -r requirements.in
-urllib3==1.26.4
+urllib3==1.26.8
     # via requests
 verboselogs==1.7
     # via -r requirements.in
 wcwidth==0.2.5
     # via prettytable
-#######################################################################################
-# Copyright ETSI Contributors and Others.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#    http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
-# implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#######################################################################################

diff --git a/snap/snapcraft.yaml b/snap/snapcraft.yaml
index d950852..92c0828 100644 (file)
--- a/snap/snapcraft.yaml
+++ b/snap/snapcraft.yaml
@@ -15,7 +15,7 @@
 # under the License.
 ##
 name: osmclient
-base: core18
+base: core20
 adopt-info: client
 summary: A python client for orchestrating OSM
 description: |
@@ -31,22 +31,28 @@ environment:
 
 apps:
   osm:
-    command: usr/bin/python3 $SNAP/usr/local/bin/osm
+    command: bin/osm
     plugs:
       - network
       - home
       - ssh-public-keys
     environment:
-      PATH: $SNAP/usr/bin:$SNAP/bin/:$PATH:$SNAP/usr/local/bin/
-      PYTHONPATH: $SNAP/usr/lib/python3/dist-packages:$PYTHONPATH
+      PATH: $SNAP/usr/bin:$SNAP/bin/:$PATH:$SNAP/usr/local/bin/:/usr/local/bin/
       MAGIC: $SNAP/usr/share/file/magic.mgc
 
 parts:
   client:
-    plugin: dump
+    plugin: python
     source: .
+    requirements:
+      - requirements-dev.txt
+      - requirements.txt
+    python-packages:
+      - pip==21.0.1
+      - setuptools==44.0.0
     build-environment:
-      - PATH: "$PATH:$SNAPCRAFT_PART_INSTALL/usr/local/bin"
+      - PIP_COMPILE: "false" # disable .pyc generation in pip
+      - PYTHONDONTWRITEBYTECODE: "false" # disable .pyc generation by setup.py
     build-packages:
       - gcc
       - git
@@ -54,26 +60,17 @@ parts:
       - wget
       - libcurl4-openssl-dev
       - libssl-dev
-    stage-packages:
-      - libmagic1
-      - python3
+      - python3-distutils
       - python3-dev
       - python3-pip
       - python3-pycurl
-      - python3-setuptools
+    stage-packages:
+      - libmagic1
+      - libcurl4-openssl-dev
+      - python3
     override-build: |
-      $SNAPCRAFT_PART_INSTALL/usr/bin/pip3 install wheel
-      $SNAPCRAFT_PART_INSTALL/usr/bin/pip3 install pyang
-      $SNAPCRAFT_PART_INSTALL/usr/bin/pip3 install pyangbind
-      PATH="$PATH:$SNAPCRAFT_PART_INSTALL/usr/local/bin"
-      BRANCH_OR_TAG=`git symbolic-ref -q --short HEAD || git describe --tags --exact-match 2>/dev/null || echo $BRANCH`
-      git clone https://osm.etsi.org/gerrit/osm/IM.git
-      cd IM/
-      git checkout $BRANCH_OR_TAG
-      $SNAPCRAFT_PART_INSTALL/usr/bin/pip3 install .
-      cd ../ && rm -rf IM/
-      $SNAPCRAFT_PART_INSTALL/usr/bin/pip3 install -r requirements.txt
-      $SNAPCRAFT_PART_INSTALL/usr/bin/pip3 install .
-      rm -rf .tox
-      snapcraftctl set-version $(git describe --match v* --tags --long --dirty)
+      VERSION=$(git describe --match v* --tags --long --dirty)
+      snapcraftctl set-version $VERSION
       snapcraftctl build
+      `find $SNAPCRAFT_PART_INSTALL -name '__pycache__' | xargs rm -r`
+      `find $SNAPCRAFT_PART_INSTALL -name 'RECORD' | xargs rm`

diff --git a/tox.ini b/tox.ini
index 083443b..ca296ee 100644 (file)
--- a/tox.ini
+++ b/tox.ini
@@ -23,7 +23,7 @@ toxworkdir = /tmp/.tox
 
 [testenv]
 usedevelop = True
-basepython = python3
+basepython = python3.8
 setenv = VIRTUAL_ENV={envdir}
          PYTHONDONTWRITEBYTECODE = 1
 deps =  -r{toxinidir}/requirements.txt
@@ -82,13 +82,18 @@ commands =
 
 #######################################################################################
 [testenv:pip-compile]
-deps =  pip-tools==5.5.0
+deps =  pip-tools==6.4.0
+skip_install = true
+whitelist_externals = bash
+        [
 commands =
-        - sh -c 'for file in requirements*.in ; do pip-compile -rU --no-header $file ;\
-        out=`echo $file | sed "s/.in/.txt/"` ; \
-        head -16 tox.ini >> $out ;\
-        done'
-whitelist_externals = sh
+        - bash -c "for file in requirements*.in ; do \
+        UNSAFE="" ; \
+        if [[ $file =~ 'dist' ]] ; then UNSAFE='--allow-unsafe' ; fi ; \
+        pip-compile -rU --no-header $UNSAFE $file ;\
+        out=`echo $file | sed 's/.in/.txt/'` ; \
+        sed -i -e '1 e head -16 tox.ini' $out ;\
+        done"
 
 
 #######################################################################################