Update from master

author Dario Faccin <dario.faccin@canonical.com>

Tue, 23 May 2023 15:44:49 +0000 (17:44 +0200)

committer Dario Faccin <dario.faccin@canonical.com>

Wed, 24 May 2023 09:38:30 +0000 (11:38 +0200)
author Dario Faccin <dario.faccin@canonical.com>
Tue, 23 May 2023 15:44:49 +0000 (17:44 +0200)
committer Dario Faccin <dario.faccin@canonical.com>
Wed, 24 May 2023 09:38:30 +0000 (11:38 +0200)
diff --git a/docker/Airflow/Dockerfile b/docker/Airflow/Dockerfile

index bf72444..1a4184c 100644 (file)
--- a/docker/Airflow/Dockerfile
+++ b/docker/Airflow/Dockerfile
@@ -15,12 +15,12 @@
  # limitations under the License.
  #######################################################################################
  
-FROM apache/airflow:2.5.2-python3.10
+FROM apache/airflow:2.5.3-python3.10
  USER root
  RUN DEBIAN_FRONTEND=noninteractive apt-get --yes update && \
      DEBIAN_FRONTEND=noninteractive apt-get --yes install \
      gcc git python3 python3-dev python3-venv python3-pip \
-    python3-setuptools wget && \
+    python3-setuptools wget zstd && \
      python3 -m pip install -U pip build
  
  USER airflow
@@ -29,14 +29,25 @@ ARG PYTHON3_OSM_COMMON_URL
  ARG PYTHON3_OSM_NGSA_URL
  RUN curl $PYTHON3_OSM_COMMON_URL -o osm_common.deb
  RUN curl $PYTHON3_OSM_NGSA_URL -o osm_ngsa.deb
-RUN mkdir /tmp/osm
-RUN dpkg-deb -x osm_common.deb /tmp/osm
-RUN dpkg-deb -x osm_ngsa.deb /tmp/osm
  
+RUN ar x osm_common.deb && \
+    zstd -d < control.tar.zst | xz > control.tar.xz && \
+    zstd -d < data.tar.zst | xz > data.tar.xz && \
+    ar -m -c -a sdsd osm_common_repacked.deb debian-binary control.tar.xz data.tar.xz && \
+    rm debian-binary control.tar.xz data.tar.xz control.tar.zst data.tar.zst
+RUN ar x osm_ngsa.deb && \
+   zstd -d < control.tar.zst | xz > control.tar.xz && \
+   zstd -d < data.tar.zst | xz > data.tar.xz && \
+   ar -m -c -a sdsd osm_ngsa_repacked.deb debian-binary control.tar.xz data.tar.xz && \
+   rm debian-binary control.tar.xz data.tar.xz control.tar.zst data.tar.zst
+
+RUN mkdir /tmp/osm
+RUN dpkg-deb -x osm_common_repacked.deb /tmp/osm
+RUN dpkg-deb -x osm_ngsa_repacked.deb /tmp/osm
  RUN mv /tmp/osm/usr/lib/python3/dist-packages/* /home/airflow/.local/lib/python3.10/site-packages/
  RUN rm -rf /tmp/osm
  
  RUN pip3 install \
-    -r /home/airflow/.local/lib/python3.10/site-packages/osm_common/requirements.txt \
-    -r /home/airflow/.local/lib/python3.10/site-packages/osm_ngsa/requirements.txt
+   -r /home/airflow/.local/lib/python3.10/site-packages/osm_common/requirements.txt \
+   -r /home/airflow/.local/lib/python3.10/site-packages/osm_ngsa/requirements.txt
  
diff --git a/docker/LCM/Dockerfile b/docker/LCM/Dockerfile

index 48fcede..a0a0935 100644 (file)
--- a/docker/LCM/Dockerfile
+++ b/docker/LCM/Dockerfile
@@ -15,7 +15,7 @@
  # limitations under the License.
  #######################################################################################
  
-FROM ubuntu:20.04 as INSTALL
+FROM ubuntu:22.04 as INSTALL
  
  ARG APT_PROXY
  RUN if [ ! -z $APT_PROXY ] ; then \
@@ -25,18 +25,21 @@ RUN if [ ! -z $APT_PROXY ] ; then \
  
  RUN DEBIAN_FRONTEND=noninteractive apt-get --yes update && \
      DEBIAN_FRONTEND=noninteractive apt-get --yes install \
-    gcc=4:9.3.* \
-    python3=3.8.* \
-    python3-dev=3.8.* \
-    python3-pip=20.0.2* \
-    python3-setuptools=45.2.* \
-    curl=7.68.*
+    gcc=4:11.* \
+    python3=3.10.* \
+    python3-dev=3.10.* \
+    python3-pip=22.0.* \
+    curl=7.81.* \
+    && rm -rf /var/lib/apt/lists/*
+
+########### End of common prepration
  
  RUN DEBIAN_FRONTEND=noninteractive apt-get --yes update && \
      DEBIAN_FRONTEND=noninteractive apt-get --yes install \
-    apt-transport-https=2.0.* \
+    apt-transport-https=2.4.* \
      gnupg2=2.2.* \
-    openssh-client=1:8.*
+    openssh-client=1:8.* \
+    && rm -rf /var/lib/apt/lists/*
  
  # https://kubernetes.io/releases/
  RUN curl -s https://packages.cloud.google.com/apt/doc/apt-key.gpg | apt-key add - \
@@ -73,7 +76,7 @@ RUN pip3 install \
      -r /usr/lib/python3/dist-packages/n2vc/requirements.txt
  
  #######################################################################################
-FROM ubuntu:20.04 as FINAL
+FROM ubuntu:22.04 as FINAL
  
  ARG APT_PROXY
  RUN if [ ! -z $APT_PROXY ] ; then \
@@ -83,9 +86,15 @@ RUN if [ ! -z $APT_PROXY ] ; then \
  
  RUN DEBIAN_FRONTEND=noninteractive apt-get --yes update && \
      DEBIAN_FRONTEND=noninteractive apt-get --yes install \
-    python3-minimal=3.8.* \
+    python3-minimal=3.10.* \
      && rm -rf /var/lib/apt/lists/*
  
+COPY --from=INSTALL /usr/lib/python3/dist-packages /usr/lib/python3/dist-packages
+COPY --from=INSTALL /usr/local/lib/python3.10/dist-packages  /usr/local/lib/python3.10/dist-packages
+
+#######################################################################################
+# End of common prepration
+
  RUN DEBIAN_FRONTEND=noninteractive apt-get --yes update && \
      DEBIAN_FRONTEND=noninteractive apt-get --yes install \
      ca-certificates \
@@ -93,9 +102,6 @@ RUN DEBIAN_FRONTEND=noninteractive apt-get --yes update && \
  
  RUN rm -f /etc/apt/apt.conf.d/proxy.conf
  
-COPY --from=INSTALL /usr/lib/python3/dist-packages /usr/lib/python3/dist-packages
-COPY --from=INSTALL /usr/local/lib/python3.8/dist-packages  /usr/local/lib/python3.8/dist-packages
-
  COPY --from=INSTALL /usr/bin/kubectl /usr/bin/kubectl
  COPY --from=INSTALL /usr/local/bin/helm /usr/local/bin/helm
  COPY --from=INSTALL /usr/local/bin/helm3 /usr/local/bin/helm3
diff --git a/docker/MON/Dockerfile b/docker/MON/Dockerfile

index cd56f53..11b6e14 100644 (file)
--- a/docker/MON/Dockerfile
+++ b/docker/MON/Dockerfile
@@ -15,7 +15,7 @@
  # limitations under the License.
  #######################################################################################
  
-FROM ubuntu:20.04 as INSTALL
+FROM ubuntu:22.04 as INSTALL
  
  ARG APT_PROXY
  RUN if [ ! -z $APT_PROXY ] ; then \
@@ -25,12 +25,15 @@ RUN if [ ! -z $APT_PROXY ] ; then \
  
  RUN DEBIAN_FRONTEND=noninteractive apt-get --yes update && \
      DEBIAN_FRONTEND=noninteractive apt-get --yes install \
-    gcc=4:9.3.* \
-    python3=3.8.* \
-    python3-dev=3.8.* \
-    python3-pip=20.0.2* \
-    python3-setuptools=45.2.* \
-    curl=7.68.*
+    gcc=4:11.* \
+    python3=3.10.* \
+    python3-dev=3.10.* \
+    python3-pip=22.0.* \
+    curl=7.81.* \
+    && rm -rf /var/lib/apt/lists/*
+
+#######################################################################################
+# End of common prepration
  
  RUN DEBIAN_FRONTEND=noninteractive apt-get --yes update && \
      DEBIAN_FRONTEND=noninteractive apt-get --yes install \
@@ -55,7 +58,8 @@ RUN pip3 install \
      -r /usr/lib/python3/dist-packages/osm_mon/requirements.txt
  
  #######################################################################################
-FROM ubuntu:20.04 as FINAL
+#######################################################################################
+FROM ubuntu:22.04 as FINAL
  
  ARG APT_PROXY
  RUN if [ ! -z $APT_PROXY ] ; then \
@@ -65,9 +69,15 @@ RUN if [ ! -z $APT_PROXY ] ; then \
  
  RUN DEBIAN_FRONTEND=noninteractive apt-get --yes update && \
      DEBIAN_FRONTEND=noninteractive apt-get --yes install \
-    python3-minimal=3.8.* \
+    python3-minimal=3.10.* \
      && rm -rf /var/lib/apt/lists/*
  
+COPY --from=INSTALL /usr/lib/python3/dist-packages /usr/lib/python3/dist-packages
+COPY --from=INSTALL /usr/local/lib/python3.10/dist-packages  /usr/local/lib/python3.10/dist-packages
+
+#######################################################################################
+# End of common prepration
+
  RUN DEBIAN_FRONTEND=noninteractive apt-get --yes update && \
      DEBIAN_FRONTEND=noninteractive apt-get --yes install \
      ca-certificates \
@@ -75,8 +85,6 @@ RUN DEBIAN_FRONTEND=noninteractive apt-get --yes update && \
  
  RUN rm -f /etc/apt/apt.conf.d/proxy.conf
  
-COPY --from=INSTALL /usr/lib/python3/dist-packages /usr/lib/python3/dist-packages
-COPY --from=INSTALL /usr/local/lib/python3.8/dist-packages  /usr/local/lib/python3.8/dist-packages
  COPY --from=INSTALL /usr/bin/osm* /usr/bin/
  COPY --from=INSTALL /usr/bin/scp /usr/bin/scp
  COPY --from=INSTALL /usr/bin/ssh-keygen /usr/bin/ssh-keygen
diff --git a/docker/NBI/Dockerfile b/docker/NBI/Dockerfile

index 8957479..d09ee05 100644 (file)
--- a/docker/NBI/Dockerfile
+++ b/docker/NBI/Dockerfile
@@ -15,7 +15,7 @@
  # limitations under the License.
  #######################################################################################
  
-FROM ubuntu:20.04 as INSTALL
+FROM ubuntu:22.04 as INSTALL
  
  ARG APT_PROXY
  RUN if [ ! -z $APT_PROXY ] ; then \
@@ -25,12 +25,15 @@ RUN if [ ! -z $APT_PROXY ] ; then \
  
  RUN DEBIAN_FRONTEND=noninteractive apt-get --yes update && \
      DEBIAN_FRONTEND=noninteractive apt-get --yes install \
-    gcc=4:9.3.* \
-    python3=3.8.* \
-    python3-dev=3.8.* \
-    python3-pip=20.0.2* \
-    python3-setuptools=45.2.* \
-    curl=7.68.*
+    gcc=4:11.* \
+    python3=3.10.* \
+    python3-dev=3.10.* \
+    python3-pip=22.0.* \
+    curl=7.81.* \
+    && rm -rf /var/lib/apt/lists/*
+
+#######################################################################################
+# End of common prepration
  
  ARG PYTHON3_OSM_COMMON_URL
  ARG PYTHON3_OSM_IM_URL
@@ -50,8 +53,11 @@ RUN pip3 install \
      -r /usr/lib/python3/dist-packages/osm_im/requirements.txt \
      -r /usr/lib/python3/dist-packages/osm_nbi/requirements.txt
  
+COPY pyangbind.patch ./
+RUN patch /usr/local/lib/python3.10/dist-packages/pyangbind/lib/yangtypes.py < ./pyangbind.patch
+
  #######################################################################################
-FROM ubuntu:20.04 as FINAL
+FROM ubuntu:22.04 as FINAL
  
  ARG APT_PROXY
  RUN if [ ! -z $APT_PROXY ] ; then \
@@ -61,13 +67,16 @@ RUN if [ ! -z $APT_PROXY ] ; then \
  
  RUN DEBIAN_FRONTEND=noninteractive apt-get --yes update && \
      DEBIAN_FRONTEND=noninteractive apt-get --yes install \
-    python3-minimal=3.8.* \
+    python3-minimal=3.10.* \
      && rm -rf /var/lib/apt/lists/*
  
-RUN rm -f /etc/apt/apt.conf.d/proxy.conf
-
  COPY --from=INSTALL /usr/lib/python3/dist-packages /usr/lib/python3/dist-packages
-COPY --from=INSTALL /usr/local/lib/python3.8/dist-packages  /usr/local/lib/python3.8/dist-packages
+COPY --from=INSTALL /usr/local/lib/python3.10/dist-packages  /usr/local/lib/python3.10/dist-packages
+
+#######################################################################################
+# End of common prepration
+
+RUN rm -f /etc/apt/apt.conf.d/proxy.conf
  
  # Creating the user for the app
  RUN groupadd -g 1000 appuser && \
diff --git a/docker/NBI/pyangbind.patch b/docker/NBI/pyangbind.patch

new file mode 100644 (file)

index 0000000..3077299
--- /dev/null
+++ b/docker/NBI/pyangbind.patch
@@ -0,0 +1,46 @@
+#######################################################################################
+# Copyright ETSI Contributors and Others.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+# implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#######################################################################################
+
+*** .tox/cover/lib/python3.10/site-packages/pyangbind/lib/yangtypes.py 2023-05-10 06:50:57.876027148 -0400
+--- .tox/cover/lib/python3.10/site-packages/pyangbind/lib/yangtypes.py 2023-05-10 06:51:11.772022417 -0400
+*************** limitations under the License.
+*** 22,27 ****
+--- 22,28 ----
+  from __future__ import unicode_literals
+
+  import collections
++ from six.moves import collections_abc
+  import copy
+  import uuid
+  from decimal import Decimal
+*************** def TypedListType(*args, **kwargs):
+*** 372,378 ****
+    if not isinstance(allowed_type, list):
+      allowed_type = [allowed_type]
+
+!   class TypedList(collections.MutableSequence):
+      _pybind_generated_by = "TypedListType"
+      _list = list()
+
+--- 373,379 ----
+    if not isinstance(allowed_type, list):
+      allowed_type = [allowed_type]
+
+!   class TypedList(collections_abc.MutableSequence):
+      _pybind_generated_by = "TypedListType"
+      _list = list()
+
diff --git a/docker/NG-UI/Dockerfile b/docker/NG-UI/Dockerfile

index 74aa1d4..418de25 100644 (file)
--- a/docker/NG-UI/Dockerfile
+++ b/docker/NG-UI/Dockerfile
@@ -33,9 +33,9 @@ RUN DEBIAN_FRONTEND=noninteractive apt-get --yes update && \
      xz-utils=5.2.*
  
  RUN curl -s https://deb.nodesource.com/gpgkey/nodesource.gpg.key | apt-key add - && \
-    echo "deb https://deb.nodesource.com/node_10.x focal main" | tee -a /etc/apt/sources.list.d/nodesource.list && \
+    echo "deb https://deb.nodesource.com/node_14.x focal main" | tee -a /etc/apt/sources.list.d/nodesource.list && \
      DEBIAN_FRONTEND=noninteractive apt-get update && \
-    DEBIAN_FRONTEND=noninteractive apt-get install -y nodejs=10.24.*
+    DEBIAN_FRONTEND=noninteractive apt-get install -y nodejs
  
  ARG OSM_NGUI_URL
  
diff --git a/docker/NGLCM/Dockerfile b/docker/NGLCM/Dockerfile

index 9ededf5..1344315 100644 (file)
--- a/docker/NGLCM/Dockerfile
+++ b/docker/NGLCM/Dockerfile
@@ -15,7 +15,7 @@
  # limitations under the License.
  #######################################################################################
  
-FROM ubuntu:20.04 as INSTALL
+FROM ubuntu:22.04 as INSTALL
  
  ARG APT_PROXY
  RUN if [ ! -z $APT_PROXY ] ; then \
@@ -25,18 +25,21 @@ RUN if [ ! -z $APT_PROXY ] ; then \
  
  RUN DEBIAN_FRONTEND=noninteractive apt-get --yes update && \
      DEBIAN_FRONTEND=noninteractive apt-get --yes install \
-    gcc=4:9.3.* \
-    python3=3.8.* \
-    python3-dev=3.8.* \
-    python3-pip=20.0.2* \
-    python3-setuptools=45.2.* \
-    curl=7.68.*
+    gcc=4:11.* \
+    python3=3.10.* \
+    python3-dev=3.10.* \
+    python3-pip=22.0.* \
+    curl=7.81.* \
+    && rm -rf /var/lib/apt/lists/*
+
+########### End of common prepration
  
  RUN DEBIAN_FRONTEND=noninteractive apt-get --yes update && \
      DEBIAN_FRONTEND=noninteractive apt-get --yes install \
-    apt-transport-https=2.0.* \
+    apt-transport-https=2.4.* \
      gnupg2=2.2.* \
-    openssh-client=1:8.*
+    openssh-client=1:8.* \
+    && rm -rf /var/lib/apt/lists/*
  
  # https://kubernetes.io/releases/
  RUN curl -s https://packages.cloud.google.com/apt/doc/apt-key.gpg | apt-key add - \
@@ -73,7 +76,7 @@ RUN pip3 install \
      -r /usr/lib/python3/dist-packages/n2vc/requirements.txt
  
  #######################################################################################
-FROM ubuntu:20.04 as FINAL
+FROM ubuntu:22.04 as FINAL
  
  ARG APT_PROXY
  RUN if [ ! -z $APT_PROXY ] ; then \
@@ -83,9 +86,15 @@ RUN if [ ! -z $APT_PROXY ] ; then \
  
  RUN DEBIAN_FRONTEND=noninteractive apt-get --yes update && \
      DEBIAN_FRONTEND=noninteractive apt-get --yes install \
-    python3-minimal=3.8.* \
+    python3-minimal=3.10.* \
      && rm -rf /var/lib/apt/lists/*
  
+COPY --from=INSTALL /usr/lib/python3/dist-packages /usr/lib/python3/dist-packages
+COPY --from=INSTALL /usr/local/lib/python3.10/dist-packages  /usr/local/lib/python3.10/dist-packages
+
+#######################################################################################
+# End of common prepration
+
  RUN DEBIAN_FRONTEND=noninteractive apt-get --yes update && \
      DEBIAN_FRONTEND=noninteractive apt-get --yes install \
      ca-certificates \
@@ -93,9 +102,6 @@ RUN DEBIAN_FRONTEND=noninteractive apt-get --yes update && \
  
  RUN rm -f /etc/apt/apt.conf.d/proxy.conf
  
-COPY --from=INSTALL /usr/lib/python3/dist-packages /usr/lib/python3/dist-packages
-COPY --from=INSTALL /usr/local/lib/python3.8/dist-packages  /usr/local/lib/python3.8/dist-packages
-
  COPY --from=INSTALL /usr/bin/kubectl /usr/bin/kubectl
  COPY --from=INSTALL /usr/local/bin/helm /usr/local/bin/helm
  COPY --from=INSTALL /usr/local/bin/helm3 /usr/local/bin/helm3
diff --git a/docker/PLA/Dockerfile b/docker/PLA/Dockerfile

index f637362..9c23262 100644 (file)
--- a/docker/PLA/Dockerfile
+++ b/docker/PLA/Dockerfile
@@ -15,7 +15,7 @@
  # limitations under the License.
  #######################################################################################
  
-FROM ubuntu:20.04 as INSTALL
+FROM ubuntu:22.04 as INSTALL
  
  ARG APT_PROXY
  RUN if [ ! -z $APT_PROXY ] ; then \
@@ -25,12 +25,15 @@ RUN if [ ! -z $APT_PROXY ] ; then \
  
  RUN DEBIAN_FRONTEND=noninteractive apt-get --yes update && \
      DEBIAN_FRONTEND=noninteractive apt-get --yes install \
-    gcc=4:9.3.* \
-    python3=3.8.* \
-    python3-dev=3.8.* \
-    python3-pip=20.0.2* \
-    python3-setuptools=45.2.* \
-    curl=7.68.*
+    gcc=4:11.* \
+    python3=3.10.* \
+    python3-dev=3.10.* \
+    python3-pip=22.0.* \
+    curl=7.81.* \
+    && rm -rf /var/lib/apt/lists/*
+
+#######################################################################################
+# End of common prepration
  
  ARG PYTHON3_OSM_COMMON_URL
  ARG PYTHON3_OSM_PLA_URL
@@ -51,7 +54,7 @@ RUN tar -zxf /minizinc.tgz && \
      mv /MiniZincIDE-2.4.2-bundle-linux /minizinc
  
  #######################################################################################
-FROM ubuntu:20.04 as FINAL
+FROM ubuntu:22.04 as FINAL
  
  ARG APT_PROXY
  RUN if [ ! -z $APT_PROXY ] ; then \
@@ -61,15 +64,19 @@ RUN if [ ! -z $APT_PROXY ] ; then \
  
  RUN DEBIAN_FRONTEND=noninteractive apt-get --yes update && \
      DEBIAN_FRONTEND=noninteractive apt-get --yes install \
-    python3-minimal=3.8.* \
+    python3-minimal=3.10.* \
      && rm -rf /var/lib/apt/lists/*
  
+COPY --from=INSTALL /usr/lib/python3/dist-packages /usr/lib/python3/dist-packages
+COPY --from=INSTALL /usr/local/lib/python3.10/dist-packages  /usr/local/lib/python3.10/dist-packages
+
+#######################################################################################
+# End of common prepration
+
  RUN rm -f /etc/apt/apt.conf.d/proxy.conf
  
  LABEL authors="Lars-Göran Magnusson"
  
-COPY --from=INSTALL /usr/lib/python3/dist-packages /usr/lib/python3/dist-packages
-COPY --from=INSTALL /usr/local/lib/python3.8/dist-packages /usr/local/lib/python3.8/dist-packages
  COPY --from=INSTALL /usr/bin/osm* /usr/bin/
  COPY --from=INSTALL /minizinc /minizinc
  
diff --git a/docker/POL/Dockerfile b/docker/POL/Dockerfile

index b807bfd..8ed8f5e 100644 (file)
--- a/docker/POL/Dockerfile
+++ b/docker/POL/Dockerfile
@@ -15,7 +15,7 @@
  # limitations under the License.
  #######################################################################################
  
-FROM ubuntu:20.04 as INSTALL
+FROM ubuntu:22.04 as INSTALL
  
  ARG APT_PROXY
  RUN if [ ! -z $APT_PROXY ] ; then \
@@ -25,12 +25,15 @@ RUN if [ ! -z $APT_PROXY ] ; then \
  
  RUN DEBIAN_FRONTEND=noninteractive apt-get --yes update && \
      DEBIAN_FRONTEND=noninteractive apt-get --yes install \
-    gcc=4:9.3.* \
-    python3=3.8.* \
-    python3-dev=3.8.* \
-    python3-pip=20.0.2* \
-    python3-setuptools=45.2.* \
-    curl=7.68.*
+    gcc=4:11.* \
+    python3=3.10.* \
+    python3-dev=3.10.* \
+    python3-pip=22.0.* \
+    curl=7.81.* \
+    && rm -rf /var/lib/apt/lists/*
+
+#######################################################################################
+# End of common prepration
  
  RUN DEBIAN_FRONTEND=noninteractive apt-get --yes update && \
      DEBIAN_FRONTEND=noninteractive apt-get --yes install \
@@ -50,7 +53,7 @@ RUN pip3 install \
      -r /usr/lib/python3/dist-packages/osm_policy_module/requirements.txt
  
  #######################################################################################
-FROM ubuntu:20.04 as FINAL
+FROM ubuntu:22.04 as FINAL
  
  ARG APT_PROXY
  RUN if [ ! -z $APT_PROXY ] ; then \
@@ -60,13 +63,15 @@ RUN if [ ! -z $APT_PROXY ] ; then \
  
  RUN DEBIAN_FRONTEND=noninteractive apt-get --yes update && \
      DEBIAN_FRONTEND=noninteractive apt-get --yes install \
-    python3-minimal=3.8.* \
+    python3-minimal=3.10.* \
      && rm -rf /var/lib/apt/lists/*
  
-RUN rm -f /etc/apt/apt.conf.d/proxy.conf
-
  COPY --from=INSTALL /usr/lib/python3/dist-packages /usr/lib/python3/dist-packages
-COPY --from=INSTALL /usr/local/lib/python3.8/dist-packages  /usr/local/lib/python3.8/dist-packages
+COPY --from=INSTALL /usr/local/lib/python3.10/dist-packages  /usr/local/lib/python3.10/dist-packages
+
+#######################################################################################
+# End of common prepration
+
  COPY --from=INSTALL /usr/bin/osm* /usr/bin/
  COPY --from=INSTALL /usr/bin/mysql /usr/bin/
  COPY --from=INSTALL /usr/bin/mysqladmin /usr/bin/
@@ -102,6 +107,10 @@ ENV OSMPOL_SQL_DATABASE_URI sqlite:///mon_sqlite.db
  
  ENV OSMPOL_GLOBAL_LOG_LEVEL INFO
  
+ENV OSMPOL_AUTOHEAL_ENABLED True
+
+ENV OSMPOL_AUTOSCALE_ENABLED True
+
  HEALTHCHECK --start-period=120s --interval=10s --timeout=5s --retries=5 \
    CMD osm-pol-healthcheck || exit 1
  
diff --git a/docker/RO/Dockerfile b/docker/RO/Dockerfile

index 64bad45..57c5f6a 100644 (file)
--- a/docker/RO/Dockerfile
+++ b/docker/RO/Dockerfile
@@ -15,7 +15,7 @@
  # limitations under the License.
  #######################################################################################
  
-FROM ubuntu:20.04 as INSTALL
+FROM ubuntu:22.04 as INSTALL
  
  ARG APT_PROXY
  RUN if [ ! -z $APT_PROXY ] ; then \
@@ -25,12 +25,15 @@ RUN if [ ! -z $APT_PROXY ] ; then \
  
  RUN DEBIAN_FRONTEND=noninteractive apt-get --yes update && \
      DEBIAN_FRONTEND=noninteractive apt-get --yes install \
-    gcc=4:9.3.* \
-    python3=3.8.* \
-    python3-dev=3.8.* \
-    python3-pip=20.0.2* \
-    python3-setuptools=45.2.* \
-    curl=7.68.*
+    gcc=4:11.* \
+    python3=3.10.* \
+    python3-dev=3.10.* \
+    python3-pip=22.0.* \
+    curl=7.81.* \
+    && rm -rf /var/lib/apt/lists/*
+
+#######################################################################################
+# End of common prepration
  
  RUN DEBIAN_FRONTEND=noninteractive apt-get --yes update && \
      DEBIAN_FRONTEND=noninteractive apt-get --yes install \
@@ -85,7 +88,8 @@ RUN pip3 install \
      -r /usr/lib/python3/dist-packages/osm_ng_ro/requirements.txt
  
  #######################################################################################
-FROM ubuntu:20.04 as FINAL
+#######################################################################################
+FROM ubuntu:22.04 as FINAL
  
  ARG APT_PROXY
  RUN if [ ! -z $APT_PROXY ] ; then \
@@ -95,13 +99,17 @@ RUN if [ ! -z $APT_PROXY ] ; then \
  
  RUN DEBIAN_FRONTEND=noninteractive apt-get --yes update && \
      DEBIAN_FRONTEND=noninteractive apt-get --yes install \
-    python3-minimal=3.8.* \
+    python3-minimal=3.10.* \
      && rm -rf /var/lib/apt/lists/*
  
+COPY --from=INSTALL /usr/lib/python3/dist-packages /usr/lib/python3/dist-packages
+COPY --from=INSTALL /usr/local/lib/python3.10/dist-packages  /usr/local/lib/python3.10/dist-packages
+
+#######################################################################################
+# End of common prepration
+
  RUN rm -f /etc/apt/apt.conf.d/proxy.conf
  
-COPY --from=INSTALL /usr/lib/python3/dist-packages /usr/lib/python3/dist-packages
-COPY --from=INSTALL /usr/local/lib/python3.8/dist-packages  /usr/local/lib/python3.8/dist-packages
  COPY --from=INSTALL /usr/bin/genisoimage /usr/bin/genisoimage
  COPY --from=INSTALL /etc/protocols /etc/protocols
  
diff --git a/docker/tests/Dockerfile b/docker/tests/Dockerfile

index 5eeda0d..2c824bc 100644 (file)
--- a/docker/tests/Dockerfile
+++ b/docker/tests/Dockerfile
@@ -15,7 +15,7 @@
  # limitations under the License.
  #######################################################################################
  
-FROM ubuntu:20.04 as FINAL
+FROM ubuntu:22.04 as FINAL
  
  ARG APT_PROXY
  RUN if [ ! -z $APT_PROXY ] ; then \
@@ -25,21 +25,25 @@ RUN if [ ! -z $APT_PROXY ] ; then \
  
  RUN DEBIAN_FRONTEND=noninteractive apt-get --yes update && \
      DEBIAN_FRONTEND=noninteractive apt-get --yes install \
-    gcc=4:9.3.* \
-    python3=3.8.* \
-    python3-dev=3.8.* \
-    python3-pip=20.0.2* \
-    python3-setuptools=45.2.* \
-    curl=7.68.*
+    gcc=4:11.* \
+    python3=3.10.* \
+    python3-dev=3.10.* \
+    python3-pip=22.0.* \
+    curl=7.81.* \
+    && rm -rf /var/lib/apt/lists/*
+
+#######################################################################################
+# End of common preparation
  
  RUN DEBIAN_FRONTEND=noninteractive apt-get --yes update && \
      DEBIAN_FRONTEND=noninteractive apt-get --yes install \
-    gettext-base=0.19.* \
-    git=1:2.25.* \
+    gettext-base=0.21* \
+    git=1:2.34.* \
      iputils-ping=3:* \
      jq=1.6* \
-    libcurl4-openssl-dev=7.68.* \
-    libssl-dev=1.1.*
+    libcurl4-openssl-dev=7.81.* \
+    libssl-dev=3.0.* \
+    libmagic1=1:5.*
  
  ARG OSM_TESTS_URL
  ARG PYTHON3_OSM_IM_URL
@@ -63,6 +67,9 @@ RUN mv /usr/share/osm-tests/robot-systest /robot-systest
  RUN mv /usr/share/osm-tests/conformance-tests/ /robot-systest/
  RUN mv /usr/share/osm-tests/charm.sh /usr/sbin/charm
  
+COPY pyangbind.patch /robot-systest
+RUN patch /usr/local/lib/python3.10/dist-packages/pyangbind/lib/yangtypes.py < /robot-systest/pyangbind.patch
+
  ARG CACHE_DATE=not_a_date
  RUN git clone \
      https://osm.etsi.org/gitlab/vnf-onboarding/osm-packages.git \
diff --git a/docker/tests/pyangbind.patch b/docker/tests/pyangbind.patch

new file mode 100644 (file)

index 0000000..3077299
--- /dev/null
+++ b/docker/tests/pyangbind.patch
@@ -0,0 +1,46 @@
+#######################################################################################
+# Copyright ETSI Contributors and Others.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+# implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#######################################################################################
+
+*** .tox/cover/lib/python3.10/site-packages/pyangbind/lib/yangtypes.py 2023-05-10 06:50:57.876027148 -0400
+--- .tox/cover/lib/python3.10/site-packages/pyangbind/lib/yangtypes.py 2023-05-10 06:51:11.772022417 -0400
+*************** limitations under the License.
+*** 22,27 ****
+--- 22,28 ----
+  from __future__ import unicode_literals
+
+  import collections
++ from six.moves import collections_abc
+  import copy
+  import uuid
+  from decimal import Decimal
+*************** def TypedListType(*args, **kwargs):
+*** 372,378 ****
+    if not isinstance(allowed_type, list):
+      allowed_type = [allowed_type]
+
+!   class TypedList(collections.MutableSequence):
+      _pybind_generated_by = "TypedListType"
+      _list = list()
+
+--- 373,379 ----
+    if not isinstance(allowed_type, list):
+      allowed_type = [allowed_type]
+
+!   class TypedList(collections_abc.MutableSequence):
+      _pybind_generated_by = "TypedListType"
+      _list = list()
+
diff --git a/installers/charm/bundles/osm/bundle.yaml b/installers/charm/bundles/osm/bundle.yaml

index 058ad3e..9dd3580 100644 (file)
--- a/installers/charm/bundles/osm/bundle.yaml
+++ b/installers/charm/bundles/osm/bundle.yaml
@@ -122,6 +122,7 @@ applications:
    vca:
      charm: osm-vca-integrator
      channel: latest/beta
+    series: focal
      scale: 1
    ingress:
      charm: nginx-ingress-integrator
diff --git a/installers/charm/keystone/charmcraft.yaml b/installers/charm/keystone/charmcraft.yaml

index 0a285a9..04b659a 100644 (file)
--- a/installers/charm/keystone/charmcraft.yaml
+++ b/installers/charm/keystone/charmcraft.yaml
@@ -34,4 +34,8 @@ bases:
            - arm64
  parts:
    charm:
-    build-packages: [git]
+    build-packages:
+      - git
+      - libffi-dev
+    charm-binary-python-packages:
+      - cryptography
diff --git a/installers/charm/osm-update-db-operator/src/db_upgrade.py b/installers/charm/osm-update-db-operator/src/db_upgrade.py

index 05cc0a0..295ce87 100644 (file)
--- a/installers/charm/osm-update-db-operator/src/db_upgrade.py
+++ b/installers/charm/osm-update-db-operator/src/db_upgrade.py
@@ -18,10 +18,276 @@ import json
  import logging
  
  from pymongo import MongoClient
+from uuid import uuid4
  
  logger = logging.getLogger(__name__)
  
  
+class MongoUpgrade1214:
+    """Upgrade MongoDB Database from OSM v12 to v14."""
+
+    @staticmethod
+    def gather_vnfr_healing_alerts(vnfr, vnfd):
+        alerts = []
+        nsr_id = vnfr["nsr-id-ref"]
+        df = vnfd.get("df", [{}])[0]
+        # Checking for auto-healing configuration
+        if "healing-aspect" in df:
+            healing_aspects = df["healing-aspect"]
+            for healing in healing_aspects:
+                for healing_policy in healing.get("healing-policy", ()):
+                    vdu_id = healing_policy["vdu-id"]
+                    vdur = next(
+                        (
+                            vdur
+                            for vdur in vnfr["vdur"]
+                            if vdu_id == vdur["vdu-id-ref"]
+                        ),
+                        {},
+                    )
+                    if not vdur:
+                        continue
+                    metric_name = "vm_status"
+                    vdu_name = vdur.get("name")
+                    vnf_member_index = vnfr["member-vnf-index-ref"]
+                    uuid = str(uuid4())
+                    name = f"healing_{uuid}"
+                    action = healing_policy
+                    # action_on_recovery = healing.get("action-on-recovery")
+                    # cooldown_time = healing.get("cooldown-time")
+                    # day1 = healing.get("day1")
+                    alert = {
+                        "uuid": uuid,
+                        "name": name,
+                        "metric": metric_name,
+                        "tags": {
+                            "ns_id": nsr_id,
+                            "vnf_member_index": vnf_member_index,
+                            "vdu_name": vdu_name,
+                        },
+                        "alarm_status": "ok",
+                        "action_type": "healing",
+                        "action": action,
+                    }
+                    alerts.append(alert)
+        return alerts
+
+    @staticmethod
+    def gather_vnfr_scaling_alerts(vnfr, vnfd):
+        alerts = []
+        nsr_id = vnfr["nsr-id-ref"]
+        df = vnfd.get("df", [{}])[0]
+        # Checking for auto-scaling configuration
+        if "scaling-aspect" in df:
+            rel_operation_types = {
+                "GE": ">=",
+                "LE": "<=",
+                "GT": ">",
+                "LT": "<",
+                "EQ": "==",
+                "NE": "!=",
+            }
+            scaling_aspects = df["scaling-aspect"]
+            all_vnfd_monitoring_params = {}
+            for ivld in vnfd.get("int-virtual-link-desc", ()):
+                for mp in ivld.get("monitoring-parameters", ()):
+                    all_vnfd_monitoring_params[mp.get("id")] = mp
+            for vdu in vnfd.get("vdu", ()):
+                for mp in vdu.get("monitoring-parameter", ()):
+                    all_vnfd_monitoring_params[mp.get("id")] = mp
+            for df in vnfd.get("df", ()):
+                for mp in df.get("monitoring-parameter", ()):
+                    all_vnfd_monitoring_params[mp.get("id")] = mp
+            for scaling_aspect in scaling_aspects:
+                scaling_group_name = scaling_aspect.get("name", "")
+                # Get monitored VDUs
+                all_monitored_vdus = set()
+                for delta in scaling_aspect.get(
+                    "aspect-delta-details", {}
+                ).get("deltas", ()):
+                    for vdu_delta in delta.get("vdu-delta", ()):
+                        all_monitored_vdus.add(vdu_delta.get("id"))
+                monitored_vdurs = list(
+                    filter(
+                        lambda vdur: vdur["vdu-id-ref"]
+                        in all_monitored_vdus,
+                        vnfr["vdur"],
+                    )
+                )
+                if not monitored_vdurs:
+                    logger.error("Scaling criteria is referring to a vnf-monitoring-param that does not contain a reference to a vdu or vnf metric")
+                    continue
+                for scaling_policy in scaling_aspect.get(
+                    "scaling-policy", ()
+                ):
+                    if scaling_policy["scaling-type"] != "automatic":
+                        continue
+                    threshold_time = scaling_policy.get(
+                        "threshold-time", "1"
+                    )
+                    cooldown_time = scaling_policy.get("cooldown-time", "0")
+                    for scaling_criteria in scaling_policy["scaling-criteria"]:
+                        monitoring_param_ref = scaling_criteria.get(
+                            "vnf-monitoring-param-ref"
+                        )
+                        vnf_monitoring_param = all_vnfd_monitoring_params[
+                            monitoring_param_ref
+                        ]
+                        for vdur in monitored_vdurs:
+                            vdu_id = vdur["vdu-id-ref"]
+                            metric_name = vnf_monitoring_param.get("performance-metric")
+                            metric_name = f"osm_{metric_name}"
+                            vdu_name = vdur["name"]
+                            vnf_member_index = vnfr["member-vnf-index-ref"]
+                            scalein_threshold = scaling_criteria.get("scale-in-threshold")
+                            # Looking for min/max-number-of-instances
+                            instances_min_number = 1
+                            instances_max_number = 1
+                            vdu_profile = df["vdu-profile"]
+                            if vdu_profile:
+                                profile = next(
+                                    item
+                                    for item in vdu_profile
+                                    if item["id"] == vdu_id
+                                )
+                                instances_min_number = profile.get("min-number-of-instances", 1)
+                                instances_max_number = profile.get("max-number-of-instances", 1)
+
+                            if scalein_threshold:
+                                uuid = str(uuid4())
+                                name = f"scalein_{uuid}"
+                                operation = scaling_criteria["scale-in-relational-operation"]
+                                rel_operator = rel_operation_types.get(operation, "<=")
+                                metric_selector = f'{metric_name}{{ns_id="{nsr_id}", vnf_member_index="{vnf_member_index}", vdu_id="{vdu_id}"}}'
+                                expression = f"(count ({metric_selector}) > {instances_min_number}) and (avg({metric_selector}) {rel_operator} {scalein_threshold})"
+                                labels = {
+                                    "ns_id": nsr_id,
+                                    "vnf_member_index": vnf_member_index,
+                                    "vdu_id": vdu_id,
+                                }
+                                prom_cfg = {
+                                    "alert": name,
+                                    "expr": expression,
+                                    "for": str(threshold_time) + "m",
+                                    "labels": labels,
+                                }
+                                action = scaling_policy
+                                action = {
+                                    "scaling-group": scaling_group_name,
+                                    "cooldown-time": cooldown_time,
+                                }
+                                alert = {
+                                    "uuid": uuid,
+                                    "name": name,
+                                    "metric": metric_name,
+                                    "tags": {
+                                        "ns_id": nsr_id,
+                                        "vnf_member_index": vnf_member_index,
+                                        "vdu_id": vdu_id,
+                                    },
+                                    "alarm_status": "ok",
+                                    "action_type": "scale_in",
+                                    "action": action,
+                                    "prometheus_config": prom_cfg,
+                                }
+                                alerts.append(alert)
+
+                            scaleout_threshold = scaling_criteria.get("scale-out-threshold")
+                            if scaleout_threshold:
+                                uuid = str(uuid4())
+                                name = f"scaleout_{uuid}"
+                                operation = scaling_criteria["scale-out-relational-operation"]
+                                rel_operator = rel_operation_types.get(operation, "<=")
+                                metric_selector = f'{metric_name}{{ns_id="{nsr_id}", vnf_member_index="{vnf_member_index}", vdu_id="{vdu_id}"}}'
+                                expression = f"(count ({metric_selector}) < {instances_max_number}) and (avg({metric_selector}) {rel_operator} {scaleout_threshold})"
+                                labels = {
+                                    "ns_id": nsr_id,
+                                    "vnf_member_index": vnf_member_index,
+                                    "vdu_id": vdu_id,
+                                }
+                                prom_cfg = {
+                                    "alert": name,
+                                    "expr": expression,
+                                    "for": str(threshold_time) + "m",
+                                    "labels": labels,
+                                }
+                                action = scaling_policy
+                                action = {
+                                    "scaling-group": scaling_group_name,
+                                    "cooldown-time": cooldown_time,
+                                }
+                                alert = {
+                                    "uuid": uuid,
+                                    "name": name,
+                                    "metric": metric_name,
+                                    "tags": {
+                                        "ns_id": nsr_id,
+                                        "vnf_member_index": vnf_member_index,
+                                        "vdu_id": vdu_id,
+                                    },
+                                    "alarm_status": "ok",
+                                    "action_type": "scale_out",
+                                    "action": action,
+                                    "prometheus_config": prom_cfg,
+                                }
+                                alerts.append(alert)
+        return alerts
+
+    @staticmethod
+    def _migrate_alerts(osm_db):
+        """Create new alerts collection.
+        """
+        if "alerts" in osm_db.list_collection_names():
+            return
+        logger.info("Entering in MongoUpgrade1214._migrate_alerts function")
+
+        # Get vnfds from MongoDB
+        logger.info("Reading VNF descriptors:")
+        vnfds = osm_db["vnfds"]
+        db_vnfds = []
+        for vnfd in vnfds.find():
+            logger.info(f'  {vnfd["_id"]}: {vnfd["description"]}')
+            db_vnfds.append(vnfd)
+
+        # Get vnfrs from MongoDB
+        logger.info("Reading VNFRs")
+        vnfrs = osm_db["vnfrs"]
+
+        # Gather healing and scaling alerts for each vnfr
+        healing_alerts = []
+        scaling_alerts = []
+        for vnfr in vnfrs.find():
+            logger.info(f'  vnfr {vnfr["_id"]}')
+            vnfd = next((sub for sub in db_vnfds if sub["_id"] == vnfr["vnfd-id"]), None)
+            healing_alerts.extend(MongoUpgrade1214.gather_vnfr_healing_alerts(vnfr, vnfd))
+            scaling_alerts.extend(MongoUpgrade1214.gather_vnfr_scaling_alerts(vnfr, vnfd))
+
+        # Add new alerts in MongoDB
+        alerts = osm_db["alerts"]
+        for alert in healing_alerts:
+            logger.info(f"Storing healing alert in MongoDB: {alert}")
+            alerts.insert_one(alert)
+        for alert in scaling_alerts:
+            logger.info(f"Storing scaling alert in MongoDB: {alert}")
+            alerts.insert_one(alert)
+
+        # Delete old alarms collections
+        logger.info("Deleting alarms and alarms_action collections")
+        alarms = osm_db["alarms"]
+        alarms.drop()
+        alarms_action = osm_db["alarms_action"]
+        alarms_action.drop()
+
+
+    @staticmethod
+    def upgrade(mongo_uri):
+        """Upgrade alerts in MongoDB."""
+        logger.info("Entering in MongoUpgrade1214.upgrade function")
+        myclient = MongoClient(mongo_uri)
+        osm_db = myclient["osm"]
+        MongoUpgrade1214._migrate_alerts(osm_db)
+
+
  class MongoUpgrade1012:
      """Upgrade MongoDB Database from OSM v10 to v12."""
  
@@ -219,6 +485,7 @@ class MongoPatch1837:
  MONGODB_UPGRADE_FUNCTIONS = {
      "9": {"10": [MongoUpgrade910.upgrade]},
      "10": {"12": [MongoUpgrade1012.upgrade]},
+    "12": {"14": [MongoUpgrade1214.upgrade]},
  }
  MYSQL_UPGRADE_FUNCTIONS = {}
  BUG_FIXES = {
diff --git a/installers/full_install_osm.sh b/installers/full_install_osm.sh

index 03be9a6..a7f46a2 100755 (executable)
--- a/installers/full_install_osm.sh
+++ b/installers/full_install_osm.sh
@@ -36,7 +36,8 @@ function usage(){
      echo -e "     -P <VCA pubkey> use VCA/juju public key file"
      echo -e "     -A <VCA apiproxy> use VCA/juju API proxy"
      echo -e "     --pla:          install the PLA module for placement support"
-    echo -e "     --ng-sa:        install Airflow and Pushgateway to get VNF and NS status (experimental)"
+    echo -e "     --old-sa:       install old Service Assurance framework (MON, POL); do not install Airflow and Pushgateway"
+    echo -e "     --ng-sa:        install new Service Assurance framework (Airflow, AlertManager and Pushgateway)  (enabled by default)"
      echo -e "     -m <MODULE>:    install OSM but only rebuild or pull the specified docker images (NG-UI, NBI, LCM, RO, MON, POL, PLA, KAFKA, MONGO, PROMETHEUS, PROMETHEUS-CADVISOR, KEYSTONE-DB, NONE)"
      echo -e "     -o <ADDON>:     ONLY (un)installs one of the addons (k8s_monitor, ng-sa)"
      echo -e "     -O <openrc file path/cloud name>: Install OSM to an OpenStack infrastructure. <openrc file/cloud name> is required. If a <cloud name> is used, the clouds.yaml file should be under ~/.config/openstack/ or /etc/openstack/"
@@ -215,13 +216,12 @@ function docker_login() {
      [ -z "${DEBUG_INSTALL}" ] || DEBUG end of function
  }
  
-function generate_docker_images() {
+function pull_docker_images() {
      [ -z "${DEBUG_INSTALL}" ] || DEBUG beginning of function
-    echo "Pulling and generating docker images"
-    [ -n "${DOCKER_REGISTRY_URL}" ] && docker_login
-
      echo "Pulling docker images"
+    [ -n "${DOCKER_REGISTRY_URL}" ] && docker_login
  
+    echo "Pulling non-OSM docker images"
      if [ -z "$TO_REBUILD" ] || echo $TO_REBUILD | grep -q KAFKA ; then
          sg docker -c "docker pull wurstmeister/zookeeper" || FATAL "cannot get zookeeper docker image"
          sg docker -c "docker pull wurstmeister/kafka:${KAFKA_TAG}" || FATAL "cannot get kafka docker image"
@@ -248,48 +248,57 @@ function generate_docker_images() {
          sg docker -c "docker pull mysql:5" || FATAL "cannot get mysql docker image"
      fi
  
-    if [ -n "$PULL_IMAGES" ]; then
-        echo "Pulling OSM docker images"
-        for module in MON POL NBI KEYSTONE RO LCM NG-UI PLA osmclient; do
+    list_of_modules="MON NBI KEYSTONE RO LCM NG-UI osmclient"
+    if [ -n "$INSTALL_NGSA" ]; then
+        list_of_modules="${list_of_modules} Airflow Webhook"
+    else
+        list_of_modules="${list_of_modules} POL"
+    fi
+    if [ -n "$INSTALL_PLA" ]; then
+        list_of_modules="${list_of_modules} PLA"
+    fi
+    echo "Pulling OSM docker images for the following modules: ${list_of_modules}"
+    for module in ${list_of_modules}; do
+        module_lower=${module,,}
+        module_tag="${OSM_DOCKER_TAG}"
+        if [ -n "${MODULE_DOCKER_TAG}" ] && echo $TO_REBUILD | grep -q $module ; then
+            module_tag="${MODULE_DOCKER_TAG}"
+        fi
+        echo "Pulling ${DOCKER_REGISTRY_URL}${DOCKER_USER}/${module_lower}:${module_tag} docker image"
+        sg docker -c "docker pull ${DOCKER_REGISTRY_URL}${DOCKER_USER}/${module_lower}:${module_tag}" || FATAL "cannot pull $module docker image"
+    done
+
+    echo "Finished pulling docker images"
+    [ -z "${DEBUG_INSTALL}" ] || DEBUG end of function
+}
+
+function generate_docker_images() {
+    [ -z "${DEBUG_INSTALL}" ] || DEBUG beginning of function
+    echo "Generating docker images"
+    _build_from=$COMMIT_ID
+    [ -z "$_build_from" ] && _build_from="latest"
+    echo "OSM Docker images generated from $_build_from"
+    LWTEMPDIR="$(mktemp -d -q --tmpdir "installosmlight.XXXXXX")"
+    trap 'rm -rf "${LWTEMPDIR}"' EXIT
+    for module in MON POL NBI KEYSTONE RO LCM NG-UI PLA; do
+        if [ -z "$TO_REBUILD" ] || echo $TO_REBUILD | grep -q ${module} ; then
              module_lower=${module,,}
              if [ $module == "PLA" -a ! -n "$INSTALL_PLA" ]; then
                  continue
              fi
-            module_tag="${OSM_DOCKER_TAG}"
-            if [ -n "${MODULE_DOCKER_TAG}" ] && echo $TO_REBUILD | grep -q $module ; then
-                module_tag="${MODULE_DOCKER_TAG}"
-            fi
-            echo "Pulling ${DOCKER_REGISTRY_URL}${DOCKER_USER}/${module_lower}:${module_tag} docker image"
-            sg docker -c "docker pull ${DOCKER_REGISTRY_URL}${DOCKER_USER}/${module_lower}:${module_tag}" || FATAL "cannot pull $module docker image"
-        done
-    else
-        _build_from=$COMMIT_ID
-        [ -z "$_build_from" ] && _build_from="latest"
-        echo "OSM Docker images generated from $_build_from"
-        LWTEMPDIR="$(mktemp -d -q --tmpdir "installosmlight.XXXXXX")"
-        trap 'rm -rf "${LWTEMPDIR}"' EXIT
-        for module in MON POL NBI KEYSTONE RO LCM NG-UI PLA; do
-            if [ -z "$TO_REBUILD" ] || echo $TO_REBUILD | grep -q ${module} ; then
-                module_lower=${module,,}
-                if [ $module == "PLA" -a ! -n "$INSTALL_PLA" ]; then
-                    continue
-                fi
-                git -C ${LWTEMPDIR} clone https://osm.etsi.org/gerrit/osm/$module
-                git -C ${LWTEMPDIR}/${module} checkout ${COMMIT_ID}
-                sg docker -c "docker build ${LWTEMPDIR}/${module} -f ${LWTEMPDIR}/${module}/docker/Dockerfile -t ${DOCKER_USER}/${module_lower} --no-cache" || FATAL "cannot build ${module} docker image"
-            fi
-        done
-        if [ -z "$TO_REBUILD" ] || echo $TO_REBUILD | grep -q osmclient; then
-            BUILD_ARGS+=(--build-arg REPOSITORY="$REPOSITORY")
-            BUILD_ARGS+=(--build-arg RELEASE="$RELEASE")
-            BUILD_ARGS+=(--build-arg REPOSITORY_KEY="$REPOSITORY_KEY")
-            BUILD_ARGS+=(--build-arg REPOSITORY_BASE="$REPOSITORY_BASE")
-            sg docker -c "docker build -t ${DOCKER_USER}/osmclient ${BUILD_ARGS[@]} -f $OSM_DEVOPS/docker/osmclient ."
+            git -C ${LWTEMPDIR} clone https://osm.etsi.org/gerrit/osm/$module
+            git -C ${LWTEMPDIR}/${module} checkout ${COMMIT_ID}
+            sg docker -c "docker build ${LWTEMPDIR}/${module} -f ${LWTEMPDIR}/${module}/docker/Dockerfile -t ${DOCKER_USER}/${module_lower} --no-cache" || FATAL "cannot build ${module} docker image"
          fi
-        echo "Finished generation of docker images"
+    done
+    if [ -z "$TO_REBUILD" ] || echo $TO_REBUILD | grep -q osmclient; then
+        BUILD_ARGS+=(--build-arg REPOSITORY="$REPOSITORY")
+        BUILD_ARGS+=(--build-arg RELEASE="$RELEASE")
+        BUILD_ARGS+=(--build-arg REPOSITORY_KEY="$REPOSITORY_KEY")
+        BUILD_ARGS+=(--build-arg REPOSITORY_BASE="$REPOSITORY_BASE")
+        sg docker -c "docker build -t ${DOCKER_USER}/osmclient ${BUILD_ARGS[@]} -f $OSM_DEVOPS/docker/osmclient ."
      fi
-
-    echo "Finished pulling and generating docker images"
+    echo "Finished generation of docker images"
      [ -z "${DEBUG_INSTALL}" ] || DEBUG end of function
  }
  
@@ -498,10 +507,11 @@ function kube_secrets(){
      kubectl create secret generic ro-db-secret -n $OSM_NAMESPACE --from-env-file=$OSM_DOCKER_WORK_DIR/ro-db.env
      kubectl create secret generic ro-secret -n $OSM_NAMESPACE --from-env-file=$OSM_DOCKER_WORK_DIR/ro.env
      kubectl create secret generic keystone-secret -n $OSM_NAMESPACE --from-env-file=$OSM_DOCKER_WORK_DIR/keystone.env
-    kubectl create secret generic pol-secret -n $OSM_NAMESPACE --from-env-file=$OSM_DOCKER_WORK_DIR/pol.env
      if [ -n "${INSTALL_NGSA}" ]; then
          kubectl create secret generic ngsa-secret -n $OSM_NAMESPACE --from-env-file=$OSM_DOCKER_WORK_DIR/ngsa.env
          kubectl create secret generic webhook-translator-secret -n $OSM_NAMESPACE --from-env-file=$OSM_DOCKER_WORK_DIR/webhook-translator.env
+    else
+        kubectl create secret generic pol-secret -n $OSM_NAMESPACE --from-env-file=$OSM_DOCKER_WORK_DIR/pol.env
      fi
      [ -z "${DEBUG_INSTALL}" ] || DEBUG end of function
  }
@@ -509,6 +519,8 @@ function kube_secrets(){
  #deploys osm pods and services
  function deploy_osm_services() {
      [ -z "${DEBUG_INSTALL}" ] || DEBUG beginning of function
+    echo "The following manifests under $OSM_K8S_WORK_DIR will be deployed:"
+    ls $OSM_K8S_WORK_DIR
      kubectl apply -n $OSM_NAMESPACE -f $OSM_K8S_WORK_DIR
      [ -z "${DEBUG_INSTALL}" ] || DEBUG end of function
  }
@@ -530,7 +542,9 @@ function deploy_osm_pla_service() {
  
  function install_osm_ngsa_service() {
      [ -z "${DEBUG_INSTALL}" ] || DEBUG beginning of function
-    $OSM_DEVOPS/installers/install_ngsa.sh -d ${OSM_HELM_WORK_DIR} -D ${OSM_DEVOPS} -t ${OSM_DOCKER_TAG} ${DEBUG_INSTALL} || \
+    NGSA_OPTS="-d ${OSM_HELM_WORK_DIR} -D ${OSM_DEVOPS} -t ${OSM_DOCKER_TAG} -U ${DOCKER_USER} ${DEBUG_INSTALL}"
+    [ -n "${DOCKER_REGISTRY_URL}" ] && NGSA_OPTS="${NGSA_OPTS} -r ${DOCKER_REGISTRY_URL}"
+    $OSM_DEVOPS/installers/install_ngsa.sh ${NGSA_OPTS} || \
      FATAL_TRACK install_osm_ngsa_service "install_ngsa.sh failed"
      [ -z "${DEBUG_INSTALL}" ] || DEBUG end of function
  }
@@ -543,7 +557,7 @@ function parse_yaml() {
      for module in $services; do
          if [ "$module" == "pla" ]; then
              if [ -n "$INSTALL_PLA" ]; then
-                echo "Updating K8s manifest file from opensourcemano\/pla:.* to ${DOCKER_REGISTRY_URL}${DOCKER_USER}\/pla:${TAG}"
+                echo "Updating K8s manifest file for ${module} from opensourcemano\/pla:.* to ${DOCKER_REGISTRY_URL}${DOCKER_USER}\/pla:${TAG}"
                  sudo sed -i "s#opensourcemano/pla:.*#${DOCKER_REGISTRY_URL}${DOCKER_USER}/pla:${TAG}#g" ${OSM_DOCKER_WORK_DIR}/osm_pla/pla.yaml
              fi
          else
@@ -553,7 +567,7 @@ function parse_yaml() {
              elif [ "$module" == "ng-mon" ]; then
                  image="mon"
              fi
-            echo "Updating K8s manifest file from opensourcemano\/${image}:.* to ${DOCKER_REGISTRY_URL}${DOCKER_USER}\/${image}:${TAG}"
+            echo "Updating K8s manifest file for ${module} from opensourcemano\/${image}:.* to ${DOCKER_REGISTRY_URL}${DOCKER_USER}\/${image}:${TAG}"
              sudo sed -i "s#opensourcemano/${image}:.*#${DOCKER_REGISTRY_URL}${DOCKER_USER}/${image}:${TAG}#g" ${OSM_K8S_WORK_DIR}/${module}.yaml
          fi
      done
@@ -561,6 +575,7 @@ function parse_yaml() {
  }
  
  function update_manifest_files() {
+    [ -z "${DEBUG_INSTALL}" ] || DEBUG beginning of function
      osm_services="nbi lcm ro pol mon ng-mon ng-ui keystone pla prometheus ng-prometheus"
      list_of_services=""
      for module in $osm_services; do
@@ -790,7 +805,8 @@ function install_osm() {
      fi
  
      # Deploy OSM services
-    [ -z "$DOCKER_NOBUILD" ] && generate_docker_images
+    [ -z "$DOCKER_NOBUILD" ] && pull_docker_images
+    [ -z "$DOCKER_NOBUILD" ] && [ -z "$PULL_IMAGES" ] && generate_docker_images
      track docker_images docker_images_ok
  
      generate_k8s_manifest_files
@@ -999,7 +1015,7 @@ DEBUG_INSTALL=""
  RELEASE="ReleaseTEN"
  REPOSITORY="stable"
  INSTALL_K8S_MONITOR=""
-INSTALL_NGSA=""
+INSTALL_NGSA="y"
  INSTALL_PLA=""
  INSTALL_VIMEMU=""
  LXD_REPOSITORY_BASE="https://osm-download.etsi.org/repository/osm/lxd"
@@ -1199,6 +1215,7 @@ while getopts ":a:b:r:n:k:u:R:D:o:O:m:N:H:S:s:t:U:P:A:l:L:K:d:p:T:f:F:-: hy" o;
              [ "${OPTARG}" == "tag" ] && continue
              [ "${OPTARG}" == "registry" ] && continue
              [ "${OPTARG}" == "pla" ] && INSTALL_PLA="y" && continue
+            [ "${OPTARG}" == "old-sa" ] && INSTALL_NGSA="" && continue
              [ "${OPTARG}" == "ng-sa" ] && INSTALL_NGSA="y" && continue
              [ "${OPTARG}" == "volume" ] && OPENSTACK_ATTACH_VOLUME="true" && continue
              [ "${OPTARG}" == "nocachelxdimages" ] && continue
diff --git a/installers/helm/values/airflow-values.yaml b/installers/helm/values/airflow-values.yaml

index 9bde121..79473aa 100644 (file)
--- a/installers/helm/values/airflow-values.yaml
+++ b/installers/helm/values/airflow-values.yaml
@@ -22,10 +22,13 @@ webserver:
        - name: airflow-ui
          port: "{{ .Values.ports.airflowUI }}"
          targetPort: "{{ .Values.ports.airflowUI }}"
+extraEnv: |
+  - name: AIRFLOW__API__AUTH_BACKENDS
+    value: 'airflow.api.auth.backend.basic_auth'
  extraEnvFrom: |
    - secretRef:
        name: ngsa-secret
  config:
    core:
-    dags_folder: "/home/airflow/.local/lib/python3.8/site-packages/osm_ngsa"
+    dags_folder: "/home/airflow/.local/lib/python3.10/site-packages/osm_ngsa"
  
diff --git a/installers/helm/values/alertmanager-values.yaml b/installers/helm/values/alertmanager-values.yaml

index 2e438cb..c140c50 100644 (file)
--- a/installers/helm/values/alertmanager-values.yaml
+++ b/installers/helm/values/alertmanager-values.yaml
@@ -34,14 +34,16 @@ config:
      group_wait: 10s
      group_interval: 2m
      receiver: default-receiver
-    repeat_interval: 3h
      routes:
      - receiver: vdu-webhook
+      repeat_interval: 15m
        matchers:
        - alertname = "vdu_down"
      - receiver: 'scaleout-webhook'
+      repeat_interval: 5m
        matchers:
        - alertname =~ "^scaleout_.*"
      - receiver: 'scalein-webhook'
+      repeat_interval: 5m
        matchers:
        - alertname =~ "^scalein_.*"
diff --git a/installers/install_ngsa.sh b/installers/install_ngsa.sh

index b90c3dc..30edc70 100755 (executable)
--- a/installers/install_ngsa.sh
+++ b/installers/install_ngsa.sh
@@ -16,7 +16,7 @@
  set +eux
  
  # Helm chart 1.6.0 correspondes to Airflow 2.3.0
-AIRFLOW_HELM_VERSION=1.6.0
+AIRFLOW_HELM_VERSION=1.9.0
  PROMPUSHGW_HELM_VERSION=1.18.2
  ALERTMANAGER_HELM_VERSION=0.22.0
  
@@ -29,6 +29,8 @@ function install_airflow() {
      # update airflow-values.yaml to use the right tag
      echo "Updating Helm values file helm/values/airflow-values.yaml to use defaultAirflowTag: ${OSM_DOCKER_TAG}"
      sudo sed -i "s#defaultAirflowTag:.*#defaultAirflowTag: ${OSM_DOCKER_TAG}#g" ${OSM_HELM_WORK_DIR}/airflow-values.yaml
+    echo "Updating Helm values file helm/values/airflow-values.yaml to use defaultAirflowRepository: ${DOCKER_REGISTRY_URL}${DOCKER_USER}/airflow"
+    sudo sed -i "s#defaultAirflowRepository:.*#defaultAirflowRepository: ${DOCKER_REGISTRY_URL}${DOCKER_USER}/airflow#g" ${OSM_HELM_WORK_DIR}/airflow-values.yaml
      if ! helm -n osm status airflow 2> /dev/null ; then
          # if it does not exist, create secrets and install
          kubectl -n osm create secret generic airflow-webserver-secret --from-literal="webserver-secret-key=$(python3 -c 'import secrets; print(secrets.token_hex(16))')"
@@ -62,6 +64,9 @@ function install_prometheus_pushgateway() {
  # Install Prometheus AlertManager helm chart
  function install_prometheus_alertmanager() {
      [ -z "${DEBUG_INSTALL}" ] || DEBUG end of function
+    # copy alertmanager-values.yaml to the destination folder
+    sudo mkdir -p ${OSM_HELM_WORK_DIR}
+    sudo cp ${OSM_DEVOPS}/installers/helm/values/alertmanager-values.yaml ${OSM_HELM_WORK_DIR}
      if ! helm -n osm status alertmanager 2> /dev/null ; then
          # if it does not exist, install
          helm repo add prometheus-community https://prometheus-community.github.io/helm-charts
@@ -77,11 +82,13 @@ function install_prometheus_alertmanager() {
  
  # main
  
+DOCKER_REGISTRY_URL=
+DOCKER_USER="opensourcemano"
  OSM_DEVOPS="/usr/share/osm-devops"
-OSM_HELM_WORK_DIR="/etc/osm/helm"
  OSM_DOCKER_TAG="13"
+OSM_HELM_WORK_DIR="/etc/osm/helm"
  
-while getopts ":D:d:t:-: " o; do
+while getopts ":D:d:t:r:U:-: " o; do
      case "${o}" in
          D)
              OSM_DEVOPS="${OPTARG}"
@@ -92,6 +99,12 @@ while getopts ":D:d:t:-: " o; do
          t)
              OSM_DOCKER_TAG="${OPTARG}"
              ;;
+        r)
+            DOCKER_REGISTRY_URL="${OPTARG}"
+            ;;
+        U)
+            DOCKER_USER="${OPTARG}"
+            ;;
          -)
              [ "${OPTARG}" == "debug" ] && DEBUG_INSTALL="y" && continue
              echo -e "Invalid option: '--$OPTARG'\n" >&2
diff --git a/jenkins/ci-pipelines/ci_stage_3.groovy b/jenkins/ci-pipelines/ci_stage_3.groovy

index e0cddea..29e438a 100644 (file)
--- a/jenkins/ci-pipelines/ci_stage_3.groovy
+++ b/jenkins/ci-pipelines/ci_stage_3.groovy
@@ -124,13 +124,13 @@ void run_robot_systest(String tagName,
  
  void archive_logs(Map remote) {
  
-    sshCommand remote: remote, command: '''mkdir -p logs'''
+    sshCommand remote: remote, command: '''mkdir -p logs/dags'''
      if (useCharmedInstaller) {
          sshCommand remote: remote, command: '''
-            for container in `kubectl get pods -n osm | grep -v operator | grep -v NAME| awk '{print $1}'`; do
-                logfile=`echo $container | cut -d- -f1`
+            for pod in `kubectl get pods -n osm | grep -v operator | grep -v NAME| awk '{print $1}'`; do
+                logfile=`echo $pod | cut -d- -f1`
                  echo "Extracting log for $logfile"
-                kubectl logs -n osm $container --timestamps=true 2>&1 > logs/$logfile.log
+                kubectl logs -n osm $pod --timestamps=true 2>&1 > logs/$logfile.log
              done
          '''
      } else {
@@ -148,6 +148,11 @@ void archive_logs(Map remote) {
                  > logs/$statefulset.log
              done
          '''
+        sshCommand remote: remote, command: '''
+            schedulerPod="$(kubectl get pods -n osm | grep airflow-scheduler| awk '{print $1; exit}')"; \
+            echo "Extracting logs from Airflow DAGs from pod ${schedulerPod}"; \
+            kubectl cp -n osm ${schedulerPod}:/opt/airflow/logs/scheduler/latest/dags logs/dags -c scheduler
+        '''
      }
  
      sh 'rm -rf logs'
@@ -456,8 +461,8 @@ node("${params.NODE}") {
                      // Ensure the VM is ready
                      sshCommand remote: remote, command: 'cloud-init status --wait'
                      // Force time sync to avoid clock drift and invalid certificates
-                    sshCommand remote: remote, command: 'sudo apt-get update'
-                    sshCommand remote: remote, command: 'sudo apt-get install -y chrony'
+                    sshCommand remote: remote, command: 'sudo apt-get -y update'
+                    sshCommand remote: remote, command: 'sudo apt-get -y install chrony'
                      sshCommand remote: remote, command: 'sudo service chrony stop'
                      sshCommand remote: remote, command: 'sudo chronyd -vq'
                      sshCommand remote: remote, command: 'sudo service chrony start'
@@ -748,7 +753,10 @@ EOF"""
              } // if (params.SAVE_ARTIFACTS_OVERRIDE || stage_archive)
          } // dir(OSM_DEVOPS)
      } finally {
-       stage('Archive Container Logs') {
+        // stage('Debug') {
+        //     sleep 900
+        // }
+        stage('Archive Container Logs') {
              if ( ARCHIVE_LOGS_FLAG ) {
                  try {
                      // Archive logs
diff --git a/tools/local-build.sh b/tools/local-build.sh

index 4ff12f2..cbaa87a 100755 (executable)
--- a/tools/local-build.sh
+++ b/tools/local-build.sh
@@ -191,6 +191,14 @@ function run_httpserver() {
      nohup python3 -m http.server ${HTTPPORT} --directory "${HTTPDDIR}" &>/dev/null &
  }
  
+function run_httpserver() {
+    EXISTING_PID=$(ps auxw | grep "http.server $HTTPPORT" | grep -v grep | awk '{print $2}')
+    if [ ! -z $EXISTING_PID ] ; then
+        kill $EXISTING_PID
+    fi
+    nohup python3 -m http.server ${HTTPPORT} --directory "${HTTPDDIR}" &>/dev/null &
+}
+
  function stage_2() {
      print_section "Performing Stage 2"
      MODULES="common devops IM LCM MON N2VC NBI NG-UI NG-SA osmclient PLA POL RO tests"
author	Dario Faccin <dario.faccin@canonical.com>
	Tue, 23 May 2023 15:44:49 +0000 (17:44 +0200)
committer	Dario Faccin <dario.faccin@canonical.com>
	Wed, 24 May 2023 09:38:30 +0000 (11:38 +0200)
docker/Airflow/Dockerfile		patch \| blob \| history
docker/LCM/Dockerfile		patch \| blob \| history
docker/MON/Dockerfile		patch \| blob \| history
docker/NBI/Dockerfile		patch \| blob \| history
docker/NBI/pyangbind.patch	[new file with mode: 0644]	patch \| blob
docker/NG-UI/Dockerfile		patch \| blob \| history
docker/NGLCM/Dockerfile		patch \| blob \| history
docker/PLA/Dockerfile		patch \| blob \| history
docker/POL/Dockerfile		patch \| blob \| history
docker/RO/Dockerfile		patch \| blob \| history
docker/tests/Dockerfile		patch \| blob \| history
docker/tests/pyangbind.patch	[new file with mode: 0644]	patch \| blob
installers/charm/bundles/osm/bundle.yaml		patch \| blob \| history
installers/charm/keystone/charmcraft.yaml		patch \| blob \| history
installers/charm/osm-update-db-operator/src/db_upgrade.py		patch \| blob \| history
installers/full_install_osm.sh		patch \| blob \| history
installers/helm/values/airflow-values.yaml		patch \| blob \| history
installers/helm/values/alertmanager-values.yaml		patch \| blob \| history
installers/install_ngsa.sh		patch \| blob \| history
jenkins/ci-pipelines/ci_stage_3.groovy		patch \| blob \| history
tools/local-build.sh		patch \| blob \| history