From 36faaff53fa49da849cbbe537706329de19deb05 Mon Sep 17 00:00:00 2001 From: Dario Faccin Date: Tue, 23 May 2023 17:44:49 +0200 Subject: [PATCH] Update from master Squashed commit of the following: commit 5ef8e951103f7084c94d1db0628e46cf82c5d06b Author: garciadeblas Date: Fri May 19 11:42:13 2023 +0200 Clarify messages and minor refactors in full_install_osm Change-Id: I33182aa1f7c53117ae0b645a4f02b3cbde7bfb1c Signed-off-by: garciadeblas commit b254c0268b0ce5969e50fd79fecd1a82fa635ce5 Author: garciadeblas Date: Mon May 22 00:47:42 2023 +0200 Update stage3 to archive logs from Airflow DAGs Change-Id: I99b6d6b9afe94135be8909c7aa96866615532389 Signed-off-by: garciadeblas commit e78f5a762565fe67b2637b560fdc829a9d71ccd0 Author: garciadeblas Date: Wed May 17 17:53:37 2023 +0200 Update tests and NBI Dockerfile to patch pyangbing and install libmagic1 Change-Id: If262e372ce26b04df5ac54961336620f27897933 Signed-off-by: garciadeblas commit 2aab0d77d06ee94a10ffed431d1b270871ce319e Author: garciadeblas Date: Wed May 17 16:24:22 2023 +0200 ubuntu20 as base image for OSM installation Change-Id: I9372211baabe88c83c5f86e32dff35686968c1d4 Signed-off-by: garciadeblas commit ef78c3c64d4a203334b7935e83e29bac56ccba97 Author: Patricia Reinoso Date: Tue May 16 08:12:58 2023 +0000 Fix charmcraft pack for keystone charm Add missing cryptography dependencies. Change-Id: Id933b7b83ad81014bdbd2832379d202e8bc07e3f Signed-off-by: Patricia Reinoso commit 3f26c2c08271e5ff2beb3ee97909a035e322761f Author: garciadeblas Date: Wed May 17 12:19:36 2023 +0200 Update stage2 to use Ubuntu22 as Openstack base image Change-Id: I91c710a86f7fba14f3e39d95e8d3fb96a7f14662 Signed-off-by: garciadeblas commit 7e3c1483c5face876176644191a9ccaf9e8c0f0e Author: garciadeblas Date: Tue May 16 11:04:40 2023 +0200 Update jenkins-stage3 to use ubuntu22 as default image and DEBUG loglevel for sshCommand Change-Id: I852e0f706176fda4f24ea4afd8a23e27280624ba Signed-off-by: garciadeblas commit 836354fe4faf5ec29acfabcaf1a79cefd86b5848 Author: garciadeblas Date: Wed May 10 18:12:00 2023 +0200 Enable NGSA installation (Airflow, AlertMgr and PushGW) by default This reverts commit 64885c283f162872a85ea05142d3ebf927aaacdc. Change-Id: I5afc552335f8f25656f1947c5b212750683b807b Signed-off-by: garciadeblas commit 4e8024f71bb6651a447f100c1c0fbd5081ae79f5 Author: Patricia Reinoso Date: Thu May 11 14:02:35 2023 +0000 Set focal series in OSM bundle. Even if model-config 'default-series' is 'focal' juju pulls the 'jammy' series for OSM charms that's why we need to explicitly specify the series. Change-Id: I4b2024f92f33817771cb40305d979213d7546af5 Signed-off-by: Patricia Reinoso commit be5864a7da380b52e846465466c3363a32d5882b Author: garciadeblas Date: Thu May 11 15:34:13 2023 +0200 Update tests/Dockerfile to make it work with Ubuntu22 base image Change-Id: I0a7db076f5e9db92a8891d13636f5ac0a845ab4a Signed-off-by: garciadeblas commit 02feb8ef9ae5f545a111759cbbe9f01b1741cc85 Author: Mark Beierl Date: Wed May 10 11:44:11 2023 -0400 Updates to Python 3.10 and Ubuntu22.04 Updates all dockerfiles for stage 3 to use newer versions Change-Id: I4d73e04e11be4e12c98467f5d24ee5daff699d89 Signed-off-by: Mark Beierl commit 54f21dc2e06cc72582026cf28b2206042f87133d Author: garciadeblas Date: Wed May 10 18:11:21 2023 +0200 Revert "Enable NGSA installation (Airflow, AlertMgr and PushGW) by default" This reverts commit 82afc03ffce54cf3415e91ed077fb2338e2de7e9. Change-Id: I511d4b9d9c7410999d135d217f33dd96f83e1d84 Signed-off-by: garciadeblas commit 1233df73c57b1c72e282218839c18b139b3b5aa6 Author: garciadeblas Date: Wed May 10 17:02:39 2023 +0200 Update Airflow Dockerfile to unpack python3.10 versions of osm_common and osm_ngsa Change-Id: I2a650b66cfdf8bb2249882947d0a47622c673806 Signed-off-by: garciadeblas commit 82afc03ffce54cf3415e91ed077fb2338e2de7e9 Author: garciadeblas Date: Thu May 4 14:20:20 2023 +0200 Enable NGSA installation (Airflow, AlertMgr and PushGW) by default Change-Id: Ia2ba77b7cfbd961a6b70f4257dab3470048d1117 Signed-off-by: garciadeblas commit 0b7ef27773763a547581023f7b72f3d193bc898f Author: garciadeblas Date: Wed May 10 10:50:34 2023 +0200 Update Airflow version to 2.5.3 and Airflow Helm chart version to 1.9.0 Change-Id: I98928685b3f3d5399f8c97d306e2ba7878b5515d Signed-off-by: garciadeblas commit 3763099835a202bb52a6d5db7b8e9c89261aec7a Author: garciadeblas Date: Tue May 9 13:33:22 2023 +0200 Update install_ngsa.sh to use docker registry and user from CICD Change-Id: I33b96d876a0dbe95846b6c6b123438f6fb17f7f0 Signed-off-by: garciadeblas commit 53b217f94a1c19c58eb3f3077779b3dcdfb741e7 Author: garciadeblas Date: Thu May 4 17:57:47 2023 +0200 Split generate_docker_images and pull_docker_images in installer Change-Id: I09fc0f0c566b481f78d818715be64dbcb855c955 Signed-off-by: garciadeblas commit 2efbda22f27241546145752035a759016dd87430 Author: garciadeblas Date: Thu May 4 13:38:46 2023 +0200 Update db_upgrade.py to append osm to metric_name to be BWC Change-Id: I8ed6497fe21a3db3ccf841795f8b92af1a95cc0b Signed-off-by: garciadeblas commit 986c308d5d89b8dbb3361ce2f1638b3285859d8e Author: garciadeblas Date: Thu May 4 11:54:30 2023 +0200 Refactor MongoUpgrade1214 in db_upgrade.py Change-Id: I1da050701c20d1a04c34457743b43a89a2786929 Signed-off-by: garciadeblas commit 05ee8ab29eb39cc34bd679e238c8051f29d93c35 Author: sritharan Date: Fri Apr 28 05:38:10 2023 +0000 Feature 10986: Autoheal switch and Autoscale switch Change-Id: I851299bf9bf57e9e50d6e1ad387766bbe13a192d Signed-off-by: sritharan commit fc5714cf86a58590c2516dff0562ce859aace267 Author: aguilard Date: Tue Apr 25 16:25:42 2023 +0000 Feature 10981: update alerts collections in Mongo from OSM v12 to v14 Change-Id: If9e7133439fa4c856d96a9108e4306a37b2fd167 Signed-off-by: aguilard commit e556fe0f1afc690e86e3d9de12e7ccea6e1edf7f Author: aguilard Date: Tue May 2 10:20:21 2023 +0000 Feature 10981: enable REST API in Airflow Change-Id: Iaae27b9917d8cc35bae6e67cce59bef3e9bd8b41 Signed-off-by: aguilard commit b798d06e0769138e57b682e446a2576b9bf2d575 Author: aguilard Date: Tue May 2 09:54:44 2023 +0000 Fix typo in file alertmanager-values.yaml Change-Id: If2c17971aab5d1c37987d18d2c24ec5864a132f1 Signed-off-by: aguilard commit 64775ea32b74f6629e7adf4e8bb397513ec20b6c Author: aguilard Date: Thu Apr 27 11:02:21 2023 +0000 Feature 10981: fix bug in Airflow DAGs directory Change-Id: I6e081306d1363401b27d7a3d4ca8d824862d05b6 Signed-off-by: aguilard commit 02609d9c7d62448a1b51b41befdfd0bc6da14d75 Author: aguilard Date: Thu Apr 27 09:47:27 2023 +0000 Feature 10981: fix bug in AlertMgr installation Change-Id: I83fd859d14fc64dc2b2e3328ce79577fffd619ce Signed-off-by: aguilard commit 8177b9cb342c3f820e5ba3dab85f633970960a95 Author: SANDHYA.JS Date: Wed Apr 26 09:48:39 2023 +0530 Fix Bug 2121: NG-UI uses unmaintained Chokidar version - Changed the node version from 10 to 14 incompatible with angular 14 version. Change-Id: Ib27ac394e2a30b9f66be530eab7fa3678d6bae01 Signed-off-by: SANDHYA.JS commit 7ff88a60b6493380746c694c0fad1b251ce08878 Author: aguilard Date: Wed Apr 26 11:26:07 2023 +0000 Feature 10981: changed repeat_interval parameter in AlertManager Change-Id: I317a7f33ef49b9e2103fedbb611efb960cb0d28f Signed-off-by: aguilard commit 0eda3f7f9927fc191d8f004e6dfaa1f7259d5429 Author: Dario Faccin Date: Thu Apr 20 16:13:50 2023 +0200 Improve stage-test script: Split charms list according to tox envlist. For newer charms the tox envlist includes lint, unit and integration: for these charms execute only lint and unit tests. For older charms the tox envlist includes black, cover, flake8, pylint, yamllint, safety: for these charms execute all tests. Change-Id: I6cfbe129440be1665f63572a1879060eccd822fd Signed-off-by: Dario Faccin commit 6db47d485db75641a282f8da13720e23d6f6dfb5 Author: Mark Beierl Date: Thu Mar 23 13:51:15 2023 -0400 OSM DB Update Charm Initial load of code for the osm-update-db-operator charm Change-Id: I2884249efaaa86f614df6c286a69f3546489b523 Signed-off-by: Mark Beierl commit f91f9c32f0d3cd4011a288235d0e4e1ba44b85d5 Author: aguilard Date: Thu Apr 13 16:57:57 2023 +0000 Feature 10981: extended Prometheus sidecar to dump alerts rules in config files Change-Id: Ic454c894b60d0b2b88b6ea81ca35a0bf4d54ebac Signed-off-by: aguilard commit 0e49b4aa3a8853ea5b9702e0bc58134d8a4bcd88 Author: garciadeblas Date: Mon Apr 17 12:38:17 2023 +0200 Feature 10981: remove mon and pol for ng-sa installation This change removes the deployment of POL for NG-SA installation. In addition, it deploys a reduced MON, which will only run mon-dashboarder. A new K8s manifest (ng-mon.yaml )file has been created for the purpose. Change-Id: I243a2710d7b883d505ff4b4d012f7d67920f0e73 Signed-off-by: garciadeblas commit cb9bb7a66ce49050e2ea005276f7d342673bbcc6 Author: aguilard Date: Tue Apr 18 16:25:19 2023 +0000 Feature 10981: fix CMD in webhook Dockerfile Change-Id: If8332c12c2f065c0a4d195873e24a98aa34b0ed4 Signed-off-by: aguilard commit 1805613aa8e2eb92a6daa7f7883ff10089447d37 Author: aguilard Date: Fri Apr 14 13:04:14 2023 +0000 Feature 10981: auto-scaling alerts rules for AlertManager Change-Id: I7e8c3f7b1dd3201b75848ae6264eaa2375a5b06b Signed-off-by: aguilard commit 63868ab3519bd2ab56e7da695e3e661ac877e9d8 Author: garciadeblas Date: Thu Apr 20 10:04:55 2023 +0200 Add nohup to http.server in tools/local-build.sh Change-Id: Ic21b33c22c069d6145ba9d60c7e3cebb75f99664 Signed-off-by: garciadeblas commit 171bbd48411e256e22156c56aa10d1cd2f8ca0da Author: garciadeblas Date: Fri Apr 14 10:00:31 2023 +0200 Update helm version to 3.11.3 Change-Id: Ic95f32cd1fc311bf93a817da90f48a17d7c2bd13 Signed-off-by: garciadeblas commit 325032a38c3cb0f433832e63241fa0a5ce5d95ea Author: garciadeblas Date: Thu Apr 13 18:07:44 2023 +0200 Replace OSM_STACK_NAME by OSM_NAMESPACE in installers scripts Change-Id: I5ce4bdc392fd64b4bed7479768b91adba53c67e4 Signed-off-by: garciadeblas commit 4c246cfd7065f2a06191e97e0d7970ef2ffaf1f9 Author: garciadeblas Date: Thu Apr 13 11:01:15 2023 +0200 Add Dockerfile for Webhook translator Change-Id: Id9a787e0fd3fd953b1b2ace190cdca6a77199f27 Signed-off-by: garciadeblas commit 2c5b33f530801667937e9147829bf8b02bd00409 Author: garciadeblas Date: Thu Apr 13 11:00:14 2023 +0200 Update tools/local-build.sh to run python http server instead of qhttp Change-Id: Id9857656e18e1487da7123e076bf00c0b9869d25 Signed-off-by: garciadeblas commit 7e26f9a0142e12dceda03d4b6e8cbbcbb2460c19 Author: garciadeblas Date: Mon Mar 27 12:49:43 2023 +0200 Remove unnecessary Makefile related to old docker image build process Change-Id: Icc304cfe7124979584405ec6635ce2c7a9861eac Signed-off-by: garciadeblas commit bacdfa852d07131631709fa16de142eb9257235e Author: garciadeblas Date: Tue Apr 4 10:52:46 2023 +0200 Feature 10981: installation of webhook translator as part of NG-SA Change-Id: I5318460103a6b89b37931bf661618251a3837d04 Signed-off-by: garciadeblas commit bae51f695443abcd0e410a92161af05b497ddc7a Author: garciadeblas Date: Tue Mar 28 18:27:20 2023 +0200 Feature 10981: installation of AlertManager as part of NG-SA Change-Id: I99bb5785081df4395be336f323d5d4ac3dfd68b6 Signed-off-by: garciadeblas commit 9ab64a953b0a46d6e09c16387c851a56c8c809cd Author: Dario Faccin Date: Mon Mar 27 10:38:22 2023 +0200 LCM integration tests: use RO charm from charmhub instead of building it locally Change-Id: I3c1aba9227d9ef5c28f559447da63035214c8ea1 Signed-off-by: Dario Faccin commit f88cb4f7140fd4e56b797deb29a41bd69c37913d Author: Dario Faccin Date: Wed Mar 15 14:55:07 2023 +0100 Integration tests for VCA Integrator Operator Change-Id: I2bc362961edb19f3a0696c779aa9eeaacc361572 Signed-off-by: Dario Faccin Signed-off-by: Mark Beierl commit 9a5ceab597f52688dd0496d67a556146a0472cd8 Author: garciadeblas Date: Tue Apr 4 10:51:51 2023 +0200 Minor indentation fixes in MON and POL K8s manifests Change-Id: Ib96f1655df650587fc6255d5f98986e1332bbb2f Signed-off-by: garciadeblas commit 473c541d5bf7a126e426490f0228f8812cc70b28 Author: Mark Beierl Date: Mon Apr 3 13:32:22 2023 -0400 Partial revert of 13026 The *.gz and *Packages are actually used in the creation of the debian repository for the installers. Change-Id: I56ba0ce478fba9bcaeb58d6f2abaf235a4eab78a Signed-off-by: Mark Beierl commit 0baeac7fd2079e87698fa1e3eea48833ee331cd7 Author: Guillermo Calvino Date: Thu Mar 23 16:14:35 2023 +0100 Adding documentation to OSM bundles Change-Id: I94b2d7467f4fba40b625acaf545dc20fc6079f8c Signed-off-by: Guillermo Calvino commit f776ffa5a6a673a62c51559d690cc86799c3269a Author: Dario Faccin Date: Fri Mar 31 16:47:35 2023 +0200 Update Dockerfile and stage-test script to run tests for charms This patch updates Dockerfile to use Ubuntu 20.04 as base for building and testing charms. This patch updates stage-test script to execute testing for charms. Tests will be executed only for charms modified by the review. This patch updates tox configuration for charms setting the python interpreter to python3.8. Change-Id: Ib9046b78d6520188cc51ac776fe60ea16479f11c Signed-off-by: Dario Faccin commit be1ab182ee509d5b458fcd1379a9101a7d2a09b1 Author: Dario Faccin Date: Mon Apr 3 13:21:19 2023 +0200 Update bundle (standalone and HA) to use MongoDB charm from edge channel Change-Id: Ie60a105a58c5838db90129f1d6d896907675a405 Signed-off-by: Dario Faccin commit 99325bd3a2891df4b88329931535619e96ae28ec Author: Dario Faccin Date: Fri Mar 31 16:56:38 2023 +0200 Remove checks for copyright in charms This patch removes the flake8 copyright plugin and configuration. Change-Id: I65e362748e16efbc48055370f8f1590d4910c000 Signed-off-by: Dario Faccin commit 75e3d990ec1cdb318f40f40676724a42e6528381 Author: garciadeblas Date: Thu Mar 23 17:31:44 2023 +0100 Update base image for Airflow to support Python 3.10 Change-Id: I4d0bd5be38faff10de4bd2dbaaa9a6010ab12732 Signed-off-by: garciadeblas commit fb220c0cbdf0936d0cd52e4527832aeadc9ac2b4 Author: garciadeblas Date: Thu Mar 23 17:29:26 2023 +0100 Update base image for Airflow to 2.5.2 Change-Id: Id73a0de10b80a4154e1816c5695d3c96de1b03fe Signed-off-by: garciadeblas commit d0303f5fff9c744aa101767166e94277c9e3259f Author: Dario Faccin Date: Fri Mar 24 11:21:19 2023 +0100 Pin Charmed Operator Framework version for charms Change-Id: Iff5659151e5678298b72e54b7b22a375bc7b7ebf Signed-off-by: Dario Faccin commit 48eea891e08c1427b711d6890308854df4ccc78b Author: garciadeblas Date: Thu Mar 23 14:04:09 2023 +0100 Fix docker tag in stage3 to coexist with periodic clean-up Change-Id: I1ce9a5de84e0bcedd7abaecfa0fb6d753b853cb7 Signed-off-by: garciadeblas commit ef27d2e228e3dcfe572dc77ce7e4342085630acf Author: garciadeblas Date: Wed Mar 22 22:03:13 2023 +0100 Remove duplicated lines in Airflow Dockerfile Change-Id: Iaeb200d498c01e53a7748293d39b6d9a0ba3cfa9 Signed-off-by: garciadeblas commit 57b89ae46f768a725e2ca7f01f7dfa97254e049a Author: Patricia Reinoso Date: Tue Mar 14 16:06:32 2023 +0000 Set K8s 1.26 in charmed OSM installation storage is deprecated: replaced by hostpath-storage Change-Id: I11dd6fc2c18f89c289ad80da696929a7c0236d63 Signed-off-by: Patricia Reinoso commit e64d8cfde60d376724327dddd17f343d1db0388d Author: Daniel Arndt Date: Fri Feb 24 09:41:14 2023 -0400 Fix unit tests for NG-UI charm Change-Id: If5b98446bb589a3346bcaf1d260a3ad2c5affd3b Signed-off-by: Daniel Arndt commit 773dd9959e5684902e84cc4751d47ccf321a4f22 Author: Daniel Arndt Date: Tue Mar 7 19:51:20 2023 -0400 Integration tests for NG UI charm Change-Id: I3c8958d54aeed84faf1ed2194bc818c1691cf755 Signed-off-by: Daniel Arndt commit 668b1469296df65fee65d62abbbbd087c69c2c04 Author: Mark Beierl Date: Wed Feb 22 13:00:58 2023 -0500 Charm cleanup Removal of obsolete charm code Change-Id: Ifc5e83457cf580d8b236a636328470c527c5c3a9 Signed-off-by: Mark Beierl commit 56db64f403eaea09530d1073640eee9b36ff2caf Author: Dario Faccin Date: Thu Feb 16 20:50:38 2023 +0100 Integration of OSM Charms with new MongoDB Change-Id: I9e723dc94ff4c5b7e691179be4e9e3c7b43b6ab0 Signed-off-by: Dario Faccin commit 740e877d19b5e77c8d7f64a85e805c0e68602d41 Author: garciadeblas Date: Tue Feb 28 12:56:07 2023 +0100 Update the artifacts stored in stage2 This change updates the patterns of the artifacts to be stored by the method `archive` in `ci_helper.groovy`. The pattern "dists/*.gz" and "dists/*Packages" corresponding to index files for debian repos are no longer required. The pattern "dist/*.whl" corresponding to Python wheel files is now required, since it is an additional artifact generated in stage2. Change-Id: Id87fcb98b2d79a9bd0b64fdaca44da8acd9e1cb1 Signed-off-by: garciadeblas commit 68bc89407c14e9e25cae41a94f1bc2e0c7eed59b Author: Daniel Arndt Date: Tue Feb 21 11:48:27 2023 -0400 Bug 2218: Fix command for `juju run-action` Change-Id: Ife2e8e9f532f3c67c7e2f71d3f77d3e4e7dc5257 Signed-off-by: Daniel Arndt commit 4734de1c25c0be183d2b415c9692af902e817da7 Author: Dario Faccin Date: Wed Feb 15 10:52:55 2023 +0100 Add OSM-MON integration tests Change-Id: I3199869880d0c9ce0784dcc623c844dd39f1180a Signed-off-by: Dario Faccin commit bb2f914425f0e7f64014d3edd6e55a83fe108975 Author: Dario Faccin Date: Wed Feb 15 09:29:55 2023 +0100 Add OSM-POL integration tests Change-Id: I140b9eb271c0f03520660b676e075b3f0d62a128 Signed-off-by: Dario Faccin Change-Id: I7020fe97c6f266c9ff3ca38a0141bcf15f606df2 Signed-off-by: Dario Faccin --- docker/Airflow/Dockerfile | 25 +- docker/LCM/Dockerfile | 34 ++- docker/MON/Dockerfile | 30 +- docker/NBI/Dockerfile | 33 ++- docker/NBI/pyangbind.patch | 46 +++ docker/NG-UI/Dockerfile | 4 +- docker/NGLCM/Dockerfile | 34 ++- docker/PLA/Dockerfile | 29 +- docker/POL/Dockerfile | 33 ++- docker/RO/Dockerfile | 30 +- docker/tests/Dockerfile | 29 +- docker/tests/pyangbind.patch | 46 +++ installers/charm/bundles/osm/bundle.yaml | 1 + installers/charm/keystone/charmcraft.yaml | 6 +- .../osm-update-db-operator/src/db_upgrade.py | 267 ++++++++++++++++++ installers/full_install_osm.sh | 111 +++++--- installers/helm/values/airflow-values.yaml | 5 +- .../helm/values/alertmanager-values.yaml | 4 +- installers/install_ngsa.sh | 19 +- jenkins/ci-pipelines/ci_stage_3.groovy | 22 +- tools/local-build.sh | 8 + 21 files changed, 651 insertions(+), 165 deletions(-) create mode 100644 docker/NBI/pyangbind.patch create mode 100644 docker/tests/pyangbind.patch diff --git a/docker/Airflow/Dockerfile b/docker/Airflow/Dockerfile index bf72444c..1a4184c2 100644 --- a/docker/Airflow/Dockerfile +++ b/docker/Airflow/Dockerfile @@ -15,12 +15,12 @@ # limitations under the License. ####################################################################################### -FROM apache/airflow:2.5.2-python3.10 +FROM apache/airflow:2.5.3-python3.10 USER root RUN DEBIAN_FRONTEND=noninteractive apt-get --yes update && \ DEBIAN_FRONTEND=noninteractive apt-get --yes install \ gcc git python3 python3-dev python3-venv python3-pip \ - python3-setuptools wget && \ + python3-setuptools wget zstd && \ python3 -m pip install -U pip build USER airflow @@ -29,14 +29,25 @@ ARG PYTHON3_OSM_COMMON_URL ARG PYTHON3_OSM_NGSA_URL RUN curl $PYTHON3_OSM_COMMON_URL -o osm_common.deb RUN curl $PYTHON3_OSM_NGSA_URL -o osm_ngsa.deb -RUN mkdir /tmp/osm -RUN dpkg-deb -x osm_common.deb /tmp/osm -RUN dpkg-deb -x osm_ngsa.deb /tmp/osm +RUN ar x osm_common.deb && \ + zstd -d < control.tar.zst | xz > control.tar.xz && \ + zstd -d < data.tar.zst | xz > data.tar.xz && \ + ar -m -c -a sdsd osm_common_repacked.deb debian-binary control.tar.xz data.tar.xz && \ + rm debian-binary control.tar.xz data.tar.xz control.tar.zst data.tar.zst +RUN ar x osm_ngsa.deb && \ + zstd -d < control.tar.zst | xz > control.tar.xz && \ + zstd -d < data.tar.zst | xz > data.tar.xz && \ + ar -m -c -a sdsd osm_ngsa_repacked.deb debian-binary control.tar.xz data.tar.xz && \ + rm debian-binary control.tar.xz data.tar.xz control.tar.zst data.tar.zst + +RUN mkdir /tmp/osm +RUN dpkg-deb -x osm_common_repacked.deb /tmp/osm +RUN dpkg-deb -x osm_ngsa_repacked.deb /tmp/osm RUN mv /tmp/osm/usr/lib/python3/dist-packages/* /home/airflow/.local/lib/python3.10/site-packages/ RUN rm -rf /tmp/osm RUN pip3 install \ - -r /home/airflow/.local/lib/python3.10/site-packages/osm_common/requirements.txt \ - -r /home/airflow/.local/lib/python3.10/site-packages/osm_ngsa/requirements.txt + -r /home/airflow/.local/lib/python3.10/site-packages/osm_common/requirements.txt \ + -r /home/airflow/.local/lib/python3.10/site-packages/osm_ngsa/requirements.txt diff --git a/docker/LCM/Dockerfile b/docker/LCM/Dockerfile index 48fcede4..a0a09355 100644 --- a/docker/LCM/Dockerfile +++ b/docker/LCM/Dockerfile @@ -15,7 +15,7 @@ # limitations under the License. ####################################################################################### -FROM ubuntu:20.04 as INSTALL +FROM ubuntu:22.04 as INSTALL ARG APT_PROXY RUN if [ ! -z $APT_PROXY ] ; then \ @@ -25,18 +25,21 @@ RUN if [ ! -z $APT_PROXY ] ; then \ RUN DEBIAN_FRONTEND=noninteractive apt-get --yes update && \ DEBIAN_FRONTEND=noninteractive apt-get --yes install \ - gcc=4:9.3.* \ - python3=3.8.* \ - python3-dev=3.8.* \ - python3-pip=20.0.2* \ - python3-setuptools=45.2.* \ - curl=7.68.* + gcc=4:11.* \ + python3=3.10.* \ + python3-dev=3.10.* \ + python3-pip=22.0.* \ + curl=7.81.* \ + && rm -rf /var/lib/apt/lists/* + +########### End of common prepration RUN DEBIAN_FRONTEND=noninteractive apt-get --yes update && \ DEBIAN_FRONTEND=noninteractive apt-get --yes install \ - apt-transport-https=2.0.* \ + apt-transport-https=2.4.* \ gnupg2=2.2.* \ - openssh-client=1:8.* + openssh-client=1:8.* \ + && rm -rf /var/lib/apt/lists/* # https://kubernetes.io/releases/ RUN curl -s https://packages.cloud.google.com/apt/doc/apt-key.gpg | apt-key add - \ @@ -73,7 +76,7 @@ RUN pip3 install \ -r /usr/lib/python3/dist-packages/n2vc/requirements.txt ####################################################################################### -FROM ubuntu:20.04 as FINAL +FROM ubuntu:22.04 as FINAL ARG APT_PROXY RUN if [ ! -z $APT_PROXY ] ; then \ @@ -83,9 +86,15 @@ RUN if [ ! -z $APT_PROXY ] ; then \ RUN DEBIAN_FRONTEND=noninteractive apt-get --yes update && \ DEBIAN_FRONTEND=noninteractive apt-get --yes install \ - python3-minimal=3.8.* \ + python3-minimal=3.10.* \ && rm -rf /var/lib/apt/lists/* +COPY --from=INSTALL /usr/lib/python3/dist-packages /usr/lib/python3/dist-packages +COPY --from=INSTALL /usr/local/lib/python3.10/dist-packages /usr/local/lib/python3.10/dist-packages + +####################################################################################### +# End of common prepration + RUN DEBIAN_FRONTEND=noninteractive apt-get --yes update && \ DEBIAN_FRONTEND=noninteractive apt-get --yes install \ ca-certificates \ @@ -93,9 +102,6 @@ RUN DEBIAN_FRONTEND=noninteractive apt-get --yes update && \ RUN rm -f /etc/apt/apt.conf.d/proxy.conf -COPY --from=INSTALL /usr/lib/python3/dist-packages /usr/lib/python3/dist-packages -COPY --from=INSTALL /usr/local/lib/python3.8/dist-packages /usr/local/lib/python3.8/dist-packages - COPY --from=INSTALL /usr/bin/kubectl /usr/bin/kubectl COPY --from=INSTALL /usr/local/bin/helm /usr/local/bin/helm COPY --from=INSTALL /usr/local/bin/helm3 /usr/local/bin/helm3 diff --git a/docker/MON/Dockerfile b/docker/MON/Dockerfile index cd56f536..11b6e14c 100644 --- a/docker/MON/Dockerfile +++ b/docker/MON/Dockerfile @@ -15,7 +15,7 @@ # limitations under the License. ####################################################################################### -FROM ubuntu:20.04 as INSTALL +FROM ubuntu:22.04 as INSTALL ARG APT_PROXY RUN if [ ! -z $APT_PROXY ] ; then \ @@ -25,12 +25,15 @@ RUN if [ ! -z $APT_PROXY ] ; then \ RUN DEBIAN_FRONTEND=noninteractive apt-get --yes update && \ DEBIAN_FRONTEND=noninteractive apt-get --yes install \ - gcc=4:9.3.* \ - python3=3.8.* \ - python3-dev=3.8.* \ - python3-pip=20.0.2* \ - python3-setuptools=45.2.* \ - curl=7.68.* + gcc=4:11.* \ + python3=3.10.* \ + python3-dev=3.10.* \ + python3-pip=22.0.* \ + curl=7.81.* \ + && rm -rf /var/lib/apt/lists/* + +####################################################################################### +# End of common prepration RUN DEBIAN_FRONTEND=noninteractive apt-get --yes update && \ DEBIAN_FRONTEND=noninteractive apt-get --yes install \ @@ -55,7 +58,8 @@ RUN pip3 install \ -r /usr/lib/python3/dist-packages/osm_mon/requirements.txt ####################################################################################### -FROM ubuntu:20.04 as FINAL +####################################################################################### +FROM ubuntu:22.04 as FINAL ARG APT_PROXY RUN if [ ! -z $APT_PROXY ] ; then \ @@ -65,9 +69,15 @@ RUN if [ ! -z $APT_PROXY ] ; then \ RUN DEBIAN_FRONTEND=noninteractive apt-get --yes update && \ DEBIAN_FRONTEND=noninteractive apt-get --yes install \ - python3-minimal=3.8.* \ + python3-minimal=3.10.* \ && rm -rf /var/lib/apt/lists/* +COPY --from=INSTALL /usr/lib/python3/dist-packages /usr/lib/python3/dist-packages +COPY --from=INSTALL /usr/local/lib/python3.10/dist-packages /usr/local/lib/python3.10/dist-packages + +####################################################################################### +# End of common prepration + RUN DEBIAN_FRONTEND=noninteractive apt-get --yes update && \ DEBIAN_FRONTEND=noninteractive apt-get --yes install \ ca-certificates \ @@ -75,8 +85,6 @@ RUN DEBIAN_FRONTEND=noninteractive apt-get --yes update && \ RUN rm -f /etc/apt/apt.conf.d/proxy.conf -COPY --from=INSTALL /usr/lib/python3/dist-packages /usr/lib/python3/dist-packages -COPY --from=INSTALL /usr/local/lib/python3.8/dist-packages /usr/local/lib/python3.8/dist-packages COPY --from=INSTALL /usr/bin/osm* /usr/bin/ COPY --from=INSTALL /usr/bin/scp /usr/bin/scp COPY --from=INSTALL /usr/bin/ssh-keygen /usr/bin/ssh-keygen diff --git a/docker/NBI/Dockerfile b/docker/NBI/Dockerfile index 89574791..d09ee050 100644 --- a/docker/NBI/Dockerfile +++ b/docker/NBI/Dockerfile @@ -15,7 +15,7 @@ # limitations under the License. ####################################################################################### -FROM ubuntu:20.04 as INSTALL +FROM ubuntu:22.04 as INSTALL ARG APT_PROXY RUN if [ ! -z $APT_PROXY ] ; then \ @@ -25,12 +25,15 @@ RUN if [ ! -z $APT_PROXY ] ; then \ RUN DEBIAN_FRONTEND=noninteractive apt-get --yes update && \ DEBIAN_FRONTEND=noninteractive apt-get --yes install \ - gcc=4:9.3.* \ - python3=3.8.* \ - python3-dev=3.8.* \ - python3-pip=20.0.2* \ - python3-setuptools=45.2.* \ - curl=7.68.* + gcc=4:11.* \ + python3=3.10.* \ + python3-dev=3.10.* \ + python3-pip=22.0.* \ + curl=7.81.* \ + && rm -rf /var/lib/apt/lists/* + +####################################################################################### +# End of common prepration ARG PYTHON3_OSM_COMMON_URL ARG PYTHON3_OSM_IM_URL @@ -50,8 +53,11 @@ RUN pip3 install \ -r /usr/lib/python3/dist-packages/osm_im/requirements.txt \ -r /usr/lib/python3/dist-packages/osm_nbi/requirements.txt +COPY pyangbind.patch ./ +RUN patch /usr/local/lib/python3.10/dist-packages/pyangbind/lib/yangtypes.py < ./pyangbind.patch + ####################################################################################### -FROM ubuntu:20.04 as FINAL +FROM ubuntu:22.04 as FINAL ARG APT_PROXY RUN if [ ! -z $APT_PROXY ] ; then \ @@ -61,13 +67,16 @@ RUN if [ ! -z $APT_PROXY ] ; then \ RUN DEBIAN_FRONTEND=noninteractive apt-get --yes update && \ DEBIAN_FRONTEND=noninteractive apt-get --yes install \ - python3-minimal=3.8.* \ + python3-minimal=3.10.* \ && rm -rf /var/lib/apt/lists/* -RUN rm -f /etc/apt/apt.conf.d/proxy.conf - COPY --from=INSTALL /usr/lib/python3/dist-packages /usr/lib/python3/dist-packages -COPY --from=INSTALL /usr/local/lib/python3.8/dist-packages /usr/local/lib/python3.8/dist-packages +COPY --from=INSTALL /usr/local/lib/python3.10/dist-packages /usr/local/lib/python3.10/dist-packages + +####################################################################################### +# End of common prepration + +RUN rm -f /etc/apt/apt.conf.d/proxy.conf # Creating the user for the app RUN groupadd -g 1000 appuser && \ diff --git a/docker/NBI/pyangbind.patch b/docker/NBI/pyangbind.patch new file mode 100644 index 00000000..30772996 --- /dev/null +++ b/docker/NBI/pyangbind.patch @@ -0,0 +1,46 @@ +####################################################################################### +# Copyright ETSI Contributors and Others. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. +####################################################################################### + +*** .tox/cover/lib/python3.10/site-packages/pyangbind/lib/yangtypes.py 2023-05-10 06:50:57.876027148 -0400 +--- .tox/cover/lib/python3.10/site-packages/pyangbind/lib/yangtypes.py 2023-05-10 06:51:11.772022417 -0400 +*************** limitations under the License. +*** 22,27 **** +--- 22,28 ---- + from __future__ import unicode_literals + + import collections ++ from six.moves import collections_abc + import copy + import uuid + from decimal import Decimal +*************** def TypedListType(*args, **kwargs): +*** 372,378 **** + if not isinstance(allowed_type, list): + allowed_type = [allowed_type] + +! class TypedList(collections.MutableSequence): + _pybind_generated_by = "TypedListType" + _list = list() + +--- 373,379 ---- + if not isinstance(allowed_type, list): + allowed_type = [allowed_type] + +! class TypedList(collections_abc.MutableSequence): + _pybind_generated_by = "TypedListType" + _list = list() + diff --git a/docker/NG-UI/Dockerfile b/docker/NG-UI/Dockerfile index 74aa1d46..418de25b 100644 --- a/docker/NG-UI/Dockerfile +++ b/docker/NG-UI/Dockerfile @@ -33,9 +33,9 @@ RUN DEBIAN_FRONTEND=noninteractive apt-get --yes update && \ xz-utils=5.2.* RUN curl -s https://deb.nodesource.com/gpgkey/nodesource.gpg.key | apt-key add - && \ - echo "deb https://deb.nodesource.com/node_10.x focal main" | tee -a /etc/apt/sources.list.d/nodesource.list && \ + echo "deb https://deb.nodesource.com/node_14.x focal main" | tee -a /etc/apt/sources.list.d/nodesource.list && \ DEBIAN_FRONTEND=noninteractive apt-get update && \ - DEBIAN_FRONTEND=noninteractive apt-get install -y nodejs=10.24.* + DEBIAN_FRONTEND=noninteractive apt-get install -y nodejs ARG OSM_NGUI_URL diff --git a/docker/NGLCM/Dockerfile b/docker/NGLCM/Dockerfile index 9ededf5b..13443153 100644 --- a/docker/NGLCM/Dockerfile +++ b/docker/NGLCM/Dockerfile @@ -15,7 +15,7 @@ # limitations under the License. ####################################################################################### -FROM ubuntu:20.04 as INSTALL +FROM ubuntu:22.04 as INSTALL ARG APT_PROXY RUN if [ ! -z $APT_PROXY ] ; then \ @@ -25,18 +25,21 @@ RUN if [ ! -z $APT_PROXY ] ; then \ RUN DEBIAN_FRONTEND=noninteractive apt-get --yes update && \ DEBIAN_FRONTEND=noninteractive apt-get --yes install \ - gcc=4:9.3.* \ - python3=3.8.* \ - python3-dev=3.8.* \ - python3-pip=20.0.2* \ - python3-setuptools=45.2.* \ - curl=7.68.* + gcc=4:11.* \ + python3=3.10.* \ + python3-dev=3.10.* \ + python3-pip=22.0.* \ + curl=7.81.* \ + && rm -rf /var/lib/apt/lists/* + +########### End of common prepration RUN DEBIAN_FRONTEND=noninteractive apt-get --yes update && \ DEBIAN_FRONTEND=noninteractive apt-get --yes install \ - apt-transport-https=2.0.* \ + apt-transport-https=2.4.* \ gnupg2=2.2.* \ - openssh-client=1:8.* + openssh-client=1:8.* \ + && rm -rf /var/lib/apt/lists/* # https://kubernetes.io/releases/ RUN curl -s https://packages.cloud.google.com/apt/doc/apt-key.gpg | apt-key add - \ @@ -73,7 +76,7 @@ RUN pip3 install \ -r /usr/lib/python3/dist-packages/n2vc/requirements.txt ####################################################################################### -FROM ubuntu:20.04 as FINAL +FROM ubuntu:22.04 as FINAL ARG APT_PROXY RUN if [ ! -z $APT_PROXY ] ; then \ @@ -83,9 +86,15 @@ RUN if [ ! -z $APT_PROXY ] ; then \ RUN DEBIAN_FRONTEND=noninteractive apt-get --yes update && \ DEBIAN_FRONTEND=noninteractive apt-get --yes install \ - python3-minimal=3.8.* \ + python3-minimal=3.10.* \ && rm -rf /var/lib/apt/lists/* +COPY --from=INSTALL /usr/lib/python3/dist-packages /usr/lib/python3/dist-packages +COPY --from=INSTALL /usr/local/lib/python3.10/dist-packages /usr/local/lib/python3.10/dist-packages + +####################################################################################### +# End of common prepration + RUN DEBIAN_FRONTEND=noninteractive apt-get --yes update && \ DEBIAN_FRONTEND=noninteractive apt-get --yes install \ ca-certificates \ @@ -93,9 +102,6 @@ RUN DEBIAN_FRONTEND=noninteractive apt-get --yes update && \ RUN rm -f /etc/apt/apt.conf.d/proxy.conf -COPY --from=INSTALL /usr/lib/python3/dist-packages /usr/lib/python3/dist-packages -COPY --from=INSTALL /usr/local/lib/python3.8/dist-packages /usr/local/lib/python3.8/dist-packages - COPY --from=INSTALL /usr/bin/kubectl /usr/bin/kubectl COPY --from=INSTALL /usr/local/bin/helm /usr/local/bin/helm COPY --from=INSTALL /usr/local/bin/helm3 /usr/local/bin/helm3 diff --git a/docker/PLA/Dockerfile b/docker/PLA/Dockerfile index f6373625..9c23262f 100644 --- a/docker/PLA/Dockerfile +++ b/docker/PLA/Dockerfile @@ -15,7 +15,7 @@ # limitations under the License. ####################################################################################### -FROM ubuntu:20.04 as INSTALL +FROM ubuntu:22.04 as INSTALL ARG APT_PROXY RUN if [ ! -z $APT_PROXY ] ; then \ @@ -25,12 +25,15 @@ RUN if [ ! -z $APT_PROXY ] ; then \ RUN DEBIAN_FRONTEND=noninteractive apt-get --yes update && \ DEBIAN_FRONTEND=noninteractive apt-get --yes install \ - gcc=4:9.3.* \ - python3=3.8.* \ - python3-dev=3.8.* \ - python3-pip=20.0.2* \ - python3-setuptools=45.2.* \ - curl=7.68.* + gcc=4:11.* \ + python3=3.10.* \ + python3-dev=3.10.* \ + python3-pip=22.0.* \ + curl=7.81.* \ + && rm -rf /var/lib/apt/lists/* + +####################################################################################### +# End of common prepration ARG PYTHON3_OSM_COMMON_URL ARG PYTHON3_OSM_PLA_URL @@ -51,7 +54,7 @@ RUN tar -zxf /minizinc.tgz && \ mv /MiniZincIDE-2.4.2-bundle-linux /minizinc ####################################################################################### -FROM ubuntu:20.04 as FINAL +FROM ubuntu:22.04 as FINAL ARG APT_PROXY RUN if [ ! -z $APT_PROXY ] ; then \ @@ -61,15 +64,19 @@ RUN if [ ! -z $APT_PROXY ] ; then \ RUN DEBIAN_FRONTEND=noninteractive apt-get --yes update && \ DEBIAN_FRONTEND=noninteractive apt-get --yes install \ - python3-minimal=3.8.* \ + python3-minimal=3.10.* \ && rm -rf /var/lib/apt/lists/* +COPY --from=INSTALL /usr/lib/python3/dist-packages /usr/lib/python3/dist-packages +COPY --from=INSTALL /usr/local/lib/python3.10/dist-packages /usr/local/lib/python3.10/dist-packages + +####################################################################################### +# End of common prepration + RUN rm -f /etc/apt/apt.conf.d/proxy.conf LABEL authors="Lars-Göran Magnusson" -COPY --from=INSTALL /usr/lib/python3/dist-packages /usr/lib/python3/dist-packages -COPY --from=INSTALL /usr/local/lib/python3.8/dist-packages /usr/local/lib/python3.8/dist-packages COPY --from=INSTALL /usr/bin/osm* /usr/bin/ COPY --from=INSTALL /minizinc /minizinc diff --git a/docker/POL/Dockerfile b/docker/POL/Dockerfile index b807bfdb..8ed8f5ed 100644 --- a/docker/POL/Dockerfile +++ b/docker/POL/Dockerfile @@ -15,7 +15,7 @@ # limitations under the License. ####################################################################################### -FROM ubuntu:20.04 as INSTALL +FROM ubuntu:22.04 as INSTALL ARG APT_PROXY RUN if [ ! -z $APT_PROXY ] ; then \ @@ -25,12 +25,15 @@ RUN if [ ! -z $APT_PROXY ] ; then \ RUN DEBIAN_FRONTEND=noninteractive apt-get --yes update && \ DEBIAN_FRONTEND=noninteractive apt-get --yes install \ - gcc=4:9.3.* \ - python3=3.8.* \ - python3-dev=3.8.* \ - python3-pip=20.0.2* \ - python3-setuptools=45.2.* \ - curl=7.68.* + gcc=4:11.* \ + python3=3.10.* \ + python3-dev=3.10.* \ + python3-pip=22.0.* \ + curl=7.81.* \ + && rm -rf /var/lib/apt/lists/* + +####################################################################################### +# End of common prepration RUN DEBIAN_FRONTEND=noninteractive apt-get --yes update && \ DEBIAN_FRONTEND=noninteractive apt-get --yes install \ @@ -50,7 +53,7 @@ RUN pip3 install \ -r /usr/lib/python3/dist-packages/osm_policy_module/requirements.txt ####################################################################################### -FROM ubuntu:20.04 as FINAL +FROM ubuntu:22.04 as FINAL ARG APT_PROXY RUN if [ ! -z $APT_PROXY ] ; then \ @@ -60,13 +63,15 @@ RUN if [ ! -z $APT_PROXY ] ; then \ RUN DEBIAN_FRONTEND=noninteractive apt-get --yes update && \ DEBIAN_FRONTEND=noninteractive apt-get --yes install \ - python3-minimal=3.8.* \ + python3-minimal=3.10.* \ && rm -rf /var/lib/apt/lists/* -RUN rm -f /etc/apt/apt.conf.d/proxy.conf - COPY --from=INSTALL /usr/lib/python3/dist-packages /usr/lib/python3/dist-packages -COPY --from=INSTALL /usr/local/lib/python3.8/dist-packages /usr/local/lib/python3.8/dist-packages +COPY --from=INSTALL /usr/local/lib/python3.10/dist-packages /usr/local/lib/python3.10/dist-packages + +####################################################################################### +# End of common prepration + COPY --from=INSTALL /usr/bin/osm* /usr/bin/ COPY --from=INSTALL /usr/bin/mysql /usr/bin/ COPY --from=INSTALL /usr/bin/mysqladmin /usr/bin/ @@ -102,6 +107,10 @@ ENV OSMPOL_SQL_DATABASE_URI sqlite:///mon_sqlite.db ENV OSMPOL_GLOBAL_LOG_LEVEL INFO +ENV OSMPOL_AUTOHEAL_ENABLED True + +ENV OSMPOL_AUTOSCALE_ENABLED True + HEALTHCHECK --start-period=120s --interval=10s --timeout=5s --retries=5 \ CMD osm-pol-healthcheck || exit 1 diff --git a/docker/RO/Dockerfile b/docker/RO/Dockerfile index 64bad452..57c5f6ae 100644 --- a/docker/RO/Dockerfile +++ b/docker/RO/Dockerfile @@ -15,7 +15,7 @@ # limitations under the License. ####################################################################################### -FROM ubuntu:20.04 as INSTALL +FROM ubuntu:22.04 as INSTALL ARG APT_PROXY RUN if [ ! -z $APT_PROXY ] ; then \ @@ -25,12 +25,15 @@ RUN if [ ! -z $APT_PROXY ] ; then \ RUN DEBIAN_FRONTEND=noninteractive apt-get --yes update && \ DEBIAN_FRONTEND=noninteractive apt-get --yes install \ - gcc=4:9.3.* \ - python3=3.8.* \ - python3-dev=3.8.* \ - python3-pip=20.0.2* \ - python3-setuptools=45.2.* \ - curl=7.68.* + gcc=4:11.* \ + python3=3.10.* \ + python3-dev=3.10.* \ + python3-pip=22.0.* \ + curl=7.81.* \ + && rm -rf /var/lib/apt/lists/* + +####################################################################################### +# End of common prepration RUN DEBIAN_FRONTEND=noninteractive apt-get --yes update && \ DEBIAN_FRONTEND=noninteractive apt-get --yes install \ @@ -85,7 +88,8 @@ RUN pip3 install \ -r /usr/lib/python3/dist-packages/osm_ng_ro/requirements.txt ####################################################################################### -FROM ubuntu:20.04 as FINAL +####################################################################################### +FROM ubuntu:22.04 as FINAL ARG APT_PROXY RUN if [ ! -z $APT_PROXY ] ; then \ @@ -95,13 +99,17 @@ RUN if [ ! -z $APT_PROXY ] ; then \ RUN DEBIAN_FRONTEND=noninteractive apt-get --yes update && \ DEBIAN_FRONTEND=noninteractive apt-get --yes install \ - python3-minimal=3.8.* \ + python3-minimal=3.10.* \ && rm -rf /var/lib/apt/lists/* +COPY --from=INSTALL /usr/lib/python3/dist-packages /usr/lib/python3/dist-packages +COPY --from=INSTALL /usr/local/lib/python3.10/dist-packages /usr/local/lib/python3.10/dist-packages + +####################################################################################### +# End of common prepration + RUN rm -f /etc/apt/apt.conf.d/proxy.conf -COPY --from=INSTALL /usr/lib/python3/dist-packages /usr/lib/python3/dist-packages -COPY --from=INSTALL /usr/local/lib/python3.8/dist-packages /usr/local/lib/python3.8/dist-packages COPY --from=INSTALL /usr/bin/genisoimage /usr/bin/genisoimage COPY --from=INSTALL /etc/protocols /etc/protocols diff --git a/docker/tests/Dockerfile b/docker/tests/Dockerfile index 5eeda0de..2c824bc3 100644 --- a/docker/tests/Dockerfile +++ b/docker/tests/Dockerfile @@ -15,7 +15,7 @@ # limitations under the License. ####################################################################################### -FROM ubuntu:20.04 as FINAL +FROM ubuntu:22.04 as FINAL ARG APT_PROXY RUN if [ ! -z $APT_PROXY ] ; then \ @@ -25,21 +25,25 @@ RUN if [ ! -z $APT_PROXY ] ; then \ RUN DEBIAN_FRONTEND=noninteractive apt-get --yes update && \ DEBIAN_FRONTEND=noninteractive apt-get --yes install \ - gcc=4:9.3.* \ - python3=3.8.* \ - python3-dev=3.8.* \ - python3-pip=20.0.2* \ - python3-setuptools=45.2.* \ - curl=7.68.* + gcc=4:11.* \ + python3=3.10.* \ + python3-dev=3.10.* \ + python3-pip=22.0.* \ + curl=7.81.* \ + && rm -rf /var/lib/apt/lists/* + +####################################################################################### +# End of common preparation RUN DEBIAN_FRONTEND=noninteractive apt-get --yes update && \ DEBIAN_FRONTEND=noninteractive apt-get --yes install \ - gettext-base=0.19.* \ - git=1:2.25.* \ + gettext-base=0.21* \ + git=1:2.34.* \ iputils-ping=3:* \ jq=1.6* \ - libcurl4-openssl-dev=7.68.* \ - libssl-dev=1.1.* + libcurl4-openssl-dev=7.81.* \ + libssl-dev=3.0.* \ + libmagic1=1:5.* ARG OSM_TESTS_URL ARG PYTHON3_OSM_IM_URL @@ -63,6 +67,9 @@ RUN mv /usr/share/osm-tests/robot-systest /robot-systest RUN mv /usr/share/osm-tests/conformance-tests/ /robot-systest/ RUN mv /usr/share/osm-tests/charm.sh /usr/sbin/charm +COPY pyangbind.patch /robot-systest +RUN patch /usr/local/lib/python3.10/dist-packages/pyangbind/lib/yangtypes.py < /robot-systest/pyangbind.patch + ARG CACHE_DATE=not_a_date RUN git clone \ https://osm.etsi.org/gitlab/vnf-onboarding/osm-packages.git \ diff --git a/docker/tests/pyangbind.patch b/docker/tests/pyangbind.patch new file mode 100644 index 00000000..30772996 --- /dev/null +++ b/docker/tests/pyangbind.patch @@ -0,0 +1,46 @@ +####################################################################################### +# Copyright ETSI Contributors and Others. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. +####################################################################################### + +*** .tox/cover/lib/python3.10/site-packages/pyangbind/lib/yangtypes.py 2023-05-10 06:50:57.876027148 -0400 +--- .tox/cover/lib/python3.10/site-packages/pyangbind/lib/yangtypes.py 2023-05-10 06:51:11.772022417 -0400 +*************** limitations under the License. +*** 22,27 **** +--- 22,28 ---- + from __future__ import unicode_literals + + import collections ++ from six.moves import collections_abc + import copy + import uuid + from decimal import Decimal +*************** def TypedListType(*args, **kwargs): +*** 372,378 **** + if not isinstance(allowed_type, list): + allowed_type = [allowed_type] + +! class TypedList(collections.MutableSequence): + _pybind_generated_by = "TypedListType" + _list = list() + +--- 373,379 ---- + if not isinstance(allowed_type, list): + allowed_type = [allowed_type] + +! class TypedList(collections_abc.MutableSequence): + _pybind_generated_by = "TypedListType" + _list = list() + diff --git a/installers/charm/bundles/osm/bundle.yaml b/installers/charm/bundles/osm/bundle.yaml index 058ad3e2..9dd35802 100644 --- a/installers/charm/bundles/osm/bundle.yaml +++ b/installers/charm/bundles/osm/bundle.yaml @@ -122,6 +122,7 @@ applications: vca: charm: osm-vca-integrator channel: latest/beta + series: focal scale: 1 ingress: charm: nginx-ingress-integrator diff --git a/installers/charm/keystone/charmcraft.yaml b/installers/charm/keystone/charmcraft.yaml index 0a285a9d..04b659ac 100644 --- a/installers/charm/keystone/charmcraft.yaml +++ b/installers/charm/keystone/charmcraft.yaml @@ -34,4 +34,8 @@ bases: - arm64 parts: charm: - build-packages: [git] + build-packages: + - git + - libffi-dev + charm-binary-python-packages: + - cryptography diff --git a/installers/charm/osm-update-db-operator/src/db_upgrade.py b/installers/charm/osm-update-db-operator/src/db_upgrade.py index 05cc0a0c..295ce875 100644 --- a/installers/charm/osm-update-db-operator/src/db_upgrade.py +++ b/installers/charm/osm-update-db-operator/src/db_upgrade.py @@ -18,10 +18,276 @@ import json import logging from pymongo import MongoClient +from uuid import uuid4 logger = logging.getLogger(__name__) +class MongoUpgrade1214: + """Upgrade MongoDB Database from OSM v12 to v14.""" + + @staticmethod + def gather_vnfr_healing_alerts(vnfr, vnfd): + alerts = [] + nsr_id = vnfr["nsr-id-ref"] + df = vnfd.get("df", [{}])[0] + # Checking for auto-healing configuration + if "healing-aspect" in df: + healing_aspects = df["healing-aspect"] + for healing in healing_aspects: + for healing_policy in healing.get("healing-policy", ()): + vdu_id = healing_policy["vdu-id"] + vdur = next( + ( + vdur + for vdur in vnfr["vdur"] + if vdu_id == vdur["vdu-id-ref"] + ), + {}, + ) + if not vdur: + continue + metric_name = "vm_status" + vdu_name = vdur.get("name") + vnf_member_index = vnfr["member-vnf-index-ref"] + uuid = str(uuid4()) + name = f"healing_{uuid}" + action = healing_policy + # action_on_recovery = healing.get("action-on-recovery") + # cooldown_time = healing.get("cooldown-time") + # day1 = healing.get("day1") + alert = { + "uuid": uuid, + "name": name, + "metric": metric_name, + "tags": { + "ns_id": nsr_id, + "vnf_member_index": vnf_member_index, + "vdu_name": vdu_name, + }, + "alarm_status": "ok", + "action_type": "healing", + "action": action, + } + alerts.append(alert) + return alerts + + @staticmethod + def gather_vnfr_scaling_alerts(vnfr, vnfd): + alerts = [] + nsr_id = vnfr["nsr-id-ref"] + df = vnfd.get("df", [{}])[0] + # Checking for auto-scaling configuration + if "scaling-aspect" in df: + rel_operation_types = { + "GE": ">=", + "LE": "<=", + "GT": ">", + "LT": "<", + "EQ": "==", + "NE": "!=", + } + scaling_aspects = df["scaling-aspect"] + all_vnfd_monitoring_params = {} + for ivld in vnfd.get("int-virtual-link-desc", ()): + for mp in ivld.get("monitoring-parameters", ()): + all_vnfd_monitoring_params[mp.get("id")] = mp + for vdu in vnfd.get("vdu", ()): + for mp in vdu.get("monitoring-parameter", ()): + all_vnfd_monitoring_params[mp.get("id")] = mp + for df in vnfd.get("df", ()): + for mp in df.get("monitoring-parameter", ()): + all_vnfd_monitoring_params[mp.get("id")] = mp + for scaling_aspect in scaling_aspects: + scaling_group_name = scaling_aspect.get("name", "") + # Get monitored VDUs + all_monitored_vdus = set() + for delta in scaling_aspect.get( + "aspect-delta-details", {} + ).get("deltas", ()): + for vdu_delta in delta.get("vdu-delta", ()): + all_monitored_vdus.add(vdu_delta.get("id")) + monitored_vdurs = list( + filter( + lambda vdur: vdur["vdu-id-ref"] + in all_monitored_vdus, + vnfr["vdur"], + ) + ) + if not monitored_vdurs: + logger.error("Scaling criteria is referring to a vnf-monitoring-param that does not contain a reference to a vdu or vnf metric") + continue + for scaling_policy in scaling_aspect.get( + "scaling-policy", () + ): + if scaling_policy["scaling-type"] != "automatic": + continue + threshold_time = scaling_policy.get( + "threshold-time", "1" + ) + cooldown_time = scaling_policy.get("cooldown-time", "0") + for scaling_criteria in scaling_policy["scaling-criteria"]: + monitoring_param_ref = scaling_criteria.get( + "vnf-monitoring-param-ref" + ) + vnf_monitoring_param = all_vnfd_monitoring_params[ + monitoring_param_ref + ] + for vdur in monitored_vdurs: + vdu_id = vdur["vdu-id-ref"] + metric_name = vnf_monitoring_param.get("performance-metric") + metric_name = f"osm_{metric_name}" + vdu_name = vdur["name"] + vnf_member_index = vnfr["member-vnf-index-ref"] + scalein_threshold = scaling_criteria.get("scale-in-threshold") + # Looking for min/max-number-of-instances + instances_min_number = 1 + instances_max_number = 1 + vdu_profile = df["vdu-profile"] + if vdu_profile: + profile = next( + item + for item in vdu_profile + if item["id"] == vdu_id + ) + instances_min_number = profile.get("min-number-of-instances", 1) + instances_max_number = profile.get("max-number-of-instances", 1) + + if scalein_threshold: + uuid = str(uuid4()) + name = f"scalein_{uuid}" + operation = scaling_criteria["scale-in-relational-operation"] + rel_operator = rel_operation_types.get(operation, "<=") + metric_selector = f'{metric_name}{{ns_id="{nsr_id}", vnf_member_index="{vnf_member_index}", vdu_id="{vdu_id}"}}' + expression = f"(count ({metric_selector}) > {instances_min_number}) and (avg({metric_selector}) {rel_operator} {scalein_threshold})" + labels = { + "ns_id": nsr_id, + "vnf_member_index": vnf_member_index, + "vdu_id": vdu_id, + } + prom_cfg = { + "alert": name, + "expr": expression, + "for": str(threshold_time) + "m", + "labels": labels, + } + action = scaling_policy + action = { + "scaling-group": scaling_group_name, + "cooldown-time": cooldown_time, + } + alert = { + "uuid": uuid, + "name": name, + "metric": metric_name, + "tags": { + "ns_id": nsr_id, + "vnf_member_index": vnf_member_index, + "vdu_id": vdu_id, + }, + "alarm_status": "ok", + "action_type": "scale_in", + "action": action, + "prometheus_config": prom_cfg, + } + alerts.append(alert) + + scaleout_threshold = scaling_criteria.get("scale-out-threshold") + if scaleout_threshold: + uuid = str(uuid4()) + name = f"scaleout_{uuid}" + operation = scaling_criteria["scale-out-relational-operation"] + rel_operator = rel_operation_types.get(operation, "<=") + metric_selector = f'{metric_name}{{ns_id="{nsr_id}", vnf_member_index="{vnf_member_index}", vdu_id="{vdu_id}"}}' + expression = f"(count ({metric_selector}) < {instances_max_number}) and (avg({metric_selector}) {rel_operator} {scaleout_threshold})" + labels = { + "ns_id": nsr_id, + "vnf_member_index": vnf_member_index, + "vdu_id": vdu_id, + } + prom_cfg = { + "alert": name, + "expr": expression, + "for": str(threshold_time) + "m", + "labels": labels, + } + action = scaling_policy + action = { + "scaling-group": scaling_group_name, + "cooldown-time": cooldown_time, + } + alert = { + "uuid": uuid, + "name": name, + "metric": metric_name, + "tags": { + "ns_id": nsr_id, + "vnf_member_index": vnf_member_index, + "vdu_id": vdu_id, + }, + "alarm_status": "ok", + "action_type": "scale_out", + "action": action, + "prometheus_config": prom_cfg, + } + alerts.append(alert) + return alerts + + @staticmethod + def _migrate_alerts(osm_db): + """Create new alerts collection. + """ + if "alerts" in osm_db.list_collection_names(): + return + logger.info("Entering in MongoUpgrade1214._migrate_alerts function") + + # Get vnfds from MongoDB + logger.info("Reading VNF descriptors:") + vnfds = osm_db["vnfds"] + db_vnfds = [] + for vnfd in vnfds.find(): + logger.info(f' {vnfd["_id"]}: {vnfd["description"]}') + db_vnfds.append(vnfd) + + # Get vnfrs from MongoDB + logger.info("Reading VNFRs") + vnfrs = osm_db["vnfrs"] + + # Gather healing and scaling alerts for each vnfr + healing_alerts = [] + scaling_alerts = [] + for vnfr in vnfrs.find(): + logger.info(f' vnfr {vnfr["_id"]}') + vnfd = next((sub for sub in db_vnfds if sub["_id"] == vnfr["vnfd-id"]), None) + healing_alerts.extend(MongoUpgrade1214.gather_vnfr_healing_alerts(vnfr, vnfd)) + scaling_alerts.extend(MongoUpgrade1214.gather_vnfr_scaling_alerts(vnfr, vnfd)) + + # Add new alerts in MongoDB + alerts = osm_db["alerts"] + for alert in healing_alerts: + logger.info(f"Storing healing alert in MongoDB: {alert}") + alerts.insert_one(alert) + for alert in scaling_alerts: + logger.info(f"Storing scaling alert in MongoDB: {alert}") + alerts.insert_one(alert) + + # Delete old alarms collections + logger.info("Deleting alarms and alarms_action collections") + alarms = osm_db["alarms"] + alarms.drop() + alarms_action = osm_db["alarms_action"] + alarms_action.drop() + + + @staticmethod + def upgrade(mongo_uri): + """Upgrade alerts in MongoDB.""" + logger.info("Entering in MongoUpgrade1214.upgrade function") + myclient = MongoClient(mongo_uri) + osm_db = myclient["osm"] + MongoUpgrade1214._migrate_alerts(osm_db) + + class MongoUpgrade1012: """Upgrade MongoDB Database from OSM v10 to v12.""" @@ -219,6 +485,7 @@ class MongoPatch1837: MONGODB_UPGRADE_FUNCTIONS = { "9": {"10": [MongoUpgrade910.upgrade]}, "10": {"12": [MongoUpgrade1012.upgrade]}, + "12": {"14": [MongoUpgrade1214.upgrade]}, } MYSQL_UPGRADE_FUNCTIONS = {} BUG_FIXES = { diff --git a/installers/full_install_osm.sh b/installers/full_install_osm.sh index 03be9a62..a7f46a28 100755 --- a/installers/full_install_osm.sh +++ b/installers/full_install_osm.sh @@ -36,7 +36,8 @@ function usage(){ echo -e " -P use VCA/juju public key file" echo -e " -A use VCA/juju API proxy" echo -e " --pla: install the PLA module for placement support" - echo -e " --ng-sa: install Airflow and Pushgateway to get VNF and NS status (experimental)" + echo -e " --old-sa: install old Service Assurance framework (MON, POL); do not install Airflow and Pushgateway" + echo -e " --ng-sa: install new Service Assurance framework (Airflow, AlertManager and Pushgateway) (enabled by default)" echo -e " -m : install OSM but only rebuild or pull the specified docker images (NG-UI, NBI, LCM, RO, MON, POL, PLA, KAFKA, MONGO, PROMETHEUS, PROMETHEUS-CADVISOR, KEYSTONE-DB, NONE)" echo -e " -o : ONLY (un)installs one of the addons (k8s_monitor, ng-sa)" echo -e " -O : Install OSM to an OpenStack infrastructure. is required. If a is used, the clouds.yaml file should be under ~/.config/openstack/ or /etc/openstack/" @@ -215,13 +216,12 @@ function docker_login() { [ -z "${DEBUG_INSTALL}" ] || DEBUG end of function } -function generate_docker_images() { +function pull_docker_images() { [ -z "${DEBUG_INSTALL}" ] || DEBUG beginning of function - echo "Pulling and generating docker images" - [ -n "${DOCKER_REGISTRY_URL}" ] && docker_login - echo "Pulling docker images" + [ -n "${DOCKER_REGISTRY_URL}" ] && docker_login + echo "Pulling non-OSM docker images" if [ -z "$TO_REBUILD" ] || echo $TO_REBUILD | grep -q KAFKA ; then sg docker -c "docker pull wurstmeister/zookeeper" || FATAL "cannot get zookeeper docker image" sg docker -c "docker pull wurstmeister/kafka:${KAFKA_TAG}" || FATAL "cannot get kafka docker image" @@ -248,48 +248,57 @@ function generate_docker_images() { sg docker -c "docker pull mysql:5" || FATAL "cannot get mysql docker image" fi - if [ -n "$PULL_IMAGES" ]; then - echo "Pulling OSM docker images" - for module in MON POL NBI KEYSTONE RO LCM NG-UI PLA osmclient; do + list_of_modules="MON NBI KEYSTONE RO LCM NG-UI osmclient" + if [ -n "$INSTALL_NGSA" ]; then + list_of_modules="${list_of_modules} Airflow Webhook" + else + list_of_modules="${list_of_modules} POL" + fi + if [ -n "$INSTALL_PLA" ]; then + list_of_modules="${list_of_modules} PLA" + fi + echo "Pulling OSM docker images for the following modules: ${list_of_modules}" + for module in ${list_of_modules}; do + module_lower=${module,,} + module_tag="${OSM_DOCKER_TAG}" + if [ -n "${MODULE_DOCKER_TAG}" ] && echo $TO_REBUILD | grep -q $module ; then + module_tag="${MODULE_DOCKER_TAG}" + fi + echo "Pulling ${DOCKER_REGISTRY_URL}${DOCKER_USER}/${module_lower}:${module_tag} docker image" + sg docker -c "docker pull ${DOCKER_REGISTRY_URL}${DOCKER_USER}/${module_lower}:${module_tag}" || FATAL "cannot pull $module docker image" + done + + echo "Finished pulling docker images" + [ -z "${DEBUG_INSTALL}" ] || DEBUG end of function +} + +function generate_docker_images() { + [ -z "${DEBUG_INSTALL}" ] || DEBUG beginning of function + echo "Generating docker images" + _build_from=$COMMIT_ID + [ -z "$_build_from" ] && _build_from="latest" + echo "OSM Docker images generated from $_build_from" + LWTEMPDIR="$(mktemp -d -q --tmpdir "installosmlight.XXXXXX")" + trap 'rm -rf "${LWTEMPDIR}"' EXIT + for module in MON POL NBI KEYSTONE RO LCM NG-UI PLA; do + if [ -z "$TO_REBUILD" ] || echo $TO_REBUILD | grep -q ${module} ; then module_lower=${module,,} if [ $module == "PLA" -a ! -n "$INSTALL_PLA" ]; then continue fi - module_tag="${OSM_DOCKER_TAG}" - if [ -n "${MODULE_DOCKER_TAG}" ] && echo $TO_REBUILD | grep -q $module ; then - module_tag="${MODULE_DOCKER_TAG}" - fi - echo "Pulling ${DOCKER_REGISTRY_URL}${DOCKER_USER}/${module_lower}:${module_tag} docker image" - sg docker -c "docker pull ${DOCKER_REGISTRY_URL}${DOCKER_USER}/${module_lower}:${module_tag}" || FATAL "cannot pull $module docker image" - done - else - _build_from=$COMMIT_ID - [ -z "$_build_from" ] && _build_from="latest" - echo "OSM Docker images generated from $_build_from" - LWTEMPDIR="$(mktemp -d -q --tmpdir "installosmlight.XXXXXX")" - trap 'rm -rf "${LWTEMPDIR}"' EXIT - for module in MON POL NBI KEYSTONE RO LCM NG-UI PLA; do - if [ -z "$TO_REBUILD" ] || echo $TO_REBUILD | grep -q ${module} ; then - module_lower=${module,,} - if [ $module == "PLA" -a ! -n "$INSTALL_PLA" ]; then - continue - fi - git -C ${LWTEMPDIR} clone https://osm.etsi.org/gerrit/osm/$module - git -C ${LWTEMPDIR}/${module} checkout ${COMMIT_ID} - sg docker -c "docker build ${LWTEMPDIR}/${module} -f ${LWTEMPDIR}/${module}/docker/Dockerfile -t ${DOCKER_USER}/${module_lower} --no-cache" || FATAL "cannot build ${module} docker image" - fi - done - if [ -z "$TO_REBUILD" ] || echo $TO_REBUILD | grep -q osmclient; then - BUILD_ARGS+=(--build-arg REPOSITORY="$REPOSITORY") - BUILD_ARGS+=(--build-arg RELEASE="$RELEASE") - BUILD_ARGS+=(--build-arg REPOSITORY_KEY="$REPOSITORY_KEY") - BUILD_ARGS+=(--build-arg REPOSITORY_BASE="$REPOSITORY_BASE") - sg docker -c "docker build -t ${DOCKER_USER}/osmclient ${BUILD_ARGS[@]} -f $OSM_DEVOPS/docker/osmclient ." + git -C ${LWTEMPDIR} clone https://osm.etsi.org/gerrit/osm/$module + git -C ${LWTEMPDIR}/${module} checkout ${COMMIT_ID} + sg docker -c "docker build ${LWTEMPDIR}/${module} -f ${LWTEMPDIR}/${module}/docker/Dockerfile -t ${DOCKER_USER}/${module_lower} --no-cache" || FATAL "cannot build ${module} docker image" fi - echo "Finished generation of docker images" + done + if [ -z "$TO_REBUILD" ] || echo $TO_REBUILD | grep -q osmclient; then + BUILD_ARGS+=(--build-arg REPOSITORY="$REPOSITORY") + BUILD_ARGS+=(--build-arg RELEASE="$RELEASE") + BUILD_ARGS+=(--build-arg REPOSITORY_KEY="$REPOSITORY_KEY") + BUILD_ARGS+=(--build-arg REPOSITORY_BASE="$REPOSITORY_BASE") + sg docker -c "docker build -t ${DOCKER_USER}/osmclient ${BUILD_ARGS[@]} -f $OSM_DEVOPS/docker/osmclient ." fi - - echo "Finished pulling and generating docker images" + echo "Finished generation of docker images" [ -z "${DEBUG_INSTALL}" ] || DEBUG end of function } @@ -498,10 +507,11 @@ function kube_secrets(){ kubectl create secret generic ro-db-secret -n $OSM_NAMESPACE --from-env-file=$OSM_DOCKER_WORK_DIR/ro-db.env kubectl create secret generic ro-secret -n $OSM_NAMESPACE --from-env-file=$OSM_DOCKER_WORK_DIR/ro.env kubectl create secret generic keystone-secret -n $OSM_NAMESPACE --from-env-file=$OSM_DOCKER_WORK_DIR/keystone.env - kubectl create secret generic pol-secret -n $OSM_NAMESPACE --from-env-file=$OSM_DOCKER_WORK_DIR/pol.env if [ -n "${INSTALL_NGSA}" ]; then kubectl create secret generic ngsa-secret -n $OSM_NAMESPACE --from-env-file=$OSM_DOCKER_WORK_DIR/ngsa.env kubectl create secret generic webhook-translator-secret -n $OSM_NAMESPACE --from-env-file=$OSM_DOCKER_WORK_DIR/webhook-translator.env + else + kubectl create secret generic pol-secret -n $OSM_NAMESPACE --from-env-file=$OSM_DOCKER_WORK_DIR/pol.env fi [ -z "${DEBUG_INSTALL}" ] || DEBUG end of function } @@ -509,6 +519,8 @@ function kube_secrets(){ #deploys osm pods and services function deploy_osm_services() { [ -z "${DEBUG_INSTALL}" ] || DEBUG beginning of function + echo "The following manifests under $OSM_K8S_WORK_DIR will be deployed:" + ls $OSM_K8S_WORK_DIR kubectl apply -n $OSM_NAMESPACE -f $OSM_K8S_WORK_DIR [ -z "${DEBUG_INSTALL}" ] || DEBUG end of function } @@ -530,7 +542,9 @@ function deploy_osm_pla_service() { function install_osm_ngsa_service() { [ -z "${DEBUG_INSTALL}" ] || DEBUG beginning of function - $OSM_DEVOPS/installers/install_ngsa.sh -d ${OSM_HELM_WORK_DIR} -D ${OSM_DEVOPS} -t ${OSM_DOCKER_TAG} ${DEBUG_INSTALL} || \ + NGSA_OPTS="-d ${OSM_HELM_WORK_DIR} -D ${OSM_DEVOPS} -t ${OSM_DOCKER_TAG} -U ${DOCKER_USER} ${DEBUG_INSTALL}" + [ -n "${DOCKER_REGISTRY_URL}" ] && NGSA_OPTS="${NGSA_OPTS} -r ${DOCKER_REGISTRY_URL}" + $OSM_DEVOPS/installers/install_ngsa.sh ${NGSA_OPTS} || \ FATAL_TRACK install_osm_ngsa_service "install_ngsa.sh failed" [ -z "${DEBUG_INSTALL}" ] || DEBUG end of function } @@ -543,7 +557,7 @@ function parse_yaml() { for module in $services; do if [ "$module" == "pla" ]; then if [ -n "$INSTALL_PLA" ]; then - echo "Updating K8s manifest file from opensourcemano\/pla:.* to ${DOCKER_REGISTRY_URL}${DOCKER_USER}\/pla:${TAG}" + echo "Updating K8s manifest file for ${module} from opensourcemano\/pla:.* to ${DOCKER_REGISTRY_URL}${DOCKER_USER}\/pla:${TAG}" sudo sed -i "s#opensourcemano/pla:.*#${DOCKER_REGISTRY_URL}${DOCKER_USER}/pla:${TAG}#g" ${OSM_DOCKER_WORK_DIR}/osm_pla/pla.yaml fi else @@ -553,7 +567,7 @@ function parse_yaml() { elif [ "$module" == "ng-mon" ]; then image="mon" fi - echo "Updating K8s manifest file from opensourcemano\/${image}:.* to ${DOCKER_REGISTRY_URL}${DOCKER_USER}\/${image}:${TAG}" + echo "Updating K8s manifest file for ${module} from opensourcemano\/${image}:.* to ${DOCKER_REGISTRY_URL}${DOCKER_USER}\/${image}:${TAG}" sudo sed -i "s#opensourcemano/${image}:.*#${DOCKER_REGISTRY_URL}${DOCKER_USER}/${image}:${TAG}#g" ${OSM_K8S_WORK_DIR}/${module}.yaml fi done @@ -561,6 +575,7 @@ function parse_yaml() { } function update_manifest_files() { + [ -z "${DEBUG_INSTALL}" ] || DEBUG beginning of function osm_services="nbi lcm ro pol mon ng-mon ng-ui keystone pla prometheus ng-prometheus" list_of_services="" for module in $osm_services; do @@ -790,7 +805,8 @@ function install_osm() { fi # Deploy OSM services - [ -z "$DOCKER_NOBUILD" ] && generate_docker_images + [ -z "$DOCKER_NOBUILD" ] && pull_docker_images + [ -z "$DOCKER_NOBUILD" ] && [ -z "$PULL_IMAGES" ] && generate_docker_images track docker_images docker_images_ok generate_k8s_manifest_files @@ -999,7 +1015,7 @@ DEBUG_INSTALL="" RELEASE="ReleaseTEN" REPOSITORY="stable" INSTALL_K8S_MONITOR="" -INSTALL_NGSA="" +INSTALL_NGSA="y" INSTALL_PLA="" INSTALL_VIMEMU="" LXD_REPOSITORY_BASE="https://osm-download.etsi.org/repository/osm/lxd" @@ -1199,6 +1215,7 @@ while getopts ":a:b:r:n:k:u:R:D:o:O:m:N:H:S:s:t:U:P:A:l:L:K:d:p:T:f:F:-: hy" o; [ "${OPTARG}" == "tag" ] && continue [ "${OPTARG}" == "registry" ] && continue [ "${OPTARG}" == "pla" ] && INSTALL_PLA="y" && continue + [ "${OPTARG}" == "old-sa" ] && INSTALL_NGSA="" && continue [ "${OPTARG}" == "ng-sa" ] && INSTALL_NGSA="y" && continue [ "${OPTARG}" == "volume" ] && OPENSTACK_ATTACH_VOLUME="true" && continue [ "${OPTARG}" == "nocachelxdimages" ] && continue diff --git a/installers/helm/values/airflow-values.yaml b/installers/helm/values/airflow-values.yaml index 9bde1214..79473aad 100644 --- a/installers/helm/values/airflow-values.yaml +++ b/installers/helm/values/airflow-values.yaml @@ -22,10 +22,13 @@ webserver: - name: airflow-ui port: "{{ .Values.ports.airflowUI }}" targetPort: "{{ .Values.ports.airflowUI }}" +extraEnv: | + - name: AIRFLOW__API__AUTH_BACKENDS + value: 'airflow.api.auth.backend.basic_auth' extraEnvFrom: | - secretRef: name: ngsa-secret config: core: - dags_folder: "/home/airflow/.local/lib/python3.8/site-packages/osm_ngsa" + dags_folder: "/home/airflow/.local/lib/python3.10/site-packages/osm_ngsa" diff --git a/installers/helm/values/alertmanager-values.yaml b/installers/helm/values/alertmanager-values.yaml index 2e438cb2..c140c505 100644 --- a/installers/helm/values/alertmanager-values.yaml +++ b/installers/helm/values/alertmanager-values.yaml @@ -34,14 +34,16 @@ config: group_wait: 10s group_interval: 2m receiver: default-receiver - repeat_interval: 3h routes: - receiver: vdu-webhook + repeat_interval: 15m matchers: - alertname = "vdu_down" - receiver: 'scaleout-webhook' + repeat_interval: 5m matchers: - alertname =~ "^scaleout_.*" - receiver: 'scalein-webhook' + repeat_interval: 5m matchers: - alertname =~ "^scalein_.*" diff --git a/installers/install_ngsa.sh b/installers/install_ngsa.sh index b90c3dc0..30edc701 100755 --- a/installers/install_ngsa.sh +++ b/installers/install_ngsa.sh @@ -16,7 +16,7 @@ set +eux # Helm chart 1.6.0 correspondes to Airflow 2.3.0 -AIRFLOW_HELM_VERSION=1.6.0 +AIRFLOW_HELM_VERSION=1.9.0 PROMPUSHGW_HELM_VERSION=1.18.2 ALERTMANAGER_HELM_VERSION=0.22.0 @@ -29,6 +29,8 @@ function install_airflow() { # update airflow-values.yaml to use the right tag echo "Updating Helm values file helm/values/airflow-values.yaml to use defaultAirflowTag: ${OSM_DOCKER_TAG}" sudo sed -i "s#defaultAirflowTag:.*#defaultAirflowTag: ${OSM_DOCKER_TAG}#g" ${OSM_HELM_WORK_DIR}/airflow-values.yaml + echo "Updating Helm values file helm/values/airflow-values.yaml to use defaultAirflowRepository: ${DOCKER_REGISTRY_URL}${DOCKER_USER}/airflow" + sudo sed -i "s#defaultAirflowRepository:.*#defaultAirflowRepository: ${DOCKER_REGISTRY_URL}${DOCKER_USER}/airflow#g" ${OSM_HELM_WORK_DIR}/airflow-values.yaml if ! helm -n osm status airflow 2> /dev/null ; then # if it does not exist, create secrets and install kubectl -n osm create secret generic airflow-webserver-secret --from-literal="webserver-secret-key=$(python3 -c 'import secrets; print(secrets.token_hex(16))')" @@ -62,6 +64,9 @@ function install_prometheus_pushgateway() { # Install Prometheus AlertManager helm chart function install_prometheus_alertmanager() { [ -z "${DEBUG_INSTALL}" ] || DEBUG end of function + # copy alertmanager-values.yaml to the destination folder + sudo mkdir -p ${OSM_HELM_WORK_DIR} + sudo cp ${OSM_DEVOPS}/installers/helm/values/alertmanager-values.yaml ${OSM_HELM_WORK_DIR} if ! helm -n osm status alertmanager 2> /dev/null ; then # if it does not exist, install helm repo add prometheus-community https://prometheus-community.github.io/helm-charts @@ -77,11 +82,13 @@ function install_prometheus_alertmanager() { # main +DOCKER_REGISTRY_URL= +DOCKER_USER="opensourcemano" OSM_DEVOPS="/usr/share/osm-devops" -OSM_HELM_WORK_DIR="/etc/osm/helm" OSM_DOCKER_TAG="13" +OSM_HELM_WORK_DIR="/etc/osm/helm" -while getopts ":D:d:t:-: " o; do +while getopts ":D:d:t:r:U:-: " o; do case "${o}" in D) OSM_DEVOPS="${OPTARG}" @@ -92,6 +99,12 @@ while getopts ":D:d:t:-: " o; do t) OSM_DOCKER_TAG="${OPTARG}" ;; + r) + DOCKER_REGISTRY_URL="${OPTARG}" + ;; + U) + DOCKER_USER="${OPTARG}" + ;; -) [ "${OPTARG}" == "debug" ] && DEBUG_INSTALL="y" && continue echo -e "Invalid option: '--$OPTARG'\n" >&2 diff --git a/jenkins/ci-pipelines/ci_stage_3.groovy b/jenkins/ci-pipelines/ci_stage_3.groovy index e0cddeaf..29e438a2 100644 --- a/jenkins/ci-pipelines/ci_stage_3.groovy +++ b/jenkins/ci-pipelines/ci_stage_3.groovy @@ -124,13 +124,13 @@ void run_robot_systest(String tagName, void archive_logs(Map remote) { - sshCommand remote: remote, command: '''mkdir -p logs''' + sshCommand remote: remote, command: '''mkdir -p logs/dags''' if (useCharmedInstaller) { sshCommand remote: remote, command: ''' - for container in `kubectl get pods -n osm | grep -v operator | grep -v NAME| awk '{print $1}'`; do - logfile=`echo $container | cut -d- -f1` + for pod in `kubectl get pods -n osm | grep -v operator | grep -v NAME| awk '{print $1}'`; do + logfile=`echo $pod | cut -d- -f1` echo "Extracting log for $logfile" - kubectl logs -n osm $container --timestamps=true 2>&1 > logs/$logfile.log + kubectl logs -n osm $pod --timestamps=true 2>&1 > logs/$logfile.log done ''' } else { @@ -148,6 +148,11 @@ void archive_logs(Map remote) { > logs/$statefulset.log done ''' + sshCommand remote: remote, command: ''' + schedulerPod="$(kubectl get pods -n osm | grep airflow-scheduler| awk '{print $1; exit}')"; \ + echo "Extracting logs from Airflow DAGs from pod ${schedulerPod}"; \ + kubectl cp -n osm ${schedulerPod}:/opt/airflow/logs/scheduler/latest/dags logs/dags -c scheduler + ''' } sh 'rm -rf logs' @@ -456,8 +461,8 @@ node("${params.NODE}") { // Ensure the VM is ready sshCommand remote: remote, command: 'cloud-init status --wait' // Force time sync to avoid clock drift and invalid certificates - sshCommand remote: remote, command: 'sudo apt-get update' - sshCommand remote: remote, command: 'sudo apt-get install -y chrony' + sshCommand remote: remote, command: 'sudo apt-get -y update' + sshCommand remote: remote, command: 'sudo apt-get -y install chrony' sshCommand remote: remote, command: 'sudo service chrony stop' sshCommand remote: remote, command: 'sudo chronyd -vq' sshCommand remote: remote, command: 'sudo service chrony start' @@ -748,7 +753,10 @@ EOF""" } // if (params.SAVE_ARTIFACTS_OVERRIDE || stage_archive) } // dir(OSM_DEVOPS) } finally { - stage('Archive Container Logs') { + // stage('Debug') { + // sleep 900 + // } + stage('Archive Container Logs') { if ( ARCHIVE_LOGS_FLAG ) { try { // Archive logs diff --git a/tools/local-build.sh b/tools/local-build.sh index 4ff12f2b..cbaa87ad 100755 --- a/tools/local-build.sh +++ b/tools/local-build.sh @@ -191,6 +191,14 @@ function run_httpserver() { nohup python3 -m http.server ${HTTPPORT} --directory "${HTTPDDIR}" &>/dev/null & } +function run_httpserver() { + EXISTING_PID=$(ps auxw | grep "http.server $HTTPPORT" | grep -v grep | awk '{print $2}') + if [ ! -z $EXISTING_PID ] ; then + kill $EXISTING_PID + fi + nohup python3 -m http.server ${HTTPPORT} --directory "${HTTPDDIR}" &>/dev/null & +} + function stage_2() { print_section "Performing Stage 2" MODULES="common devops IM LCM MON N2VC NBI NG-UI NG-SA osmclient PLA POL RO tests" -- 2.25.1