From 44cfa9f40abcbcce54af8d3b01a804244482df19 Mon Sep 17 00:00:00 2001 From: beierlm Date: Thu, 17 Feb 2022 21:36:31 +0000 Subject: [PATCH 01/15] LTS Upgrade Documentation Adds documentation on how to upgrade from v9 or v10 to 10.1.0 LTS --- 18-tacacs-based-authentication.md | 2 +- 19-lts-upgrade.md | 174 ++++++++++++++++++++++++++++++ 2 files changed, 175 insertions(+), 1 deletion(-) create mode 100644 19-lts-upgrade.md diff --git a/18-tacacs-based-authentication.md b/18-tacacs-based-authentication.md index 049fe89..fda1a31 100644 --- a/18-tacacs-based-authentication.md +++ b/18-tacacs-based-authentication.md @@ -36,4 +36,4 @@ OSM NB has a plugin which can interact with TACACS server to authenticate users. - If user profile is removed from TACACS server, OSM still maintains the profile and will not be removed until next login. As the profile is not available in TACACS, the authentication shall fail and user profile will be removed from OSM. - \ No newline at end of file + diff --git a/19-lts-upgrade.md b/19-lts-upgrade.md new file mode 100644 index 0000000..923a3e4 --- /dev/null +++ b/19-lts-upgrade.md @@ -0,0 +1,174 @@ +# 19 LTS Upgrade + +## Introduction + +Starting with version 10.1.0 of OSM, every even numbered release will receive two years of community support. This document covers the steps needed for upgrading OSM. Depending on the installation method, there are two methods for upgrading OSM to an LTS version. + +## Upgrade of 10.0.3 to 10.1.0 LTS + +### Community Installation Option + +### Charmed Installation Option + +#### Install pre-LTS Version + +For the purpose of testing, we can start with 10.0.3 of OSM. +```bash +wget https://osm-download.etsi.org/ftp/osm-10.0-ten/install_osm.sh +chmod +x ./install_osm.sh +./install_osm.sh --charmed --tag 10.0.3 +``` + +#### Stop all OSM Services + +```bash +juju scale-application grafana 0 +juju scale-application prometheus 0 +juju scale-application kafka 0 +juju scale-application keystone 0 +juju scale-application lcm 0 +juju scale-application mon 0 +juju scale-application nbi 0 +juju scale-application ng-ui 0 +juju scale-application pla 0 +juju scale-application pol 0 +juju scale-application ro 0 +juju scale-application zookeeper 0 +``` + +#### Backup the Databases + +```bash +mariadb_unit=$(juju status | grep -i mariadb | tail -1 | awk -F" " '{print $1}' | tr -d '[*]') +maridb_pod=$(microk8s.kubectl get pod -n osm | grep -i mariadb | tail -1 | awk -F" " '{print $1}') +juju run-action $mariadb_unit backup --wait -m osm +microk8s.kubectl cp osm/$maridb_pod:/var/lib/mysql/backup.sql.gz backup.sql.gz + + +mongodb_unit=$(juju status | grep -i mongodb | tail -1 | awk -F" " '{print $1}'| tr -d '[*]') +mongodb_pod=$(microk8s.kubectl get pod -n osm | grep -i mongodb | tail -1 | awk -F" " '{print $1}') +juju run-action $mongodb_unit backup --wait -m osm +microk8s.kubectl cp osm/$mongodb_pod:/data/backup.archive backup.archive +``` + +#### Remove Deployed OSM Application + +```bash +juju destroy-model osm --destroy-storage -y --force --no-wait +``` + +#### Upgrade Juju + +```bash +sudo snap switch --channel 2.9/stable juju +sudo snap refresh +juju switch osm-vca:admin/controller +juju upgrade-model +``` + +#### Upgrade MicroK8s + +```bash +sudo snap refresh microk8s --channel=1.23/stable +``` + +#### Install OSM 10.1.0 LTS + +```bash +wget https://osm-download.etsi.org/repository/osm/debian/ReleaseTEN-daily/pool/devops/osm-devops_10.0.3.post24-1_all.deb +sudo apt remove --purge osm-devops +sudo apt install ./osm-devops_10.0.3.post24-1_all.deb +/usr/share/osm-devops/installers/charmed_install.sh --vca osm-vca --tag releaseten-daily #10.1.0 +``` + +#### Stop all OSM Services + +```bash +juju scale-application grafana 0 +juju scale-application prometheus 0 +juju scale-application kafka-k8s 0 +juju scale-application keystone 0 +juju scale-application lcm 0 +juju scale-application mon 0 +juju scale-application nbi 0 +juju scale-application ng-ui 0 +juju scale-application pla 0 +juju scale-application pol 0 +juju scale-application ro 0 +juju scale-application zookeeper-k8s 0 +``` + +#### Restore the Databases + +```bash +mariadb_unit=$(juju status | grep -i mariadb | tail -1 | awk -F" " '{print $1}' | tr -d '[*]') +mariadb_pod=$(microk8s.kubectl get pod -n osm | grep -i mariadb | tail -1 | awk -F" " '{print $1}') +microk8s.kubectl cp backup.sql.gz osm/$mariadb_pod:/var/lib/mysql/backup.sql.gz +juju run-action $mariadb_unit backup --wait -m osm + +mongodb_pod=$(microk8s.kubectl get pod -n osm | grep -i mongodb | tail -1 | awk -F" " '{print $1}') +microk8s.kubectl cp backup.archive osm/$mongodb_pod:/data/backup.archive +microk8s.kubectl exec -n osm -it $mongodb_pod -- mongorestore --gzip --archive=/data/backup.archive +``` + +#### Perform Database Migration + +##### Keystone Database Updates + +Start the Keystone container. +``` +juju scale-application keystone 1 +``` + +1. Keystone URL Endpoint update +```bash +mariadb_unit=$(juju status | grep -i mariadb | tail -1 | awk -F" " '{print $1}' | tr -d '[*]') +juju run --unit $mariadb_unit -- mysql -posm4u -Dkeystone \ + --execute 'UPDATE endpoint SET url="http://osm-keystone:5000/v3/" WHERE url="http://keystone:5000/v3/";' +``` + +2. DB Sync command from Keystone to update schema to installed version + +```bash +keystone_pod=$(microk8s.kubectl get pod -n osm | grep -i keystone | tail -1 | awk -F" " '{print $1}') +juju run --unit $keystone_unit -- keystone-manage db_sync --expand +``` + +#### OSM Application Database Updates + +A helper charm has been created to assist in the database updates. Build, deploy and run the action as follows: + +```bash +sudo snap install charmcraft --classic + +git clone https://github.com/charmed-osm/osm-update-db-operator.git +cd osm-update-db-operator +juju switch osm +juju deploy ./osm-update-db_ubuntu-20.04-amd64.charm +``` +```bash +juju switch osm +mongodb_app=$(juju status | grep -i mongodb | head -1 | awk -F" " '{print $1}') +mongodb_ip=$(juju status | grep -i $mongodb_app/.*\\* | head -1 | awk -F " " '{print $4}') + + +mongodb_uri=mongodb://mongodb:27017 + +juju switch update-db +juju config osm-update-db mongodb-uri=$mongodb_uri + +juju run-action --wait osm-update-db/0 update-db \ + current-version=9 \ + target-version=10 \ + mongodb-only=True + +juju run-action --wait osm-update-db/0 apply-patch \ + bug_number=1837 +``` + +The charm can now be removed. + +```bash +juju remove-application osm-update-db +juju destroy-model update-db +``` -- GitLab From 6fea1b2616ddce6f2abe1ffcea068678e9e4285e Mon Sep 17 00:00:00 2001 From: beierlm Date: Fri, 18 Feb 2022 01:55:01 +0000 Subject: [PATCH 02/15] Update 19-lts-upgrade.md --- 19-lts-upgrade.md | 39 ++++++++++++++++++++++++++++----------- 1 file changed, 28 insertions(+), 11 deletions(-) diff --git a/19-lts-upgrade.md b/19-lts-upgrade.md index 923a3e4..98722d0 100644 --- a/19-lts-upgrade.md +++ b/19-lts-upgrade.md @@ -141,34 +141,51 @@ A helper charm has been created to assist in the database updates. Build, deplo ```bash sudo snap install charmcraft --classic +charmcraft build git clone https://github.com/charmed-osm/osm-update-db-operator.git cd osm-update-db-operator + juju switch osm juju deploy ./osm-update-db_ubuntu-20.04-amd64.charm ``` + +Run the upgrade as follows. + ```bash -juju switch osm -mongodb_app=$(juju status | grep -i mongodb | head -1 | awk -F" " '{print $1}') -mongodb_ip=$(juju status | grep -i $mongodb_app/.*\\* | head -1 | awk -F " " '{print $4}') +juju config osm-update-db mongodb-uri=mongodb://mongodb:27017 +juju run-action --wait osm-update-db/0 apply-patch bug-number=1837 +``` -mongodb_uri=mongodb://mongodb:27017 +##### Upgrading From v9.0 Versions -juju switch update-db -juju config osm-update-db mongodb-uri=$mongodb_uri +If the ugrade is from v9.0 to 10.1.0 LTS, the following must also be run to update the database. +```bash juju run-action --wait osm-update-db/0 update-db \ current-version=9 \ target-version=10 \ - mongodb-only=True - -juju run-action --wait osm-update-db/0 apply-patch \ - bug_number=1837 + mongodb-only=True ``` The charm can now be removed. ```bash juju remove-application osm-update-db -juju destroy-model update-db +``` + +#### Restart all OSM Services + +```bash +juju scale-application grafana 1 +juju scale-application prometheus 1 +juju scale-application kafka-k8s 1 +juju scale-application lcm 1 +juju scale-application mon 1 +juju scale-application nbi 1 +juju scale-application ng-ui 1 +juju scale-application pla 1 +juju scale-application pol 1 +juju scale-application ro 1 +juju scale-application zookeeper-k8s 1 ``` -- GitLab From e34398847a4eff04e2c640d76e5baa3e4be8e141 Mon Sep 17 00:00:00 2001 From: beierlm Date: Fri, 18 Feb 2022 15:57:26 +0000 Subject: [PATCH 03/15] Update 19-lts-upgrade.md --- 19-lts-upgrade.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/19-lts-upgrade.md b/19-lts-upgrade.md index 98722d0..f26b2f7 100644 --- a/19-lts-upgrade.md +++ b/19-lts-upgrade.md @@ -6,7 +6,7 @@ Starting with version 10.1.0 of OSM, every even numbered release will receive tw ## Upgrade of 10.0.3 to 10.1.0 LTS -### Community Installation Option +### Kubernetes Installation Option ### Charmed Installation Option -- GitLab From fc30d322559dc010d4be3a5f9ca9261255a53176 Mon Sep 17 00:00:00 2001 From: beierlm Date: Fri, 18 Feb 2022 17:58:34 +0000 Subject: [PATCH 04/15] Update 19-lts-upgrade.md --- 19-lts-upgrade.md | 78 ++++++++++++++++++++++++++++++++++++----------- 1 file changed, 60 insertions(+), 18 deletions(-) diff --git a/19-lts-upgrade.md b/19-lts-upgrade.md index f26b2f7..878efdd 100644 --- a/19-lts-upgrade.md +++ b/19-lts-upgrade.md @@ -4,22 +4,60 @@ Starting with version 10.1.0 of OSM, every even numbered release will receive two years of community support. This document covers the steps needed for upgrading OSM. Depending on the installation method, there are two methods for upgrading OSM to an LTS version. -## Upgrade of 10.0.3 to 10.1.0 LTS +## Upgrade of Pre-LTS to 10.1.0 LTS + +This procedure covers both upgrade from 9.1.5 or 10.0.3 to 10.1.0 LTS. Where necessary, additional steps for 9.1.5 are shown. There are two installation methods, each with its own set of procedures: + +* [Kubernetes Installation Option](#kubernetes-installation-option) +* [Charmed Installation Option](#charmed-nstallation-option) ### Kubernetes Installation Option -### Charmed Installation Option +The following steps are to be followed for an upgrade to LTS: + +* [Stop all OSM Services](#k8s-1) +* []() +* []() -#### Install pre-LTS Version +#### Stop all OSM Services -For the purpose of testing, we can start with 10.0.3 of OSM. ```bash -wget https://osm-download.etsi.org/ftp/osm-10.0-ten/install_osm.sh -chmod +x ./install_osm.sh -./install_osm.sh --charmed --tag 10.0.3 +kubectl -n osm scale deployment/grafana --replicas=0 +kubectl -n osm scale statefulset/prometheus --replicas=0 +kubectl -n osm scale statefulset/kafka --replicas=0 +kubectl -n osm scale deployment/keystone --replicas=0 +kubectl -n osm scale deployment/lcm --replicas=0 +kubectl -n osm scale deployment/mon --replicas=0 +kubectl -n osm scale deployment/nbi --replicas=0 +kubectl -n osm scale deployment/ng-ui --replicas=0 +kubectl -n osm scale deployment/pla --replicas=0 +kubectl -n osm scale deployment/pol --replicas=0 +kubectl -n osm scale deployment/ro --replicas=0 +kubectl -n osm scale statefulset/zookeeper --replicas=0 ``` -#### Stop all OSM Services +__Note__ if PLA was not installed, you can ignore the error about `deployments.apps "pla" not found`. + +#### Backup the Databases + + +### Charmed Installation Option + +The following steps are to be followed for an upgrade to LTS: + +* [Stop all OSM Services](#charm-1) +* [Backup the Databases](#charm-2) +* [Remove Deployed OSM Application](#charm-3) +* [Upgrade Juju](#charm-4) +* [Upgrade MicroK8s](#charm-5) +* [Install OSM 10.1.0 LTS](#charm-6) +* [Stop New OSM Services](#charm-7) +* [Restore the Databases](#charm-8) +* [Perform Database Migration](#charm-9) +* [Restart all OSM Services](#charm-10) + + +#### Stop all OSM Services ```bash juju scale-application grafana 0 @@ -36,7 +74,7 @@ juju scale-application ro 0 juju scale-application zookeeper 0 ``` -#### Backup the Databases +#### Backup the Databases ```bash mariadb_unit=$(juju status | grep -i mariadb | tail -1 | awk -F" " '{print $1}' | tr -d '[*]') @@ -51,13 +89,13 @@ juju run-action $mongodb_unit backup --wait -m osm microk8s.kubectl cp osm/$mongodb_pod:/data/backup.archive backup.archive ``` -#### Remove Deployed OSM Application +#### Remove Deployed OSM Application ```bash juju destroy-model osm --destroy-storage -y --force --no-wait ``` -#### Upgrade Juju +#### Upgrade Juju ```bash sudo snap switch --channel 2.9/stable juju @@ -66,13 +104,13 @@ juju switch osm-vca:admin/controller juju upgrade-model ``` -#### Upgrade MicroK8s +#### Upgrade MicroK8s ```bash sudo snap refresh microk8s --channel=1.23/stable ``` -#### Install OSM 10.1.0 LTS +#### Install OSM 10.1.0 LTS ```bash wget https://osm-download.etsi.org/repository/osm/debian/ReleaseTEN-daily/pool/devops/osm-devops_10.0.3.post24-1_all.deb @@ -81,7 +119,9 @@ sudo apt install ./osm-devops_10.0.3.post24-1_all.deb /usr/share/osm-devops/installers/charmed_install.sh --vca osm-vca --tag releaseten-daily #10.1.0 ``` -#### Stop all OSM Services +#### Stop all OSM Services + +As we need to restore the database, we are going to stop all the OSM services once again. ```bash juju scale-application grafana 0 @@ -98,7 +138,7 @@ juju scale-application ro 0 juju scale-application zookeeper-k8s 0 ``` -#### Restore the Databases +#### Restore the Databases ```bash mariadb_unit=$(juju status | grep -i mariadb | tail -1 | awk -F" " '{print $1}' | tr -d '[*]') @@ -111,7 +151,7 @@ microk8s.kubectl cp backup.archive osm/$mongodb_pod:/data/backup.archive microk8s.kubectl exec -n osm -it $mongodb_pod -- mongorestore --gzip --archive=/data/backup.archive ``` -#### Perform Database Migration +#### Perform Database Migration ##### Keystone Database Updates @@ -134,7 +174,7 @@ keystone_pod=$(microk8s.kubectl get pod -n osm | grep -i keystone | tail -1 | aw juju run --unit $keystone_unit -- keystone-manage db_sync --expand ``` -#### OSM Application Database Updates +##### OSM Application Database Updates A helper charm has been created to assist in the database updates. Build, deploy and run the action as follows: @@ -174,7 +214,9 @@ The charm can now be removed. juju remove-application osm-update-db ``` -#### Restart all OSM Services +#### Restart all OSM Services + +Now that the databases are migrated to the new version, we can restart the services. ```bash juju scale-application grafana 1 -- GitLab From 6707139e59b3eb5b8a01109f35953909472e130a Mon Sep 17 00:00:00 2001 From: beierlm Date: Tue, 22 Feb 2022 15:11:18 +0000 Subject: [PATCH 05/15] Update 19-lts-upgrade.md --- 19-lts-upgrade.md | 94 ++++++++++++++++++++++++++++++++++++++--------- 1 file changed, 76 insertions(+), 18 deletions(-) diff --git a/19-lts-upgrade.md b/19-lts-upgrade.md index 878efdd..6cd5d3f 100644 --- a/19-lts-upgrade.md +++ b/19-lts-upgrade.md @@ -16,8 +16,14 @@ This procedure covers both upgrade from 9.1.5 or 10.0.3 to 10.1.0 LTS. Where ne The following steps are to be followed for an upgrade to LTS: * [Stop all OSM Services](#k8s-1) -* []() -* []() +* [Backup the Databases](#k8s-2) +* [](#k8s-4) +* [](#k8s-5) +* [](#k8s-6) +* [](#k8s-7) +* [](#k8s-8) +* [](#k8s-9) +* [](#k8s-10) #### Stop all OSM Services @@ -40,6 +46,40 @@ __Note__ if PLA was not installed, you can ignore the error about `deployments.a #### Backup the Databases +```bash +mysql_pod=$(kubectl get pod -n osm | grep -i mysql | tail -1 | awk -F" " '{print $1}') +kubectl exec -n osm -it $mysql_pod -- bash -c \ + 'mysqldump -u root -p$MYSQL_ROOT_PASSWORD --single-transaction --all-databases' \ + | gzip > backup.sql.gz + +mongodb_unit=$(juju status | grep -i mongodb | tail -1 | awk -F" " '{print $1}'| tr -d '[*]') +mongodb_pod=$(kubectl get pod -n osm | grep -i mongodb | grep -v operator | tail -1 | awk -F" " '{print $1}') +juju run-action $mongodb_unit backup --wait -m osm +kubectl cp osm/$mongodb_pod:/data/backup.archive backup.archive +``` + +#### Remove Deployed OSM Application + +```bash +/usr/share/osm-devops/installers/install_osm.sh --uninstall +``` + +#### Install OSM 10.1.0 LTS + +```bash +wget https://osm-download.etsi.org/repository/osm/debian/ReleaseTEN-daily/pool/devops/osm-devops_10.0.3.post24-1_all.deb +sudo apt remove --purge -y osm-devops +sudo apt install ./osm-devops_10.0.3.post24-1_all.deb +/usr/share/osm-devops/installers/install_osm.sh -t releaseten-daily #10.1.0 +``` + +__NOTE__ This does not work - need help getting a procudure for upgrade here. + + + + + +--------------------------------------------------------------------------------------------------------------- ### Charmed Installation Option @@ -59,10 +99,12 @@ The following steps are to be followed for an upgrade to LTS: #### Stop all OSM Services +##### Version 10.0.3 + ```bash juju scale-application grafana 0 juju scale-application prometheus 0 -juju scale-application kafka 0 +juju scale-application kafka-k8s 0 juju scale-application keystone 0 juju scale-application lcm 0 juju scale-application mon 0 @@ -71,7 +113,24 @@ juju scale-application ng-ui 0 juju scale-application pla 0 juju scale-application pol 0 juju scale-application ro 0 -juju scale-application zookeeper 0 +juju scale-application zookeeper-k8s 0 +``` + +##### Version 9.1.5 + +```bash +juju scale-application grafana-k8s 0 +juju scale-application prometheus-k8s 0 +juju scale-application kafka-k8s 0 +juju scale-application keystone 0 +juju scale-application lcm-k8s 0 +juju scale-application mon-k8s 0 +juju scale-application nbi 0 +juju scale-application ng-ui 0 +juju scale-application pla 0 +juju scale-application pol-k8s 0 +juju scale-application ro-k8s 0 +juju scale-application zookeeper-k8s 0 ``` #### Backup the Databases @@ -82,7 +141,6 @@ maridb_pod=$(microk8s.kubectl get pod -n osm | grep -i mariadb | tail -1 | awk - juju run-action $mariadb_unit backup --wait -m osm microk8s.kubectl cp osm/$maridb_pod:/var/lib/mysql/backup.sql.gz backup.sql.gz - mongodb_unit=$(juju status | grep -i mongodb | tail -1 | awk -F" " '{print $1}'| tr -d '[*]') mongodb_pod=$(microk8s.kubectl get pod -n osm | grep -i mongodb | tail -1 | awk -F" " '{print $1}') juju run-action $mongodb_unit backup --wait -m osm @@ -95,7 +153,7 @@ microk8s.kubectl cp osm/$mongodb_pod:/data/backup.archive backup.archive juju destroy-model osm --destroy-storage -y --force --no-wait ``` -#### Upgrade Juju +#### Upgrade Juju ```bash sudo snap switch --channel 2.9/stable juju @@ -104,22 +162,22 @@ juju switch osm-vca:admin/controller juju upgrade-model ``` -#### Upgrade MicroK8s +#### Upgrade MicroK8s ```bash sudo snap refresh microk8s --channel=1.23/stable ``` -#### Install OSM 10.1.0 LTS +#### Install OSM 10.1.0 LTS ```bash wget https://osm-download.etsi.org/repository/osm/debian/ReleaseTEN-daily/pool/devops/osm-devops_10.0.3.post24-1_all.deb -sudo apt remove --purge osm-devops +sudo apt remove -y --purge osm-devops sudo apt install ./osm-devops_10.0.3.post24-1_all.deb /usr/share/osm-devops/installers/charmed_install.sh --vca osm-vca --tag releaseten-daily #10.1.0 ``` -#### Stop all OSM Services +#### Stop all OSM Services As we need to restore the database, we are going to stop all the OSM services once again. @@ -138,20 +196,20 @@ juju scale-application ro 0 juju scale-application zookeeper-k8s 0 ``` -#### Restore the Databases +#### Restore the Databases ```bash mariadb_unit=$(juju status | grep -i mariadb | tail -1 | awk -F" " '{print $1}' | tr -d '[*]') mariadb_pod=$(microk8s.kubectl get pod -n osm | grep -i mariadb | tail -1 | awk -F" " '{print $1}') microk8s.kubectl cp backup.sql.gz osm/$mariadb_pod:/var/lib/mysql/backup.sql.gz -juju run-action $mariadb_unit backup --wait -m osm +juju run-action $mariadb_unit restore --wait -m osm mongodb_pod=$(microk8s.kubectl get pod -n osm | grep -i mongodb | tail -1 | awk -F" " '{print $1}') microk8s.kubectl cp backup.archive osm/$mongodb_pod:/data/backup.archive microk8s.kubectl exec -n osm -it $mongodb_pod -- mongorestore --gzip --archive=/data/backup.archive ``` -#### Perform Database Migration +#### Perform Database Migration ##### Keystone Database Updates @@ -163,14 +221,14 @@ juju scale-application keystone 1 1. Keystone URL Endpoint update ```bash mariadb_unit=$(juju status | grep -i mariadb | tail -1 | awk -F" " '{print $1}' | tr -d '[*]') -juju run --unit $mariadb_unit -- mysql -posm4u -Dkeystone \ - --execute 'UPDATE endpoint SET url="http://osm-keystone:5000/v3/" WHERE url="http://keystone:5000/v3/";' +juju run --unit $mariadb_unit -- bash -c 'mysql -p${MARIADB_ROOT_PASSWORD} -Dkeystone \ + --execute "UPDATE endpoint SET url=\"http://osm-keystone:5000/v3/\" WHERE url=\"http://keystone:5000/v3/\"";' ``` 2. DB Sync command from Keystone to update schema to installed version ```bash -keystone_pod=$(microk8s.kubectl get pod -n osm | grep -i keystone | tail -1 | awk -F" " '{print $1}') +keystone_unit=$(juju status | grep -i keystone | tail -1 | awk -F" " '{print $1}' | tr -d '[*]') juju run --unit $keystone_unit -- keystone-manage db_sync --expand ``` @@ -181,9 +239,9 @@ A helper charm has been created to assist in the database updates. Build, deplo ```bash sudo snap install charmcraft --classic -charmcraft build git clone https://github.com/charmed-osm/osm-update-db-operator.git cd osm-update-db-operator +charmcraft build juju switch osm juju deploy ./osm-update-db_ubuntu-20.04-amd64.charm @@ -214,7 +272,7 @@ The charm can now be removed. juju remove-application osm-update-db ``` -#### Restart all OSM Services +#### Restart all OSM Services Now that the databases are migrated to the new version, we can restart the services. -- GitLab From 03d410346ffdc12fd43ea034924e00429ebe7373 Mon Sep 17 00:00:00 2001 From: beierlm Date: Thu, 24 Feb 2022 16:53:42 +0000 Subject: [PATCH 06/15] Update 19-lts-upgrade.md --- 19-lts-upgrade.md | 203 ++++++++++++++++++++++++++++++++++++++++------ 1 file changed, 180 insertions(+), 23 deletions(-) diff --git a/19-lts-upgrade.md b/19-lts-upgrade.md index 6cd5d3f..875819a 100644 --- a/19-lts-upgrade.md +++ b/19-lts-upgrade.md @@ -17,6 +17,7 @@ The following steps are to be followed for an upgrade to LTS: * [Stop all OSM Services](#k8s-1) * [Backup the Databases](#k8s-2) +* [](#k8s-3) * [](#k8s-4) * [](#k8s-5) * [](#k8s-6) @@ -64,15 +65,10 @@ kubectl cp osm/$mongodb_pod:/data/backup.archive backup.archive /usr/share/osm-devops/installers/install_osm.sh --uninstall ``` -#### Install OSM 10.1.0 LTS +#### Update OSM 10.1.0 LTS -```bash -wget https://osm-download.etsi.org/repository/osm/debian/ReleaseTEN-daily/pool/devops/osm-devops_10.0.3.post24-1_all.deb -sudo apt remove --purge -y osm-devops -sudo apt install ./osm-devops_10.0.3.post24-1_all.deb -/usr/share/osm-devops/installers/install_osm.sh -t releaseten-daily #10.1.0 ``` - +``` __NOTE__ This does not work - need help getting a procudure for upgrade here. @@ -99,37 +95,37 @@ The following steps are to be followed for an upgrade to LTS: #### Stop all OSM Services -##### Version 10.0.3 +##### Version 9.1.5 ```bash -juju scale-application grafana 0 -juju scale-application prometheus 0 +juju scale-application grafana-k8s 0 +juju scale-application prometheus-k8s 0 juju scale-application kafka-k8s 0 juju scale-application keystone 0 -juju scale-application lcm 0 -juju scale-application mon 0 +juju scale-application lcm-k8s 0 +juju scale-application mon-k8s 0 juju scale-application nbi 0 juju scale-application ng-ui 0 juju scale-application pla 0 -juju scale-application pol 0 -juju scale-application ro 0 +juju scale-application pol-k8s 0 +juju scale-application ro-k8s 0 juju scale-application zookeeper-k8s 0 ``` -##### Version 9.1.5 +##### Version 10.0.3 ```bash -juju scale-application grafana-k8s 0 -juju scale-application prometheus-k8s 0 +juju scale-application grafana 0 +juju scale-application prometheus 0 juju scale-application kafka-k8s 0 juju scale-application keystone 0 -juju scale-application lcm-k8s 0 -juju scale-application mon-k8s 0 +juju scale-application lcm 0 +juju scale-application mon 0 juju scale-application nbi 0 juju scale-application ng-ui 0 juju scale-application pla 0 -juju scale-application pol-k8s 0 -juju scale-application ro-k8s 0 +juju scale-application pol 0 +juju scale-application ro 0 juju scale-application zookeeper-k8s 0 ``` @@ -156,8 +152,7 @@ juju destroy-model osm --destroy-storage -y --force --no-wait #### Upgrade Juju ```bash -sudo snap switch --channel 2.9/stable juju -sudo snap refresh +sudo snap refresh juju --channel 2.9/stable juju switch osm-vca:admin/controller juju upgrade-model ``` @@ -174,6 +169,8 @@ sudo snap refresh microk8s --channel=1.23/stable wget https://osm-download.etsi.org/repository/osm/debian/ReleaseTEN-daily/pool/devops/osm-devops_10.0.3.post24-1_all.deb sudo apt remove -y --purge osm-devops sudo apt install ./osm-devops_10.0.3.post24-1_all.deb +unset OSM_USERNAME +unset OSM_PASSWORD /usr/share/osm-devops/installers/charmed_install.sh --vca osm-vca --tag releaseten-daily #10.1.0 ``` @@ -289,3 +286,163 @@ juju scale-application pol 1 juju scale-application ro 1 juju scale-application zookeeper-k8s 1 ``` + +# Testing Upgrade + +## Changing Credentials + +We will change some default passwords, and create some additional users to ensure RBAC still works. + +```bash +osm user-update admin --password 'osm4u' +``` + +```bash +osm project-create --domain-name default test_project_1 +``` + +```bash +osm user-create test_admin_1 --projects admin --project-role-mappings 'test_project_1,project_user' --password testadmin --domain-name default +osm user-update test_admin_1 --remove-project-role 'admin,project_admin' +osm user-create test_member_1 --projects admin --project-role-mappings 'test_project_1,project_user' --password testmember --domain-name default +osm user-update test_member_1 --remove-project-role 'admin,project_admin' +``` + + +## Selection of Packages + +In order to test that the upgrade does not impact existing operations, we will deploy a series of network services and slices prior to the upgrade and then verify all functionality post upgrade. What follows is a list of packages and tests to run. All packages will come from https://osm.etsi.org/gitlab/vnf-onboarding/osm-packages + +### Basic OpenStack VNF + +The following two packages will be used: +1. https://osm.etsi.org/gitlab/vnf-onboarding/osm-packages/-/tree/master/hackfest_basic_ns +2. https://osm.etsi.org/gitlab/vnf-onboarding/osm-packages/-/tree/master/hackfest_basic_vnf + +#### Preparation + +```bash +osm package-build hackfest_basic_ns +osm package-build hackfest_basic_vnf +osm upload-package hackfest_basic_vnf +osm upload-package hackfest_basic_ns + +osm ns-create --ns_name hackfest_basic-ns \ + --nsd_name hackfest_basic-ns \ + --vim_account etsi-vim \ + --config \ + '{vld: [ {name: mgmtnet, vim-network-name: osm-ext} ] }' +``` + +### Basic OpenStack VNF with Monitoring + +#### Preparation + +```bash +osm package-build hackfest_basic_metrics_vnf +osm package-build hackfest_basic_metrics_ns +osm upload-package ./hackfest_basic_metrics_vnf.tar.gz +osm upload-package ./hackfest_basic_metrics_ns.tar.gz + +osm ns-create --ns_name hackfest_basic_metrics_ns \ + --nsd_name hackfest_basic-ns-metrics \ + --vim_account etsi-vim \ + --config '{vld: [ {name: mgmtnet, vim-network-name: osm-ext} ] }' +``` + +### Basic OpenStack VNF with Proxy Charms + +#### Preparation + +```bash +osm package-build charm-packages/ha_proxy_charm_vnf +osm package-build charm-packages/ha_proxy_charm_ns +osm upload-package charm-packages/ha_proxy_charm_vnf.tar.gz +osm upload-package charm-packages/ha_proxy_charm_ns.tar.gz + +osm ns-create --ns_name ha_proxy_charm-ns \ +--nsd_name ha_proxy_charm-ns \ +--vim_account etsi-vim \ +--config '{vld: [ {name: mgmtnet, vim-network-name: osm-ext} ] }' + +osm ns-action ha_proxy_charm-ns \ + --vnf_name 1 \ + --action_name touch \ + --params '{filename: file-001.dat}' +``` + +### Basic OpenStack VNF with Native Charms + +#### Preparation + +```bash +osm package-build charm-packages/native_charm_vnf +osm package-build charm-packages/native_charm_ns +osm upload-package charm-packages/native_charm_vnf.tar.gz +osm upload-package charm-packages/native_charm_ns.tar.gz + +osm ns-create --ns_name native_charm-ns \ + --nsd_name native_charm-ns \ + --vim_account etsi-vim \ + --config '{vld: [ {name: mgmtnet, vim-network-name: osm-ext} ] }' + +osm ns-action native_charm-ns \ + --vnf_name 1 \ + --vdu_id mgmtVM \ + --action_name touch \ + --params '{filename: file-001.dat}' +``` + +### KNF with Helm + +#### Preparation + +```bash +cat << EOF > ~/openldap-params.yaml + +vld: +- name: mgmtnet + vim-network-name: osm-ext +additionalParamsForVnf: +- member-vnf-index: openldap + additionalParamsForKdu: + - kdu_name: ldap + additionalParams: + adminPassword: osm4u + configPassword: osm4u + env: +        LDAP_ORGANISATION: "Example Inc." +        LDAP_DOMAIN: "example.org" +        LDAP_BACKEND: "hdb" +        LDAP_TLS: "true" +        LDAP_TLS_ENFORCE: "false" +        LDAP_REMOVE_CONFIG_AFTER_SETUP: "true" +EOF + +osm package-build openldap_knf +osm package-build openldap_ns +osm upload-package ./openldap_knf.tar.gz +osm upload-package ./openldap_ns.tar.gz + +osm ns-create --ns_name openldap_ns \ + --nsd_name openldap_ns \ + --vim_account etsi-vim \ + --config_file ~/openldap-params.yaml +``` + +### KNF with Juju + +#### Preparation + +```bash +osm package-build squid_metrics_cnf +osm package-build squid_metrics_cnf_ns +osm upload-package ./squid_metrics_cnf.tar.gz +osm upload-package ./squid_metrics_cnf_ns.tar.gz + +osm ns-create --ns_name squid_cnf_ns \ + --nsd_name squid_cnf_ns \ + --vim_account dimension \ + --config '{vld: [ {name: mgmtnet, vim-network-name: osm-ext} ] }' +``` + -- GitLab From cf4faa30aaf3c21afc26c4e1be98889767a3c2d9 Mon Sep 17 00:00:00 2001 From: beierlm Date: Fri, 25 Feb 2022 14:59:19 +0000 Subject: [PATCH 07/15] Update 19-lts-upgrade.md --- 19-lts-upgrade.md | 56 ++++++++++++++++++++++++++++++++++++++++++----- 1 file changed, 50 insertions(+), 6 deletions(-) diff --git a/19-lts-upgrade.md b/19-lts-upgrade.md index 875819a..db2fb93 100644 --- a/19-lts-upgrade.md +++ b/19-lts-upgrade.md @@ -65,11 +65,23 @@ kubectl cp osm/$mongodb_pod:/data/backup.archive backup.archive /usr/share/osm-devops/installers/install_osm.sh --uninstall ``` -#### Update OSM 10.1.0 LTS +#### Upgrade Juju + +```bash +sudo snap refresh juju --channel 2.9/stable +juju switch osm-vca:admin/controller +juju upgrade-model +``` + +#### Upgrade Kubernetes + +Documentation for how to upgrade Kubernetes can be found at https://kubernetes.io/docs/tasks/administer-cluster/kubeadm/kubeadm-upgrade/ + +#### Update OSM 10.1.0 LTS ``` ``` -__NOTE__ This does not work - need help getting a procudure for upgrade here. +__NOTE__ Need help getting a procudure for upgrade here. @@ -129,8 +141,36 @@ juju scale-application ro 0 juju scale-application zookeeper-k8s 0 ``` +Wait for all the applications to scale to 0. The output of `juju status` should look similar to the following, with only `maridb-k8s` and `mongodb-k8s` units left. +``` +Model Controller Cloud/Region Version SLA Timestamp +osm osm-vca microk8s/localhost 2.8.13 unsupported 20:57:44Z + +App Version Status Scale Charm Store Rev OS Address Notes +grafana docker.io/ubuntu/grafana@sh... active 0 grafana jujucharms 4 kubernetes 10.152.183.45 +kafka-k8s rocks.canonical.com:443/wur... active 0 kafka-k8s jujucharms 21 kubernetes 10.152.183.248 +keystone keystone:10.0.3 active 0 keystone jujucharms 9 kubernetes 10.152.183.114 +lcm lcm:10.0.3 active 0 lcm jujucharms 8 kubernetes 10.152.183.70 +mariadb-k8s rocks.canonical.com:443/mar... active 1 mariadb-k8s jujucharms 35 kubernetes 10.152.183.177 +mon mon:10.0.3 active 0 mon jujucharms 5 kubernetes 10.152.183.227 +mongodb-k8s mongo:latest active 1 mongodb-k8s jujucharms 29 kubernetes 10.152.183.63 +nbi nbi:10.0.3 active 0 nbi jujucharms 12 kubernetes 10.152.183.163 +ng-ui ng-ui:10.0.3 active 0 ng-ui jujucharms 21 kubernetes 10.152.183.180 +pla pla:10.0.3 active 0 pla jujucharms 9 kubernetes 10.152.183.7 +pol pol:10.0.3 active 0 pol jujucharms 4 kubernetes 10.152.183.104 +prometheus docker.io/ed1000/prometheus... active 0 prometheus jujucharms 4 kubernetes 10.152.183.120 +ro ro:10.0.3 active 0 ro jujucharms 4 kubernetes 10.152.183.159 +zookeeper-k8s rocks.canonical.com:443/k8s... active 0 zookeeper-k8s jujucharms 37 kubernetes 10.152.183.201 + +Unit Workload Agent Address Ports Message +mariadb-k8s/0* active idle 10.1.244.152 3306/TCP ready +mongodb-k8s/0* active idle 10.1.244.156 27017/TCP ready +``` + #### Backup the Databases +Once all the units show a scale of 0, proceed to performing the database backup. + ```bash mariadb_unit=$(juju status | grep -i mariadb | tail -1 | awk -F" " '{print $1}' | tr -d '[*]') maridb_pod=$(microk8s.kubectl get pod -n osm | grep -i mariadb | tail -1 | awk -F" " '{print $1}') @@ -181,7 +221,7 @@ As we need to restore the database, we are going to stop all the OSM services on ```bash juju scale-application grafana 0 juju scale-application prometheus 0 -juju scale-application kafka-k8s 0 +juju scale-application kafka 0 juju scale-application keystone 0 juju scale-application lcm 0 juju scale-application mon 0 @@ -190,7 +230,7 @@ juju scale-application ng-ui 0 juju scale-application pla 0 juju scale-application pol 0 juju scale-application ro 0 -juju scale-application zookeeper-k8s 0 +juju scale-application zookeeper 0 ``` #### Restore the Databases @@ -199,7 +239,7 @@ juju scale-application zookeeper-k8s 0 mariadb_unit=$(juju status | grep -i mariadb | tail -1 | awk -F" " '{print $1}' | tr -d '[*]') mariadb_pod=$(microk8s.kubectl get pod -n osm | grep -i mariadb | tail -1 | awk -F" " '{print $1}') microk8s.kubectl cp backup.sql.gz osm/$mariadb_pod:/var/lib/mysql/backup.sql.gz -juju run-action $mariadb_unit restore --wait -m osm +juju run-action --wait -m osm $mariadb_unit restore mongodb_pod=$(microk8s.kubectl get pod -n osm | grep -i mongodb | tail -1 | awk -F" " '{print $1}') microk8s.kubectl cp backup.archive osm/$mongodb_pod:/data/backup.archive @@ -211,7 +251,8 @@ microk8s.kubectl exec -n osm -it $mongodb_pod -- mongorestore --gzip --archive=/ ##### Keystone Database Updates Start the Keystone container. -``` + +```bash juju scale-application keystone 1 ``` @@ -227,6 +268,9 @@ juju run --unit $mariadb_unit -- bash -c 'mysql -p${MARIADB_ROOT_PASSWORD} -Dkey ```bash keystone_unit=$(juju status | grep -i keystone | tail -1 | awk -F" " '{print $1}' | tr -d '[*]') juju run --unit $keystone_unit -- keystone-manage db_sync --expand +juju run-action --wait -m osm $keystone_unit db-sync + + ``` ##### OSM Application Database Updates -- GitLab From 4a1b004673d3aa3ed59cd27e16bf834e7a89a1c4 Mon Sep 17 00:00:00 2001 From: beierlm Date: Sat, 26 Feb 2022 12:32:43 +0000 Subject: [PATCH 08/15] Update 19-lts-upgrade.md --- 19-lts-upgrade.md | 51 ++++++++++++++++++++++++----------------------- 1 file changed, 26 insertions(+), 25 deletions(-) diff --git a/19-lts-upgrade.md b/19-lts-upgrade.md index db2fb93..2241cfc 100644 --- a/19-lts-upgrade.md +++ b/19-lts-upgrade.md @@ -91,7 +91,9 @@ __NOTE__ Need help getting a procudure for upgrade here. ### Charmed Installation Option -The following steps are to be followed for an upgrade to LTS: +For Charmed OSM Installation, the procedure is to maintain the database content while redeploying the application using Juju. Rather than adding a series of commands to manually redeploy, we can simply download and run the LTS installer to recreate OSM after removing the non-LTS software. + +The following steps will upgrade OSM to the LTS version: * [Stop all OSM Services](#charm-1) * [Backup the Databases](#charm-2) @@ -187,16 +189,29 @@ microk8s.kubectl cp osm/$mongodb_pod:/data/backup.archive backup.archive ```bash juju destroy-model osm --destroy-storage -y --force --no-wait +microk8s.kubectl delete namespaces osm ``` #### Upgrade Juju +The following commands will upgrade the OSM controller. + ```bash sudo snap refresh juju --channel 2.9/stable juju switch osm-vca:admin/controller juju upgrade-model ``` +Next, for any native or proxy charms, upgrade each model. + +```bash +for model in $(juju models --format json | jq .models[].name | tr -d \") ; do + juju switch $model + juju upgrade-model +done +``` + + #### Upgrade MicroK8s ```bash @@ -219,9 +234,6 @@ unset OSM_PASSWORD As we need to restore the database, we are going to stop all the OSM services once again. ```bash -juju scale-application grafana 0 -juju scale-application prometheus 0 -juju scale-application kafka 0 juju scale-application keystone 0 juju scale-application lcm 0 juju scale-application mon 0 @@ -230,15 +242,15 @@ juju scale-application ng-ui 0 juju scale-application pla 0 juju scale-application pol 0 juju scale-application ro 0 -juju scale-application zookeeper 0 ``` #### Restore the Databases ```bash -mariadb_unit=$(juju status | grep -i mariadb | tail -1 | awk -F" " '{print $1}' | tr -d '[*]') +mariadb_unit=$(juju status | grep -i mariadb | tail -1 | awk -F" " '{print $1}' | tr -d '[*]') mariadb_pod=$(microk8s.kubectl get pod -n osm | grep -i mariadb | tail -1 | awk -F" " '{print $1}') microk8s.kubectl cp backup.sql.gz osm/$mariadb_pod:/var/lib/mysql/backup.sql.gz +juju run --unit $mariadb_unit -- bash -c 'mysql -p${MARIADB_ROOT_PASSWORD} --execute "DROP DATABASE keystone"' juju run-action --wait -m osm $mariadb_unit restore mongodb_pod=$(microk8s.kubectl get pod -n osm | grep -i mongodb | tail -1 | awk -F" " '{print $1}') @@ -267,10 +279,7 @@ juju run --unit $mariadb_unit -- bash -c 'mysql -p${MARIADB_ROOT_PASSWORD} -Dkey ```bash keystone_unit=$(juju status | grep -i keystone | tail -1 | awk -F" " '{print $1}' | tr -d '[*]') -juju run --unit $keystone_unit -- keystone-manage db_sync --expand juju run-action --wait -m osm $keystone_unit db-sync - - ``` ##### OSM Application Database Updates @@ -318,9 +327,6 @@ juju remove-application osm-update-db Now that the databases are migrated to the new version, we can restart the services. ```bash -juju scale-application grafana 1 -juju scale-application prometheus 1 -juju scale-application kafka-k8s 1 juju scale-application lcm 1 juju scale-application mon 1 juju scale-application nbi 1 @@ -328,9 +334,11 @@ juju scale-application ng-ui 1 juju scale-application pla 1 juju scale-application pol 1 juju scale-application ro 1 -juju scale-application zookeeper-k8s 1 ``` +At this point, OSM LTS is operational and ready to use. + + # Testing Upgrade ## Changing Credentials @@ -339,6 +347,7 @@ We will change some default passwords, and create some additional users to ensur ```bash osm user-update admin --password 'osm4u' +export OSM_PASSWORD=osm4u ``` ```bash @@ -357,11 +366,11 @@ osm user-update test_member_1 --remove-project-role 'admin,project_admin' In order to test that the upgrade does not impact existing operations, we will deploy a series of network services and slices prior to the upgrade and then verify all functionality post upgrade. What follows is a list of packages and tests to run. All packages will come from https://osm.etsi.org/gitlab/vnf-onboarding/osm-packages -### Basic OpenStack VNF +```bash +git clone -j4 --recursive https://osm.etsi.org/gitlab/vnf-onboarding/osm-packages.git +``` -The following two packages will be used: -1. https://osm.etsi.org/gitlab/vnf-onboarding/osm-packages/-/tree/master/hackfest_basic_ns -2. https://osm.etsi.org/gitlab/vnf-onboarding/osm-packages/-/tree/master/hackfest_basic_vnf +### Basic OpenStack VNF #### Preparation @@ -396,8 +405,6 @@ osm ns-create --ns_name hackfest_basic_metrics_ns \ ### Basic OpenStack VNF with Proxy Charms -#### Preparation - ```bash osm package-build charm-packages/ha_proxy_charm_vnf osm package-build charm-packages/ha_proxy_charm_ns @@ -417,8 +424,6 @@ osm ns-action ha_proxy_charm-ns \ ### Basic OpenStack VNF with Native Charms -#### Preparation - ```bash osm package-build charm-packages/native_charm_vnf osm package-build charm-packages/native_charm_ns @@ -439,8 +444,6 @@ osm ns-action native_charm-ns \ ### KNF with Helm -#### Preparation - ```bash cat << EOF > ~/openldap-params.yaml @@ -476,8 +479,6 @@ osm ns-create --ns_name openldap_ns \ ### KNF with Juju -#### Preparation - ```bash osm package-build squid_metrics_cnf osm package-build squid_metrics_cnf_ns -- GitLab From d2e2169f09b4ab54f7f8dd87aac1608c36b06c3e Mon Sep 17 00:00:00 2001 From: beierlm Date: Tue, 1 Mar 2022 09:39:09 +0000 Subject: [PATCH 09/15] Update 19-lts-upgrade.md --- 19-lts-upgrade.md | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/19-lts-upgrade.md b/19-lts-upgrade.md index 2241cfc..d550c24 100644 --- a/19-lts-upgrade.md +++ b/19-lts-upgrade.md @@ -192,6 +192,15 @@ juju destroy-model osm --destroy-storage -y --force --no-wait microk8s.kubectl delete namespaces osm ``` +##### Ingress for Upgrade from 9.1.5 + +If this is an upgrade from 9.1.5, the following commands should be run at this time to flush any legacy ingress descriptors. + +```bash +microk8s.kubectl disable ingress +microk8s.kubectl enable ingress +``` + #### Upgrade Juju The following commands will upgrade the OSM controller. @@ -221,9 +230,9 @@ sudo snap refresh microk8s --channel=1.23/stable #### Install OSM 10.1.0 LTS ```bash -wget https://osm-download.etsi.org/repository/osm/debian/ReleaseTEN-daily/pool/devops/osm-devops_10.0.3.post24-1_all.deb +wget https://osm-download.etsi.org/repository/osm/debian/ReleaseTEN-daily/pool/devops/osm-devops_10.0.3.post27-1_all.deb sudo apt remove -y --purge osm-devops -sudo apt install ./osm-devops_10.0.3.post24-1_all.deb +sudo apt install ./osm-devops_10.0.3.post27-1_all.deb unset OSM_USERNAME unset OSM_PASSWORD /usr/share/osm-devops/installers/charmed_install.sh --vca osm-vca --tag releaseten-daily #10.1.0 @@ -255,7 +264,7 @@ juju run-action --wait -m osm $mariadb_unit restore mongodb_pod=$(microk8s.kubectl get pod -n osm | grep -i mongodb | tail -1 | awk -F" " '{print $1}') microk8s.kubectl cp backup.archive osm/$mongodb_pod:/data/backup.archive -microk8s.kubectl exec -n osm -it $mongodb_pod -- mongorestore --gzip --archive=/data/backup.archive +microk8s.kubectl exec -n osm -it $mongodb_pod -- mongorestore --drop --gzip --archive=/data/backup.archive ``` #### Perform Database Migration -- GitLab From 26c108499e07037760b62fccaf3ae69340465079 Mon Sep 17 00:00:00 2001 From: beierlm Date: Fri, 4 Mar 2022 12:32:27 +0000 Subject: [PATCH 10/15] Update 19-lts-upgrade.md --- 19-lts-upgrade.md | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/19-lts-upgrade.md b/19-lts-upgrade.md index d550c24..3b4dc04 100644 --- a/19-lts-upgrade.md +++ b/19-lts-upgrade.md @@ -188,7 +188,7 @@ microk8s.kubectl cp osm/$mongodb_pod:/data/backup.archive backup.archive #### Remove Deployed OSM Application ```bash -juju destroy-model osm --destroy-storage -y --force --no-wait +juju destroy-model osm --destroy-storage -y --force microk8s.kubectl delete namespaces osm ``` @@ -207,8 +207,7 @@ The following commands will upgrade the OSM controller. ```bash sudo snap refresh juju --channel 2.9/stable -juju switch osm-vca:admin/controller -juju upgrade-model +juju upgrade-controller ``` Next, for any native or proxy charms, upgrade each model. @@ -230,12 +229,17 @@ sudo snap refresh microk8s --channel=1.23/stable #### Install OSM 10.1.0 LTS ```bash -wget https://osm-download.etsi.org/repository/osm/debian/ReleaseTEN-daily/pool/devops/osm-devops_10.0.3.post27-1_all.deb sudo apt remove -y --purge osm-devops -sudo apt install ./osm-devops_10.0.3.post27-1_all.deb unset OSM_USERNAME unset OSM_PASSWORD -/usr/share/osm-devops/installers/charmed_install.sh --vca osm-vca --tag releaseten-daily #10.1.0 + +wget https://osm-download.etsi.org/ftp/osm-10.0-ten/install_osm.sh +chmod +x ./install_osm.sh +./install_osm -R 10.1.0rc1 -r testing --charmed -vca osm-vca --tag 10.1.0-rc1 + +WIP: RC1 testing was apparently not signed +sudo sed -i "s|\[arch=amd64] https://osm-download.etsi.org|[trusted=yes arch=amd64] https://osm-download.etsi.org|" /etc/apt/sources.list + ``` #### Stop all OSM Services -- GitLab From 2cd104dab86bdf5781a89e47f6fa1665cf699841 Mon Sep 17 00:00:00 2001 From: beierlm Date: Wed, 9 Mar 2022 12:56:34 +0000 Subject: [PATCH 11/15] Update 19-lts-upgrade.md --- 19-lts-upgrade.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/19-lts-upgrade.md b/19-lts-upgrade.md index 3b4dc04..c6e1ba8 100644 --- a/19-lts-upgrade.md +++ b/19-lts-upgrade.md @@ -235,7 +235,7 @@ unset OSM_PASSWORD wget https://osm-download.etsi.org/ftp/osm-10.0-ten/install_osm.sh chmod +x ./install_osm.sh -./install_osm -R 10.1.0rc1 -r testing --charmed -vca osm-vca --tag 10.1.0-rc1 +./install_osm.sh -R 10.1.0rc1 -r testing --charmed --vca osm-vca --tag 10.1.0-rc1 WIP: RC1 testing was apparently not signed sudo sed -i "s|\[arch=amd64] https://osm-download.etsi.org|[trusted=yes arch=amd64] https://osm-download.etsi.org|" /etc/apt/sources.list -- GitLab From b95128a00ca7d89b73843d4b618955145e0b1cd8 Mon Sep 17 00:00:00 2001 From: garciadeblas Date: Wed, 9 Mar 2022 17:29:43 +0100 Subject: [PATCH 12/15] Update 19-lts-upgrade with instructions for community-based installations Signed-off-by: garciadeblas --- 19-lts-upgrade.md | 144 +++++++++++++++++++++++++++++++++++----------- 1 file changed, 111 insertions(+), 33 deletions(-) diff --git a/19-lts-upgrade.md b/19-lts-upgrade.md index c6e1ba8..9bf9d75 100644 --- a/19-lts-upgrade.md +++ b/19-lts-upgrade.md @@ -9,7 +9,7 @@ Starting with version 10.1.0 of OSM, every even numbered release will receive tw This procedure covers both upgrade from 9.1.5 or 10.0.3 to 10.1.0 LTS. Where necessary, additional steps for 9.1.5 are shown. There are two installation methods, each with its own set of procedures: * [Kubernetes Installation Option](#kubernetes-installation-option) -* [Charmed Installation Option](#charmed-nstallation-option) +* [Charmed Installation Option](#charmed-installation-option) ### Kubernetes Installation Option @@ -17,14 +17,14 @@ The following steps are to be followed for an upgrade to LTS: * [Stop all OSM Services](#k8s-1) * [Backup the Databases](#k8s-2) -* [](#k8s-3) -* [](#k8s-4) -* [](#k8s-5) -* [](#k8s-6) -* [](#k8s-7) -* [](#k8s-8) -* [](#k8s-9) -* [](#k8s-10) +* [Backup existing OSM manifests](#k8s-3) +* [Upgrade Juju](#k8s-4) +* [Upgrade Kubernetes](#k8s-5) +* [Upgrade charmed services](#k8s-6) +* [Upgrade OSM to 10.1.0 LTS](#k8s-7) +* [Stop New OSM Service](#k8s-8) +* [Perform Database Migration](#k8s-9) +* [Restart all OSM services](#k8s-10) #### Stop all OSM Services @@ -47,6 +47,8 @@ __Note__ if PLA was not installed, you can ignore the error about `deployments.a #### Backup the Databases +Once all the deployments and statafulsets show 0 replicas, proceed to performing the database backup. + ```bash mysql_pod=$(kubectl get pod -n osm | grep -i mysql | tail -1 | awk -F" " '{print $1}') kubectl exec -n osm -it $mysql_pod -- bash -c \ @@ -59,10 +61,10 @@ juju run-action $mongodb_unit backup --wait -m osm kubectl cp osm/$mongodb_pod:/data/backup.archive backup.archive ``` -#### Remove Deployed OSM Application +#### Backup existing OSM manifests ```bash -/usr/share/osm-devops/installers/install_osm.sh --uninstall +cp -bR /etc/osm/docker/osm_pods backup_osm_manifests ``` #### Upgrade Juju @@ -75,19 +77,99 @@ juju upgrade-model #### Upgrade Kubernetes -Documentation for how to upgrade Kubernetes can be found at https://kubernetes.io/docs/tasks/administer-cluster/kubeadm/kubeadm-upgrade/ +Documentation for how to upgrade Kubernetes can be found at + +#### Upgrade charmed services + +Mongo needs to be updated. + +__NOTE__: Need help getting a procedure for upgrade Mongo here. + +#### Upgrade OSM to 10.1.0 LTS + +```bash +sudo apt-get update +sudo apt-get install -y osm-devops python3-osm-im python3-osmclient +sudo cp -R /usr/share/osm-devops/installers/docker/osm_pods /etc/osm/docker/osm_pods +sudo rm /etc/osm/docker/mongo.yaml +kubectl -n osm apply -f /etc/osm/docker/osm_pods +``` + +#### Stop all OSM Services + +As we need to restore the database, we are going to stop all the OSM services once again. + +```bash +kubectl -n osm scale deployment/grafana --replicas=0 +kubectl -n osm scale statefulset/prometheus --replicas=0 +kubectl -n osm scale statefulset/kafka --replicas=0 +kubectl -n osm scale deployment/keystone --replicas=0 +kubectl -n osm scale deployment/lcm --replicas=0 +kubectl -n osm scale deployment/mon --replicas=0 +kubectl -n osm scale deployment/nbi --replicas=0 +kubectl -n osm scale deployment/ng-ui --replicas=0 +kubectl -n osm scale deployment/pla --replicas=0 +kubectl -n osm scale deployment/pol --replicas=0 +kubectl -n osm scale deployment/ro --replicas=0 +kubectl -n osm scale statefulset/zookeeper --replicas=0 +``` + +#### Perform Database Migration -#### Update OSM 10.1.0 LTS +##### Keystone Database Updates + +Update the database: +```bash +mysql_pod=$(kubectl get pod -n osm | grep -i mysql | tail -1 | awk -F" " '{print $1}') +kubectl exec -n osm -it $mysql_pod -- bash -c '' ``` + +__NOTE__: Need help getting a procedure for upgrade Keystone here. + +##### OSM Application Database Updates + +A helper charm has been created to assist in the database updates. Build, deploy and run the action as follows: + +```bash +sudo snap install charmcraft --classic + +git clone https://github.com/charmed-osm/osm-update-db-operator.git +cd osm-update-db-operator +charmcraft build + +juju switch osm +juju deploy ./osm-update-db_ubuntu-20.04-amd64.charm ``` -__NOTE__ Need help getting a procudure for upgrade here. +Run the upgrade as follows. +```bash +juju config osm-update-db mongodb-uri=mongodb://mongodb:27017 +juju run-action --wait osm-update-db/0 apply-patch bug-number=1837 +``` +#### Restart all OSM Services ---------------------------------------------------------------------------------------------------------------- +Now that the databases are migrated to the new version, we can restart the services. + +```bash +kubectl -n osm scale deployment/grafana --replicas=1 +kubectl -n osm scale statefulset/prometheus --replicas=1 +kubectl -n osm scale statefulset/kafka --replicas=1 +kubectl -n osm scale deployment/keystone --replicas=1 +kubectl -n osm scale deployment/lcm --replicas=1 +kubectl -n osm scale deployment/mon --replicas=1 +kubectl -n osm scale deployment/nbi --replicas=1 +kubectl -n osm scale deployment/ng-ui --replicas=1 +kubectl -n osm scale deployment/pla --replicas=1 +kubectl -n osm scale deployment/pol --replicas=1 +kubectl -n osm scale deployment/ro --replicas=1 +kubectl -n osm scale statefulset/zookeeper --replicas=1 +``` + +At this point, OSM LTS is operational and ready to use. ### Charmed Installation Option @@ -106,7 +188,6 @@ The following steps will upgrade OSM to the LTS version: * [Perform Database Migration](#charm-9) * [Restart all OSM Services](#charm-10) - #### Stop all OSM Services ##### Version 9.1.5 @@ -219,7 +300,6 @@ for model in $(juju models --format json | jq .models[].name | tr -d \") ; do done ``` - #### Upgrade MicroK8s ```bash @@ -242,7 +322,7 @@ sudo sed -i "s|\[arch=amd64] https://osm-download.etsi.org|[trusted=yes arch=amd ``` -#### Stop all OSM Services +#### Stop New OSM Services As we need to restore the database, we are going to stop all the OSM services once again. @@ -271,7 +351,7 @@ microk8s.kubectl cp backup.archive osm/$mongodb_pod:/data/backup.archive microk8s.kubectl exec -n osm -it $mongodb_pod -- mongorestore --drop --gzip --archive=/data/backup.archive ``` -#### Perform Database Migration +#### Perform Database Migration ##### Keystone Database Updates @@ -297,7 +377,7 @@ juju run-action --wait -m osm $keystone_unit db-sync ##### OSM Application Database Updates -A helper charm has been created to assist in the database updates. Build, deploy and run the action as follows: +A helper charm has been created to assist in the database updates. Build, deploy and run the action as follows: ```bash sudo snap install charmcraft --classic @@ -351,10 +431,9 @@ juju scale-application ro 1 At this point, OSM LTS is operational and ready to use. +## Testing Upgrade -# Testing Upgrade - -## Changing Credentials +### Changing Credentials We will change some default passwords, and create some additional users to ensure RBAC still works. @@ -374,8 +453,7 @@ osm user-create test_member_1 --projects admin --project-role-mappings 'test_pro osm user-update test_member_1 --remove-project-role 'admin,project_admin' ``` - -## Selection of Packages +### Selection of Packages In order to test that the upgrade does not impact existing operations, we will deploy a series of network services and slices prior to the upgrade and then verify all functionality post upgrade. What follows is a list of packages and tests to run. All packages will come from https://osm.etsi.org/gitlab/vnf-onboarding/osm-packages @@ -383,9 +461,9 @@ In order to test that the upgrade does not impact existing operations, we will d git clone -j4 --recursive https://osm.etsi.org/gitlab/vnf-onboarding/osm-packages.git ``` -### Basic OpenStack VNF +#### Basic OpenStack VNF -#### Preparation +##### Preparation ```bash osm package-build hackfest_basic_ns @@ -400,9 +478,9 @@ osm ns-create --ns_name hackfest_basic-ns \ '{vld: [ {name: mgmtnet, vim-network-name: osm-ext} ] }' ``` -### Basic OpenStack VNF with Monitoring +#### Basic OpenStack VNF with Monitoring -#### Preparation +##### Preparation ```bash osm package-build hackfest_basic_metrics_vnf @@ -416,7 +494,7 @@ osm ns-create --ns_name hackfest_basic_metrics_ns \ --config '{vld: [ {name: mgmtnet, vim-network-name: osm-ext} ] }' ``` -### Basic OpenStack VNF with Proxy Charms +#### Basic OpenStack VNF with Proxy Charms ```bash osm package-build charm-packages/ha_proxy_charm_vnf @@ -435,7 +513,7 @@ osm ns-action ha_proxy_charm-ns \ --params '{filename: file-001.dat}' ``` -### Basic OpenStack VNF with Native Charms +#### Basic OpenStack VNF with Native Charms ```bash osm package-build charm-packages/native_charm_vnf @@ -455,7 +533,7 @@ osm ns-action native_charm-ns \ --params '{filename: file-001.dat}' ``` -### KNF with Helm +#### KNF with Helm ```bash cat << EOF > ~/openldap-params.yaml @@ -490,7 +568,7 @@ osm ns-create --ns_name openldap_ns \ --config_file ~/openldap-params.yaml ``` -### KNF with Juju +#### KNF with Juju ```bash osm package-build squid_metrics_cnf -- GitLab From 6605db98b1113a35e645ea1d9b6f92c21eb8f740 Mon Sep 17 00:00:00 2001 From: rodriguezgar Date: Mon, 14 Mar 2022 12:56:13 +0000 Subject: [PATCH 13/15] Typos correction --- 19-lts-upgrade.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/19-lts-upgrade.md b/19-lts-upgrade.md index 9bf9d75..a128ca2 100644 --- a/19-lts-upgrade.md +++ b/19-lts-upgrade.md @@ -256,9 +256,9 @@ Once all the units show a scale of 0, proceed to performing the database backup. ```bash mariadb_unit=$(juju status | grep -i mariadb | tail -1 | awk -F" " '{print $1}' | tr -d '[*]') -maridb_pod=$(microk8s.kubectl get pod -n osm | grep -i mariadb | tail -1 | awk -F" " '{print $1}') +mariadb_pod=$(microk8s.kubectl get pod -n osm | grep -i mariadb | tail -1 | awk -F" " '{print $1}') juju run-action $mariadb_unit backup --wait -m osm -microk8s.kubectl cp osm/$maridb_pod:/var/lib/mysql/backup.sql.gz backup.sql.gz +microk8s.kubectl cp osm/$mariadb_pod:/var/lib/mysql/backup.sql.gz backup.sql.gz mongodb_unit=$(juju status | grep -i mongodb | tail -1 | awk -F" " '{print $1}'| tr -d '[*]') mongodb_pod=$(microk8s.kubectl get pod -n osm | grep -i mongodb | tail -1 | awk -F" " '{print $1}') @@ -278,8 +278,8 @@ microk8s.kubectl delete namespaces osm If this is an upgrade from 9.1.5, the following commands should be run at this time to flush any legacy ingress descriptors. ```bash -microk8s.kubectl disable ingress -microk8s.kubectl enable ingress +microk8s disable ingress +microk8s enable ingress ``` #### Upgrade Juju -- GitLab From aa5201b09c7004e438b19d535f5a11709c80821d Mon Sep 17 00:00:00 2001 From: garciadeblas Date: Fri, 25 Mar 2022 13:06:24 +0100 Subject: [PATCH 14/15] Update 19-lts-upgrade with fixes for community-based installations Signed-off-by: garciadeblas --- 19-lts-upgrade.md | 66 ++++++++++++++++++++++++++++++++--------------- 1 file changed, 45 insertions(+), 21 deletions(-) diff --git a/19-lts-upgrade.md b/19-lts-upgrade.md index a128ca2..3179076 100644 --- a/19-lts-upgrade.md +++ b/19-lts-upgrade.md @@ -18,13 +18,15 @@ The following steps are to be followed for an upgrade to LTS: * [Stop all OSM Services](#k8s-1) * [Backup the Databases](#k8s-2) * [Backup existing OSM manifests](#k8s-3) -* [Upgrade Juju](#k8s-4) -* [Upgrade Kubernetes](#k8s-5) -* [Upgrade charmed services](#k8s-6) -* [Upgrade OSM to 10.1.0 LTS](#k8s-7) -* [Stop New OSM Service](#k8s-8) -* [Perform Database Migration](#k8s-9) -* [Restart all OSM services](#k8s-10) +* [Remove Deployed Charmed Services](#k8s-4) +* [Upgrade Juju](#k8s-5) +* [Upgrade Kubernetes](#k8s-6) +* [Deploy Charmed Services](#k8s-7) +* [Upgrade OSM to 10.1.0 LTS](#k8s-8) +* [Stop New OSM Service](#k8s-9) +* [Restore the Databases](#k8s-10) +* [Perform Database Migration](#k8s-11) +* [Restart all OSM services](#k8s-12) #### Stop all OSM Services @@ -67,7 +69,13 @@ kubectl cp osm/$mongodb_pod:/data/backup.archive backup.archive cp -bR /etc/osm/docker/osm_pods backup_osm_manifests ``` -#### Upgrade Juju +#### Remove Deployed Charmed Services + +```bash +juju destroy-model osm --destroy-storage -y --force +``` + +#### Upgrade Juju ```bash sudo snap refresh juju --channel 2.9/stable @@ -75,17 +83,18 @@ juju switch osm-vca:admin/controller juju upgrade-model ``` -#### Upgrade Kubernetes +#### Upgrade Kubernetes Documentation for how to upgrade Kubernetes can be found at -#### Upgrade charmed services +#### Deploy Charmed Services -Mongo needs to be updated. - -__NOTE__: Need help getting a procedure for upgrade Mongo here. +```bash +juju add-model osm k8scloud +juju deploy ch:mongodb-k8s -m osm +``` -#### Upgrade OSM to 10.1.0 LTS +#### Upgrade OSM to 10.1.0 LTS ```bash sudo apt-get update @@ -95,7 +104,7 @@ sudo rm /etc/osm/docker/mongo.yaml kubectl -n osm apply -f /etc/osm/docker/osm_pods ``` -#### Stop all OSM Services +#### Stop all OSM Services As we need to restore the database, we are going to stop all the OSM services once again. @@ -114,7 +123,22 @@ kubectl -n osm scale deployment/ro --replicas=0 kubectl -n osm scale statefulset/zookeeper --replicas=0 ``` -#### Perform Database Migration +#### Restore the Databases + +```bash +mysql_pod=$(kubectl get pod -n osm | grep -i mysql | tail -1 | awk -F" " '{print $1}') +kubectl cp backup.sql.gz osm/$mysql_pod:/var/lib/mysql/backup.sql.gz +kubectl exec -n osm -it $mysql_pod -- bash -c \ + 'mysql -uroot -p${MYSQL_ROOT_PASSWORD} --execute "DROP DATABASE keystone"' +kubectl exec -n osm -it $mysql_pod -- bash -c \ + 'zcat backup.sql.gz | mysql -uroot -p${MYSQL_ROOT_PASSWORD}' + +mongodb_pod=$(kubectl get pod -n osm | grep -i mongodb | grep -v operator | tail -1 | awk -F" " '{print $1}') +kubectl cp backup.archive osm/$mongodb_pod:/data/backup.archive +kubectl exec -n osm -it $mongodb_pod -- mongorestore --drop --gzip --archive=/data/backup.archive +``` + +#### Perform Database Migration ##### Keystone Database Updates @@ -122,11 +146,11 @@ Update the database: ```bash mysql_pod=$(kubectl get pod -n osm | grep -i mysql | tail -1 | awk -F" " '{print $1}') -kubectl exec -n osm -it $mysql_pod -- bash -c '' +kubectl exec -n osm -it $mysql_pod -- bash -c \ + 'mysql -uroot -p${MYSQL_ROOT_PASSWORD} -Dkeystone \ + --execute "UPDATE endpoint SET url=\"http://osm-keystone:5000/v3/\" WHERE url=\"http://keystone:5000/v3/\"";' ``` -__NOTE__: Need help getting a procedure for upgrade Keystone here. - ##### OSM Application Database Updates A helper charm has been created to assist in the database updates. Build, deploy and run the action as follows: @@ -150,7 +174,7 @@ juju config osm-update-db mongodb-uri=mongodb://mongodb:27017 juju run-action --wait osm-update-db/0 apply-patch bug-number=1837 ``` -#### Restart all OSM Services +#### Restart all OSM Services Now that the databases are migrated to the new version, we can restart the services. @@ -224,7 +248,7 @@ juju scale-application ro 0 juju scale-application zookeeper-k8s 0 ``` -Wait for all the applications to scale to 0. The output of `juju status` should look similar to the following, with only `maridb-k8s` and `mongodb-k8s` units left. +Wait for all the applications to scale to 0. The output of `juju status` should look similar to the following, with only `mariadb-k8s` and `mongodb-k8s` units left. ``` Model Controller Cloud/Region Version SLA Timestamp osm osm-vca microk8s/localhost 2.8.13 unsupported 20:57:44Z -- GitLab From b4a5dbb90e5cb49c1e29e16565e89d82b96fc116 Mon Sep 17 00:00:00 2001 From: beierlm Date: Fri, 25 Mar 2022 12:46:16 +0000 Subject: [PATCH 15/15] Points to final installer tag --- 18-tacacs-based-authentication.md | 2 +- 19-lts-upgrade.md | 6 +----- 2 files changed, 2 insertions(+), 6 deletions(-) diff --git a/18-tacacs-based-authentication.md b/18-tacacs-based-authentication.md index fda1a31..049fe89 100644 --- a/18-tacacs-based-authentication.md +++ b/18-tacacs-based-authentication.md @@ -36,4 +36,4 @@ OSM NB has a plugin which can interact with TACACS server to authenticate users. - If user profile is removed from TACACS server, OSM still maintains the profile and will not be removed until next login. As the profile is not available in TACACS, the authentication shall fail and user profile will be removed from OSM. - + \ No newline at end of file diff --git a/19-lts-upgrade.md b/19-lts-upgrade.md index 3179076..1863860 100644 --- a/19-lts-upgrade.md +++ b/19-lts-upgrade.md @@ -339,11 +339,7 @@ unset OSM_PASSWORD wget https://osm-download.etsi.org/ftp/osm-10.0-ten/install_osm.sh chmod +x ./install_osm.sh -./install_osm.sh -R 10.1.0rc1 -r testing --charmed --vca osm-vca --tag 10.1.0-rc1 - -WIP: RC1 testing was apparently not signed -sudo sed -i "s|\[arch=amd64] https://osm-download.etsi.org|[trusted=yes arch=amd64] https://osm-download.etsi.org|" /etc/apt/sources.list - +./install_osm.sh --charmed --vca osm-vca --tag 10.1.0 ``` #### Stop New OSM Services -- GitLab