From 6db47d485db75641a282f8da13720e23d6f6dfb5 Mon Sep 17 00:00:00 2001 From: Mark Beierl Date: Thu, 23 Mar 2023 13:51:15 -0400 Subject: [PATCH] OSM DB Update Charm Initial load of code for the osm-update-db-operator charm Change-Id: I2884249efaaa86f614df6c286a69f3546489b523 Signed-off-by: Mark Beierl --- .../charm/osm-update-db-operator/.gitignore | 23 + .../charm/osm-update-db-operator/.jujuignore | 17 + .../osm-update-db-operator/CONTRIBUTING.md | 74 ++++ .../charm/osm-update-db-operator/LICENSE | 202 +++++++++ .../charm/osm-update-db-operator/README.md | 80 ++++ .../charm/osm-update-db-operator/actions.yaml | 42 ++ .../osm-update-db-operator/charmcraft.yaml | 26 ++ .../charm/osm-update-db-operator/config.yaml | 29 ++ .../osm-update-db-operator/metadata.yaml | 19 + .../osm-update-db-operator/pyproject.toml | 53 +++ .../osm-update-db-operator/requirements.txt | 16 + .../charm/osm-update-db-operator/src/charm.py | 119 +++++ .../osm-update-db-operator/src/db_upgrade.py | 275 ++++++++++++ .../tests/integration/test_charm.py | 48 ++ .../tests/unit/test_charm.py | 165 +++++++ .../tests/unit/test_db_upgrade.py | 413 ++++++++++++++++++ .../charm/osm-update-db-operator/tox.ini | 104 +++++ jenkins/ci-pipelines/ci_stage_2.groovy | 1 + jenkins/ci-pipelines/ci_stage_3.groovy | 1 + 19 files changed, 1707 insertions(+) create mode 100644 installers/charm/osm-update-db-operator/.gitignore create mode 100644 installers/charm/osm-update-db-operator/.jujuignore create mode 100644 installers/charm/osm-update-db-operator/CONTRIBUTING.md create mode 100644 installers/charm/osm-update-db-operator/LICENSE create mode 100644 installers/charm/osm-update-db-operator/README.md create mode 100644 installers/charm/osm-update-db-operator/actions.yaml create mode 100644 installers/charm/osm-update-db-operator/charmcraft.yaml create mode 100644 installers/charm/osm-update-db-operator/config.yaml create mode 100644 installers/charm/osm-update-db-operator/metadata.yaml create mode 100644 installers/charm/osm-update-db-operator/pyproject.toml create mode 100644 installers/charm/osm-update-db-operator/requirements.txt create mode 100755 installers/charm/osm-update-db-operator/src/charm.py create mode 100644 installers/charm/osm-update-db-operator/src/db_upgrade.py create mode 100644 installers/charm/osm-update-db-operator/tests/integration/test_charm.py create mode 100644 installers/charm/osm-update-db-operator/tests/unit/test_charm.py create mode 100644 installers/charm/osm-update-db-operator/tests/unit/test_db_upgrade.py create mode 100644 installers/charm/osm-update-db-operator/tox.ini diff --git a/installers/charm/osm-update-db-operator/.gitignore b/installers/charm/osm-update-db-operator/.gitignore new file mode 100644 index 00000000..c2501574 --- /dev/null +++ b/installers/charm/osm-update-db-operator/.gitignore @@ -0,0 +1,23 @@ +# Copyright 2022 Canonical Ltd. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +venv/ +build/ +*.charm +.coverage +coverage.xml +__pycache__/ +*.py[cod] +.vscode +.tox diff --git a/installers/charm/osm-update-db-operator/.jujuignore b/installers/charm/osm-update-db-operator/.jujuignore new file mode 100644 index 00000000..ddb544e6 --- /dev/null +++ b/installers/charm/osm-update-db-operator/.jujuignore @@ -0,0 +1,17 @@ +# Copyright 2022 Canonical Ltd. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +/venv +*.py[cod] +*.charm diff --git a/installers/charm/osm-update-db-operator/CONTRIBUTING.md b/installers/charm/osm-update-db-operator/CONTRIBUTING.md new file mode 100644 index 00000000..4d706713 --- /dev/null +++ b/installers/charm/osm-update-db-operator/CONTRIBUTING.md @@ -0,0 +1,74 @@ + +# Contributing + +## Overview + +This documents explains the processes and practices recommended for contributing enhancements to +the Update DB charm. + +- Generally, before developing enhancements to this charm, you should consider [opening an issue + ](https://github.com/gcalvinos/update-db-operator/issues) explaining your use case. +- If you would like to chat with us about your use-cases or proposed implementation, you can reach + us at [Canonical Mattermost public channel](https://chat.charmhub.io/charmhub/channels/charm-dev) + or [Discourse](https://discourse.charmhub.io/). The primary author of this charm is available on + the Mattermost channel as `@davigar15`. +- Familiarising yourself with the [Charmed Operator Framework](https://juju.is/docs/sdk) library + will help you a lot when working on new features or bug fixes. +- All enhancements require review before being merged. Code review typically examines + - code quality + - test coverage + - user experience for Juju administrators this charm. +- Please help us out in ensuring easy to review branches by rebasing your pull request branch onto + the `main` branch. This also avoids merge commits and creates a linear Git commit history. + +## Developing + +You can use the environments created by `tox` for development: + +```shell +tox --notest -e unit +source .tox/unit/bin/activate +``` + +### Testing + +```shell +tox -e fmt # update your code according to linting rules +tox -e lint # code style +tox -e unit # unit tests +# tox -e integration # integration tests +tox # runs 'lint' and 'unit' environments +``` + +## Build charm + +Build the charm in this git repository using: + +```shell +charmcraft pack +``` + +### Deploy + +```bash +# Create a model +juju add-model test-update-db +# Enable DEBUG logging +juju model-config logging-config="=INFO;unit=DEBUG" +# Deploy the charm +juju deploy ./update-db_ubuntu-20.04-amd64.charm \ + --resource update-db-image=ubuntu:latest +``` diff --git a/installers/charm/osm-update-db-operator/LICENSE b/installers/charm/osm-update-db-operator/LICENSE new file mode 100644 index 00000000..d6456956 --- /dev/null +++ b/installers/charm/osm-update-db-operator/LICENSE @@ -0,0 +1,202 @@ + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/installers/charm/osm-update-db-operator/README.md b/installers/charm/osm-update-db-operator/README.md new file mode 100644 index 00000000..2ee8f6e4 --- /dev/null +++ b/installers/charm/osm-update-db-operator/README.md @@ -0,0 +1,80 @@ + + +# OSM Update DB Operator + +[![code style](https://img.shields.io/badge/code%20style-black-000000.svg)](https://github.com/psf/black/tree/main) + +## Description + +Charm used to update the OSM databases during an OSM upgrade process. To be used you should have an instance of OSM running that you may want to upgrade + +## Usage + +### Deploy the charm (locally) + +```shell +juju add-model update-db +juju deploy osm-update-db-operator --series focal +``` + +Set MongoDB and MySQL URIs: + +```shell +juju config osm-update-db-operator mysql-uri= +juju config osm-update-db-operator mongodb-uri= +``` + +### Updating the databases + +In case we want to update both databases, we need to run the following command: + +```shell +juju run-action osm-update-db-operator/0 update-db current-version= target-version= +# Example: +juju run-action osm-update-db-operator/0 update-db current-version=9 target-version=10 +``` + +In case only you just want to update MongoDB, then we can use a flag 'mongodb-only=True': + +```shell +juju run-action osm-update-db-operator/0 update-db current-version=9 target-version=10 mongodb-only=True +``` + +In case only you just want to update MySQL database, then we can use a flag 'mysql-only=True': + +```shell +juju run-action osm-update-db-operator/0 update-db current-version=9 target-version=10 mysql-only=True +``` + +You can check if the update of the database was properly done checking the result of the command: + +```shell +juju show-action-output +``` + +### Fixes for bugs + +Updates de database to apply the changes needed to fix a bug. You need to specify the bug number. Example: + +```shell +juju run-action osm-update-db-operator/0 apply-patch bug-number=1837 +``` + +## Contributing + +Please see the [Juju SDK docs](https://juju.is/docs/sdk) for guidelines +on enhancements to this charm following best practice guidelines, and +`CONTRIBUTING.md` for developer guidance. diff --git a/installers/charm/osm-update-db-operator/actions.yaml b/installers/charm/osm-update-db-operator/actions.yaml new file mode 100644 index 00000000..aba1ee32 --- /dev/null +++ b/installers/charm/osm-update-db-operator/actions.yaml @@ -0,0 +1,42 @@ +# Copyright 2022 Canonical Ltd. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +update-db: + description: | + Updates the Mongodb and MySQL with the new data needed for the target OSM + params: + current-version: + type: integer + description: "Current version of Charmed OSM - Example: 9" + target-version: + type: integer + description: "Final version of OSM after the update - Example: 10" + mysql-only: + type: boolean + description: "if True the update is only applied for mysql database" + mongodb-only: + type: boolean + description: "if True the update is only applied for mongo database" + required: + - current-version + - target-version +apply-patch: + description: | + Updates de database to apply the changes needed to fix a bug + params: + bug-number: + type: integer + description: "The number of the bug that needs to be fixed" + required: + - bug-number diff --git a/installers/charm/osm-update-db-operator/charmcraft.yaml b/installers/charm/osm-update-db-operator/charmcraft.yaml new file mode 100644 index 00000000..31c233b5 --- /dev/null +++ b/installers/charm/osm-update-db-operator/charmcraft.yaml @@ -0,0 +1,26 @@ +# Copyright 2022 Canonical Ltd. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +type: "charm" +bases: + - build-on: + - name: "ubuntu" + channel: "20.04" + run-on: + - name: "ubuntu" + channel: "20.04" +parts: + charm: + build-packages: + - git diff --git a/installers/charm/osm-update-db-operator/config.yaml b/installers/charm/osm-update-db-operator/config.yaml new file mode 100644 index 00000000..3b7190b5 --- /dev/null +++ b/installers/charm/osm-update-db-operator/config.yaml @@ -0,0 +1,29 @@ +# Copyright 2022 Canonical Ltd. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +options: + log-level: + description: "Log Level" + type: string + default: "INFO" + mongodb-uri: + type: string + description: | + MongoDB URI (external database) + mongodb://:/ + mysql-uri: + type: string + description: | + Mysql URI with the following format: + mysql://:@:/ diff --git a/installers/charm/osm-update-db-operator/metadata.yaml b/installers/charm/osm-update-db-operator/metadata.yaml new file mode 100644 index 00000000..b058591f --- /dev/null +++ b/installers/charm/osm-update-db-operator/metadata.yaml @@ -0,0 +1,19 @@ +# Copyright 2022 Canonical Ltd. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +name: osm-update-db-operator +description: | + Charm to update the OSM databases +summary: | + Charm to update the OSM databases diff --git a/installers/charm/osm-update-db-operator/pyproject.toml b/installers/charm/osm-update-db-operator/pyproject.toml new file mode 100644 index 00000000..3fae1741 --- /dev/null +++ b/installers/charm/osm-update-db-operator/pyproject.toml @@ -0,0 +1,53 @@ +# Copyright 2022 Canonical Ltd. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +# Testing tools configuration +[tool.coverage.run] +branch = true + +[tool.coverage.report] +show_missing = true + +[tool.pytest.ini_options] +minversion = "6.0" +log_cli_level = "INFO" + +# Formatting tools configuration +[tool.black] +line-length = 99 +target-version = ["py38"] + +[tool.isort] +profile = "black" + +# Linting tools configuration +[tool.flake8] +max-line-length = 99 +max-doc-length = 99 +max-complexity = 10 +exclude = [".git", "__pycache__", ".tox", "build", "dist", "*.egg_info", "venv"] +select = ["E", "W", "F", "C", "N", "R", "D", "H"] +# Ignore W503, E501 because using black creates errors with this +# Ignore D107 Missing docstring in __init__ +ignore = ["W503", "E501", "D107"] +# D100, D101, D102, D103: Ignore missing docstrings in tests +per-file-ignores = ["tests/*:D100,D101,D102,D103,D104"] +docstring-convention = "google" +# Check for properly formatted copyright header in each file +copyright-check = "True" +copyright-author = "Canonical Ltd." +copyright-regexp = "Copyright\\s\\d{4}([-,]\\d{4})*\\s+%(author)s" + +[tool.bandit] +tests = ["B201", "B301"] diff --git a/installers/charm/osm-update-db-operator/requirements.txt b/installers/charm/osm-update-db-operator/requirements.txt new file mode 100644 index 00000000..b488dba4 --- /dev/null +++ b/installers/charm/osm-update-db-operator/requirements.txt @@ -0,0 +1,16 @@ +# Copyright 2022 Canonical Ltd. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +ops < 2.2 +pymongo == 3.12.3 diff --git a/installers/charm/osm-update-db-operator/src/charm.py b/installers/charm/osm-update-db-operator/src/charm.py new file mode 100755 index 00000000..32db2f76 --- /dev/null +++ b/installers/charm/osm-update-db-operator/src/charm.py @@ -0,0 +1,119 @@ +#!/usr/bin/env python3 +# Copyright 2022 Canonical Ltd. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +"""Update DB charm module.""" + +import logging + +from ops.charm import CharmBase +from ops.framework import StoredState +from ops.main import main +from ops.model import ActiveStatus, BlockedStatus + +from db_upgrade import MongoUpgrade, MysqlUpgrade + +logger = logging.getLogger(__name__) + + +class UpgradeDBCharm(CharmBase): + """Upgrade DB Charm operator.""" + + _stored = StoredState() + + def __init__(self, *args): + super().__init__(*args) + + # Observe events + event_observe_mapping = { + self.on.update_db_action: self._on_update_db_action, + self.on.apply_patch_action: self._on_apply_patch_action, + self.on.config_changed: self._on_config_changed, + } + for event, observer in event_observe_mapping.items(): + self.framework.observe(event, observer) + + @property + def mongo(self): + """Create MongoUpgrade object if the configuration has been set.""" + mongo_uri = self.config.get("mongodb-uri") + return MongoUpgrade(mongo_uri) if mongo_uri else None + + @property + def mysql(self): + """Create MysqlUpgrade object if the configuration has been set.""" + mysql_uri = self.config.get("mysql-uri") + return MysqlUpgrade(mysql_uri) if mysql_uri else None + + def _on_config_changed(self, _): + mongo_uri = self.config.get("mongodb-uri") + mysql_uri = self.config.get("mysql-uri") + if not mongo_uri and not mysql_uri: + self.unit.status = BlockedStatus("mongodb-uri and/or mysql-uri must be set") + return + self.unit.status = ActiveStatus() + + def _on_update_db_action(self, event): + """Handle the update-db action.""" + current_version = str(event.params["current-version"]) + target_version = str(event.params["target-version"]) + mysql_only = event.params.get("mysql-only") + mongodb_only = event.params.get("mongodb-only") + try: + results = {} + if mysql_only and mongodb_only: + raise Exception("cannot set both mysql-only and mongodb-only options to True") + if mysql_only: + self._upgrade_mysql(current_version, target_version) + results["mysql"] = "Upgraded successfully" + elif mongodb_only: + self._upgrade_mongodb(current_version, target_version) + results["mongodb"] = "Upgraded successfully" + else: + self._upgrade_mysql(current_version, target_version) + results["mysql"] = "Upgraded successfully" + self._upgrade_mongodb(current_version, target_version) + results["mongodb"] = "Upgraded successfully" + event.set_results(results) + except Exception as e: + event.fail(f"Failed DB Upgrade: {e}") + + def _upgrade_mysql(self, current_version, target_version): + logger.debug("Upgrading mysql") + if self.mysql: + self.mysql.upgrade(current_version, target_version) + else: + raise Exception("mysql-uri not set") + + def _upgrade_mongodb(self, current_version, target_version): + logger.debug("Upgrading mongodb") + if self.mongo: + self.mongo.upgrade(current_version, target_version) + else: + raise Exception("mongo-uri not set") + + def _on_apply_patch_action(self, event): + bug_number = event.params["bug-number"] + logger.debug("Patching bug number {}".format(str(bug_number))) + try: + if self.mongo: + self.mongo.apply_patch(bug_number) + else: + raise Exception("mongo-uri not set") + except Exception as e: + event.fail(f"Failed Patch Application: {e}") + + +if __name__ == "__main__": # pragma: no cover + main(UpgradeDBCharm, use_juju_for_storage=True) diff --git a/installers/charm/osm-update-db-operator/src/db_upgrade.py b/installers/charm/osm-update-db-operator/src/db_upgrade.py new file mode 100644 index 00000000..05cc0a0c --- /dev/null +++ b/installers/charm/osm-update-db-operator/src/db_upgrade.py @@ -0,0 +1,275 @@ +# Copyright 2022 Canonical Ltd. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +"""Upgrade DB charm module.""" + +import json +import logging + +from pymongo import MongoClient + +logger = logging.getLogger(__name__) + + +class MongoUpgrade1012: + """Upgrade MongoDB Database from OSM v10 to v12.""" + + @staticmethod + def _remove_namespace_from_k8s(nsrs, nsr): + namespace = "kube-system:" + if nsr["_admin"].get("deployed"): + k8s_list = [] + for k8s in nsr["_admin"]["deployed"].get("K8s"): + if k8s.get("k8scluster-uuid"): + k8s["k8scluster-uuid"] = k8s["k8scluster-uuid"].replace(namespace, "", 1) + k8s_list.append(k8s) + myquery = {"_id": nsr["_id"]} + nsrs.update_one(myquery, {"$set": {"_admin.deployed.K8s": k8s_list}}) + + @staticmethod + def _update_nsr(osm_db): + """Update nsr. + + Add vim_message = None if it does not exist. + Remove "namespace:" from k8scluster-uuid. + """ + if "nsrs" not in osm_db.list_collection_names(): + return + logger.info("Entering in MongoUpgrade1012._update_nsr function") + + nsrs = osm_db["nsrs"] + for nsr in nsrs.find(): + logger.debug(f"Updating {nsr['_id']} nsr") + for key, values in nsr.items(): + if isinstance(values, list): + item_list = [] + for value in values: + if isinstance(value, dict) and value.get("vim_info"): + index = list(value["vim_info"].keys())[0] + if not value["vim_info"][index].get("vim_message"): + value["vim_info"][index]["vim_message"] = None + item_list.append(value) + myquery = {"_id": nsr["_id"]} + nsrs.update_one(myquery, {"$set": {key: item_list}}) + MongoUpgrade1012._remove_namespace_from_k8s(nsrs, nsr) + + @staticmethod + def _update_vnfr(osm_db): + """Update vnfr. + + Add vim_message to vdur if it does not exist. + Copy content of interfaces into interfaces_backup. + """ + if "vnfrs" not in osm_db.list_collection_names(): + return + logger.info("Entering in MongoUpgrade1012._update_vnfr function") + mycol = osm_db["vnfrs"] + for vnfr in mycol.find(): + logger.debug(f"Updating {vnfr['_id']} vnfr") + vdur_list = [] + for vdur in vnfr["vdur"]: + if vdur.get("vim_info"): + index = list(vdur["vim_info"].keys())[0] + if not vdur["vim_info"][index].get("vim_message"): + vdur["vim_info"][index]["vim_message"] = None + if vdur["vim_info"][index].get( + "interfaces", "Not found" + ) != "Not found" and not vdur["vim_info"][index].get("interfaces_backup"): + vdur["vim_info"][index]["interfaces_backup"] = vdur["vim_info"][index][ + "interfaces" + ] + vdur_list.append(vdur) + myquery = {"_id": vnfr["_id"]} + mycol.update_one(myquery, {"$set": {"vdur": vdur_list}}) + + @staticmethod + def _update_k8scluster(osm_db): + """Remove namespace from helm-chart and helm-chart-v3 id.""" + if "k8sclusters" not in osm_db.list_collection_names(): + return + logger.info("Entering in MongoUpgrade1012._update_k8scluster function") + namespace = "kube-system:" + k8sclusters = osm_db["k8sclusters"] + for k8scluster in k8sclusters.find(): + if k8scluster["_admin"].get("helm-chart") and k8scluster["_admin"]["helm-chart"].get( + "id" + ): + if k8scluster["_admin"]["helm-chart"]["id"].startswith(namespace): + k8scluster["_admin"]["helm-chart"]["id"] = k8scluster["_admin"]["helm-chart"][ + "id" + ].replace(namespace, "", 1) + if k8scluster["_admin"].get("helm-chart-v3") and k8scluster["_admin"][ + "helm-chart-v3" + ].get("id"): + if k8scluster["_admin"]["helm-chart-v3"]["id"].startswith(namespace): + k8scluster["_admin"]["helm-chart-v3"]["id"] = k8scluster["_admin"][ + "helm-chart-v3" + ]["id"].replace(namespace, "", 1) + myquery = {"_id": k8scluster["_id"]} + k8sclusters.update_one(myquery, {"$set": k8scluster}) + + @staticmethod + def upgrade(mongo_uri): + """Upgrade nsr, vnfr and k8scluster in DB.""" + logger.info("Entering in MongoUpgrade1012.upgrade function") + myclient = MongoClient(mongo_uri) + osm_db = myclient["osm"] + MongoUpgrade1012._update_nsr(osm_db) + MongoUpgrade1012._update_vnfr(osm_db) + MongoUpgrade1012._update_k8scluster(osm_db) + + +class MongoUpgrade910: + """Upgrade MongoDB Database from OSM v9 to v10.""" + + @staticmethod + def upgrade(mongo_uri): + """Add parameter alarm status = OK if not found in alarms collection.""" + myclient = MongoClient(mongo_uri) + osm_db = myclient["osm"] + collist = osm_db.list_collection_names() + + if "alarms" in collist: + mycol = osm_db["alarms"] + for x in mycol.find(): + if not x.get("alarm_status"): + myquery = {"_id": x["_id"]} + mycol.update_one(myquery, {"$set": {"alarm_status": "ok"}}) + + +class MongoPatch1837: + """Patch Bug 1837 on MongoDB.""" + + @staticmethod + def _update_nslcmops_params(osm_db): + """Updates the nslcmops collection to change the additional params to a string.""" + logger.info("Entering in MongoPatch1837._update_nslcmops_params function") + if "nslcmops" in osm_db.list_collection_names(): + nslcmops = osm_db["nslcmops"] + for nslcmop in nslcmops.find(): + if nslcmop.get("operationParams"): + if nslcmop["operationParams"].get("additionalParamsForVnf") and isinstance( + nslcmop["operationParams"].get("additionalParamsForVnf"), list + ): + string_param = json.dumps( + nslcmop["operationParams"]["additionalParamsForVnf"] + ) + myquery = {"_id": nslcmop["_id"]} + nslcmops.update_one( + myquery, + { + "$set": { + "operationParams": {"additionalParamsForVnf": string_param} + } + }, + ) + elif nslcmop["operationParams"].get("primitive_params") and isinstance( + nslcmop["operationParams"].get("primitive_params"), dict + ): + string_param = json.dumps(nslcmop["operationParams"]["primitive_params"]) + myquery = {"_id": nslcmop["_id"]} + nslcmops.update_one( + myquery, + {"$set": {"operationParams": {"primitive_params": string_param}}}, + ) + + @staticmethod + def _update_vnfrs_params(osm_db): + """Updates the vnfrs collection to change the additional params to a string.""" + logger.info("Entering in MongoPatch1837._update_vnfrs_params function") + if "vnfrs" in osm_db.list_collection_names(): + mycol = osm_db["vnfrs"] + for vnfr in mycol.find(): + if vnfr.get("kdur"): + kdur_list = [] + for kdur in vnfr["kdur"]: + if kdur.get("additionalParams") and not isinstance( + kdur["additionalParams"], str + ): + kdur["additionalParams"] = json.dumps(kdur["additionalParams"]) + kdur_list.append(kdur) + myquery = {"_id": vnfr["_id"]} + mycol.update_one( + myquery, + {"$set": {"kdur": kdur_list}}, + ) + vnfr["kdur"] = kdur_list + + @staticmethod + def patch(mongo_uri): + """Updates the database to change the additional params from dict to a string.""" + logger.info("Entering in MongoPatch1837.patch function") + myclient = MongoClient(mongo_uri) + osm_db = myclient["osm"] + MongoPatch1837._update_nslcmops_params(osm_db) + MongoPatch1837._update_vnfrs_params(osm_db) + + +MONGODB_UPGRADE_FUNCTIONS = { + "9": {"10": [MongoUpgrade910.upgrade]}, + "10": {"12": [MongoUpgrade1012.upgrade]}, +} +MYSQL_UPGRADE_FUNCTIONS = {} +BUG_FIXES = { + 1837: MongoPatch1837.patch, +} + + +class MongoUpgrade: + """Upgrade MongoDB Database.""" + + def __init__(self, mongo_uri): + self.mongo_uri = mongo_uri + + def upgrade(self, current, target): + """Validates the upgrading path and upgrades the DB.""" + self._validate_upgrade(current, target) + for function in MONGODB_UPGRADE_FUNCTIONS.get(current)[target]: + function(self.mongo_uri) + + def _validate_upgrade(self, current, target): + """Check if the upgrade path chosen is possible.""" + logger.info("Validating the upgrade path") + if current not in MONGODB_UPGRADE_FUNCTIONS: + raise Exception(f"cannot upgrade from {current} version.") + if target not in MONGODB_UPGRADE_FUNCTIONS[current]: + raise Exception(f"cannot upgrade from version {current} to {target}.") + + def apply_patch(self, bug_number: int) -> None: + """Checks the bug-number and applies the fix in the database.""" + if bug_number not in BUG_FIXES: + raise Exception(f"There is no patch for bug {bug_number}") + patch_function = BUG_FIXES[bug_number] + patch_function(self.mongo_uri) + + +class MysqlUpgrade: + """Upgrade Mysql Database.""" + + def __init__(self, mysql_uri): + self.mysql_uri = mysql_uri + + def upgrade(self, current, target): + """Validates the upgrading path and upgrades the DB.""" + self._validate_upgrade(current, target) + for function in MYSQL_UPGRADE_FUNCTIONS[current][target]: + function(self.mysql_uri) + + def _validate_upgrade(self, current, target): + """Check if the upgrade path chosen is possible.""" + logger.info("Validating the upgrade path") + if current not in MYSQL_UPGRADE_FUNCTIONS: + raise Exception(f"cannot upgrade from {current} version.") + if target not in MYSQL_UPGRADE_FUNCTIONS[current]: + raise Exception(f"cannot upgrade from version {current} to {target}.") diff --git a/installers/charm/osm-update-db-operator/tests/integration/test_charm.py b/installers/charm/osm-update-db-operator/tests/integration/test_charm.py new file mode 100644 index 00000000..cc9e0be2 --- /dev/null +++ b/installers/charm/osm-update-db-operator/tests/integration/test_charm.py @@ -0,0 +1,48 @@ +# Copyright 2022 Canonical Ltd. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import base64 +import logging +from pathlib import Path + +import pytest +import yaml +from pytest_operator.plugin import OpsTest + +logger = logging.getLogger(__name__) + +METADATA = yaml.safe_load(Path("./metadata.yaml").read_text()) + + +@pytest.mark.abort_on_fail +async def test_build_and_deploy(ops_test: OpsTest): + """Build the charm-under-test and deploy it together with related charms. + + Assert on the unit status before any relations/configurations take place. + """ + await ops_test.model.set_config({"update-status-hook-interval": "10s"}) + # build and deploy charm from local source folder + charm = await ops_test.build_charm(".") + resources = { + "update-db-image": METADATA["resources"]["update-db-image"]["upstream-source"], + } + await ops_test.model.deploy(charm, resources=resources, application_name="update-db") + await ops_test.model.wait_for_idle(apps=["update-db"], status="active", timeout=1000) + assert ops_test.model.applications["update-db"].units[0].workload_status == "active" + + await ops_test.model.set_config({"update-status-hook-interval": "60m"}) + + +def base64_encode(phrase: str) -> str: + return base64.b64encode(phrase.encode("utf-8")).decode("utf-8") diff --git a/installers/charm/osm-update-db-operator/tests/unit/test_charm.py b/installers/charm/osm-update-db-operator/tests/unit/test_charm.py new file mode 100644 index 00000000..a0f625db --- /dev/null +++ b/installers/charm/osm-update-db-operator/tests/unit/test_charm.py @@ -0,0 +1,165 @@ +# Copyright 2022 Canonical Ltd. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import unittest +from unittest.mock import Mock, patch + +from ops.model import ActiveStatus, BlockedStatus, MaintenanceStatus +from ops.testing import Harness + +from charm import UpgradeDBCharm + + +class TestCharm(unittest.TestCase): + def setUp(self): + self.harness = Harness(UpgradeDBCharm) + self.addCleanup(self.harness.cleanup) + self.harness.begin() + + def test_initial_config(self): + self.assertEqual(self.harness.model.unit.status, MaintenanceStatus("")) + + def test_config_changed(self): + self.harness.update_config({"mongodb-uri": "foo"}) + self.assertEqual(self.harness.model.unit.status, ActiveStatus()) + + def test_config_changed_blocked(self): + self.harness.update_config({"log-level": "DEBUG"}) + self.assertEqual( + self.harness.model.unit.status, + BlockedStatus("mongodb-uri and/or mysql-uri must be set"), + ) + + def test_update_db_fail_only_params(self): + action_event = Mock( + params={ + "current-version": 9, + "target-version": 10, + "mysql-only": True, + "mongodb-only": True, + } + ) + self.harness.charm._on_update_db_action(action_event) + self.assertEqual( + action_event.fail.call_args, + [("Failed DB Upgrade: cannot set both mysql-only and mongodb-only options to True",)], + ) + + @patch("charm.MongoUpgrade") + @patch("charm.MysqlUpgrade") + def test_update_db_mysql(self, mock_mysql_upgrade, mock_mongo_upgrade): + self.harness.update_config({"mysql-uri": "foo"}) + action_event = Mock( + params={ + "current-version": 9, + "target-version": 10, + "mysql-only": True, + "mongodb-only": False, + } + ) + self.harness.charm._on_update_db_action(action_event) + mock_mysql_upgrade().upgrade.assert_called_once() + mock_mongo_upgrade.assert_not_called() + + @patch("charm.MongoUpgrade") + @patch("charm.MysqlUpgrade") + def test_update_db_mongo(self, mock_mysql_upgrade, mock_mongo_upgrade): + self.harness.update_config({"mongodb-uri": "foo"}) + action_event = Mock( + params={ + "current-version": 7, + "target-version": 10, + "mysql-only": False, + "mongodb-only": True, + } + ) + self.harness.charm._on_update_db_action(action_event) + mock_mongo_upgrade().upgrade.assert_called_once() + mock_mysql_upgrade.assert_not_called() + + @patch("charm.MongoUpgrade") + def test_update_db_not_configured_mongo_fail(self, mock_mongo_upgrade): + action_event = Mock( + params={ + "current-version": 7, + "target-version": 10, + "mysql-only": False, + "mongodb-only": True, + } + ) + self.harness.charm._on_update_db_action(action_event) + mock_mongo_upgrade.assert_not_called() + self.assertEqual( + action_event.fail.call_args, + [("Failed DB Upgrade: mongo-uri not set",)], + ) + + @patch("charm.MysqlUpgrade") + def test_update_db_not_configured_mysql_fail(self, mock_mysql_upgrade): + action_event = Mock( + params={ + "current-version": 7, + "target-version": 10, + "mysql-only": True, + "mongodb-only": False, + } + ) + self.harness.charm._on_update_db_action(action_event) + mock_mysql_upgrade.assert_not_called() + self.assertEqual( + action_event.fail.call_args, + [("Failed DB Upgrade: mysql-uri not set",)], + ) + + @patch("charm.MongoUpgrade") + @patch("charm.MysqlUpgrade") + def test_update_db_mongodb_and_mysql(self, mock_mysql_upgrade, mock_mongo_upgrade): + self.harness.update_config({"mongodb-uri": "foo"}) + self.harness.update_config({"mysql-uri": "foo"}) + action_event = Mock( + params={ + "current-version": 7, + "target-version": 10, + "mysql-only": False, + "mongodb-only": False, + } + ) + self.harness.charm._on_update_db_action(action_event) + mock_mysql_upgrade().upgrade.assert_called_once() + mock_mongo_upgrade().upgrade.assert_called_once() + + @patch("charm.MongoUpgrade") + def test_apply_patch(self, mock_mongo_upgrade): + self.harness.update_config({"mongodb-uri": "foo"}) + action_event = Mock( + params={ + "bug-number": 57, + } + ) + self.harness.charm._on_apply_patch_action(action_event) + mock_mongo_upgrade().apply_patch.assert_called_once() + + @patch("charm.MongoUpgrade") + def test_apply_patch_fail(self, mock_mongo_upgrade): + action_event = Mock( + params={ + "bug-number": 57, + } + ) + self.harness.charm._on_apply_patch_action(action_event) + mock_mongo_upgrade.assert_not_called() + self.assertEqual( + action_event.fail.call_args, + [("Failed Patch Application: mongo-uri not set",)], + ) diff --git a/installers/charm/osm-update-db-operator/tests/unit/test_db_upgrade.py b/installers/charm/osm-update-db-operator/tests/unit/test_db_upgrade.py new file mode 100644 index 00000000..50affdd2 --- /dev/null +++ b/installers/charm/osm-update-db-operator/tests/unit/test_db_upgrade.py @@ -0,0 +1,413 @@ +# Copyright 2022 Canonical Ltd. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import logging +import unittest +from unittest.mock import MagicMock, Mock, call, patch + +import db_upgrade +from db_upgrade import ( + MongoPatch1837, + MongoUpgrade, + MongoUpgrade910, + MongoUpgrade1012, + MysqlUpgrade, +) + +logger = logging.getLogger(__name__) + + +class TestUpgradeMongo910(unittest.TestCase): + @patch("db_upgrade.MongoClient") + def test_upgrade_mongo_9_10(self, mock_mongo_client): + mock_db = MagicMock() + alarms = Mock() + alarms.find.return_value = [{"_id": "1", "alarm_status": "1"}] + collection_dict = {"alarms": alarms, "other": {}} + mock_db.list_collection_names.return_value = collection_dict + mock_db.__getitem__.side_effect = collection_dict.__getitem__ + mock_mongo_client.return_value = {"osm": mock_db} + MongoUpgrade910.upgrade("mongo_uri") + alarms.update_one.assert_not_called() + + @patch("db_upgrade.MongoClient") + def test_upgrade_mongo_9_10_no_alarms(self, mock_mongo_client): + mock_db = Mock() + mock_db.__getitem__ = Mock() + + mock_db.list_collection_names.return_value = {"other": {}} + mock_db.alarms.return_value = None + mock_mongo_client.return_value = {"osm": mock_db} + self.assertIsNone(MongoUpgrade910.upgrade("mongo_uri")) + + @patch("db_upgrade.MongoClient") + def test_upgrade_mongo_9_10_no_alarm_status(self, mock_mongo_client): + mock_db = MagicMock() + alarms = Mock() + alarms.find.return_value = [{"_id": "1"}] + collection_dict = {"alarms": alarms, "other": {}} + mock_db.list_collection_names.return_value = collection_dict + mock_db.__getitem__.side_effect = collection_dict.__getitem__ + mock_db.alarms.return_value = alarms + mock_mongo_client.return_value = {"osm": mock_db} + MongoUpgrade910.upgrade("mongo_uri") + alarms.update_one.assert_called_once_with({"_id": "1"}, {"$set": {"alarm_status": "ok"}}) + + +class TestUpgradeMongo1012(unittest.TestCase): + def setUp(self): + self.mock_db = MagicMock() + self.nsrs = Mock() + self.vnfrs = Mock() + self.k8s_clusters = Mock() + + @patch("db_upgrade.MongoClient") + def test_update_nsr_empty_nsrs(self, mock_mongo_client): + self.nsrs.find.return_value = [] + collection_list = {"nsrs": self.nsrs} + self.mock_db.__getitem__.side_effect = collection_list.__getitem__ + self.mock_db.list_collection_names.return_value = collection_list + mock_mongo_client.return_value = {"osm": self.mock_db} + MongoUpgrade1012.upgrade("mongo_uri") + + @patch("db_upgrade.MongoClient") + def test_update_nsr_empty_nsr(self, mock_mongo_client): + nsr = MagicMock() + nsr_values = {"_id": "2", "_admin": {}} + nsr.__getitem__.side_effect = nsr_values.__getitem__ + nsr.items.return_value = [] + self.nsrs.find.return_value = [nsr] + collection_list = {"nsrs": self.nsrs} + self.mock_db.__getitem__.side_effect = collection_list.__getitem__ + self.mock_db.list_collection_names.return_value = collection_list + mock_mongo_client.return_value = {"osm": self.mock_db} + MongoUpgrade1012.upgrade("mongo_uri") + + @patch("db_upgrade.MongoClient") + def test_update_nsr_add_vim_message(self, mock_mongo_client): + nsr = MagicMock() + vim_info1 = {"vim_info_key1": {}} + vim_info2 = {"vim_info_key2": {"vim_message": "Hello"}} + nsr_items = {"nsr_item_key": [{"vim_info": vim_info1}, {"vim_info": vim_info2}]} + nsr_values = {"_id": "2", "_admin": {}} + nsr.__getitem__.side_effect = nsr_values.__getitem__ + nsr.items.return_value = nsr_items.items() + self.nsrs.find.return_value = [nsr] + collection_list = {"nsrs": self.nsrs} + self.mock_db.__getitem__.side_effect = collection_list.__getitem__ + self.mock_db.list_collection_names.return_value = collection_list + mock_mongo_client.return_value = {"osm": self.mock_db} + MongoUpgrade1012.upgrade("mongo_uri") + expected_vim_info = {"vim_info_key1": {"vim_message": None}} + expected_vim_info2 = {"vim_info_key2": {"vim_message": "Hello"}} + self.assertEqual(vim_info1, expected_vim_info) + self.assertEqual(vim_info2, expected_vim_info2) + self.nsrs.update_one.assert_called_once_with({"_id": "2"}, {"$set": nsr_items}) + + @patch("db_upgrade.MongoClient") + def test_update_nsr_admin(self, mock_mongo_client): + nsr = MagicMock() + k8s = [{"k8scluster-uuid": "namespace"}, {"k8scluster-uuid": "kube-system:k8s"}] + admin = {"deployed": {"K8s": k8s}} + nsr_values = {"_id": "2", "_admin": admin} + nsr.__getitem__.side_effect = nsr_values.__getitem__ + nsr_items = {} + nsr.items.return_value = nsr_items.items() + self.nsrs.find.return_value = [nsr] + collection_list = {"nsrs": self.nsrs} + self.mock_db.__getitem__.side_effect = collection_list.__getitem__ + self.mock_db.list_collection_names.return_value = collection_list + mock_mongo_client.return_value = {"osm": self.mock_db} + MongoUpgrade1012.upgrade("mongo_uri") + expected_k8s = [{"k8scluster-uuid": "namespace"}, {"k8scluster-uuid": "k8s"}] + self.nsrs.update_one.assert_called_once_with( + {"_id": "2"}, {"$set": {"_admin.deployed.K8s": expected_k8s}} + ) + + @patch("db_upgrade.MongoClient") + def test_update_vnfr_empty_vnfrs(self, mock_mongo_client): + self.vnfrs.find.return_value = [{"_id": "10", "vdur": []}] + collection_list = {"vnfrs": self.vnfrs} + self.mock_db.__getitem__.side_effect = collection_list.__getitem__ + self.mock_db.list_collection_names.return_value = collection_list + mock_mongo_client.return_value = {"osm": self.mock_db} + MongoUpgrade1012.upgrade("mongo_uri") + self.vnfrs.update_one.assert_called_once_with({"_id": "10"}, {"$set": {"vdur": []}}) + + @patch("db_upgrade.MongoClient") + def test_update_vnfr_no_vim_info(self, mock_mongo_client): + vdur = {"other": {}} + vnfr = {"_id": "10", "vdur": [vdur]} + self.vnfrs.find.return_value = [vnfr] + collection_list = {"vnfrs": self.vnfrs} + self.mock_db.__getitem__.side_effect = collection_list.__getitem__ + self.mock_db.list_collection_names.return_value = collection_list + mock_mongo_client.return_value = {"osm": self.mock_db} + MongoUpgrade1012.upgrade("mongo_uri") + self.assertEqual(vdur, {"other": {}}) + self.vnfrs.update_one.assert_called_once_with({"_id": "10"}, {"$set": {"vdur": [vdur]}}) + + @patch("db_upgrade.MongoClient") + def test_update_vnfr_vim_message_not_conditions_matched(self, mock_mongo_client): + vim_info = {"vim_message": "HelloWorld"} + vim_infos = {"key1": vim_info, "key2": "value2"} + vdur = {"vim_info": vim_infos, "other": {}} + vnfr = {"_id": "10", "vdur": [vdur]} + self.vnfrs.find.return_value = [vnfr] + collection_list = {"vnfrs": self.vnfrs} + self.mock_db.__getitem__.side_effect = collection_list.__getitem__ + self.mock_db.list_collection_names.return_value = collection_list + mock_mongo_client.return_value = {"osm": self.mock_db} + MongoUpgrade1012.upgrade("mongo_uri") + expected_vim_info = {"vim_message": "HelloWorld"} + self.assertEqual(vim_info, expected_vim_info) + self.vnfrs.update_one.assert_called_once_with({"_id": "10"}, {"$set": {"vdur": [vdur]}}) + + @patch("db_upgrade.MongoClient") + def test_update_vnfr_vim_message_is_missing(self, mock_mongo_client): + vim_info = {"interfaces_backup": "HelloWorld"} + vim_infos = {"key1": vim_info, "key2": "value2"} + vdur = {"vim_info": vim_infos, "other": {}} + vnfr = {"_id": "10", "vdur": [vdur]} + self.vnfrs.find.return_value = [vnfr] + collection_list = {"vnfrs": self.vnfrs} + self.mock_db.__getitem__.side_effect = collection_list.__getitem__ + self.mock_db.list_collection_names.return_value = collection_list + mock_mongo_client.return_value = {"osm": self.mock_db} + MongoUpgrade1012.upgrade("mongo_uri") + expected_vim_info = {"vim_message": None, "interfaces_backup": "HelloWorld"} + self.assertEqual(vim_info, expected_vim_info) + self.vnfrs.update_one.assert_called_once_with({"_id": "10"}, {"$set": {"vdur": [vdur]}}) + + @patch("db_upgrade.MongoClient") + def test_update_vnfr_interfaces_backup_is_updated(self, mock_mongo_client): + vim_info = {"interfaces": "HelloWorld", "vim_message": "ByeWorld"} + vim_infos = {"key1": vim_info, "key2": "value2"} + vdur = {"vim_info": vim_infos, "other": {}} + vnfr = {"_id": "10", "vdur": [vdur]} + self.vnfrs.find.return_value = [vnfr] + collection_list = {"vnfrs": self.vnfrs} + self.mock_db.__getitem__.side_effect = collection_list.__getitem__ + self.mock_db.list_collection_names.return_value = collection_list + mock_mongo_client.return_value = {"osm": self.mock_db} + MongoUpgrade1012.upgrade("mongo_uri") + expected_vim_info = { + "interfaces": "HelloWorld", + "vim_message": "ByeWorld", + "interfaces_backup": "HelloWorld", + } + self.assertEqual(vim_info, expected_vim_info) + self.vnfrs.update_one.assert_called_once_with({"_id": "10"}, {"$set": {"vdur": [vdur]}}) + + @patch("db_upgrade.MongoClient") + def test_update_k8scluster_empty_k8scluster(self, mock_mongo_client): + self.k8s_clusters.find.return_value = [] + collection_list = {"k8sclusters": self.k8s_clusters} + self.mock_db.__getitem__.side_effect = collection_list.__getitem__ + self.mock_db.list_collection_names.return_value = collection_list + mock_mongo_client.return_value = {"osm": self.mock_db} + MongoUpgrade1012.upgrade("mongo_uri") + + @patch("db_upgrade.MongoClient") + def test_update_k8scluster_replace_namespace_in_helm_chart(self, mock_mongo_client): + helm_chart = {"id": "kube-system:Hello", "other": {}} + k8s_cluster = {"_id": "8", "_admin": {"helm-chart": helm_chart}} + self.k8s_clusters.find.return_value = [k8s_cluster] + collection_list = {"k8sclusters": self.k8s_clusters} + self.mock_db.__getitem__.side_effect = collection_list.__getitem__ + self.mock_db.list_collection_names.return_value = collection_list + mock_mongo_client.return_value = {"osm": self.mock_db} + MongoUpgrade1012.upgrade("mongo_uri") + expected_helm_chart = {"id": "Hello", "other": {}} + expected_k8s_cluster = {"_id": "8", "_admin": {"helm-chart": expected_helm_chart}} + self.k8s_clusters.update_one.assert_called_once_with( + {"_id": "8"}, {"$set": expected_k8s_cluster} + ) + + @patch("db_upgrade.MongoClient") + def test_update_k8scluster_replace_namespace_in_helm_chart_v3(self, mock_mongo_client): + helm_chart_v3 = {"id": "kube-system:Hello", "other": {}} + k8s_cluster = {"_id": "8", "_admin": {"helm-chart-v3": helm_chart_v3}} + self.k8s_clusters.find.return_value = [k8s_cluster] + collection_list = {"k8sclusters": self.k8s_clusters} + self.mock_db.__getitem__.side_effect = collection_list.__getitem__ + self.mock_db.list_collection_names.return_value = collection_list + mock_mongo_client.return_value = {"osm": self.mock_db} + MongoUpgrade1012.upgrade("mongo_uri") + expected_helm_chart_v3 = {"id": "Hello", "other": {}} + expected_k8s_cluster = {"_id": "8", "_admin": {"helm-chart-v3": expected_helm_chart_v3}} + self.k8s_clusters.update_one.assert_called_once_with( + {"_id": "8"}, {"$set": expected_k8s_cluster} + ) + + +class TestPatch1837(unittest.TestCase): + def setUp(self): + self.mock_db = MagicMock() + self.vnfrs = Mock() + self.nslcmops = Mock() + + @patch("db_upgrade.MongoClient") + def test_update_vnfrs_params_no_vnfrs_or_nslcmops(self, mock_mongo_client): + collection_dict = {"other": {}} + self.mock_db.list_collection_names.return_value = collection_dict + mock_mongo_client.return_value = {"osm": self.mock_db} + MongoPatch1837.patch("mongo_uri") + + @patch("db_upgrade.MongoClient") + def test_update_vnfrs_params_no_kdur(self, mock_mongo_client): + self.vnfrs.find.return_value = {"_id": "1"} + collection_dict = {"vnfrs": self.vnfrs, "other": {}} + self.mock_db.list_collection_names.return_value = collection_dict + mock_mongo_client.return_value = {"osm": self.mock_db} + MongoPatch1837.patch("mongo_uri") + + @patch("db_upgrade.MongoClient") + def test_update_vnfrs_params_kdur_without_additional_params(self, mock_mongo_client): + kdur = [{"other": {}}] + self.vnfrs.find.return_value = [{"_id": "1", "kdur": kdur}] + collection_dict = {"vnfrs": self.vnfrs, "other": {}} + self.mock_db.list_collection_names.return_value = collection_dict + self.mock_db.__getitem__.side_effect = collection_dict.__getitem__ + mock_mongo_client.return_value = {"osm": self.mock_db} + MongoPatch1837.patch("mongo_uri") + self.vnfrs.update_one.assert_called_once_with({"_id": "1"}, {"$set": {"kdur": kdur}}) + + @patch("db_upgrade.MongoClient") + def test_update_vnfrs_params_kdur_two_additional_params(self, mock_mongo_client): + kdur1 = {"additionalParams": "additional_params", "other": {}} + kdur2 = {"additionalParams": 4, "other": {}} + kdur = [kdur1, kdur2] + self.vnfrs.find.return_value = [{"_id": "1", "kdur": kdur}] + collection_dict = {"vnfrs": self.vnfrs, "other": {}} + self.mock_db.list_collection_names.return_value = collection_dict + self.mock_db.__getitem__.side_effect = collection_dict.__getitem__ + mock_mongo_client.return_value = {"osm": self.mock_db} + MongoPatch1837.patch("mongo_uri") + self.vnfrs.update_one.assert_called_once_with( + {"_id": "1"}, {"$set": {"kdur": [kdur1, {"additionalParams": "4", "other": {}}]}} + ) + + @patch("db_upgrade.MongoClient") + def test_update_nslcmops_params_no_nslcmops(self, mock_mongo_client): + self.nslcmops.find.return_value = [] + collection_dict = {"nslcmops": self.nslcmops, "other": {}} + self.mock_db.list_collection_names.return_value = collection_dict + self.mock_db.__getitem__.side_effect = collection_dict.__getitem__ + mock_mongo_client.return_value = {"osm": self.mock_db} + MongoPatch1837.patch("mongo_uri") + + @patch("db_upgrade.MongoClient") + def test_update_nslcmops_additional_params(self, mock_mongo_client): + operation_params_list = {"additionalParamsForVnf": [1, 2, 3]} + operation_params_dict = {"primitive_params": {"dict_key": 5}} + nslcmops1 = {"_id": "1", "other": {}} + nslcmops2 = {"_id": "2", "operationParams": operation_params_list, "other": {}} + nslcmops3 = {"_id": "3", "operationParams": operation_params_dict, "other": {}} + self.nslcmops.find.return_value = [nslcmops1, nslcmops2, nslcmops3] + collection_dict = {"nslcmops": self.nslcmops, "other": {}} + self.mock_db.list_collection_names.return_value = collection_dict + self.mock_db.__getitem__.side_effect = collection_dict.__getitem__ + mock_mongo_client.return_value = {"osm": self.mock_db} + MongoPatch1837.patch("mongo_uri") + call1 = call( + {"_id": "2"}, {"$set": {"operationParams": {"additionalParamsForVnf": "[1, 2, 3]"}}} + ) + call2 = call( + {"_id": "3"}, {"$set": {"operationParams": {"primitive_params": '{"dict_key": 5}'}}} + ) + expected_calls = [call1, call2] + self.nslcmops.update_one.assert_has_calls(expected_calls) + + +class TestMongoUpgrade(unittest.TestCase): + def setUp(self): + self.mongo = MongoUpgrade("http://fake_mongo:27017") + self.upgrade_function = Mock() + self.patch_function = Mock() + db_upgrade.MONGODB_UPGRADE_FUNCTIONS = {"9": {"10": [self.upgrade_function]}} + db_upgrade.BUG_FIXES = {1837: self.patch_function} + + def test_validate_upgrade_fail_target(self): + valid_current = "9" + invalid_target = "7" + with self.assertRaises(Exception) as context: + self.mongo._validate_upgrade(valid_current, invalid_target) + self.assertEqual("cannot upgrade from version 9 to 7.", str(context.exception)) + + def test_validate_upgrade_fail_current(self): + invalid_current = "7" + invalid_target = "8" + with self.assertRaises(Exception) as context: + self.mongo._validate_upgrade(invalid_current, invalid_target) + self.assertEqual("cannot upgrade from 7 version.", str(context.exception)) + + def test_validate_upgrade_pass(self): + valid_current = "9" + valid_target = "10" + self.assertIsNone(self.mongo._validate_upgrade(valid_current, valid_target)) + + @patch("db_upgrade.MongoUpgrade._validate_upgrade") + def test_update_mongo_success(self, mock_validate): + valid_current = "9" + valid_target = "10" + mock_validate.return_value = "" + self.mongo.upgrade(valid_current, valid_target) + self.upgrade_function.assert_called_once() + + def test_validate_apply_patch(self): + bug_number = 1837 + self.mongo.apply_patch(bug_number) + self.patch_function.assert_called_once() + + def test_validate_apply_patch_invalid_bug_fail(self): + bug_number = 2 + with self.assertRaises(Exception) as context: + self.mongo.apply_patch(bug_number) + self.assertEqual("There is no patch for bug 2", str(context.exception)) + self.patch_function.assert_not_called() + + +class TestMysqlUpgrade(unittest.TestCase): + def setUp(self): + self.mysql = MysqlUpgrade("mysql://fake_mysql:23023") + self.upgrade_function = Mock() + db_upgrade.MYSQL_UPGRADE_FUNCTIONS = {"9": {"10": [self.upgrade_function]}} + + def test_validate_upgrade_mysql_fail_current(self): + invalid_current = "7" + invalid_target = "8" + with self.assertRaises(Exception) as context: + self.mysql._validate_upgrade(invalid_current, invalid_target) + self.assertEqual("cannot upgrade from 7 version.", str(context.exception)) + + def test_validate_upgrade_mysql_fail_target(self): + valid_current = "9" + invalid_target = "7" + with self.assertRaises(Exception) as context: + self.mysql._validate_upgrade(valid_current, invalid_target) + self.assertEqual("cannot upgrade from version 9 to 7.", str(context.exception)) + + def test_validate_upgrade_mysql_success(self): + valid_current = "9" + valid_target = "10" + self.assertIsNone(self.mysql._validate_upgrade(valid_current, valid_target)) + + @patch("db_upgrade.MysqlUpgrade._validate_upgrade") + def test_upgrade_mysql_success(self, mock_validate): + valid_current = "9" + valid_target = "10" + mock_validate.return_value = "" + self.mysql.upgrade(valid_current, valid_target) + self.upgrade_function.assert_called_once() diff --git a/installers/charm/osm-update-db-operator/tox.ini b/installers/charm/osm-update-db-operator/tox.ini new file mode 100644 index 00000000..bcf628a8 --- /dev/null +++ b/installers/charm/osm-update-db-operator/tox.ini @@ -0,0 +1,104 @@ +# Copyright 2022 Canonical Ltd. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +[tox] +skipsdist=True +skip_missing_interpreters = True +envlist = lint, unit + +[vars] +src_path = {toxinidir}/src/ +tst_path = {toxinidir}/tests/ +;lib_path = {toxinidir}/lib/charms/ +all_path = {[vars]src_path} {[vars]tst_path} + +[testenv] +basepython = python3 +setenv = + PYTHONPATH = {toxinidir}:{toxinidir}/lib:{[vars]src_path} + PYTHONBREAKPOINT=ipdb.set_trace +passenv = + PYTHONPATH + HOME + PATH + CHARM_BUILD_DIR + MODEL_SETTINGS + HTTP_PROXY + HTTPS_PROXY + NO_PROXY + +[testenv:fmt] +description = Apply coding style standards to code +deps = + black + isort +commands = + isort {[vars]all_path} + black {[vars]all_path} + +[testenv:lint] +description = Check code against coding style standards +deps = + black + flake8>= 4.0.0, < 5.0.0 + flake8-docstrings + flake8-copyright + flake8-builtins + # prospector[with_everything] + pylint + pyproject-flake8 + pep8-naming + isort + codespell + yamllint + -r{toxinidir}/requirements.txt +commands = + codespell {toxinidir}/*.yaml {toxinidir}/*.ini {toxinidir}/*.md \ + {toxinidir}/*.toml {toxinidir}/*.txt {toxinidir}/.github + # prospector -A -F -T + pylint -E {[vars]src_path} + yamllint -d '\{extends: default, ignore: "build\n.tox" \}' . + # pflake8 wrapper supports config from pyproject.toml + pflake8 {[vars]all_path} + isort --check-only --diff {[vars]all_path} + black --check --diff {[vars]all_path} + +[testenv:unit] +description = Run unit tests +deps = + pytest + pytest-mock + pytest-cov + coverage[toml] + -r{toxinidir}/requirements.txt +commands = + pytest --ignore={[vars]tst_path}integration --cov={[vars]src_path} --cov-report=xml + coverage report + +[testenv:security] +description = Run security tests +deps = + bandit + safety +commands = + bandit -r {[vars]src_path} + - safety check + +[testenv:integration] +description = Run integration tests +deps = + pytest + pytest-operator +commands = + pytest -v --tb native --ignore={[vars]tst_path}unit --log-cli-level=INFO -s {posargs} diff --git a/jenkins/ci-pipelines/ci_stage_2.groovy b/jenkins/ci-pipelines/ci_stage_2.groovy index 0691dcc8..c3880b02 100644 --- a/jenkins/ci-pipelines/ci_stage_2.groovy +++ b/jenkins/ci-pipelines/ci_stage_2.groovy @@ -139,6 +139,7 @@ def ci_pipeline(mdg,url_prefix,project,branch,refspec,revision,do_stage_3,artifa 'installers/charm/osm-ng-ui', 'installers/charm/osm-pol', 'installers/charm/osm-ro', + 'installers/charm/osm-update-db-operator', 'installers/charm/prometheus', 'installers/charm/vca-integrator-operator', ] diff --git a/jenkins/ci-pipelines/ci_stage_3.groovy b/jenkins/ci-pipelines/ci_stage_3.groovy index fb5120df..e0cddeaf 100644 --- a/jenkins/ci-pipelines/ci_stage_3.groovy +++ b/jenkins/ci-pipelines/ci_stage_3.groovy @@ -690,6 +690,7 @@ EOF""" 'osm-pol', 'osm-ro', 'osm-prometheus', + 'osm-update-db-operator', 'osm-vca-integrator', ] for (charm in charms) { -- 2.25.1