From 10be7c984475707be716708dfdd2d91a655158dc Mon Sep 17 00:00:00 2001 From: Benjamin Diaz Date: Tue, 5 Feb 2019 17:44:42 -0300 Subject: [PATCH] Adds migration engine for peewee ORM bug 590 Change-Id: If6752715d455cbeb3dc2633e9b62f79360fef767 Signed-off-by: Benjamin Diaz --- debian/python3-osm-policy-module.postinst | 4 +- docker/Dockerfile | 2 +- osm_policy_module/cmd/policy_module_agent.py | 4 +- osm_policy_module/core/agent.py | 2 +- osm_policy_module/core/database.py | 11 +- osm_policy_module/migrations/001_initial.py | 120 ++++++++++++++++++ osm_policy_module/migrations/__init__.py | 23 ++++ osm_policy_module/migrations/conf.py | 26 ++++ .../tests/integration/test_policy_agent.py | 14 +- requirements.txt | 8 +- setup.py | 4 +- 11 files changed, 194 insertions(+), 24 deletions(-) create mode 100644 osm_policy_module/migrations/001_initial.py create mode 100644 osm_policy_module/migrations/__init__.py create mode 100644 osm_policy_module/migrations/conf.py diff --git a/debian/python3-osm-policy-module.postinst b/debian/python3-osm-policy-module.postinst index 253eaaa..fe73f0b 100644 --- a/debian/python3-osm-policy-module.postinst +++ b/debian/python3-osm-policy-module.postinst @@ -24,9 +24,9 @@ echo "Installing python dependencies via pip..." pip3 install aiokafka==0.4.* -pip3 install peewee==3.1.* +pip3 install peewee==3.8.* pip3 install jsonschema==2.6.* -pip3 install six==1.11.* pip3 install pyyaml==3.* pip3 install pymysql +pip3 install peewee-migrate==1.1.* echo "Installation of python dependencies finished" \ No newline at end of file diff --git a/docker/Dockerfile b/docker/Dockerfile index 7f57036..3c00b7a 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -45,6 +45,6 @@ ENV OSMPOL_DATABASE_URI mongodb://mongo:27017 ENV OSMPOL_SQL_DATABASE_URI sqlite:///policy_module.db -ENV OSMPOL_GLOBAL_LOG_LEVEL INFO +ENV OSMPOL_GLOBAL_LOGLEVEL INFO CMD osm-policy-agent diff --git a/osm_policy_module/cmd/policy_module_agent.py b/osm_policy_module/cmd/policy_module_agent.py index 050cf32..af2f602 100644 --- a/osm_policy_module/cmd/policy_module_agent.py +++ b/osm_policy_module/cmd/policy_module_agent.py @@ -48,8 +48,8 @@ def main(): log = logging.getLogger(__name__) log.debug("Config: %s", cfg.conf) log.info("Initializing database...") - db_manager = DatabaseManager() - db_manager.init_db(cfg) + db_manager = DatabaseManager(cfg) + db_manager.create_tables() log.info("Database initialized correctly.") log.info("Starting policy module agent...") loop = asyncio.get_event_loop() diff --git a/osm_policy_module/core/agent.py b/osm_policy_module/core/agent.py index 482f911..65c741f 100644 --- a/osm_policy_module/core/agent.py +++ b/osm_policy_module/core/agent.py @@ -52,7 +52,7 @@ class PolicyModuleAgent: self.db_client = CommonDbClient(config) self.mon_client = MonClient(config, loop=self.loop) self.lcm_client = LcmClient(config, loop=self.loop) - self.database_manager = DatabaseManager() + self.database_manager = DatabaseManager(config) self.msg_bus = MessageBusClient(config) def run(self): diff --git a/osm_policy_module/core/database.py b/osm_policy_module/core/database.py index 330d8c5..3ca2e33 100644 --- a/osm_policy_module/core/database.py +++ b/osm_policy_module/core/database.py @@ -23,10 +23,13 @@ ## import datetime import logging +import os from peewee import CharField, IntegerField, ForeignKeyField, Model, TextField, AutoField, DateTimeField, Proxy +from peewee_migrate import Router from playhouse.db_url import connect +from osm_policy_module import migrations from osm_policy_module.core.config import Config log = logging.getLogger(__name__) @@ -69,13 +72,13 @@ class ScalingAlarm(BaseModel): class DatabaseManager: - def init_db(self, config: Config): + def __init__(self, config: Config): db.initialize(connect(config.get('sql', 'database_uri'))) - self.create_tables() - def create_tables(self): + def create_tables(self) -> None: with db.atomic(): - db.create_tables([ScalingGroup, ScalingPolicy, ScalingCriteria, ScalingAlarm]) + router = Router(db, os.path.dirname(migrations.__file__)) + router.run() def get_alarm(self, alarm_uuid: str): with db.atomic(): diff --git a/osm_policy_module/migrations/001_initial.py b/osm_policy_module/migrations/001_initial.py new file mode 100644 index 0000000..257a06c --- /dev/null +++ b/osm_policy_module/migrations/001_initial.py @@ -0,0 +1,120 @@ +# -*- coding: utf-8 -*- + +# Copyright 2018 Whitestack, LLC +# ************************************************************* + +# This file is part of OSM Monitoring module +# All Rights Reserved to Whitestack, LLC + +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at + +# http://www.apache.org/licenses/LICENSE-2.0 + +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +# For those usages not covered by the Apache License, Version 2.0 please +# contact: bdiaz@whitestack.com or glavado@whitestack.com +## +"""Peewee migrations -- 001_initial.py. + +Some examples (model - class or model name):: + + > Model = migrator.orm['model_name'] # Return model in current state by name + + > migrator.sql(sql) # Run custom SQL + > migrator.python(func, *args, **kwargs) # Run python code + > migrator.create_model(Model) # Create a model (could be used as decorator) + > migrator.remove_model(model, cascade=True) # Remove a model + > migrator.add_fields(model, **fields) # Add fields to a model + > migrator.change_fields(model, **fields) # Change fields + > migrator.remove_fields(model, *field_names, cascade=True) + > migrator.rename_field(model, old_field_name, new_field_name) + > migrator.rename_table(model, new_table_name) + > migrator.add_index(model, *col_names, unique=False) + > migrator.drop_index(model, *col_names) + > migrator.add_not_null(model, *field_names) + > migrator.drop_not_null(model, *field_names) + > migrator.add_default(model, field_name, default) + +""" + +import peewee as pw + +SQL = pw.SQL + + +def migrate(migrator, database, fake=False, **kwargs): + """Write your migrations here.""" + + @migrator.create_model + class BaseModel(pw.Model): + id = pw.AutoField() + + class Meta: + table_name = "basemodel" + + @migrator.create_model + class ScalingGroup(pw.Model): + id = pw.AutoField() + nsr_id = pw.CharField(max_length=255) + vnf_member_index = pw.IntegerField() + name = pw.CharField(max_length=255) + content = pw.TextField() + + class Meta: + table_name = "scalinggroup" + + @migrator.create_model + class ScalingPolicy(pw.Model): + id = pw.AutoField() + name = pw.CharField(max_length=255) + cooldown_time = pw.IntegerField() + last_scale = pw.DateTimeField() + scaling_group = pw.ForeignKeyField(backref='scaling_policies', column_name='scaling_group_id', field='id', + model=migrator.orm['scalinggroup'], on_delete='CASCADE') + + class Meta: + table_name = "scalingpolicy" + + @migrator.create_model + class ScalingCriteria(pw.Model): + id = pw.AutoField() + name = pw.CharField(max_length=255) + scaling_policy = pw.ForeignKeyField(backref='scaling_criterias', column_name='scaling_policy_id', field='id', + model=migrator.orm['scalingpolicy'], on_delete='CASCADE') + + class Meta: + table_name = "scalingcriteria" + + @migrator.create_model + class ScalingAlarm(pw.Model): + id = pw.AutoField() + alarm_uuid = pw.CharField(max_length=255, unique=True) + action = pw.CharField(max_length=255) + vnf_member_index = pw.IntegerField() + vdu_name = pw.CharField(max_length=255) + scaling_criteria = pw.ForeignKeyField(backref='scaling_alarms', column_name='scaling_criteria_id', field='id', + model=migrator.orm['scalingcriteria'], on_delete='CASCADE') + + class Meta: + table_name = "scalingalarm" + + +def rollback(migrator, database, fake=False, **kwargs): + """Write your rollback migrations here.""" + + migrator.remove_model('scalingalarm') + + migrator.remove_model('scalingcriteria') + + migrator.remove_model('scalingpolicy') + + migrator.remove_model('scalinggroup') + + migrator.remove_model('basemodel') diff --git a/osm_policy_module/migrations/__init__.py b/osm_policy_module/migrations/__init__.py new file mode 100644 index 0000000..d81308a --- /dev/null +++ b/osm_policy_module/migrations/__init__.py @@ -0,0 +1,23 @@ +# -*- coding: utf-8 -*- + +# Copyright 2018 Whitestack, LLC +# ************************************************************* + +# This file is part of OSM Monitoring module +# All Rights Reserved to Whitestack, LLC + +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at + +# http://www.apache.org/licenses/LICENSE-2.0 + +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +# For those usages not covered by the Apache License, Version 2.0 please +# contact: bdiaz@whitestack.com or glavado@whitestack.com +## diff --git a/osm_policy_module/migrations/conf.py b/osm_policy_module/migrations/conf.py new file mode 100644 index 0000000..c86cad9 --- /dev/null +++ b/osm_policy_module/migrations/conf.py @@ -0,0 +1,26 @@ +# -*- coding: utf-8 -*- + +# Copyright 2018 Whitestack, LLC +# ************************************************************* + +# This file is part of OSM Monitoring module +# All Rights Reserved to Whitestack, LLC + +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at + +# http://www.apache.org/licenses/LICENSE-2.0 + +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +# For those usages not covered by the Apache License, Version 2.0 please +# contact: bdiaz@whitestack.com or glavado@whitestack.com +## +import os + +DATABASE = os.getenv('OSMPOL_SQL_DATABASE_URI', 'sqlite:///policy_module.db') diff --git a/osm_policy_module/tests/integration/test_policy_agent.py b/osm_policy_module/tests/integration/test_policy_agent.py index 68ab1f4..7ad9944 100644 --- a/osm_policy_module/tests/integration/test_policy_agent.py +++ b/osm_policy_module/tests/integration/test_policy_agent.py @@ -30,7 +30,7 @@ from unittest.mock import patch, Mock from kafka import KafkaProducer from osm_common.dbmongo import DbMongo -from peewee import SqliteDatabase +from playhouse.db_url import connect from osm_policy_module.common.common_db_client import CommonDbClient from osm_policy_module.common.mon_client import MonClient @@ -420,19 +420,17 @@ vnfd_record_mock = { } } -test_db = SqliteDatabase(':memory:') - MODELS = [ScalingGroup, ScalingPolicy, ScalingCriteria, ScalingAlarm] class PolicyModuleAgentTest(unittest.TestCase): def setUp(self): super() - database.db = test_db - test_db.bind(MODELS) - test_db.connect() - test_db.drop_tables(MODELS) - test_db.create_tables(MODELS) + database.db.initialize(connect('sqlite://')) + database.db.bind(MODELS) + database.db.connect() + database.db.drop_tables(MODELS) + database.db.create_tables(MODELS) self.loop = asyncio.new_event_loop() def tearDown(self): diff --git a/requirements.txt b/requirements.txt index 5665a39..b949cbd 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,7 +1,7 @@ aiokafka==0.4.* -peewee==3.1.* +peewee==3.8.* jsonschema==2.6.* -six==1.11.* pyyaml==3.* -pymysql -git+https://osm.etsi.org/gerrit/osm/common.git#egg=osm-common +pymysql==0.9.* +peewee-migrate==1.1.* +git+https://osm.etsi.org/gerrit/osm/common.git@v5.0#egg=osm-common diff --git a/setup.py b/setup.py index 1d42c16..8b3996d 100644 --- a/setup.py +++ b/setup.py @@ -49,11 +49,11 @@ setup( include_package_data=True, install_requires=[ "aiokafka==0.4.*", - "peewee==3.1.*", + "peewee==3.8.*", "jsonschema==2.6.*", - "six==1.11.*", "pyyaml==3.*", "pymysql", + "peewee-migrate==1.1.*", "osm-common", ], entry_points={ -- 2.17.1