From: tierno Date: Mon, 10 Dec 2018 15:24:05 +0000 (+0000) Subject: bug 608 Database migration recovering with docker relaunch X-Git-Tag: v7.1.0rc1~2 X-Git-Url: https://osm.etsi.org/gitweb/?p=osm%2Fopenvim.git;a=commitdiff_plain;h=6714bc31b0ecf2bf801e75e23eb72254e3d18d78 bug 608 Database migration recovering with docker relaunch Change-Id: I4dec9b53ad08fa2a67c9b5d9d9f7b0f22605fa5e Signed-off-by: tierno --- diff --git a/database_utils/init_vim_db.sh b/database_utils/init_vim_db.sh index d4a801e..aca7673 100755 --- a/database_utils/init_vim_db.sh +++ b/database_utils/init_vim_db.sh @@ -1,7 +1,7 @@ #!/bin/bash ## -# Copyright 2015 Telefónica Investigación y Desarrollo, S.A.U. +# Copyright 2015 Telefonica Investigacion y Desarrollo, S.A.U. # This file is part of openvim # All Rights Reserved. # @@ -162,7 +162,8 @@ DBPORT_="-P$DBPORT" echo " loading ${DIRNAME}/vim_db_structure.sql" -sed -e "s/{{vim_db}}/$DBNAME/" ${DIRNAME}/vim_db_structure.sql | mysql $DEF_EXTRA_FILE_PARAM +sed -e "s/{{vim_db}}/$DBNAME/" ${DIRNAME}/vim_db_structure.sql | mysql $DEF_EXTRA_FILE_PARAM || + ! echo "ERROR at init $DBNAME" || exit 1 echo " loading ${DIRNAME}/host_ranking.sql" mysql $DEF_EXTRA_FILE_PARAM $DBNAME < ${DIRNAME}/host_ranking.sql diff --git a/database_utils/migrate_vim_db.sh b/database_utils/migrate_vim_db.sh index 9d59b2b..73e6cbd 100755 --- a/database_utils/migrate_vim_db.sh +++ b/database_utils/migrate_vim_db.sh @@ -1,7 +1,7 @@ #!/bin/bash ## -# Copyright 2015 Telefónica Investigación y Desarrollo, S.A.U. +# Copyright 2015 Telefonica Investigacion y Desarrollo, S.A.U. # This file is part of openvim # All Rights Reserved. # @@ -24,6 +24,7 @@ # #Upgrade/Downgrade openvim database preserving the content # +DBUTILS="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" DBUSER="vim" DBPASS="" @@ -32,6 +33,8 @@ DBHOST="" DBPORT="3306" DBNAME="vim_db" QUIET_MODE="" +BACKUP_DIR="" +BACKUP_FILE="" #TODO update it with the last database version LAST_DB_VERSION=23 @@ -50,11 +53,12 @@ function usage(){ echo -e " -P PORT database port. '$DBPORT' by default" echo -e " -h HOST database host. 'localhost' by default" echo -e " -d NAME database name. '$DBNAME' by default. Prompts if DB access fails" + echo -e " -b DIR backup folder where to create rollback backup file" echo -e " -q --quiet: Do not prompt for credentials and exit if cannot access to database" echo -e " --help shows this help" } -while getopts ":u:p:P:h:d:q-:" o; do +while getopts ":u:p:b:P:h:d:q-:" o; do case "${o}" in u) DBUSER="$OPTARG" @@ -71,6 +75,9 @@ while getopts ":u:p:P:h:d:q-:" o; do h) DBHOST="$OPTARG" ;; + b) + BACKUP_DIR="$OPTARG" + ;; q) export QUIET_MODE=yes ;; @@ -142,7 +149,6 @@ done DBCMD="mysql $DEF_EXTRA_FILE_PARAM $DBNAME" #echo DBCMD $DBCMD -#GET DATABASE VERSION # check that the database seems a openvim database if ! echo -e "show create table instances;\nshow create table numas" | $DBCMD >/dev/null 2>&1 then @@ -150,22 +156,6 @@ then exit 1; fi -if ! echo 'show create table schema_version;' | $DBCMD >/dev/null 2>&1 -then - DATABASE_VER="0.0" - DATABASE_VER_NUM=0 -else - DATABASE_VER_NUM=`echo "select max(version_int) from schema_version;" | $DBCMD | tail -n+2` - DATABASE_VER=`echo "select version from schema_version where version_int='$DATABASE_VER_NUM';" | $DBCMD | tail -n+2` - [ "$DATABASE_VER_NUM" -lt 0 -o "$DATABASE_VER_NUM" -gt 100 ] && - echo " Error can not get database version ($DATABASE_VER?)" >&2 && exit 1 - #echo "_${DATABASE_VER_NUM}_${DATABASE_VER}" -fi - -[ "$DATABASE_VER_NUM" -gt "$LAST_DB_VERSION" ] && - echo "Database has been upgraded with a newer version of this script. Use this version to downgrade" >&2 && - exit 1 - #GET DATABASE TARGET VERSION #DB_VERSION=0 #[ $OPENVIM_VER_NUM -gt 1091 ] && DATABASE_TARGET_VER_NUM=1 #>0.1.91 => 1 @@ -792,19 +782,31 @@ function downgrade_from_23(){ # TODO ... put functions here -# echo "db version = "${DATABASE_VER_NUM} -[ $DB_VERSION -eq $DATABASE_VER_NUM ] && echo " current database version '$DATABASE_VER_NUM' is ok" && exit 0 -# Create a backup database content -TEMPFILE2="$(mktemp -q --tmpdir "backupdb.XXXXXX.sql")" -trap 'rm -f "$TEMPFILE2"' EXIT -mysqldump $DEF_EXTRA_FILE_PARAM --add-drop-table --add-drop-database --routines --databases $DBNAME > $TEMPFILE2 +function del_schema_version_process() +{ + echo "DELETE FROM schema_version WHERE version_int='0';" | $DBCMD || + ! echo " ERROR writing on schema_version" >&2 || exit 1 +} + +function set_schema_version_process() +{ + echo "INSERT INTO schema_version (version_int, version, openvim_ver, comments, date) VALUES "\ + "(0, '0.0', '0.0.0', 'migration from $DATABASE_VER_NUM to $DB_VERSION backup: $BACKUP_FILE',"\ + "'$(date +%Y-%m-%d)');" | $DBCMD || + ! echo " Cannot set database at migration process writing into schema_version" >&2 || exit 1 + +} -function rollback_db() +function rollback_db() { - cat $TEMPFILE2 | mysql $DEF_EXTRA_FILE_PARAM && echo " Aborted! Rollback database OK" || - echo " Aborted! Rollback database FAIL" - exit 1 + if echo $DATABASE_PROCESS | grep -q init ; then # Empty database. No backup needed + echo " Aborted! Rollback database not needed" && exit 1 + else # migration a non empty database or Recovering a migration process + cat $BACKUP_FILE | mysql $DEF_EXTRA_FILE_PARAM && echo " Aborted! Rollback database OK" && + del_schema_version_process && rm -f "$BACKUP_FILE" && exit 1 + echo " Aborted! Rollback database FAIL" && exit 1 + fi } function sql() # send a sql command @@ -813,27 +815,86 @@ function sql() # send a sql command return 0 } -#UPGRADE DATABASE step by step -while [ $DB_VERSION -gt $DATABASE_VER_NUM ] -do - echo " upgrade database from version '$DATABASE_VER_NUM' to '$((DATABASE_VER_NUM+1))'" - DATABASE_VER_NUM=$((DATABASE_VER_NUM+1)) - upgrade_to_${DATABASE_VER_NUM} - #FILE_="${DIRNAME}/upgrade_to_${DATABASE_VER_NUM}.sh" - #[ ! -x "$FILE_" ] && echo "Error, can not find script '$FILE_' to upgrade" >&2 && exit -1 - #$FILE_ || exit -1 # if fail return -done +function migrate() +{ + #UPGRADE DATABASE step by step + while [ $DB_VERSION -gt $DATABASE_VER_NUM ] + do + echo " upgrade database from version '$DATABASE_VER_NUM' to '$((DATABASE_VER_NUM+1))'" + DATABASE_VER_NUM=$((DATABASE_VER_NUM+1)) + upgrade_to_${DATABASE_VER_NUM} + #FILE_="${DIRNAME}/upgrade_to_${DATABASE_VER_NUM}.sh" + #[ ! -x "$FILE_" ] && echo "Error, can not find script '$FILE_' to upgrade" >&2 && exit -1 + #$FILE_ || exit -1 # if fail return + done -#DOWNGRADE DATABASE step by step -while [ $DB_VERSION -lt $DATABASE_VER_NUM ] -do - echo " downgrade database from version '$DATABASE_VER_NUM' to '$((DATABASE_VER_NUM-1))'" - #FILE_="${DIRNAME}/downgrade_from_${DATABASE_VER_NUM}.sh" - #[ ! -x "$FILE_" ] && echo "Error, can not find script '$FILE_' to downgrade" >&2 && exit -1 - #$FILE_ || exit -1 # if fail return - downgrade_from_${DATABASE_VER_NUM} - DATABASE_VER_NUM=$((DATABASE_VER_NUM-1)) -done + #DOWNGRADE DATABASE step by step + while [ $DB_VERSION -lt $DATABASE_VER_NUM ] + do + echo " downgrade database from version '$DATABASE_VER_NUM' to '$((DATABASE_VER_NUM-1))'" + #FILE_="${DIRNAME}/downgrade_from_${DATABASE_VER_NUM}.sh" + #[ ! -x "$FILE_" ] && echo "Error, can not find script '$FILE_' to downgrade" >&2 && exit -1 + #$FILE_ || exit -1 # if fail return + downgrade_from_${DATABASE_VER_NUM} + DATABASE_VER_NUM=$((DATABASE_VER_NUM-1)) + done +} + + +# check if current database is ok +function check_migration_needed() +{ + DATABASE_VER_NUM=`echo "select max(version_int) from schema_version;" | $DBCMD | tail -n+2` || + ! echo " ERROR cannot read from schema_version" || exit 1 + + if [[ -z "$DATABASE_VER_NUM" ]] || [[ "$DATABASE_VER_NUM" -lt 0 ]] || [[ "$DATABASE_VER_NUM" -gt 100 ]] ; then + echo " Error can not get database version ($DATABASE_VER_NUM?)" >&2 + exit 1 + fi + + [[ $DB_VERSION -eq $DATABASE_VER_NUM ]] && echo " current database version '$DATABASE_VER_NUM' is ok" && return 1 + [[ "$DATABASE_VER_NUM" -gt "$LAST_DB_VERSION" ]] && + echo "Database has been upgraded with a newer version of this script. Use this version to downgrade" >&2 && + exit 1 + return 0 +} + +DATABASE_PROCESS=`echo "select comments from schema_version where version_int=0;" | $DBCMD | tail -n+2` || + ! echo " ERROR cannot read from schema_version" || exit 1 +if [[ -z "$DATABASE_PROCESS" ]] ; then # migration a non empty database + check_migration_needed || exit 0 + # Create a backup database content + [[ -n "$BACKUP_DIR" ]] && BACKUP_FILE="$(mktemp -q "${BACKUP_DIR}/backupdb.XXXXXX.sql")" + [[ -z "$BACKUP_DIR" ]] && BACKUP_FILE="$(mktemp -q --tmpdir "backupdb.XXXXXX.sql")" + mysqldump $DEF_EXTRA_FILE_PARAM --add-drop-table --add-drop-database --routines --databases $DBNAME > $BACKUP_FILE || + ! echo "Cannot create Backup file '$BACKUP_FILE'" >&2 || exit 1 + echo " Backup file '$BACKUP_FILE' created" + # Set schema version + set_schema_version_process + migrate + del_schema_version_process + rm -f "$BACKUP_FILE" +elif echo $DATABASE_PROCESS | grep -q init ; then # Empty database. No backup needed + echo " Migrating an empty database" + if check_migration_needed ; then + migrate + fi + del_schema_version_process + +else # Recover Migration process + BACKUP_FILE=${DATABASE_PROCESS##*backup: } + [[ -f "$BACKUP_FILE" ]] || ! echo "Previous migration process fail and cannot recover backup file '$BACKUP_FILE'" >&2 || + exit 1 + echo " Previous migration was killed. Restoring database from rollback file'$BACKUP_FILE'" + cat $BACKUP_FILE | mysql $DEF_EXTRA_FILE_PARAM || ! echo " Cannot load backup file '$BACKUP_FILE'" >&2 || exit 1 + if check_migration_needed ; then + set_schema_version_process + migrate + fi + del_schema_version_process + rm -f "$BACKUP_FILE" +fi +exit 0 #echo done diff --git a/database_utils/vim_db_structure.sql b/database_utils/vim_db_structure.sql index 342c20c..d6b7cb7 100644 --- a/database_utils/vim_db_structure.sql +++ b/database_utils/vim_db_structure.sql @@ -19,11 +19,11 @@ * contact with: nfvlabs@tid.es **/ --- MySQL dump 10.13 Distrib 5.7.20, for Linux (x86_64) +-- MySQL dump 10.13 Distrib 5.7.24, for Linux (x86_64) -- -- Host: localhost Database: {{vim_db}} -- ------------------------------------------------------ --- Server version 5.7.20-0ubuntu0.16.04.1 +-- Server version 5.7.24 /*!40101 SET @OLD_CHARACTER_SET_CLIENT=@@CHARACTER_SET_CLIENT */; /*!40101 SET @OLD_CHARACTER_SET_RESULTS=@@CHARACTER_SET_RESULTS */; @@ -102,6 +102,7 @@ CREATE TABLE `hosts` ( `ranking` smallint(6) NOT NULL DEFAULT '0', `created_at` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP, `features` varchar(255) DEFAULT NULL, + `hypervisors` varchar(255) NOT NULL DEFAULT 'kvm', `user` varchar(64) NOT NULL, `password` varchar(64) DEFAULT NULL, `keyfile` varchar(255) DEFAULT NULL, @@ -171,6 +172,8 @@ DROP TABLE IF EXISTS `instances`; CREATE TABLE `instances` ( `uuid` varchar(36) NOT NULL, `flavor_id` varchar(36) NOT NULL, + `hypervisor` enum('kvm','xen-unik','xenhvm') NOT NULL DEFAULT 'kvm', + `os_image_type` varchar(24) NOT NULL DEFAULT 'other', `image_id` varchar(36) NOT NULL, `name` varchar(64) NOT NULL, `description` varchar(255) DEFAULT NULL, @@ -984,17 +987,14 @@ DELIMITER ; /*!40101 SET COLLATION_CONNECTION=@OLD_COLLATION_CONNECTION */; /*!40111 SET SQL_NOTES=@OLD_SQL_NOTES */; --- Dump completed on 2017-11-23 10:32:25 +-- Dump completed on 2018-12-10 14:58:11 - - - --- MySQL dump 10.13 Distrib 5.7.20, for Linux (x86_64) +-- MySQL dump 10.13 Distrib 5.7.24, for Linux (x86_64) -- -- Host: localhost Database: {{vim_db}} -- ------------------------------------------------------ --- Server version 5.7.20-0ubuntu0.16.04.1 +-- Server version 5.7.24 /*!40101 SET @OLD_CHARACTER_SET_CLIENT=@@CHARACTER_SET_CLIENT */; /*!40101 SET @OLD_CHARACTER_SET_RESULTS=@@CHARACTER_SET_RESULTS */; @@ -1013,7 +1013,31 @@ DELIMITER ; LOCK TABLES `schema_version` WRITE; /*!40000 ALTER TABLE `schema_version` DISABLE KEYS */; -INSERT INTO `schema_version` VALUES (1,'0.1','0.2.00','insert schema_version; alter nets with last_error column','2015-05-05'),(2,'0.2','0.2.03','update Procedure UpdateSwitchPort','2015-05-06'),(3,'0.3','0.2.5','New Procedure GetAllAvailablePorts','2015-07-09'),(4,'0.4','0.3.1','Remove unique index VLAN at resources_port','2015-09-04'),(5,'0.5','0.4.1','Add ip_address to ports','2015-09-04'),(6,'0.6','0.4.2','Enlarging name at database','2016-02-01'),(7,'0.7','0.4.4','Add bind_net to net table','2016-02-12'),(8,'0.8','0.4.10','add column checksum to images','2016-09-30'),(9,'0.9','0.5.1','increase length of columns path and name to 255 in table images, and change length of column name to 255 in table flavors','2017-01-10'),(10,'0.10','0.5.2','change ports type, adding instance:ovs','2017-02-01'),(11,'0.11','0.5.4','Add gateway_ip colum to nets','2017-02-13'),(12,'0.12','0.5.5','Add of_controller table','2017-02-17'),(13,'0.13','0.5.6','Add of_port_mapings table','2017-03-09'),(14,'0.14','0.5.7','Add switch_mac, ofc_id colum to ports and resources_port tables','2017-03-09'),(15,'0.15','0.5.8','Add ofc_id colum to of_flows','2017-03-15'),(16,'0.16','0.5.9','Add last_error and status colum to ofcs','2017-03-17'),(17,'0.17','0.5.10','Add pci to unique index dpid port/mac at of_port_mappings','2017-04-05'),(18,'0.18','0.5.13','Add region to nets, change vlan unique index','2017-05-03'),(19,'0.19','0.5.15','Add keyfile to hosts','2017-05-23'),(20,'0.20','0.5.17','Add image_size to instance_devices','2017-06-01'),(21,'0.21','0.5.18','Add routes, links and dns to inets','2017-06-21'),(22,'0.22','0.5.21','Changed type of ram in flavors from SMALLINT to MEDIUMINT','2017-11-14'); +INSERT INTO `schema_version` VALUES +(0,'0.0','0.0.0','Database in init process','2015-05-05'), +(1,'0.1','0.2.00','insert schema_version; alter nets with last_error column','2015-05-05'), +(2,'0.2','0.2.03','update Procedure UpdateSwitchPort','2015-05-06'), +(3,'0.3','0.2.5','New Procedure GetAllAvailablePorts','2015-07-09'), +(4,'0.4','0.3.1','Remove unique index VLAN at resources_port','2015-09-04'), +(5,'0.5','0.4.1','Add ip_address to ports','2015-09-04'), +(6,'0.6','0.4.2','Enlarging name at database','2016-02-01'), +(7,'0.7','0.4.4','Add bind_net to net table','2016-02-12'), +(8,'0.8','0.4.10','add column checksum to images','2016-09-30'), +(9,'0.9','0.5.1','increase length of columns path and name to 255 in table images, and change length of column name to 255 in table flavors','2017-01-10'), +(10,'0.10','0.5.2','change ports type, adding instance:ovs','2017-02-01'), +(11,'0.11','0.5.4','Add gateway_ip colum to nets','2017-02-13'), +(12,'0.12','0.5.5','Add of_controller table','2017-02-17'), +(13,'0.13','0.5.6','Add of_port_mapings table','2017-03-09'), +(14,'0.14','0.5.7','Add switch_mac, ofc_id colum to ports and resources_port tables','2017-03-09'), +(15,'0.15','0.5.8','Add ofc_id colum to of_flows','2017-03-15'), +(16,'0.16','0.5.9','Add last_error and status colum to ofcs','2017-03-17'), +(17,'0.17','0.5.10','Add pci to unique index dpid port/mac at of_port_mappings','2017-04-05'), +(18,'0.18','0.5.13','Add region to nets, change vlan unique index','2017-05-03'), +(19,'0.19','0.5.15','Add keyfile to hosts','2017-05-23'), +(20,'0.20','0.5.17','Add image_size to instance_devices','2017-06-01'), +(21,'0.21','0.5.18','Add routes, links and dns to inets','2017-06-21'), +(22,'0.22','0.5.21','Changed type of ram in flavors from SMALLINT to MEDIUMINT','2017-11-14'), +(23,'0.23','0.5.24','Add hypervisor, os_type to instances and add hypervisors to hosts','2018-03-20'); /*!40000 ALTER TABLE `schema_version` ENABLE KEYS */; UNLOCK TABLES; /*!40103 SET TIME_ZONE=@OLD_TIME_ZONE */; @@ -1026,4 +1050,5 @@ UNLOCK TABLES; /*!40101 SET COLLATION_CONNECTION=@OLD_COLLATION_CONNECTION */; /*!40111 SET SQL_NOTES=@OLD_SQL_NOTES */; --- Dump completed on 2017-11-23 10:32:25 +-- Dump completed on 2018-12-10 14:58:11 +