MT#56542 Add tools to prepare sql and json dumps

The main tool ngcp-build-schema-files: 1. initializes and starts a dedicated mysql instance in tmpfs. 2. prepares schemes. 3. runs ngcp-dump-json.pl and mysqldump to get sql and json files. 4. stops and deletes mysql instance. The helper ngcp-dump-json.pl creates json dump of the schema. Change-Id: I65968d0f9136b619f9adc5ed735daca0bd129e6e
2 years ago · a5d44761c1
parent 17a527a751
commit a5d44761c1
3 changed files with 270 additions and 0 deletions
--- a/build_utils/ngcp-build-schema-files
+++ b/build_utils/ngcp-build-schema-files
@ -0,0 +1,121 @@
+#!/bin/bash
+
+set -euEo pipefail
+
+ngcp_dump_json_cmd="/usr/share/ngcp-db-schema/helper/ngcp-dump-json.pl"
+if [[ ! -x "${ngcp_dump_json_cmd}" ]]; then
+  echo "Can't find executable ${ngcp_dump_json_cmd}" >&2
+  echo "Please install ngcp-db-schema package" >&2
+  exit 1
+fi
+
+mariadb_dir=$(mktemp -d /tmp/mariadb-build.XXXXXX)
+chown mysql:mysql "${mariadb_dir}"
+
+declare -a mysql_install_options
+mysql_install_options+=("--no-defaults")
+mysql_install_options+=("--verbose")
+mysql_install_options+=("--skip-test-db")
+mysql_install_options+=("--innodb_encrypt_tables=off")
+mysql_install_options+=("--innodb_encrypt_log=0")
+mysql_install_options+=("--encrypt_binlog=0")
+mysql_install_options+=("--datadir=${mariadb_dir}")
+mysql_install_options+=("--user=mysql")
+mysql_install_options+=("--skip-networking")
+mysql_install_options+=("--sql-mode=NO_ENGINE_SUBSTITUTION")
+mysql_install_options+=("--character-set-server=utf8")
+mysql_install_options+=("--collation-server=utf8_general_ci")
+
+mysql_install_db "${mysql_install_options[@]}"
+
+declare -a mysqld_options
+mysqld_options+=("--no-defaults")
+mysqld_options+=("--pid-file=${mariadb_dir}/mysqld.pid")
+mysqld_options+=("--datadir=${mariadb_dir}")
+mysqld_options+=("--user=mysql")
+mysqld_options+=("--socket=${mariadb_dir}/mysqld.sock")
+mysqld_options+=("--log-error=${mariadb_dir}/mysqld.log")
+mysqld_options+=("--skip-networking")
+mysqld_options+=("--sql-mode=NO_ENGINE_SUBSTITUTION")
+mysqld_options+=("--character-set-server=utf8")
+mysqld_options+=("--init-connect='SET NAMES utf8; SET collation_connection = utf8_unicode_ci; SET sql_mode = STRICT_TRANS_TABLES'")
+mysqld_options+=("--collation-server=utf8_general_ci")
+
+mariadbd "${mysqld_options[@]}" &>/dev/null &
+
+tries=0
+max_tries=10
+started=false
+while [[ "${tries}" -le "${max_tries}" ]]; do
+  if [[ -S "${mariadb_dir}/mysqld.sock" ]]; then
+    started=true
+    break
+  fi
+  tries=$((tries + 1))
+  sleep 1
+done
+
+if ! "${started}"; then
+  echo "Mysql server wasn't started" >&2
+  exit 1
+fi
+
+mysql -S "${mariadb_dir}/mysqld.sock" -e \
+  "GRANT ALL PRIVILEGES ON *.* TO sipwise@localhost IDENTIFIED BY 'sipwise' WITH GRANT OPTION;"
+
+echo "Running ngcp-update-db-schema..."
+MYSQL_SOCKET="${mariadb_dir}/mysqld.sock" AUTOMATED_INSTALL_MODE=1 \
+  SKIP_SYNC_DB=1 /usr/sbin/ngcp-update-db-schema -f
+echo "Done"
+
+declare -a schemes=()
+schemes+=("accounting")
+schemes+=("billing")
+schemes+=("carrier")
+schemes+=("fileshare")
+schemes+=("kamailio")
+schemes+=("ldap")
+schemes+=("ngcp")
+schemes+=("prosody")
+schemes+=("provisioning")
+schemes+=("sipstats")
+schemes+=("stats")
+schemes+=("syslog")
+
+for schema in "${schemes[@]}"; do
+  echo "Running ngcp-dump-json.pl for schema '${schema}'..."
+  "${ngcp_dump_json_cmd}" \
+    --socket="${mariadb_dir}/mysqld.sock" \
+    --schema="${schema}"
+  echo "Done"
+
+  echo "Running mysqldump for schema '${schema}'..."
+  echo "SET FOREIGN_KEY_CHECKS=0;" > "${schema}.sql"
+  echo "SET NAMES utf8;" >> "${schema}.sql"
+  echo "SET SESSION autocommit=0;" >> "${schema}.sql"
+  echo "SET SESSION unique_checks=0;" >> "${schema}.sql"
+  echo "CREATE DATABASE ${schema};" >> "${schema}.sql"
+  echo "USE ${schema};" >> "${schema}.sql"
+
+  # Replace all the dates in format YYYY-MM-DD HH:MM:SS with NOW() function
+  mysqldump "${schema}" --compact --skip-extended-insert \
+    -S "${mariadb_dir}/mysqld.sock" \
+    | sed -r "s/'[[:digit:]]{4}-[[:digit:]]{2}-[[:digit:]]{2} [[:digit:]]{2}:[[:digit:]]{2}:[[:digit:]]{2}'/NOW()/g" \
+    > "${schema}.sql"
+  echo "Done"
+done
+
+tries=0
+max_tries=10
+stopped=false
+mysql_pid=$(cat "${mariadb_dir}/mysqld.pid")
+if [[ -z "${mysql_pid}" ]]; then
+  echo "Can't get mysql pid from ${mariadb_dir}/mysqld.pid" >&2
+  exit 1
+fi
+
+echo "Stopping mysql instance in '${mariadb_dir}'..."
+kill -9 "${mysql_pid}"
+
+echo "Removing mysql instance in '${mariadb_dir}'..."
+rm -rf "${mariadb_dir}"
--- a/build_utils/ngcp-dump-json.pl
+++ b/build_utils/ngcp-dump-json.pl
@ -0,0 +1,147 @@
+#!/usr/bin/env perl
+
+use strict;
+use warnings;
+
+use DBI;
+use JSON::XS;
+use Getopt::Long;
+use Carp;
+
+my $queries = {
+tables => <<"__SQL__"
+SELECT
+  TABLE_NAME,
+  ENGINE,
+  TABLE_COLLATION,
+  CREATE_OPTIONS,
+  TABLE_NAME AS key_col
+FROM information_schema.TABLES
+  WHERE TABLE_SCHEMA = ?
+    AND TABLE_TYPE = 'BASE TABLE'
+ORDER BY TABLE_NAME
+__SQL__
+,
+
+columns => <<"__SQL__"
+SELECT
+  c.TABLE_NAME,
+  c.COLUMN_NAME,
+  c.COLUMN_DEFAULT,
+  c.IS_NULLABLE,
+  c.COLUMN_TYPE,
+  c.EXTRA,
+  c.CHARACTER_SET_NAME,
+  c.COLLATION_NAME,
+  c.ORDINAL_POSITION,
+  CONCAT(c.TABLE_NAME, '/', c.COLUMN_NAME) AS key_col
+FROM information_schema.COLUMNS c
+  INNER JOIN information_schema.TABLES t
+    ON c.TABLE_NAME = t.TABLE_NAME
+WHERE t.TABLE_TYPE = 'BASE TABLE'
+  AND c.TABLE_SCHEMA = t.TABLE_SCHEMA
+  AND t.TABLE_SCHEMA = ?
+ORDER BY c.TABLE_NAME, c.COLUMN_NAME
+__SQL__
+,
+
+indexes => <<"__SQL__"
+SELECT
+  TABLE_NAME,
+  INDEX_NAME,
+  NON_UNIQUE,
+  COLUMN_NAME,
+  SEQ_IN_INDEX,
+  COLLATION,
+  SUB_PART,
+  NULLABLE,
+  INDEX_TYPE,
+  CONCAT(TABLE_NAME, '/', INDEX_NAME, '/',  SEQ_IN_INDEX) AS key_col
+FROM information_schema.STATISTICS
+WHERE TABLE_SCHEMA = ?
+ORDER BY TABLE_NAME, INDEX_NAME, COLUMN_NAME
+__SQL__
+,
+
+constraints => <<"__SQL__"
+SELECT
+  rc.CONSTRAINT_NAME,
+  rc.TABLE_NAME,
+  rc.REFERENCED_TABLE_NAME,
+  rc.UPDATE_RULE,
+  rc.DELETE_RULE,
+  cu.REFERENCED_COLUMN_NAME,
+  cu.COLUMN_NAME,
+  CONCAT(  rc.TABLE_NAME, '/', rc.CONSTRAINT_NAME, '/',
+    cu.COLUMN_NAME, '/', rc.REFERENCED_TABLE_NAME, '/',
+    cu.REFERENCED_COLUMN_NAME) AS key_col
+FROM information_schema.REFERENTIAL_CONSTRAINTS rc
+  LEFT JOIN information_schema.KEY_COLUMN_USAGE cu
+    ON (rc.CONSTRAINT_NAME=cu.CONSTRAINT_NAME
+      AND rc.CONSTRAINT_SCHEMA=cu.CONSTRAINT_SCHEMA)
+WHERE rc.CONSTRAINT_SCHEMA = ?
+ORDER BY CONSTRAINT_NAME, rc.TABLE_NAME, cu.COLUMN_NAME
+__SQL__
+,
+
+triggers => <<"__SQL__"
+SELECT
+  TRIGGER_NAME,
+  EVENT_MANIPULATION,
+  ACTION_STATEMENT,
+  ACTION_TIMING,
+  EVENT_OBJECT_SCHEMA,
+  EVENT_OBJECT_TABLE,
+  CONCAT(TRIGGER_NAME, '/', EVENT_OBJECT_TABLE) AS key_col
+FROM information_schema.TRIGGERS
+WHERE TRIGGER_SCHEMA = ?
+ORDER BY EVENT_OBJECT_TABLE, TRIGGER_NAME
+__SQL__
+,
+
+views => <<"__SQL__"
+SELECT
+  TABLE_NAME AS key_col,
+  VIEW_DEFINITION
+FROM information_schema.VIEWS
+WHERE TABLE_SCHEMA = ?
+ORDER BY TABLE_NAME
+__SQL__
+,
+
+routines => <<"__SQL__"
+SELECT
+  ROUTINE_NAME AS key_col,
+  ROUTINE_DEFINITION,
+  ROUTINE_TYPE
+FROM information_schema.ROUTINES
+WHERE ROUTINE_SCHEMA = ?
+__SQL__
+,
+};
+
+my $argv;
+GetOptions(
+    'socket=s' => \$argv->{socket},
+    'schema=s' => \$argv->{schema},
+);
+
+my $dbh = DBI->connect("DBI:mysql:;mysql_socket=$argv->{socket}",
+    'root',
+    '',
+    {RaiseError => 1}) or
+        croak("Can't connect to db: DBI:mysql:;mysql_socket=$argv->{socket}");
+
+my $coder = JSON::XS->new->utf8->pretty->allow_nonref;
+$coder->canonical([]);
+
+my @objects = qw(tables columns indexes constraints views routines);
+my $schema = {};
+foreach my $object (@objects) {
+    my $rows = $dbh->selectall_hashref($queries->{$object}, 'key_col', undef, $argv->{schema});
+    $schema->{$object} = $rows;
+}
+
+my $utf8_encoded_json_text = $coder->encode($schema);
+open(my $fh, '>', "$argv->{schema}.json") or croak("Can't write file $argv->{schema}.json");
+print $fh $utf8_encoded_json_text;
--- a/debian/ngcp-db-schema.install
+++ b/debian/ngcp-db-schema.install
@ -1,2 +1,4 @@
+build_utils/ngcp-build-schema-files       usr/share/ngcp-db-schema/helper/
+build_utils/ngcp-dump-json.pl       usr/share/ngcp-db-schema/helper/
 db_scripts                  usr/share/ngcp-db-schema/
 ngcp-update-db-schema       usr/sbin/