mariadb-dump based database replication

we list the databases on the primary and replica server,
determine which dbs the replica server needs, then send
those dbs in a single tcp stream with mariadb-dump.

supposedly, the CHANGE MASTER sql statements are automatically included.
however, it is unclear how or whether replication is working here.
also, even if replication is working right now, it probably got set up
by the old code that used mariabackup's "xtra" slave info file to get
the binlog position. So the testing needs to be set up to make sure that
this is all OK...

1 files changed, 84 insertions, 35 deletions

diff --git a/src/mariadb-push-replica.sh b/src/mariadb-push-replica.sh
index a7345dc..7134864 100755
--- a/src/mariadb-push-replica.sh
+++ b/src/mariadb-push-replica.sh
@@ -33,7 +33,13 @@ then
 fi
 
 primary_host=$(hostname --fqdn)
-replica_host=${1:-$default_replica_host}
+if [ $# = 0 ]
+then
+        replica_host=$default_replica_host
+else
+        replica_host=${1:-$default_replica_host}
+        shift
+fi
 replication_user=replication
 
 run_primary()
@@ -114,7 +120,7 @@ send_backup()
         rsync -zaR -- /./"${mariabackup_target_dir#/}" "$1":/
 }
 
-stop_database_server_and_remove_database_files()
+stop_mariadb_server_and_remove_database_files()
 {
         livedb=/var/lib/mysql
         set -e
@@ -136,6 +142,7 @@ restore_from_backup()
 {
         mariabackup_target_dir=/var/mariadb/backup
         set -e
+        stop_mariadb_server_and_remove_database_files
         set -- \
                 --force-non-empty-directories \
                 -u root \
@@ -188,7 +195,7 @@ mostly_silent_unless_error()
         return $r
 }
 
-enable_replication()
+enable_replication_via_mariabackup_xtra_info()
 {
         primary_host=$1
         replication_user=$2
@@ -286,57 +293,99 @@ from
 .
 }
 
-list_databases_needed_on_replica()
-{
-        set -- "$(mktemp)" "$(mktemp)"
-        run_primary mariadb_list_databases | sort -u >"$1"
-        run_replica mariadb_list_databases | sort -u >"$2"
-        comm -23 -- "$1" "$2"
-        rm -- "$1" "$2"
-}
-
 mariabackup_create_replica_databases()
 {
         run_primary create_backup
         run_primary send_backup "$replica_host"
-        run_replica stop_database_server_and_remove_database_files
         run_replica restore_from_backup
+        run_primary create_replication_user \
+                "$replica_host" \
+                "$replication_user" \
+                "$replication_password"
+        run_replica enable_replication_via_mariabackup_xtra_info \
+                "$primary_host" \
+                "$replication_user" \
+                "$replication_password"
+}
+
+printlines()
+{
+        printf '%s\n' "$@"
+}
+
+printarray()
+{
+        declare -n _PRINTARRAY_VARNAME="$1"
+        printf '%s\n' "${_PRINTARRAY_VARNAME[@]}"
+}
+
+# Call run_replica from here to avoid
+# piping the database back to caller
+# unnecessarily
+send_mariadb_dump()
+{
+        (set -x
+        mariadb-dump "${@:2}") |
+        replica_host="$1" run_replica pipe_into_mariadb
 }
 
-mariadbdump_create_replica_databases()
+pipe_into_mariadb()
 {
-        list_databases_needed_on_replica
-        comm -13 -- \
-                <(list_databases_needed_on_replica | tee /dev/stderr) \
-                <(printf '%s\n' "$@")
+        pv -f | mariadb
+}
+
+mariadbdump_transfer_missing_databases()
+{
+        declare -a primary_dbs
+        mapfile -t primary_dbs < \
+                <(</dev/null run_primary \
+                        mariadb_list_databases | sort -u)
+
+        declare -a replica_dbs
+        mapfile -t replica_dbs < \
+                <(</dev/null run_replica \
+                        mariadb_list_databases | sort -u)
+
+        declare -a dbs_missing_from_replica
+        mapfile -t dbs_missing_from_replica < \
+                <(comm -23 -- \
+                        <(printarray primary_dbs) \
+                        <(printarray replica_dbs))
+
+        declare -a to_replicate
+        if [ $# = 0 ]
+        then
+                to_replicate=("${dbs_missing_from_replica[@]}")
+        else
+                mapfile -t to_replicate < \
+                        <(comm -12 -- \
+                                <(printarray dbs_missing_from_replica) \
+                                <(printlines "$@"))
+        fi
+        if [ ${#to_replicate[@]} -gt 0 ]
+        then
+                run_primary \
+                        send_mariadb_dump "$replica_host" \
+                        --master-data \
+                        --gtid \
+                        --single-transaction \
+                        --databases "${to_replicate[@]}"
+        fi
 }
 
 main()
 {
         set -e
         check_input
+
         run_primary set_server_id
-        run_both list_databases
 
         # run_primary check_db
         # run_replica check_db
         # showvars replica
 
-        run_primary create_replication_user \
-                "$replica_host" \
-                "$replication_user" \
-                "$replication_password"
-
-        # mariadbdump_create_replica
-        run_both list_databases
-        mariabackup_create_replica_databases
-        run_both list_databases
-
-        run_replica check_db
-        run_replica enable_replication \
-                "$primary_host" \
-                "$replication_user" \
-                "$replication_password"
+        mariadbdump_transfer_missing_databases "$@"
+        run_replica list_databases
         run_both list_databases
 }
 
@@ -358,7 +407,7 @@ from
 .
 }
 
-if true
+if false
 then
         cleanup_after_test
 fi