path: root/src/mariadb-push-replica.sh

#!/bin/bash
set -e
PATH=$(dirname "$0"):$PATH
source rpc.bash

datadir=/etc/hosting-tools
default_replica_host_file=$datadir/default-replica-host
replication_password_file=$datadir/replication-password

if [ -r "$default_replica_host_file" ]
then
	read default_replica_host < "$default_replica_host_file"
fi

gen_password()
{
	generated_password_length=32
	generated_password=$(tr -cd a-zA-Z0-9 < /dev/urandom |
		head -c "$generated_password_length")
	[ "${#generated_password}" -eq "$generated_password_length" ]
	printf '%s\n' "$generated_password"
}

if ! [ -e "$replication_password_file" ]
then
	gen_password > "$replication_password_file"
fi

if [ -r "$replication_password_file" ]
then
	read replication_password < "$replication_password_file" 
	[ "${#replication_password}" -ge 30 ]
fi

primary_host=$(hostname --fqdn)
if [ $# = 0 ]
then
	replica_host=$default_replica_host
else
	replica_host=${1:-$default_replica_host}
	shift
fi
replication_user=replication

run_primary()
{
	(set -x
	: primary : $1 $2 ${3:+ ...})
	BASH_RPC_REMOTE_DEST=$primary_host remote_run_function "$@"
}

run_replica()
{
	(set -x
	: replica : $1 $2 ${3:+ ...})
	BASH_RPC_REMOTE_DEST=$replica_host remote_run_function "$@"
}

show_hostnames()
{
	printf \
		"==> %s %s <==\n    %s\n" \
		"$(hostname -A)" \
		"$(hostname -I)" \
		"$(uptime)"
}

check_db()
{
ls -lFC --color /var/lib/mysql
mariadb -t "$@" <<.
select
	user()
,	@@hostname
,	@@server_id
,	@@gtid_slave_pos
,	@@sql_log_bin
;
select
	@@hostname
,	schema_name as 'database'
from
	information_schema.schemata
;
.
}

create_backup()
{
	mostly_silent_unless_error create_backup_verbose "$@"
}

create_backup_verbose()
{
	set -e
	mariabackup_target_dir=/var/mariadb/backup
	binlog_info_file=$mariabackup_target_dir/xtrabackup_binlog_info 
	[ -e "$binlog_info_file" ] && return
	mkdir -p "$mariabackup_target_dir"
	set -- \
		-u root \
		--target-dir="$mariabackup_target_dir" \
		"$@"
	set -x
	mariabackup --backup --rsync "$@"
	mariabackup --prepare "$@"
	[ -e "$binlog_info_file" ]
}

delete_backup()
{
	set -x
	rm -r /var/mariadb/backup
}

send_backup()
{
	mariabackup_target_dir=/var/mariadb/backup
	set -x
	rsync -zaR -- /./"${mariabackup_target_dir#/}" "$1":/
}

stop_mariadb_server_and_remove_database_files()
{
	livedb=/var/lib/mysql
	set -e
	set -o pipefail
	[ -e "$livedb" ] || return 0

	if [ "$(systemctl is-active mariadb)" = active ]
	then
		systemctl stop mariadb
	fi
	livedb_backup=$livedb~$(date -Ins)
	mv -v -T -- "$livedb" "$livedb_backup"
	mkdir "$livedb"
	chown --reference="$livedb_backup" "$livedb"
	chmod --reference="$livedb_backup" "$livedb"
}

restore_from_backup()
{
	mariabackup_target_dir=/var/mariadb/backup
	set -e
	stop_mariadb_server_and_remove_database_files
	set -- \
		--force-non-empty-directories \
		-u root \
		--target-dir="$mariabackup_target_dir"
	set -- "$@"
	if mostly_silent_unless_error \
		mariabackup --move-back "$@"
	then
		# Can't believe mariabackup
		# is so primitive as to
		# recommend this chown in its
		# documentation
		chown -R mysql:mysql /var/lib/mysql
		systemctl start mariadb
		set_server_id
	else
		exit $?
	fi
}

silent_unless_error()
{
	set -- "$(mktemp)" "$@"
	if
		"${@:2}" >"$1" 2>&1
	then
		local r=0
	else
		local r=$?
		cat "$1" >&2
	fi
	rm "$1"
	return $r
}

mostly_silent_unless_error()
{
	echo "+ $*" >&2
	set -- "$(mktemp)" "$@"
	if
		"${@:2}" >"$1" 2>&1
	then
		tail -n3 "$1" >&2
		local r=0
	else
		local r=$?
		cat "$1" >&2
	fi
	rm "$1"
	return $r
}

enable_replication_via_mariabackup_xtra_info()
{
	primary_host=$1
	replication_user=$2
	replication_password=$3
	mariabackup_target_dir=/var/mariadb/backup
	binlog_info_file=$mariabackup_target_dir/xtrabackup_binlog_info 
	set -e
	read master_log_file master_log_pos gtid_slave_pos < $binlog_info_file
	tee /dev/stderr <<END | mariadb --skip-reconnect -t
STOP SLAVE;
SET GLOBAL gtid_slave_pos = "$gtid_slave_pos";
CHANGE MASTER TO
  MASTER_HOST='$primary_host',
  MASTER_USER='$replication_user',
  MASTER_PASSWORD='$replication_password',
  MASTER_USE_GTID=slave_pos;
START SLAVE;
SHOW SLAVE STATUS\G
END
	set -x
	rm -r /var/mariadb/backup
}

create_replication_user()
{
	mariadb --skip-reconnect -t <<END
CREATE OR REPLACE USER '$2'@'$1' IDENTIFIED BY '$3';
GRANT REPLICATION SLAVE ON *.*  TO '$2'@'$1';
END
}

showvars()
{
	for h in "$@"
	do
		run_$h mariadb --skip-reconnect -t <<END
select VARIABLE_NAME,SESSION_VALUE,GLOBAL_VALUE 
from INFORMATION_SCHEMA.SYSTEM_VARIABLES
where VARIABLE_NAME LIKE '%SLAVE%' \G
END
	done
}

check_input()
{
	[ "$primary_host" ]
	[ "$replica_host" ]
	[ "$replication_user" ]
	[ "$replication_password" ]
	dns_check_servers
}

dns_check_servers()
{
	primary_ipv4=$(dig +short -ta "$primary_host")
	replica_ipv4=$(dig +short -ta "$replica_host")
	echo "primary: $primary_host $primary_ipv4"
	echo "replica: $replica_host $replica_ipv4"
}

truncated_machineid_decimal_string_int32()
{
	systemd-id128 machine-id |
	sha256sum |
	( read -n8 &&
	printf '%u\n' 0x"$REPLY" )
}

run_both()
{
	set -e
	r=0
	run_primary "$@" || r=$?
	run_replica "$@"
	return $r
}

set_server_id()
{
	set -e
	chosen_id=$(truncated_machineid_decimal_string_int32)
	[ "$chosen_id" -gt 1 ]
	mariadb --skip-reconnect -t <<END
set global server_id = $chosen_id;
END
}

mariadb_list_databases()
{
mariadb -Bsss "$@" <<.
select
	schema_name
from
	information_schema.schemata;
.
}

mariabackup_create_replica_databases()
{
	run_primary create_backup
	run_primary send_backup "$replica_host"
	run_replica restore_from_backup
	run_primary create_replication_user \
		"$replica_host" \
		"$replication_user" \
		"$replication_password"
	run_replica enable_replication_via_mariabackup_xtra_info \
		"$primary_host" \
		"$replication_user" \
		"$replication_password"
}

printlines()
{
	printf '%s\n' "$@"
}

printarray()
{
	declare -n _PRINTARRAY_VARNAME="$1"
	printf '%s\n' "${_PRINTARRAY_VARNAME[@]}"
}

# Call run_replica from here to avoid
# piping the database back to caller
# unnecessarily.  Better would be
# a direct connect from mariadb client
# to remote mariadb server; but that
# requires transmitting credentials.
# Credential-transporter.  Transporter-transporter.
# Well, a transporter protein is more like a provenance tag,
# and the transporter code is the receptor to the transporter
# which is the provenance checker.  But anyway we have that
# with ssh, except we don't: we are assuming the primary
# has the ssh root of the replica!  Insanity!  Now this only works
# because the code runs on the primary; unless we forward the
# ssh auth with the ssh agent; but that only makes the security
# flaw temporary not solved; in fact, the replica should receive
# the transmission on some limited authorization channel; which
# _could_ be ssh; in fact, the dump could transparently be either
# live or else cached on the server side; it could be the
# rsync.net backup even; but ... it needs to include
# the btrfs snap similarly ... .
send_mariadb_dump()
{
	(
		set -x
		mariadb-dump "${@:2}"
	) |
	replica_host="$1" run_replica receive_mariadb_dump
}

receive_mariadb_dump()
{
	pv -f | mariadb --skip-reconnect
}

save_array()
{
	declare -n _to_save="$1"
	case "$2 $3" in
		'from zfile' )
			mapfile -d '' -t _to_save < "$4"
			;;
		'from lines' )
			mapfile -t _to_save < "$4"
			;;
		* )
			false
			;;
	esac
}

mariadb_scan_databases()
{
	declare -g -a primary_dbs
	save_array primary_dbs from lines <(
		run_primary mariadb_list_databases </dev/null |
		sort -u
	)

	declare -g -a replica_dbs
	save_array replica_dbs from lines <(
		run_replica mariadb_list_databases </dev/null |
		sort -u
	)

	declare -g -a primary_dbs_not_on_replica
	save_array primary_dbs_not_on_replica from zfile <(
		subtract_arrays primary_dbs replica_dbs
	)
}

mariadbdump_transfer_missing_databases()
{
	mariadb_scan_databases
	declare -a to_replicate
	if [ $# = 0 ]
	then
		to_replicate=("${primary_dbs_not_on_replica[@]}")
		printf "Missing on ${replica_host}: %s\n" \
			"${to_replicate[@]}"
		exit
	else
		save_array to_replicate from lines \
			<(intersection_lines \
				<(printarray primary_dbs_not_on_replica) \
				<(printlines "$@" | sort -u))
	fi
	mariadbdump_transfer_databases \
		"$replica_host" \
		"${to_replicate[@]}"
}

mariadbdump_transfer_databases()
{
	[ $# -ge 2 ] || return
	run_primary \
		send_mariadb_dump "$1" \
		--master-data \
		--gtid \
		--single-transaction \
		--databases "${@:2}"
}

intersection_lines()
{
	comm -12 -- "$1" "$2"
}

subtract_arrays()
{
	declare -n _a="$1" _b="$2"
	comm -z -23 -- \
		<(printf '%s\0' "${_a[@]}") \
		<(printf '%s\0' "${_b[@]}")
}

main()
{
	set -e
	check_input

	run_primary set_server_id

	# run_primary check_db
	# run_replica check_db
	# showvars replica

	mariadbdump_transfer_missing_databases "$@"
	run_replica list_databases
	run_both list_databases
}

cleanup_after_test()
{
	run_primary delete_backup || true
}

list_databases()
{
mariadb -t "$@" <<.
select
	@@hostname
,	@@server_id
,	count(schema_name) as 'databases'
,	@@gtid_slave_pos
from
	information_schema.schemata
;
select
	schema_name as 'database'
from
	information_schema.schemata
;
.
}

if false
then
	cleanup_after_test
fi
main "$@"
exit $?