From f1ad32d5aebf572a4f54fbcfb190f238aaac05e2 Mon Sep 17 00:00:00 2001 From: eleksir Date: Mon, 5 Jul 2021 16:56:01 +0300 Subject: [PATCH] Add migration scripts and doc. When I move bot from perl-5.30.0 to perl-5.32.1 I encounter with BDB0091 DB_VERSION_MISMATCH error And it looks like I have to export essential data on older perl to some portable format and then import it on newer perl environment. --- Migration.txt | 30 +++++++++++++ bin/cache_export.pl | 106 ++++++++++++++++++++++++++++++++++++++++++++ bin/cache_import.pl | 86 +++++++++++++++++++++++++++++++++++ 3 files changed, 222 insertions(+) create mode 100644 Migration.txt create mode 100644 bin/cache_export.pl create mode 100644 bin/cache_import.pl diff --git a/Migration.txt b/Migration.txt new file mode 100644 index 0000000..df3a55f --- /dev/null +++ b/Migration.txt @@ -0,0 +1,30 @@ +Migration procedure. + +When you require to perform this procedure? +- When perl version is changed +- When migration to other OS or distribution occurs + +Why do i have to perform this procedure? +- Some modules are vendored with bot +- BDB can vary its on-disk format depending on dbd version that linked with + perl interpreter itself and its apropriate module. + +How can i guess that i miss to perform this procedure? +Most notable sympthom is 'BDB0091 DB_VERSION_MISMATCH' in logs and bot admin settings are lost. + + +Procedure steps. + +On source system perform following steps: +1. stop the bot, to unlock and close all files. +2. make a backup +3. prepare target system and copy bot to target system + +On target system perform these steps: +1. remove vendor_perl dir in bot' directory +2. run "bash bootstrap.sh" (no quotes) and wait until it suscessfuly build and + installl required modules. +3. run perl bin/cache_import.pl script. It should create cache and exit + silently. + +Now it is possible to run bot without settings loss. diff --git a/bin/cache_export.pl b/bin/cache_export.pl new file mode 100644 index 0000000..a84d82a --- /dev/null +++ b/bin/cache_export.pl @@ -0,0 +1,106 @@ +#!/usr/bin/perl + +use 5.018; +use strict; +use warnings; +use utf8; +use open qw (:std :utf8); +use English qw ( -no_match_vars ); +use version; our $VERSION = qw (1.0); + +my $workdir; + +# before we run, change working dir +BEGIN { + use Cwd qw (chdir abs_path); + my @CWD = split /\//xms, abs_path ($PROGRAM_NAME); + if ($#CWD > 1) { $#CWD = $#CWD - 2; } + $workdir = join '/', @CWD; + chdir $workdir; +} + +use lib ("$workdir/lib", "$workdir/vendor_perl", "$workdir/vendor_perl/lib/perl5"); +use CHI; +use CHI::Driver::BerkeleyDB; +use File::Basename qw (dirname); +use File::Path qw (make_path); +use JSON::XS; +use Encode; + +use BotLib::Conf qw (LoadConf); + +my $c = LoadConf (); +my $cachedir = $c->{cachedir}; +my $datadir = dirname $cachedir; +my $export = 0; +my $exportInst = 0; + +sub dumpCache { + my $namespace = shift; + my $cacheExportDir = sprintf '%s/cacheexport', $datadir; + + unless (-d $cacheExportDir) { + make_path ($cacheExportDir) || die "Unable to create $cacheExportDir: $OS_ERROR\n"; + } + + my $filename = sprintf '%s/%s', $cacheExportDir, $exportInst; + + my $cache = CHI->new ( + driver => 'BerkeleyDB', + root_dir => $cachedir, + namespace => $namespace + ); + + my $j; + my $writefile = 0; + + foreach my $k ($cache->get_keys ()) { + $k = decode ('UTF-8', $k); + $j->{$k} = decode ('UTF-8', $cache->get ($k)); + $writefile = 1; + } + + if ($writefile) { + say "Dumping $namespace"; ## no critic (InputOutput::RequireCheckedSyscalls) + my $jsonencoder = JSON::XS->new->pretty (); + open (my $FILE, '>', $filename) or die "Unable to dump cache namespace $namespace to $filename: $OS_ERROR\n"; + print $FILE $jsonencoder->encode ($j) or die "Unable to dump cache namespace $namespace to $filename: $OS_ERROR\n";; + close $FILE; ## no critic (InputOutput::RequireCheckedSyscalls) + my $EXPORTFD; + + if ($export) { + open ($EXPORTFD, '>>', "$cacheExportDir/exportfiles.txt") or die "Unable to open for appending $cacheExportDir/exportfiles.txt: $OS_ERROR\n"; + } else { + open ($EXPORTFD, '>', "$cacheExportDir/exportfiles.txt") or die "Unable to create $cacheExportDir/exportfiles.txt: $OS_ERROR\n"; + $export = 1; + } + + printf ($EXPORTFD "%s,%s\n", $namespace, $filename) or die "Unable to write to $cacheExportDir/exportfiles.txt: $OS_ERROR\n"; + close $EXPORTFD; ## no critic (InputOutput::RequireCheckedSyscalls) + $exportInst++; + } + + return; +} + +opendir (my $DIR, $cachedir) or die "Unable to open $cachedir: $OS_ERROR\n"; + +while (readdir $DIR) { + if (/^BotLib\+3a\+3aAdmin_censor_(.+)\.db$/) { + dumpCache ("BotLib::Admin_censor_$1"); + } elsif (/^BotLib\+3a\+3aAdmin_fortune.db$/) { + dumpCache ('BotLib::Admin_fortune'); + } elsif (/BotLib\+3a\+3aAdmin_plugin_(.+)\.db$/) { + dumpCache ("BotLib::Admin_plugin_$1"); + } elsif (/BotLib\+3a\+3aAdmin_(.+)\.db$/) { + dumpCache ("BotLib::Admin_$1"); + } elsif (/^BotLib\+3a\+3aImage\+3a\+3aFlickr\.db$/) { + dumpCache ('BotLib::Image::Flickr'); + } elsif (/^BotLib\+3a\+3aKarma_(.+)\.db$/) { + dumpCache ("BotLib::Karma_$1"); + } +} + +closedir $DIR; +exit 0; + diff --git a/bin/cache_import.pl b/bin/cache_import.pl new file mode 100644 index 0000000..159c162 --- /dev/null +++ b/bin/cache_import.pl @@ -0,0 +1,86 @@ +#!/usr/bin/perl + +use 5.018; +use strict; +use warnings; +use utf8; +use open qw (:std :utf8); +use English qw ( -no_match_vars ); +use version; our $VERSION = qw (1.0); + +my $workdir; + +# before we run, change working dir +BEGIN { + use Cwd qw (chdir abs_path); + my @CWD = split /\//xms, abs_path ($PROGRAM_NAME); + if ($#CWD > 1) { $#CWD = $#CWD - 2; } + $workdir = join '/', @CWD; + chdir $workdir; +} + +use lib ("$workdir/lib", "$workdir/vendor_perl", "$workdir/vendor_perl/lib/perl5"); +use CHI; +use CHI::Driver::BerkeleyDB; +use File::Basename qw (dirname); +use File::Path qw (make_path); +use JSON::XS; +use Encode; + +use BotLib::Conf qw (LoadConf); + +my $c = LoadConf (); +my $cachedir = $c->{cachedir}; +my $datadir = dirname $cachedir; +my $export = 0; + +sub createCache { + my $namespace = shift; + my $filename = shift; + say "Creating $namespace from $filename"; ## no critic (InputOutput::RequireCheckedSyscalls) + + my $cache = CHI->new ( + driver => 'BerkeleyDB', + root_dir => $cachedir, + namespace => $namespace + ); + + open my $FH, '<', $filename or die "Unable to open $filename: $OS_ERROR\n"; + binmode $FH; + my $len = (stat $filename) [7]; + my $json; + use bytes; + my $readlen = read $FH, $json, $len; + no bytes; + + unless ($readlen) { + close $FH; ## no critic (InputOutput::RequireCheckedSyscalls + die "Unable to read $filename: $OS_ERROR\n"; + } + + if ($readlen != $len) { + close $FH; ## no critic (InputOutput::RequireCheckedSyscalls + die "File $filename is $len bytes on disk, but we read only $readlen bytes\n"; + } + + close $FH; ## no critic (InputOutput::RequireCheckedSyscalls + my $j = JSON::XS->new->utf8->relaxed; + my $dump = $j->decode ($json); + + foreach my $key (keys (%{$dump})) { + $cache->set ($key, $dump->{$key}, 'never'); + } + + return; +} + +open (my $FH, '<', "$datadir/cacheexport/exportfiles.txt") or die "Unable to open $datadir/cacheexport/exportfiles.txt: $OS_ERROR\n"; + +while (<$FH>) { + chomp; + my ($namespace, $filename) = split /\,/, $_, 2; + createCache ($namespace, $filename); +} + +close $FH; ## no critic (InputOutput::RequireCheckedSyscalls) +exit 0;