add ceph maint. helpers

ceph_osd_kickin.pl

@@ -0,0 +1,86 @@
+#!/usr/bin/perl
+
+use warnings "all";
+use strict;
+use Fcntl;
+
+sub get_ceph_status(); # defined - not healthy | undef - healthy
+sub mark_osd_out($);
+sub logger($);
+sub restore_prio(@);
+
+if ((int(@ARGV) < 1) or ($ARGV[0] eq '--help') or ($ARGV[0] eq '-h')) {
+	print "\n    Usage:   $0 osd_num osd_num osd_num\n\n";
+	print "    Example: user\@host\# $0 10 25 44 128 9 36 33 78 91\n\n";
+	exit 0;
+}
+
+my @osd_list = @ARGV;
+my $PACK = 2;
+my $OSDINDEX = 0;
+
+while ( 1 ) {
+	while ($PACK >0) {
+		if (defined($osd_list[$OSDINDEX])) {
+			logger("Mark $osd_list[$OSDINDEX] as in");
+			mark_osd_in($osd_list[$OSDINDEX]);
+		} else {
+# last (and not full) pack of osds, we _can_ quit now
+			logger("All OSD marked as in");
+			exit 0;
+		}
+
+		$OSDINDEX = $OSDINDEX + 1;
+		$PACK = $PACK - 1;
+		sleep 300;
+	}
+
+	while (defined(get_ceph_status())) {
+		sleep 60;
+	}
+
+	$PACK = 2;
+	restore_prio(@osd_list);
+}
+
+sub get_ceph_status() {
+	my $status = `/bin/ceph -s`;
+	my @msg = split(/\n/, $status);
+# by default we assume that ceph is unhealthy, with misplaced pgs, and degraded pgs
+	my $healthy = undef;
+	my $misplaced = undef;
+	my $degraded = undef;
+
+	foreach (@msg) {
+		return undef if ($_ =~ /health HEALTH_OK$/);
+		$degraded = 1 if ($_ =~ /recovery \d+\/\d+ objects degraded/);
+		$misplaced = 1 if ($_ =~ /recovery \d+\/\d+ objects misplaced/)
+	}
+
+	if ((defined($degraded)) or (defined($misplaced))) {
+		return 1;
+	}
+	
+	return undef;
+}
+	
+sub mark_osd_in($) {
+	my $osd = shift;
+	`/bin/ceph osd in $osd`;
+	sleep(90);
+	`./prio-recovery.sh $osd`;
+}
+	
+sub logger($) {
+	my $msg = shift;
+	syswrite STDOUT, "$msg\n";
+}
+
+sub restore_prio(@) {
+	my @osds = @_;
+
+	foreach(@osds) {
+		`./prio-client.sh $_`
+	}
+}
+

ceph_osd_kickoff_mt.pl

@@ -0,0 +1,141 @@
+#!/usr/bin/perl
+
+use warnings "all";
+use strict;
+use Fcntl;
+use threads;
+use threads::shared;
+
+
+sub get_ceph_status(); # defined - not healthy | undef - healthy
+sub get_osd_status($); # give osd id, returns 1 if osd have pgs | undef - no pg left on osd
+sub mark_osd_out($);
+sub logger($);
+sub metabackup(@);
+sub worker(@);
+
+
+if ((int(@ARGV) < 1) or ($ARGV[0] eq '--help') or ($ARGV[0] eq '-h')) {
+	print "\n    Usage:   $0 osd_num osd_num osd_num\n\n";
+	print "    Example: user\@host\# $0 10 25 44 128 9 36 33 78 91\n\n";
+	exit 0;
+}
+
+my @osd_list = @ARGV;
+my $PACK = 5;
+my $OSDINDEX = 0;
+my %JOBS;
+my $OSD;
+my @threads;
+
+for (my $i = 0; $i < @osd_list; $i++) {
+	if ($OSDINDEX >= $PACK) {
+		$OSDINDEX = 0;
+	}
+	
+	${JOBS}{$OSDINDEX}{$osd_list[$i]} = 1;
+	$OSDINDEX++;	
+}
+
+# ${JOBS}{$OSDINDEX}{OSD_ID} = 1
+#  hash of hashes of hashes
+
+foreach (keys(%JOBS)) {
+	my @jobs = keys($JOBS{$_});
+	push @threads, threads->create(\&worker, @jobs);
+	sleep (300); # do not start all pack in one shot	
+}
+
+# wait 'till jobs are complete
+foreach (@threads) {
+	$_->join();
+}
+
+print "All requested OSDS are out, performing backup\n";
+metabackup(@osd_list);
+
+exit 0;
+
+sub get_ceph_status() {
+	my $status = `/bin/ceph -s`;
+	my @msg = split(/\n/, $status);
+# by default we assume that ceph is unhealthy, with misplaced pgs, and degraded pgs
+	my $healthy = undef;
+	my $misplaced = undef;
+	my $degraded = undef;
+
+	foreach (@msg) {
+		return undef if ($_ =~ /health HEALTH_OK$/);
+		$degraded = 1 if ($_ =~ /recovery \d+\/\d+ objects degraded/);
+		$misplaced = 1 if ($_ =~ /recovery \d+\/\d+ objects misplaced/)
+	}
+
+	if ((defined($degraded)) or (defined($misplaced))) {
+		return 1;
+	}
+	
+	return undef;
+}
+
+sub get_osd_status($) {
+	my $osd = shift;
+	my $status = `ceph osd df`;
+	
+	foreach (split(/\n/, $status)) {
+		if ($_ =~ /^ ? ?$osd /) {
+			$status = $_;
+		}
+	}
+	
+	if ($status =~ / 0.$/) {
+# zero placement groups here
+		return undef;
+	}
+	
+	return 1;
+}
+
+sub mark_osd_out($) {
+	my $osd = shift;
+	`/bin/ceph osd out $osd`;
+}
+	
+sub logger($) {
+	my $msg = shift;
+	syswrite STDOUT, "$msg\n";
+}
+
+sub metabackup(@) {
+	my @list = @_;
+
+# just to make sure :)
+	while (defined(get_ceph_status())) {
+		sleep 60;
+	}
+
+	`mount|sudo grep ceph >/ceph-mountmap`;
+
+        system('/bin/sudo', '/bin/mkdir', "/root/ceph");
+	chdir('/var/lib/ceph/osd');
+	my $osd;
+
+	foreach $osd (@list) {
+		system('/bin/sudo', "/bin/systemctl", "stop", sprintf("ceph-osd@%s", $osd));
+		sleep 1;
+		system('/bin/sudo', '/bin/tar', '-zcf', sprintf("/root/ceph/ceph-%s.tar.gz", $osd), sprintf("ceph-%s", $osd));
+	}
+}
+
+sub worker(@) {
+	my @OSD = @_;
+
+	foreach my $osd (@OSD) {
+		mark_osd_out($osd);
+		sleep(15); # ceph is slow
+
+		while (defined(get_osd_status($osd))) {
+			sleep 150;
+		}
+	}
+}
+

prio-client.sh

@@ -0,0 +1,15 @@
+#!/bin/bash
+
+OSD="$1"
+
+if [ "$OSD" = "" ]; then
+        echo "OSD ID is not set"
+        exit 0
+fi
+
+echo "==> $OSD"
+ceph tell osd.$OSD injectargs '--osd_recovery_op_priority 1'
+ceph tell osd.$OSD injectargs '--osd_recovery_threads 6'
+ceph tell osd.$OSD injectargs '--osd_recovery_max_active 1'
+ceph tell osd.$OSD injectargs '--osd_max_backfills 1'
+

prio-recovery.sh

@@ -0,0 +1,14 @@
+#!/bin/bash
+
+OSD="$1"
+
+if [ "$OSD" = "" ]; then
+        echo "OSD ID is not set"
+        exit 0
+fi
+
+ceph tell osd.$OSD injectargs '--osd_recovery_op_priority 10'
+ceph tell osd.$OSD injectargs '--osd_recovery_threads 10'
+ceph tell osd.$OSD injectargs '--osd_recovery_max_active 3'
+ceph tell osd.$OSD injectargs '--osd_max_backfills 10'
+