From a29223b97f5dce1e2f986559b9c431850e2c8a12 Mon Sep 17 00:00:00 2001 From: eleksir Date: Mon, 17 Jul 2017 16:30:13 +0300 Subject: [PATCH] add ceph maint. helpers --- ceph_osd_kickin.pl | 86 +++++++++++++++++++++++++ ceph_osd_kickoff_mt.pl | 141 +++++++++++++++++++++++++++++++++++++++++ prio-client.sh | 15 +++++ prio-recovery.sh | 14 ++++ 4 files changed, 256 insertions(+) create mode 100755 ceph_osd_kickin.pl create mode 100755 ceph_osd_kickoff_mt.pl create mode 100644 prio-client.sh create mode 100644 prio-recovery.sh diff --git a/ceph_osd_kickin.pl b/ceph_osd_kickin.pl new file mode 100755 index 0000000..7daf658 --- /dev/null +++ b/ceph_osd_kickin.pl @@ -0,0 +1,86 @@ +#!/usr/bin/perl + +use warnings "all"; +use strict; +use Fcntl; + +sub get_ceph_status(); # defined - not healthy | undef - healthy +sub mark_osd_out($); +sub logger($); +sub restore_prio(@); + +if ((int(@ARGV) < 1) or ($ARGV[0] eq '--help') or ($ARGV[0] eq '-h')) { + print "\n Usage: $0 osd_num osd_num osd_num\n\n"; + print " Example: user\@host\# $0 10 25 44 128 9 36 33 78 91\n\n"; + exit 0; +} + +my @osd_list = @ARGV; +my $PACK = 2; +my $OSDINDEX = 0; + +while ( 1 ) { + while ($PACK >0) { + if (defined($osd_list[$OSDINDEX])) { + logger("Mark $osd_list[$OSDINDEX] as in"); + mark_osd_in($osd_list[$OSDINDEX]); + } else { +# last (and not full) pack of osds, we _can_ quit now + logger("All OSD marked as in"); + exit 0; + } + + $OSDINDEX = $OSDINDEX + 1; + $PACK = $PACK - 1; + sleep 300; + } + + while (defined(get_ceph_status())) { + sleep 60; + } + + $PACK = 2; + restore_prio(@osd_list); +} + +sub get_ceph_status() { + my $status = `/bin/ceph -s`; + my @msg = split(/\n/, $status); +# by default we assume that ceph is unhealthy, with misplaced pgs, and degraded pgs + my $healthy = undef; + my $misplaced = undef; + my $degraded = undef; + + foreach (@msg) { + return undef if ($_ =~ /health HEALTH_OK$/); + $degraded = 1 if ($_ =~ /recovery \d+\/\d+ objects degraded/); + $misplaced = 1 if ($_ =~ /recovery \d+\/\d+ objects misplaced/) + } + + if ((defined($degraded)) or (defined($misplaced))) { + return 1; + } + + return undef; +} + +sub mark_osd_in($) { + my $osd = shift; + `/bin/ceph osd in $osd`; + sleep(90); + `./prio-recovery.sh $osd`; +} + +sub logger($) { + my $msg = shift; + syswrite STDOUT, "$msg\n"; +} + +sub restore_prio(@) { + my @osds = @_; + + foreach(@osds) { + `./prio-client.sh $_` + } +} + diff --git a/ceph_osd_kickoff_mt.pl b/ceph_osd_kickoff_mt.pl new file mode 100755 index 0000000..7c60ea6 --- /dev/null +++ b/ceph_osd_kickoff_mt.pl @@ -0,0 +1,141 @@ +#!/usr/bin/perl + +use warnings "all"; +use strict; +use Fcntl; +use threads; +use threads::shared; + + +sub get_ceph_status(); # defined - not healthy | undef - healthy +sub get_osd_status($); # give osd id, returns 1 if osd have pgs | undef - no pg left on osd +sub mark_osd_out($); +sub logger($); +sub metabackup(@); +sub worker(@); + + +if ((int(@ARGV) < 1) or ($ARGV[0] eq '--help') or ($ARGV[0] eq '-h')) { + print "\n Usage: $0 osd_num osd_num osd_num\n\n"; + print " Example: user\@host\# $0 10 25 44 128 9 36 33 78 91\n\n"; + exit 0; +} + +my @osd_list = @ARGV; +my $PACK = 5; +my $OSDINDEX = 0; +my %JOBS; +my $OSD; +my @threads; + +for (my $i = 0; $i < @osd_list; $i++) { + if ($OSDINDEX >= $PACK) { + $OSDINDEX = 0; + } + + ${JOBS}{$OSDINDEX}{$osd_list[$i]} = 1; + $OSDINDEX++; +} + +# ${JOBS}{$OSDINDEX}{OSD_ID} = 1 +# hash of hashes of hashes + +foreach (keys(%JOBS)) { + my @jobs = keys($JOBS{$_}); + push @threads, threads->create(\&worker, @jobs); + sleep (300); # do not start all pack in one shot +} + +# wait 'till jobs are complete +foreach (@threads) { + $_->join(); +} + +print "All requested OSDS are out, performing backup\n"; +metabackup(@osd_list); + +exit 0; + +sub get_ceph_status() { + my $status = `/bin/ceph -s`; + my @msg = split(/\n/, $status); +# by default we assume that ceph is unhealthy, with misplaced pgs, and degraded pgs + my $healthy = undef; + my $misplaced = undef; + my $degraded = undef; + + foreach (@msg) { + return undef if ($_ =~ /health HEALTH_OK$/); + $degraded = 1 if ($_ =~ /recovery \d+\/\d+ objects degraded/); + $misplaced = 1 if ($_ =~ /recovery \d+\/\d+ objects misplaced/) + } + + if ((defined($degraded)) or (defined($misplaced))) { + return 1; + } + + return undef; +} + +sub get_osd_status($) { + my $osd = shift; + my $status = `ceph osd df`; + + foreach (split(/\n/, $status)) { + if ($_ =~ /^ ? ?$osd /) { + $status = $_; + } + } + + if ($status =~ / 0.$/) { +# zero placement groups here + return undef; + } + + return 1; +} + +sub mark_osd_out($) { + my $osd = shift; + `/bin/ceph osd out $osd`; +} + +sub logger($) { + my $msg = shift; + syswrite STDOUT, "$msg\n"; +} + +sub metabackup(@) { + my @list = @_; + +# just to make sure :) + while (defined(get_ceph_status())) { + sleep 60; + } + + `mount|sudo grep ceph >/ceph-mountmap`; + + system('/bin/sudo', '/bin/mkdir', "/root/ceph"); + chdir('/var/lib/ceph/osd'); + my $osd; + + foreach $osd (@list) { + system('/bin/sudo', "/bin/systemctl", "stop", sprintf("ceph-osd@%s", $osd)); + sleep 1; + system('/bin/sudo', '/bin/tar', '-zcf', sprintf("/root/ceph/ceph-%s.tar.gz", $osd), sprintf("ceph-%s", $osd)); + } +} + +sub worker(@) { + my @OSD = @_; + + foreach my $osd (@OSD) { + mark_osd_out($osd); + sleep(15); # ceph is slow + + while (defined(get_osd_status($osd))) { + sleep 150; + } + } +} + diff --git a/prio-client.sh b/prio-client.sh new file mode 100644 index 0000000..3e26707 --- /dev/null +++ b/prio-client.sh @@ -0,0 +1,15 @@ +#!/bin/bash + +OSD="$1" + +if [ "$OSD" = "" ]; then + echo "OSD ID is not set" + exit 0 +fi + +echo "==> $OSD" +ceph tell osd.$OSD injectargs '--osd_recovery_op_priority 1' +ceph tell osd.$OSD injectargs '--osd_recovery_threads 6' +ceph tell osd.$OSD injectargs '--osd_recovery_max_active 1' +ceph tell osd.$OSD injectargs '--osd_max_backfills 1' + diff --git a/prio-recovery.sh b/prio-recovery.sh new file mode 100644 index 0000000..458583e --- /dev/null +++ b/prio-recovery.sh @@ -0,0 +1,14 @@ +#!/bin/bash + +OSD="$1" + +if [ "$OSD" = "" ]; then + echo "OSD ID is not set" + exit 0 +fi + +ceph tell osd.$OSD injectargs '--osd_recovery_op_priority 10' +ceph tell osd.$OSD injectargs '--osd_recovery_threads 10' +ceph tell osd.$OSD injectargs '--osd_recovery_max_active 3' +ceph tell osd.$OSD injectargs '--osd_max_backfills 10' +