Initial commit
This commit is contained in:
eleksir
10
README.md
Normal file
10
README.md
Normal file
@@ -0,0 +1,10 @@
|
||||
## CEPH storage helper scripts
|
||||
|
||||
* ceph-osd-list.pl - another listing presetation of listing node<->osd
|
||||
* ceph_osd_kickin.pl - kicks in big amount of osds in cluster by small batches
|
||||
* ceph_osd_kickoff_mt.pl - ejects big amount of osds from cluster by small batches
|
||||
* ceph_osd_pool_discovery.pl - script for zabbix' pool discovery
|
||||
* ceph_osd_pool_stats.pl - zabbix monitoring script
|
||||
* ceph_reweight.pl - carefuly reweights osd when "near full osd" cluster health warning appears
|
||||
* prio-client.sh - gives priority to client io operations
|
||||
* prio-recovery.sh - gives priority to recovery io operations
|
||||
22
ceph-osd-list.pl
Normal file
22
ceph-osd-list.pl
Normal file
@@ -0,0 +1,22 @@
|
||||
#!/usr/bin/perl
|
||||
#
|
||||
use strict;
|
||||
use warnings "all";
|
||||
use JSON::PP;
|
||||
|
||||
my $jstr = `ceph osd tree --format=json`;
|
||||
|
||||
my $json = decode_json $jstr;
|
||||
|
||||
my @nodes = @{ $json->{'nodes'} };
|
||||
|
||||
foreach my $node (@nodes) {
|
||||
if ($node->{'type'} eq 'host') {
|
||||
print "Хост: $node->{'name'}\n";
|
||||
my @children = @{ $node->{'children'} };
|
||||
|
||||
foreach my $child (@children) {
|
||||
print "Диск: osd.$child\n";
|
||||
}
|
||||
}
|
||||
}
|
||||
86
ceph_osd_kickin.pl
Normal file
86
ceph_osd_kickin.pl
Normal file
@@ -0,0 +1,86 @@
|
||||
#!/usr/bin/perl
|
||||
|
||||
use warnings "all";
|
||||
use strict;
|
||||
use Fcntl;
|
||||
|
||||
sub get_ceph_status(); # defined - not healthy | undef - healthy
|
||||
sub mark_osd_out($);
|
||||
sub logger($);
|
||||
sub restore_prio(@);
|
||||
|
||||
if ((int(@ARGV) < 1) or ($ARGV[0] eq '--help') or ($ARGV[0] eq '-h')) {
|
||||
print "\n Usage: $0 osd_num osd_num osd_num\n\n";
|
||||
print " Example: user\@host\# $0 10 25 44 128 9 36 33 78 91\n\n";
|
||||
exit 0;
|
||||
}
|
||||
|
||||
my @osd_list = @ARGV;
|
||||
my $PACK = 2;
|
||||
my $OSDINDEX = 0;
|
||||
|
||||
while ( 1 ) {
|
||||
while ($PACK >0) {
|
||||
if (defined($osd_list[$OSDINDEX])) {
|
||||
logger("Mark $osd_list[$OSDINDEX] as in");
|
||||
mark_osd_in($osd_list[$OSDINDEX]);
|
||||
} else {
|
||||
# last (and not full) pack of osds, we _can_ quit now
|
||||
logger("All OSD marked as in");
|
||||
exit 0;
|
||||
}
|
||||
|
||||
$OSDINDEX = $OSDINDEX + 1;
|
||||
$PACK = $PACK - 1;
|
||||
sleep 300;
|
||||
}
|
||||
|
||||
while (defined(get_ceph_status())) {
|
||||
sleep 60;
|
||||
}
|
||||
|
||||
$PACK = 2;
|
||||
restore_prio(@osd_list);
|
||||
}
|
||||
|
||||
sub get_ceph_status() {
|
||||
my $status = `/bin/ceph -s`;
|
||||
my @msg = split(/\n/, $status);
|
||||
# by default we assume that ceph is unhealthy, with misplaced pgs, and degraded pgs
|
||||
my $healthy = undef;
|
||||
my $misplaced = undef;
|
||||
my $degraded = undef;
|
||||
|
||||
foreach (@msg) {
|
||||
return undef if ($_ =~ /health HEALTH_OK$/);
|
||||
$degraded = 1 if ($_ =~ /recovery \d+\/\d+ objects degraded/);
|
||||
$misplaced = 1 if ($_ =~ /recovery \d+\/\d+ objects misplaced/)
|
||||
}
|
||||
|
||||
if ((defined($degraded)) or (defined($misplaced))) {
|
||||
return 1;
|
||||
}
|
||||
|
||||
return undef;
|
||||
}
|
||||
|
||||
sub mark_osd_in($) {
|
||||
my $osd = shift;
|
||||
`/bin/ceph osd in $osd`;
|
||||
sleep(90);
|
||||
`./prio-recovery.sh $osd`;
|
||||
}
|
||||
|
||||
sub logger($) {
|
||||
my $msg = shift;
|
||||
syswrite STDOUT, "$msg\n";
|
||||
}
|
||||
|
||||
sub restore_prio(@) {
|
||||
my @osds = @_;
|
||||
|
||||
foreach(@osds) {
|
||||
`./prio-client.sh $_`
|
||||
}
|
||||
}
|
||||
|
||||
141
ceph_osd_kickoff_mt.pl
Normal file
141
ceph_osd_kickoff_mt.pl
Normal file
@@ -0,0 +1,141 @@
|
||||
#!/usr/bin/perl
|
||||
|
||||
use warnings "all";
|
||||
use strict;
|
||||
use Fcntl;
|
||||
use threads;
|
||||
use threads::shared;
|
||||
|
||||
|
||||
sub get_ceph_status(); # defined - not healthy | undef - healthy
|
||||
sub get_osd_status($); # give osd id, returns 1 if osd have pgs | undef - no pg left on osd
|
||||
sub mark_osd_out($);
|
||||
sub logger($);
|
||||
sub metabackup(@);
|
||||
sub worker(@);
|
||||
|
||||
|
||||
if ((int(@ARGV) < 1) or ($ARGV[0] eq '--help') or ($ARGV[0] eq '-h')) {
|
||||
print "\n Usage: $0 osd_num osd_num osd_num\n\n";
|
||||
print " Example: user\@host\# $0 10 25 44 128 9 36 33 78 91\n\n";
|
||||
exit 0;
|
||||
}
|
||||
|
||||
my @osd_list = @ARGV;
|
||||
my $PACK = 5;
|
||||
my $OSDINDEX = 0;
|
||||
my %JOBS;
|
||||
my $OSD;
|
||||
my @threads;
|
||||
|
||||
for (my $i = 0; $i < @osd_list; $i++) {
|
||||
if ($OSDINDEX >= $PACK) {
|
||||
$OSDINDEX = 0;
|
||||
}
|
||||
|
||||
${JOBS}{$OSDINDEX}{$osd_list[$i]} = 1;
|
||||
$OSDINDEX++;
|
||||
}
|
||||
|
||||
# ${JOBS}{$OSDINDEX}{OSD_ID} = 1
|
||||
# hash of hashes of hashes
|
||||
|
||||
foreach (keys(%JOBS)) {
|
||||
my @jobs = keys($JOBS{$_});
|
||||
push @threads, threads->create(\&worker, @jobs);
|
||||
sleep (300); # do not start all pack in one shot
|
||||
}
|
||||
|
||||
# wait 'till jobs are complete
|
||||
foreach (@threads) {
|
||||
$_->join();
|
||||
}
|
||||
|
||||
print "All requested OSDS are out, performing backup\n";
|
||||
metabackup(@osd_list);
|
||||
|
||||
exit 0;
|
||||
|
||||
sub get_ceph_status() {
|
||||
my $status = `/bin/ceph -s`;
|
||||
my @msg = split(/\n/, $status);
|
||||
# by default we assume that ceph is unhealthy, with misplaced pgs, and degraded pgs
|
||||
my $healthy = undef;
|
||||
my $misplaced = undef;
|
||||
my $degraded = undef;
|
||||
|
||||
foreach (@msg) {
|
||||
return undef if ($_ =~ /health HEALTH_OK$/);
|
||||
$degraded = 1 if ($_ =~ /recovery \d+\/\d+ objects degraded/);
|
||||
$misplaced = 1 if ($_ =~ /recovery \d+\/\d+ objects misplaced/)
|
||||
}
|
||||
|
||||
if ((defined($degraded)) or (defined($misplaced))) {
|
||||
return 1;
|
||||
}
|
||||
|
||||
return undef;
|
||||
}
|
||||
|
||||
sub get_osd_status($) {
|
||||
my $osd = shift;
|
||||
my $status = `ceph osd df`;
|
||||
|
||||
foreach (split(/\n/, $status)) {
|
||||
if ($_ =~ /^ ? ?$osd /) {
|
||||
$status = $_;
|
||||
}
|
||||
}
|
||||
|
||||
if ($status =~ / 0.$/) {
|
||||
# zero placement groups here
|
||||
return undef;
|
||||
}
|
||||
|
||||
return 1;
|
||||
}
|
||||
|
||||
sub mark_osd_out($) {
|
||||
my $osd = shift;
|
||||
`/bin/ceph osd out $osd`;
|
||||
}
|
||||
|
||||
sub logger($) {
|
||||
my $msg = shift;
|
||||
syswrite STDOUT, "$msg\n";
|
||||
}
|
||||
|
||||
sub metabackup(@) {
|
||||
my @list = @_;
|
||||
|
||||
# just to make sure :)
|
||||
while (defined(get_ceph_status())) {
|
||||
sleep 60;
|
||||
}
|
||||
|
||||
`mount|sudo grep ceph >/ceph-mountmap`;
|
||||
|
||||
system('/bin/sudo', '/bin/mkdir', "/root/ceph");
|
||||
chdir('/var/lib/ceph/osd');
|
||||
my $osd;
|
||||
|
||||
foreach $osd (@list) {
|
||||
system('/bin/sudo', "/bin/systemctl", "stop", sprintf("ceph-osd@%s", $osd));
|
||||
sleep 1;
|
||||
system('/bin/sudo', '/bin/tar', '-zcf', sprintf("/root/ceph/ceph-%s.tar.gz", $osd), sprintf("ceph-%s", $osd));
|
||||
}
|
||||
}
|
||||
|
||||
sub worker(@) {
|
||||
my @OSD = @_;
|
||||
|
||||
foreach my $osd (@OSD) {
|
||||
mark_osd_out($osd);
|
||||
sleep(15); # ceph is slow
|
||||
|
||||
while (defined(get_osd_status($osd))) {
|
||||
sleep 150;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
24
ceph_osd_pool_discovery.pl
Normal file
24
ceph_osd_pool_discovery.pl
Normal file
@@ -0,0 +1,24 @@
|
||||
#!/usr/bin/perl
|
||||
|
||||
use strict;
|
||||
use warnings "all";
|
||||
|
||||
my $str = `ceph osd pool stats`;
|
||||
my @arr = split(/\n\n/, $str);
|
||||
|
||||
my $msg = "{\n \"data\": [\n";
|
||||
|
||||
foreach my $p (@arr) {
|
||||
my @str = split(/\s+/, $p);
|
||||
next if ((not defined($str[1])) or ($str[1] eq ''));
|
||||
next if ((not defined($str[3])) or ($str[3] eq ''));
|
||||
{# in jinja template engine this is comment, so in real script fix this #}
|
||||
$msg .= sprintf(" {\n \"\{#POOLID}\": \"%s\",\n", $str[3]);
|
||||
$msg .= sprintf(" \"\{#POOLNAME}\": \"%s\"\n },\n", $str[1]);
|
||||
}
|
||||
chop($msg);
|
||||
chop($msg);
|
||||
|
||||
$msg .= "\n ]\n}\n";
|
||||
print $msg;
|
||||
|
||||
93
ceph_osd_pool_stats.pl
Normal file
93
ceph_osd_pool_stats.pl
Normal file
@@ -0,0 +1,93 @@
|
||||
#!/usr/bin/perl
|
||||
|
||||
use strict;
|
||||
use warnings "all";
|
||||
|
||||
sub usage();
|
||||
# print usage banner if we have not enough arg
|
||||
usage() if ((not defined($ARGV[1])) or ($ARGV[1] eq ''));
|
||||
|
||||
my $pflag = 0;
|
||||
my $paramflag = 0;
|
||||
|
||||
my $str = `ceph osd pool stats`;
|
||||
my @arr = split(/\n\n/, $str);
|
||||
|
||||
my $msg = "{\n \"data\": [\n";
|
||||
|
||||
foreach my $p (@arr) {
|
||||
next unless (defined((split(/\s+/, $p))[1]));
|
||||
next unless ((split(/\s+/, $p))[1] eq $ARGV[0]);
|
||||
|
||||
if ((split(/\s+/, $p))[4] eq 'nothing') {
|
||||
syswrite STDOUT, '0';
|
||||
exit 0;
|
||||
}
|
||||
|
||||
my @data = split(/\n/, $p);
|
||||
|
||||
foreach my $l (@data) {
|
||||
if (substr($l, 2, 6) eq 'client') {
|
||||
if ($ARGV[1] eq 'crops') {
|
||||
syswrite STDOUT, (split(/\s/, $l))[10];
|
||||
} elsif ($ARGV[1] eq 'cwops') {
|
||||
syswrite STDOUT, (split(/\s/, $l))[13];
|
||||
} elsif ($ARGV[1] eq 'crbps') {
|
||||
if ((split(/\s/, $l))[5] =~ /^k/) {
|
||||
syswrite STDOUT, (split(/\s/, $l))[4] * 1024;
|
||||
} elsif ((split(/\s/, $l))[5] =~ /^M/) {
|
||||
syswrite STDOUT, (split(/\s/, $l))[4] * 1024 * 1024;
|
||||
} elsif ((split(/\s/, $l))[5] =~ /^G/) {
|
||||
syswrite STDOUT, (split(/\s/, $l))[4] * 1024 * 1024 * 1024;
|
||||
}
|
||||
} elsif ($ARGV[1] eq 'cwbps') {
|
||||
if ((split(/\s/, $l))[5] =~ /^k/) {
|
||||
syswrite STDOUT, (split(/\s/, $l))[4] * 1024;
|
||||
} elsif ((split(/\s/, $l))[5] =~ /^M/) {
|
||||
syswrite STDOUT, (split(/\s/, $l))[4] * 1024 * 1024;
|
||||
} elsif ((split(/\s/, $l))[5] =~ /^G/) {
|
||||
syswrite STDOUT, (split(/\s/, $l))[4] * 1024 * 1024 * 1024;
|
||||
}
|
||||
}
|
||||
} elsif (substr($l, 2, 8) eq 'recovery') {
|
||||
if ($ARGV[1] eq 'rbps') {
|
||||
if ((split(/\s/, $l))[5] =~ /^k/) {
|
||||
syswrite STDOUT, (split(/\s/, $l))[4] * 1024;
|
||||
} elsif ((split(/\s/, $l))[5] =~ /^M/) {
|
||||
syswrite STDOUT, (split(/\s/, $l))[4] * 1024 * 1024;
|
||||
} elsif ((split(/\s/, $l))[5] =~ /^G/) {
|
||||
syswrite STDOUT, (split(/\s/, $l))[4] * 1024 * 1024 * 1024;
|
||||
} else {
|
||||
# i never seen stats in bytes per second, so as an exception printout just zero
|
||||
syswrite STDOUT, '0';
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
exit 0;
|
||||
|
||||
sub usage () {
|
||||
print <<EOF;
|
||||
Usage: $0 <poolname> <parameter>
|
||||
|
||||
Where parameter can be:
|
||||
crops - client reads per second
|
||||
cwops - client writes per second
|
||||
crbps - client read kbytes per second
|
||||
cwbps - client write kbytes per second
|
||||
rbps - recovery io in bytes per second
|
||||
EOF
|
||||
exit 0;
|
||||
}
|
||||
|
||||
__END__
|
||||
|
||||
pool vms.sata.zone3 id 13
|
||||
nothing is going on
|
||||
|
||||
pool volumes.sata.zone1 id 14
|
||||
recovery io 1418 MB/s, 360 objects/s
|
||||
client io 24445 kB/s rd, 37316 kB/s wr, 975 op/s rd, 1115 op/s wr
|
||||
|
||||
65
ceph_reweight.pl
Normal file
65
ceph_reweight.pl
Normal file
@@ -0,0 +1,65 @@
|
||||
#!/usr/bin/perl
|
||||
#
|
||||
# This script reweights osd by 0.02 if utilization overgrows 85%
|
||||
# it also reweight osd 80% full osds by 0.01 as a "bonus".
|
||||
# It assumes that ceph cluster is less than 75-77% packed with data.
|
||||
|
||||
use strict;
|
||||
use warnings "all";
|
||||
use Fcntl;
|
||||
|
||||
use JSON::PP;
|
||||
use Data::Dumper;
|
||||
sub help();
|
||||
|
||||
my $dry = 1;
|
||||
|
||||
if (defined($ARGV[0]) and ($ARGV[0] eq '--do-it')) {
|
||||
my $jstr = `ceph osd df --format=json-pretty`;
|
||||
$jstr =~ s/\-nan/0/gi; # ceph tends to return broken json if custer is not in perfect shape
|
||||
my $j = decode_json $jstr;
|
||||
|
||||
foreach my $osd (@{$j->{'nodes'}}) {
|
||||
# skip all unrelated objects
|
||||
next unless (defined($osd->{'utilization'}));
|
||||
# skip all objects that are not 80% full
|
||||
next if ($osd->{'utilization'} < 80.000001);
|
||||
my $new_weight = 1;
|
||||
|
||||
if (($osd->{'utilization'} >= 80.000001) and ($osd->{'utilization'} < 85.000001)) {
|
||||
$new_weight = $osd->{'reweight'} - 0.01;
|
||||
printf ("osd.%s is %02.6f%% full, reweight it %01.6f -> %01.6f\n", $osd->{'id'}, $osd->{'utilization'}, $osd->{'reweight'}, $new_weight);
|
||||
}
|
||||
|
||||
if ($osd->{'utilization'} >= 85.000001) {
|
||||
$new_weight = $osd->{'reweight'} - 0.02;
|
||||
printf ("osd.%s is %02.6f%% full, reweight it %01.6f -> %01.6f\n", $osd->{'id'}, $osd->{'utilization'}, $osd->{'reweight'}, $new_weight);
|
||||
}
|
||||
|
||||
if (defined($ARGV[1])) {
|
||||
help () if ($ARGV[1] ne '--dry');
|
||||
} else {
|
||||
system('ceph', 'osd', 'reweght', sprintf("%d", $osd->{'id'}), sprintf("%f", $new_weight));
|
||||
}
|
||||
}
|
||||
} else {
|
||||
help();
|
||||
}
|
||||
|
||||
sub help() {
|
||||
print <<MSG;
|
||||
Usage:
|
||||
$0 prints this help
|
||||
$0 --do-it just reweghts osds
|
||||
$0 --do-it --dry will print what should be done
|
||||
|
||||
it reweghts 80% full osds by 0.01 and 85%+ full osds by 0.02
|
||||
|
||||
Note: you should wait until rebalance ends in order to understand state of
|
||||
cluster utilization.
|
||||
MSG
|
||||
|
||||
return undef;
|
||||
}
|
||||
|
||||
exit 0;
|
||||
15
prio-client.sh
Normal file
15
prio-client.sh
Normal file
@@ -0,0 +1,15 @@
|
||||
#!/bin/bash
|
||||
|
||||
OSD="$1"
|
||||
|
||||
if [ "$OSD" = "" ]; then
|
||||
echo "OSD ID is not set"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
echo "==> $OSD"
|
||||
ceph tell osd.$OSD injectargs '--osd_recovery_op_priority 1'
|
||||
ceph tell osd.$OSD injectargs '--osd_recovery_threads 6'
|
||||
ceph tell osd.$OSD injectargs '--osd_recovery_max_active 1'
|
||||
ceph tell osd.$OSD injectargs '--osd_max_backfills 1'
|
||||
|
||||
14
prio-recovery.sh
Normal file
14
prio-recovery.sh
Normal file
@@ -0,0 +1,14 @@
|
||||
#!/bin/bash
|
||||
|
||||
OSD="$1"
|
||||
|
||||
if [ "$OSD" = "" ]; then
|
||||
echo "OSD ID is not set"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
ceph tell osd.$OSD injectargs '--osd_recovery_op_priority 10'
|
||||
ceph tell osd.$OSD injectargs '--osd_recovery_threads 10'
|
||||
ceph tell osd.$OSD injectargs '--osd_recovery_max_active 3'
|
||||
ceph tell osd.$OSD injectargs '--osd_max_backfills 10'
|
||||
|
||||
Reference in New Issue
Block a user