commit d6e2f9f211b66ee7ba2efd2a6911cb27a33e2ef9 Author: eleksir Date: Sun Sep 23 14:24:20 2018 +0300 Initial commit diff --git a/README.md b/README.md new file mode 100644 index 0000000..3b1bed0 --- /dev/null +++ b/README.md @@ -0,0 +1,10 @@ +## CEPH storage helper scripts + +* ceph-osd-list.pl - another listing presetation of listing node<->osd +* ceph_osd_kickin.pl - kicks in big amount of osds in cluster by small batches +* ceph_osd_kickoff_mt.pl - ejects big amount of osds from cluster by small batches +* ceph_osd_pool_discovery.pl - script for zabbix' pool discovery +* ceph_osd_pool_stats.pl - zabbix monitoring script +* ceph_reweight.pl - carefuly reweights osd when "near full osd" cluster health warning appears +* prio-client.sh - gives priority to client io operations +* prio-recovery.sh - gives priority to recovery io operations diff --git a/ceph-osd-list.pl b/ceph-osd-list.pl new file mode 100644 index 0000000..2c1ce45 --- /dev/null +++ b/ceph-osd-list.pl @@ -0,0 +1,22 @@ +#!/usr/bin/perl +# +use strict; +use warnings "all"; +use JSON::PP; + +my $jstr = `ceph osd tree --format=json`; + +my $json = decode_json $jstr; + +my @nodes = @{ $json->{'nodes'} }; + +foreach my $node (@nodes) { + if ($node->{'type'} eq 'host') { + print "Хост: $node->{'name'}\n"; + my @children = @{ $node->{'children'} }; + + foreach my $child (@children) { + print "Диск: osd.$child\n"; + } + } +} diff --git a/ceph_osd_kickin.pl b/ceph_osd_kickin.pl new file mode 100644 index 0000000..7daf658 --- /dev/null +++ b/ceph_osd_kickin.pl @@ -0,0 +1,86 @@ +#!/usr/bin/perl + +use warnings "all"; +use strict; +use Fcntl; + +sub get_ceph_status(); # defined - not healthy | undef - healthy +sub mark_osd_out($); +sub logger($); +sub restore_prio(@); + +if ((int(@ARGV) < 1) or ($ARGV[0] eq '--help') or ($ARGV[0] eq '-h')) { + print "\n Usage: $0 osd_num osd_num osd_num\n\n"; + print " Example: user\@host\# $0 10 25 44 128 9 36 33 78 91\n\n"; + exit 0; +} + +my @osd_list = @ARGV; +my $PACK = 2; +my $OSDINDEX = 0; + +while ( 1 ) { + while ($PACK >0) { + if (defined($osd_list[$OSDINDEX])) { + logger("Mark $osd_list[$OSDINDEX] as in"); + mark_osd_in($osd_list[$OSDINDEX]); + } else { +# last (and not full) pack of osds, we _can_ quit now + logger("All OSD marked as in"); + exit 0; + } + + $OSDINDEX = $OSDINDEX + 1; + $PACK = $PACK - 1; + sleep 300; + } + + while (defined(get_ceph_status())) { + sleep 60; + } + + $PACK = 2; + restore_prio(@osd_list); +} + +sub get_ceph_status() { + my $status = `/bin/ceph -s`; + my @msg = split(/\n/, $status); +# by default we assume that ceph is unhealthy, with misplaced pgs, and degraded pgs + my $healthy = undef; + my $misplaced = undef; + my $degraded = undef; + + foreach (@msg) { + return undef if ($_ =~ /health HEALTH_OK$/); + $degraded = 1 if ($_ =~ /recovery \d+\/\d+ objects degraded/); + $misplaced = 1 if ($_ =~ /recovery \d+\/\d+ objects misplaced/) + } + + if ((defined($degraded)) or (defined($misplaced))) { + return 1; + } + + return undef; +} + +sub mark_osd_in($) { + my $osd = shift; + `/bin/ceph osd in $osd`; + sleep(90); + `./prio-recovery.sh $osd`; +} + +sub logger($) { + my $msg = shift; + syswrite STDOUT, "$msg\n"; +} + +sub restore_prio(@) { + my @osds = @_; + + foreach(@osds) { + `./prio-client.sh $_` + } +} + diff --git a/ceph_osd_kickoff_mt.pl b/ceph_osd_kickoff_mt.pl new file mode 100644 index 0000000..7c60ea6 --- /dev/null +++ b/ceph_osd_kickoff_mt.pl @@ -0,0 +1,141 @@ +#!/usr/bin/perl + +use warnings "all"; +use strict; +use Fcntl; +use threads; +use threads::shared; + + +sub get_ceph_status(); # defined - not healthy | undef - healthy +sub get_osd_status($); # give osd id, returns 1 if osd have pgs | undef - no pg left on osd +sub mark_osd_out($); +sub logger($); +sub metabackup(@); +sub worker(@); + + +if ((int(@ARGV) < 1) or ($ARGV[0] eq '--help') or ($ARGV[0] eq '-h')) { + print "\n Usage: $0 osd_num osd_num osd_num\n\n"; + print " Example: user\@host\# $0 10 25 44 128 9 36 33 78 91\n\n"; + exit 0; +} + +my @osd_list = @ARGV; +my $PACK = 5; +my $OSDINDEX = 0; +my %JOBS; +my $OSD; +my @threads; + +for (my $i = 0; $i < @osd_list; $i++) { + if ($OSDINDEX >= $PACK) { + $OSDINDEX = 0; + } + + ${JOBS}{$OSDINDEX}{$osd_list[$i]} = 1; + $OSDINDEX++; +} + +# ${JOBS}{$OSDINDEX}{OSD_ID} = 1 +# hash of hashes of hashes + +foreach (keys(%JOBS)) { + my @jobs = keys($JOBS{$_}); + push @threads, threads->create(\&worker, @jobs); + sleep (300); # do not start all pack in one shot +} + +# wait 'till jobs are complete +foreach (@threads) { + $_->join(); +} + +print "All requested OSDS are out, performing backup\n"; +metabackup(@osd_list); + +exit 0; + +sub get_ceph_status() { + my $status = `/bin/ceph -s`; + my @msg = split(/\n/, $status); +# by default we assume that ceph is unhealthy, with misplaced pgs, and degraded pgs + my $healthy = undef; + my $misplaced = undef; + my $degraded = undef; + + foreach (@msg) { + return undef if ($_ =~ /health HEALTH_OK$/); + $degraded = 1 if ($_ =~ /recovery \d+\/\d+ objects degraded/); + $misplaced = 1 if ($_ =~ /recovery \d+\/\d+ objects misplaced/) + } + + if ((defined($degraded)) or (defined($misplaced))) { + return 1; + } + + return undef; +} + +sub get_osd_status($) { + my $osd = shift; + my $status = `ceph osd df`; + + foreach (split(/\n/, $status)) { + if ($_ =~ /^ ? ?$osd /) { + $status = $_; + } + } + + if ($status =~ / 0.$/) { +# zero placement groups here + return undef; + } + + return 1; +} + +sub mark_osd_out($) { + my $osd = shift; + `/bin/ceph osd out $osd`; +} + +sub logger($) { + my $msg = shift; + syswrite STDOUT, "$msg\n"; +} + +sub metabackup(@) { + my @list = @_; + +# just to make sure :) + while (defined(get_ceph_status())) { + sleep 60; + } + + `mount|sudo grep ceph >/ceph-mountmap`; + + system('/bin/sudo', '/bin/mkdir', "/root/ceph"); + chdir('/var/lib/ceph/osd'); + my $osd; + + foreach $osd (@list) { + system('/bin/sudo', "/bin/systemctl", "stop", sprintf("ceph-osd@%s", $osd)); + sleep 1; + system('/bin/sudo', '/bin/tar', '-zcf', sprintf("/root/ceph/ceph-%s.tar.gz", $osd), sprintf("ceph-%s", $osd)); + } +} + +sub worker(@) { + my @OSD = @_; + + foreach my $osd (@OSD) { + mark_osd_out($osd); + sleep(15); # ceph is slow + + while (defined(get_osd_status($osd))) { + sleep 150; + } + } +} + diff --git a/ceph_osd_pool_discovery.pl b/ceph_osd_pool_discovery.pl new file mode 100644 index 0000000..629883c --- /dev/null +++ b/ceph_osd_pool_discovery.pl @@ -0,0 +1,24 @@ +#!/usr/bin/perl + +use strict; +use warnings "all"; + +my $str = `ceph osd pool stats`; +my @arr = split(/\n\n/, $str); + +my $msg = "{\n \"data\": [\n"; + +foreach my $p (@arr) { + my @str = split(/\s+/, $p); + next if ((not defined($str[1])) or ($str[1] eq '')); + next if ((not defined($str[3])) or ($str[3] eq '')); +{# in jinja template engine this is comment, so in real script fix this #} + $msg .= sprintf(" {\n \"\{#POOLID}\": \"%s\",\n", $str[3]); + $msg .= sprintf(" \"\{#POOLNAME}\": \"%s\"\n },\n", $str[1]); +} +chop($msg); +chop($msg); + +$msg .= "\n ]\n}\n"; +print $msg; + diff --git a/ceph_osd_pool_stats.pl b/ceph_osd_pool_stats.pl new file mode 100644 index 0000000..3cd4174 --- /dev/null +++ b/ceph_osd_pool_stats.pl @@ -0,0 +1,93 @@ +#!/usr/bin/perl + +use strict; +use warnings "all"; + +sub usage(); +# print usage banner if we have not enough arg +usage() if ((not defined($ARGV[1])) or ($ARGV[1] eq '')); + +my $pflag = 0; +my $paramflag = 0; + +my $str = `ceph osd pool stats`; +my @arr = split(/\n\n/, $str); + +my $msg = "{\n \"data\": [\n"; + +foreach my $p (@arr) { + next unless (defined((split(/\s+/, $p))[1])); + next unless ((split(/\s+/, $p))[1] eq $ARGV[0]); + + if ((split(/\s+/, $p))[4] eq 'nothing') { + syswrite STDOUT, '0'; + exit 0; + } + + my @data = split(/\n/, $p); + + foreach my $l (@data) { + if (substr($l, 2, 6) eq 'client') { + if ($ARGV[1] eq 'crops') { + syswrite STDOUT, (split(/\s/, $l))[10]; + } elsif ($ARGV[1] eq 'cwops') { + syswrite STDOUT, (split(/\s/, $l))[13]; + } elsif ($ARGV[1] eq 'crbps') { + if ((split(/\s/, $l))[5] =~ /^k/) { + syswrite STDOUT, (split(/\s/, $l))[4] * 1024; + } elsif ((split(/\s/, $l))[5] =~ /^M/) { + syswrite STDOUT, (split(/\s/, $l))[4] * 1024 * 1024; + } elsif ((split(/\s/, $l))[5] =~ /^G/) { + syswrite STDOUT, (split(/\s/, $l))[4] * 1024 * 1024 * 1024; + } + } elsif ($ARGV[1] eq 'cwbps') { + if ((split(/\s/, $l))[5] =~ /^k/) { + syswrite STDOUT, (split(/\s/, $l))[4] * 1024; + } elsif ((split(/\s/, $l))[5] =~ /^M/) { + syswrite STDOUT, (split(/\s/, $l))[4] * 1024 * 1024; + } elsif ((split(/\s/, $l))[5] =~ /^G/) { + syswrite STDOUT, (split(/\s/, $l))[4] * 1024 * 1024 * 1024; + } + } + } elsif (substr($l, 2, 8) eq 'recovery') { + if ($ARGV[1] eq 'rbps') { + if ((split(/\s/, $l))[5] =~ /^k/) { + syswrite STDOUT, (split(/\s/, $l))[4] * 1024; + } elsif ((split(/\s/, $l))[5] =~ /^M/) { + syswrite STDOUT, (split(/\s/, $l))[4] * 1024 * 1024; + } elsif ((split(/\s/, $l))[5] =~ /^G/) { + syswrite STDOUT, (split(/\s/, $l))[4] * 1024 * 1024 * 1024; + } else { + # i never seen stats in bytes per second, so as an exception printout just zero + syswrite STDOUT, '0'; + } + } + } + } +} + +exit 0; + +sub usage () { + print < + +Where parameter can be: +crops - client reads per second +cwops - client writes per second +crbps - client read kbytes per second +cwbps - client write kbytes per second +rbps - recovery io in bytes per second +EOF + exit 0; +} + +__END__ + +pool vms.sata.zone3 id 13 + nothing is going on + +pool volumes.sata.zone1 id 14 + recovery io 1418 MB/s, 360 objects/s + client io 24445 kB/s rd, 37316 kB/s wr, 975 op/s rd, 1115 op/s wr + diff --git a/ceph_reweight.pl b/ceph_reweight.pl new file mode 100644 index 0000000..e71c3d1 --- /dev/null +++ b/ceph_reweight.pl @@ -0,0 +1,65 @@ +#!/usr/bin/perl +# +# This script reweights osd by 0.02 if utilization overgrows 85% +# it also reweight osd 80% full osds by 0.01 as a "bonus". +# It assumes that ceph cluster is less than 75-77% packed with data. + +use strict; +use warnings "all"; +use Fcntl; + +use JSON::PP; +use Data::Dumper; +sub help(); + +my $dry = 1; + +if (defined($ARGV[0]) and ($ARGV[0] eq '--do-it')) { + my $jstr = `ceph osd df --format=json-pretty`; + $jstr =~ s/\-nan/0/gi; # ceph tends to return broken json if custer is not in perfect shape + my $j = decode_json $jstr; + + foreach my $osd (@{$j->{'nodes'}}) { +# skip all unrelated objects + next unless (defined($osd->{'utilization'})); +# skip all objects that are not 80% full + next if ($osd->{'utilization'} < 80.000001); + my $new_weight = 1; + + if (($osd->{'utilization'} >= 80.000001) and ($osd->{'utilization'} < 85.000001)) { + $new_weight = $osd->{'reweight'} - 0.01; + printf ("osd.%s is %02.6f%% full, reweight it %01.6f -> %01.6f\n", $osd->{'id'}, $osd->{'utilization'}, $osd->{'reweight'}, $new_weight); + } + + if ($osd->{'utilization'} >= 85.000001) { + $new_weight = $osd->{'reweight'} - 0.02; + printf ("osd.%s is %02.6f%% full, reweight it %01.6f -> %01.6f\n", $osd->{'id'}, $osd->{'utilization'}, $osd->{'reweight'}, $new_weight); + } + + if (defined($ARGV[1])) { + help () if ($ARGV[1] ne '--dry'); + } else { + system('ceph', 'osd', 'reweght', sprintf("%d", $osd->{'id'}), sprintf("%f", $new_weight)); + } + } +} else { + help(); +} + +sub help() { + print < $OSD" +ceph tell osd.$OSD injectargs '--osd_recovery_op_priority 1' +ceph tell osd.$OSD injectargs '--osd_recovery_threads 6' +ceph tell osd.$OSD injectargs '--osd_recovery_max_active 1' +ceph tell osd.$OSD injectargs '--osd_max_backfills 1' + diff --git a/prio-recovery.sh b/prio-recovery.sh new file mode 100644 index 0000000..458583e --- /dev/null +++ b/prio-recovery.sh @@ -0,0 +1,14 @@ +#!/bin/bash + +OSD="$1" + +if [ "$OSD" = "" ]; then + echo "OSD ID is not set" + exit 0 +fi + +ceph tell osd.$OSD injectargs '--osd_recovery_op_priority 10' +ceph tell osd.$OSD injectargs '--osd_recovery_threads 10' +ceph tell osd.$OSD injectargs '--osd_recovery_max_active 3' +ceph tell osd.$OSD injectargs '--osd_max_backfills 10' +