eleksir/misc
1
Fork 0
Code Activity

cr.pl: added --noclean option, fix alignment

Browse Source
This commit is contained in:
eleksir
2017-11-15 18:13:30 +03:00
parent dfe37e2f24
commit c1dbcb52d3
1 changed files with 306 additions and 232 deletions
Download Patch File Download Diff File Expand all files Collapse all files

318
cr.pl Normal file → Executable file
View File

@@ -1,5 +1,6 @@
#!/usr/bin/perl #!/usr/bin/perl
# TODO: --clean option to cleanup repo accrding to metadata # TODO: implement paths fool proof paths (add trailing backslash if it absent on url and path)
#
use strict; use strict;
use warnings "all"; use warnings "all";
@@ -17,29 +18,46 @@ sub urlencode($);
sub getmetadata(); sub getmetadata();
sub help($); sub help($);
sub path_normalize_by_string_manipulation($); sub path_normalize_by_string_manipulation($);
sub getfilenames($);
help(0) unless (defined $ARGV[0]);
help(1) unless (defined $ARGV[1]); help(1) unless (defined $ARGV[1]);
help(0) if ($ARGV[0] eq '--help'); help(0) if ($ARGV[0] eq '--help');
help(0) if ($ARGV[0] eq '-h'); help(0) if ($ARGV[0] eq '-h');
my $noclean = 0;
if (defined($ARGV[0]) and $ARGV[0] eq '--noclean') {
$noclean = 1;
}
my $BASEURL = $ARGV[0]; my $BASEURL = $ARGV[0];
my $BASEDIR = $ARGV[1]; my $BASEDIR = $ARGV[1];
my $primaryxmlgz = getmetadata(); if ($noclean == 1) {
help(1) unless (defined $ARGV[2]);
$BASEURL = $ARGV[1];
$BASEDIR = $ARGV[2];
}
my @REPOFILES;
my @FILES;
my $primaryxmlgz = getmetadata();
my $fh = new IO::Zlib; my $fh = new IO::Zlib;
my $xmldata = ''; my $xmldata = '';
if ($fh->open($BASEDIR . $primaryxmlgz, "rb")) { if ($fh->open($BASEDIR . $primaryxmlgz, "rb")) {
while (<$fh>) { while (<$fh>) {
$xmldata .= $_; $xmldata .= $_;
} }
$fh->close; $fh->close;
} else { } else {
syswrite STDERR, sprintf("An error has occured while reading %s.", $BASEDIR . 'repodata/primary.xml.gz'); syswrite STDERR, sprintf("An error has occured while reading %s.", $BASEDIR . 'repodata/primary.xml.gz');
exit 1; exit 1;
} }
my $p1 = new XML::Parser(Style => 'Tree'); my $p1 = new XML::Parser(Style => 'Tree');
@@ -49,42 +67,74 @@ my @array = @{ $ref->[1] };
for (my $i = 0; $i < @array; $i++) { for (my $i = 0; $i < @array; $i++) {
if (defined($array[$i]) and ($array[$i] eq 'package')) { if (defined($array[$i]) and ($array[$i] eq 'package')) {
$i++; $i++;
my @array2 = @{ $array[$i] }; my @array2 = @{ $array[$i] };
my $size = -1; my $size = -1;
my $link; my $link;
for (my $j = 0; $j < @array2; $j++) { for (my $j = 0; $j < @array2; $j++) {
if (defined($array2[$j]) and ($array2[$j] eq 'size')) { if (defined($array2[$j]) and ($array2[$j] eq 'size')) {
$j++; $j++;
$size = $array2[$j][0]{'package'}; $size = $array2[$j][0]{'package'};
} }
if (defined($array2[$j]) and ($array2[$j] eq 'location')) { if (defined($array2[$j]) and ($array2[$j] eq 'location')) {
$j++; $j++;
$link = $array2[$j][0]{'href'}; $link = $array2[$j][0]{'href'};
# syswrite STDOUT, "+ " . $array2[$j][0]{'href'} . "\n"; # syswrite STDOUT, "+ " . $array2[$j][0]{'href'} . "\n";
} }
} }
dlfunc($link, $size); dlfunc($link, $size);
undef $size; undef $size;
undef $link; undef $link;
@array2 = -1; @array2 = -1;
} }
}
exit 0 if ($noclean == 1);
my $searchpath = $BASEDIR;
chop $searchpath;
getfilenames($searchpath);
my @filesToDelete;
foreach my $fileOnDisk (@FILES) {
my $flag = 0;
foreach my $fileInXML (@REPOFILES) {
if ($fileInXML eq $fileOnDisk) {
$flag = 1;
last;
}
}
if ($flag == 0) {
push @filesToDelete, $fileOnDisk;
}
}
foreach my $file (@filesToDelete) {
unless (-d $file) {
unlink $file;
}
} }
exit 0; exit 0;
sub help($) { sub help($) {
my $rc = shift; my $rc = shift;
my $help = <<'HELP'; my $help = <<'HELP';
Usage: cr.pl source destination Usage: cr.pl [--noclean] source destination
Where Where
--noclean is optional argument, if it specified, no cleanup operation will
be performed after rpm downloading complete.
Source must be http(s) url which trimmed at repodata folder. If full url is Source must be http(s) url which trimmed at repodata folder. If full url is
http://mirror.yandex.ru/centos/7.4.1708/os/x86_64/repodata/repomd.xml http://mirror.yandex.ru/centos/7.4.1708/os/x86_64/repodata/repomd.xml
source must be given as: source must be given as:
@@ -96,137 +146,161 @@ For example:
/var/www/htdocs/repo/ /var/www/htdocs/repo/
HELP HELP
syswrite STDOUT, $help; syswrite STDOUT, $help;
exit $rc; exit $rc;
} }
sub dlfunc(@) { sub dlfunc(@) {
my $url = shift; # short (relative) URL without BASE part my $url = shift; # short (relative) URL without BASE part
my $size = shift; my $size = shift;
my $wgetpath; my $wgetpath;
my $savepath = path_normalize_by_string_manipulation ($BASEDIR . $url); my $savepath = path_normalize_by_string_manipulation ($BASEDIR . $url);
if (substr($savepath, 0, length($BASEDIR)) ne $BASEDIR) { if (substr($savepath, 0, length($BASEDIR)) ne $BASEDIR) {
syswrite STDERR, sprintf("Skipping package %s, because it out of destdir %s, looks like quirky metadata.\n", $savepath, $BASEDIR); syswrite STDERR, sprintf("Skipping package %s, because it out of destdir %s, looks like quirky metadata.\n", $savepath, $BASEDIR);
return; return;
} }
# do not re-download already downloaded rpm packages push @REPOFILES, $savepath;
unless ($size == -1) {
if (-f $savepath) { # do not re-download already downloaded rpm packages
my $filesize = (stat($savepath))[7]; unless ($size == -1) {
if ($size == $filesize) { if (-f $savepath) {
return; my $filesize = (stat($savepath))[7];
}
}
} if ($size == $filesize) {
return;
}
}
my $basedir; (undef, $basedir) = fileparse($savepath); }
unless (-d $basedir) { my $basedir; (undef, $basedir) = fileparse($savepath);
my $ret = make_path ($basedir);
if ($ret < 1) { unless (-d $basedir) {
syswrite STDERR, sprintf("Unable to create directory %s .\n", $basedir); my $ret = make_path ($basedir);
exit 1;
}
}
if (-f "/bin/wget") { if ($ret < 1) {
$wgetpath = '/bin/wget'; syswrite STDERR, sprintf("Unable to create directory %s .\n", $basedir);
} elsif (-f "/usr/bin/wget") { exit 1;
$wgetpath = '/usr/bin/wget'; }
} elsif (-f "/usr/local/bin/wget") { }
$wgetpath = '/usr/local/bin/wget';
} else {
syswrite STDERR, "Please install wget. (Unable to find it in std locations)\n";
exit 1;
}
$url = $BASEURL . $url; if (-f "/bin/wget") {
$url = urlencode($url); $wgetpath = '/bin/wget';
system($wgetpath, '--no-check-certificate', '-q', '-T', '20', '-O', $savepath, '-o', '/dev/null', $url); } elsif (-f "/usr/bin/wget") {
$wgetpath = '/usr/bin/wget';
} elsif (-f "/usr/local/bin/wget") {
$wgetpath = '/usr/local/bin/wget';
} else {
syswrite STDERR, "Please install wget. (Unable to find it in std locations)\n";
exit 1;
}
undef $savepath; $url = $BASEURL . $url;
undef $url; $url = urlencode($url);
system($wgetpath, '--no-check-certificate', '-q', '-T', '20', '-O', $savepath, '-o', '/dev/null', $url);
undef $savepath;
undef $url;
} }
sub urlencode($) { sub urlencode($) {
my $url = shift; my $url = shift;
my $urlobj = url $url; my $urlobj = url $url;
$url = $urlobj->as_string; $url = $urlobj->as_string;
undef $urlobj; undef $urlobj;
return $url; return $url;
} }
sub getmetadata() { sub getmetadata() {
dlfunc('repodata/repomd.xml', -1); dlfunc('repodata/repomd.xml', -1);
my $p1 = new XML::Parser(Style => 'Tree'); my $p1 = new XML::Parser(Style => 'Tree');
my $ref = $p1->parsefile($BASEDIR . 'repodata/repomd.xml'); my $ref = $p1->parsefile($BASEDIR . 'repodata/repomd.xml');
#print Dumper $ref->[1][8][12][0]{'href'}; #print Dumper $ref->[1][8][12][0]{'href'};
# must be 'data'-------^ # must be 'data'-------^
# must be 'location'------^ # must be 'location'------^
my @array = @{ $ref->[1] }; my @array = @{ $ref->[1] };
my $primaryxmlgz; my $primaryxmlgz;
for (my $i = 0; $i < @array; $i++) { for (my $i = 0; $i < @array; $i++) {
if (defined($array[$i]) and ($array[$i] eq 'data')){ if (defined($array[$i]) and ($array[$i] eq 'data')){
$i++; $i++;
my @array2 = @{ $array[$i] }; my @array2 = @{ $array[$i] };
for (my $j = 0; $j < @array2; $j++) { for (my $j = 0; $j < @array2; $j++) {
if (defined($array2[$j]) and ($array2[$j] eq 'location')) { if (defined($array2[$j]) and ($array2[$j] eq 'location')) {
$j++; $j++;
dlfunc($array2[$j][0]{'href'}, -1); dlfunc($array2[$j][0]{'href'}, -1);
if ($array2[$j][0]{'href'} =~ /primary\.xml\.gz$/) { if ($array2[$j][0]{'href'} =~ /primary\.xml\.gz$/) {
$primaryxmlgz = $array2[$j][0]{'href'}; $primaryxmlgz = $array2[$j][0]{'href'};
} }
} }
} }
@array2 = -1; @array2 = -1;
} }
} }
return $primaryxmlgz; return $primaryxmlgz;
} }
sub path_normalize_by_string_manipulation ($) { sub path_normalize_by_string_manipulation ($) {
my $path = shift; my $path = shift;
# canonpath does string manipulation, but does not remove "..". # canonpath does string manipulation, but does not remove "..".
my $ret = File::Spec->canonpath($path); my $ret = File::Spec->canonpath($path);
# Let's remove ".." by using a regex. # Let's remove ".." by using a regex.
while ($ret =~ s{ while ($ret =~ s{
(^|/) # Either the beginning of the string, or a slash, save as $1 (^|/) # Either the beginning of the string, or a slash, save as $1
( # Followed by one of these: ( # Followed by one of these:
[^/]| # * Any one character (except slash, obviously) [^/]| # * Any one character (except slash, obviously)
[^./][^/]| # * Two characters where [^./][^/]| # * Two characters where
[^/][^./]| # they are not ".." [^/][^./]| # they are not ".."
[^/][^/][^/]+ # * Three or more characters [^/][^/][^/]+ # * Three or more characters
) # Followed by: ) # Followed by:
/\.\./ # "/", followed by "../" /\.\./ # "/", followed by "../"
}{$1}x }{$1}x
) { ) {
# Repeat this substitution until not possible anymore. # Repeat this substitution until not possible anymore.
} }
# Re-adding the trailing slash, if needed. # Re-adding the trailing slash, if needed.
if ($path =~ m!/$! && $ret !~ m!/$!) { if ($path =~ m!/$! && $ret !~ m!/$!) {
$ret .= '/'; $ret .= '/';
} }
return $ret; return $ret;
} }
sub getfilenames($) {
my $path = shift;
opendir (DIR, $path) or return "";
my @localfiles;
@localfiles =
map { $path . '/' . $_ }
grep { !/^\.{1,2}$/ }
readdir (DIR);
@FILES = (@FILES, @localfiles);
foreach (@localfiles) {
next if(-l $_);
if( -d $_){
getfilenames($_);
}
}
}
__END__ __END__