cr.pl: added --noclean option, fix alignment
This commit is contained in:
eleksir
538
cr.pl
Normal file → Executable file
538
cr.pl
Normal file → Executable file
@@ -1,232 +1,306 @@
|
|||||||
#!/usr/bin/perl
|
#!/usr/bin/perl
|
||||||
# TODO: --clean option to cleanup repo accrding to metadata
|
# TODO: implement paths fool proof paths (add trailing backslash if it absent on url and path)
|
||||||
|
#
|
||||||
use strict;
|
|
||||||
use warnings "all";
|
use strict;
|
||||||
use Fcntl;
|
use warnings "all";
|
||||||
use File::Basename;
|
use Fcntl;
|
||||||
use File::Path qw(make_path);
|
use File::Basename;
|
||||||
use File::Spec;
|
use File::Path qw(make_path);
|
||||||
use XML::Parser;
|
use File::Spec;
|
||||||
use IO::Zlib;
|
use XML::Parser;
|
||||||
use URI::URL;
|
use IO::Zlib;
|
||||||
use Data::Dumper;
|
use URI::URL;
|
||||||
|
use Data::Dumper;
|
||||||
sub dlfunc(@);
|
|
||||||
sub urlencode($);
|
sub dlfunc(@);
|
||||||
sub getmetadata();
|
sub urlencode($);
|
||||||
sub help($);
|
sub getmetadata();
|
||||||
sub path_normalize_by_string_manipulation($);
|
sub help($);
|
||||||
|
sub path_normalize_by_string_manipulation($);
|
||||||
help(1) unless (defined $ARGV[1]);
|
sub getfilenames($);
|
||||||
help(0) if ($ARGV[0] eq '--help');
|
|
||||||
help(0) if ($ARGV[0] eq '-h');
|
help(0) unless (defined $ARGV[0]);
|
||||||
|
help(1) unless (defined $ARGV[1]);
|
||||||
my $BASEURL = $ARGV[0];
|
|
||||||
my $BASEDIR = $ARGV[1];
|
help(0) if ($ARGV[0] eq '--help');
|
||||||
|
help(0) if ($ARGV[0] eq '-h');
|
||||||
my $primaryxmlgz = getmetadata();
|
|
||||||
|
my $noclean = 0;
|
||||||
my $fh = new IO::Zlib;
|
|
||||||
my $xmldata = '';
|
if (defined($ARGV[0]) and $ARGV[0] eq '--noclean') {
|
||||||
|
$noclean = 1;
|
||||||
if ($fh->open($BASEDIR . $primaryxmlgz, "rb")) {
|
}
|
||||||
|
|
||||||
while (<$fh>) {
|
my $BASEURL = $ARGV[0];
|
||||||
$xmldata .= $_;
|
my $BASEDIR = $ARGV[1];
|
||||||
}
|
|
||||||
|
if ($noclean == 1) {
|
||||||
$fh->close;
|
help(1) unless (defined $ARGV[2]);
|
||||||
} else {
|
$BASEURL = $ARGV[1];
|
||||||
syswrite STDERR, sprintf("An error has occured while reading %s.", $BASEDIR . 'repodata/primary.xml.gz');
|
$BASEDIR = $ARGV[2];
|
||||||
exit 1;
|
}
|
||||||
}
|
|
||||||
|
my @REPOFILES;
|
||||||
my $p1 = new XML::Parser(Style => 'Tree');
|
my @FILES;
|
||||||
my $ref = $p1->parse($xmldata);
|
|
||||||
undef $xmldata;
|
my $primaryxmlgz = getmetadata();
|
||||||
my @array = @{ $ref->[1] };
|
my $fh = new IO::Zlib;
|
||||||
|
my $xmldata = '';
|
||||||
for (my $i = 0; $i < @array; $i++) {
|
|
||||||
|
if ($fh->open($BASEDIR . $primaryxmlgz, "rb")) {
|
||||||
if (defined($array[$i]) and ($array[$i] eq 'package')) {
|
|
||||||
$i++;
|
while (<$fh>) {
|
||||||
my @array2 = @{ $array[$i] };
|
$xmldata .= $_;
|
||||||
my $size = -1;
|
}
|
||||||
my $link;
|
|
||||||
|
$fh->close;
|
||||||
for (my $j = 0; $j < @array2; $j++) {
|
} else {
|
||||||
|
syswrite STDERR, sprintf("An error has occured while reading %s.", $BASEDIR . 'repodata/primary.xml.gz');
|
||||||
if (defined($array2[$j]) and ($array2[$j] eq 'size')) {
|
exit 1;
|
||||||
$j++;
|
}
|
||||||
$size = $array2[$j][0]{'package'};
|
|
||||||
}
|
my $p1 = new XML::Parser(Style => 'Tree');
|
||||||
|
my $ref = $p1->parse($xmldata);
|
||||||
if (defined($array2[$j]) and ($array2[$j] eq 'location')) {
|
undef $xmldata;
|
||||||
$j++;
|
my @array = @{ $ref->[1] };
|
||||||
$link = $array2[$j][0]{'href'};
|
|
||||||
# syswrite STDOUT, "+ " . $array2[$j][0]{'href'} . "\n";
|
for (my $i = 0; $i < @array; $i++) {
|
||||||
}
|
|
||||||
|
if (defined($array[$i]) and ($array[$i] eq 'package')) {
|
||||||
}
|
$i++;
|
||||||
|
my @array2 = @{ $array[$i] };
|
||||||
dlfunc($link, $size);
|
my $size = -1;
|
||||||
undef $size;
|
my $link;
|
||||||
undef $link;
|
|
||||||
@array2 = -1;
|
for (my $j = 0; $j < @array2; $j++) {
|
||||||
}
|
|
||||||
}
|
if (defined($array2[$j]) and ($array2[$j] eq 'size')) {
|
||||||
|
$j++;
|
||||||
exit 0;
|
$size = $array2[$j][0]{'package'};
|
||||||
|
}
|
||||||
sub help($) {
|
|
||||||
my $rc = shift;
|
if (defined($array2[$j]) and ($array2[$j] eq 'location')) {
|
||||||
my $help = <<'HELP';
|
$j++;
|
||||||
Usage: cr.pl source destination
|
$link = $array2[$j][0]{'href'};
|
||||||
|
# syswrite STDOUT, "+ " . $array2[$j][0]{'href'} . "\n";
|
||||||
Where
|
}
|
||||||
Source must be http(s) url which trimmed at repodata folder. If full url is
|
|
||||||
http://mirror.yandex.ru/centos/7.4.1708/os/x86_64/repodata/repomd.xml
|
}
|
||||||
source must be given as:
|
|
||||||
http://mirror.yandex.ru/centos/7.4.1708/os/x86_64/
|
dlfunc($link, $size);
|
||||||
|
undef $size;
|
||||||
Destination is the folder where all files and metadata will be downloaded, if
|
undef $link;
|
||||||
folder does not exist it will be created. Must contain trailng slash char.
|
@array2 = -1;
|
||||||
For example:
|
}
|
||||||
/var/www/htdocs/repo/
|
}
|
||||||
HELP
|
|
||||||
|
exit 0 if ($noclean == 1);
|
||||||
syswrite STDOUT, $help;
|
|
||||||
exit $rc;
|
my $searchpath = $BASEDIR;
|
||||||
}
|
chop $searchpath;
|
||||||
|
getfilenames($searchpath);
|
||||||
sub dlfunc(@) {
|
my @filesToDelete;
|
||||||
my $url = shift; # short (relative) URL without BASE part
|
|
||||||
my $size = shift;
|
foreach my $fileOnDisk (@FILES) {
|
||||||
my $wgetpath;
|
my $flag = 0;
|
||||||
my $savepath = path_normalize_by_string_manipulation ($BASEDIR . $url);
|
|
||||||
|
foreach my $fileInXML (@REPOFILES) {
|
||||||
if (substr($savepath, 0, length($BASEDIR)) ne $BASEDIR) {
|
if ($fileInXML eq $fileOnDisk) {
|
||||||
syswrite STDERR, sprintf("Skipping package %s, because it out of destdir %s, looks like quirky metadata.\n", $savepath, $BASEDIR);
|
$flag = 1;
|
||||||
return;
|
last;
|
||||||
}
|
}
|
||||||
|
}
|
||||||
# do not re-download already downloaded rpm packages
|
|
||||||
unless ($size == -1) {
|
if ($flag == 0) {
|
||||||
|
push @filesToDelete, $fileOnDisk;
|
||||||
if (-f $savepath) {
|
}
|
||||||
my $filesize = (stat($savepath))[7];
|
}
|
||||||
|
|
||||||
if ($size == $filesize) {
|
|
||||||
return;
|
foreach my $file (@filesToDelete) {
|
||||||
}
|
unless (-d $file) {
|
||||||
}
|
unlink $file;
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
my $basedir; (undef, $basedir) = fileparse($savepath);
|
exit 0;
|
||||||
|
|
||||||
unless (-d $basedir) {
|
sub help($) {
|
||||||
my $ret = make_path ($basedir);
|
my $rc = shift;
|
||||||
|
my $help = <<'HELP';
|
||||||
if ($ret < 1) {
|
Usage: cr.pl [--noclean] source destination
|
||||||
syswrite STDERR, sprintf("Unable to create directory %s .\n", $basedir);
|
|
||||||
exit 1;
|
Where
|
||||||
}
|
--noclean is optional argument, if it specified, no cleanup operation will
|
||||||
}
|
be performed after rpm downloading complete.
|
||||||
|
|
||||||
if (-f "/bin/wget") {
|
Source must be http(s) url which trimmed at repodata folder. If full url is
|
||||||
$wgetpath = '/bin/wget';
|
http://mirror.yandex.ru/centos/7.4.1708/os/x86_64/repodata/repomd.xml
|
||||||
} elsif (-f "/usr/bin/wget") {
|
source must be given as:
|
||||||
$wgetpath = '/usr/bin/wget';
|
http://mirror.yandex.ru/centos/7.4.1708/os/x86_64/
|
||||||
} elsif (-f "/usr/local/bin/wget") {
|
|
||||||
$wgetpath = '/usr/local/bin/wget';
|
Destination is the folder where all files and metadata will be downloaded, if
|
||||||
} else {
|
folder does not exist it will be created. Must contain trailng slash char.
|
||||||
syswrite STDERR, "Please install wget. (Unable to find it in std locations)\n";
|
For example:
|
||||||
exit 1;
|
/var/www/htdocs/repo/
|
||||||
}
|
HELP
|
||||||
|
|
||||||
$url = $BASEURL . $url;
|
syswrite STDOUT, $help;
|
||||||
$url = urlencode($url);
|
exit $rc;
|
||||||
system($wgetpath, '--no-check-certificate', '-q', '-T', '20', '-O', $savepath, '-o', '/dev/null', $url);
|
}
|
||||||
|
|
||||||
undef $savepath;
|
sub dlfunc(@) {
|
||||||
undef $url;
|
my $url = shift; # short (relative) URL without BASE part
|
||||||
}
|
my $size = shift;
|
||||||
|
my $wgetpath;
|
||||||
sub urlencode($) {
|
my $savepath = path_normalize_by_string_manipulation ($BASEDIR . $url);
|
||||||
my $url = shift;
|
|
||||||
my $urlobj = url $url;
|
if (substr($savepath, 0, length($BASEDIR)) ne $BASEDIR) {
|
||||||
$url = $urlobj->as_string;
|
syswrite STDERR, sprintf("Skipping package %s, because it out of destdir %s, looks like quirky metadata.\n", $savepath, $BASEDIR);
|
||||||
undef $urlobj;
|
return;
|
||||||
return $url;
|
}
|
||||||
}
|
|
||||||
|
push @REPOFILES, $savepath;
|
||||||
sub getmetadata() {
|
|
||||||
dlfunc('repodata/repomd.xml', -1);
|
# do not re-download already downloaded rpm packages
|
||||||
|
unless ($size == -1) {
|
||||||
my $p1 = new XML::Parser(Style => 'Tree');
|
|
||||||
my $ref = $p1->parsefile($BASEDIR . 'repodata/repomd.xml');
|
if (-f $savepath) {
|
||||||
|
my $filesize = (stat($savepath))[7];
|
||||||
#print Dumper $ref->[1][8][12][0]{'href'};
|
|
||||||
# must be 'data'-------^
|
if ($size == $filesize) {
|
||||||
# must be 'location'------^
|
return;
|
||||||
|
}
|
||||||
my @array = @{ $ref->[1] };
|
}
|
||||||
my $primaryxmlgz;
|
|
||||||
|
}
|
||||||
for (my $i = 0; $i < @array; $i++) {
|
|
||||||
|
my $basedir; (undef, $basedir) = fileparse($savepath);
|
||||||
if (defined($array[$i]) and ($array[$i] eq 'data')){
|
|
||||||
$i++;
|
unless (-d $basedir) {
|
||||||
my @array2 = @{ $array[$i] };
|
my $ret = make_path ($basedir);
|
||||||
|
|
||||||
for (my $j = 0; $j < @array2; $j++) {
|
if ($ret < 1) {
|
||||||
if (defined($array2[$j]) and ($array2[$j] eq 'location')) {
|
syswrite STDERR, sprintf("Unable to create directory %s .\n", $basedir);
|
||||||
$j++;
|
exit 1;
|
||||||
dlfunc($array2[$j][0]{'href'}, -1);
|
}
|
||||||
|
}
|
||||||
if ($array2[$j][0]{'href'} =~ /primary\.xml\.gz$/) {
|
|
||||||
$primaryxmlgz = $array2[$j][0]{'href'};
|
if (-f "/bin/wget") {
|
||||||
}
|
$wgetpath = '/bin/wget';
|
||||||
}
|
} elsif (-f "/usr/bin/wget") {
|
||||||
|
$wgetpath = '/usr/bin/wget';
|
||||||
}
|
} elsif (-f "/usr/local/bin/wget") {
|
||||||
|
$wgetpath = '/usr/local/bin/wget';
|
||||||
@array2 = -1;
|
} else {
|
||||||
}
|
syswrite STDERR, "Please install wget. (Unable to find it in std locations)\n";
|
||||||
|
exit 1;
|
||||||
}
|
}
|
||||||
|
|
||||||
return $primaryxmlgz;
|
$url = $BASEURL . $url;
|
||||||
}
|
$url = urlencode($url);
|
||||||
|
system($wgetpath, '--no-check-certificate', '-q', '-T', '20', '-O', $savepath, '-o', '/dev/null', $url);
|
||||||
sub path_normalize_by_string_manipulation ($) {
|
|
||||||
my $path = shift;
|
undef $savepath;
|
||||||
|
undef $url;
|
||||||
# canonpath does string manipulation, but does not remove "..".
|
}
|
||||||
my $ret = File::Spec->canonpath($path);
|
|
||||||
|
sub urlencode($) {
|
||||||
# Let's remove ".." by using a regex.
|
my $url = shift;
|
||||||
while ($ret =~ s{
|
my $urlobj = url $url;
|
||||||
(^|/) # Either the beginning of the string, or a slash, save as $1
|
$url = $urlobj->as_string;
|
||||||
( # Followed by one of these:
|
undef $urlobj;
|
||||||
[^/]| # * Any one character (except slash, obviously)
|
return $url;
|
||||||
[^./][^/]| # * Two characters where
|
}
|
||||||
[^/][^./]| # they are not ".."
|
|
||||||
[^/][^/][^/]+ # * Three or more characters
|
sub getmetadata() {
|
||||||
) # Followed by:
|
dlfunc('repodata/repomd.xml', -1);
|
||||||
/\.\./ # "/", followed by "../"
|
|
||||||
}{$1}x
|
my $p1 = new XML::Parser(Style => 'Tree');
|
||||||
) {
|
my $ref = $p1->parsefile($BASEDIR . 'repodata/repomd.xml');
|
||||||
# Repeat this substitution until not possible anymore.
|
|
||||||
}
|
#print Dumper $ref->[1][8][12][0]{'href'};
|
||||||
|
# must be 'data'-------^
|
||||||
# Re-adding the trailing slash, if needed.
|
# must be 'location'------^
|
||||||
if ($path =~ m!/$! && $ret !~ m!/$!) {
|
|
||||||
$ret .= '/';
|
my @array = @{ $ref->[1] };
|
||||||
}
|
my $primaryxmlgz;
|
||||||
|
|
||||||
return $ret;
|
for (my $i = 0; $i < @array; $i++) {
|
||||||
}
|
|
||||||
__END__
|
if (defined($array[$i]) and ($array[$i] eq 'data')){
|
||||||
|
$i++;
|
||||||
|
my @array2 = @{ $array[$i] };
|
||||||
|
|
||||||
|
for (my $j = 0; $j < @array2; $j++) {
|
||||||
|
if (defined($array2[$j]) and ($array2[$j] eq 'location')) {
|
||||||
|
$j++;
|
||||||
|
dlfunc($array2[$j][0]{'href'}, -1);
|
||||||
|
|
||||||
|
if ($array2[$j][0]{'href'} =~ /primary\.xml\.gz$/) {
|
||||||
|
$primaryxmlgz = $array2[$j][0]{'href'};
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
}
|
||||||
|
|
||||||
|
@array2 = -1;
|
||||||
|
}
|
||||||
|
|
||||||
|
}
|
||||||
|
|
||||||
|
return $primaryxmlgz;
|
||||||
|
}
|
||||||
|
|
||||||
|
sub path_normalize_by_string_manipulation ($) {
|
||||||
|
my $path = shift;
|
||||||
|
|
||||||
|
# canonpath does string manipulation, but does not remove "..".
|
||||||
|
my $ret = File::Spec->canonpath($path);
|
||||||
|
|
||||||
|
# Let's remove ".." by using a regex.
|
||||||
|
while ($ret =~ s{
|
||||||
|
(^|/) # Either the beginning of the string, or a slash, save as $1
|
||||||
|
( # Followed by one of these:
|
||||||
|
[^/]| # * Any one character (except slash, obviously)
|
||||||
|
[^./][^/]| # * Two characters where
|
||||||
|
[^/][^./]| # they are not ".."
|
||||||
|
[^/][^/][^/]+ # * Three or more characters
|
||||||
|
) # Followed by:
|
||||||
|
/\.\./ # "/", followed by "../"
|
||||||
|
}{$1}x
|
||||||
|
) {
|
||||||
|
# Repeat this substitution until not possible anymore.
|
||||||
|
}
|
||||||
|
|
||||||
|
# Re-adding the trailing slash, if needed.
|
||||||
|
if ($path =~ m!/$! && $ret !~ m!/$!) {
|
||||||
|
$ret .= '/';
|
||||||
|
}
|
||||||
|
|
||||||
|
return $ret;
|
||||||
|
}
|
||||||
|
|
||||||
|
sub getfilenames($) {
|
||||||
|
my $path = shift;
|
||||||
|
|
||||||
|
opendir (DIR, $path) or return "";
|
||||||
|
my @localfiles;
|
||||||
|
@localfiles =
|
||||||
|
map { $path . '/' . $_ }
|
||||||
|
grep { !/^\.{1,2}$/ }
|
||||||
|
readdir (DIR);
|
||||||
|
@FILES = (@FILES, @localfiles);
|
||||||
|
|
||||||
|
foreach (@localfiles) {
|
||||||
|
next if(-l $_);
|
||||||
|
|
||||||
|
if( -d $_){
|
||||||
|
getfilenames($_);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
}
|
||||||
|
|
||||||
|
__END__
|
||||||
|
|||||||
Reference in New Issue
Block a user