cr.pl added. Rpm repository cloning script. Tested on elasticsearch repo.

cr.pl

@@ -0,0 +1,161 @@
+#!/usr/bin/perl
+
+use strict;
+use warnings "all";
+use Fcntl;
+use File::Basename;
+use File::Path qw(make_path);
+use XML::Parser;
+use IO::Zlib;
+use URI::URL;
+
+sub dlfunc($);
+sub urlencode($);
+sub getmetadata();
+sub help($);
+
+help(1) unless (defined $ARGV[1]);
+help(0) if ($ARGV[0] eq '--help');
+help(0) if ($ARGV[0] eq '-h');
+
+my $BASEURL = $ARGV[0];
+my $BASEDIR = $ARGV[1];
+
+getmetadata();
+
+my $fh = new IO::Zlib;
+my $xmldata = '';
+
+if ($fh->open($BASEDIR . "repodata/primary.xml.gz", "rb")) {
+
+	while (<$fh>) {
+		$xmldata .= $_;
+	}
+
+	$fh->close;
+} else {
+	syswrite STDERR, sprintf("An error has occured while reading %s.", $BASEDIR . 'repodata/primary.xml.gz');
+	exit 1;
+}
+
+my $p1 = new XML::Parser(Style => 'Tree');
+my $ref = $p1->parse($xmldata);
+undef $xmldata;
+my @array = @{ $ref->[1] };
+
+for (my $i = 0; $i < @array; $i++) {
+
+	if (defined($array[$i]) and ($array[$i] eq 'package')) {
+		$i++;
+		my @array2 = @{ $array[$i] };
+
+		for (my $j = 0; $j < @array2; $j++) {
+
+			if (defined($array2[$j]) and ($array2[$j] eq 'location')) {
+				$j++;
+				dlfunc($array2[$j][0]{'href'});
+#				syswrite STDOUT, "+ " . $array2[$j][0]{'href'} . "\n";
+			}
+
+		}
+
+		@array2 = -1;
+	}
+}
+
+exit 0;
+
+sub help($) {
+	my $rc = shift;
+	my $help = <<'HELP';
+Usage: cr.pl source destination
+
+Where
+Source must be http(s) url which trimmed at repodata folder. If full url is
+http://mirror.yandex.ru/centos/7.4.1708/os/x86_64/repodata/repomd.xml
+source must be given as:
+http://mirror.yandex.ru/centos/7.4.1708/os/x86_64/
+
+Destination is the folder where all files and metadata will be downloaded, if
+folder does not exist it will be created. Must contain trailng slash char.
+For example:
+/var/www/htdocs/repo/
+HELP
+
+	syswrite STDOUT, $help;
+	exit $rc;
+}
+
+sub dlfunc($) {
+	my $url = shift; # short (relative) URL without BASE part
+		my $wgetpath;
+
+	if (-f "/bin/wget") {
+		$wgetpath = '/bin/wget';
+	} elsif (-f "/usr/bin/wget") {
+		$wgetpath = '/usr/bin/wget';
+	} elsif (-f "/usr/local/bin/wget") {
+		$wgetpath = '/usr/local/bin/wget';
+	}
+
+	my $savepath = $BASEDIR . $url;
+	my $basedir; (undef, $basedir) = fileparse($savepath);
+	
+	unless (-d $basedir) {
+		my $ret = make_path ($basedir);
+
+		if ($ret < 1) {
+			syswrite STDERR, sprintf("Unable to create directory %s .\n", $basedir);
+			exit 1;
+		}
+	}
+
+	$url = $BASEURL . $url;
+	$url = urlencode($url);
+	system($wgetpath, '--no-check-certificate', '-q', '-T', '20', '-O', $savepath, '-o', '/dev/null', $url);
+
+	undef $savepath;
+	undef $url; 
+}
+
+sub urlencode($) {
+	my $url = shift;
+	my $urlobj = url $url;
+	$url = $urlobj->as_string;
+	undef $urlobj;
+	return $url;
+}
+
+sub getmetadata() {
+	dlfunc('repodata/repomd.xml');
+
+	my $p1 = new XML::Parser(Style => 'Tree');
+	my $ref = $p1->parsefile($BASEDIR . 'repodata/repomd.xml');
+
+#print Dumper $ref->[1][8][12][0]{'href'};
+#  must be 'data'-------^
+#  must be 'location'------^
+
+	my @array = @{ $ref->[1] };
+
+	for (my $i = 0; $i < @array; $i++) {
+
+		if (defined($array[$i]) and ($array[$i] eq 'data')){
+			$i++;
+			my @array2 = @{ $array[$i] };
+
+			for (my $j = 0; $j < @array2; $j++) {
+				if (defined($array2[$j]) and ($array2[$j] eq 'location')) {
+					$j++;
+					dlfunc($array2[$j][0]{'href'});
+				}
+
+			}
+
+			@array2 = -1;
+		}
+
+	}
+
+}
+__END__