eleksir/slackmetarep
0
Fork 0
Code Activity

Initial commit

Browse Source
This commit is contained in:
eleksir
2020-11-29 15:18:14 +03:00
commit c3f04b6c7f
16 changed files with 1623 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
.gitignore vendored Normal file
View File

@@ -0,0 +1,2 @@
vendor_perl
data/config.json

94
.perlcriticrc Normal file
View File

@@ -0,0 +1,94 @@
#!/usr/bin/env perlcritic
severity = 5
verbose = 8
# minorify some of warnings that are not really critical
[-InputOutput::RequireCheckedClose]
# What can we handle in case of error?
#[-Modules::ProhibitExcessMainComplexity]
# Noone cares
[-Subroutines::ProhibitExcessComplexity]
# Noone cares
#[-Subroutines::ProhibitExplicitReturnUndef]
# But it's very handy sometimes!
[-ValuesAndExpressions::ProhibitMagicNumbers]
# this policy makes too much false positives
#[-RegularExpressions::RequireLineBoundaryMatching]
# We use /m as needed
#[-RegularExpressions::RequireExtendedFormatting]
# We do not use extended formatting of regex, they still not THAT complex
#[-RegularExpressions::RequireDotMatchAnything]
# huh?
[-RegularExpressions::ProhibitEscapedMetacharacters]
# Looks like it pointless. How can i match simple dot without putting \. in regex? fuck off, man
#[-ControlStructures::ProhibitCascadingIfElse]
# I like long if..else chains
[-ControlStructures::ProhibitPostfixControls]
# In short lines we can afford ourselves to use postfix controls
# but ofcorse we should avoid such constructions in long lines.
[-Subroutines::ProhibitSubroutinePrototypes]
# Contrary to Perl::Critic we consider prototypes as useful, even if
# the compile-time checks of Perl are not perfect, Perl may give some
# hints.
[-CodeLayout::RequireTidyCode]
# Impossible to complete, because of on higher linting levels "## no critic..."
# require be per-case on same line where critique is, but tidy align them
# one line down.
[-CodeLayout::ProhibitParensWithBuiltins]
# duh... in some cases it reduces readbility
[-ValuesAndExpressions::ProhibitEmptyQuotes]
# cause we consider following correct and useful: my $var = '';
# the recomended use of q{} looks too complex: my $var = q{};
[-ValuesAndExpressions::ProhibitNoisyQuotes]
# Other people, other opinions: "," is better readable than qw(,)
#[-BuiltinFunctions::ProhibitSleepViaSelect]
# We do not wish to use too much external modules.
# select undef, undef, undef, 0.25; # actually is okay if you add comment that
# it is sleep for 0.25 seconds
[-NamingConventions::Capitalization]
# duh...
# Noone care, really
[-ControlStructures::ProhibitUnlessBlocks]
# They should not be too complex - only one condition should be checked,
# that's all
#[-Miscellanea::ProhibitTies]
# Just allow it. It's very handy stuff and extremely simple and safe enugh to use.
[-ValuesAndExpressions::RequireNumberSeparators]
# Not really
#[-InputOutput::RequireBracedFileHandleWithPrint]
# We prefer not to use braced file handles
#[-Documentation::PodSpelling]
# It doesn't help, but emits annoying errors
#[Documentation::RequirePodSections]
# Considering this as "soft" requirement. For modules that is (or will be)
# stand-alone libs well formed documentation is highly desired.
#lib_sections = NAME | VERSION | DESCRIPTION | AUTHOR | LICENSE AND COPYRIGHT
[-InputOutput::RequireBriefOpen]
# Yes, i do!
[-ValuesAndExpressions::ProhibitLongChainsOfMethodCalls]
# But i do really want to use them in JSON::XS declaration!

21
LICENSE Normal file
View File

@@ -0,0 +1,21 @@
MIT License
Copyright (c) 2020 Sergei Fedosoff <eleksir@gmail.com>
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
SOFTWARE.

154
README.md Normal file
View File

@@ -0,0 +1,154 @@
# Slackmetarep - Slackware repository metadata generator
## About
Slackware repository metadata generator and upload server.
## What for?
For receiving package uploads and creating metadata for slackware package repositories.
Main idea is to use it in automated CI process.
## What this application is not for?
It is not designed for large scale repositories or for use as a public service.
## How it works?
Upload file to repository via PUT request:
```bash
curl -X PUT --data-binary "@/bin/bash" -H 'auth: welcometothejungle' http://reposerver.tld/api/upload/repository/file.ext
```
Where **repository** is same that set in config.json and **file.ext** is package or piece of metadata
When all chunks of metadata and package are uploaded, you may issue metagen command:
```bash
curl -H 'repo: repository' 'http://reposerver.tld/api/metagen'
```
### What metadata?
As an example for building this application data format in AlienBob repository is taken. There is some different files
that contain pieces of information for generating metadata comptible with one used for Slackware-14.2 repositories.
More detailed information in [docs/metadata_en.md](docs/metadata_en.md)
So this tool needs these metadata chunks to properly write metadata:
* packagename-version-arch-suffix.lst
* packagename-version-arch-suffix.meta
* packagename-version-arch-suffix.tgz.md5
* packagename-version-arch-suffix.txt
Additional JSON-formatted metadata if required for buildinfo module to work. An example of it:
```json
{
"dmenu" : {
"SRC" : [
"dmenu-4.8.tar.gz"
]
},
"docker" : {
"DEPS" : [
"containerd",
"docker-proxy",
"tini",
"google-go-lang"
],
"GROUP" : {
"docker" : "281"
},
"SRC" : [
"docker-ce-19.03.13.tar.gz"
],
"USER" : {
"docker" : {
"GROUP" : "docker",
"HOME" : "/var/lib/docker",
"SHELL" : "/bin/false",
"UID" : "281"
}
}
},
"perf" : {
"DEPS" : [
"libunwind",
"kernel-headers",
"kernel-source"
]
}
}
```
But this data intended to help in package build script that prepares build environment, runs SlackBuilds, upload
files to repo and kicks metadata generation process.
All metadata required and generated by this application very-very close to one found in AlienBob's
[repositories](http://www.slackware.com/~alien/slackbuilds/), except no changelogs and rss are generated.
## Installation
In order to run this application, you need to "bootstrap" it - download and build
all required dependencies and libraries.
You'll need "Development Tools" or similar group of packages, perl, perl-devel,
perl-local-lib, perl-app-cpanm, zlib-devel, openssl-devel, libdb4-devel (Berkeley DB devel), make.
After installng required dependencies it is possible to run:
```bash
bash bootstrap.sh
```
and all libraries should be downloaded, built, tested and installed.
## How to run this?
Via [uWSGI](https://github.com/unbit/uwsgi).
```bash
#!/bin/ash
exec 2>&1
LOGDIR=/var/log/uwsgi
PIDDIR=/var/run/uwsgi
WORKDIR=/var/www/slackmetarep
mkdir -p $LOGDIR
chown -R apache:apache $LOGDIR
mkdir -p $PIDDIR
cd $WORKDIR
# for 100% sure
export PERL5LIB="${WORKDIR}/vendor_perl"
export PERL_LIB="$PERL5LIB"
exec sudo -u apache -g apache /usr/bin/uwsgi \
--yaml ${WORKDIR}/slackmetarep.yaml \
--pidfile ${PIDDIR}/slackmetarep.pid \
--perl-local-lib ${PERL5LIB}
```
At least i use uwsgi for this. Starman suits well too and should work without any problems. You can also try
[Nginx UNIT](https://unit.nginx.org/), but it was totally not tested.
## How to contribute?
Got something about this project? Make PR.
## Who should i thank?
Slackware mainatiners for their continous labour. Notably [AlienBob](https://docs.slackware.com/wiki:user:alienbob) and
SBo team.
## What is missing here?
At least proper documentaion of metadata format generated by this particular application.

86
Slackmetarep/Buildinfo.pm Normal file
View File

@@ -0,0 +1,86 @@
package Slackmetarep::Buildinfo;
use strict;
use warnings;
use vars qw/$VERSION/;
use English qw( -no_match_vars );
use Carp;
use JSON::XS;
use Slackmetarep::Conf qw(loadConf);
use Exporter qw(import);
our @EXPORT_OK = qw(buildinfo);
$VERSION = '1.0';
my $c = loadConf();
sub buildinfo ($) {
my $repo = shift;
my ($status, $content, $msg) = ('400', 'text/plain', "Bad Request?\n");
return ($status, $content, $msg) unless ($repo =~ /\//xmsg);
my ($config, $package) = split(/\//xms, $repo);
unless (defined($c->{buildinfo}->{$config})) {
$msg = "Config repo $config is not defined in config.\n";
return ($status, $content, $msg);
}
open (my $jhandle, '<', $c->{buildinfo}->{$config}) or do {
$status = '500';
$msg = "Unable to open $c->{buildinfo}->{$config}; $OS_ERROR\n";
carp '[FATA] ' . $msg;
return ($status, $content, $msg);
};
my $len = (stat($c->{buildinfo}->{$config}))[7];
my $json;
my $readlen = read ($jhandle, $json, $len);
close $jhandle; ## no critic (InputOutput::RequireCheckedSyscalls
unless (defined($readlen)) {
$status = '500';
$msg = "Unable to read $c->{buildinfo}->{$config}: $OS_ERROR";
carp '[FATA] ' . $msg;
return ($status, $content, $msg);
}
if ($len != $readlen) {
$status = '500';
$msg = "Size of $c->{buildinfo}->{$config} $len bytes but actually read $readlen bytes";
carp '[FATA] ' . $msg;
return ('500', $content, "Unable to read $c->{buildinfo}->{$config}\n");
}
my $j = eval { decode_json($json) } or do {
$status = '500';
$msg = "Error during decoding $c->{buildinfo}->{$config}; $EVAL_ERROR\n";
carp '[FATA] ' . $msg;
return ($status, $content, $msg);
};
unless (defined($j->{$package})) {
$msg = "$package is not defined in config for $config.\n";
carp '[ERRO] ' . $msg;
return ($status, $content, $msg);
}
$json = JSON::XS->new->pretty->canonical->indent(1)->encode($j->{$package}) or do {
$status = '500';
$msg = "Unable to encode json.\n";
carp '[ERRO] ' . $msg;
return ($status, $content, $msg);
};
$status = '200';
$content = 'application/json';
$msg = $json;
return ($status, $content, $msg);
}
1;
# vim: set ft=perl noet ai ts=4 sw=4 sts=4:

43
Slackmetarep/Conf.pm Normal file
View File

@@ -0,0 +1,43 @@
package Slackmetarep::Conf;
# loads config
use 5.018;
use strict;
use warnings;
use utf8;
use open qw(:std :utf8);
use English qw( -no_match_vars );
use vars qw/$VERSION/;
use JSON::XS;
use Exporter qw(import);
our @EXPORT_OK = qw(loadConf);
$VERSION = '1.0';
sub loadConf {
my $c = 'data/config.json';
open my $CH, '<', $c || die "[FATA] No conf at $c: $OS_ERROR\n";
my $len = (stat ($c)) [7];
my $json;
my $readlen = read $CH, $json, $len;
unless ($readlen) {
close $CH; ## no critic (InputOutput::RequireCheckedSyscalls
die "[FATA] Unable to read $c: $OS_ERROR\n";
}
if ($readlen != $len) {
close $CH; ## no critic (InputOutput::RequireCheckedSyscalls
die "[FATA] File $c is $len bytes on disk, but we read only $readlen bytes\n";
}
close $CH; ## no critic (InputOutput::RequireCheckedSyscalls
my $j = JSON::XS->new->utf8->relaxed;
return $j->decode ($json);
}
1;
# vim: set ft=perl noet ai ts=4 sw=4 sts=4:

387
Slackmetarep/Metagen.pm Normal file
View File

@@ -0,0 +1,387 @@
# Most of this data is generated during package build process and being uploaded
# to pool alongside with packages themselves. So we can easily and fast regen meta.
# Assume that whole operation takes resonable amount of time to perform it syncronously
# metadata consists of
# * CHECKSUMS.md5 or CHECKSUMS.md5.gz - checksum of each file in repo
# * ChangeLog.txt - optional, contains history of changes, this script does not
# handle it
# * FILELIST.TXT - whole list of files that are part of repostory data or
# metadata
# * PACKAGES.TXT - packages description with metainfo such as relative path in
# repo, compressed size and uncompressed size
# * MANIFEST.bz2 - list of files in each package
# TODO: make metadata in atomic way
# TODO: validate metadata
package Slackmetarep::Metagen;
use warnings;
use strict;
use utf8;
use open qw(:std :utf8);
use vars qw/$VERSION/;
use English qw( -no_match_vars );
use Carp;
use Fcntl;
use Compress::Raw::Bzip2 qw(BZ_RUN_OK BZ_STREAM_END);
use Compress::Raw::Zlib;
use POSIX qw(strftime);
use Slackmetarep::Conf qw(loadConf);
use Exporter qw(import);
our @EXPORT_OK = qw(metagen);
$VERSION = '1.0';
sub metagen($);
sub __pdate; # generates current date in pretty format
sub __trim($);
sub __readfile($);
sub __dirlist($);
sub __writefile(@);
sub __bzdata($);
sub __gzdata($);
sub metagen ($) {
my $dir = shift;
my $c = loadConf();
$dir = $c->{metagen}->{$dir};
unless (defined $dir || ($dir eq '')) {
return ('400', 'text/plain', "No such repository\n");
}
my $dirlist = __dirlist ($dir);
return ('500', 'text/plain', $dirlist->{error}) if (defined $dirlist->{error});
my @filelist = @{$dirlist->{dir}};
$dirlist = undef;
# manifest
my $buf = '';
foreach my $filename (@filelist) {
if ($filename =~ /\.lst/xmsg) {
my $file = __readfile("$dir/$filename");
return ('500', 'text/plain', $file->{error}) if (defined $file->{'error'});
$buf .= __trim $file->{data};
$buf .= "\n\n\n";
$file->{data} = ''; $file = undef;
}
}
my $bzdata = __bzdata (\$buf);
return ('500', 'text/plain', $bzdata->{error}) if (defined $bzdata->{error});
my $manifest = __writefile ("$dir/MANIFEST.bz2", \$bzdata->{data});
return ('500', 'text/plain', $manifest->{error}) if (defined $manifest->{error});
$bzdata->{data} = ''; $bzdata = undef;
# checksums
$buf = << 'MSG';
These are the MD5 message digests for the files in this directory.
If you want to test your files, use 'md5sum' and compare the values to
the ones listed here.
To test all these files, use this command:
tail +13 CHECKSUMS.md5 | md5sum --check | less
'md5sum' can be found in the GNU coreutils package on ftp.gnu.org in
/pub/gnu, or at any GNU mirror site.
MD5 message digest Filename
MSG
foreach my $filename (@filelist) {
next if ($filename =~ /CHECKSUMS\.md5$/xmsg);
next if ($filename =~ /CHECKSUMS.md5.gz$/xmsg);
if ($filename =~ /\.md5/xmsg) {
my $file = __readfile("$dir/$filename");
return ('500', 'text/plain', $file->{error}) if (defined $file->{'error'});
$buf .= $file->{data};
$file->{data} = ''; $file = undef;
}
}
my $checksum = __writefile ("$dir/CHECKSUMS.md5", \$buf);
return ('500', 'text/plain', $checksum->{error}) if (defined $checksum->{error});
my $gzipchecksum = __gzdata(\$buf);
return ('500', 'text/plain', $gzipchecksum->{error}) if (defined $gzipchecksum->{error});
$buf = '';
my $checksumgz = __writefile ("$dir/CHECKSUMS.md5.gz", \$gzipchecksum->{data});
return ('500', 'text/plain', $checksumgz->{error}) if (defined $checksumgz->{error});
$gzipchecksum->{data} = ''; $gzipchecksum = undef;
# packages
$buf = sprintf "PACKAGES.TXT; %s\n\n\n", __pdate;
foreach my $filename (@filelist) {
if ($filename =~ /\.meta/xmsg) {
my $file = __readfile("$dir/$filename");
return ('500', 'text/plain', $file->{error}) if (defined $file->{'error'});
$buf .= __trim ($file->{data});
$buf .= "\n\n";
$file->{data} = ''; $file = undef;
}
}
my $packages = __writefile ("$dir/PACKAGES.TXT", \$buf);
return ('500', 'text/plain', $packages->{error}) if (defined $packages->{error});
my $gzippackages = __gzdata(\$buf);
return ('500', 'text/plain', $gzippackages->{error}) if (defined $gzippackages->{error});
$buf = '';
my $packagesgz = __writefile ("$dir/PACKAGES.TXT.gz", \$gzippackages->{data});
return ('500', 'text/plain', $packagesgz->{error}) if (defined $packagesgz->{error});
$gzippackages->{data} = ''; $gzippackages = undef;
# filelist
$buf = __pdate;
$buf .= "\n\n";
$buf .= << 'BUFFER';
Here is the file list for this directory,
maintained by eleksir <eleksir@gmail.com> .
If you are using a mirror site and find missing or extra files
in the subdirectories, please have the archive administrator
refresh the mirror.
BUFFER
# dir listing is not actual, so we need to re-read it
$#filelist = -1;
$dirlist = __dirlist ($dir);
return ('500', 'text/plain', $dirlist->{error}) if (defined $dirlist->{error});
@filelist = @{$dirlist->{dir}};
$dirlist = undef;
while (my $file = shift (@filelist)) {
my $filename = sprintf ('%s/%s', $dir, $file);
my @stat = stat $filename;
my @date = localtime ($stat[10]);
$buf .= sprintf (
"-rw-r--r-- %d root root % 8d %04d-%02d-%02d %02d:%02d ./%s\n",
$stat[3],
$stat[7],
$date[5] + 1900,
$date[4],
$date[3],
$date[2],
$date[1],
$file
);
}
$#filelist = -1;
my $filelist = __writefile ("$dir/FILELIST.TXT", \$buf);
return ('500', 'text/plain', $filelist->{error}) if (defined $filelist->{error});
$buf = '';
return ('200', 'text/plain', "Done\n");
}
sub __pdate {
my @time = gmtime(time);
my @DAYOFWEEK = qw(Sun Mon Tue Wed Thu Fri Sat);
my @MONTH = qw(Jan Feb Mar Apr May Jun Jul Aug Sep Nov Dec);
return strftime "$DAYOFWEEK[$time[6]] $MONTH[$time[4] - 1] %e %T UTC %Y", @time;
}
sub __trim ($) {
my $str = shift;
while (substr ($str, 0, 1) =~ /^\s$/xms) {
$str = substr ($str, 1);
}
while (substr ($str, -1, 1) =~ /^\s$/xms) {
chop ($str);
}
return $str;
}
sub __readfile ($) {
my $file = shift;
my $ret;
open (my $FILEHANDLE, '<', $file) or do {
my $msg = "Unable to open file $file: $OS_ERROR";
carp "[FATA] $msg";
$ret->{error} = $msg;
return $ret;
};
binmode $FILEHANDLE;
my $buf;
my $len = (stat $file)[7];
my $readlen = read ($FILEHANDLE, $buf, $len);
close $FILEHANDLE; ## no critic (InputOutput::RequireCheckedSyscalls)
unless (defined $readlen) {
my $msg = "Unable to read $file: $OS_ERROR";
carp "[FATA] $msg";
$ret->{error} = $msg;
return $ret;
}
if ($readlen != $len) {
my $msg = "Unable to read $file: amount of read bytes does not match with file size";
carp "[FATA] $msg";
$ret->{error} = $msg;
return $ret;
}
$ret->{data} = $buf;
return $ret;
}
sub __writefile (@) {
my $file = shift;
my $dataref = shift;
my $ret;
sysopen (my $FILEHANDLE, $file, O_WRONLY|O_TRUNC|O_CREAT) or do {
my $msg = "Unable to open file $file: $OS_ERROR";
carp "[FATA] $msg";
$ret->{error} = $msg;
return $ret;
};
binmode $FILEHANDLE;
use bytes;
my $len = length ${$dataref};
no bytes;
my $writelen = syswrite ($FILEHANDLE, ${$dataref}, $len);
close $FILEHANDLE; ## no critic (InputOutput::RequireCheckedSyscalls)
unless (defined $writelen) {
my $msg = "Unable to write $file: $OS_ERROR";
carp "[FATA] $msg";
$ret->{error} = $msg;
return $ret;
}
if ($writelen != $len) {
my $msg = "Unable to write $file: amount of read bytes does not match with file size";
carp "[FATA] $msg";
$ret->{error} = $msg;
return $ret;
}
$ret->{success} = 1;
return $ret;
}
sub __dirlist ($) {
my $dir = shift;
my $res;
my @files;
opendir (my $dirhandle, $dir) or do {
my $msg = "Unable to open $dir: $OS_ERROR";
carp "[FATA] $msg";
$res->{error} = $msg;
return $msg;
};
while (readdir $dirhandle) {
next unless (-f "$dir/$_");
push @files, $_;
}
closedir $dirhandle;
@files = sort @files;
$res->{dir} = \@files;
return $res;
}
sub __bzdata ($) {
my $databufref = shift;
my $ret;
my $bzdata;
my $bz = Compress::Raw::Bzip2->new (1, 9, 0);
unless (defined $bz) {
my $msg = 'Unable to create bz object';
carp "[FATA] $msg";
$ret->{error} = $msg;
return $ret;
}
if ($bz->bzdeflate(${$databufref}, $bzdata) != BZ_RUN_OK) {
my $msg = 'Unable to perofrm bzip2 compression';
carp "[FATA] $msg";
$ret->{error} = $msg;
return $ret;
}
if ($bz->bzflush($bzdata) != BZ_RUN_OK) {
my $msg = 'Unable to flush bz buffer';
carp "[FATA] $msg";
$ret->{error} = $msg;
return $ret;
}
if ($bz->bzclose($bzdata) != BZ_STREAM_END) {
my $msg = 'Unable to flush and close bz buffer';
carp "[FATA] $msg";
$ret->{error} = $msg;
return $ret;
}
$ret->{data} = $bzdata;
return $ret;
}
sub __gzdata ($) {
my $databufref = shift;
my $ret;
my $gzdata;
my $gz = Compress::Raw::Zlib::Deflate->new (
-Level => Z_BEST_COMPRESSION,
-CRC32 => 1,
-ADLER32=> 1,
-WindowBits => WANT_GZIP
);
unless (defined $gz) {
my $msg = 'Unable to create gz object';
carp "[FATA] $msg";
$ret->{error} = $msg;
return $ret;
}
if ($gz->deflate(${$databufref}, $gzdata) != Z_OK) {
my $msg = 'Unable to deflate';
carp "[FATA] $msg";
$ret->{error} = $msg;
return $ret;
}
if ($gz->flush($gzdata) != Z_OK) {
my $msg = 'Unable to flush gz object';
carp "[FATA] $msg";
$ret->{error} = $msg;
return $ret;
}
$ret->{data} = $gzdata;
return $ret;
}
1;
# vim: set ft=perl noet ai ts=4 sw=4 sts=4 :

99
Slackmetarep/Upload.pm Normal file
View File

@@ -0,0 +1,99 @@
package Slackmetarep::Upload;
use strict;
use warnings;
use vars qw/$VERSION/;
use English qw( -no_match_vars );
use Carp;
use Fcntl;
use Slackmetarep::Conf qw(loadConf);
use Exporter qw(import);
our @EXPORT_OK = qw(upload);
$VERSION = '1.0';
my $c = loadConf ();
sub upload {
my $input = shift; # file descriptor with data being uploaded
my $len = shift; # expected data length (from client)
my $name = shift; # upload "dir" and filename
my ($status, $content, $msg) = ('400', 'text/plain', "Bad Request?\n");
return ($status, $content, "No Content-Length supplied.\n") unless (defined $len);
return ($status, $content, "No name supplied.\n") unless (defined $name);
my $d;
($d, $name) = split (/\//xms, $name, 2);
return ($status, $content, "Name does not match pattern.\n") unless ($name =~ /^[[:alnum:]_\-\+\.]+$/xmsg);
my $match;
foreach my $uploaddir (keys(%{$c->{upload}->{dir}})) {
if ($d eq $uploaddir) {
$match = 1;
last;
}
}
return ($status, $content, "Incorrect destination dir.\n") unless $match; # incorrect destination in url
$name = sprintf('%s/%s', $c->{upload}->{dir}->{$d}, $name);
if ($len > 0) {
if (sysopen (my $FH, $name, O_CREAT|O_TRUNC|O_WRONLY)) {
my $buf;
my $readlen = 0;
my $totalread = 0;
my $buflen = 524288; # 512 kbytes, looks sane enough
if ($len < $buflen) {
$buflen = $len;
}
do {
$readlen = $input->read($buf, $buflen);
my $written = syswrite $FH, $buf, $readlen;
unless (defined($written)) { # out of space?
close $FH; ## no critic (InputOutput::RequireCheckedSyscalls)
unlink $name;
carp "[FATA] Unable to write to $name: $OS_ERROR";
return ('500', $content, "An error has occured during upload: $OS_ERROR\n");
}
if ($readlen != $written) {
close $FH; ## no critic (InputOutput::RequireCheckedSyscalls)
unlink $name;
carp "[FATA] Must write $readlen bytes, but actualy wrote $written bytes to $name";
return ('500', $content, "An error has occured during upload: $OS_ERROR\n");
}
$totalread += $readlen;
} while ($readlen == $buflen);
close $FH; ## no critic (InputOutput::RequireCheckedSyscalls)
if ($totalread != $len) {
($content, $msg) = ($content, "Content-Length does not match amount of recieved bytes.\n");
} else {
($status, $content, $msg) = ('201', $content, "Uploaded.\n");
}
} else {
carp "[FATA] Unable to open file $name: $OS_ERROR";
($status, $content, $msg) = ('500', $content, "Unable to write: $OS_ERROR\n");
}
} else {
$msg = "Incorrect Content-Length\n";
}
return ($status, $content, $msg);
}
1;
# vim: set ft=perl noet ai ts=4 sw=4 sts=4 :

5
bootstrap.sh Normal file
View File

@@ -0,0 +1,5 @@
#!/bin/bash
cpanm -l vendor_perl --self-contained --force --update --installdeps .
# vim: set ft=sh noet ai ts=4 sw=4 sts=4:

20
confcheck.pl Executable file
View File

@@ -0,0 +1,20 @@
#!/usr/bin/perl
use 5.018;
use strict;
use warnings;
use utf8;
use open qw(:std :utf8);
use English qw( -no_match_vars );
use JSON::XS;
use lib qw(./vendor_perl ./vendor_perl/lib/perl5);
use Slackmetarep::Conf;
use version; our $VERSION = qv(1.0);
my $CONF = loadConf();
my $j = JSON::XS->new->pretty->canonical->indent(1);
say $j->encode ($CONF); ## no critic (InputOutput::RequireCheckedSyscalls)
# vim: set ft=perl noet ai ts=4 sw=4 sts=4:

8
cpanfile Normal file
View File

@@ -0,0 +1,8 @@
requires 'Compress::Raw::Bzip2';
requires 'Compress::Raw::Zlib';
requires 'Exporter';
requires 'Fcntl';
requires 'JSON::XS';
requires 'POSIX';
requires 'Plack::Util';
requires 'local::lib';

25
data/sample_config.json Normal file
View File

@@ -0,0 +1,25 @@
{
"api" : {
"prefix" : "/api"
},
"upload" : {
"dir" : {
"thrirdparty": "/var/www/REPO/slackware/thrird-party/14.2/x86_64",
"thrird-party": "/var/www/REPO/slackware/thrird-party/14.2/x86_64",
"slackbuild-thrird-party": "/var/www/REPO/slackware/thrird-party/14.2/x86_64",
"slackbuild-thrirdparty": "/var/www/REPO/slackware/thrird-party/14.2/x86_64"
},
"auth": "welcometothejungle"
},
"metagen" : {
"thrirdparty" : "/var/www/REPO/slackware/thrird-party/14.2/x86_64",
"thrird-party" : "/var/www/REPO/slackware/thrird-party/14.2/x86_64",
"slackbuild-thrird-party" : "/var/www/REPO/slackware/thrird-party/14.2/x86_64",
"slackbuild-thrirdparty" : "/var/www/REPO/slackware/thrird-party/14.2/x86_64"
},
"buildinfo" : {
"thrirdparty" : "/var/www/config-slackbuild-thrird-party.json",
"slackbuild-thrirdparty" : "/var/www/config-slackbuild-thrird-party.json",
"slackbuild-thrird-party" : "/var/www/config-slackbuild-thrird-party.json"
}
}

303
docs/metadata_en.md Normal file
View File

@@ -0,0 +1,303 @@
# Metadata of slackware packages repository
To install packages from repository, repository package list is required. Also description is desired. All this infrmation called metadata.
## Standard metadata format
Metadata from DVD and in official repo in internet both identical. Format od data by itself allows some variations.
Metadata is split across some files for ease of it creation and parsing.
### list of metadata files in main repo
**CHECKSUMS.md5, CHECKSUMS.md5.asc, ChangeLog.txt, FILELIST.TXT, GPG-KEY, PACKAGES.TXT**
### List of metadata files in sub-repos
There are sub-repositories in main repository pool, it just sub-dirs here with its own pack of metadata files. It is: patches, pasture, extra.
**Patches** - updates.
**Pasture** - obsoleted packages that have compatibility issues, not supported by authors or for other reasons are not included in main packages list.
**Extra** - new or alternative packages that for some reasons not in main package list.
Sub-repository **patches** exist only in release version of distro. There is no patches in -current.
There is different metadata files in sub-repo:
**CHECKSUMS.md5, CHECKSUMS.md5.asc, FILE_LIST, MANIFEST.bz2, PACKAGES.TXT.**
### Metadata description
#### CHECKSUMS.md5
This is md5-checksim for each file in repository available at the moment of CHECKSUM.m5 creation. Header of this file contains instruction how check yoour copy intergity:
```text
These are the MD5 message digests for the files in this directory.
If you want to test your files, use 'md5sum' and compare the values to
the ones listed here.
To test all these files, use this command:
tail +13 CHECKSUMS.md5 | md5sum -c --quiet - | less
'md5sum' can be found in the GNU coreutils package on ftp.gnu.org in
/pub/gnu, or at any GNU mirror site.
MD5 message digest Filename
```
There is 1 extra new line between header and actual data. There is no enpty lines in the body of this file.
```text
2c4180b57c277ad3cd38865a466a4c15 ./ANNOUNCE.14_2
85891676a985eb6c87170083e1df75de ./CHANGES_AND_HINTS.TXT
```
and so on for all other files. Important checksums here is only for packages itselves, accompanying .txt and respective asc files. MD5 for other files can be ommited.
***Notes to format of this file:***
* first, no empty lined allowed in data body
* second, there must be exactly one newline after last record in file
* thrird, file name always begins with "./" (no quotes).
#### CHECKSUMS.md5.asc
This file sries gpg signature for CHECKSUMS.md5.
GPG signature is highly desired but not required. It ensures that packages with matching md5 is authored and placed to repository by owner of closed part of gpg key. In other word it ensures that package is genuine.
#### ChangeLog.txt
All package changes are stored in this file. This file exist in main repo and contains also changes that made to sub-repostory.
Changes are recorded here in "batches":
```text
Sat Nov 2 18:52:12 UTC 2019
a/aaa_terminfo-6.1_20191026-x86_64-1.txz: Upgraded.
a/cryptsetup-2.2.2-x86_64-1.txz: Upgraded.
a/lvm2-2.03.06-x86_64-1.txz: Upgraded.
d/Cython-0.29.14-x86_64-1.txz: Upgraded.
l/ncurses-6.1_20191026-x86_64-1.txz: Upgraded.
Restore the --without-normal option to skip static libraries as used in 14.2.
Thanks to Richard Narron.
x/xterm-349-x86_64-2.txz: Rebuilt.
In /etc/app-defaults/XTerm, use terminus-medium instead of terminus-bold.
Thanks to igadoter.
+--------------------------+
```
Note the delimeter at the bottom of the batch.
This file is optional and can be found not only in official repo but for example in AlienBob's. By the way in AlienBob's reop this file also *begins* with delimeter and contains extra newline between batches.
#### FILELIST.TXT
List of all files in repository. Means all related files for endusers, but as a rule it contains all files in repository.
Thist file contains header:
```text
Sat Nov 2 19:03:12 UTC 2019
Here is the file list for this directory. If you are using a
mirror site and find missing or extra files in the disk
subdirectories, please have the archive administrator refresh
the mirror.
```
Note the date of file generation and one extra new line after header.
File body looks like that:
```text
drwxr-xr-x 12 root root 4096 2019-11-02 18:52 .
-rw-r--r-- 1 root root 10064 2016-06-30 18:39 ./ANNOUNCE.14_2
-rw-r--r-- 1 root root 14642 2019-10-18 21:18 ./CHANGES_AND_HINTS.TXT
```
Similar listing you can get by invoking next command:
```bash
ls -lAn --time-style=long-iso "$dir"
```
but please note that in ls output there is no "." directory and file path will not begin with "./" which is required.
File must end with newline symbol.
#### GPG-KEY
This is public part of GPG-key you can check integrity of files that is signed with apropiate closed key (files with satelite files with **.asc** extension).
How to use it:
Check the key for correctness:
```bash
gpg --keyid-format long --show-key GPG-KEY
```
Import it to our keyring:
```bash
gpg --import GPG-KEY
```
Then check:
```bash
gpg --verify CHECKSUMS.md5.asc CHECKSUMS.md5
```
#### PACKAGES.TXT
Repository packages list (kind of). Classic header looks like this:
```text
PACKAGES.TXT; Sat Nov 2 19:01:20 UTC 2019
This file provides details on the Slackware packages found
in the ./slackware64/ directory.
Total size of all packages (compressed): 2612 MB
Total size of all packages (uncompressed): 11323 MB
```
Header begins with newline and ends with 2 newlines.
Technically whole header can be "compressed" to this form withoun any loss:
```text
PACKAGES.TXT; Sun Nov 3 12:47:53 UTC 2019
```
In this particular case file does not begin with newline and contains only one newline in the end of header. AlienBob do it like this.
Body itself consists of batches of strings related to each package. Batch separator is extra newline. Example of package record from official repository:
```text
PACKAGE NAME: ConsoleKit2-1.0.0-x86_64-4.txz
PACKAGE LOCATION: ./slackware64/l
PACKAGE SIZE (compressed): 148 K
PACKAGE SIZE (uncompressed): 780 K
PACKAGE DESCRIPTION:
ConsoleKit2: ConsoleKit2 (user, login, and seat tracking framework)
ConsoleKit2:
ConsoleKit2: ConsoleKit2 is a framework for defining and tracking users, login
ConsoleKit2: sessions, and seats.
ConsoleKit2:
ConsoleKit2: Homepage: https://github.com/ConsoleKit2/ConsoleKit2
ConsoleKit2:
```
Example from thrird-party repo:
```text
PACKAGE NAME: SDL2-2.0.10-x86_64-1.txz
PACKAGE LOCATION: .
PACKAGE SIZE (compressed): 609 K
PACKAGE SIZE (uncompressed): 3260 K
PACKAGE REQUIRED:
PACKAGE CONFLICTS:
PACKAGE SUGGESTS:
PACKAGE DESCRIPTION:
SDL2:
SDL2: Simple DirectMedia Layer is a cross-platform development library
SDL2: designed to provide low-level access to audio, keyboard, mouse,
SDL2: joystick, and graphics hardware ia via OpenGL. It is used by
SDL2: video playback software, emulators, and games.
SDL2:
SDL2: Homepage: http://www.libsdl.org
SDL2:
SDL2:
SDL2:
```
As you can see everythin that goes after "PACKAGE DESCRIPTION:" is contents of apropriate slack-desc from package.
Also thrird party repository contains extended metainformation - dependencies, conflicts and recommended packages.
In official repository this file ends with 2 newlines, AlienBob makes only one and it works fine.
#### MANIFEST.bz2
In official repository it is placed only in sub-repositries, for thrird-parties it placed in main repository.
It doesen't contain any header. All records are separated by extra 2 newlines. File ends with 2 extra newlines.
Example of one record:
```text
++========================================
||
|| Package: ./aspell-word-lists/aspell-am-0.03_1-x86_64-5.txz
||
++========================================
drwxr-xr-x root/root 0 2016-06-06 15:13 ./
drwxr-xr-x root/root 0 2016-06-06 15:13 install/
-rw-r--r-- root/root 190 2016-06-06 15:13 install/slack-desc
drwxr-xr-x root/root 0 2016-06-06 15:13 usr/
drwxr-xr-x root/root 0 2016-06-06 15:13 usr/lib64/
drwxr-xr-x root/root 0 2016-06-06 15:13 usr/lib64/aspell/
-rw-r--r-- root/root 101 2016-06-06 15:13 usr/lib64/aspell/am.dat
-rw-r--r-- root/root 282544 2016-06-06 15:13 usr/lib64/aspell/am.rws
-rw-r--r-- root/root 426 2016-06-06 15:13 usr/lib64/aspell/am_affix.dat
-rw-r--r-- root/root 1871 2016-06-06 15:13 usr/lib64/aspell/am_phonet.dat
-rw-r--r-- root/root 72 2016-06-06 15:13 usr/lib64/aspell/amharic.alias
-rw-r--r-- root/root 25615 2016-06-06 15:13 usr/lib64/aspell/s-ethi.cmap
-rw-r--r-- root/root 70 2016-06-06 15:13 usr/lib64/aspell/am.multi
-rw-r--r-- root/root 13356 2016-06-06 15:13 usr/lib64/aspell/s-ethi.cset
drwxr-xr-x root/root 0 2016-06-06 15:13 usr/doc/
drwxr-xr-x root/root 0 2016-06-06 15:13 usr/doc/aspell-am-0.03_1/
-rw-r--r-- root/root 394 2004-12-22 20:27 usr/doc/aspell-am-0.03_1/Copyright
-rw-r--r-- root/root 2434 2004-12-28 08:15 usr/doc/aspell-am-0.03_1/README
```
## Metadata format extensions
As extension to official repo it may note existance of gzipped files: CHECKSUMS.md5.gz ChangeLog.txt.gz PACKAGES.TXT.gz. In case of gpg signature of uncompressed files exist, it also must me generated for compressed files too.
There is additional fields in PACKAGES.TXT:
```text
PACKAGE REQUIRED:
PACKAGE CONFLICTS:
PACKAGE SUGGESTS:
```
Their purpos is obviously more flexible package management.
## Methods of repo metsdata creation
There is to approaches to metadata generation:
* lazy - made with pre-generated additional information chunks (or database) that are gathered durin package build process.
* full - extract all metadata directly from packages.
### Lazy method
It been used by AlienBob in his repo. Some additional files being generated during package build process and being uploaded to repository. When repository metadata generator run, it parses these additional files and makes metadata for repo.
It is also possible to store additional metainformation abaout packages in some sort of database for same puposes.
There is some benefits in this approach. It is faster to parse pre-generated metadata than extract it from package. it is especially notable on large repositories.
As a side effect there are some extra files in repository. But you can mitigate it by using database, but it adds some complexity to solution of repo metadata problem.
### Full data methos
I do not know application area of this method. But official repositories does not contain any extra files.
This method suits for small repositories.
Tradeoffs of this method is cpu and disk intensive process of metadata generation because of package unpacking and inspection occurs during metadata generation.
Benefits is obvious. You need no extra files in repo, just packages itselves.

298
docs/metadata_ru.md Normal file
View File

@@ -0,0 +1,298 @@
# Метаданные репозитория
Чтобы установить пакеты из репозитория, необходимо знать список пакетов, которые там есть, кроме того желательно иметь описание для каждого пакета и некоторую техническую информацию.
## Стандартный формат метаданных
Метаданные на DVD ROM и в официальном репозитории в интернете по своему формату одинаковые. Сам по себе формат допускает некоторые небольшие вольности, подозреваю, что это связано с тем, что разбор этих данных происходит стандартными утилитами командной строки, которые позволяют в некоторых местах использовать относительно "свободный" формат.
Метаданные размещены в нескольких текстовых файлах, для простоты их генерации и разбора.
### Список файлов метаданных основного репозитория
**CHECKSUMS.md5, CHECKSUMS.md5.asc, ChangeLog.txt, FILELIST.TXT, GPG-KEY, PACKAGES.TXT**
### Список файлов метаданных суб-репозиториев
Суб-репозитории - это подкаталоги с тематической подборкой пакетов. Их бывает 3: patches, pasture, extra.
**Patches** - это обновления.
**Pasture** - это пакеты, которые были в прошлых выпусках дистрибутива, но не вошли в текущий по тем или иным причинам.
**Extra** - это новые пакеты, которые были предложены к использованию в текущем выпуске, но по тем или иным причинам не были включены в основной список пакетов.
Суб-репозиторий **patches** присутствует только в репозитории с релизным вариантом дистрибутива, то есть его нет, например, в репозитории -current.
Метаданные субрепозиториев немного отличаются.
**CHECKSUMS.md5, CHECKSUMS.md5.asc, FILE_LIST, MANIFEST.bz2, PACKAGES.TXT.**
### Описание метаданных
#### CHECKSUMS.md5
Это файл с checksum-ами в формате md5, в заголовке файла есть описание, как его сгенерировать:
```text
These are the MD5 message digests for the files in this directory.
If you want to test your files, use 'md5sum' and compare the values to
the ones listed here.
To test all these files, use this command:
tail +13 CHECKSUMS.md5 | md5sum -c --quiet - | less
'md5sum' can be found in the GNU coreutils package on ftp.gnu.org in
/pub/gnu, or at any GNU mirror site.
MD5 message digest Filename
```
После заголовка идёт 1 перенос строки, то есть между последней строкой заголовка и первой строкой тела этого файла пустых строк нет.
Само тело это действительно вывод указанной команды:
```text
2c4180b57c277ad3cd38865a466a4c15 ./ANNOUNCE.14_2
85891676a985eb6c87170083e1df75de ./CHANGES_AND_HINTS.TXT
```
и так далее для всех файлов в репозитории. Значимыми здесь являются файлы пакетов и сопровождающие их asc и txt файлы. Остальные - за компанию, для них вычислять md5 необязательно.
***Замечания к формату тела файла:***
* во-первых, не должно быть пустых строк
* во вторых, файл чётко должен заканчиваться на один снос строки, после последней записи
* в третьих, имя файла относительно и всегда начинается на "./" (без кавычек).
#### CHECKSUMS.md5.asc
В этом файле хранится gpg-подпись для CHECKSUMS.md5.
GPG-подпись желательна, но не обязательна. Она удостоверяет, что пакет был положен в репозиторий владельцем закрытого gpg-ключа, соответственно, он подлинный.
#### ChangeLog.txt
Здесь записываются все изменения. Этот файл встречается в основном репозитории и в него входят изменения, в том числе внесённые в суб-репозитории.
Изменения записываются "пачками":
```text
Sat Nov 2 18:52:12 UTC 2019
a/aaa_terminfo-6.1_20191026-x86_64-1.txz: Upgraded.
a/cryptsetup-2.2.2-x86_64-1.txz: Upgraded.
a/lvm2-2.03.06-x86_64-1.txz: Upgraded.
d/Cython-0.29.14-x86_64-1.txz: Upgraded.
l/ncurses-6.1_20191026-x86_64-1.txz: Upgraded.
Restore the --without-normal option to skip static libraries as used in 14.2.
Thanks to Richard Narron.
x/xterm-349-x86_64-2.txz: Rebuilt.
In /etc/app-defaults/XTerm, use terminus-medium instead of terminus-bold.
Thanks to igadoter.
+--------------------------+
```
Файл опционален, он входит в состав не только официальных репозиториев, но и встречается, например, у AlienBob-а. Кстати, у последнего этот файл начинается с разделителя и после каждой "пачки" вставляется дополнительная пустая строка.
#### FILELIST.TXT
Список всех файлов в репозитории. Имеются в виду, конечно, значимые файлы, но, как правило, генерят просто список всех файлов, начиная с корня репозитория.
Файл содержит следующий заголовок:
```text
Sat Nov 2 19:03:12 UTC 2019
Here is the file list for this directory. If you are using a
mirror site and find missing or extra files in the disk
subdirectories, please have the archive administrator refresh
the mirror.
```
После заголовка и перед телом файла присутствует пустая строка.
Тело файла выглядит примерно так:
```text
drwxr-xr-x 12 root root 4096 2019-11-02 18:52 .
-rw-r--r-- 1 root root 10064 2016-06-30 18:39 ./ANNOUNCE.14_2
-rw-r--r-- 1 root root 14642 2019-10-18 21:18 ./CHANGES_AND_HINTS.TXT
```
похожий список можно получить в выводе команды:
```bash
ls -lAn --time-style=long-iso "$dir"
```
однако в выводе не будет каталога "." и все файлы не будут начинаться "./".
Файл заканчивается на снос строки.
#### GPG-KEY
Это публичная часть ключа, с помощью которой можно проверять целостность файлов, подписанных gpg (при которых присутствует файл-сателлит с расширением **.asc**).
А воспользоваться этим инструментом можно примерно так:
Проверяем, что ключ правильный:
```text
gpg --keyid-format long --show-key GPG-KEY
```
Импортируем его к себе в keyring:
```text
gpg --import GPG-KEY
```
Собственно, проверяем:
```text
gpg --verify CHECKSUMS.md5.asc CHECKSUMS.md5
```
#### PACKAGES.TXT
Список пакетов в репозитории. Классический заголовок файла выглядит так:
```text
PACKAGES.TXT; Sat Nov 2 19:01:20 UTC 2019
This file provides details on the Slackware packages found
in the ./slackware64/ directory.
Total size of all packages (compressed): 2612 MB
Total size of all packages (uncompressed): 11323 MB
```
Заголовок файла начинается с пустой строки, а заканчивается 2 (*двумя!*) пустыми строками.
Технически это всё свободно (без потери работоспособности) может ужиматься до такой записи:
```text
PACKAGES.TXT; Sun Nov 3 12:47:53 UTC 2019
```
В этом случае заголовок не начинается с пустой строки и заканчивается на одну пустую строку. Так делает AlienBob.
Само тело состоит из записей (набор строк), разделённых пустой строкой. Пример записи из основного репозитория:
```text
PACKAGE NAME: ConsoleKit2-1.0.0-x86_64-4.txz
PACKAGE LOCATION: ./slackware64/l
PACKAGE SIZE (compressed): 148 K
PACKAGE SIZE (uncompressed): 780 K
PACKAGE DESCRIPTION:
ConsoleKit2: ConsoleKit2 (user, login, and seat tracking framework)
ConsoleKit2:
ConsoleKit2: ConsoleKit2 is a framework for defining and tracking users, login
ConsoleKit2: sessions, and seats.
ConsoleKit2:
ConsoleKit2: Homepage: https://github.com/ConsoleKit2/ConsoleKit2
ConsoleKit2:
```
Пример записи из стороннего репозитория:
```text
PACKAGE NAME: SDL2-2.0.10-x86_64-1.txz
PACKAGE LOCATION: .
PACKAGE SIZE (compressed): 609 K
PACKAGE SIZE (uncompressed): 3260 K
PACKAGE REQUIRED:
PACKAGE CONFLICTS:
PACKAGE SUGGESTS:
PACKAGE DESCRIPTION:
SDL2:
SDL2: Simple DirectMedia Layer is a cross-platform development library
SDL2: designed to provide low-level access to audio, keyboard, mouse,
SDL2: joystick, and graphics hardware ia via OpenGL. It is used by
SDL2: video playback software, emulators, and games.
SDL2:
SDL2: Homepage: http://www.libsdl.org
SDL2:
SDL2:
SDL2:
```
как видим, всё, что идёт после "PACKAGE DESCRIPTION:" - это фрагмент файла slack-desc с описанием пакета.
Соответственно, сторонний репозиторий содержит расширенную информацию: зависимости, конфликты, рекомендованные пакеты
В случае официального репозитория, файл заканчивается на 2 пустые строки. AlienBob в своём стороннем репозитории довольствуется одной пустой строкой и это работает.
#### MANIFEST.bz2
Для официального репозитория он лежит только в суб-репозиториях, для сторонних репозиториев его кладут в основной репозиторий. Содержит список файлов в пакете.
Не содержит заголовка. Записи разделяются двумя пустыми строками.
```text
++========================================
||
|| Package: ./aspell-word-lists/aspell-am-0.03_1-x86_64-5.txz
||
++========================================
drwxr-xr-x root/root 0 2016-06-06 15:13 ./
drwxr-xr-x root/root 0 2016-06-06 15:13 install/
-rw-r--r-- root/root 190 2016-06-06 15:13 install/slack-desc
drwxr-xr-x root/root 0 2016-06-06 15:13 usr/
drwxr-xr-x root/root 0 2016-06-06 15:13 usr/lib64/
drwxr-xr-x root/root 0 2016-06-06 15:13 usr/lib64/aspell/
-rw-r--r-- root/root 101 2016-06-06 15:13 usr/lib64/aspell/am.dat
-rw-r--r-- root/root 282544 2016-06-06 15:13 usr/lib64/aspell/am.rws
-rw-r--r-- root/root 426 2016-06-06 15:13 usr/lib64/aspell/am_affix.dat
-rw-r--r-- root/root 1871 2016-06-06 15:13 usr/lib64/aspell/am_phonet.dat
-rw-r--r-- root/root 72 2016-06-06 15:13 usr/lib64/aspell/amharic.alias
-rw-r--r-- root/root 25615 2016-06-06 15:13 usr/lib64/aspell/s-ethi.cmap
-rw-r--r-- root/root 70 2016-06-06 15:13 usr/lib64/aspell/am.multi
-rw-r--r-- root/root 13356 2016-06-06 15:13 usr/lib64/aspell/s-ethi.cset
drwxr-xr-x root/root 0 2016-06-06 15:13 usr/doc/
drwxr-xr-x root/root 0 2016-06-06 15:13 usr/doc/aspell-am-0.03_1/
-rw-r--r-- root/root 394 2004-12-22 20:27 usr/doc/aspell-am-0.03_1/Copyright
-rw-r--r-- root/root 2434 2004-12-28 08:15 usr/doc/aspell-am-0.03_1/README
```
## Расширения стандартного формата метаданных
В качестве расширения к официальному формату можно указать наличие сжатых версий файлов CHECKSUMS.md5.gz ChangeLog.txt.gz PACKAGES.TXT.gz. В данном случае если у несжатой версии есть подпись в asc-файле, то подписывать надо и сжатый файл тоже.
Кроме дополнительных сжатых файлов появились новые поля в PACKAGES.TXT:
```text
PACKAGE REQUIRED:
PACKAGE CONFLICTS:
PACKAGE SUGGESTS:
```
Их назначение вполне очевидно - более гибкое и более "умное" управление процессом установки пакетов.
## Подходы к генерации метаданных
Технически есть 2 метода генерации метаданных:
* Ленивый - с помощью подготовленных файликов, содержащих максимум метаинформации (либо с помощью БД) эта информация собирается в процессе подготовки пакета
* Полный - извлечение метаинформации непосредственно из готовых пакетов.
### Ленивый подход
Его применяет AlienBob в своём репозитории. В процессе сборки пакета генерируется несколько дополнительных файликов, из которых в последствии извлекается информация для создания метаданных репозитория. Возможен другой вариант - складывать метаданные о пакете в БД и в процессе генерации метаданных о репозитории извлекать информацию о каждом пакете уже из неё.
К плюсам этого подхода можно отнести прежде всего дешевизну его использования, так как по файликам с подготовленными данными пробегаться значительно проще и быстрее, чем распаковывать каждый пакет и из полученных данных строит метаданные для репозитория, это особенно заметно на больших пакетах.
К минусам этого подхода можно отнести его неуниверсальность и наличие в репозитории дополнительных артефактов, впрочем, последнее можно нивелировать, задействовав БД как хранилище промежуточной метаинформации.
### Полный подход
Область применения - затрудняюсь сказать. Однако официальные репозитории не содержат дополнительных артефактов.
Этот подход вполне подходит для небольших репозиториев, так как позволяет генерировать метаданные из любой кучки пакетов.
К минусам этого подхода очевидно имеет смысл приписать его дороговизну как по iops-ам, так и по процессору: на каждую перегенерацию метаданных придётся распаковывать весь набор пакетов в репозитории.

61
slackmetarep.psgi Normal file
View File

@@ -0,0 +1,61 @@
## no critic (Modules::RequireExplicitPackage)
use strict;
use warnings;
# my plugins
use lib qw(. ./vendor_perl ./vendor_perl/lib/perl5);
use Slackmetarep::Conf qw(loadConf);
use Slackmetarep::Upload qw(upload);
use Slackmetarep::Metagen qw(metagen);
use Slackmetarep::Buildinfo qw(buildinfo);
use version; our $VERSION = qv(1.0);
my $CONF = loadConf ();
if ($CONF->{api}->{prefix} eq '/') { $CONF->{api}->{prefix} = ''; }
my $prefix = $CONF->{api}->{prefix};
# that is not module.
my $app = sub { ## no critic (Modules::RequireEndWithOne)
my $env = shift;
my $msg = "Your Opinion is very important for us, please stand by.\n";
my $status = '404';
my $content = 'text/plain';
if ($env->{PATH_INFO} =~ /$prefix\/upload\/(.+)/xmsg) {
my $upload = $1;
($status, $content, $msg) = ('400', $content, "Bad Request?\n");
if (defined($env->{HTTP_AUTH}) && ($env->{HTTP_AUTH} eq $CONF->{upload}->{auth})) {
if (($upload !~ /\.\./xmsg) && ($upload =~ /^[_\-\+\/\.[:alnum:]]+$/xmsg)) {
if (defined($env->{CONTENT_LENGTH}) && ($env->{CONTENT_LENGTH} > 0)) {
($status, $content, $msg) = upload ($env->{'psgi.input'}, $env->{CONTENT_LENGTH}, $upload);
}
} else {
$msg = "Something wrong with upload path.\n";
}
} else {
($status, $content, $msg) = ('403', $content, "You're not allowed here. Fuck off.\n");
}
} elsif ($env->{PATH_INFO} eq "$prefix/metagen") {
if (defined($env->{HTTP_REPO})) {
($status, $content, $msg) = metagen ($env->{HTTP_REPO});
}
} elsif ($env->{PATH_INFO} eq "$prefix/buildinfo") {
if (defined($env->{HTTP_REPO})) {
($status, $content, $msg) = buildinfo ($env->{HTTP_REPO});
}
}
return [
$status,
[ 'Content-Type' => $content, 'Content-Length' => length $msg ],
[ $msg ],
];
};
__END__
# vim: set ft=perl noet ai ts=4 sw=4 sts=4:

17
slackmetarep.yaml Normal file
View File

@@ -0,0 +1,17 @@
uwsgi:
http-socket: 127.0.0.1:3082
tcp-nodelay: true
tcp-fastopen: 64
chdir: /var/www/slackmetarep
plugins: psgi
psgi: /var/www/slackmetarep/slackmetarep.psgi
processes: 1
enable-threads: false
thunder-lock: true
logformat: %(ctime) %(var.REMOTE_ADDR) %(var.REQUEST_METHOD) %(var.REQUEST_URI) %(status) %(rsize) %(msecs)
need-app: true
log-reopen: yes
reload-mercy: 5
buffer-size: 32768
logto: /var/log/uwsgi/slackmetarep.log
die-on-term: true