aboutsummaryrefslogtreecommitdiff
path: root/bin
diff options
context:
space:
mode:
Diffstat (limited to 'bin')
-rw-r--r--bin/README.txt6
-rwxr-xr-xbin/makepkgdb.pl235
-rwxr-xr-xbin/whatis2sqlite.pl83
3 files changed, 324 insertions, 0 deletions
diff --git a/bin/README.txt b/bin/README.txt
new file mode 100644
index 0000000..e0baa08
--- /dev/null
+++ b/bin/README.txt
@@ -0,0 +1,6 @@
+Scripts that generate the databases used by the bot plugins.
+
+These are in Perl, because they're pretty complex and I just started
+learning Python last week (and I've used perl for over 20 years).
+
+See the comments at the top of each script for instructions.
diff --git a/bin/makepkgdb.pl b/bin/makepkgdb.pl
new file mode 100755
index 0000000..31838d3
--- /dev/null
+++ b/bin/makepkgdb.pl
@@ -0,0 +1,235 @@
+#!/usr/bin/perl -w
+
+# create database for limnoria bot Slackpkg plugin.
+# reads PACKAGES.txt from the main (slackware64) and extra
+# trees. Also has to have a copy of /var/log/scripts, since
+# PACKAGES.txt doesn't have symlinks, and a copy of
+# /var/log/packages since PACKAGES.txt doesn't have file contents.
+
+# the output is a huge pile of SQL insert statements, like half
+# a million of them. best way to run this IMO:
+# $ makepkgdb.pl > db.sql
+# $ rm -f /path/to/dbfile
+# $ sqlite3 -bail /path/to/dbfile < db.sql
+
+# this script was only tested for slackware 14.2. When 15.0 gets
+# released, we'll rebuild the database using this script, which
+# might or might not need to be modified.
+
+# special cases: /lib64/incoming gets the /incoming removed, and
+# we manually add /bin/bash.
+
+### configurables:
+
+@pkglists = (
+ '/data/mirrors/slackware/slackware64-14.2/PACKAGES.TXT',
+ '/data/mirrors/slackware/slackware64-14.2/extra/PACKAGES.TXT',
+);
+
+# ...the above list could have included pasture and testing, but
+# for that to work, you'd also have to have all the packages from
+# pasture and testing installed (we don't right now).
+
+# these 2 are copies of /var/log/packages and /var/log/scripts from
+# a full install of 14.2, with no 3rd-party packages added:
+$pkgdir = "/home/urchlay/var.log/packages";
+$scriptsdir = "/home/urchlay/var.log/scripts";
+
+### end configurables.
+
+sub init_db {
+ #unlink("testdb.sqlite3");
+ #open P, "|sqlite3 testdb.sqlite3";
+ #select P;
+
+ print <<EOF;
+pragma journal_mode = memory;
+begin transaction;
+
+-- should include extra (and maybe pasture?)
+create table categories (
+ id integer primary key not null,
+ name text(10) not null
+);
+
+-- compressed and uncompressed size?
+create table packages (
+ id integer primary key not null,
+ name varchar not null,
+ descrip varchar not null,
+ category integer not null,
+ foreign key(category) references categories(id)
+);
+
+-- path should be something bigger than a varchar? (does it matter, for sqlite?)
+create table files (
+ id integer primary key not null,
+ path varchar not null,
+ symlink boolean not null default 0,
+ package integer not null,
+ foreign key(package) references packages(id),
+ check(symlink in (0,1))
+);
+
+EOF
+}
+
+
+# turn e.g. bash-completion-2.2-noarch-3 into bash-completion
+# (supports arbitrary number of - in the package name).
+sub getpkgname {
+ my $p = reverse shift;
+ (undef, undef, undef, $p) = split("-", $p, 4);
+ return reverse $p;
+}
+
+# return ref-to-array of symlinks extracted from a scripts/ file.
+sub getsymlinks {
+ my @links = ();
+ open my $f, "<$_[0]" or return \@links;
+ while(<$f>) {
+ next unless /^\( cd (\S+) ; ln -sf \S+ (\S+) \)/;
+ my $dir = $1;
+ my $file = $2;
+ #warn "link $dir/$file\n" if $_[0] =~ /bash-4/;
+ push @links, "$dir/$file";
+ }
+ close $f;
+ #warn "links: " . join(" ", @links) . "\n" if $_[0] =~ /bash-4/;
+ return \@links;
+}
+
+# return ref-to-array of files extracted from a packages/ file.
+sub getfiles {
+ my @files = ();
+ open my $f, "<$_[0]" or return \@files;
+ while(<$f>) {
+ last if /^FILE LIST:/;
+ }
+ while(<$f>) {
+ chomp;
+ next if m,^\./$,;
+ next if m,^install/,;
+ s,^lib64/incoming,lib64,;
+ push @files, $_;
+ }
+ close $f;
+ push @files, "bin/bash" if $_[0] =~ /\/bash-[0-9]/;
+ return \@files;
+}
+
+# given a package db entry as a multi-line string, extract the
+# short description from it.
+sub getdescrip {
+ my $pkgname = shift;
+ my $blob = shift;
+ if($blob =~ /^$pkgname:\s+(\S.*)$/m) {
+ my $line = $1;
+ if($line =~ /^$pkgname (?:- )?\((.+)\)/) {
+ $line = $1;
+ }
+ return $line;
+ }
+ return '(no description)';
+}
+
+# given a package db entry as a multi-line string, extract the
+# category from it.
+sub getcategory {
+ my $pkgname = shift;
+ my $blob = shift;
+ if($blob =~ /^PACKAGE LOCATION:\s+\.\/(?:(extra)|(?:slackware64\/(\w+)))/m) {
+ return $1 ? $1 : $2;
+ }
+ return "(unknown category)";
+}
+
+sub getfilelist {
+ my $pkgname = shift;
+ return @{$files{$pkgname}};
+}
+
+sub getsymlinklist {
+ my $pkgname = shift;
+ if(defined $symlinks{$pkgname}) {
+ return @{$symlinks{$pkgname}};
+ } else {
+ return ();
+ }
+}
+
+$catcount = 0;
+sub getcategory_id {
+ my $catname = shift;
+ my $cat_id = $catname2id{$catname};
+ if(defined $cat_id) {
+ return $cat_id;
+ }
+ $catcount++;
+ print "insert into categories values($catcount, '$catname');\n";
+ $catname2id{$catname} = $catcount;
+ return $catcount;
+}
+
+### main()
+
+# build hash of all symlinks, indexed by package name.
+warn "reading symlinks from $scriptsdir/*\n";
+while(<$scriptsdir/*>) {
+ s,.*/,,;
+ my $p = getpkgname($_);
+ $symlinks{$p} = getsymlinks($scriptsdir . "/" . $_);
+}
+
+warn "reading filelists from $pkgdir/*\n";
+while(<$pkgdir/*>) {
+ s,.*/,,;
+ my $p = getpkgname($_);
+ $files{$p} = getfiles($pkgdir . "/" . $_);
+}
+
+#print for getfilelist('bash');
+#die;
+
+init_db();
+
+$pkg_id = 0;
+for(@pkglists) {
+ local $/ = '';
+ open my $l, "<$_" or die $!;
+ warn "reading package list from $_\n";
+ while(<$l>) {
+ next unless /^PACKAGE NAME:\s+(\S+)/;
+ my $pkgname = getpkgname($1);
+ my $descrip = getdescrip($pkgname, $_);
+ my $category = getcategory($pkgname, $_);
+ my $cat_id = getcategory_id($category);
+ $pkg_id++;
+
+ $descrip =~ s,','',g;
+ print "insert into packages values ($pkg_id, '$pkgname', '$descrip', $cat_id);\n";
+
+ my @filelist = getfilelist($pkgname);
+ my @symlinklist = getsymlinklist($pkgname);
+
+ for my $f (@filelist) {
+ print "insert into files values (null, '/$f', 0, $pkg_id);\n"
+ }
+
+ for my $f (@symlinklist) {
+ print "insert into files values (null, '/$f', 1, $pkg_id);\n"
+ }
+
+ #print "=== $cat_id: $category/$pkgname - $descrip\n";
+ #print "files: " . join("\n ", @filelist) . "\n";
+ #print "links: " . join("\n ", @symlinklist) . "\n\n";
+
+ }
+ close $l;
+}
+
+print "commit;\n";
+
+exit 0;
+
+
diff --git a/bin/whatis2sqlite.pl b/bin/whatis2sqlite.pl
new file mode 100755
index 0000000..297a82a
--- /dev/null
+++ b/bin/whatis2sqlite.pl
@@ -0,0 +1,83 @@
+#!/usr/bin/perl -w
+
+# whatis2sqlite.pl - create sqlite3 whatis database for use with
+# limnoria Manpages plugin.
+
+# Usage:
+
+# $ rm -f /path/to/Manpages.sqlite3
+# $ perl whatis2sqlite.pl [whatis-file] | sqlite /path/to/Manpages.sqlite3
+
+# then reload the Manpages plugin in the bot.
+
+# For Slackware 14.2, the whatis-file comes from /usr/man/whatis.
+# For 15.0, we'll have to generate it according to the directions
+# in "man whatis":
+# $ whatis -w '*' | sort > whatis
+# Note that man-db's databases will have to already exist, for that to work.
+
+push @ARGV, 'whatis' unless @ARGV;
+
+print <<EOF;
+pragma journal_mode = memory;
+
+begin transaction;
+
+create table whatis (
+ id integer primary key not null,
+ page varchar not null,
+ section char(5) not null,
+ desc varchar not null
+);
+EOF
+
+while(<>) {
+ my($name, $desc, $sect, $alias);
+ chomp;
+
+ # 14.2's whatis has some garbage entries, skip them.
+ next if /^struct/;
+ next if /^and (put|with)/;
+ next if /bernd\.warken/;
+
+ s/\s+$//g;
+ ($name, $desc) = split /\s+-\s+/, $_, 2;
+ $name =~ s/\s+/ /g;
+ ($sect) = $name =~ /\(([^)]+)\)/;
+ next if $sect eq '3p'; # symlink
+
+ $alias = 0;
+ if($name =~ /\[([^]]+)\]/) {
+ $alias = $1;
+ }
+
+ $name =~ s,\s.*,,;
+ $sect =~ s/^(.).*$/$1/; # no "3x", etc.
+
+ # 14.2's whatis has some wrong sections, fix them.
+ $sect = '8' if $sect eq 'v'; # ebtables
+ $sect = '1' if $sect eq 'P'; # cdparanoia
+ $sect = '1' if $sect eq 'o'; # rclock, rxvt
+
+ #print "$sect, $name, '$alias' => $desc\n";
+
+ make_sql($name, $sect, $desc);
+ make_sql($alias, $sect, $desc) if $alias;
+}
+
+print "COMMIT;\n";
+
+sub make_sql {
+ my $page = shift;
+ my $sect = shift;
+ my $desc = shift;
+
+ return if $seen{"$page^^$sect"}++;
+
+ # N.B. we don't escape quotes here because 14.2's whatis
+ # doesn't contain any double-quotes. When 15 is released,
+ # better check again!
+ print <<EOF
+insert into whatis values (null, "$page", "$sect", "$desc");
+EOF
+}