From 15128b94b496324a1c0afd6cbdd7fffbdaa18ba7 Mon Sep 17 00:00:00 2001 From: "B. Watson" Date: Thu, 17 Jun 2021 01:58:33 -0400 Subject: Initial commit. --- bin/README.txt | 6 ++ bin/makepkgdb.pl | 235 +++++++++++++++++++++++++++++++++++++++++++++++++++ bin/whatis2sqlite.pl | 83 ++++++++++++++++++ 3 files changed, 324 insertions(+) create mode 100644 bin/README.txt create mode 100755 bin/makepkgdb.pl create mode 100755 bin/whatis2sqlite.pl (limited to 'bin') diff --git a/bin/README.txt b/bin/README.txt new file mode 100644 index 0000000..e0baa08 --- /dev/null +++ b/bin/README.txt @@ -0,0 +1,6 @@ +Scripts that generate the databases used by the bot plugins. + +These are in Perl, because they're pretty complex and I just started +learning Python last week (and I've used perl for over 20 years). + +See the comments at the top of each script for instructions. diff --git a/bin/makepkgdb.pl b/bin/makepkgdb.pl new file mode 100755 index 0000000..31838d3 --- /dev/null +++ b/bin/makepkgdb.pl @@ -0,0 +1,235 @@ +#!/usr/bin/perl -w + +# create database for limnoria bot Slackpkg plugin. +# reads PACKAGES.txt from the main (slackware64) and extra +# trees. Also has to have a copy of /var/log/scripts, since +# PACKAGES.txt doesn't have symlinks, and a copy of +# /var/log/packages since PACKAGES.txt doesn't have file contents. + +# the output is a huge pile of SQL insert statements, like half +# a million of them. best way to run this IMO: +# $ makepkgdb.pl > db.sql +# $ rm -f /path/to/dbfile +# $ sqlite3 -bail /path/to/dbfile < db.sql + +# this script was only tested for slackware 14.2. When 15.0 gets +# released, we'll rebuild the database using this script, which +# might or might not need to be modified. + +# special cases: /lib64/incoming gets the /incoming removed, and +# we manually add /bin/bash. + +### configurables: + +@pkglists = ( + '/data/mirrors/slackware/slackware64-14.2/PACKAGES.TXT', + '/data/mirrors/slackware/slackware64-14.2/extra/PACKAGES.TXT', +); + +# ...the above list could have included pasture and testing, but +# for that to work, you'd also have to have all the packages from +# pasture and testing installed (we don't right now). + +# these 2 are copies of /var/log/packages and /var/log/scripts from +# a full install of 14.2, with no 3rd-party packages added: +$pkgdir = "/home/urchlay/var.log/packages"; +$scriptsdir = "/home/urchlay/var.log/scripts"; + +### end configurables. + +sub init_db { + #unlink("testdb.sqlite3"); + #open P, "|sqlite3 testdb.sqlite3"; + #select P; + + print <) { + next unless /^\( cd (\S+) ; ln -sf \S+ (\S+) \)/; + my $dir = $1; + my $file = $2; + #warn "link $dir/$file\n" if $_[0] =~ /bash-4/; + push @links, "$dir/$file"; + } + close $f; + #warn "links: " . join(" ", @links) . "\n" if $_[0] =~ /bash-4/; + return \@links; +} + +# return ref-to-array of files extracted from a packages/ file. +sub getfiles { + my @files = (); + open my $f, "<$_[0]" or return \@files; + while(<$f>) { + last if /^FILE LIST:/; + } + while(<$f>) { + chomp; + next if m,^\./$,; + next if m,^install/,; + s,^lib64/incoming,lib64,; + push @files, $_; + } + close $f; + push @files, "bin/bash" if $_[0] =~ /\/bash-[0-9]/; + return \@files; +} + +# given a package db entry as a multi-line string, extract the +# short description from it. +sub getdescrip { + my $pkgname = shift; + my $blob = shift; + if($blob =~ /^$pkgname:\s+(\S.*)$/m) { + my $line = $1; + if($line =~ /^$pkgname (?:- )?\((.+)\)/) { + $line = $1; + } + return $line; + } + return '(no description)'; +} + +# given a package db entry as a multi-line string, extract the +# category from it. +sub getcategory { + my $pkgname = shift; + my $blob = shift; + if($blob =~ /^PACKAGE LOCATION:\s+\.\/(?:(extra)|(?:slackware64\/(\w+)))/m) { + return $1 ? $1 : $2; + } + return "(unknown category)"; +} + +sub getfilelist { + my $pkgname = shift; + return @{$files{$pkgname}}; +} + +sub getsymlinklist { + my $pkgname = shift; + if(defined $symlinks{$pkgname}) { + return @{$symlinks{$pkgname}}; + } else { + return (); + } +} + +$catcount = 0; +sub getcategory_id { + my $catname = shift; + my $cat_id = $catname2id{$catname}; + if(defined $cat_id) { + return $cat_id; + } + $catcount++; + print "insert into categories values($catcount, '$catname');\n"; + $catname2id{$catname} = $catcount; + return $catcount; +} + +### main() + +# build hash of all symlinks, indexed by package name. +warn "reading symlinks from $scriptsdir/*\n"; +while(<$scriptsdir/*>) { + s,.*/,,; + my $p = getpkgname($_); + $symlinks{$p} = getsymlinks($scriptsdir . "/" . $_); +} + +warn "reading filelists from $pkgdir/*\n"; +while(<$pkgdir/*>) { + s,.*/,,; + my $p = getpkgname($_); + $files{$p} = getfiles($pkgdir . "/" . $_); +} + +#print for getfilelist('bash'); +#die; + +init_db(); + +$pkg_id = 0; +for(@pkglists) { + local $/ = ''; + open my $l, "<$_" or die $!; + warn "reading package list from $_\n"; + while(<$l>) { + next unless /^PACKAGE NAME:\s+(\S+)/; + my $pkgname = getpkgname($1); + my $descrip = getdescrip($pkgname, $_); + my $category = getcategory($pkgname, $_); + my $cat_id = getcategory_id($category); + $pkg_id++; + + $descrip =~ s,','',g; + print "insert into packages values ($pkg_id, '$pkgname', '$descrip', $cat_id);\n"; + + my @filelist = getfilelist($pkgname); + my @symlinklist = getsymlinklist($pkgname); + + for my $f (@filelist) { + print "insert into files values (null, '/$f', 0, $pkg_id);\n" + } + + for my $f (@symlinklist) { + print "insert into files values (null, '/$f', 1, $pkg_id);\n" + } + + #print "=== $cat_id: $category/$pkgname - $descrip\n"; + #print "files: " . join("\n ", @filelist) . "\n"; + #print "links: " . join("\n ", @symlinklist) . "\n\n"; + + } + close $l; +} + +print "commit;\n"; + +exit 0; + + diff --git a/bin/whatis2sqlite.pl b/bin/whatis2sqlite.pl new file mode 100755 index 0000000..297a82a --- /dev/null +++ b/bin/whatis2sqlite.pl @@ -0,0 +1,83 @@ +#!/usr/bin/perl -w + +# whatis2sqlite.pl - create sqlite3 whatis database for use with +# limnoria Manpages plugin. + +# Usage: + +# $ rm -f /path/to/Manpages.sqlite3 +# $ perl whatis2sqlite.pl [whatis-file] | sqlite /path/to/Manpages.sqlite3 + +# then reload the Manpages plugin in the bot. + +# For Slackware 14.2, the whatis-file comes from /usr/man/whatis. +# For 15.0, we'll have to generate it according to the directions +# in "man whatis": +# $ whatis -w '*' | sort > whatis +# Note that man-db's databases will have to already exist, for that to work. + +push @ARGV, 'whatis' unless @ARGV; + +print <) { + my($name, $desc, $sect, $alias); + chomp; + + # 14.2's whatis has some garbage entries, skip them. + next if /^struct/; + next if /^and (put|with)/; + next if /bernd\.warken/; + + s/\s+$//g; + ($name, $desc) = split /\s+-\s+/, $_, 2; + $name =~ s/\s+/ /g; + ($sect) = $name =~ /\(([^)]+)\)/; + next if $sect eq '3p'; # symlink + + $alias = 0; + if($name =~ /\[([^]]+)\]/) { + $alias = $1; + } + + $name =~ s,\s.*,,; + $sect =~ s/^(.).*$/$1/; # no "3x", etc. + + # 14.2's whatis has some wrong sections, fix them. + $sect = '8' if $sect eq 'v'; # ebtables + $sect = '1' if $sect eq 'P'; # cdparanoia + $sect = '1' if $sect eq 'o'; # rclock, rxvt + + #print "$sect, $name, '$alias' => $desc\n"; + + make_sql($name, $sect, $desc); + make_sql($alias, $sect, $desc) if $alias; +} + +print "COMMIT;\n"; + +sub make_sql { + my $page = shift; + my $sect = shift; + my $desc = shift; + + return if $seen{"$page^^$sect"}++; + + # N.B. we don't escape quotes here because 14.2's whatis + # doesn't contain any double-quotes. When 15 is released, + # better check again! + print <