From 15128b94b496324a1c0afd6cbdd7fffbdaa18ba7 Mon Sep 17 00:00:00 2001 From: "B. Watson" Date: Thu, 17 Jun 2021 01:58:33 -0400 Subject: Initial commit. --- Manpages/README.md | 1 + Manpages/__init__.py | 48 +++++++++ Manpages/config.py | 36 +++++++ Manpages/local/__init__.py | 1 + Manpages/plugin.py | 99 ++++++++++++++++++ Manpages/plugin.py.old | 98 ++++++++++++++++++ Manpages/test.py | 15 +++ README.txt | 1 + SlackTools/README.md | 1 + SlackTools/__init__.py | 48 +++++++++ SlackTools/config.py | 36 +++++++ SlackTools/local/__init__.py | 1 + SlackTools/plugin.py | 202 +++++++++++++++++++++++++++++++++++++ SlackTools/plugin.py.old | 94 +++++++++++++++++ SlackTools/test.py | 15 +++ bin/README.txt | 6 ++ bin/makepkgdb.pl | 235 +++++++++++++++++++++++++++++++++++++++++++ bin/whatis2sqlite.pl | 83 +++++++++++++++ 18 files changed, 1020 insertions(+) create mode 100644 Manpages/README.md create mode 100644 Manpages/__init__.py create mode 100644 Manpages/config.py create mode 100644 Manpages/local/__init__.py create mode 100644 Manpages/plugin.py create mode 100644 Manpages/plugin.py.old create mode 100644 Manpages/test.py create mode 100644 README.txt create mode 100644 SlackTools/README.md create mode 100644 SlackTools/__init__.py create mode 100644 SlackTools/config.py create mode 100644 SlackTools/local/__init__.py create mode 100644 SlackTools/plugin.py create mode 100644 SlackTools/plugin.py.old create mode 100644 SlackTools/test.py create mode 100644 bin/README.txt create mode 100755 bin/makepkgdb.pl create mode 100755 bin/whatis2sqlite.pl diff --git a/Manpages/README.md b/Manpages/README.md new file mode 100644 index 0000000..b12ae49 --- /dev/null +++ b/Manpages/README.md @@ -0,0 +1 @@ +Provides Unix manpage lookups diff --git a/Manpages/__init__.py b/Manpages/__init__.py new file mode 100644 index 0000000..c726c08 --- /dev/null +++ b/Manpages/__init__.py @@ -0,0 +1,48 @@ +### +# Copyright (c) 2021, B. Watson +# All rights reserved. +# +# +### + +""" +Manpages: Provides Unix manpage lookups +""" + +import sys +import supybot +from supybot import world + +# Use this for the version of this plugin. +__version__ = "0.0.1" + +# XXX Replace this with an appropriate author or supybot.Author instance. +__author__ = supybot.Author('B. Watson', 'Urchlay', 'yalhcru@gmail.com') + +# This is a dictionary mapping supybot.Author instances to lists of +# contributions. +__contributors__ = {} + +# This is a url where the most recent plugin package can be downloaded. +__url__ = '' + +from . import config +from . import plugin +if sys.version_info >= (3, 4): + from importlib import reload +else: + from imp import reload +# In case we're being reloaded. +reload(config) +reload(plugin) +# Add more reloads here if you add third-party modules and want them to be +# reloaded when this plugin is reloaded. Don't forget to import them as well! + +if world.testing: + from . import test + +Class = plugin.Class +configure = config.configure + + +# vim:set shiftwidth=4 tabstop=4 expandtab textwidth=79: diff --git a/Manpages/config.py b/Manpages/config.py new file mode 100644 index 0000000..b0ab3ff --- /dev/null +++ b/Manpages/config.py @@ -0,0 +1,36 @@ +### +# Copyright (c) 2021, B. Watson +# All rights reserved. +# +# +### + +from supybot import conf, registry +try: + from supybot.i18n import PluginInternationalization + _ = PluginInternationalization('Manpages') +except: + # Placeholder that allows to run the plugin on a bot + # without the i18n module + _ = lambda x: x + + +def configure(advanced): + # This will be called by supybot to configure this module. advanced is + # a bool that specifies whether the user identified themself as an advanced + # user or not. You should effect your configuration by manipulating the + # registry as appropriate. + from supybot.questions import expect, anything, something, yn + conf.registerPlugin('Manpages', True) + + +Manpages = conf.registerPlugin('Manpages') +# This is where your configuration variables (if any) should go. For example: +# conf.registerGlobalValue(Manpages, 'someConfigVariableName', +# registry.Boolean(False, _("""Help for someConfigVariableName."""))) + +conf.registerGlobalValue(Manpages, 'dbpath', + registry.String("/home/slackfacts/supybot/Manpages.sqlite3", _("""Path to sqite3 database."""))) + +conf.registerGlobalValue(Manpages, 'maxresults', + registry.Integer(5, _("""Maximum number of results to send to client."""))) diff --git a/Manpages/local/__init__.py b/Manpages/local/__init__.py new file mode 100644 index 0000000..e86e97b --- /dev/null +++ b/Manpages/local/__init__.py @@ -0,0 +1 @@ +# Stub so local is a module, used for third-party modules diff --git a/Manpages/plugin.py b/Manpages/plugin.py new file mode 100644 index 0000000..e3913d1 --- /dev/null +++ b/Manpages/plugin.py @@ -0,0 +1,99 @@ +### +# Copyright (c) 2021, B. Watson +# All rights reserved. +# +# +### + +import os +import subprocess +import sys +import re +import sqlite3 + +from supybot import utils, plugins, ircutils, callbacks +from supybot.commands import * +import supybot.utils.minisix as minisix + +try: + from supybot.i18n import PluginInternationalization + _ = PluginInternationalization('Manpages') +except ImportError: + # Placeholder that allows to run the plugin on a bot + # without the i18n module + _ = lambda x: x + + +class Manpages(callbacks.Plugin): + """Provides Unix manpage lookups""" + threaded = True + + db = None + + def InitDB(self): + if self.db is None: + filename = self.registryValue('dbpath') + if(os.path.exists(filename)): + self.db = sqlite3.connect(filename) + return self.db + + # man() is a *very* limited subset of what the real + # man command does. Should maybe be called whatis instead, + # use the Aka plugin to create an alias: + # load Aka + # aka add whatis man $* + def man(self, irc, msg, args, sect_or_page, page): + """ [
|-k] + + Look up man pages and show their summaries. Without -k, searches + man pages by name (same as the OS "whatis" or "man -f" commands). + With -k, also searches descriptions (same as OS "man -k" or "apropos"). + With
, limits search to a single man section (otherwise all + sections are searched). + """ + + db = self.InitDB(); + if db is None: + irc.error("manpage database doesn't exist", Raise=True) + + maxresults = self.registryValue('maxresults') + if msg.channel is None: + # private message, increase limit + maxresults *= 5 + + cursor = db.cursor() + + errmsg = "No manual entry for " + + selectfrom = "select page, section, desc from whatis " + orderby = " order by section " + limit = " limit " + str(maxresults + 1) + " " + + if page is None: + cursor.execute(selectfrom + "where page glob ?" + orderby + limit, (sect_or_page,)) + errmsg = errmsg + sect_or_page + else: + if sect_or_page == "-k": + glob = "*" + page + "*" + cursor.execute(selectfrom + "where page glob ? or desc glob ? " + orderby + limit, (glob, glob)) + errmsg = "No man pages matching " + page + else: + cursor.execute(selectfrom + "where section=? and page glob ? " + orderby + limit, (sect_or_page, page)) + errmsg = errmsg + page + " in section " + sect_or_page + + result = cursor.fetchall() + + lines = [] + + if len(result) == 0: + irc.reply(errmsg) + else: + for (page, section, desc) in result: + lines.append(ircutils.bold(page + "(" + section + ")") + ": " + desc) + if(len(result) > maxresults): + lines.append("[too many results, only showing first " + str(maxresults) + "]") + irc.replies(lines, joiner=' | ') + + man = thread(wrap(man, ['somethingWithoutSpaces', optional('somethingWithoutSpaces')])) + +Class = Manpages diff --git a/Manpages/plugin.py.old b/Manpages/plugin.py.old new file mode 100644 index 0000000..b60b4dd --- /dev/null +++ b/Manpages/plugin.py.old @@ -0,0 +1,98 @@ +### +# Copyright (c) 2021, B. Watson +# All rights reserved. +# +# +### + +import os +import subprocess +import sys +import re + +from supybot import utils, plugins, ircutils, callbacks +from supybot.commands import * +import supybot.utils.minisix as minisix + +try: + from supybot.i18n import PluginInternationalization + _ = PluginInternationalization('Manpages') +except ImportError: + # Placeholder that allows to run the plugin on a bot + # without the i18n module + _ = lambda x: x + + +# The regex stuff is to turn this: +# ls (1) - list directory contents +# into this: +# ls(1) - list directory contents + +class Manpages(callbacks.Plugin): + """Provides Unix manpage and whereis lookups""" + threaded = True + + re_sub_spaces = re.compile(r"\s\s+") + re_sub_paren = re.compile(r"\s\(") + + def Command(self, irc, msg, args): + try: + with open(os.devnull) as null: + child = subprocess.Popen(args, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + stdin=null) + except OSError as e: + irc.error('Man page lookup failed.', Raise=True) + (out, err) = child.communicate() + child.wait() + out = out + err + if minisix.PY3: + lines = [i.decode('utf-8').rstrip() for i in out.splitlines()] + lines = list(map(str, lines)) + else: + lines = out.splitlines() + lines = list(map(str.rstrip, lines)) + lines = filter(None, lines) + return lines + + # man() is a *very* limited subset of what the real + # man command does. Should maybe be called whatis instead, + # use the Aka plugin to create an alias: + # load Aka + # aka add whatis man $* + def man(self, irc, msg, args, sect_or_page, page): + """ Usage: man
, or man for all sections """ + + cmd = [ '/usr/bin/man', '-f' ] + + try: + sect = int(sect_or_page) + if page is None: + irc.error("Missing ", Raise=True) + else: + cmd.append("-s") + cmd.append(sect_or_page) + cmd.append(page) + except ValueError: + if page is None: + cmd.append(sect_or_page) + else: + irc.error("Invalid
", Raise=True) + + lines = self.Command(irc, msg, cmd) + + lines = [self.re_sub_spaces.sub(' ', i) for i in lines] + lines = [self.re_sub_paren.sub('(', i) for i in lines] + irc.replies(lines, joiner=' | ') + + man = thread(wrap(man, ['somethingWithoutSpaces', optional('somethingWithoutSpaces')])) + + # whereis() is a limited subset of the real whereis. + def whereis(self, irc, msg, args, name): + """ """ + lines = self.Command(irc, msg, [ '/usr/bin/whereis', name ]) + irc.replies(lines, joiner=' | ') + whereis = thread(wrap(whereis, ['somethingWithoutSpaces'])) + +Class = Manpages diff --git a/Manpages/test.py b/Manpages/test.py new file mode 100644 index 0000000..53e803a --- /dev/null +++ b/Manpages/test.py @@ -0,0 +1,15 @@ +### +# Copyright (c) 2021, B. Watson +# All rights reserved. +# +# +### + +from supybot.test import * + + +class ManpagesTestCase(PluginTestCase): + plugins = ('Manpages',) + + +# vim:set shiftwidth=4 tabstop=4 expandtab textwidth=79: diff --git a/README.txt b/README.txt new file mode 100644 index 0000000..b1b1557 --- /dev/null +++ b/README.txt @@ -0,0 +1 @@ +slackfacts bot plugins diff --git a/SlackTools/README.md b/SlackTools/README.md new file mode 100644 index 0000000..a694e68 --- /dev/null +++ b/SlackTools/README.md @@ -0,0 +1 @@ +Provides Slackware package database lookups. diff --git a/SlackTools/__init__.py b/SlackTools/__init__.py new file mode 100644 index 0000000..fe66e85 --- /dev/null +++ b/SlackTools/__init__.py @@ -0,0 +1,48 @@ +### +# Copyright (c) 2021, B. Watson +# All rights reserved. +# +# +### + +""" +SlackTools: Provides Unix manpage lookups +""" + +import sys +import supybot +from supybot import world + +# Use this for the version of this plugin. +__version__ = "0.0.1" + +# XXX Replace this with an appropriate author or supybot.Author instance. +__author__ = supybot.Author('B. Watson', 'Urchlay', 'yalhcru@gmail.com') + +# This is a dictionary mapping supybot.Author instances to lists of +# contributions. +__contributors__ = {} + +# This is a url where the most recent plugin package can be downloaded. +__url__ = '' + +from . import config +from . import plugin +if sys.version_info >= (3, 4): + from importlib import reload +else: + from imp import reload +# In case we're being reloaded. +reload(config) +reload(plugin) +# Add more reloads here if you add third-party modules and want them to be +# reloaded when this plugin is reloaded. Don't forget to import them as well! + +if world.testing: + from . import test + +Class = plugin.Class +configure = config.configure + + +# vim:set shiftwidth=4 tabstop=4 expandtab textwidth=79: diff --git a/SlackTools/config.py b/SlackTools/config.py new file mode 100644 index 0000000..f660d44 --- /dev/null +++ b/SlackTools/config.py @@ -0,0 +1,36 @@ +### +# Copyright (c) 2021, B. Watson +# All rights reserved. +# +# +### + +from supybot import conf, registry +try: + from supybot.i18n import PluginInternationalization + _ = PluginInternationalization('SlackTools') +except: + # Placeholder that allows to run the plugin on a bot + # without the i18n module + _ = lambda x: x + + +def configure(advanced): + # This will be called by supybot to configure this module. advanced is + # a bool that specifies whether the user identified themself as an advanced + # user or not. You should effect your configuration by manipulating the + # registry as appropriate. + from supybot.questions import expect, anything, something, yn + conf.registerPlugin('SlackTools', True) + + +SlackTools = conf.registerPlugin('SlackTools') +# This is where your configuration variables (if any) should go. For example: +# conf.registerGlobalValue(SlackTools, 'someConfigVariableName', +# registry.Boolean(False, _("""Help for someConfigVariableName."""))) + +conf.registerGlobalValue(SlackTools, 'dbpath', + registry.String("/home/slackfacts/supybot/SlackTools.sqlite3", _("""Path to sqite3 database."""))) + +conf.registerGlobalValue(SlackTools, 'maxresults', + registry.Integer(5, _("""Maximum number of results to send to client."""))) diff --git a/SlackTools/local/__init__.py b/SlackTools/local/__init__.py new file mode 100644 index 0000000..e86e97b --- /dev/null +++ b/SlackTools/local/__init__.py @@ -0,0 +1 @@ +# Stub so local is a module, used for third-party modules diff --git a/SlackTools/plugin.py b/SlackTools/plugin.py new file mode 100644 index 0000000..e4cc8bb --- /dev/null +++ b/SlackTools/plugin.py @@ -0,0 +1,202 @@ +### +# Copyright (c) 2021, B. Watson +# All rights reserved. +# +# +### + +import os +import subprocess +import sys +import re +import sqlite3 + +from supybot import utils, plugins, ircutils, callbacks +from supybot.commands import * +import supybot.utils.minisix as minisix + +try: + from supybot.i18n import PluginInternationalization + _ = PluginInternationalization('SlackTools') +except ImportError: + # Placeholder that allows to run the plugin on a bot + # without the i18n module + _ = lambda x: x + +# This code could stand to be refactored. Lots of almost-identical +# code in FileQuery and PkgQuery. Also, I'm pretty much a python +# n00b, so the code might read a little weird if you're a python +# expert (I learned just enough of the language, on the fly, to +# get this stuff written). That's also why the database-creation +# script is in perl (I'm much more familiar and competent with perl). + +# notes about the database: + +# it gets created by the makepkgdb.pl script, which requires: +# - PACKAGES.txt and extra/PACKAGES.txt from the installer. +# - copies of /var/log/scripts and /var/log/packages from a full +# slackware install (including /extra but not /testing nor +# /pasture) with no 3rd-party packages installed. +# see the comments in makepkgdb.pl for more info. + +# The database only contains packages from one slackware version +# and arch. Currently that's x86_64 14.2. Only packages from the +# main slackware64/ and extra/ trees are in the db. If there's any +# demand for it, it wouldn't be hard to support multiple slack +# versions and/or arches... or add pasture/ and/or testing/ to the db. + +# It would have been easy to include package sizes in the db, but +# I didn't think it would be that useful. + +# The files table also includes symlinks. + +# /lib64/incoming/ paths get their /incoming removed before being +# inserted in the db. + +# file paths are stored *with* the leading slash, so e.g. /bin/ls +# rather than the bin/ls. This is unlike usual slackpkg and +# /var/log/packages usage, but it seems more intuitive for users. + + +class SlackTools(callbacks.Plugin): + """Provides Slackware package db lookups""" + threaded = True + + db = None + + def getMaxResults(self, msg): + maxresults = self.registryValue('maxresults') + if msg.channel is None: + # private message, increase limit + maxresults *= 5 + return maxresults + + def InitDB(self): + if self.db is None: + filename = self.registryValue('dbpath') + if(os.path.exists(filename)): + self.db = sqlite3.connect(filename) + return self.db + + def PkgQuery(self, irc, msg, searchtype, term): + db = self.InitDB(); + if db is None: + irc.error("slackpkg database doesn't exist", Raise=True) + + maxresults = self.getMaxResults(msg) + cursor = db.cursor() + if(searchtype == 'pkg'): + cursor.execute("select c.name, p.name, p.descrip from categories c, packages p where c.id=p.category and p.name glob ? order by c.name, p.name limit ?", (term, maxresults+1)) + else: + cursor.execute("select distinct c.name, p.name, p.descrip from categories c, packages p, files f where c.id=p.category and p.id=f.package and f.path glob ? order by c.name, p.name limit ?", (term, maxresults+1)) + + result = cursor.fetchall() + + lines = [] + + if len(result) == 0: + irc.reply("no matching packages") + else: + for (category, pkg, descrip) in result: + lines.append(ircutils.bold(category + "/" + pkg) + ": " + descrip) + if(len(result) > maxresults): + lines.append("[too many results, only showing first " + str(maxresults) + "]") + irc.replies(lines, joiner=' | ') + + + # search for packages by name + def pkgsearch(self, irc, msg, args, pkg): + """ + + Search the Slackware package database for packages named like . + This is a case-sensitive substring match (e.g. "core" matches "coreutils" + and "xapian-core"). You can use * for globbing, e.g. "c*s" to find all + packages whose names begin with "c" and end with "s". + """ + + self.PkgQuery(irc, msg, 'pkg', "*" + pkg + "*") + + pkgsearch = thread(wrap(pkgsearch, ['somethingWithoutSpaces'])) + + # search for packages by contents + def filesearch(self, irc, msg, args, file): + """ + + Search the Slackware package database for packages that contain files + named . This is a case-sensitive exact match, by default (e.g. + "/bin/ls" does not match "/usr/bin/ls" nor "/bin/lsattr"). You can use * + for globbing, e.g. "/bin/ls*" to match all files in /bin whose names begin + with "ls". + """ + + self.PkgQuery(irc, msg, 'file', file) + + filesearch = thread(wrap(filesearch, ['somethingWithoutSpaces'])) + + # run filesearch or pkgsearch, save some typing. + def pkg(self, irc, msg, args, fileflag, term): + """ [-f] + + Without -f, same as pkgsearch. With -f, same as filesearch. + See the help for pkgsearch and/or filesearch for details. + """ + + if(fileflag): + self.PkgQuery(irc, msg, 'file', term) + else: + self.PkgQuery(irc, msg, 'pkg', "*" + term + "*") + + pkg = thread(wrap(pkg, [optional(('literal', ['-f'])), 'somethingWithoutSpaces'])) + + def FileQuery(self, irc, msg, term): + db = self.InitDB(); + if db is None: + irc.error("slackpkg database doesn't exist", Raise=True) + + maxresults = self.getMaxResults(msg) + cursor = db.cursor() + cursor.execute("select path, symlink from files where path glob ? order by path limit ?", (term, maxresults+1)) + + result = cursor.fetchall() + + lines = [] + + if len(result) == 0: + irc.reply("nothing found") + else: + for (file,symlink) in result: + if(symlink): + file = file + " [l]" + lines.append(file) + if(len(result) > maxresults): + lines.append("[too many results, only showing first " + str(maxresults) + "]") + irc.replies(lines, joiner=' | ') + + # subset of locate/slocate/mlocate. + def locate(self, irc, msg, args, file): + """ + + Search the package database for files named like . + This is a case-sensitive substring match (basically works like the + actual 'locate' command). You can use * for globbing. + """ + + self.FileQuery(irc, msg, '*' + file + '*') + + locate = thread(wrap(locate, ['somethingWithoutSpaces'])) + + # faux 'which' command. + def which(self, irc, msg, args, file): + """ + + Search the package database for files named like */bin/. + This is similar to the OS "which" command, except it doesn't search + a $PATH, it doesn't know whether the file has +x permission, it can + return multiple results, and you can use * for globbing. + """ + + self.FileQuery(irc, msg, '*/bin/' + file) + + which = thread(wrap(which, ['somethingWithoutSpaces'])) + +Class = SlackTools diff --git a/SlackTools/plugin.py.old b/SlackTools/plugin.py.old new file mode 100644 index 0000000..f50a55d --- /dev/null +++ b/SlackTools/plugin.py.old @@ -0,0 +1,94 @@ +### +# Copyright (c) 2021, B. Watson +# All rights reserved. +# +# +### + +import os +import subprocess +import sys +import re +import sqlite3 + +from supybot import utils, plugins, ircutils, callbacks +from supybot.commands import * +import supybot.utils.minisix as minisix + +try: + from supybot.i18n import PluginInternationalization + _ = PluginInternationalization('Slackpkg') +except ImportError: + # Placeholder that allows to run the plugin on a bot + # without the i18n module + _ = lambda x: x + + +class Slackpkg(callbacks.Plugin): + """Provides Slackware package db lookups""" + threaded = True + + db = None + + def InitDB(self): + if self.db is None: + filename = self.registryValue('dbpath') + if(os.path.exists(filename)): + self.db = sqlite3.connect(filename) + return self.db + + # search for packages by name + def pkgsearch(self, irc, msg, args, pkg): + """ Usage: pkg (wildcards allowed)""" + + db = self.InitDB(); + if db is None: + irc.error("slackpkg database doesn't exist", Raise=True) + + maxresults = self.registryValue('maxresults') + cursor = db.cursor() + cursor.execute("select c.name, p.name, p.descrip from categories c, packages p where c.id=p.category and p.name glob ? order by c.name, p.name limit ?", (pkg, maxresults+1)) + result = cursor.fetchall() + + lines = [] + + if len(result) == 0: + irc.reply("no matching packages") + else: + for (category, pkg, descrip) in result: + lines.append(format('%s/%s - %s', category, pkg, descrip)) + if(len(result) > maxresults): + lines.append("[too many results, only showing first " + str(maxresults) + "]") + irc.replies(lines, joiner=' | ') + + pkgsearch = thread(wrap(pkg, ['somethingWithoutSpaces'])) + + # search for packages by contents + def pkgfile(self, irc, msg, args, pkg): + """ Usage: pkgfile (wildcards allowed)""" + + db = self.InitDB(); + if db is None: + irc.error("slackpkg database doesn't exist", Raise=True) + + maxresults = self.registryValue('maxresults') + cursor = db.cursor() + cursor.execute("select distinct c.name, p.name, p.descrip from categories c, packages p, files f where c.id=p.category and p.id=f.package and f.path glob ? order by c.name, p.name limit ?", (pkg, maxresults+1)) + result = cursor.fetchall() + + lines = [] + + if len(result) == 0: + irc.reply("no matching packages") + else: + for (category, pkg, descrip) in result: + lines.append(format('%s/%s - %s', category, pkg, descrip)) + if(len(result) > maxresults): + lines.append("[too many results, only showing first " + str(maxresults) + "]") + irc.replies(lines, joiner=' | ') + + pkgfile = thread(wrap(pkgfile, ['somethingWithoutSpaces'])) + + #def which(self, irc, msg, args, filename): + +Class = Slackpkg diff --git a/SlackTools/test.py b/SlackTools/test.py new file mode 100644 index 0000000..14d770b --- /dev/null +++ b/SlackTools/test.py @@ -0,0 +1,15 @@ +### +# Copyright (c) 2021, B. Watson +# All rights reserved. +# +# +### + +from supybot.test import * + + +class SlackToolsTestCase(PluginTestCase): + plugins = ('SlackTools',) + + +# vim:set shiftwidth=4 tabstop=4 expandtab textwidth=79: diff --git a/bin/README.txt b/bin/README.txt new file mode 100644 index 0000000..e0baa08 --- /dev/null +++ b/bin/README.txt @@ -0,0 +1,6 @@ +Scripts that generate the databases used by the bot plugins. + +These are in Perl, because they're pretty complex and I just started +learning Python last week (and I've used perl for over 20 years). + +See the comments at the top of each script for instructions. diff --git a/bin/makepkgdb.pl b/bin/makepkgdb.pl new file mode 100755 index 0000000..31838d3 --- /dev/null +++ b/bin/makepkgdb.pl @@ -0,0 +1,235 @@ +#!/usr/bin/perl -w + +# create database for limnoria bot Slackpkg plugin. +# reads PACKAGES.txt from the main (slackware64) and extra +# trees. Also has to have a copy of /var/log/scripts, since +# PACKAGES.txt doesn't have symlinks, and a copy of +# /var/log/packages since PACKAGES.txt doesn't have file contents. + +# the output is a huge pile of SQL insert statements, like half +# a million of them. best way to run this IMO: +# $ makepkgdb.pl > db.sql +# $ rm -f /path/to/dbfile +# $ sqlite3 -bail /path/to/dbfile < db.sql + +# this script was only tested for slackware 14.2. When 15.0 gets +# released, we'll rebuild the database using this script, which +# might or might not need to be modified. + +# special cases: /lib64/incoming gets the /incoming removed, and +# we manually add /bin/bash. + +### configurables: + +@pkglists = ( + '/data/mirrors/slackware/slackware64-14.2/PACKAGES.TXT', + '/data/mirrors/slackware/slackware64-14.2/extra/PACKAGES.TXT', +); + +# ...the above list could have included pasture and testing, but +# for that to work, you'd also have to have all the packages from +# pasture and testing installed (we don't right now). + +# these 2 are copies of /var/log/packages and /var/log/scripts from +# a full install of 14.2, with no 3rd-party packages added: +$pkgdir = "/home/urchlay/var.log/packages"; +$scriptsdir = "/home/urchlay/var.log/scripts"; + +### end configurables. + +sub init_db { + #unlink("testdb.sqlite3"); + #open P, "|sqlite3 testdb.sqlite3"; + #select P; + + print <) { + next unless /^\( cd (\S+) ; ln -sf \S+ (\S+) \)/; + my $dir = $1; + my $file = $2; + #warn "link $dir/$file\n" if $_[0] =~ /bash-4/; + push @links, "$dir/$file"; + } + close $f; + #warn "links: " . join(" ", @links) . "\n" if $_[0] =~ /bash-4/; + return \@links; +} + +# return ref-to-array of files extracted from a packages/ file. +sub getfiles { + my @files = (); + open my $f, "<$_[0]" or return \@files; + while(<$f>) { + last if /^FILE LIST:/; + } + while(<$f>) { + chomp; + next if m,^\./$,; + next if m,^install/,; + s,^lib64/incoming,lib64,; + push @files, $_; + } + close $f; + push @files, "bin/bash" if $_[0] =~ /\/bash-[0-9]/; + return \@files; +} + +# given a package db entry as a multi-line string, extract the +# short description from it. +sub getdescrip { + my $pkgname = shift; + my $blob = shift; + if($blob =~ /^$pkgname:\s+(\S.*)$/m) { + my $line = $1; + if($line =~ /^$pkgname (?:- )?\((.+)\)/) { + $line = $1; + } + return $line; + } + return '(no description)'; +} + +# given a package db entry as a multi-line string, extract the +# category from it. +sub getcategory { + my $pkgname = shift; + my $blob = shift; + if($blob =~ /^PACKAGE LOCATION:\s+\.\/(?:(extra)|(?:slackware64\/(\w+)))/m) { + return $1 ? $1 : $2; + } + return "(unknown category)"; +} + +sub getfilelist { + my $pkgname = shift; + return @{$files{$pkgname}}; +} + +sub getsymlinklist { + my $pkgname = shift; + if(defined $symlinks{$pkgname}) { + return @{$symlinks{$pkgname}}; + } else { + return (); + } +} + +$catcount = 0; +sub getcategory_id { + my $catname = shift; + my $cat_id = $catname2id{$catname}; + if(defined $cat_id) { + return $cat_id; + } + $catcount++; + print "insert into categories values($catcount, '$catname');\n"; + $catname2id{$catname} = $catcount; + return $catcount; +} + +### main() + +# build hash of all symlinks, indexed by package name. +warn "reading symlinks from $scriptsdir/*\n"; +while(<$scriptsdir/*>) { + s,.*/,,; + my $p = getpkgname($_); + $symlinks{$p} = getsymlinks($scriptsdir . "/" . $_); +} + +warn "reading filelists from $pkgdir/*\n"; +while(<$pkgdir/*>) { + s,.*/,,; + my $p = getpkgname($_); + $files{$p} = getfiles($pkgdir . "/" . $_); +} + +#print for getfilelist('bash'); +#die; + +init_db(); + +$pkg_id = 0; +for(@pkglists) { + local $/ = ''; + open my $l, "<$_" or die $!; + warn "reading package list from $_\n"; + while(<$l>) { + next unless /^PACKAGE NAME:\s+(\S+)/; + my $pkgname = getpkgname($1); + my $descrip = getdescrip($pkgname, $_); + my $category = getcategory($pkgname, $_); + my $cat_id = getcategory_id($category); + $pkg_id++; + + $descrip =~ s,','',g; + print "insert into packages values ($pkg_id, '$pkgname', '$descrip', $cat_id);\n"; + + my @filelist = getfilelist($pkgname); + my @symlinklist = getsymlinklist($pkgname); + + for my $f (@filelist) { + print "insert into files values (null, '/$f', 0, $pkg_id);\n" + } + + for my $f (@symlinklist) { + print "insert into files values (null, '/$f', 1, $pkg_id);\n" + } + + #print "=== $cat_id: $category/$pkgname - $descrip\n"; + #print "files: " . join("\n ", @filelist) . "\n"; + #print "links: " . join("\n ", @symlinklist) . "\n\n"; + + } + close $l; +} + +print "commit;\n"; + +exit 0; + + diff --git a/bin/whatis2sqlite.pl b/bin/whatis2sqlite.pl new file mode 100755 index 0000000..297a82a --- /dev/null +++ b/bin/whatis2sqlite.pl @@ -0,0 +1,83 @@ +#!/usr/bin/perl -w + +# whatis2sqlite.pl - create sqlite3 whatis database for use with +# limnoria Manpages plugin. + +# Usage: + +# $ rm -f /path/to/Manpages.sqlite3 +# $ perl whatis2sqlite.pl [whatis-file] | sqlite /path/to/Manpages.sqlite3 + +# then reload the Manpages plugin in the bot. + +# For Slackware 14.2, the whatis-file comes from /usr/man/whatis. +# For 15.0, we'll have to generate it according to the directions +# in "man whatis": +# $ whatis -w '*' | sort > whatis +# Note that man-db's databases will have to already exist, for that to work. + +push @ARGV, 'whatis' unless @ARGV; + +print <) { + my($name, $desc, $sect, $alias); + chomp; + + # 14.2's whatis has some garbage entries, skip them. + next if /^struct/; + next if /^and (put|with)/; + next if /bernd\.warken/; + + s/\s+$//g; + ($name, $desc) = split /\s+-\s+/, $_, 2; + $name =~ s/\s+/ /g; + ($sect) = $name =~ /\(([^)]+)\)/; + next if $sect eq '3p'; # symlink + + $alias = 0; + if($name =~ /\[([^]]+)\]/) { + $alias = $1; + } + + $name =~ s,\s.*,,; + $sect =~ s/^(.).*$/$1/; # no "3x", etc. + + # 14.2's whatis has some wrong sections, fix them. + $sect = '8' if $sect eq 'v'; # ebtables + $sect = '1' if $sect eq 'P'; # cdparanoia + $sect = '1' if $sect eq 'o'; # rclock, rxvt + + #print "$sect, $name, '$alias' => $desc\n"; + + make_sql($name, $sect, $desc); + make_sql($alias, $sect, $desc) if $alias; +} + +print "COMMIT;\n"; + +sub make_sql { + my $page = shift; + my $sect = shift; + my $desc = shift; + + return if $seen{"$page^^$sect"}++; + + # N.B. we don't escape quotes here because 14.2's whatis + # doesn't contain any double-quotes. When 15 is released, + # better check again! + print <