ropeylog: stop caching the json; cache the result of parsing it. also, add command-line options and verbose debug info.HEADmaster

1 files changed, 130 insertions, 59 deletions

diff --git a/ropeylog b/ropeylog
index 5aefe21..66aae99 100755
--- a/ropeylog
+++ b/ropeylog
@@ -1,18 +1,19 @@
 #!/usr/bin/perl -w
 
-###############
-# These 2 are configurable, change as needed:
-
-$user_agent = 'Mozilla/5.0 (compatible; ropeylog-repology-bot/1.0; +https://github.com/SlackBuildsOrg/slackbuilds)';
-
+# Defaults for the -g, -o, -v options:
 $git_root = $ENV{SBO_GITROOT} || "/slackbuilds/www/slackbuilds/15.0/";
-
 $outdir = "$ENV{HOME}/www";
+$verbose = 0;
+
+###############
+# This shouldn't need changing.
+$user_agent = 'Mozilla/5.0 (compatible; ropeylog-repology-bot/1.0; +https://github.com/SlackBuildsOrg/slackbuilds)';
 
 ###############
 
 use JSON::XS;                  # from SBo perl-JSON-XS
 use POSIX qw/strftime getcwd/; # bundled with Slackware's perl
+use Getopt::Long;              # ditto
 
 ($SELF = $0) =~ s,.*/,,;
 
@@ -26,15 +27,29 @@ use POSIX qw/strftime getcwd/; # bundled with Slackware's perl
 
 ###############
 
+sub warning {
+	warn "$SELF: warning: " . join(" ", @_). "\n";
+}
+
+sub fatal {
+	die "$SELF: fatal: " . join(" ", @_). "\n";
+}
+
+sub info {
+	return unless $verbose;
+	warn "$SELF: " . join(" ", @_). "\n";
+}
+
+###############
+
 sub usage {
 	print <<EOF;
-$SELF - generate SBo Maintainer Status report (HTML)
+$SELF - generate SBo Maintainer Status reports (HTML)
 
-Usage: $SELF [-d] [output-dir]
+Usage: $SELF [-d] [-g <git-dir>] [-o <output-dir>] [-s] [-v] 
 
-With the -d option, fresh package status data is fetched from
-repology.org. Without it, the most recent data (cached in
-~/.ropeylog/) is used.
+Generates HTML maintainer status reports, suitable for use on the
+SlackBuilds.org site.
 
 The HTML pages are written to the [output-dir] directory. If any file
 already exists, it is backed up with a .bak extension.
@@ -47,6 +62,31 @@ EOF
 	for(@reports) {
 		printf "%20s | %-s\n", $_->[0], get_page_name($_->[1]);
 	}
+	print <<EOF;
+
+Options:
+
+  -d, --download
+    Download fresh package status data from repology.org.
+    Without this option, cached data from ~/.ropeylog/ is used.
+
+  -o <output-dir>, --output-dir=<output-dir>
+    Write HTML reports to this directory. The default is:
+    $outdir
+
+  -g <git-dir>
+    Read .info files and git logs from this tree. The default is:
+    $git_root
+
+  -s, --save-json
+    Save the JSON data downloaded from repology.org. Only useful
+    for debugging purposes.
+
+  -v, --verbose
+    Print verbose information on what ropeylog is doing, including
+    download progress from wget. Only useful for debugging.
+EOF
+
 }
 
 ###############
@@ -58,36 +98,60 @@ sub make_api_url {
 	$url .= "?status=outdated&inrepo=slackbuilds";
 }
 
+# note to self: wget needs to write to an actual file, not stdout,
+# because repology's json api doesn't support continuing, if the
+# transfer dies part of the way through. wgetting to stdout results
+# in incomplete files if that happens.
 sub wget {
 	my $url = shift;
-	my $output = shift;
+	my $file = shift;
 
+	my $q = $verbose ? "" : "-q";
 	my $cmd =
-		"wget --compression=gzip --user-agent '$user_agent' -O$output '$url'";
+		"wget $q --compression=gzip --user-agent '$user_agent' -O$file '$url'";
 
-	warn "$cmd\n";
+	info "$cmd";
 
 	system($cmd);
+
+	undef local $/;
+	open my $f, "<$file" or fatal $!;
+	my $resp = <$f>;
+	close $f;
+	unlink $file unless $save_json;
+
+	info "sleeping 1s to comply with repology.org spec";
 	sleep 1;
+
+	return $resp;
 }
 
 sub download {
 	my $firstpkg = $_[0] || "";
+	my $count = $_[1];
 	my $lastpkg = "";
-	my $filename = sprintf("%s/%03d.json", $cache_dir, $_[1]);
 
-	wget(make_api_url($_[0]), $filename);
+	my $filename = sprintf("%s/%03d.json", $cache_dir, $count);
+	my $json = wget(make_api_url($_[0]), $filename);
 
-	my $j = parse_one_file($filename);
+	eval { $j = decode_json($json); };
+	fatal "failed to decode JSON: $@\n" if $@;
 
 	for $proj (keys %{$j}) {
 		$lastpkg = $proj if $proj gt $lastpkg;
+		my $arr = $$j{$proj};
+		for my $hr (@{$arr}) {
+			next unless $hr->{repo} eq 'slackbuilds';
+			my $catbuild = $hr->{visiblename};
+			$outdated{$catbuild}++ if $hr->{status} eq 'outdated';
+		}
 	}
 
 	return (($lastpkg eq $firstpkg) ? undef : $lastpkg);
 }
 
 sub download_json_files {
+	info "downloading JSON results from repology.org";
 	my $lastpkg;
 	my $count = 1;
 
@@ -98,37 +162,29 @@ sub download_json_files {
 
 ###############
 
-sub read_file {
-	undef local $/;
-	local @ARGV = shift;
-	$_ = <>;
+sub get_cache_filename {
+	return "$cache_dir/outdated";
 }
 
-sub parse_one_file {
-	my $file = shift;
-	my $j;
-
-	eval { $j = decode_json(read_file($file)); };
-	die "failed to decode JSON: $@\n" if $@;
-
-	return $j;
+sub write_cache {
+	my $fname = get_cache_filename();
+	info "saving outdated build list to cache: $fname";
+	open my $f, ">$fname" or fatal "$fname: $!";
+	for my $pkg (sort { $a cmp $b } keys %outdated) {
+		print $f $pkg . "\n";
+	}
+	close $f;
 }
 
-sub parse_json_files {
-	die "$SELF: nothing to parse, run '$SELF download' first.\n"
-		unless -e "$cache_dir/001.json";
-
-	for(<$cache_dir/*.json>) {
-		my $j = parse_one_file($_);
-		for my $proj (keys %{$j}) {
-			my $arr = $$j{$proj};
-			for my $hr (@{$arr}) {
-				next unless $hr->{repo} eq 'slackbuilds';
-				my $catbuild = $hr->{visiblename};
-				$outdated{$catbuild}++ if $hr->{status} eq 'outdated';
-			}
-		}
+sub read_cache {
+	my $fname = get_cache_filename();
+	info "loading outdated build list from cache: $fname";
+	open my $f, "<$fname" or fatal "$fname: $!\n*** You must run '$SELF -d' to initialize the cache.";
+	while(<$f>) {
+		chomp;
+		$outdated{$_}++;
 	}
+	close $f;
 }
 
 ###############
@@ -156,6 +212,7 @@ sub read_info_file {
 }
 
 sub parse_info_files {
+	info "reading info files from $git_root";
 	my $files = `git $git_args ls-files '*/*/*.info'`;
 	for my $info (split /\n/, $files) {
 		$total_builds++;
@@ -171,9 +228,11 @@ sub parse_info_files {
 		push @{$maintbuilds{$maintainer}}, $catbuild;
 	}
 	@maintainers = keys %name2email;
+	info "read $total_builds .info files, " . @maintainers . " maintainers"
 }
 
-sub get_git_maintainers {
+sub parse_git_log {
+	info "reading & parsing git log";
 
 	open my $in, "git $git_args log " .
 		"--pretty=format:%h%x09%an%x09%ae%x09%as%x09%s|"
@@ -372,26 +431,38 @@ sub print_all_reports {
 ###############
 
 # main()
+Getopt::Long::Configure("bundling");
+GetOptions(
+	"download|d"    => \$do_download,
+	"save-json|s"   => \$save_json,
+	"git-dir|g"     => \$git_root,
+	"output-dir|o"  => \$outdir,
+	"help|h"        => sub { usage(); exit 0; },
+	"verbose|v"     => \$verbose,
+);
+
+$start_time = time;
+
 $cache_dir = "$ENV{HOME}/.$SELF";
-system("mkdir -p $cache_dir");
+system("mkdir -p $cache_dir $outdir");
+fatal "can't create $cache_dir" unless -d $cache_dir;
+fatal "can't create $cache_dir" unless -d $outdir;
+info "\$cache_dir: $cache_dir";
+info "\$outdir: $outdir";
 
 $git_args = "--work-tree $git_root --git-dir $git_root/.git";
+info "\$git_args: $git_args";
 
-if(@ARGV) {
-	if($ARGV[0] eq '-d') {
-		shift;
-		for(<$cache_dir/*.json>) {
-			unlink $_;
-		}
-		download_json_files();
-	} elsif($ARGV[0] =~ /help/) {
-		usage();
-		exit 0;
-	}
+if($do_download) {
+	download_json_files();
+	write_cache();
+} else {
+	read_cache();
 }
-$outdir = $ARGV[0] if $ARGV[0];
 
-parse_json_files();
 parse_info_files();
-get_git_maintainers();
+parse_git_log();
 print_all_reports();
+
+info "elapsed time: " . (time - $start_time) . " sec";
+exit 0;