Add -n/--cached option, handle wget failure (fatally), really get rid of "mkdir -p", sleep(1) between downloads.

2 files changed, 36 insertions, 11 deletions

diff --git a/TODO b/TODO
index 06cdded..0483144 100644
--- a/TODO
+++ b/TODO
@@ -1,8 +1,6 @@
 TODO list for greypool
 
-- add option to use cached data, even if expired.
-
-- "sleep 1" between API downloads.
+- use custom wgetrc.
 
 - handle ^C gracefully during wget downloads.
 
@@ -15,7 +13,9 @@ TODO list for greypool
   - search for repos by partial name
 
 - there's a bug I haven't looked for yet that causes the list to
-  be incomplete occasionally. NFI why yet.
+  be incomplete occasionally. Pretty sure this happens if an ignored
+  package comes at the end of a JSON file. The packages listed in each
+  file are in random order so it's hard to trigger on purpose.
 
 - de-uglify and refactor the code. it's a mess.
 
@@ -27,6 +27,9 @@ TODO list for greypool
 - handle (unlikely) possibility of commas in CSV fields. is there a
   single universally-supported correct way to do this?
 
+- transforms in config file. regex to change my version number into
+  what repology tracks.
+
 Maybe:
 
 - allow multiple repos in config file. check them all by default, add
diff --git a/greypool b/greypool
index f4a1a89..5c9abd9 100755
--- a/greypool
+++ b/greypool
@@ -25,7 +25,7 @@ use POSIX 'strftime';
 use feature 'unicode_strings';
 
 $VERSION = "0.0.0";
-$verbose = $force = $html = 0;
+$verbose = $force_download = $html = 0;
 $stamp = strftime("%Y%m%d", localtime);
 
 # all the parsed data gets shoved in here.
@@ -61,9 +61,10 @@ sub parse_opts {
 	Getopt::Long::Configure("bundling");
 	GetOptions(
 		"verbose|v" => \$verbose,
-		"force|f" => \$force,
+		"force|f" => \$force_download,
 		"html|h" => \$html,
 		"html-fragment|H" => \$html_frag,
+		"cached|n" => \$no_download,
 		"csv|s" => \$csv_output,
 		"version|V" => sub { print "$VERSION\n"; exit 0; },
 		"doc|help" => sub { exec("perldoc $0"); },
@@ -76,6 +77,9 @@ sub parse_opts {
 	$html = 1 if $html_frag;
 	$html = 0 if $csv_output;
 
+	fatal "can't use -f/--force and -n/--cached together"
+		if $force_download && $no_download;
+
 	$email = shift @ARGV;
 	fatal "extra junk on the command line after '$email'" if @ARGV;
 
@@ -105,12 +109,16 @@ sub get_repo_list {
 		"wget -qO- --user-agent '$user_agent' " .
 		'https://repology.org/repositories/statistics' .
 		'|';
+	fatal "failed to run wget: $!\n" unless $f;
 	while(<$f>) {
 		if(m,<a\s*href="/repository/([^"/]+)">([^<]+)</a>,) {
 			$repo_list{$1} = $2;
 		}
 	}
-	close $;
+	my $status = close $f;
+	if($status) {
+		fatal "wget failed, exit status " . ($status >> 8);
+	}
 }
 
 sub show_repo_list {
@@ -348,6 +356,7 @@ sub url_escape {
 }
 
 sub cache_ok {
+	return 1 if $no_download;
 	return 0 unless -e $cachestampfile;
 	my $oldstamp = read_file($cachestampfile);
 	chomp($oldstamp);
@@ -362,8 +371,8 @@ sub stamp_cache {
 }
 
 sub check_cache {
-	system("mkdir -p $cachedir");
-	return if cache_ok();
+	mkdir_p $cachedir;
+	return if cache_ok() || $no_download;
 	for(<$cachedir/*>) {
 		unlink $_;
 	}
@@ -385,10 +394,13 @@ sub download {
 	my $num = shift;
 
 	my $filename = $cachedir . "/" . $num . ".json";
-	if(!$force && (-e $filename)) {
+	if(!$force_download && (-e $filename)) {
 		info "$filename found in cache, not downloading";
 		return;
 	}
+	if($no_download && (! -e $filename)) {
+		fatal "-n / --cached option given, but no cached data in $cachedir";
+	}
 
 	my $qopt = $verbose ? "" : "-q";
 
@@ -400,7 +412,11 @@ sub download {
 		"'$url'";
 
 	info $cmd;
-	system($cmd);
+	my $status = system("cmd");
+	if($status) {
+		fatal "wget failed, exit status " . ($status >> 8);
+	}
+	sleep 1; # repology wants to limit requests to 1 per sec.
 }
 
 sub generate_report {
@@ -591,6 +607,12 @@ Re-download the JSON files, even if we have a cached copy less than
 24 hours old. Use this option with caution: excessive requests to the
 repology.org server may cause the server to block your IP address.
 
+=item B<-n>, B<--cached>
+
+Do not download JSON files from the repology API. Use cached data,
+even if it's outdated. If there is no cached data, you'll get a fatal
+error.
+
 =item B<-h>, B<--html>
 
 Generate report as an HTML page, rather than the default plain text.