#!/bin/bash # sbodl, download slackbuilds.org source files # 20230513 bkw: move to the sbo-maintainer-tools repo, clean up for release. # 20191222 bkw: handle broken symlinks in current dir. # change default cache dir. # 20170306 bkw: add caching # Don't edit the next line; use "make version" instead. VERSION=0.9.2.1 : < [-h | --help] B [-m | --man ] B --version B [ [-a | --archive] | [-f | --force] | [-n | --no-content-disposition] ] [ -- I ...] =head1 DESCRIPTION B reads the .info file in the current directory and downloads the source files listed there, if they don't already exist. Then it checks the md5sums of the source files (whether they were just downloaded or not). To avoid extra network overhead, the downloaded files are stored in a cache directory (see ENVIRONMENT), and maintained as symlinks in the current directory. This allows you to remove the symlinks prior to creating a tarball of your SlackBuild directory for upload, without losing the actual files. It also allows builds that use the same source files to share them (no need to re-download). By default, B<--content-disposition> is used. If you want to disable it, use the B<-n>, B<--no-content-disposition> option. If you need to download files for a different architecture (e.g. 32-bit files when running on a 64-bit OS), you can override ARCH in the environment: "ARCH=i686 sbodl" would download the 32-bit source file(s). This only matters if there's a DOWNLOAD_x86_64 URL in the .info file. =head1 OPTIONS B: Any arguments after B<--> are passed through to B(1) verbatim. =over 4 =item B<-a>, B<--archive> Download from the SBo Source Archive (sbosrcarch) rather than the original URL(s). Use this if the original site is down or the file has gone 404. See ENVIRONMENT, if you need to change the sbosrcarch URL. =item B<-f>, B<--force> Force downloading the file(s), even if they already exist in the current directory or in the cache. =item B<-n>, B<--no-content-disposition> Ignore the I header; save the file using the filename in the URL rather than the name suggested in the HTTP headers. This option just passes B<--no-content-disposition> to B. =item B<--help>, B<--doc> Show this documentation in your pager. =item B<--version> Show version number and exit. =item B<--man> Output this documentation, formatted as a man page, on standard output. =back =head1 ENVIRONMENT =over 4 =item B The URL of the sbosrcarch instance. The only known one at the time of this writing is B, but that may change in the future (or you may run your own private instance). This is the URL of the directory containing the B and B dirs, and should not end with a trailing slash. =item B Where to cache downloaded files. By default, this is B<$HOME/sbodl-cache>. This directory will be created if it does not exist. =item B Used to determine which files to download; default is autodetect. Only matters for builds with a separate 64-bit download URL. Use B or B for 32-bit x86, B for 64-bit. Note that there's no point setting this to e.g. B, as the .info file format doesn't have a DOWNLOAD_aarch64 field (yet). =back =head1 EXIT STATUS 0 (success) if all files were downloaded or found in the cache, and their md5sums all match the .info file. 1 (failure) if any download failed, or if at least one md5sum failed to match. =head1 BUGS One very rare but nasty one: all the cached files are stored in the same directory. If two SlackBuilds happen to have the same filename for their downloads (but they're actually different files), you'll get md5sum failures. This is most noticeable with I builds, with filenames like B<1.cabal>, B<2.cabal>, etc. At some point this will be fixed. For now, if this happens, use the B<-f> option. Not really a bug: since B<--content-disposition> is used, occasionally the downloaded filename won't match the filename in the URL, and B will warn that it can't find the downloaded file. I left this enabled specifically to detect URLs that have this problem: mostly, these are github URLs and there's a way to rewrite them to work either way. B SlackBuild scripts that deal with URLs like this can handle either possible filename, or else they use a URL that uses the same filename either way. If you have to, use B<-n> to ignore content disposition. =head1 AUTHOR B. Watson , aka Urchlay on Libera IRC. =head1 SEE ALSO B(1), B(1), B(1), B(8), B(1) =cut EOF SELF=$( basename $0 ) DEFCACHEDIR=~/sbodl-cache CACHEDIR=${SBODL_CACHEDIR:-$DEFCACHEDIR} DEFARCHIVE=https://slackware.uk/sbosrcarch ARCHIVE=${SBODL_ARCHIVE:-$DEFARCHIVE} if [ -t 1 ]; then RED="\033[1;31m" GREEN="\033[1;32m" YELLOW="\033[1;33m" COLOR_OFF="\033[0m" fi die() { echo "$SELF: $*" 2>&1 exit 1 } CONTDISP="--content-disposition" # 20231205 bkw: fancy argument checking. note that getopt is part of # util-linux; do not confuse it with bash's built-in getopts (with an "s"). OPTS=$( getopt -n 1.sh -o vmhafn -l help,man,doc,version,archive,force,no-content-disposition -- "$@" ) eval set -- "$OPTS" while true; do case "$1" in -h|--help|--doc) exec perldoc "$0" ;; -m|--man) exec pod2man --stderr -s1 -csbo-maintainer-tools -r$VERSION "$0" ;; -v|--version) echo $VERSION ; exit 0 ;; -a|--archive) USEARCHIVE="yes" ; shift ; continue ;; -f|--force) FORCEDL="yes" ; shift ; continue ;; -n) CONTDISP="--no-content-disposition" ; shift ; continue ;; --) shift; break ;; esac done if [ "$*" != "" ]; then echo -e "${YELLOW}Passing extra arguments to wget: ${RED}$@${COLOR_OFF}" fi [ -d "$CACHEDIR/" ] || mkdir -p $CACHEDIR || die "Can't create $CACHEDIR" source ./$( basename $( pwd ) ).info \ || die "No .info file, are you sure this is a SBo directory? Try '$SELF --help'" # This stanza copied from the SBo template for 14.1: if [ -z "$ARCH" ]; then case "$( uname -m )" in i?86) ARCH=i586 ;; arm*) ARCH=arm ;; *) ARCH=$( uname -m ) ;; esac fi if [ "$ARCH" = "x86_64" ]; then DL=${DOWNLOAD_x86_64:-$DOWNLOAD} SUM=${MD5SUM_x86_64:-$MD5SUM} else DL=$DOWNLOAD SUM=${MD5SUM} fi if [ -z "$DL" ]; then die "Bad .info file (no DOWNLOAD= or DOWNLOAD_x86_64=)." fi if [ "$DL" = "UNSUPPORTED" ]; then die "$PRGNAM is unsupported on $ARCH" fi # save passed-in command line args for use with wget WGETARGS="$@" EXIT=0 set $SUM for dl in $DL; do if [ "$USEARCHIVE" = "yes" ]; then file="$( echo "$dl" | sed 's,.*/,,' )" a=$( echo $1 | cut -b1 ) b=$( echo $1 | cut -b2 ) dl="$ARCHIVE/by-md5/$a/$b/$1/$file" fi EXTRAWGETARGS="$CONTDISP" case "$dl" in # sourceforge can't use content-disposition, because some of its # mirrors add '?viasf=1' to the filename in the header. *sourceforge.net/*|*.sf.net/*) EXTRAWGETARGS="--user-agent wget" ;; *) ;; esac FILE=$( echo "$dl" | sed 's,.*/,,' ) # ignore (rm) broken symlinks [ -L "$FILE" ] && ! [ -e "$FILE" ] && FORCEDL=yes [ "$FORCEDL" = "yes" ] && rm -f "$FILE" if [ -f "$FILE" -a ! -L "$FILE" ]; then # file exists and is a regular file, cache it mv -b "$FILE" "$CACHEDIR" ln -s "$CACHEDIR/$FILE" "$FILE" elif [ -e "$FILE" ]; then # don't do anything : elif [ "$FORCEDL" != "yes" ] && [ -e "$CACHEDIR/$FILE" ]; then ln -s "$CACHEDIR/$FILE" "$FILE" else wget $EXTRAWGETARGS $WGETARGS "$dl" || die "Download failed, try '$SELF -a'." if [ -e "$FILE" ]; then mv -b "$FILE" "$CACHEDIR" ln -s "$CACHEDIR/$FILE" "$FILE" fi fi if [ -e "$FILE" ]; then GOTSUM="$( md5sum "$FILE" | cut -d' ' -f1 )" if [ "$1" != "$GOTSUM" ]; then echo -e "${RED}WARN: md5sum doesn't match${COLOR_OFF}\\n expected: $1\\n got: $GOTSUM" case "$FILE" in ?.cabal) echo -e "${YELLOW}HINT:${COLOR_OFF} for ?.cabal files: if the md5sum fails, retry with '$SELF -f'." ;; esac EXIT=1 else echo -e "${GREEN}md5sum matches OK${COLOR_OFF}: $1" fi else echo -e "${RED}WARN${COLOR_OFF}: can't find downloaded file $FILE, try '$SELF -a'?" EXIT=1 fi shift done exit $EXIT