store html in repo/ and use locking

1 files changed, 31 insertions, 12 deletions

diff --git a/fdroids.sh b/fdroids.sh
index 369c05e..b06d9a7 100755
--- a/fdroids.sh
+++ b/fdroids.sh
@@ -1,30 +1,49 @@
-#!/bin/sh
-set -ex
+#!/bin/bash
+set -e
 quiet() { "$@" >/dev/null 2>&1; }
 quiet dpkg-query -W html-xml-utils || sudo apt install html-xml-utils
 stale() { [ $(( $(date +%s) - $(stat -c%Y "$1") )) -gt $(( 60*60*12 )) ]; }
+wget_with_log()
+{
+        touch "$1"
+        flock "$1" wget -a "$1" "${@:2}"
+}
 if [ -t 0 ]
 then
         exec < fdroids.txt
 fi
 while read url
 do
+        baseurl=https://f-droid.org
+        url=${url%/}
         case "$url" in
-                https://f-droid.org/en/packages/*/)
-                        pkg=${url#https://f-droid.org/en/packages/}
-                        pkg=${pkg%/}
-                        ;;
-                *) continue ;;
+                $baseurl/??/packages/*/* | $baseurl/packages/*/* )
+                  continue ;;
+                $baseurl/??/packages/* | $baseurl/packages/* )
+                  pkg=${url#*/packages/} ;;
+                * ) continue ;;
         esac
-        html=$pkg.html
+        html=repo/$pkg/index.html
         if [ ! -e "$html" ] || stale "$html"
         then
-                wget -a wget-log.txt -c -O "$html" "$url" || { rm -f "$html"; break; }
+            (
+                echo "$pkg" >&2
+                mkdir -p "${html%/*}"
+                cd "${html%/*}" || continue
+                if wget_with_log wget-log.txt -c -O "$html" "$url"
+                then
+                  touch "$html"
+                else
+                  continue
+                fi
                 link=$(hxwls "$html" | egrep "/repo/${pkg}_[0-9]+\.apk\$" | sort -n | tail -n1)
-                if [ "$link" ]
+                if [ "$link" ] &&
+                        wget_with_log wget-log.txt -c "$link" "$link.asc"
                 then
-                        mkdir -p repo
-                        (cd repo && wget -c "$link" "$link.asc" -a "$pkg".wget-log.txt)
+                        continue
+                else
+                        echo "failed to fetch $pkg" >&2
                 fi
+            )
         fi
 done