Merge pull request 'feature: validate commits in reverse date-time order when MAX_COMMITS_TO_CHECK is used' (#12) from date-order into main

Reviewed-on: #12
Reviewed-by: Artur Neumann <artur@jankaritech.eu>

.gitea/workflows/validate.yaml

@@ -1,14 +1,3 @@
-location / {
-    proxy_pass        http://localhost:5232/;
-    proxy_set_header  X-Script-Name /radicale;
-    proxy_set_header  X-Forwarded-For $proxy_add_x_forwarded_for;
-    proxy_set_header  X-Forwarded-Host $host;
-    proxy_set_header  X-Forwarded-Port $server_port;
-    proxy_set_header  X-Forwarded-Proto $scheme;
-    proxy_set_header  Host $http_host;
-    proxy_pass_header Authorization;
-}
-
 name: Validate Trusted Timestamps Actions Demo
 run-name: ${{ gitea.actor }} is validating the trusted timestamps of all commits 🚀
 on: [push]

.timestampltv/crls/E1D782A8E191BEEF6BCA1691B5AAB494A6249BF3.crl

@@ -2,17 +2,17 @@
 MIIC5jCCAc4CAQEwDQYJKoZIhvcNAQELBQAwcjELMAkGA1UEBhMCVVMxFTATBgNV
 BAoTDERpZ2lDZXJ0IEluYzEZMBcGA1UECxMQd3d3LmRpZ2ljZXJ0LmNvbTExMC8G
 A1UEAxMoRGlnaUNlcnQgU0hBMiBBc3N1cmVkIElEIFRpbWVzdGFtcGluZyBDQRcN
-MjUwMTI0MDUxNDI4WhcNMjUwMTMxMDUxNDI4WjCB9TAhAhAL2v0LKRQzmpYSZqw1
+MjUwMjEyMDUyMjQ1WhcNMjUwMjE5MDUyMjQ1WjCB9TAhAhAL2v0LKRQzmpYSZqw1
 OkdEFw0xNjEwMjQxNzQyNDlaMCECEAH40oMtKRkZcbNQw9u8pQAXDTE2MTExMTE1
 MjEzNFowIQIQClKwbEb16yWgi9U/3Ht4hhcNMTgwOTAzMTIxMTQyWjAhAhAFlx7K
 SlmJinvPTLfjd5doFw0xOTA5MzAwODE2MjRaMCECEAxFkEkmQLBOlEh/jEwCeJAX
 DTIxMDIwOTIyMzk0MlowIQIQBMvnUVSd49EL7YN0yV7iRBcNMjEwMjA5MjMyMzM3
 WjAhAhALmUrhw5aLANVesgZ0jpseFw0yMTAyMDkyMzI0MjNaoDAwLjAfBgNVHSME
-GDAWgBT0tuEgHf4prtLkYaWyoiWyyBc1bjALBgNVHRQEBAICDNwwDQYJKoZIhvcN
-AQELBQADggEBAGuGW4lrI1pz4IwilL1u3rFRWD43/2Cu0+Pc1+tRx+QqB42aB0Jl
-esdIRc7t7bZ+5wmJyl5DTToQ3Vm7v34dXlblmmlJ2IM+1BKNEO4jMg82i4CFHtaE
-1e2lTfCOKR7YiTmUv/E44jAeQNJbt3k/6gnpDTGafJTIybYNh3uVDtC8Iiun4DKH
-x1qe0qzuixF2TDdTRgPP293nShxNJP5G9G5JaOGSreVOItwEhI+GP6rrPffcanfJ
-v7ghEutuJCE2BGZkqL5iEGgAbMYhFitCu58rfwCHF78uz8T/kxbe5Ax2Zu1IV3is
-kuc5vOHsT/GFYnMC4PZn9J9eYKLE6mzr0SY=
+GDAWgBT0tuEgHf4prtLkYaWyoiWyyBc1bjALBgNVHRQEBAICDO8wDQYJKoZIhvcN
+AQELBQADggEBALr4VopJYkMfQ97HiyqytcWRY/vgyU/LxOwlH0/1DBSeeObQB0Nj
+uF7vcF2bhbpnxba7gvzOPryudwtbqquf2cl3CJG6MC2D8Nk1XzntDnpxCjVSfsAr
+158zAWPevyiuj3yzFz04mYALt/ZmOJMTF0vyKN8cg5bwfLu3itV6b6vhpuloIhRc
+Hmsbgr3BtCVHkf4vJWq/qKDEMcOhSrJ6wxGCzVyphenewSIbVcogj19cRZDFPWOC
+3sAy/GY3Rz0qK30tDvNbE1uum8gy5ijXFmepJ/lEetRCvrIsxTsXJOj0tqVZfIIQ
+E1YWUZ57TiBBrdS+dTgmRxkN/zaAfYVAIck=
 -----END X509 CRL-----

hooks/timestamping

@@ -553,7 +553,7 @@ download_crls_for_chain() {
       local URL=$(openssl x509 -inform PEM -in $EXTRACTED_CERT -text -noout \
                   | awk '/CRL Distribution Points:/{f=1} f && /URI:/ {print; exit}' \
                   | sed 's/^.*URI://1')
-      if curl "$URL" --output "$CRL_TMP" &> "$OUT_STREAM"; then
+      if curl -L "$URL" --output "$CRL_TMP" &> "$OUT_STREAM"; then
         if openssl crl -in "$CRL_TMP" -inform DER -noout &> "$OUT_STREAM"; then
           openssl crl -in "$CRL_TMP" -inform DER >> "$OUTPUT_FILE"
         elif openssl crl -in "$CRL_TMP" -inform PEM -noout &> "$OUT_STREAM"; then

hooks/validate.sh

@@ -42,6 +42,10 @@ if [[ ! -d "$DIR" ]]; then DIR="$PWD"; fi
 . "$DIR/timestamping"
 
 declare -i MINVERSION=$TIMESTAMPING_VERSION
+declare -i MAX_COMMITS_TO_CHECK=0
+declare -A PROCESSED_COMMIT
+declare -A COMMITS
+declare -A COMMIT_TIMES
 
 while [[ $# -gt 0 ]]; do
   KEY="$1"
@@ -61,6 +65,16 @@ while [[ $# -gt 0 ]]; do
       shift # past argument
       shift # past value
       ;;
+    -max|--maxcommits)
+      INTEGER_REGEX='^[0-9]+$'
+      if ! [[ "$2" =~ $INTEGER_REGEX ]]; then
+        echo_error "$KEY: expected positive integer"
+        exit 1
+      fi
+      MAX_COMMITS_TO_CHECK="$2"
+      shift # past argument
+      shift # past value
+      ;;
     -v|--verbose)
       OUT_STREAM=/dev/stdout
       shift # past argument
@@ -89,6 +103,10 @@ fi
 # tokens, the function will return 0 but echo a warning about the invalid token.
 validate_commit() {
   local COMMIT_HASH="$1"
+  if [[ ${PROCESSED_COMMIT[$COMMIT_HASH]} ]]; then
+    log "validate_commit for $COMMIT_HASH has already been validated"
+    return 0
+  fi
   log "validate_commit for $COMMIT_HASH"
 
   local TIMESTAMP_COMMIT_VERSION
@@ -275,9 +293,11 @@ validate_commit() {
   #assert that all extracted timestamps have been processed
   assert "[ $NUM_PROCESSED -eq $NUM_EXTRACTED ]" "All extracted token must be processed."
 
+  PROCESSED_COMMIT[$COMMIT_HASH]=1
+
   if [ $NUM_VALID -gt 0 ]; then
     if [ $NUM_INVALID -gt 0 ]; then
-      echo_warning "Warning: While commit $COMMIT_HASH contains $NUM_VALID valid timestamp tokens and thus is considered proppely timestamped, it also contains $NUM_INVALID invalid timestamp tokens."
+      echo_warning "Warning: While commit $COMMIT_HASH contains $NUM_VALID valid timestamp tokens and thus is considered properly timestamped, it also contains $NUM_INVALID invalid timestamp tokens."
     fi
     DATE_STRING=$(date -d @"$EARLIEST_VALID_UNIX_TIME")
     echo_info "Commit $COMMIT_HASH, which timestamps commit $PARENT_HASH at $DATE_STRING, contains $NUM_VALID valid timestamp tokens."
@@ -293,6 +313,13 @@ validate_commit() {
 # param1: commit hash
 # returns: 0 if the validation of the commit and all its ancestors succeeded
 validate_commit_and_parents() {
+  # If MAX_COMMITS_TO_CHECK is zero (or a negative number) then that is understood as "infinity".
+  # So finish if we have reached the limit, and if the limit is not "infinity".
+  NUM_COMMITS_CHECKED=${#PROCESSED_COMMIT[@]}
+  if [[ ${NUM_COMMITS_CHECKED} -ge ${MAX_COMMITS_TO_CHECK} ]] && [[ ${MAX_COMMITS_TO_CHECK} -ge 1 ]]; then
+    # enough commits have already been checked, so return early
+    return 0;
+  fi
   local COMMIT_HASH="$1"
   log "validate_commit_and_parents for $COMMIT_HASH"
 
@@ -300,6 +327,7 @@ validate_commit_and_parents() {
   if ! validate_commit "$COMMIT_HASH"; then
     ALL_PASSED=false
   fi
+  NUM_COMMITS_CHECKED=${#PROCESSED_COMMIT[@]}
   local PARENTS=$(git cat-file -p "$COMMIT_HASH" | awk '/^$/{exit} /parent/ {print}' | sed 's/parent //')
   #iterate over all parents of commit
   if [ ! -z "$PARENTS" ]; then
@@ -315,6 +343,64 @@ validate_commit_and_parents() {
   return 1
 }
 
+# Recursive function to find all ancestors of commit
+# param1: commit hash
+# creates an array COMMITS, key is the commit hash, value is the commit time (Unix epoch seconds)
+# the array contains all commits found in all paths from the passed-in commit hash back to the root commit of the repo
+# the array is global so it can be accessed after the function returns
+find_all_commits() {
+  local COMMIT_HASH="$1"
+  log "find_all_commits for $COMMIT_HASH"
+  # git show "ct" format returns the commit time as Unix epoch seconds
+  COMMIT_TIME=$(git show --no-patch --format=%ct "$COMMIT_HASH")
+  COMMITS[$COMMIT_HASH]="${COMMIT_TIME}"
+
+  local PARENTS=$(git cat-file -p "$COMMIT_HASH" | awk '/^$/{exit} /parent/ {print}' | sed 's/parent //')
+  # iterate over all parents of commit
+  if [ ! -z "$PARENTS" ]; then
+    while read PARENT_HASH; do
+      if [[ ${COMMITS[$PARENT_HASH]} ]]; then
+        log "commit $PARENT_HASH has already been processed"
+      else
+        find_all_commits "$PARENT_HASH"
+      fi
+    done <<< $(printf "%s" "$PARENTS")
+  fi
+}
+
+# Validate the commits in the COMMITS array, up to MAX_COMMITS_TO_CHECK
+# returns: 0 if the validation of the commits succeeded
+validate_commits() {
+  ALL_PASSED=true
+  # create an associative array with keys using the Unix epoch commit time and value the commit hash
+  # this array can be easily used to sort in (forward or reverse) order of time
+  for HASH in "${!COMMITS[@]}"; do
+    UNIX_EPOCH_TIME="${COMMITS[$HASH]}"
+    # two commits could have the exact same Unix epoch in seconds
+    # so make that unique by appending an "x" and the hash
+    UNIQUE_KEY="${UNIX_EPOCH_TIME}x${HASH}"
+    COMMIT_TIMES[$UNIQUE_KEY]="${HASH}"
+  done
+  # sort into reverse order
+  SORTED_KEYS=($(printf "%s\n" "${!COMMIT_TIMES[@]}" | sort -r))
+  # process the commits from latest time to oldest time
+  ALL_PASSED=true
+  for ENTRY in "${SORTED_KEYS[@]}"; do
+    COMMIT_HASH=${COMMIT_TIMES[${ENTRY}]}
+    log "${ENTRY} has value ${COMMIT_HASH}"
+    NUM_COMMITS_CHECKED=${#PROCESSED_COMMIT[@]}
+    if [[ ${NUM_COMMITS_CHECKED} -lt ${MAX_COMMITS_TO_CHECK} ]]; then
+      if ! validate_commit "$COMMIT_HASH"; then
+        ALL_PASSED=false
+      fi
+    fi
+  done
+  if [ "$ALL_PASSED" = true ]; then
+    return 0
+  fi
+  return 1
+}
+
 echo_info "Checking repository integrity..."
 #check git repository integrity
 if ! git fsck --full --strict --no-progress --no-dangling "$COMMIT_HASH"; then
@@ -326,10 +412,21 @@ echo ""
 
 echo_info "Validating timestamps. This may take a while..."
 echo ""
-if validate_commit_and_parents "$COMMIT_HASH"; then
-  echo_success "Validation OK: All timestamped commits in the commit history of $COMMIT_HASH contain at least one valid timestamp."
-  exit 0
+if [[ ${MAX_COMMITS_TO_CHECK} -ge 1 ]]; then
+  find_all_commits "$COMMIT_HASH"
+  if validate_commits; then
+    echo_success "Validation OK: ${NUM_COMMITS_CHECKED} timestamped commits in the commit history of $COMMIT_HASH contain at least one valid timestamp."
+    exit 0
+  else
+    echo_error "Validation Failed: There are timestamped commits in the commit history of $COMMIT_HASH which do not contain any valid timestamps."
+    exit 1
+  fi
 else
-  echo_error "Validation Failed: There are timestamped commits in the commit history of $COMMIT_HASH which do not contain any valid timestamps."
-  exit 1
+  if validate_commit_and_parents "$COMMIT_HASH"; then
+    echo_success "Validation OK: All timestamped commits in the commit history of $COMMIT_HASH contain at least one valid timestamp."
+    exit 0
+  else
+    echo_error "Validation Failed: There are timestamped commits in the commit history of $COMMIT_HASH which do not contain any valid timestamps."
+    exit 1
+  fi
 fi