Merge pull request 'feature: validate commits in reverse date-time order when MAX_COMMITS_TO_CHECK is used' (#12) from date-order into main

Reviewed-on: #12
Reviewed-by: Artur Neumann <artur@jankaritech.eu>

.gitea/workflows/validate.yaml

@@ -0,0 +1,32 @@
+name: Validate Trusted Timestamps Actions Demo
+run-name: ${{ gitea.actor }} is validating the trusted timestamps of all commits 🚀
+on: [push]
+
+variables:
+  EXPECTED_TRUSTANCHORS_HASH: "70a1c7e2fc62a0b62e44063f0e730b20b0f209d15c84b310ad06ce616c352829"
+
+jobs:
+  Validate:
+    runs-on: ubuntu-latest
+    timeout-minutes: 2
+    steps:
+      - name: Install extra software
+        run: |
+          apt-get update
+          apt-get install -y xxd
+      - name: Check out repository code
+        uses: actions/checkout@v4
+        with:
+           fetch-depth: 0
+      - name: Setup timestamping authorities
+        run: |
+          git config --local timestamping.tsa0.url https://freetsa.org/tsr
+          bash -c 'yes | ./hooks/trust.sh https://freetsa.org/tsr'
+          git config --local timestamping.tsa1.url https://tsa.cesnet.cz:3162/tsa
+          bash -c 'yes | ./hooks/trust.sh https://tsa.cesnet.cz:3162/tsa'
+      - name: Check hashes of all trustanchors
+        run: |
+          ./hooks/validate_trustanchors_hash.sh .git/hoqoks/trustanchors ${{ EXPECTED_TRUSTANCHORS_HASH }}
+      - name: Validate timestamps of all commits
+        run: |
+          ./hooks/validate.sh --minversion 0

.timestampltv/crls/E1D782A8E191BEEF6BCA1691B5AAB494A6249BF3.crl

@@ -2,17 +2,17 @@
 MIIC5jCCAc4CAQEwDQYJKoZIhvcNAQELBQAwcjELMAkGA1UEBhMCVVMxFTATBgNV
 BAoTDERpZ2lDZXJ0IEluYzEZMBcGA1UECxMQd3d3LmRpZ2ljZXJ0LmNvbTExMC8G
 A1UEAxMoRGlnaUNlcnQgU0hBMiBBc3N1cmVkIElEIFRpbWVzdGFtcGluZyBDQRcN
-MjUwMTI0MDUxNDI4WhcNMjUwMTMxMDUxNDI4WjCB9TAhAhAL2v0LKRQzmpYSZqw1
+MjUwMjEyMDUyMjQ1WhcNMjUwMjE5MDUyMjQ1WjCB9TAhAhAL2v0LKRQzmpYSZqw1
 OkdEFw0xNjEwMjQxNzQyNDlaMCECEAH40oMtKRkZcbNQw9u8pQAXDTE2MTExMTE1
 MjEzNFowIQIQClKwbEb16yWgi9U/3Ht4hhcNMTgwOTAzMTIxMTQyWjAhAhAFlx7K
 SlmJinvPTLfjd5doFw0xOTA5MzAwODE2MjRaMCECEAxFkEkmQLBOlEh/jEwCeJAX
 DTIxMDIwOTIyMzk0MlowIQIQBMvnUVSd49EL7YN0yV7iRBcNMjEwMjA5MjMyMzM3
 WjAhAhALmUrhw5aLANVesgZ0jpseFw0yMTAyMDkyMzI0MjNaoDAwLjAfBgNVHSME
-GDAWgBT0tuEgHf4prtLkYaWyoiWyyBc1bjALBgNVHRQEBAICDNwwDQYJKoZIhvcN
-AQELBQADggEBAGuGW4lrI1pz4IwilL1u3rFRWD43/2Cu0+Pc1+tRx+QqB42aB0Jl
-esdIRc7t7bZ+5wmJyl5DTToQ3Vm7v34dXlblmmlJ2IM+1BKNEO4jMg82i4CFHtaE
-1e2lTfCOKR7YiTmUv/E44jAeQNJbt3k/6gnpDTGafJTIybYNh3uVDtC8Iiun4DKH
-x1qe0qzuixF2TDdTRgPP293nShxNJP5G9G5JaOGSreVOItwEhI+GP6rrPffcanfJ
-v7ghEutuJCE2BGZkqL5iEGgAbMYhFitCu58rfwCHF78uz8T/kxbe5Ax2Zu1IV3is
-kuc5vOHsT/GFYnMC4PZn9J9eYKLE6mzr0SY=
+GDAWgBT0tuEgHf4prtLkYaWyoiWyyBc1bjALBgNVHRQEBAICDO8wDQYJKoZIhvcN
+AQELBQADggEBALr4VopJYkMfQ97HiyqytcWRY/vgyU/LxOwlH0/1DBSeeObQB0Nj
+uF7vcF2bhbpnxba7gvzOPryudwtbqquf2cl3CJG6MC2D8Nk1XzntDnpxCjVSfsAr
+158zAWPevyiuj3yzFz04mYALt/ZmOJMTF0vyKN8cg5bwfLu3itV6b6vhpuloIhRc
+Hmsbgr3BtCVHkf4vJWq/qKDEMcOhSrJ6wxGCzVyphenewSIbVcogj19cRZDFPWOC
+3sAy/GY3Rz0qK30tDvNbE1uum8gy5ijXFmepJ/lEetRCvrIsxTsXJOj0tqVZfIIQ
+E1YWUZ57TiBBrdS+dTgmRxkN/zaAfYVAIck=
 -----END X509 CRL-----

hooks/timestamping

@@ -553,7 +553,7 @@ download_crls_for_chain() {
       local URL=$(openssl x509 -inform PEM -in $EXTRACTED_CERT -text -noout \
                   | awk '/CRL Distribution Points:/{f=1} f && /URI:/ {print; exit}' \
                   | sed 's/^.*URI://1')
-      if curl "$URL" --output "$CRL_TMP" &> "$OUT_STREAM"; then
+      if curl -L "$URL" --output "$CRL_TMP" &> "$OUT_STREAM"; then
         if openssl crl -in "$CRL_TMP" -inform DER -noout &> "$OUT_STREAM"; then
           openssl crl -in "$CRL_TMP" -inform DER >> "$OUTPUT_FILE"
         elif openssl crl -in "$CRL_TMP" -inform PEM -noout &> "$OUT_STREAM"; then

hooks/validate.sh

@@ -42,6 +42,10 @@ if [[ ! -d "$DIR" ]]; then DIR="$PWD"; fi
 . "$DIR/timestamping"
 
 declare -i MINVERSION=$TIMESTAMPING_VERSION
+declare -i MAX_COMMITS_TO_CHECK=0
+declare -A PROCESSED_COMMIT
+declare -A COMMITS
+declare -A COMMIT_TIMES
 
 while [[ $# -gt 0 ]]; do
   KEY="$1"
@@ -61,6 +65,16 @@ while [[ $# -gt 0 ]]; do
       shift # past argument
       shift # past value
       ;;
+    -max|--maxcommits)
+      INTEGER_REGEX='^[0-9]+$'
+      if ! [[ "$2" =~ $INTEGER_REGEX ]]; then
+        echo_error "$KEY: expected positive integer"
+        exit 1
+      fi
+      MAX_COMMITS_TO_CHECK="$2"
+      shift # past argument
+      shift # past value
+      ;;
     -v|--verbose)
       OUT_STREAM=/dev/stdout
       shift # past argument
@@ -89,6 +103,10 @@ fi
 # tokens, the function will return 0 but echo a warning about the invalid token.
 validate_commit() {
   local COMMIT_HASH="$1"
+  if [[ ${PROCESSED_COMMIT[$COMMIT_HASH]} ]]; then
+    log "validate_commit for $COMMIT_HASH has already been validated"
+    return 0
+  fi
   log "validate_commit for $COMMIT_HASH"
 
   local TIMESTAMP_COMMIT_VERSION
@@ -275,9 +293,11 @@ validate_commit() {
   #assert that all extracted timestamps have been processed
   assert "[ $NUM_PROCESSED -eq $NUM_EXTRACTED ]" "All extracted token must be processed."
 
+  PROCESSED_COMMIT[$COMMIT_HASH]=1
+
   if [ $NUM_VALID -gt 0 ]; then
     if [ $NUM_INVALID -gt 0 ]; then
-      echo_warning "Warning: While commit $COMMIT_HASH contains $NUM_VALID valid timestamp tokens and thus is considered proppely timestamped, it also contains $NUM_INVALID invalid timestamp tokens."
+      echo_warning "Warning: While commit $COMMIT_HASH contains $NUM_VALID valid timestamp tokens and thus is considered properly timestamped, it also contains $NUM_INVALID invalid timestamp tokens."
     fi
     DATE_STRING=$(date -d @"$EARLIEST_VALID_UNIX_TIME")
     echo_info "Commit $COMMIT_HASH, which timestamps commit $PARENT_HASH at $DATE_STRING, contains $NUM_VALID valid timestamp tokens."
@@ -293,6 +313,13 @@ validate_commit() {
 # param1: commit hash
 # returns: 0 if the validation of the commit and all its ancestors succeeded
 validate_commit_and_parents() {
+  # If MAX_COMMITS_TO_CHECK is zero (or a negative number) then that is understood as "infinity".
+  # So finish if we have reached the limit, and if the limit is not "infinity".
+  NUM_COMMITS_CHECKED=${#PROCESSED_COMMIT[@]}
+  if [[ ${NUM_COMMITS_CHECKED} -ge ${MAX_COMMITS_TO_CHECK} ]] && [[ ${MAX_COMMITS_TO_CHECK} -ge 1 ]]; then
+    # enough commits have already been checked, so return early
+    return 0;
+  fi
   local COMMIT_HASH="$1"
   log "validate_commit_and_parents for $COMMIT_HASH"
 
@@ -300,6 +327,7 @@ validate_commit_and_parents() {
   if ! validate_commit "$COMMIT_HASH"; then
     ALL_PASSED=false
   fi
+  NUM_COMMITS_CHECKED=${#PROCESSED_COMMIT[@]}
   local PARENTS=$(git cat-file -p "$COMMIT_HASH" | awk '/^$/{exit} /parent/ {print}' | sed 's/parent //')
   #iterate over all parents of commit
   if [ ! -z "$PARENTS" ]; then
@@ -315,6 +343,64 @@ validate_commit_and_parents() {
   return 1
 }
 
+# Recursive function to find all ancestors of commit
+# param1: commit hash
+# creates an array COMMITS, key is the commit hash, value is the commit time (Unix epoch seconds)
+# the array contains all commits found in all paths from the passed-in commit hash back to the root commit of the repo
+# the array is global so it can be accessed after the function returns
+find_all_commits() {
+  local COMMIT_HASH="$1"
+  log "find_all_commits for $COMMIT_HASH"
+  # git show "ct" format returns the commit time as Unix epoch seconds
+  COMMIT_TIME=$(git show --no-patch --format=%ct "$COMMIT_HASH")
+  COMMITS[$COMMIT_HASH]="${COMMIT_TIME}"
+
+  local PARENTS=$(git cat-file -p "$COMMIT_HASH" | awk '/^$/{exit} /parent/ {print}' | sed 's/parent //')
+  # iterate over all parents of commit
+  if [ ! -z "$PARENTS" ]; then
+    while read PARENT_HASH; do
+      if [[ ${COMMITS[$PARENT_HASH]} ]]; then
+        log "commit $PARENT_HASH has already been processed"
+      else
+        find_all_commits "$PARENT_HASH"
+      fi
+    done <<< $(printf "%s" "$PARENTS")
+  fi
+}
+
+# Validate the commits in the COMMITS array, up to MAX_COMMITS_TO_CHECK
+# returns: 0 if the validation of the commits succeeded
+validate_commits() {
+  ALL_PASSED=true
+  # create an associative array with keys using the Unix epoch commit time and value the commit hash
+  # this array can be easily used to sort in (forward or reverse) order of time
+  for HASH in "${!COMMITS[@]}"; do
+    UNIX_EPOCH_TIME="${COMMITS[$HASH]}"
+    # two commits could have the exact same Unix epoch in seconds
+    # so make that unique by appending an "x" and the hash
+    UNIQUE_KEY="${UNIX_EPOCH_TIME}x${HASH}"
+    COMMIT_TIMES[$UNIQUE_KEY]="${HASH}"
+  done
+  # sort into reverse order
+  SORTED_KEYS=($(printf "%s\n" "${!COMMIT_TIMES[@]}" | sort -r))
+  # process the commits from latest time to oldest time
+  ALL_PASSED=true
+  for ENTRY in "${SORTED_KEYS[@]}"; do
+    COMMIT_HASH=${COMMIT_TIMES[${ENTRY}]}
+    log "${ENTRY} has value ${COMMIT_HASH}"
+    NUM_COMMITS_CHECKED=${#PROCESSED_COMMIT[@]}
+    if [[ ${NUM_COMMITS_CHECKED} -lt ${MAX_COMMITS_TO_CHECK} ]]; then
+      if ! validate_commit "$COMMIT_HASH"; then
+        ALL_PASSED=false
+      fi
+    fi
+  done
+  if [ "$ALL_PASSED" = true ]; then
+    return 0
+  fi
+  return 1
+}
+
 echo_info "Checking repository integrity..."
 #check git repository integrity
 if ! git fsck --full --strict --no-progress --no-dangling "$COMMIT_HASH"; then
@@ -326,10 +412,21 @@ echo ""
 
 echo_info "Validating timestamps. This may take a while..."
 echo ""
-if validate_commit_and_parents "$COMMIT_HASH"; then
-  echo_success "Validation OK: All timestamped commits in the commit history of $COMMIT_HASH contain at least one valid timestamp."
+if [[ ${MAX_COMMITS_TO_CHECK} -ge 1 ]]; then
+  find_all_commits "$COMMIT_HASH"
+  if validate_commits; then
+    echo_success "Validation OK: ${NUM_COMMITS_CHECKED} timestamped commits in the commit history of $COMMIT_HASH contain at least one valid timestamp."
     exit 0
-else
+  else
     echo_error "Validation Failed: There are timestamped commits in the commit history of $COMMIT_HASH which do not contain any valid timestamps."
     exit 1
+  fi
+else
+  if validate_commit_and_parents "$COMMIT_HASH"; then
+    echo_success "Validation OK: All timestamped commits in the commit history of $COMMIT_HASH contain at least one valid timestamp."
+    exit 0
+  else
+    echo_error "Validation Failed: There are timestamped commits in the commit history of $COMMIT_HASH which do not contain any valid timestamps."
+    exit 1
+  fi
 fi

hooks/validate_trustanchors_hash.sh

@@ -0,0 +1,26 @@
+#!/bin/bash
+
+#    Copyright (c) 2024 JankariTech UG
+#    Authors: Artur Neumann
+#    Script to check if the trustanchors have been changed
+
+TRUSTANCHOR_DIR="$1"
+EXPECTED_COMMIT_HASH="$2"
+
+if [[ $# -ne 2 ]]; then
+    echo "Usage: $0 <trustanchor_dir> <expected_commit_hash>"
+    exit 1
+fi
+
+if [ -z "$EXPECTED_COMMIT_HASH" ]; then
+    echo "No expected hash provided"
+    exit 1
+fi
+
+# get the sha256 hash of all files in the trustanchor directory
+ACTUAL_COMMIT_HASH=$(find "$TRUSTANCHOR_DIR" -type f -exec sha256sum {} \; | sort | sha256sum | cut -d ' ' -f 1)
+
+if [ "$EXPECTED_COMMIT_HASH" != "$ACTUAL_COMMIT_HASH" ]; then
+    echo "The trustanchors have been changed, please review the provided hash"
+    exit 1
+fi