From 96b7608966b0e06f757b2a5fc464d53b610ca432 Mon Sep 17 00:00:00 2001
From: root <root@localhost>
Date: Thu, 14 Feb 2013 00:00:00 +0000
Subject: Repack the encrypted remote regularly

Use a simple but slow method of repacking the remote repository.
Download (and verify) all packs not marked 'keep', and repack those into
a new packfile. The new packfile is marked 'keep' with generation 1.
After PUT is called on the manifest, we remove the redundant old
packfiles.

The generation number will allow further iterations of repacking to be
implemented later.
---
 git-remote-gcrypt | 135 +++++++++++++++++++++++++++++++++++++++++++++++++++++-
 1 file changed, 133 insertions(+), 2 deletions(-)

(limited to 'git-remote-gcrypt')

diff --git a/git-remote-gcrypt b/git-remote-gcrypt
index 61f8921..c912f99 100755
--- a/git-remote-gcrypt
+++ b/git-remote-gcrypt
@@ -18,10 +18,14 @@ Repoid=
 Packkey_bytes=33  # 33 random bytes for passphrase, still compatible if changed
 Hashtype=SHA224   # incompatible if changed
 Packpfx="pack :${Hashtype}:"
+Keeppfx="keep :${Hashtype}:"
 
 Branchlist=
 Packlist=
+Keeplist=
 Extension_list=
+Repack_limit=25
+Packlist_delete=
 
 Recipients=
 Signers=
@@ -112,6 +116,17 @@ gitception_put()
 		git update-ref "$Gref" "$commit_id"
 }
 
+# Remove giturl $1, file $2
+# depends on previous GET like put
+gitception_remove()
+{
+	local tree_id= commit_id= tab_="	"
+	# $2 is a filename from the repo format
+	tree_id=$(git ls-tree "$Gref" | xgrep -v -E '\b'"$2"'$' | git mktree) &&
+		commit_id=$(anon_commit "$tree_id") &&
+		git update-ref "$Gref" "$commit_id"
+}
+
 gitception_new_repo()
 {
 	local empty_tree=4b825dc642cb6eb9a060e54bf8d69288fbee4904
@@ -198,6 +213,32 @@ PUTREPO()
 	fi
 }
 
+# For repo $1, delete all newline-separated files in $2
+REMOVE()
+{
+	local fn_=
+	if isurl ssh "$1"
+	then
+		splitcolon "${1#ssh://}"
+		(exec 0>&- ; ssh "$prefix_" "cd $suffix_; rm $2")
+	elif isurl sftp "$1"
+	then
+		# FIXME
+		echo_info "sftp: Ignore remove request $1/$2"
+	elif isurl rsync "$1"
+	then
+		xecho "$2" | rsync -I -W -v -r --delete --include-from=- \
+			--exclude='*' "$Localdir"/ "${1#rsync://}/" >&2
+	elif islocalrepo "$1"
+	then
+		(cd "$1"; rm $2)
+	else
+		for fn_ in $2; do
+			gitception_remove "${1#gitception://}" "$fn_"
+		done
+	fi
+}
+
 CLEAN_FINAL()
 {
 	if isurl ssh "$1" || isurl sftp "$1" || islocalrepo "$1" || isurl rsync "$1"
@@ -372,11 +413,83 @@ ensure_connected()
 
 	Branchlist=$(xecho "$manifest_" | xgrep -E '^[0-9a-f]{40} ')
 	Packlist=$(xecho "$manifest_" | xgrep "^$Packpfx")
+	Keeplist=$(xecho "$manifest_" | xgrep "^keep")
 	Extension_list=$(xecho "$manifest_" | xgrep "^extn ")
 	rcv_repoid=$(xecho "$manifest_" | xgrep "^repo ")
 	iseq "$(repoidstr)" "$rcv_repoid" || echo_die "Repository id mismatch!"
 }
 
+# $1 is new pack id $2 key
+# set did_repack=yes if repacked
+repack_if_needed()
+{
+	local pack_= rcv_id= packline_= premote_= key_= pkeep_= n_=
+
+	# $TmpPack_Encrypted set in caller
+
+	did_repack=no
+	isnonnull "$Packlist" || return 0
+
+	premote_=$(xecho "$Packlist" | cut -f 1-2 -d ' ')
+	pkeep_=$(xecho "$Keeplist" | cut -f 2 -d ' ')
+
+	if isnull "$pkeep_"; then
+		n_=$(xecho "$Packlist" | wc -l)
+	else
+		n_=$(xecho "$Packlist" | grep -v -F -e "$pkeep_" | wc -l)
+	fi
+	if [ $Repack_limit -gt "$n_" ]; then
+		return
+	fi
+	echo_info "Repacking remote $NAME, ..."
+
+	rm -r -f "$Localdir/pack"
+	mkdir -p "$Localdir/pack"
+	DECRYPT "$2" < "$TmpPack_Encrypted" |
+		git index-pack -v --stdin "$Localdir/pack/${1}.pack" >/dev/null
+
+	xecho "$premote_" | while read packline_
+	do
+		isnonnull "$packline_" || continue
+		if isnonnull "$pkeep_" &&
+			xecho "$packline_" | grep -q -F -e "$pkeep_"
+		then
+			continue
+		fi
+		pack_=${packline_#"$Packpfx"}
+		GET "$URL" "$pack_" "$TmpPack_Encrypted"
+		rcv_id=$(pack_hash < "$TmpPack_Encrypted")
+		if isnoteq "$rcv_id" "$pack_"
+		then
+			echo_die "Packfile $pack_ does not match digest!"
+		fi
+		key_=$(xecho "$Packlist" | grep "$pack_" | cut -f 3 -d ' ')
+		DECRYPT "$key_" < "$TmpPack_Encrypted" |
+		git index-pack -v --stdin "$Localdir/pack/${pack_}.pack" >/dev/null
+	done
+	key_=$(genkey "$Packkey_bytes")
+
+	git verify-pack -v "$Localdir"/pack/*.idx | grep -E '^[0-9a-f]{40}' |
+		cut -f 1 -d ' ' |
+		GIT_ALTERNATE_OBJECT_DIRECTORIES=$Localdir \
+		git pack-objects --stdout | ENCRYPT "$key_" > "$TmpPack_Encrypted"
+
+	# Truncate packlist to only the kept packs
+	if isnull "$pkeep_"; then
+		Packlist_delete=$premote_
+		Packlist=
+	else
+		Packlist_delete=$(xecho "$premote_" | xgrep -v -F -e "$pkeep_")
+		Packlist=$(xecho "$Packlist" | xgrep -F -e "$pkeep_")
+	fi
+
+	pack_id=$(pack_hash < "$TmpPack_Encrypted")
+	Packlist=$(append "$Packlist" "$Packpfx$pack_id $key_")
+	Keeplist=$(append "$Keeplist" "$Keeppfx$pack_id 1")
+	rm -r -f "$Localdir/pack"
+	did_repack=yes
+}
+
 do_capabilities()
 {
 	echo_git fetch
@@ -510,7 +623,15 @@ EOF
 	if [ -s "$TmpObjlist" ]
 	then
 		pack_id=$(pack_hash < "$TmpPack_Encrypted")
-		Packlist=$(append "$Packlist" "$Packpfx$pack_id $key_")
+		did_repack=
+		repack_if_needed "$pack_id" "$key_"
+
+		if isnoteq "$did_repack" yes
+		then
+			Packlist=$(append "$Packlist" "$Packpfx$pack_id $key_")
+		fi
+		# else, repack rewrote Packlist
+
 	fi
 
 	# Generate manifest
@@ -519,7 +640,7 @@ EOF
 
 	TmpManifest_Enc="$Localdir/manifest.$$"
 
-	(xecho "$Branchlist"; xecho "$Packlist";
+	(xecho "$Branchlist"; xecho "$Packlist"; xecho "$Keeplist";
 	repoidstr; xecho "$Extension_list") |
 		PRIVENCRYPT "$Recipients" > "$TmpManifest_Enc"
 
@@ -534,6 +655,16 @@ EOF
 	# Upload manifest
 	PUT "$URL" "$Repoid" "$TmpManifest_Enc"
 
+	# Delete packs
+	if isnonnull "$Packlist_delete"; then
+		REMOVE "$URL" "$(xecho "$Packlist_delete" | while read packline_
+		do
+			isnonnull "$packline_" || continue
+			pack_=${packline_#"$Packpfx"}
+			xecho "$pack_"
+		done)"
+	fi
+
 	PUT_FINAL "$URL"
 
 	rm -f "$TmpManifest_Enc"
-- 
cgit v1.2.3