From 84ac13f5a944d0b5b74dfa58c771aac3072220e0 Mon Sep 17 00:00:00 2001 From: root Date: Sat, 10 Nov 2012 09:25:35 +0400 Subject: Refactor fetch and repack --- git-remote-gcrypt | 233 +++++++++++++++++++++++------------------------------- 1 file changed, 101 insertions(+), 132 deletions(-) (limited to 'git-remote-gcrypt') diff --git a/git-remote-gcrypt b/git-remote-gcrypt index 736702f..e584cc3 100755 --- a/git-remote-gcrypt +++ b/git-remote-gcrypt @@ -88,34 +88,21 @@ splitcolon() setvar "$3" "${1#*:}" } -# if $1 contains $2 -contains() -{ - isnull "${1##*"$2"*}" -} - # Pick words from each line # $1 return variable name -# $2 field list "1,2,3" -# $3 input value -pick_fields() +# $2 input value +pick_fields_1_2() { - local f_line_= f_result_= f_mask_= f_ret_var= f_oifs="$IFS" IFS= - f_ret_var=$1 - f_mask_=$2 + local f_ret= f_line= f_var= f_oifs="$IFS" IFS= + f_var=$1 IFS=$Newline - for f_line_ in $3 + for f_line in $2 do IFS=$f_oifs - # split $f_line_ into words and pick them out - set -- $f_line_ - f_line_= - ! contains "$f_mask_" 1 || f_line_=${1:-} - ! contains "$f_mask_" 2 || f_line_="$f_line_ ${2:-}" - ! contains "$f_mask_" 3 || f_line_="$f_line_ ${3:-}" - append_to @f_result_ "${f_line_# }" + set -- $f_line + f_ret=$f_ret"${1:-} ${2:-}"$Newline done - setvar "$f_ret_var" "$f_result_" + setvar "$f_var" "${f_ret#$Newline}" } # Take all lines matching $2 (full line) @@ -557,37 +544,60 @@ ensure_connected() isnull "$r_name" || git config "remote.$r_name.gcrypt-id" "$r_repoid" } -# $1 is the packline pack :SHA256:abc1231.. -fetch_decrypt_pack() +# $1 is the hash type (SHA256 etc) +# $2 the pack id +# $3 the key +get_verify_decrypt_pack() { - local rcv_id= r_key= r_htype= r_pack= - splitcolon "${1#pack :}" @r_htype @r_pack + local rcv_id= tmp_encrypted= + tmp_encrypted="$Localdir/tmp_pack_ENCRYPTED_.$$" + GET "$URL" "$2" "$tmp_encrypted" && + rcv_id=$(gpg_hash "$1" < "$tmp_encrypted") && + iseq "$rcv_id" "$2" || echo_die "Packfile $2 does not match digest!" + DECRYPT "$3" < "$tmp_encrypted" + rm -f "$tmp_encrypted" +} - if isnoteq "$r_htype" SHA256 && isnoteq "$r_htype" SHA224 && - isnoteq "$r_htype" SHA384 && isnoteq "$r_htype" SHA512 - then - echo_die "Packline malformed: $1" - fi - GET "$URL" "$r_pack" "$TmpPack_Encrypted" && - rcv_id=$(gpg_hash "$r_htype" < "$TmpPack_Encrypted") && - iseq "$rcv_id" "$r_pack" || - echo_die "Packfile $r_pack does not match digest!" - filter_to @r_key "pack :${r_htype}:$r_pack *" "$Packlist" - pick_fields @r_key 3 "$r_key" - DECRYPT "$r_key" < "$TmpPack_Encrypted" -} - -# $1 is new pack id $2 key, $3, $4 return variables -# set $3 to yes if repacked -# $4 to list of packfiles to delete -repack_if_needed() +# download all packlines (pack :SHA256:a32abc1231) from stdin (or die) +# $1 destdir (when repack, else "") +get_pack_files() { - local pack_= packline_= premote_= key_= pkeep_= n_= m_= \ - orig_ifs= kline_= r_line= r_list_new= + local pack_id= r_pack_key_line= r_htype= r_pack= key_= + while read -r _ pack_id # </dev/null + xecho "pack $pack_id" >> "$Localdir/have_packs$GITCEPTION" + else + git index-pack -v --stdin "$1/${r_pack}.pack" >/dev/null + fi + done +} + +# Download and unpack remote packfiles +# $1 is objects tmpfile +# $2 return var for list of packfiles to delete +repack_if_needed() +{ + local n_= m_= kline_= r_line= r_keep_packlist= r_del_list= - setvar "$3" no isnonnull "$Packlist" || return 0 if isnonnull "${GCRYPT_FULL_REPACK:-}" @@ -596,63 +606,38 @@ repack_if_needed() Repack_limit=1 fi - pick_fields @premote_ 1,2 "$Packlist" - pick_fields @pkeep_ 2 "$Keeplist" + pick_fields_1_2 @r_del_list "$Packlist" n_=$(line_count "$Packlist") - m_=$(line_count "$pkeep_") - if [ "$Repack_limit" -gt "$(($n_ - $m_))" ]; then + m_=$(line_count "$Keeplist") + if iseq 0 "$(( $Repack_limit < ($n_ - $m_) ))"; then return fi echo_info "Repacking remote $NAME, ..." rm -r -f "$Localdir/pack" mkdir -p "$Localdir/pack" - DECRYPT "$2" < "$TmpPack_Encrypted" | - git index-pack -v --stdin "$Localdir/pack/${1}.pack" >/dev/null - - xecho "$premote_" | while read packline_ - do - isnonnull "$packline_" || continue - if isnonnull "$pkeep_" && - xecho "$packline_" | grep -q -e "$pkeep_" - then - continue - fi - pack_=${packline_#$Packpat} - fetch_decrypt_pack "$packline_" | - git index-pack -v --stdin "$Localdir/pack/${pack_}.pack" >/dev/null - done - key_=$(genkey "$Packkey_bytes") - git verify-pack -v "$Localdir"/pack/*.idx | grep -E '^[0-9a-f]{40}' | - cut -f 1 -d ' ' | - GIT_ALTERNATE_OBJECT_DIRECTORIES=$Localdir \ - git pack-objects --stdout | ENCRYPT "$key_" > "$TmpPack_Encrypted" - - # Truncate packlist to only the kept packs - if isnull "$pkeep_"; then - setvar "$4" "$premote_" - Packlist= - else - setvar "$4" "$(xecho "$premote_" | xgrep -v -e "$pkeep_")" - orig_ifs=$IFS - IFS=$Newline - for kline_ in $pkeep_ + # Split packages to keep and to repack + if isnonnull "$Keeplist"; then + while read -r _ kline_ _ # <> "$1" + + Packlist=$r_keep_packlist + setvar "$2" "$r_del_list" } do_capabilities() @@ -685,23 +670,13 @@ do_list() do_fetch() { - # The PACK id is the hash of the encrypted git packfile. - # We only download packs mentioned in the encrypted manifest, - # and check their digest when received. - local pack_= packline_= pneed_= premote_= + # Download packs in the manifest that don't appear in have_packs + local pneed_= premote_= ensure_connected - if isnull "$Packlist" - then - echo_git # end with blank line - return - fi - - TmpPack_Encrypted="$Localdir/tmp_pack_ENCRYPTED_.$$" - # The `+` for $GITCEPTION is pointless but we will be safe for stacking - pick_fields @premote_ 1,2 "$Packlist" + pick_fields_1_2 @premote_ "$Packlist" if [ -s "$Localdir/have_packs+" ] then pneed_=$(xecho "$premote_" | xgrep -v -x -f "$Localdir/have_packs+") @@ -709,16 +684,8 @@ do_fetch() pneed_=$premote_ fi - xecho "$pneed_" | while read packline_ - do - isnonnull "$packline_" || continue - fetch_decrypt_pack "$packline_" | - git index-pack -v --stdin >/dev/null - # add to local pack list - xecho "${packline_}" >> "$Localdir/have_packs$GITCEPTION" - done + xecho "$pneed_" | get_pack_files - rm -f "$TmpPack_Encrypted" echo_git # end with blank line } @@ -729,8 +696,8 @@ do_push() # Each git packfile is encrypted and then named for the encrypted # file's hash. The manifest is updated with the pack id. # The manifest is encrypted. - local r_revlist= line_= pack_id= key_= obj_= \ - r_did_repack= r_pack_delete= r_src= r_dst= + local r_revlist= line_= pack_id= r_pack_id= key_= obj_= \ + r_pack_delete= r_src= r_dst= tmp_encrypted= tmp_objlist= ensure_connected @@ -763,25 +730,26 @@ do_push() $1 EOF - TmpPack_Encrypted="$Localdir/tmp_pack_ENCRYPTED_.$$" - TmpObjlist="$Localdir/tmp_packrevlist.$$" - key_=$(genkey "$Packkey_bytes") + tmp_encrypted="$Localdir/tmp_pack_ENCRYPTED_.$$" + tmp_objlist="$Localdir/tmp_packrevlist.$$" - xecho "$r_revlist" | git rev-list --objects --stdin -- | - tee "$TmpObjlist" | - git pack-objects --stdout | ENCRYPT "$key_">"$TmpPack_Encrypted" + xecho "$r_revlist" | git rev-list --objects --stdin -- > "$tmp_objlist" # Only send pack if we have any objects to send - if [ -s "$TmpObjlist" ] + if [ -s "$tmp_objlist" ] then - pack_id=$(pack_hash < "$TmpPack_Encrypted") - repack_if_needed "$pack_id" "$key_" @r_did_repack @r_pack_delete + repack_if_needed "$tmp_objlist" @r_pack_delete - if isnoteq "$r_did_repack" yes + key_=$(genkey "$Packkey_bytes") + GIT_ALTERNATE_OBJECT_DIRECTORIES=$Localdir \ + git pack-objects --stdout < "$tmp_objlist" | + ENCRYPT "$key_" > "$tmp_encrypted" + pack_id=$(pack_hash < "$tmp_encrypted") + + append_to @Packlist "pack :${Hashtype}:$pack_id $key_" + if isnonnull "$r_pack_delete" then - append_to @Packlist "pack :${Hashtype}:$pack_id $key_" + append_to @Keeplist "keep :${Hashtype}:$pack_id 1" fi - # else, repack rewrote Packlist - fi # Generate manifest @@ -799,20 +767,21 @@ $Extnlist EOF # Upload pack - if [ -s "$TmpObjlist" ] + if [ -s "$tmp_objlist" ] then - PUT "$URL" "$pack_id" "$TmpPack_Encrypted" + PUT "$URL" "$pack_id" "$tmp_encrypted" fi # Upload manifest PUT "$URL" "$Manifestfile" "$TmpManifest_Enc" - rm -f "$TmpPack_Encrypted" - rm -f "$TmpObjlist" + rm -f "$tmp_encrypted" + rm -f "$tmp_objlist" rm -f "$TmpManifest_Enc" # Delete packs if isnonnull "$r_pack_delete"; then + rm -r -f "$Localdir/pack" REMOVE "$URL" "$(xecho "$r_pack_delete" | while read packline_ do isnonnull "$packline_" || continue -- cgit v1.2.3