diff --git a/creation_repo.sh b/creation_repo.sh index ec278fa..72eaabb 100755 --- a/creation_repo.sh +++ b/creation_repo.sh @@ -52,7 +52,9 @@ if [ ! -d $REPO_NAME ]; then git commit -m"first 1M sample created" git tag start create_random_file 'sample1' '1M' - git add sample1 + touch texte + echo "un soleil bleu">texte + git add sample1 texte git commit -m"second 1M sample created" git branch secondary git checkout secondary diff --git a/notes.odt b/notes.odt index 0d1c8a0..d2f8ff3 100644 Binary files a/notes.odt and b/notes.odt differ diff --git a/performance_tests.sh b/performance_tests.sh index 3f73b61..cb90f4c 100755 --- a/performance_tests.sh +++ b/performance_tests.sh @@ -31,17 +31,40 @@ DESCRIPTION TEST1: --single-branch cloning TEST2: --depth=1 --no-single-branch cloning TEST3: --depth=1 cloning - TEST4: sparse-checking 1M sample0 only cloning + TEST4: --depth=1 with reflog and gc cloning + TEST5: sparse-checking 1M sample0 cloning _________________ - TEST5: classic fetching and merging after addition of a 1M file - TEST6: classic fetching and merging, after removal of a 1M file - TEST7: classic fetching and merging, after addition then removal of a 1M file - TEST8: --depth=1 fetching and merging, after addition of a 1M file - TEST9: --depth=1 fetching and merging, after removal of a 1M file - TEST10: --depth=1 fetching and merging, after addition then removal of 1M a file - TEST17: --depth=1 fetching and merging, after addition of a 1M file in submodule - TEST18: --depth=1 fetching and merging, after removal of a 1M file in submodule - TEST19: --depth=1 fetching and merging, after addition then removal of a 1M file in submodule" + TEST6: classic fetching+checking out after addition of a 1M file + TEST7: classic fetching+checking out after removal of a 1M file + TEST8: classic fetching+checking out after addition then removal of a 1M file + + TEST9: --depth=1 fetching+checking out after addition of a 1M file + TEST10: --depth=1 fetching+checking out after removal of a 1M file + TEST11: --depth=1 fetching+checking out after addition then removal of 1M a file + + TEST12: --depth=1 fetching+checking out with reflog annd gc after addition of a 1M file + TEST13: --depth=1 fetching+checking out with reflog annd gc after removal of a 1M file + TEST14: --depth=1 fetching+checking out with reflog annd gc after addition then removal of a 1M file + + TEST15: --depth=1 fetching+ --reset=hard after addition of a 1M file + TEST16: --depth=1 fetching+ --reset=hard after removal of a 1M file + TEST17: --depth=1 fetching+ --reset=hard after addition then removal of a 1M file + + TEST18: --depth=1 fetching+ --reset=hard and reflog and gc after addition of a 1M file + TEST19: --depth=1 fetching+ --reset=hard and reflog and gc after removal of a 1M file + TEST20: --depth=1 fetching+ --reset=hard and reflog and gc after addition then removal of a 1M file + + TEST21: --depth=1 fetching+checking out after addition of a 1M file in submodule + TEST22: --depth=1 fetching+checking out after removal of a 1M file in submodule + TEST23: --depth=1 fetching+checking out after addition then removal of a 1M file in submodule + + TEST24: --depth=1 fetching+merging -X theirs with reflog and gc after addition of a 1M file + TEST25: --depth=1 fetching+merging -X theirs with reflog and gc after removal of a 1M file + TEST26: --depth=1 fetching+merging -X theirs with reflog and gc after addition then removal of a 1M file + + TEST27: --depth=1 fetching+merging -s ours with reflog and gc after addition of a 1M file + TEST28: --depth=1 fetching+merging -s ours with reflog and gc after removal of a 1M file + TEST29: --depth=1 fetching+merging -s ours with reflog and gc after addition then removal of a 1M file" } #USEFUL FUNCTIONS FOR THE TESTS @@ -109,7 +132,7 @@ test2(){ test3(){ section TEST3 echo "TEST 3: --depth=1 with single-branch (default))" - git clone --recurse-submodules --recurse-submodules --progress --single-branch --no-local --depth=1 $1 &> cloning_text + git clone --recurse-submodules --progress --single-branch --no-local --depth=1 $1 &> cloning_text cd $REPO_NAME run git submodule update --init --recursive --force --remote --progress --depth=1 &> /dev/null cd .. @@ -124,7 +147,7 @@ test3(){ test4(){ section TEST4 echo "TEST 4: --depth=1 with single-branch (default) and reflog and gc" - git clone --recurse-submodules --recurse-submodules --progress --single-branch --no-local --depth=1 $1 &> cloning_text + git clone --recurse-submodules --progress --single-branch --no-local --depth=1 $1 &> cloning_text cd $REPO_NAME run git submodule update --init --recursive --force --remote --progress --depth=1 &> /dev/null git reset --hard @@ -149,7 +172,7 @@ test5(){ #pulling from the remote with sparse-checking enabled run git remote add -f origin ../$1 &> /dev/null run git submodule update --init --recursive --force --depth=1 --remote &> /dev/null - run git fetch --progress --tags --depth=1 origin &> /dev/null + run git fetch --progress --tags --depth=1 --prune --prune-tags origin &> /dev/null git checkout -f origin/main &> /dev/null get_storage_used . echo "memory usage: $mem" @@ -164,7 +187,7 @@ test6(){ section TEST6 run echo 'TEST 6: after addition of a 1M file' #initialization - git clone --recurse-submodules --progress --no-local $1 &> /dev/null + git clone --recurse-submodules --progress --depth=1 --no-local $1 &> /dev/null cd $REPO_NAME #run git submodule update --init --recursive --force --depth=1 get_storage_used . @@ -177,7 +200,7 @@ test6(){ cd ../../$REPO_NAME #fetching run git submodule update --init --recursive --force --remote &> /dev/null - run git fetch --progress --tags origin &> /dev/null + run git fetch --progress --tags --prune --prune-tags origin &> /dev/null git checkout -f origin/main &> /dev/null get_storage_used . mem_after=$mem @@ -192,7 +215,7 @@ test7(){ section TEST7 run echo 'TEST 7: after removal of a 1M file' #initialization - git clone --recurse-submodules --progress --no-local $1 &> /dev/null + git clone --recurse-submodules --progress --depth=1 --no-local $1 &> /dev/null cd $REPO_NAME get_storage_used . mem_before=$mem @@ -204,7 +227,7 @@ test7(){ cd ../../$REPO_NAME #fetching run git submodule update --init --recursive --force --remote &> /dev/null - run git fetch --progress --tags origin &> /dev/null + run git fetch --progress --tags --prune --prune-tags origin &> /dev/null git checkout -f origin/main &> /dev/null get_storage_used . mem_after=$mem @@ -220,7 +243,7 @@ test8(){ section TEST8 run echo 'TEST 8: after addition then removal of a 1M file' #initialization - git clone --recurse-submodules --progress --no-local $1 &> /dev/null + git clone --recurse-submodules --progress --depth=1 --no-local $1 &> /dev/null cd $REPO_NAME get_storage_used . mem_before=$mem @@ -235,7 +258,7 @@ test8(){ cd ../../$REPO_NAME #fetching run git submodule update --init --recursive --force --remote &> /dev/null - run git fetch --progress --tags origin &> /dev/null + run git fetch --progress --tags --prune --prune-tags origin &> /dev/null git checkout -f origin/main &> /dev/null get_storage_used . mem_after=$mem @@ -252,9 +275,8 @@ test9(){ section TEST9 run echo 'TEST 9: after addition of a 1M file' #initialization - git clone --recurse-submodules --progress --no-local $1 &> /dev/null + git clone --recurse-submodules --progress --depth=1 --no-local $1 &> /dev/null cd $REPO_NAME - echo "un truc écrit en plus" >> untexte get_storage_used . mem_before=$mem #modification of the remote repo @@ -264,7 +286,7 @@ test9(){ run git commit --quiet -m"fourth 1M sample created" cd ../../$REPO_NAME run git submodule update --init --recursive --force --depth=1 --remote &> /dev/null - run git fetch --progress --tags --depth=1 origin &> /dev/null + run git fetch --progress --tags --depth=1 --prune --prune-tags origin &> /dev/null git checkout -f origin/main &> /dev/null get_storage_used . mem_after=$mem @@ -279,7 +301,7 @@ test10(){ section TEST10 run echo 'TEST 10: after removal of a 1M file' #initialization - git clone --recurse-submodules --progress --no-local $1 &> /dev/null + git clone --recurse-submodules --progress --depth=1 --no-local $1 &> /dev/null cd $REPO_NAME get_storage_used . mem_before=$mem @@ -291,7 +313,7 @@ test10(){ cd ../../$REPO_NAME #fetching run git submodule update --init --recursive --force --depth=1 --remote &> /dev/null - git fetch --progress --tags --prune --depth=1 origin &> /dev/null + git fetch --progress --tags --depth=1 --prune --prune-tags origin &> /dev/null git checkout -f origin/main &> /dev/null get_storage_used . mem_after=$mem @@ -305,7 +327,7 @@ test10(){ test11(){ section TEST11 run echo 'TEST 11: after addition then removal of a 1M file' - git clone --recurse-submodules --progress --no-local $1 &> /dev/null + git clone --recurse-submodules --progress --depth=1 --no-local $1 &> /dev/null cd $REPO_NAME get_storage_used . mem_before=$mem @@ -320,7 +342,7 @@ test11(){ cd ../../$REPO_NAME #fetching run git submodule update --init --recursive --force --depth=1 --remote &> /dev/null - run git fetch --progress --tags --depth=1 origin &> /dev/null + run git fetch --progress --tags --depth=1 --prune --prune-tags origin &> /dev/null git checkout -f origin/main &> /dev/null get_storage_used . mem_after=$mem @@ -337,9 +359,8 @@ test12(){ section TEST12 run echo 'TEST 12: after addition of a 1M file' #initialization - git clone --recurse-submodules --progress --no-local $1 &> /dev/null + git clone --recurse-submodules --progress --depth=1 --no-local $1 &> /dev/null cd $REPO_NAME - echo "un truc écrit en plus" >> untexte get_storage_used . mem_before=$mem #modification of the remote repo @@ -349,7 +370,7 @@ test12(){ run git commit --quiet -m"fourth 1M sample created" cd ../../$REPO_NAME run git submodule update --init --recursive --force --depth=1 --remote &> /dev/null - run git fetch --progress --tags --depth=1 origin &> /dev/null + run git fetch --progress --tags --depth=1 --prune --prune-tags origin &> /dev/null git checkout -f origin/main &> /dev/null git reflog expire --expire=now --all &> /dev/null git gc --aggressive --prune=now &> /dev/null @@ -366,7 +387,7 @@ test13(){ section TEST13 run echo 'TEST 13: after removal of a 1M file' #initialization - git clone --recurse-submodules --progress --no-local $1 &> /dev/null + git clone --recurse-submodules --progress --depth=1 --no-local $1 &> /dev/null cd $REPO_NAME get_storage_used . mem_before=$mem @@ -378,7 +399,7 @@ test13(){ cd ../../$REPO_NAME #fetching run git submodule update --init --recursive --force --depth=1 --remote &> /dev/null - git fetch --progress --tags --prune --depth=1 origin &> /dev/null + git fetch --progress --tags --depth=1 --prune --prune-tags origin &> /dev/null git checkout -f origin/main &> /dev/null git reflog expire --expire=now --all &> /dev/null git gc --aggressive --prune=now &> /dev/null @@ -394,7 +415,7 @@ test13(){ test14(){ section TEST14 run echo 'TEST 14: after addition then removal of a 1M file' - git clone --recurse-submodules --progress --no-local $1 &> /dev/null + git clone --recurse-submodules --progress --depth=1 --no-local $1 &> /dev/null cd $REPO_NAME get_storage_used . mem_before=$mem @@ -409,7 +430,7 @@ test14(){ cd ../../$REPO_NAME #fetching run git submodule update --init --recursive --force --depth=1 --remote &> /dev/null - run git fetch --progress --tags --depth=1 origin &> /dev/null + run git fetch --progress --tags --depth=1 --prune --prune-tags origin &> /dev/null git checkout -f origin/main &> /dev/null git reflog expire --expire=now --all &> /dev/null git gc --aggressive --prune=now &> /dev/null @@ -428,9 +449,8 @@ test15(){ section TEST15 run echo 'TEST 15: after addition of a 1M file' #initialization - git clone --recurse-submodules --progress --no-local $1 &> /dev/null + git clone --recurse-submodules --progress --depth=1 --no-local $1 &> /dev/null cd $REPO_NAME - echo "un truc écrit en plus" >> untexte get_storage_used . mem_before=$mem #modification of the remote repo @@ -440,7 +460,7 @@ test15(){ run git commit --quiet -m"fourth 1M sample created" cd ../../$REPO_NAME run git submodule update --init --recursive --force --depth=1 --remote &> /dev/null - run git fetch --progress --tags --depth=1 origin &> /dev/null + run git fetch --progress --tags --depth=1 --prune --prune-tags origin &> /dev/null git reset --hard origin/main &> /dev/null get_storage_used . mem_after=$mem @@ -455,7 +475,7 @@ test16(){ section TEST16 run echo 'TEST 16: after removal of a 1M file' #initialization - git clone --recurse-submodules --progress --no-local $1 &> /dev/null + git clone --recurse-submodules --progress --depth=1 --no-local $1 &> /dev/null cd $REPO_NAME get_storage_used . mem_before=$mem @@ -467,7 +487,7 @@ test16(){ cd ../../$REPO_NAME #fetching run git submodule update --init --recursive --force --depth=1 --remote &> /dev/null - git fetch --progress --tags --prune --depth=1 origin &> /dev/null + git fetch --progress --tags --depth=1 --prune --prune-tags origin &> /dev/null git reset --hard origin/main &> /dev/null get_storage_used . mem_after=$mem @@ -481,7 +501,7 @@ test16(){ test17(){ section TEST17 run echo 'TEST 17: after addition then removal of a 1M file' - git clone --recurse-submodules --progress --no-local $1 &> /dev/null + git clone --recurse-submodules --progress --depth=1 --no-local $1 &> /dev/null cd $REPO_NAME get_storage_used . mem_before=$mem @@ -496,7 +516,7 @@ test17(){ cd ../../$REPO_NAME #fetching run git submodule update --init --recursive --force --depth=1 --remote &> /dev/null - run git fetch --progress --tags --depth=1 origin &> /dev/null + run git fetch --progress --tags --depth=1 --prune --prune-tags origin &> /dev/null git reset --hard origin/main &> /dev/null get_storage_used . mem_after=$mem @@ -511,7 +531,7 @@ test17(){ test18(){ section TEST18 run echo 'TEST 18: after addition of a 1M file' - git clone --recurse-submodules --progress --no-local $1 &> /dev/null + git clone --recurse-submodules --progress --depth=1 --no-local $1 &> /dev/null cd $REPO_NAME get_storage_used . mem_before=$mem @@ -523,7 +543,7 @@ test18(){ cd ../../$REPO_NAME #fetching run git submodule update --init --recursive --force --depth=1 --remote &> /dev/null - run git fetch --progress --tags --depth=1 origin &> /dev/null + run git fetch --progress --tags --depth=1 --prune --prune-tags origin &> /dev/null git reset --hard origin/main &> /dev/null git reflog expire --expire=now --all &> /dev/null git gc --aggressive --prune=now &> /dev/null @@ -540,7 +560,7 @@ test19(){ section TEST19 run echo 'TEST 19: after removal of a 1M file' #initialization - git clone --recurse-submodules --progress --no-local $1 &> /dev/null + git clone --recurse-submodules --progress --depth=1 --no-local $1 &> /dev/null cd $REPO_NAME get_storage_used . mem_before=$mem @@ -552,7 +572,7 @@ test19(){ cd ../../$REPO_NAME #fetching run git submodule update --init --recursive --force --depth=1 --remote &> /dev/null - git fetch --progress --tags --prune --depth=1 origin &> /dev/null + git fetch --progress --tags --depth=1 --prune --prune-tags origin &> /dev/null git reset --hard origin/main &> /dev/null git reflog expire --expire=now --all &> /dev/null git gc --aggressive --prune=now &> /dev/null @@ -568,7 +588,7 @@ test19(){ test20(){ section TEST20 run echo 'TEST 20: after addition then removal of a 1M file' - git clone --recurse-submodules --progress --no-local $1 &> /dev/null + git clone --recurse-submodules --progress --depth=1 --no-local $1 &> /dev/null cd $REPO_NAME get_storage_used . mem_before=$mem @@ -583,7 +603,7 @@ test20(){ cd ../../$REPO_NAME #fetching run git submodule update --init --recursive --force --depth=1 --remote &> /dev/null - run git fetch --progress --tags --depth=1 origin &> /dev/null + run git fetch --progress --tags --depth=1 --prune --prune-tags origin &> /dev/null git reset --hard origin/main &> /dev/null git reflog expire --expire=now --all &> /dev/null git gc --aggressive --prune=now &> /dev/null @@ -601,7 +621,7 @@ test21(){ section TEST21 run echo 'TEST 21: after addition of a 1M file' if [ "$WITH_SUBMODULE" = "true" ]; then - git clone --recurse-submodules --progress --no-local $1 &> /dev/null + git clone --recurse-submodules --progress --depth=1 --no-local $1 &> /dev/null cd $REPO_NAME get_storage_used . mem_before=$mem @@ -614,7 +634,7 @@ test21(){ cd ../../$REPO_NAME #fetching run git submodule update --init --recursive --force --depth=1 --remote &> /dev/null - run git fetch --progress --tags --depth=1 origin &> /dev/null + run git fetch --progress --tags --depth=1 --prune --prune-tags origin &> /dev/null git checkout -f origin/main &> /dev/null get_storage_used . mem_after=$mem @@ -633,7 +653,7 @@ test22(){ section TEST22 run echo 'TEST 22: after removal of a 1M file' if [ "$WITH_SUBMODULE" = "true" ]; then - git clone --recurse-submodules --progress --no-local $1 &> /dev/null + git clone --recurse-submodules --progress --depth=1 --no-local $1 &> /dev/null cd $REPO_NAME get_storage_used . mem_before=$mem @@ -646,7 +666,7 @@ test22(){ cd ../../$REPO_NAME #fetching run git submodule update --init --recursive --force --depth=1 --remote &> /dev/null - run git fetch --progress --tags --depth=1 origin &> /dev/null + run git fetch --progress --tags --depth=1 --prune --prune-tags origin &> /dev/null git checkout -f origin/main &> /dev/null get_storage_used . mem_after=$mem @@ -665,7 +685,7 @@ test23(){ section TEST23 run echo 'TEST 23: after addition then removal of a 1M file' if [ "$WITH_SUBMODULE" = "true" ]; then - git clone --recurse-submodules --progress --no-local $1 &> /dev/null + git clone --recurse-submodules --progress --depth=1 --no-local $1 &> /dev/null cd $REPO_NAME get_storage_used . mem_before=$mem @@ -681,7 +701,7 @@ test23(){ cd ../../$REPO_NAME #fetching run git submodule update --init --recursive --force --depth=1 --remote &> /dev/null - run git fetch --progress --tags --depth=1 origin &> /dev/null + run git fetch --progress --tags --depth=1 --prune --prune-tags origin &> /dev/null git checkout -f origin/main &> /dev/null get_storage_used . mem_after=$mem @@ -696,6 +716,199 @@ test23(){ fi } +# --depth=1 fetching+merging -X theirs with reflog and gc +test24(){ + section TEST24 + run echo 'TEST 24: after addition of a 1M file' + #initialization + git clone --recurse-submodules --progress --depth=1 --no-local $1 &> /dev/null + cd $REPO_NAME + get_storage_used . + mem_before=$mem + #modification of the remote repo + cd ../$REMOTE + create_random_file 'sample5' '1M' #adding a 1M file + run git add sample5 + run git commit --quiet -m"fourth 1M sample created" + cd ../../$REPO_NAME + run git submodule update --init --recursive --force --depth=1 --remote &> /dev/null + run git fetch --progress --tags --depth=1 --prune --prune-tags origin &> /dev/null + git merge -X theirs --allow-unrelated-histories &> /dev/null + git reflog expire --expire=now --all &> /dev/null + git gc --aggressive --prune=now &> /dev/null + get_storage_used . + mem_after=$mem + mem=$(($mem_after-$mem_before)) + echo "memory usage: +$mem" + cd ../$REMOTE + git reset --hard -q HEAD~1 + cd ../.. + rm -rf performance_testing +} +test25(){ + section TEST25 + run echo 'TEST 25: after removal of a 1M file' + #initialization + git clone --recurse-submodules --progress --depth=1 --no-local $1 &> /dev/null + cd $REPO_NAME + get_storage_used . + mem_before=$mem + #modification of the remote repo + cd ../$REMOTE + run rm sample0 + run git add sample0 + run git commit --quiet -m"1M sample0 deleted" + cd ../../$REPO_NAME + #fetching + run git submodule update --init --recursive --force --depth=1 --remote &> /dev/null + git fetch --progress --tags --depth=1 --prune --prune-tags origin &> /dev/null + git merge -X theirs --allow-unrelated-histories &> /dev/null + git reflog expire --expire=now --all &> /dev/null + git gc --aggressive --prune=now &> /dev/null + get_storage_used . + mem_after=$mem + mem=$(($mem_after-$mem_before)) + echo "memory usage: $mem" + cd ../$REMOTE + git reset --hard -q HEAD~1 + cd ../.. + rm -rf performance_testing +} +test26(){ + section TEST26 + run echo 'TEST 26: after addition then removal of a 1M file' + git clone --recurse-submodules --progress --depth=1 --no-local $1 &> /dev/null + cd $REPO_NAME + get_storage_used . + mem_before=$mem + #modification of the remote repo + cd ../$REMOTE + create_random_file 'sample5' '1M' #adding a 1M file + run git add sample5 + run git commit --quiet -m"fourth 1M sample created" + run rm sample5 + run git add sample5 + run git commit --quiet -m"1M "sample5" deleted" + cd ../../$REPO_NAME + #fetching + run git submodule update --init --recursive --force --depth=1 --remote &> /dev/null + run git fetch --progress --tags --depth=1 --prune --prune-tags origin &> /dev/null + git merge -X theirs --allow-unrelated-histories &> /dev/null + git reflog expire --expire=now --all &> /dev/null + git gc --aggressive --prune=now &> /dev/null + get_storage_used . + mem_after=$mem + mem=$(($mem_after-$mem_before)) + echo "memory usage: $mem" + cd ../$REMOTE + git reset --hard -q HEAD~2 + cd ../.. + rm -rf performance_testing +} + +# --depth=1 fetching+merging -s ours with reflog and gc +test27(){ + section TEST27 + run echo 'TEST 27: after addition of a 1M file' + #initialization + git clone --recurse-submodules --progress --depth=1 --no-local $1 &> /dev/null + cd $REPO_NAME + get_storage_used . + mem_before=$mem + #modification of the remote repo + cd ../$REMOTE + create_random_file 'sample5' '1M' #adding a 1M file + run git add sample5 + run git commit --quiet -m"fourth 1M sample created" + cd ../../$REPO_NAME + run git submodule update --init --recursive --force --depth=1 --remote &> /dev/null + run git fetch --progress --tags --depth=1 --prune --prune-tags origin &> /dev/null + git switch -c temp origin/main &> /dev/null #creating a temporary branch identical to origin/main + git merge -s ours --allow-unrelated-histories main &> /dev/null + git checkout main &> /dev/null + git merge --allow-unrelated-histories temp &> /dev/null + git branch -D temp &> /dev/null + git reflog expire --expire=now --all &> /dev/null + git gc --aggressive --prune=now &> /dev/null + get_storage_used . + mem_after=$mem + mem=$(($mem_after-$mem_before)) + echo "memory usage: +$mem" + cd ../$REMOTE + git reset --hard -q HEAD~1 + cd ../.. + rm -rf performance_testing +} +test28(){ + section TEST28 + run echo 'TEST 28: after removal of a 1M file' + #initialization + git clone --recurse-submodules --progress --depth=1 --no-local $1 &> /dev/null + cd $REPO_NAME + get_storage_used . + mem_before=$mem + #modification of the remote repo + cd ../$REMOTE + run rm sample0 + run git add sample0 + run git commit --quiet -m"1M sample0 deleted" + cd ../../$REPO_NAME + #fetching + run git submodule update --init --recursive --force --depth=1 --remote &> /dev/null + git fetch --progress --tags --depth=1 --prune --prune-tags origin &> /dev/null + git switch -c temp origin/main &> /dev/null #creating a temporary branch identical to origin/main + git merge -s ours --allow-unrelated-histories main &> /dev/null + git checkout main &> /dev/null + git merge --allow-unrelated-histories temp &> /dev/null + git branch -D temp &> /dev/null + git reflog expire --expire=now --all &> /dev/null + git gc --aggressive --prune=now &> /dev/null + get_storage_used . + mem_after=$mem + mem=$(($mem_after-$mem_before)) + echo "memory usage: $mem" + cd ../$REMOTE + git reset --hard -q HEAD~1 + cd ../.. + rm -rf performance_testing +} +test29(){ + section TEST29 + run echo 'TEST 29: after addition then removal of a 1M file' + git clone --recurse-submodules --progress --depth=1 --no-local $1 &> /dev/null + cd $REPO_NAME + get_storage_used . + mem_before=$mem + #modification of the remote repo + cd ../$REMOTE + create_random_file 'sample5' '1M' #adding a 1M file + run git add sample5 + run git commit --quiet -m"fourth 1M sample created" + run rm sample5 + run git add sample5 + run git commit --quiet -m"1M "sample5" deleted" + cd ../../$REPO_NAME + #fetching + run git submodule update --init --recursive --force --depth=1 --remote &> /dev/null + run git fetch --progress --tags --depth=1 --prune --prune-tags origin &> /dev/null + git switch -c temp origin/main &> /dev/null #creating a temporary branch identical to origin/main + git merge -s ours --allow-unrelated-histories main &> /dev/null + git checkout main &> /dev/null + #git merge --allow-unrelated-histories temp &> /dev/null + git branch -D temp &> /dev/null + git reflog expire --expire=now --all &> /dev/null + git gc --aggressive --prune=now &> /dev/null + get_storage_used . + mem_after=$mem + mem=$(($mem_after-$mem_before)) + echo "memory usage: $mem" + cd ../$REMOTE + git reset --hard -q HEAD~2 + cd ../.. + rm -rf performance_testing +} + + while getopts ":hn:a" option; do case $option in h) # display Help @@ -719,30 +932,39 @@ if [ "$ALL_TESTS" = "true" ]; then test4 $REMOTE test5 $REMOTE echo $(section "Tests on the updating of the repository") - section "classic fetching" + section "classic fetching+checking out" test6 $REMOTE test7 $REMOTE test8 $REMOTE - echo $(section "fetching with --depth=1") + echo $(section "fetching+checking out with --depth=1") test9 $REMOTE test10 $REMOTE test11 $REMOTE - echo $(section "--depth=1 fetching with reflog and gc") + echo $(section "--depth=1 fetching+checking out reflog and gc") test12 $REMOTE test13 $REMOTE test14 $REMOTE - echo $(section "--depth=1 fetching with reset --hard") + echo $(section "--depth=1 fetching+ reset --hard") test15 $REMOTE test16 $REMOTE test17 $REMOTE - echo $(section "--depth=1 fetching with reset --hard and reflog and gc") + echo $(section "--depth=1 fetching+ reset --hard and reflog and gc") test18 $REMOTE test19 $REMOTE test20 $REMOTE - echo $(section "--depth=1 fetching after modification applied in submodule") + echo $(section "--depth=1 fetching+checking out after modification applied in submodule") test21 $REMOTE test22 $REMOTE test23 $REMOTE + echo $(section "--depth=1 fetching+merging -X theirs with reflog and gc") + test24 $REMOTE + test25 $REMOTE + test26 $REMOTE + echo $(section "--depth=1 fetching+merging -s ours with reflog and gc") + test27 $REMOTE + test28 $REMOTE + test29 $REMOTE + elif [ -n "$TEST_NUM" ]; then case $TEST_NUM in @@ -774,8 +996,40 @@ elif [ -n "$TEST_NUM" ]; then test12 $REMOTE;; 13) test13 $REMOTE;; + 14) + test14 $REMOTE;; + 15) + test15 $REMOTE;; + 16) + test16 $REMOTE;; + 17) + test17 $REMOTE;; + 18) + test18 $REMOTE;; + 19) + test19 $REMOTE;; + 20) + test20 $REMOTE;; 21) test21 $REMOTE;; + 22) + test22 $REMOTE;; + 23) + test23 $REMOTE;; + 24) + test24 $REMOTE;; + 25) + test25 $REMOTE;; + 26) + test26 $REMOTE;; + 26) + test26 $REMOTE;; + 27) + test27 $REMOTE;; + 28) + test28 $REMOTE;; + 29) + test29 $REMOTE;; *) echo "Error: Invalid test number" die;; diff --git a/readme.md b/readme.md index db030d8..f2eed5c 100644 --- a/readme.md +++ b/readme.md @@ -1,40 +1,34 @@ ## Testing of differeng cloning methods Our objective is to find the least-consuming method in terms of memory and bandwidth resources. We are interested in cloning one specific state of the repository. We are not interested in its history or the possibility to change it from the server where it has been cloned. The first step is done by the script creation_repo.sh which creates an adequate repository to act as a local remote. The testing in itself is done by performance_tests.sh. -NB:The large file storage is not tested by this script because it relies on additional remote stockage and we are interested in reducing the overall stockage for environmental purposes. ## Creation of the test repository -The script creation_repo.sh creates a performance_testing repository. +The script creation_repo.sh creates a remote/performance_testing repository. NAME creation_repo.sh SYNOPSIS - creation_repo.sh [-h] + creation_repo.sh [-h] [-s] DESCRIPTION - It creates a remote directory in the current directory, then create a remote/performance_testing git repository. This git repository is filled with randomly generated binary. + This script creates a ./remote directory in the current directory, then creates a remote/performance_testing git repository. + This git repository is filled with randomly generated binary files described in the readme.md. OPTIONS -h prints the help. + -s creates a submodule remote/submodule_for_performance_testing and includes it in remote/performance_testing. " Here is a history of the commits: ### branch main -commit f665df376fa57880b5edfb85b156c7703601980e (tag: start) -sample0 1M created -commit a17812c13a69986c1103e0d06ff0a0b59a3a4a63 -sample1 1M created -commit bd9d8e9421c3253abba47b9def480f1b2d595568 -sample3 1M created -commit 0e599f3b47bf3200dc6e2734df0c6e655c0d8dde -sample4 5M created -commit effb3cab262e6e1e8242e524e1862b6f77d7ad38 -sample4 5M deleted +commit sample0 1M created (tag: start) +commit sample1 1M created +commit sample3 1M created +commit sample4 5M created +commit sample4 5M deleted +[if -s is selected] commit adding submodule_for_performance_testing module ### branch secondary -commit 643cdbbda81f4c4f26513b9d6ecca3d436e97040 -sample0 1M created -commit b4bac1f2a2c1b4ab325751748b9496cccc65b082 -sample1 1M created -commit e69695f08e79c160045b3319297597a9d8c9b513 -sample2 500K created +commit sample0 1M created +commit sample1 1M created +commit sample2 500K created -Which gives us the latest state: +Which gives the latest state: ### branch main sample0 1M sample1 1M @@ -44,8 +38,16 @@ sample0 1M sample1 1M sample2 500K +If the -s option is selected, submodule_for_performance_testing is created with this history of commits: +### branch main +commit first 1M sample created + +which gives the latest state: +### branch main +sub_sample0 + ## Testing -The script performance_tests.sh measures memory and bandwidth usage for different git commands. It sources creation_repo.sh in order to have an adequate repository to test on. It also sources the script driglibash-base. +The script performance_tests.sh measures memory and bandwidth usage for different git commands. It sources the script driglibash-base. It executes creation_repo.sh in order to have an adequate repository to test on. NAME performance_tests.sh SYNOPSIS @@ -55,13 +57,42 @@ OPTIONS -n number executes test number -h prints the help. DESCRIPTION - This script is in writing. It allows you to measure memory and bandwidth usage. The first four test different cloning methods. Te following change the local remote by adding a 1M "sample5" file before testing fetching and merging commands. + This script is in writing. It allows you to measure memory and bandwidth usage. The first four test different cloning methods. Te following apply changes to the local remote before testing fetching and merging commands. TEST0: classic cloning - TEST1: --single-branch - TEST2: --depth=1 --no-single-branch - TEST3: --depth=1 - TEST4: sparse-checking 1M sample0 only - _________________________________________ - TEST5: classic fetching and merging - TEST6: - \ No newline at end of file + TEST1: --single-branch cloning + TEST2: --depth=1 --no-single-branch cloning + TEST3: --depth=1 cloning + TEST4: --depth=1 with reflog and gc cloning + TEST5: sparse-checking 1M sample0 cloning + _________________ + TEST6: classic fetching+checking out after addition of a 1M file + TEST7: classic fetching+checking out after removal of a 1M file + TEST8: classic fetching+checking out after addition then removal of a 1M file + + TEST9: --depth=1 fetching+checking out after addition of a 1M file + TEST10: --depth=1 fetching+checking out after removal of a 1M file + TEST11: --depth=1 fetching+checking out after addition then removal of 1M a file + + TEST12: --depth=1 fetching+checking out with reflog annd gc after addition of a 1M file + TEST13: --depth=1 fetching+checking out with reflog annd gc after removal of a 1M file + TEST14: --depth=1 fetching+checking out with reflog annd gc after addition then removal of a 1M file + + TEST15: --depth=1 fetching+ --reset=hard after addition of a 1M file + TEST16: --depth=1 fetching+ --reset=hard after removal of a 1M file + TEST17: --depth=1 fetching+ --reset=hard after addition then removal of a 1M file + + TEST18: --depth=1 fetching+ --reset=hard and reflog and gc after addition of a 1M file + TEST19: --depth=1 fetching+ --reset=hard and reflog and gc after removal of a 1M file + TEST20: --depth=1 fetching+ --reset=hard and reflog and gc after addition then removal of a 1M file + + TEST21: --depth=1 fetching+checking out after addition of a 1M file in submodule + TEST22: --depth=1 fetching+checking out after removal of a 1M file in submodule + TEST23: --depth=1 fetching+checking out after addition then removal of a 1M file in submodule + + TEST24: --depth=1 fetching+merging -X theirs with reflog and gc after addition of a 1M file + TEST25: --depth=1 fetching+merging -X theirs with reflog and gc after removal of a 1M file + TEST26: --depth=1 fetching+merging -X theirs with reflog and gc after addition then removal of a 1M file + + TEST27: --depth=1 fetching+merging -s ours with reflog and gc after addition of a 1M file + TEST28: --depth=1 fetching+merging -s ours with reflog and gc after removal of a 1M file + TEST29: --depth=1 fetching+merging -s ours with reflog and gc after addition then removal of a 1M file" \ No newline at end of file diff --git a/texte_explicatif.txt b/texte_explicatif.txt index 4639368..d82d68a 100644 --- a/texte_explicatif.txt +++ b/texte_explicatif.txt @@ -11,7 +11,7 @@ Comment minimiser la consommation en ressources mémoire et flux de données d'u Le dépôt Git créé n'enverra aucune donnée au remote. Il n'aura aucune utilité de l'historique. - Il pourra éventuellement conserver certains fichiers locaux en plus de ses clonages Git. Il incluera les éventuels submodules. Il peut vouloir télécharger un commit d'une certaine branche ou tag. + Il pourra éventuellement conserver certains fichiers locaux en plus de ses clonages Git. En cas de conflit, le remote aura toujours raison. Il incluera les éventuels submodules. Il peut vouloir télécharger un commit d'une certaine branche ou tag. # Procédé @@ -78,6 +78,7 @@ git gc --aggressive --prune=now aggressive invoque repack et prend plus de temps. repack défait et refait les packs, qui sont des unités de compression. +Cette combinaison ne permet pas de garder des fichiers compilés, néanmoins elle est extrêmement efficace en terme de gains de mémoire. # Détails Voici un résumé de différentes pistes explorées afin de réduire l'empreinte du dépôt Git. @@ -98,5 +99,157 @@ C'est un mécanisme très intéressant, que nous ne retenons pas pour la même ## Suppression de l'historique L'usage de la commande git filter-branch est déconseillé par la documentation Git. Elle présente plusieurs failles de sécurité et de performance. Elle permet de réécrire l'historique des branches grâce à des filtres. +La librairie Java repo-cleaner fonctionne, néanmoins la documentation Git estime que la librairie Python filter-repo est plus rapide et plus sûre. Nous ne souhaitons pas devoir installer Python ou Java donc ne creusons pas plus ces deux possibilités. + +Nous souhaitons supprimer tout l'historique sans filtrer, donc la commande git fetch --depth=1 suivie d'un git checkout ou d'un git reset nous convient. + +## checkout ? merge ? reset ? +Une fois que l'on a fetched les modifications dans notre dossier local remote/, quelle est la meilleure façon de les appliquer à notre index et working directory ? + +Une première idée est d'utiliser git merge. On ne souhaite pas résoudre de conflits manuellement. Le remote doit toujours prévaloir sur les différences locales. +Les commandes git merge -s ours (applique la stratégie ours) et git merge -X theirs (applique la stratégie ort avec l'option theirs) sont intéressantes. +### git merge -X theirs +Cette commande applique une stratégie ort qui en cas de conflit, donne la prévalence à theirs. +Néanmoins, puisque l'on travaille en --depth=1, les deux branches n'ont pas d'ancêtre commun, et on doit d'ailleurs fournir l'option --allow-unrelated-histories. L'absence d'ancêtre commun empêche Git de reconnaître les similitudes à l'intérieur d'un même fichier. N'importe quelle modification d'un fichier tracé sur ours, même sur une nouvelle ligne, causera ainsi un conflit et sera écrasée. Cette commande permet tout de même de sauvegarder les fichiers nouvellement créés sur ours, qu'ils soient committés ou non. +Elle peut donc se montrer intéressante dans notre cas. +Un gros inconvénient de cette commande est en cas de délétion d'un fichier sur theirs : il ne sera pas supprimé sur ours. + +### git merge -s ours +git merge -s ours va ignorer tous les changements et créations de fichiers committés sur theirs. Elle va également ignorer les modifications non committées. En revanche, les créations de fichier non-committées sont conservées. C'est le même résultat qu'avec la commande git reset --hard. +C'est une option intéressante dans notre cas, car on pourrait vouloir sauvegarder uniquement des fichiers de compilation non committés. +Comme l'option git merge -s theirs n'existe pas, on doit faire une petite manipulation : +#on veut merge origin/main sur main, en donnant la prévalence à origin/main +#création d'une nouvelle branche temporaire temp que l'on check out, sourcée sur origin/main +git switch -c temp origin/main +#merge de main sur temp, en donnant la prévalence à temp qui est identique à origin/main +git merge -s ours --allow-unrelated-histories main +#on retourne sur main +git checkout main +#on merge temp. +git merge --allow-unrelated-histories temp +#suppression de temp +git branch -D temp + +### git checkout –force -B main origin/main + +Cette commande est équivalente à git merge -s ours décrite ci-dessus. Elle évite la petite manipulation de branche temporaire. En revanche, on finit en detached HEAD state, ce qui n'est pas un problème dans notre cas puisque l'on ne souhaite pas push de modification depuis notre dépôt. + +### git reset --hard +git reset --hard va ignorer tous les changements et créations de fichiers committés sur theirs. Elle va également ignorer les modifications non committées. En revanche, les créations de fichier non-committées sont conservées. C'est le même résultat qu'avec la commande git merge -s ours, mais en une ligne. +C'est une option intéressante dans notre cas, car on pourrait vouloir sauvegarder uniquement des fichiers de compilation non committés. + +Les tests nous montrent que les options les plus économes en mémoire sont git merge -s ours et git --reset hard, qui font la même chose. Néanmoins git reset --hard tient en une ligne et n'implique pas la création d'une branche temporaire, c'est donc celle que nous retenons. + + +RESULTATS des tests : + + ======================================= Tests on the initial populating of the repository +============================================================= TEST0 +TEST 0: classic cloning. +memory usage: 22668 +bandwidth usage (submodule excluded): 8.49 MiB +============================================================= TEST1 +TEST 1: --single-branch cloning. +memory usage: 22168 +bandwidth usage (submodule excluded): 8.00 MiB +============================================================= TEST2 +TEST 2: --depth=1 --no-single-branch +memory usage: 17552 +bandwidth usage (submodule excluded): 3.49 MiB +============================================================= TEST3 +TEST 3: --depth=1 with single-branch (default)) +memory usage: 17052 +bandwidth usage (submodule excluded): 3.00 MiB +============================================================= TEST4 +TEST 4: --depth=1 with single-branch (default) and reflog and gc +HEAD is now at 23700cf adding submodule_for_performance_testing module +memory usage: 17056 +bandwidth usage (submodule excluded): 3.00 MiB +============================================================= TEST5 +TEST 5 : sparse-checking only sample0 with depth=1 +memory usage: 10060 +bandwidth usage (submodule excluded): unknown +============================================ Tests on the updating of the repository +================================================= classic fetching+checking out +============================================================= TEST6 +TEST 6: after addition of a 1M file +memory usage: +2108 +============================================================= TEST7 +TEST 7: after removal of a 1M file +memory usage: -972 +============================================================= TEST8 +TEST 8: after addition then removal of a 1M file +memory usage: 1088 +============================================== fetching+checking out with --depth=1 +============================================================= TEST9 +TEST 9: after addition of a 1M file +memory usage: +2112 +============================================================= TEST10 +TEST 10: after removal of a 1M file +memory usage: -968 +============================================================= TEST11 +TEST 11: after addition then removal of a 1M file +memory usage: 48 +========================================= --depth=1 fetching+checking out reflog and gc +============================================================= TEST12 +TEST 12: after addition of a 1M file +memory usage: +2052 +============================================================= TEST13 +TEST 13: after removal of a 1M file +memory usage: -1020 +============================================================= TEST14 +TEST 14: after addition then removal of a 1M file +memory usage: 4 +================================================ --depth=1 fetching+ reset --hard +============================================================= TEST15 +TEST 15: after addition of a 1M file +memory usage: +2116 +============================================================= TEST16 +TEST 16: after removal of a 1M file +memory usage: -964 +============================================================= TEST17 +TEST 17: after addition then removal of a 1M file +memory usage: 52 +======================================= --depth=1 fetching+ reset --hard and reflog and gc +============================================================= TEST18 +TEST 18: after addition of a 1M file +memory usage: 2056 +============================================================= TEST19 +TEST 19: after removal of a 1M file +memory usage: -1016 +============================================================= TEST20 +TEST 20: after addition then removal of a 1M file +memory usage: 8 +============================ --depth=1 fetching+checking out after modification applied in submodule +============================================================= TEST21 +TEST 21: after addition of a 1M file +memory usage: 2112 +============================================================= TEST22 +TEST 22: after removal of a 1M file +memory usage: -976 +============================================================= TEST23 +TEST 23: after addition then removal of a 1M file +memory usage: 48 +==================================== --depth=1 fetching+merging -X theirs with reflog and gc +============================================================= TEST24 +TEST 24: after addition of a 1M file +memory usage: +2056 +============================================================= TEST25 +TEST 25: after removal of a 1M file +memory usage: 8 +============================================================= TEST26 +TEST 26: after addition then removal of a 1M file +memory usage: 8 +===================================== --depth=1 fetching+merging -s ours with reflog and gc +============================================================= TEST27 +TEST 27: after addition of a 1M file +memory usage: +2056 +============================================================= TEST28 +TEST 28: after removal of a 1M file +memory usage: -1016 +============================================================= TEST29 +TEST 29: after addition then removal of a 1M file +memory usage: 8 +