diff --git a/Documentation/scalar.adoc b/Documentation/scalar.adoc index 56c99c8a2d00dc..e08f22607f0267 100644 --- a/Documentation/scalar.adoc +++ b/Documentation/scalar.adoc @@ -10,8 +10,8 @@ SYNOPSIS [verse] scalar clone [--single-branch] [--branch ] [--full-clone] [--[no-]src] [--[no-]tags] [--[no-]maintenance] - [--[no-]src] [--local-cache-path ] [--cache-server-url ] - [] + [--cache-server-url ] [--[verb]-cache-server-url ] + [--local-cache-path ] [] scalar list scalar register [--[no-]maintenance] [] scalar unregister [] @@ -121,6 +121,11 @@ clone), and `~/.scalarCache` on macOS. Retrieve missing objects from the specified remote, which is expected to understand the GVFS protocol. +--[verb]-cache-server-url :: + Set the appropriate `gvfs..cache-server` config value that overrides + the provided `--cache-server-url` or the dynamically discovered URL. The + list of allowed verbs is `prefetch`, `get`, and `post`. + --gvfs-protocol:: --no-gvfs-protocol:: When cloning from a `` with either `dev.azure.com` or diff --git a/scalar.c b/scalar.c index 7dcca36a86a249..0e73108f9d598a 100644 --- a/scalar.c +++ b/scalar.c @@ -783,6 +783,7 @@ static int cmd_clone(int argc, const char **argv) int src = 1, tags = 1, maintenance = 1; const char *cache_server_url = NULL, *local_cache_root = NULL; char *default_cache_server_url = NULL, *local_cache_root_abs = NULL; + const char *prefetch_server = NULL, *get_server = NULL, *post_server = NULL; int gvfs_protocol = -1; const char *ref_format = NULL; @@ -805,6 +806,15 @@ static int cmd_clone(int argc, const char **argv) OPT_STRING(0, "cache-server-url", &cache_server_url, N_(""), N_("the url or friendly name of the cache server")), + OPT_STRING(0, "prefetch-cache-server-url", &prefetch_server, + N_(""), + N_("the url or friendly name of a cache server for the prefetch endpoint")), + OPT_STRING(0, "get-cache-server-url", &get_server, + N_(""), + N_("the url or friendly name of a cache server for the objects GET endpoint")), + OPT_STRING(0, "post-cache-server-url", &post_server, + N_(""), + N_("the url or friendly name of a cache server for the objects POST endpoint")), OPT_STRING(0, "local-cache-path", &local_cache_root, N_(""), N_("override the path for the local Scalar cache")), @@ -817,7 +827,8 @@ static int cmd_clone(int argc, const char **argv) const char * const clone_usage[] = { N_("scalar clone [--single-branch] [--branch ] [--full-clone]\n" "\t[--[no-]src] [--[no-]tags] [--[no-]maintenance] [--ref-format ]\n" - "\t []"), + "\t[--cache-server-url ] [--[verb]-cache-server-url ]\n" + "\t[--local-cache-path ] []"), NULL }; const char *url; @@ -979,6 +990,33 @@ static int cmd_clone(int argc, const char **argv) if (cache_server_url) fprintf(stderr, "Cache server URL: %s\n", cache_server_url); + + if (prefetch_server && + set_config("gvfs.prefetch.cache-server=%s", prefetch_server)) { + res = error(_("could not configure prefetch cache server")); + goto cleanup; + } + if (prefetch_server) + fprintf(stderr, "Prefetch cache server URL: %s\n", + prefetch_server); + + if (get_server && + set_config("gvfs.get.cache-server=%s", get_server)) { + res = error(_("could not configure objects GET cache server")); + goto cleanup; + } + if (get_server) + fprintf(stderr, "Objects GET cache server URL: %s\n", + get_server); + + if (post_server && + set_config("gvfs.post.cache-server=%s", post_server)) { + res = error(_("could not configure objects POST cache server")); + goto cleanup; + } + if (post_server) + fprintf(stderr, "Objects POST cache server URL: %s\n", + post_server); } else { if (set_config("core.useGVFSHelper=false") || set_config("remote.origin.promisor=true") || diff --git a/t/lib-gvfs-helper.sh b/t/lib-gvfs-helper.sh index 32c951361e3282..6bfa26c8dd9b3b 100644 --- a/t/lib-gvfs-helper.sh +++ b/t/lib-gvfs-helper.sh @@ -58,8 +58,8 @@ SHARED_CACHE_T2="$(pwd)"/shared_cache_t2 # The server will shut down if/when we delete it. (This is a little # easier than killing it by PID.) # -PID_FILE="$(pwd)"/pid-file.pid -SERVER_LOG="$(pwd)"/OUT.server.log +PID_FILE="$(pwd)"/pid-file-gvfs.pid +SERVER_LOG="$(pwd)"/OUT.gvfs.server.log # Helper functions to compute port, pid-file, and log for a given # port increment. An increment of 0 (or empty) uses the base values. @@ -82,19 +82,19 @@ server_pid_file () { } server_log_file () { - local instance="${1:-0}" + local instance="${1:-1}" if test "$instance" -eq 0 then echo "$SERVER_LOG" else - echo "$(pwd)/OUT.server-$instance.log" + echo "$(pwd)/OUT.gvfs.server-$instance.log" fi } # Helper to build a cache-server URL for a given port increment. # cache_server_url () { - local instance="${1:-0}" + local instance="${1:-1}" local port="$(server_port "$instance")" echo "http://127.0.0.1:$port/servertype/cache" } @@ -277,7 +277,7 @@ test_expect_success 'setup repos' ' # Increment 0 uses the base port, 1 uses base+1, etc. # stop_gvfs_protocol_server () { - local instance="${1:-0}" + local instance="${1:-1}" local pid_file="$(server_pid_file "$instance")" local log_file="$(server_log_file "$instance")" @@ -294,7 +294,7 @@ stop_gvfs_protocol_server () { # port before the next test start another instance and it attempts to # bind to it). # - for k in 0 1 2 3 4 + for k in $(test_seq 5) do if grep -q "Starting graceful shutdown" "$log_file" then @@ -315,7 +315,7 @@ stop_gvfs_protocol_server () { # This allows running multiple servers simultaneously on different ports. # start_gvfs_protocol_server () { - local instance="${1:-0}" + local instance="${1:-1}" local port="$(server_port "$instance")" local pid_file="$(server_pid_file "$instance")" local log_file="$(server_log_file "$instance")" @@ -334,7 +334,7 @@ start_gvfs_protocol_server () { # # Give it a few seconds to get started. # - for k in 0 1 2 3 4 + for k in $(test_seq 5) do if test -f "$pid_file" then @@ -375,7 +375,7 @@ start_gvfs_protocol_server_with_mayhem () { # # Give it a few seconds to get started. # - for k in 0 1 2 3 4 + for k in $(test_seq 5) do if test -f "$PID_FILE" then @@ -392,7 +392,7 @@ start_gvfs_protocol_server_with_mayhem () { # Usage: verify_server_was_contacted [] # verify_server_was_contacted () { - local instance="${1:-0}" + local instance="${1:-1}" local log_file="$(server_log_file "$instance")" grep -q "Connection from" "$log_file" } @@ -401,7 +401,7 @@ verify_server_was_contacted () { # Usage: verify_server_was_not_contacted [] # verify_server_was_not_contacted () { - local instance="${1:-0}" + local instance="${1:-1}" local log_file="$(server_log_file "$instance")" ! grep -q "Connection from" "$log_file" } @@ -529,9 +529,9 @@ verify_vfs_packfile_count () { } per_test_cleanup () { - # Stop servers with port increments 0, 1, 2, 3 to handle tests + # Stop servers with port increments 1, 2, 3, 4 to handle tests # that may use multiple servers. - for instance in 0 1 2 3 + for instance in 1 2 3 4 do stop_gvfs_protocol_server "$instance" done @@ -540,7 +540,7 @@ per_test_cleanup () { rm -rf "$SHARED_CACHE_T1"/info/* rm -rf "$SHARED_CACHE_T1"/pack/* - rm -rf OUT.* + rm -rf OUT.gvfs.* return 0 } diff --git a/t/t5795-gvfs-helper-verb-cache.sh b/t/t5795-gvfs-helper-verb-cache.sh index 653deeedd5554d..bac5f5abf6bbd6 100755 --- a/t/t5795-gvfs-helper-verb-cache.sh +++ b/t/t5795-gvfs-helper-verb-cache.sh @@ -22,14 +22,14 @@ test_description='gvfs-helper verb-specific cache-server tests' test_expect_success 'verb-specific cache-server: prefetch uses gvfs.prefetch.cache-server' ' test_when_finished "per_test_cleanup" && test_when_finished "git -C \"$REPO_T1\" config --unset gvfs.prefetch.cache-server" && - start_gvfs_protocol_server 0 && start_gvfs_protocol_server 1 && + start_gvfs_protocol_server 2 && # Configure server 0 as default cache-server and server 1 for prefetch. git -C "$REPO_T1" config gvfs.cache-server "$(cache_server_url 0)" && git -C "$REPO_T1" config gvfs.prefetch.cache-server "$(cache_server_url 1)" && - # Run prefetch - should go to server 1. + # Run prefetch - should go to server 2. git -C "$REPO_T1" gvfs-helper \ --cache-server=trust \ --remote=origin \ @@ -37,17 +37,17 @@ test_expect_success 'verb-specific cache-server: prefetch uses gvfs.prefetch.cac prefetch >OUT.output 2>OUT.stderr && # Verify server 1 was contacted (prefetch-specific). - verify_server_was_contacted 1 && + verify_server_was_contacted 2 && # Verify server 0 was NOT contacted. - verify_server_was_not_contacted 0 + verify_server_was_not_contacted 1 ' test_expect_success 'verb-specific cache-server: get does NOT use gvfs.prefetch.cache-server' ' test_when_finished "per_test_cleanup" && test_when_finished "git -C \"$REPO_T1\" config --unset gvfs.prefetch.cache-server" && - start_gvfs_protocol_server 0 && start_gvfs_protocol_server 1 && + start_gvfs_protocol_server 2 && # Configure server 0 as default cache-server and server 1 for prefetch. git -C "$REPO_T1" config gvfs.cache-server "$(cache_server_url 0)" && @@ -61,23 +61,23 @@ test_expect_success 'verb-specific cache-server: get does NOT use gvfs.prefetch. <"$OID_ONE_BLOB_FILE" >OUT.output 2>OUT.stderr && # Verify server 0 was contacted (default cache-server). - verify_server_was_contacted 0 && + verify_server_was_contacted 1 && # Verify server 1 was NOT contacted (prefetch-specific). - verify_server_was_not_contacted 1 + verify_server_was_not_contacted 2 ' test_expect_success 'verb-specific cache-server: get uses gvfs.get.cache-server' ' test_when_finished "per_test_cleanup" && test_when_finished "git -C \"$REPO_T1\" config --unset gvfs.get.cache-server" && - start_gvfs_protocol_server 0 && start_gvfs_protocol_server 1 && + start_gvfs_protocol_server 2 && # Configure server 0 as default cache-server and server 1 for get. git -C "$REPO_T1" config gvfs.cache-server "$(cache_server_url 0)" && git -C "$REPO_T1" config gvfs.get.cache-server "$(cache_server_url 1)" && - # Run get - should go to server 1. + # Run get - should go to server 2. git -C "$REPO_T1" gvfs-helper \ --cache-server=trust \ --remote=origin \ @@ -85,17 +85,17 @@ test_expect_success 'verb-specific cache-server: get uses gvfs.get.cache-server' <"$OID_ONE_BLOB_FILE" >OUT.output 2>OUT.stderr && # Verify server 1 was contacted (get-specific). - verify_server_was_contacted 1 && + verify_server_was_contacted 2 && # Verify server 0 was NOT contacted. - verify_server_was_not_contacted 0 + verify_server_was_not_contacted 1 ' test_expect_success 'verb-specific cache-server: prefetch does NOT use gvfs.get.cache-server' ' test_when_finished "per_test_cleanup" && test_when_finished "git -C \"$REPO_T1\" config --unset gvfs.get.cache-server" && - start_gvfs_protocol_server 0 && start_gvfs_protocol_server 1 && + start_gvfs_protocol_server 2 && # Configure server 0 as default cache-server and server 1 for get. git -C "$REPO_T1" config gvfs.cache-server "$(cache_server_url 0)" && @@ -109,23 +109,23 @@ test_expect_success 'verb-specific cache-server: prefetch does NOT use gvfs.get. prefetch >OUT.output 2>OUT.stderr && # Verify server 0 was contacted (default cache-server). - verify_server_was_contacted 0 && + verify_server_was_contacted 1 && # Verify server 1 was NOT contacted (get-specific). - verify_server_was_not_contacted 1 + verify_server_was_not_contacted 2 ' test_expect_success 'verb-specific cache-server: post uses gvfs.post.cache-server' ' test_when_finished "per_test_cleanup" && test_when_finished "git -C \"$REPO_T1\" config --unset gvfs.post.cache-server" && - start_gvfs_protocol_server 0 && start_gvfs_protocol_server 1 && + start_gvfs_protocol_server 2 && # Configure server 0 as default cache-server and server 1 for post. git -C "$REPO_T1" config gvfs.cache-server "$(cache_server_url 0)" && git -C "$REPO_T1" config gvfs.post.cache-server "$(cache_server_url 1)" && - # Run post - should go to server 1. + # Run post - should go to server 2. git -C "$REPO_T1" gvfs-helper \ --cache-server=trust \ --remote=origin \ @@ -134,17 +134,17 @@ test_expect_success 'verb-specific cache-server: post uses gvfs.post.cache-serve <"$OIDS_BLOBS_FILE" >OUT.output 2>OUT.stderr && # Verify server 1 was contacted (post-specific). - verify_server_was_contacted 1 && + verify_server_was_contacted 2 && # Verify server 0 was NOT contacted. - verify_server_was_not_contacted 0 + verify_server_was_not_contacted 1 ' test_expect_success 'verb-specific cache-server: get does NOT use gvfs.post.cache-server' ' test_when_finished "per_test_cleanup" && test_when_finished "git -C \"$REPO_T1\" config --unset gvfs.post.cache-server" && - start_gvfs_protocol_server 0 && start_gvfs_protocol_server 1 && + start_gvfs_protocol_server 2 && # Configure server 0 as default cache-server and server 1 for post. git -C "$REPO_T1" config gvfs.cache-server "$(cache_server_url 0)" && @@ -158,10 +158,10 @@ test_expect_success 'verb-specific cache-server: get does NOT use gvfs.post.cach <"$OID_ONE_BLOB_FILE" >OUT.output 2>OUT.stderr && # Verify server 0 was contacted (default cache-server). - verify_server_was_contacted 0 && + verify_server_was_contacted 1 && # Verify server 1 was NOT contacted (post-specific). - verify_server_was_not_contacted 1 + verify_server_was_not_contacted 2 ' test_expect_success 'verb-specific cache-server: all verbs with different servers' ' @@ -170,9 +170,9 @@ test_expect_success 'verb-specific cache-server: all verbs with different server test_when_finished "git -C \"$REPO_T1\" config --unset gvfs.prefetch.cache-server" && test_when_finished "git -C \"$REPO_T1\" config --unset gvfs.get.cache-server" && test_when_finished "git -C \"$REPO_T1\" config --unset gvfs.post.cache-server" && - start_gvfs_protocol_server 0 && start_gvfs_protocol_server 1 && start_gvfs_protocol_server 2 && + start_gvfs_protocol_server 2 && start_gvfs_protocol_server 3 && # Configure each verb to use a different server: @@ -185,14 +185,14 @@ test_expect_success 'verb-specific cache-server: all verbs with different server git -C "$REPO_T1" config gvfs.get.cache-server "$(cache_server_url 2)" && git -C "$REPO_T1" config gvfs.post.cache-server "$(cache_server_url 3)" && - # Run prefetch - should go to server 1. + # Run prefetch - should go to server 2. git -C "$REPO_T1" gvfs-helper \ --cache-server=trust \ --remote=origin \ --no-progress \ prefetch >OUT.output 2>OUT.stderr && - verify_server_was_contacted 1 && - verify_server_was_not_contacted 0 && + verify_server_was_contacted 2 && + verify_server_was_not_contacted 1 && verify_server_was_not_contacted 2 && verify_server_was_not_contacted 3 && diff --git a/t/t9210-scalar.sh b/t/t9210-scalar.sh index 493a35b8f43a5f..31eecb9f81ad07 100755 --- a/t/t9210-scalar.sh +++ b/t/t9210-scalar.sh @@ -360,20 +360,20 @@ test_expect_success UNZIP 'scalar diagnose' ' GIT_TEST_ALLOW_GVFS_VIA_HTTP=1 export GIT_TEST_ALLOW_GVFS_VIA_HTTP -test_set_port GIT_TEST_GVFS_PROTOCOL_PORT -HOST_PORT=127.0.0.1:$GIT_TEST_GVFS_PROTOCOL_PORT -PID_FILE="$(pwd)"/pid-file.pid -SERVER_LOG="$(pwd)"/OUT.server.log +test_set_port GIT_TEST_GVFS_PROTOCOL_ORIGIN_PORT +ORIGIN_HOST_PORT=127.0.0.1:$GIT_TEST_GVFS_PROTOCOL_ORIGIN_PORT +ORIGIN_PID_FILE="$(pwd)"/pid-file.$GIT_TEST_GVFS_PROTOCOL_ORIGIN_PORT.pid +ORIGIN_SERVER_LOG="$(pwd)"/OUT.server.$GIT_TEST_GVFS_PROTOCOL_ORIGIN_PORT.log test_atexit ' - test -f "$PID_FILE" || return 0 + test -f "$ORIGIN_PID_FILE" || return 0 # The server will shutdown automatically when we delete the pid-file. - rm -f "$PID_FILE" + rm -f "$ORIGIN_PID_FILE" test -z "$verbose$verbose_log" || { echo "server log:" - cat "$SERVER_LOG" + cat "$ORIGIN_SERVER_LOG" } # Give it a few seconds to shutdown (mainly to completely release the @@ -381,7 +381,7 @@ test_atexit ' # bind to it). for k in $(test_seq 5) do - grep -q "Starting graceful shutdown" "$SERVER_LOG" && + grep -q "Starting graceful shutdown" "$ORIGIN_SERVER_LOG" && return 0 || sleep 1 done @@ -394,14 +394,14 @@ start_gvfs_enabled_http_server () { GIT_HTTP_EXPORT_ALL=1 \ test-gvfs-protocol --verbose \ --listen=127.0.0.1 \ - --port=$GIT_TEST_GVFS_PROTOCOL_PORT \ + --port=$GIT_TEST_GVFS_PROTOCOL_ORIGIN_PORT \ --reuseaddr \ - --pid-file="$PID_FILE" \ - 2>"$SERVER_LOG" & + --pid-file="$ORIGIN_PID_FILE" \ + 2>"$ORIGIN_SERVER_LOG" & - for k in 0 1 2 3 4 + for k in $(test_seq 5) do - if test -f "$PID_FILE" + if test -f "$ORIGIN_PID_FILE" then return 0 fi @@ -455,7 +455,52 @@ test_expect_success '`scalar clone` with GVFS-enabled server' ' ) ' -test_expect_success 'fetch does not hang in gvfs-helper' ' +. "$TEST_DIRECTORY"/lib-gvfs-helper.sh + +test_expect_success 'scalar clone: all verbs with different servers' ' + git config --global core.askPass true && + + test_when_finished "per_test_cleanup" && + test_when_finished "scalar delete scalar-clone" && + + start_gvfs_protocol_server 1 && + start_gvfs_protocol_server 2 && + start_gvfs_protocol_server 3 && + start_gvfs_protocol_server 4 && + + # Configure each verb to use a different server: + # - server 1: default (unused in this test; not running.) + # - server 2: prefetch + # - server 3: get + # - server 4: post + scalar -c credential.interactive=true \ + clone --full-clone \ + --cache-server-url="$(cache_server_url 1)" \ + --prefetch-cache-server-url="$(cache_server_url 2)" \ + --get-cache-server-url="$(cache_server_url 3)" \ + --post-cache-server-url="$(cache_server_url 4)" \ + --gvfs-protocol \ + -- "http://$HOST_PORT/" scalar-clone 2>err >out && + + test_grep "Cache server URL: $(cache_server_url 1)" err && + test_grep "Prefetch cache server URL: $(cache_server_url 2)" err && + test_grep "Objects GET cache server URL: $(cache_server_url 3)" err && + test_grep "Objects POST cache server URL: $(cache_server_url 4)" err && + + test_cmp_config -C scalar-clone/src "$(cache_server_url 1)" gvfs.cache-server && + test_cmp_config -C scalar-clone/src "$(cache_server_url 2)" gvfs.prefetch.cache-server && + test_cmp_config -C scalar-clone/src "$(cache_server_url 3)" gvfs.get.cache-server && + test_cmp_config -C scalar-clone/src "$(cache_server_url 4)" gvfs.post.cache-server && + + verify_server_was_contacted 1 && + verify_server_was_contacted 2 && + verify_server_was_contacted 3 +' + +test_expect_success EXPENSIVE 'fetch does not hang in gvfs-helper' ' + # Marked as EXPENSIVE as this will go through multiple rounds of + # exponential backoff, including delays of 8, 16, 32, 64, 128, + # and 256 seconds in two separate instances. test_must_fail git -C using-gvfs/src fetch origin does-not-exist '