diff --git a/src/config.c b/src/config.c index 7f3a90826c2..489a2cfbdfd 100644 --- a/src/config.c +++ b/src/config.c @@ -3065,6 +3065,7 @@ standardConfig configs[] = { createIntConfig("swap-debug-swapout-notify-delay-micro", NULL, MODIFIABLE_CONFIG, -1, INT_MAX, server.swap_debug_swapout_notify_delay_micro, 0, INTEGER_CONFIG, NULL, NULL), createIntConfig("swap-debug-before-exec-swap-delay-micro", NULL, MODIFIABLE_CONFIG, 0, INT_MAX, server.swap_debug_before_exec_swap_delay_micro, 0, INTEGER_CONFIG, NULL, NULL), createIntConfig("swap-debug-init-rocksdb-delay-micro", NULL, MODIFIABLE_CONFIG, 0, INT_MAX, server.swap_debug_init_rocksdb_delay_micro, 0, INTEGER_CONFIG, NULL, NULL), + createIntConfig("swap-debug-scale-down-delay-micro", NULL, MODIFIABLE_CONFIG, 0, INT_MAX, server.swap_debug_scale_down_delay_micro, 0, INTEGER_CONFIG, NULL, NULL), createIntConfig("swap-debug-bgsave-metalen-addition", NULL, MODIFIABLE_CONFIG, INT_MIN, INT_MAX, server.swap_debug_bgsave_metalen_addition, 0, INTEGER_CONFIG, NULL, NULL), createIntConfig("swap-debug-compaction-filter-delay-micro", NULL, MODIFIABLE_CONFIG, 0, INT_MAX, server.swap_debug_compaction_filter_delay_micro, 0, INTEGER_CONFIG, NULL, NULL), createIntConfig("swap-debug-rdb-key-save-delay-micro", NULL, MODIFIABLE_CONFIG, -1, INT_MAX, server.swap_debug_rdb_key_save_delay_micro, 0, INTEGER_CONFIG, NULL, NULL), diff --git a/src/ctrip_swap_server.h b/src/ctrip_swap_server.h index 390584bbe03..3138111fa45 100644 --- a/src/ctrip_swap_server.h +++ b/src/ctrip_swap_server.h @@ -166,6 +166,7 @@ typedef struct swapBatchLimitsConfig { int swap_debug_bgsave_metalen_addition; \ int swap_debug_compaction_filter_delay_micro; \ int swap_debug_rdb_key_save_delay_micro; \ + int swap_debug_scale_down_delay_micro; /* debug: sub-thread sleeps before start_idle_time=-1, enlarging race window for scale-down bug reproduction */ \ int swap_rordb_load_incremental_fsync; \ /* repl swap */ \ int swap_repl_workers; /* num of repl worker clients */ \ diff --git a/src/ctrip_swap_thread.c b/src/ctrip_swap_thread.c index 5e070b45839..19c4a7b37df 100644 --- a/src/ctrip_swap_thread.c +++ b/src/ctrip_swap_thread.c @@ -56,6 +56,11 @@ void *swapThreadMain (void *arg) { } pthread_cond_wait(&thread->cond, &thread->lock); } + if (server.swap_debug_scale_down_delay_micro) { + pthread_mutex_unlock(&thread->lock); + usleep(server.swap_debug_scale_down_delay_micro); + pthread_mutex_lock(&thread->lock); + } thread->start_idle_time = -1; // During the process of copying a linked list, encountering data corruption could lead to the main thread getting stuck on pthread_mutex_lock, making it impossible to terminate the program normally. In this case, AddressSanitizer (ASan) fails to print the detection results, and no core dump file will be generated. processing_reqs = thread->pending_reqs; @@ -186,7 +191,9 @@ int swapThreadsAutoScaleDownIfNeeded(void) { pthread_mutex_lock(&thread->lock); start_idle_time = thread->start_idle_time; pthread_mutex_unlock(&thread->lock); - if (start_idle_time == -1) return 0; + size_t inflight_reqs; + atomicGet(thread->inflight_reqs, inflight_reqs); + if (start_idle_time == -1 || inflight_reqs > 0) return 0; if (((ustime() - start_idle_time) / 1000000) > server.swap_threads_auto_scale_down_idle_seconds) { return swapThreadsAutoScaleDown(); } diff --git a/tests/swap/unit/swap_thread_race_scaledown.tcl b/tests/swap/unit/swap_thread_race_scaledown.tcl new file mode 100644 index 00000000000..d71385dbbf6 --- /dev/null +++ b/tests/swap/unit/swap_thread_race_scaledown.tcl @@ -0,0 +1,132 @@ + + +start_server {tags {"swap_thread_race_scaledown"} overrides { + save "" + swap-threads-auto-scale-max 5 +}} { + set host [srv 0 host] + set port [srv 0 port] + set r1 [srv 0 client] + set r2 [redis $host $port] + + proc wait_keys_cold {r prefix count {wait_ms 8000}} { + set deadline [expr {[clock milliseconds] + $wait_ms}] + while {[clock milliseconds] < $deadline} { + set cold 0 + for {set i 0} {$i < $count} {incr i} { + set info "" + catch {set info [$r swap object "$prefix:$i"]} + if {[string match "*value: *" $info] && \ + [string match "*cold_meta: swap_type=*" $info]} { + incr cold + } + } + if {$cold == $count} { return $cold } + after 100 + } + return 0 + } + + test {scale-down race: assert inflight_reqs==0 crash via serverCron} { + $r1 config set swap-threads-auto-scale-down-idle-seconds 300 + $r1 swap.debug thread auto-scale-up + after 200 + + set info [$r1 info swap] + set tn 0 + regexp {swap_thread_num:(\d+)} $info -> tn + assert {$tn >= 5} + + after 1500 + + + $r1 config set swap-batch-limit "IN 1 1048576 OUT 1 1048576 DEL 1 1048576" + $r1 config set swap-threads-auto-scale-up-threshold 1 + + + set key_count 5 + set prefix "scalerace" + for {set i 0} {$i < $key_count} {incr i} { + $r1 set "$prefix:$i" [string repeat "v" 4096] + $r1 swap.evict "$prefix:$i" + } + set cold [wait_keys_cold $r1 $prefix $key_count 8000] + assert_equal $cold $key_count + + + set warmup_conns {} + for {set i 0} {$i < $key_count} {incr i} { + lappend warmup_conns [redis $host $port] + } + set i 0 + foreach rc $warmup_conns { + $rc deferred 1 + $rc get "$prefix:$i" + incr i + } + + foreach rc $warmup_conns { catch {$rc read} } + + + for {set i 0} {$i < $key_count} {incr i} { + catch {$r1 swap.evict "$prefix:$i"} + } + set cold [wait_keys_cold $r1 $prefix $key_count 5000] + assert_equal $cold $key_count + + + after 1200 + + + $r1 config set swap-debug-scale-down-delay-micro 500000 + + set t0 [clock milliseconds] + + + $r1 config set swap-threads-auto-scale-down-idle-seconds 0 + + + set race_conns {} + for {set i 0} {$i < $key_count} {incr i} { + lappend race_conns [redis $host $port] + } + set i 0 + foreach rc $race_conns { + $rc deferred 1 + $rc get "$prefix:$i" + incr i + } + + + after 50 + + + set tp0 [clock milliseconds] + catch {$r1 ping} + set tp1 [clock milliseconds] + + after 100 + + + set crash_detected 0 + set check_deadline [expr {[clock milliseconds] + 2000}] + while {[clock milliseconds] < $check_deadline} { + after 50 + if {[catch {$r2 ping}]} { + set crash_detected 1 + break + } + } + + set elapsed [expr {[clock milliseconds] - $t0}] + + catch {$r1 config set swap-debug-scale-down-delay-micro 0} + catch {$r1 config set swap-threads-auto-scale-down-idle-seconds 300} + catch {$r1 config set swap-threads-auto-scale-up-threshold 32} + catch {$r1 config set swap-batch-limit "IN 64 67108864 OUT 64 67108864 DEL 64 67108864"} + + + assert_equal [$r2 ping] "PONG" + + } +} diff --git a/tests/test_helper.tcl b/tests/test_helper.tcl index 15d5b4c62e5..dd20be9a60f 100644 --- a/tests/test_helper.tcl +++ b/tests/test_helper.tcl @@ -156,6 +156,7 @@ set ::disk_tests { swap/unit/import_mode swap/unit/metascan_multidb swap/unit/ltrim_check + swap/unit/swap_thread_race_scaledown } set ::all_tests [concat $::gtid_tests $::all_tests]