[hotfix] crash bug when swap thread scaledown

src/config.c

@@ -3065,6 +3065,7 @@ standardConfig configs[] = {
     createIntConfig("swap-debug-swapout-notify-delay-micro", NULL, MODIFIABLE_CONFIG, -1, INT_MAX, server.swap_debug_swapout_notify_delay_micro, 0, INTEGER_CONFIG, NULL, NULL),
     createIntConfig("swap-debug-before-exec-swap-delay-micro", NULL, MODIFIABLE_CONFIG, 0, INT_MAX, server.swap_debug_before_exec_swap_delay_micro, 0, INTEGER_CONFIG, NULL, NULL),
     createIntConfig("swap-debug-init-rocksdb-delay-micro", NULL, MODIFIABLE_CONFIG, 0, INT_MAX, server.swap_debug_init_rocksdb_delay_micro, 0, INTEGER_CONFIG, NULL, NULL),
+    createIntConfig("swap-debug-scale-down-delay-micro", NULL, MODIFIABLE_CONFIG, 0, INT_MAX, server.swap_debug_scale_down_delay_micro, 0, INTEGER_CONFIG, NULL, NULL),
     createIntConfig("swap-debug-bgsave-metalen-addition", NULL, MODIFIABLE_CONFIG, INT_MIN, INT_MAX, server.swap_debug_bgsave_metalen_addition, 0, INTEGER_CONFIG, NULL, NULL),
     createIntConfig("swap-debug-compaction-filter-delay-micro", NULL, MODIFIABLE_CONFIG, 0, INT_MAX, server.swap_debug_compaction_filter_delay_micro, 0, INTEGER_CONFIG, NULL, NULL),
     createIntConfig("swap-debug-rdb-key-save-delay-micro", NULL, MODIFIABLE_CONFIG, -1, INT_MAX, server.swap_debug_rdb_key_save_delay_micro, 0, INTEGER_CONFIG, NULL, NULL),

src/ctrip_swap_server.h

@@ -166,6 +166,7 @@ typedef struct swapBatchLimitsConfig {
     int swap_debug_bgsave_metalen_addition; \
     int swap_debug_compaction_filter_delay_micro; \
     int swap_debug_rdb_key_save_delay_micro;  \
+    int swap_debug_scale_down_delay_micro; /* debug: sub-thread sleeps before start_idle_time=-1, enlarging race window for scale-down bug reproduction */ \
     int swap_rordb_load_incremental_fsync;  \
     /* repl swap */ \
     int swap_repl_workers;   /* num of repl worker clients */ \

src/ctrip_swap_thread.c

@@ -56,6 +56,11 @@ void *swapThreadMain (void *arg) {
             }
             pthread_cond_wait(&thread->cond, &thread->lock);
         }
+        if (server.swap_debug_scale_down_delay_micro) {
+            pthread_mutex_unlock(&thread->lock);
+            usleep(server.swap_debug_scale_down_delay_micro);
+            pthread_mutex_lock(&thread->lock);
+        }
         thread->start_idle_time = -1;
         // During the process of copying a linked list, encountering data corruption could lead to the main thread getting stuck on pthread_mutex_lock, making it impossible to terminate the program normally. In this case, AddressSanitizer (ASan) fails to print the detection results, and no core dump file will be generated.
         processing_reqs = thread->pending_reqs;
@@ -186,7 +191,9 @@ int swapThreadsAutoScaleDownIfNeeded(void) {
         pthread_mutex_lock(&thread->lock);
         start_idle_time = thread->start_idle_time;
         pthread_mutex_unlock(&thread->lock);
-        if (start_idle_time == -1) return 0;
+        size_t inflight_reqs;
+        atomicGet(thread->inflight_reqs, inflight_reqs);
+        if (start_idle_time == -1 || inflight_reqs > 0) return 0;
         if (((ustime() - start_idle_time) / 1000000) > server.swap_threads_auto_scale_down_idle_seconds) {
             return swapThreadsAutoScaleDown();
         }

tests/swap/unit/swap_thread_race_scaledown.tcl

@@ -0,0 +1,132 @@
+
+
+start_server {tags {"swap_thread_race_scaledown"} overrides {
+    save ""
+    swap-threads-auto-scale-max 5
+}} {
+    set host [srv 0 host]
+    set port [srv 0 port]
+    set r1 [srv 0 client]
+    set r2 [redis $host $port]
+
+    proc wait_keys_cold {r prefix count {wait_ms 8000}} {
+        set deadline [expr {[clock milliseconds] + $wait_ms}]
+        while {[clock milliseconds] < $deadline} {
+            set cold 0
+            for {set i 0} {$i < $count} {incr i} {
+                set info ""
+                catch {set info [$r swap object "$prefix:$i"]}
+                if {[string match "*value: <nil>*" $info] && \
+                    [string match "*cold_meta: swap_type=*" $info]} {
+                    incr cold
+                }
+            }
+            if {$cold == $count} { return $cold }
+            after 100
+        }
+        return 0
+    }
+
+    test {scale-down race: assert inflight_reqs==0 crash via serverCron} {
+        $r1 config set swap-threads-auto-scale-down-idle-seconds 300
+        $r1 swap.debug thread auto-scale-up
+        after 200
+
+        set info [$r1 info swap]
+        set tn 0
+        regexp {swap_thread_num:(\d+)} $info -> tn
+        assert {$tn >= 5}
+
+        after 1500
+
+
+        $r1 config set swap-batch-limit "IN 1 1048576 OUT 1 1048576 DEL 1 1048576"
+        $r1 config set swap-threads-auto-scale-up-threshold 1
+
+
+        set key_count 5
+        set prefix "scalerace"
+        for {set i 0} {$i < $key_count} {incr i} {
+            $r1 set "$prefix:$i" [string repeat "v" 4096]
+            $r1 swap.evict "$prefix:$i"
+        }
+        set cold [wait_keys_cold $r1 $prefix $key_count 8000]
+        assert_equal $cold $key_count
+
+
+        set warmup_conns {}
+        for {set i 0} {$i < $key_count} {incr i} {
+            lappend warmup_conns [redis $host $port]
+        }
+        set i 0
+        foreach rc $warmup_conns {
+            $rc deferred 1
+            $rc get "$prefix:$i"
+            incr i
+        }
+
+        foreach rc $warmup_conns { catch {$rc read} }
+
+
+        for {set i 0} {$i < $key_count} {incr i} {
+            catch {$r1 swap.evict "$prefix:$i"}
+        }
+        set cold [wait_keys_cold $r1 $prefix $key_count 5000]
+        assert_equal $cold $key_count
+
+
+        after 1200
+
+
+        $r1 config set swap-debug-scale-down-delay-micro 500000
+
+        set t0 [clock milliseconds]
+
+
+        $r1 config set swap-threads-auto-scale-down-idle-seconds 0
+
+
+        set race_conns {}
+        for {set i 0} {$i < $key_count} {incr i} {
+            lappend race_conns [redis $host $port]
+        }
+        set i 0
+        foreach rc $race_conns {
+            $rc deferred 1
+            $rc get "$prefix:$i"
+            incr i
+        }
+
+
+        after 50
+
+
+        set tp0 [clock milliseconds]
+        catch {$r1 ping}
+        set tp1 [clock milliseconds]
+
+        after 100
+
+
+        set crash_detected 0
+        set check_deadline [expr {[clock milliseconds] + 2000}]
+        while {[clock milliseconds] < $check_deadline} {
+            after 50
+            if {[catch {$r2 ping}]} {
+                set crash_detected 1
+                break
+            }
+        }
+
+        set elapsed [expr {[clock milliseconds] - $t0}]
+
+        catch {$r1 config set swap-debug-scale-down-delay-micro 0}
+        catch {$r1 config set swap-threads-auto-scale-down-idle-seconds 300}
+        catch {$r1 config set swap-threads-auto-scale-up-threshold 32}
+        catch {$r1 config set swap-batch-limit "IN 64 67108864 OUT 64 67108864 DEL 64 67108864"}
+
+
+        assert_equal [$r2 ping] "PONG"
+
+    }
+}

tests/test_helper.tcl

@@ -156,6 +156,7 @@ set ::disk_tests {
     swap/unit/import_mode
     swap/unit/metascan_multidb
     swap/unit/ltrim_check
+    swap/unit/swap_thread_race_scaledown
 }
 
 set ::all_tests [concat $::gtid_tests $::all_tests]