1proc start_bg_complex_data {host port db ops} {
2    set tclsh [info nameofexecutable]
3    exec $tclsh tests/helpers/bg_complex_data.tcl $host $port $db $ops &
4}
5
6proc stop_bg_complex_data {handle} {
7    catch {exec /bin/kill -9 $handle}
8}
9
10# Creates a master-slave pair and breaks the link continuously to force
11# partial resyncs attempts, all this while flooding the master with
12# write queries.
13#
14# You can specifiy backlog size, ttl, delay before reconnection, test duration
15# in seconds, and an additional condition to verify at the end.
16proc test_psync {descr duration backlog_size backlog_ttl delay cond} {
17    start_server {tags {"repl"}} {
18        start_server {} {
19
20            set master [srv -1 client]
21            set master_host [srv -1 host]
22            set master_port [srv -1 port]
23            set slave [srv 0 client]
24
25            $master config set repl-backlog-size $backlog_size
26            $master config set repl-backlog-ttl $backlog_ttl
27
28            set load_handle0 [start_bg_complex_data $master_host $master_port 9 100000]
29            set load_handle1 [start_bg_complex_data $master_host $master_port 11 100000]
30            set load_handle2 [start_bg_complex_data $master_host $master_port 12 100000]
31
32            test {Slave should be able to synchronize with the master} {
33                $slave slaveof $master_host $master_port
34                wait_for_condition 50 100 {
35                    [lindex [r role] 0] eq {slave} &&
36                    [lindex [r role] 3] eq {connected}
37                } else {
38                    fail "Replication not started."
39                }
40            }
41
42            # Check that the background clients are actually writing.
43            test {Detect write load to master} {
44                wait_for_condition 50 100 {
45                    [$master dbsize] > 100
46                } else {
47                    fail "Can't detect write load from background clients."
48                }
49            }
50
51            test "Test replication partial resync: $descr" {
52                # Now while the clients are writing data, break the maste-slave
53                # link multiple times.
54                for {set j 0} {$j < $duration*10} {incr j} {
55                    after 100
56                    # catch {puts "MASTER [$master dbsize] keys, SLAVE [$slave dbsize] keys"}
57
58                    if {($j % 20) == 0} {
59                        catch {
60                            if {$delay} {
61                                $slave multi
62                                $slave client kill $master_host:$master_port
63                                $slave debug sleep $delay
64                                $slave exec
65                            } else {
66                                $slave client kill $master_host:$master_port
67                            }
68                        }
69                    }
70                }
71                stop_bg_complex_data $load_handle0
72                stop_bg_complex_data $load_handle1
73                stop_bg_complex_data $load_handle2
74                set retry 10
75                while {$retry && ([$master debug digest] ne [$slave debug digest])}\
76                {
77                    after 1000
78                    incr retry -1
79                }
80                assert {[$master dbsize] > 0}
81
82                if {[$master debug digest] ne [$slave debug digest]} {
83                    set csv1 [csvdump r]
84                    set csv2 [csvdump {r -1}]
85                    set fd [open /tmp/repldump1.txt w]
86                    puts -nonewline $fd $csv1
87                    close $fd
88                    set fd [open /tmp/repldump2.txt w]
89                    puts -nonewline $fd $csv2
90                    close $fd
91                    puts "Master - Slave inconsistency"
92                    puts "Run diff -u against /tmp/repldump*.txt for more info"
93                }
94                assert_equal [r debug digest] [r -1 debug digest]
95                eval $cond
96            }
97        }
98    }
99}
100
101test_psync {ok psync} 6 1000000 3600 0 {
102    assert {[s -1 sync_partial_ok] > 0}
103}
104
105test_psync {no backlog} 6 100 3600 0.5 {
106    assert {[s -1 sync_partial_err] > 0}
107}
108
109test_psync {ok after delay} 3 100000000 3600 3 {
110    assert {[s -1 sync_partial_ok] > 0}
111}
112
113test_psync {backlog expired} 3 100000000 1 3 {
114    assert {[s -1 sync_partial_err] > 0}
115}
116