1# Copyright (c) 2011-present, Facebook, Inc.  All rights reserved.
2#  This source code is licensed under both the GPLv2 (found in the
3#  COPYING file in the root directory) and Apache 2.0 License
4#  (found in the LICENSE.Apache file in the root directory).
5
6from advisor.db_log_parser import NO_COL_FAMILY
7from advisor.db_options_parser import DatabaseOptions
8from advisor.rule_parser import Condition, OptionCondition
9import os
10import unittest
11
12
13class TestDatabaseOptions(unittest.TestCase):
14    def setUp(self):
15        self.this_path = os.path.abspath(os.path.dirname(__file__))
16        self.og_options = os.path.join(
17            self.this_path, 'input_files/OPTIONS-000005'
18        )
19        misc_options = [
20            'bloom_bits = 4', 'rate_limiter_bytes_per_sec = 1024000'
21        ]
22        # create the options object
23        self.db_options = DatabaseOptions(self.og_options, misc_options)
24        # perform clean-up before running tests
25        self.generated_options = os.path.join(
26            self.this_path, '../temp/OPTIONS_testing.tmp'
27        )
28        if os.path.isfile(self.generated_options):
29            os.remove(self.generated_options)
30
31    def test_get_options_diff(self):
32        old_opt = {
33            'DBOptions.stats_dump_freq_sec': {NO_COL_FAMILY: '20'},
34            'CFOptions.write_buffer_size': {
35                'default': '1024000',
36                'col_fam_A': '128000',
37                'col_fam_B': '128000000'
38            },
39            'DBOptions.use_fsync': {NO_COL_FAMILY: 'true'},
40            'DBOptions.max_log_file_size': {NO_COL_FAMILY: '128000000'}
41        }
42        new_opt = {
43            'bloom_bits': {NO_COL_FAMILY: '4'},
44            'CFOptions.write_buffer_size': {
45                'default': '128000000',
46                'col_fam_A': '128000',
47                'col_fam_C': '128000000'
48            },
49            'DBOptions.use_fsync': {NO_COL_FAMILY: 'true'},
50            'DBOptions.max_log_file_size': {NO_COL_FAMILY: '0'}
51        }
52        diff = DatabaseOptions.get_options_diff(old_opt, new_opt)
53
54        expected_diff = {
55            'DBOptions.stats_dump_freq_sec': {NO_COL_FAMILY: ('20', None)},
56            'bloom_bits': {NO_COL_FAMILY: (None, '4')},
57            'CFOptions.write_buffer_size': {
58                'default': ('1024000', '128000000'),
59                'col_fam_B': ('128000000', None),
60                'col_fam_C': (None, '128000000')
61            },
62            'DBOptions.max_log_file_size': {NO_COL_FAMILY: ('128000000', '0')}
63        }
64        self.assertDictEqual(diff, expected_diff)
65
66    def test_is_misc_option(self):
67        self.assertTrue(DatabaseOptions.is_misc_option('bloom_bits'))
68        self.assertFalse(
69            DatabaseOptions.is_misc_option('DBOptions.stats_dump_freq_sec')
70        )
71
72    def test_set_up(self):
73        options = self.db_options.get_all_options()
74        self.assertEqual(22, len(options.keys()))
75        expected_misc_options = {
76            'bloom_bits': '4', 'rate_limiter_bytes_per_sec': '1024000'
77        }
78        self.assertDictEqual(
79            expected_misc_options, self.db_options.get_misc_options()
80        )
81        self.assertListEqual(
82            ['default', 'col_fam_A'], self.db_options.get_column_families()
83        )
84
85    def test_get_options(self):
86        opt_to_get = [
87            'DBOptions.manual_wal_flush', 'DBOptions.db_write_buffer_size',
88            'bloom_bits', 'CFOptions.compaction_filter_factory',
89            'CFOptions.num_levels', 'rate_limiter_bytes_per_sec',
90            'TableOptions.BlockBasedTable.block_align', 'random_option'
91        ]
92        options = self.db_options.get_options(opt_to_get)
93        expected_options = {
94            'DBOptions.manual_wal_flush': {NO_COL_FAMILY: 'false'},
95            'DBOptions.db_write_buffer_size': {NO_COL_FAMILY: '0'},
96            'bloom_bits': {NO_COL_FAMILY: '4'},
97            'CFOptions.compaction_filter_factory': {
98                'default': 'nullptr', 'col_fam_A': 'nullptr'
99            },
100            'CFOptions.num_levels': {'default': '7', 'col_fam_A': '5'},
101            'rate_limiter_bytes_per_sec': {NO_COL_FAMILY: '1024000'},
102            'TableOptions.BlockBasedTable.block_align': {
103                'default': 'false', 'col_fam_A': 'true'
104            }
105        }
106        self.assertDictEqual(expected_options, options)
107
108    def test_update_options(self):
109        # add new, update old, set old
110        # before updating
111        expected_old_opts = {
112            'DBOptions.db_log_dir': {NO_COL_FAMILY: None},
113            'DBOptions.manual_wal_flush': {NO_COL_FAMILY: 'false'},
114            'bloom_bits': {NO_COL_FAMILY: '4'},
115            'CFOptions.num_levels': {'default': '7', 'col_fam_A': '5'},
116            'TableOptions.BlockBasedTable.block_restart_interval': {
117                'col_fam_A': '16'
118            }
119        }
120        get_opts = list(expected_old_opts.keys())
121        options = self.db_options.get_options(get_opts)
122        self.assertEqual(expected_old_opts, options)
123        # after updating options
124        update_opts = {
125            'DBOptions.db_log_dir': {NO_COL_FAMILY: '/dev/shm'},
126            'DBOptions.manual_wal_flush': {NO_COL_FAMILY: 'true'},
127            'bloom_bits': {NO_COL_FAMILY: '2'},
128            'CFOptions.num_levels': {'col_fam_A': '7'},
129            'TableOptions.BlockBasedTable.block_restart_interval': {
130                'default': '32'
131            },
132            'random_misc_option': {NO_COL_FAMILY: 'something'}
133        }
134        self.db_options.update_options(update_opts)
135        update_opts['CFOptions.num_levels']['default'] = '7'
136        update_opts['TableOptions.BlockBasedTable.block_restart_interval'] = {
137            'default': '32', 'col_fam_A': '16'
138        }
139        get_opts.append('random_misc_option')
140        options = self.db_options.get_options(get_opts)
141        self.assertDictEqual(update_opts, options)
142        expected_misc_options = {
143            'bloom_bits': '2',
144            'rate_limiter_bytes_per_sec': '1024000',
145            'random_misc_option': 'something'
146        }
147        self.assertDictEqual(
148            expected_misc_options, self.db_options.get_misc_options()
149        )
150
151    def test_generate_options_config(self):
152        # make sure file does not exist from before
153        self.assertFalse(os.path.isfile(self.generated_options))
154        self.db_options.generate_options_config('testing')
155        self.assertTrue(os.path.isfile(self.generated_options))
156
157    def test_check_and_trigger_conditions(self):
158        # options only from CFOptions
159        # setup the OptionCondition objects to check and trigger
160        update_dict = {
161            'CFOptions.level0_file_num_compaction_trigger': {'col_fam_A': '4'},
162            'CFOptions.max_bytes_for_level_base': {'col_fam_A': '10'}
163        }
164        self.db_options.update_options(update_dict)
165        cond1 = Condition('opt-cond-1')
166        cond1 = OptionCondition.create(cond1)
167        cond1.set_parameter(
168            'options', [
169                'CFOptions.level0_file_num_compaction_trigger',
170                'TableOptions.BlockBasedTable.block_restart_interval',
171                'CFOptions.max_bytes_for_level_base'
172            ]
173        )
174        cond1.set_parameter(
175            'evaluate',
176            'int(options[0])*int(options[1])-int(options[2])>=0'
177        )
178        # only DBOptions
179        cond2 = Condition('opt-cond-2')
180        cond2 = OptionCondition.create(cond2)
181        cond2.set_parameter(
182            'options', [
183                'DBOptions.db_write_buffer_size',
184                'bloom_bits',
185                'rate_limiter_bytes_per_sec'
186            ]
187        )
188        cond2.set_parameter(
189            'evaluate',
190            '(int(options[2]) * int(options[1]) * int(options[0]))==0'
191        )
192        # mix of CFOptions and DBOptions
193        cond3 = Condition('opt-cond-3')
194        cond3 = OptionCondition.create(cond3)
195        cond3.set_parameter(
196            'options', [
197                'DBOptions.db_write_buffer_size',  # 0
198                'CFOptions.num_levels',  # 5, 7
199                'bloom_bits'  # 4
200            ]
201        )
202        cond3.set_parameter(
203            'evaluate', 'int(options[2])*int(options[0])+int(options[1])>6'
204        )
205        self.db_options.check_and_trigger_conditions([cond1, cond2, cond3])
206
207        cond1_trigger = {'col_fam_A': ['4', '16', '10']}
208        self.assertDictEqual(cond1_trigger, cond1.get_trigger())
209        cond2_trigger = {NO_COL_FAMILY: ['0', '4', '1024000']}
210        self.assertDictEqual(cond2_trigger, cond2.get_trigger())
211        cond3_trigger = {'default': ['0', '7', '4']}
212        self.assertDictEqual(cond3_trigger, cond3.get_trigger())
213
214
215if __name__ == '__main__':
216    unittest.main()
217