1#!/usr/bin/env python
2# Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved.
3from __future__ import absolute_import, division, print_function, unicode_literals
4
5import os
6import glob
7import os.path
8import shutil
9import subprocess
10import time
11import unittest
12import tempfile
13import re
14
15def my_check_output(*popenargs, **kwargs):
16    """
17    If we had python 2.7, we should simply use subprocess.check_output.
18    This is a stop-gap solution for python 2.6
19    """
20    if 'stdout' in kwargs:
21        raise ValueError('stdout argument not allowed, it will be overridden.')
22    process = subprocess.Popen(stderr=subprocess.PIPE, stdout=subprocess.PIPE,
23                               *popenargs, **kwargs)
24    output, unused_err = process.communicate()
25    retcode = process.poll()
26    if retcode:
27        cmd = kwargs.get("args")
28        if cmd is None:
29            cmd = popenargs[0]
30        raise Exception("Exit code is not 0.  It is %d.  Command: %s" %
31                (retcode, cmd))
32    return output.decode('utf-8')
33
34def run_err_null(cmd):
35    return os.system(cmd + " 2>/dev/null ")
36
37class LDBTestCase(unittest.TestCase):
38    def setUp(self):
39        self.TMP_DIR  = tempfile.mkdtemp(prefix="ldb_test_")
40        self.DB_NAME = "testdb"
41
42    def tearDown(self):
43        assert(self.TMP_DIR.strip() != "/"
44                and self.TMP_DIR.strip() != "/tmp"
45                and self.TMP_DIR.strip() != "/tmp/") #Just some paranoia
46
47        shutil.rmtree(self.TMP_DIR)
48
49    def dbParam(self, dbName):
50        return "--db=%s" % os.path.join(self.TMP_DIR, dbName)
51
52    def assertRunOKFull(self, params, expectedOutput, unexpected=False,
53                        isPattern=False):
54        """
55        All command-line params must be specified.
56        Allows full flexibility in testing; for example: missing db param.
57        """
58        output = my_check_output("./ldb %s |grep -v \"Created bg thread\"" %
59                            params, shell=True)
60        if not unexpected:
61            if isPattern:
62                self.assertNotEqual(expectedOutput.search(output.strip()),
63                                    None)
64            else:
65                self.assertEqual(output.strip(), expectedOutput.strip())
66        else:
67            if isPattern:
68                self.assertEqual(expectedOutput.search(output.strip()), None)
69            else:
70                self.assertNotEqual(output.strip(), expectedOutput.strip())
71
72    def assertRunFAILFull(self, params):
73        """
74        All command-line params must be specified.
75        Allows full flexibility in testing; for example: missing db param.
76        """
77        try:
78
79            my_check_output("./ldb %s >/dev/null 2>&1 |grep -v \"Created bg \
80                thread\"" % params, shell=True)
81        except Exception:
82            return
83        self.fail(
84            "Exception should have been raised for command with params: %s" %
85            params)
86
87    def assertRunOK(self, params, expectedOutput, unexpected=False):
88        """
89        Uses the default test db.
90        """
91        self.assertRunOKFull("%s %s" % (self.dbParam(self.DB_NAME), params),
92                             expectedOutput, unexpected)
93
94    def assertRunFAIL(self, params):
95        """
96        Uses the default test db.
97        """
98        self.assertRunFAILFull("%s %s" % (self.dbParam(self.DB_NAME), params))
99
100    def testSimpleStringPutGet(self):
101        print("Running testSimpleStringPutGet...")
102        self.assertRunFAIL("put x1 y1")
103        self.assertRunOK("put --create_if_missing x1 y1", "OK")
104        self.assertRunOK("get x1", "y1")
105        self.assertRunFAIL("get x2")
106
107        self.assertRunOK("put x2 y2", "OK")
108        self.assertRunOK("get x1", "y1")
109        self.assertRunOK("get x2", "y2")
110        self.assertRunFAIL("get x3")
111
112        self.assertRunOK("scan --from=x1 --to=z", "x1 : y1\nx2 : y2")
113        self.assertRunOK("put x3 y3", "OK")
114
115        self.assertRunOK("scan --from=x1 --to=z", "x1 : y1\nx2 : y2\nx3 : y3")
116        self.assertRunOK("scan", "x1 : y1\nx2 : y2\nx3 : y3")
117        self.assertRunOK("scan --from=x", "x1 : y1\nx2 : y2\nx3 : y3")
118
119        self.assertRunOK("scan --to=x2", "x1 : y1")
120        self.assertRunOK("scan --from=x1 --to=z --max_keys=1", "x1 : y1")
121        self.assertRunOK("scan --from=x1 --to=z --max_keys=2",
122                "x1 : y1\nx2 : y2")
123
124        self.assertRunOK("scan --from=x1 --to=z --max_keys=3",
125                "x1 : y1\nx2 : y2\nx3 : y3")
126        self.assertRunOK("scan --from=x1 --to=z --max_keys=4",
127                "x1 : y1\nx2 : y2\nx3 : y3")
128        self.assertRunOK("scan --from=x1 --to=x2", "x1 : y1")
129        self.assertRunOK("scan --from=x2 --to=x4", "x2 : y2\nx3 : y3")
130        self.assertRunFAIL("scan --from=x4 --to=z") # No results => FAIL
131        self.assertRunFAIL("scan --from=x1 --to=z --max_keys=foo")
132
133        self.assertRunOK("scan", "x1 : y1\nx2 : y2\nx3 : y3")
134
135        self.assertRunOK("delete x1", "OK")
136        self.assertRunOK("scan", "x2 : y2\nx3 : y3")
137
138        self.assertRunOK("delete NonExistentKey", "OK")
139        # It is weird that GET and SCAN raise exception for
140        # non-existent key, while delete does not
141
142        self.assertRunOK("checkconsistency", "OK")
143
144    def dumpDb(self, params, dumpFile):
145        return 0 == run_err_null("./ldb dump %s > %s" % (params, dumpFile))
146
147    def loadDb(self, params, dumpFile):
148        return 0 == run_err_null("cat %s | ./ldb load %s" % (dumpFile, params))
149
150    def writeExternSst(self, params, inputDumpFile, outputSst):
151        return 0 == run_err_null("cat %s | ./ldb write_extern_sst %s %s"
152                % (inputDumpFile, outputSst, params))
153
154    def ingestExternSst(self, params, inputSst):
155        return 0 == run_err_null("./ldb ingest_extern_sst %s %s"
156                                     % (inputSst, params))
157
158    def testStringBatchPut(self):
159        print("Running testStringBatchPut...")
160        self.assertRunOK("batchput x1 y1 --create_if_missing", "OK")
161        self.assertRunOK("scan", "x1 : y1")
162        self.assertRunOK("batchput x2 y2 x3 y3 \"x4 abc\" \"y4 xyz\"", "OK")
163        self.assertRunOK("scan", "x1 : y1\nx2 : y2\nx3 : y3\nx4 abc : y4 xyz")
164        self.assertRunFAIL("batchput")
165        self.assertRunFAIL("batchput k1")
166        self.assertRunFAIL("batchput k1 v1 k2")
167
168    def testCountDelimDump(self):
169        print("Running testCountDelimDump...")
170        self.assertRunOK("batchput x.1 x1 --create_if_missing", "OK")
171        self.assertRunOK("batchput y.abc abc y.2 2 z.13c pqr", "OK")
172        self.assertRunOK("dump --count_delim", "x => count:1\tsize:5\ny => count:2\tsize:12\nz => count:1\tsize:8")
173        self.assertRunOK("dump --count_delim=\".\"", "x => count:1\tsize:5\ny => count:2\tsize:12\nz => count:1\tsize:8")
174        self.assertRunOK("batchput x,2 x2 x,abc xabc", "OK")
175        self.assertRunOK("dump --count_delim=\",\"", "x => count:2\tsize:14\nx.1 => count:1\tsize:5\ny.2 => count:1\tsize:4\ny.abc => count:1\tsize:8\nz.13c => count:1\tsize:8")
176
177    def testCountDelimIDump(self):
178        print("Running testCountDelimIDump...")
179        self.assertRunOK("batchput x.1 x1 --create_if_missing", "OK")
180        self.assertRunOK("batchput y.abc abc y.2 2 z.13c pqr", "OK")
181        self.assertRunOK("idump --count_delim", "x => count:1\tsize:5\ny => count:2\tsize:12\nz => count:1\tsize:8")
182        self.assertRunOK("idump --count_delim=\".\"", "x => count:1\tsize:5\ny => count:2\tsize:12\nz => count:1\tsize:8")
183        self.assertRunOK("batchput x,2 x2 x,abc xabc", "OK")
184        self.assertRunOK("idump --count_delim=\",\"", "x => count:2\tsize:14\nx.1 => count:1\tsize:5\ny.2 => count:1\tsize:4\ny.abc => count:1\tsize:8\nz.13c => count:1\tsize:8")
185
186    def testInvalidCmdLines(self):
187        print("Running testInvalidCmdLines...")
188        # db not specified
189        self.assertRunFAILFull("put 0x6133 0x6233 --hex --create_if_missing")
190        # No param called he
191        self.assertRunFAIL("put 0x6133 0x6233 --he --create_if_missing")
192        # max_keys is not applicable for put
193        self.assertRunFAIL("put 0x6133 0x6233 --max_keys=1 --create_if_missing")
194        # hex has invalid boolean value
195
196    def testHexPutGet(self):
197        print("Running testHexPutGet...")
198        self.assertRunOK("put a1 b1 --create_if_missing", "OK")
199        self.assertRunOK("scan", "a1 : b1")
200        self.assertRunOK("scan --hex", "0x6131 : 0x6231")
201        self.assertRunFAIL("put --hex 6132 6232")
202        self.assertRunOK("put --hex 0x6132 0x6232", "OK")
203        self.assertRunOK("scan --hex", "0x6131 : 0x6231\n0x6132 : 0x6232")
204        self.assertRunOK("scan", "a1 : b1\na2 : b2")
205        self.assertRunOK("get a1", "b1")
206        self.assertRunOK("get --hex 0x6131", "0x6231")
207        self.assertRunOK("get a2", "b2")
208        self.assertRunOK("get --hex 0x6132", "0x6232")
209        self.assertRunOK("get --key_hex 0x6132", "b2")
210        self.assertRunOK("get --key_hex --value_hex 0x6132", "0x6232")
211        self.assertRunOK("get --value_hex a2", "0x6232")
212        self.assertRunOK("scan --key_hex --value_hex",
213                "0x6131 : 0x6231\n0x6132 : 0x6232")
214        self.assertRunOK("scan --hex --from=0x6131 --to=0x6133",
215                "0x6131 : 0x6231\n0x6132 : 0x6232")
216        self.assertRunOK("scan --hex --from=0x6131 --to=0x6132",
217                "0x6131 : 0x6231")
218        self.assertRunOK("scan --key_hex", "0x6131 : b1\n0x6132 : b2")
219        self.assertRunOK("scan --value_hex", "a1 : 0x6231\na2 : 0x6232")
220        self.assertRunOK("batchput --hex 0x6133 0x6233 0x6134 0x6234", "OK")
221        self.assertRunOK("scan", "a1 : b1\na2 : b2\na3 : b3\na4 : b4")
222        self.assertRunOK("delete --hex 0x6133", "OK")
223        self.assertRunOK("scan", "a1 : b1\na2 : b2\na4 : b4")
224        self.assertRunOK("checkconsistency", "OK")
225
226    def testTtlPutGet(self):
227        print("Running testTtlPutGet...")
228        self.assertRunOK("put a1 b1 --ttl --create_if_missing", "OK")
229        self.assertRunOK("scan --hex", "0x6131 : 0x6231", True)
230        self.assertRunOK("dump --ttl ", "a1 ==> b1", True)
231        self.assertRunOK("dump --hex --ttl ",
232                         "0x6131 ==> 0x6231\nKeys in range: 1")
233        self.assertRunOK("scan --hex --ttl", "0x6131 : 0x6231")
234        self.assertRunOK("get --value_hex a1", "0x6231", True)
235        self.assertRunOK("get --ttl a1", "b1")
236        self.assertRunOK("put a3 b3 --create_if_missing", "OK")
237        # fails because timstamp's length is greater than value's
238        self.assertRunFAIL("get --ttl a3")
239        self.assertRunOK("checkconsistency", "OK")
240
241    def testInvalidCmdLines(self):  # noqa: F811 T25377293 Grandfathered in
242        print("Running testInvalidCmdLines...")
243        # db not specified
244        self.assertRunFAILFull("put 0x6133 0x6233 --hex --create_if_missing")
245        # No param called he
246        self.assertRunFAIL("put 0x6133 0x6233 --he --create_if_missing")
247        # max_keys is not applicable for put
248        self.assertRunFAIL("put 0x6133 0x6233 --max_keys=1 --create_if_missing")
249        # hex has invalid boolean value
250        self.assertRunFAIL("put 0x6133 0x6233 --hex=Boo --create_if_missing")
251
252    def testDumpLoad(self):
253        print("Running testDumpLoad...")
254        self.assertRunOK("batchput --create_if_missing x1 y1 x2 y2 x3 y3 x4 y4",
255                "OK")
256        self.assertRunOK("scan", "x1 : y1\nx2 : y2\nx3 : y3\nx4 : y4")
257        origDbPath = os.path.join(self.TMP_DIR, self.DB_NAME)
258
259        # Dump and load without any additional params specified
260        dumpFilePath = os.path.join(self.TMP_DIR, "dump1")
261        loadedDbPath = os.path.join(self.TMP_DIR, "loaded_from_dump1")
262        self.assertTrue(self.dumpDb("--db=%s" % origDbPath, dumpFilePath))
263        self.assertTrue(self.loadDb(
264            "--db=%s --create_if_missing" % loadedDbPath, dumpFilePath))
265        self.assertRunOKFull("scan --db=%s" % loadedDbPath,
266                "x1 : y1\nx2 : y2\nx3 : y3\nx4 : y4")
267
268        # Dump and load in hex
269        dumpFilePath = os.path.join(self.TMP_DIR, "dump2")
270        loadedDbPath = os.path.join(self.TMP_DIR, "loaded_from_dump2")
271        self.assertTrue(self.dumpDb("--db=%s --hex" % origDbPath, dumpFilePath))
272        self.assertTrue(self.loadDb(
273            "--db=%s --hex --create_if_missing" % loadedDbPath, dumpFilePath))
274        self.assertRunOKFull("scan --db=%s" % loadedDbPath,
275                "x1 : y1\nx2 : y2\nx3 : y3\nx4 : y4")
276
277        # Dump only a portion of the key range
278        dumpFilePath = os.path.join(self.TMP_DIR, "dump3")
279        loadedDbPath = os.path.join(self.TMP_DIR, "loaded_from_dump3")
280        self.assertTrue(self.dumpDb(
281            "--db=%s --from=x1 --to=x3" % origDbPath, dumpFilePath))
282        self.assertTrue(self.loadDb(
283            "--db=%s --create_if_missing" % loadedDbPath, dumpFilePath))
284        self.assertRunOKFull("scan --db=%s" % loadedDbPath, "x1 : y1\nx2 : y2")
285
286        # Dump upto max_keys rows
287        dumpFilePath = os.path.join(self.TMP_DIR, "dump4")
288        loadedDbPath = os.path.join(self.TMP_DIR, "loaded_from_dump4")
289        self.assertTrue(self.dumpDb(
290            "--db=%s --max_keys=3" % origDbPath, dumpFilePath))
291        self.assertTrue(self.loadDb(
292            "--db=%s --create_if_missing" % loadedDbPath, dumpFilePath))
293        self.assertRunOKFull("scan --db=%s" % loadedDbPath,
294                "x1 : y1\nx2 : y2\nx3 : y3")
295
296        # Load into an existing db, create_if_missing is not specified
297        self.assertTrue(self.dumpDb("--db=%s" % origDbPath, dumpFilePath))
298        self.assertTrue(self.loadDb("--db=%s" % loadedDbPath, dumpFilePath))
299        self.assertRunOKFull("scan --db=%s" % loadedDbPath,
300                "x1 : y1\nx2 : y2\nx3 : y3\nx4 : y4")
301
302        # Dump and load with WAL disabled
303        dumpFilePath = os.path.join(self.TMP_DIR, "dump5")
304        loadedDbPath = os.path.join(self.TMP_DIR, "loaded_from_dump5")
305        self.assertTrue(self.dumpDb("--db=%s" % origDbPath, dumpFilePath))
306        self.assertTrue(self.loadDb(
307            "--db=%s --disable_wal --create_if_missing" % loadedDbPath,
308            dumpFilePath))
309        self.assertRunOKFull("scan --db=%s" % loadedDbPath,
310                "x1 : y1\nx2 : y2\nx3 : y3\nx4 : y4")
311
312        # Dump and load with lots of extra params specified
313        extraParams = " ".join(["--bloom_bits=14", "--block_size=1024",
314                                "--auto_compaction=true",
315                                "--write_buffer_size=4194304",
316                                "--file_size=2097152"])
317        dumpFilePath = os.path.join(self.TMP_DIR, "dump6")
318        loadedDbPath = os.path.join(self.TMP_DIR, "loaded_from_dump6")
319        self.assertTrue(self.dumpDb(
320            "--db=%s %s" % (origDbPath, extraParams), dumpFilePath))
321        self.assertTrue(self.loadDb(
322            "--db=%s %s --create_if_missing" % (loadedDbPath, extraParams),
323            dumpFilePath))
324        self.assertRunOKFull("scan --db=%s" % loadedDbPath,
325                "x1 : y1\nx2 : y2\nx3 : y3\nx4 : y4")
326
327        # Dump with count_only
328        dumpFilePath = os.path.join(self.TMP_DIR, "dump7")
329        loadedDbPath = os.path.join(self.TMP_DIR, "loaded_from_dump7")
330        self.assertTrue(self.dumpDb(
331            "--db=%s --count_only" % origDbPath, dumpFilePath))
332        self.assertTrue(self.loadDb(
333            "--db=%s --create_if_missing" % loadedDbPath, dumpFilePath))
334        # DB should have atleast one value for scan to work
335        self.assertRunOKFull("put --db=%s k1 v1" % loadedDbPath, "OK")
336        self.assertRunOKFull("scan --db=%s" % loadedDbPath, "k1 : v1")
337
338        # Dump command fails because of typo in params
339        dumpFilePath = os.path.join(self.TMP_DIR, "dump8")
340        self.assertFalse(self.dumpDb(
341            "--db=%s --create_if_missing" % origDbPath, dumpFilePath))
342
343    def testIDumpBasics(self):
344        print("Running testIDumpBasics...")
345        self.assertRunOK("put a val --create_if_missing", "OK")
346        self.assertRunOK("put b val", "OK")
347        self.assertRunOK(
348                "idump", "'a' seq:1, type:1 => val\n"
349                "'b' seq:2, type:1 => val\nInternal keys in range: 2")
350        self.assertRunOK(
351                "idump --input_key_hex --from=%s --to=%s" % (hex(ord('a')),
352                                                             hex(ord('b'))),
353                "'a' seq:1, type:1 => val\nInternal keys in range: 1")
354
355    def testMiscAdminTask(self):
356        print("Running testMiscAdminTask...")
357        # These tests need to be improved; for example with asserts about
358        # whether compaction or level reduction actually took place.
359        self.assertRunOK("batchput --create_if_missing x1 y1 x2 y2 x3 y3 x4 y4",
360                "OK")
361        self.assertRunOK("scan", "x1 : y1\nx2 : y2\nx3 : y3\nx4 : y4")
362        origDbPath = os.path.join(self.TMP_DIR, self.DB_NAME)
363
364        self.assertTrue(0 == run_err_null(
365            "./ldb compact --db=%s" % origDbPath))
366        self.assertRunOK("scan", "x1 : y1\nx2 : y2\nx3 : y3\nx4 : y4")
367
368        self.assertTrue(0 == run_err_null(
369            "./ldb reduce_levels --db=%s --new_levels=2" % origDbPath))
370        self.assertRunOK("scan", "x1 : y1\nx2 : y2\nx3 : y3\nx4 : y4")
371
372        self.assertTrue(0 == run_err_null(
373            "./ldb reduce_levels --db=%s --new_levels=3" % origDbPath))
374        self.assertRunOK("scan", "x1 : y1\nx2 : y2\nx3 : y3\nx4 : y4")
375
376        self.assertTrue(0 == run_err_null(
377            "./ldb compact --db=%s --from=x1 --to=x3" % origDbPath))
378        self.assertRunOK("scan", "x1 : y1\nx2 : y2\nx3 : y3\nx4 : y4")
379
380        self.assertTrue(0 == run_err_null(
381            "./ldb compact --db=%s --hex --from=0x6131 --to=0x6134"
382            % origDbPath))
383        self.assertRunOK("scan", "x1 : y1\nx2 : y2\nx3 : y3\nx4 : y4")
384
385        #TODO(dilip): Not sure what should be passed to WAL.Currently corrupted.
386        self.assertTrue(0 == run_err_null(
387            "./ldb dump_wal --db=%s --walfile=%s --header" % (
388                origDbPath, os.path.join(origDbPath, "LOG"))))
389        self.assertRunOK("scan", "x1 : y1\nx2 : y2\nx3 : y3\nx4 : y4")
390
391    def testCheckConsistency(self):
392        print("Running testCheckConsistency...")
393
394        dbPath = os.path.join(self.TMP_DIR, self.DB_NAME)
395        self.assertRunOK("put x1 y1 --create_if_missing", "OK")
396        self.assertRunOK("put x2 y2", "OK")
397        self.assertRunOK("get x1", "y1")
398        self.assertRunOK("checkconsistency", "OK")
399
400        sstFilePath = my_check_output("ls %s" % os.path.join(dbPath, "*.sst"),
401                                      shell=True)
402
403        # Modify the file
404        my_check_output("echo 'evil' > %s" % sstFilePath, shell=True)
405        self.assertRunFAIL("checkconsistency")
406
407        # Delete the file
408        my_check_output("rm -f %s" % sstFilePath, shell=True)
409        self.assertRunFAIL("checkconsistency")
410
411    def dumpLiveFiles(self, params, dumpFile):
412        return 0 == run_err_null("./ldb dump_live_files %s > %s" % (
413            params, dumpFile))
414
415    def testDumpLiveFiles(self):
416        print("Running testDumpLiveFiles...")
417
418        dbPath = os.path.join(self.TMP_DIR, self.DB_NAME)
419        self.assertRunOK("put x1 y1 --create_if_missing", "OK")
420        self.assertRunOK("put x2 y2", "OK")
421        dumpFilePath = os.path.join(self.TMP_DIR, "dump1")
422        self.assertTrue(self.dumpLiveFiles("--db=%s" % dbPath, dumpFilePath))
423        self.assertRunOK("delete x1", "OK")
424        self.assertRunOK("put x3 y3", "OK")
425        dumpFilePath = os.path.join(self.TMP_DIR, "dump2")
426        self.assertTrue(self.dumpLiveFiles("--db=%s" % dbPath, dumpFilePath))
427
428    def getManifests(self, directory):
429        return glob.glob(directory + "/MANIFEST-*")
430
431    def getSSTFiles(self, directory):
432        return glob.glob(directory + "/*.sst")
433
434    def getWALFiles(self, directory):
435        return glob.glob(directory + "/*.log")
436
437    def copyManifests(self, src, dest):
438        return 0 == run_err_null("cp " + src + " " + dest)
439
440    def testManifestDump(self):
441        print("Running testManifestDump...")
442        dbPath = os.path.join(self.TMP_DIR, self.DB_NAME)
443        self.assertRunOK("put 1 1 --create_if_missing", "OK")
444        self.assertRunOK("put 2 2", "OK")
445        self.assertRunOK("put 3 3", "OK")
446        # Pattern to expect from manifest_dump.
447        num = "[0-9]+"
448        st = ".*"
449        subpat = st + " seq:" + num + ", type:" + num
450        regex = num + ":" + num + "\[" + subpat + ".." + subpat + "\]"
451        expected_pattern = re.compile(regex)
452        cmd = "manifest_dump --db=%s"
453        manifest_files = self.getManifests(dbPath)
454        self.assertTrue(len(manifest_files) == 1)
455        # Test with the default manifest file in dbPath.
456        self.assertRunOKFull(cmd % dbPath, expected_pattern,
457                             unexpected=False, isPattern=True)
458        self.copyManifests(manifest_files[0], manifest_files[0] + "1")
459        manifest_files = self.getManifests(dbPath)
460        self.assertTrue(len(manifest_files) == 2)
461        # Test with multiple manifest files in dbPath.
462        self.assertRunFAILFull(cmd % dbPath)
463        # Running it with the copy we just created should pass.
464        self.assertRunOKFull((cmd + " --path=%s")
465                             % (dbPath, manifest_files[1]),
466                             expected_pattern, unexpected=False,
467                             isPattern=True)
468        # Make sure that using the dump with --path will result in identical
469        # output as just using manifest_dump.
470        cmd = "dump --path=%s"
471        self.assertRunOKFull((cmd)
472                             % (manifest_files[1]),
473                             expected_pattern, unexpected=False,
474                             isPattern=True)
475
476    def testSSTDump(self):
477        print("Running testSSTDump...")
478
479        dbPath = os.path.join(self.TMP_DIR, self.DB_NAME)
480        self.assertRunOK("put sst1 sst1_val --create_if_missing", "OK")
481        self.assertRunOK("put sst2 sst2_val", "OK")
482        self.assertRunOK("get sst1", "sst1_val")
483
484        # Pattern to expect from SST dump.
485        regex = ".*Sst file format:.*"
486        expected_pattern = re.compile(regex)
487
488        sst_files = self.getSSTFiles(dbPath)
489        self.assertTrue(len(sst_files) >= 1)
490        cmd = "dump --path=%s"
491        self.assertRunOKFull((cmd)
492                             % (sst_files[0]),
493                             expected_pattern, unexpected=False,
494                             isPattern=True)
495
496    def testWALDump(self):
497        print("Running testWALDump...")
498
499        dbPath = os.path.join(self.TMP_DIR, self.DB_NAME)
500        self.assertRunOK("put wal1 wal1_val --create_if_missing", "OK")
501        self.assertRunOK("put wal2 wal2_val", "OK")
502        self.assertRunOK("get wal1", "wal1_val")
503
504        # Pattern to expect from WAL dump.
505        regex = "^Sequence,Count,ByteSize,Physical Offset,Key\(s\).*"
506        expected_pattern = re.compile(regex)
507
508        wal_files = self.getWALFiles(dbPath)
509        self.assertTrue(len(wal_files) >= 1)
510        cmd = "dump --path=%s"
511        self.assertRunOKFull((cmd)
512                             % (wal_files[0]),
513                             expected_pattern, unexpected=False,
514                             isPattern=True)
515
516    def testListColumnFamilies(self):
517        print("Running testListColumnFamilies...")
518        self.assertRunOK("put x1 y1 --create_if_missing", "OK")
519        cmd = "list_column_families | grep -v \"Column families\""
520        # Test on valid dbPath.
521        self.assertRunOK(cmd, "{default}")
522        # Test on empty path.
523        self.assertRunFAIL(cmd)
524
525    def testColumnFamilies(self):
526        print("Running testColumnFamilies...")
527        dbPath = os.path.join(self.TMP_DIR, self.DB_NAME)  # noqa: F841 T25377293 Grandfathered in
528        self.assertRunOK("put cf1_1 1 --create_if_missing", "OK")
529        self.assertRunOK("put cf1_2 2 --create_if_missing", "OK")
530        self.assertRunOK("put cf1_3 3 --try_load_options", "OK")
531        # Given non-default column family to single CF DB.
532        self.assertRunFAIL("get cf1_1 --column_family=two")
533        self.assertRunOK("create_column_family two", "OK")
534        self.assertRunOK("put cf2_1 1 --create_if_missing --column_family=two",
535                         "OK")
536        self.assertRunOK("put cf2_2 2 --create_if_missing --column_family=two",
537                         "OK")
538        self.assertRunOK("delete cf1_2", "OK")
539        self.assertRunOK("create_column_family three", "OK")
540        self.assertRunOK("delete cf2_2 --column_family=two", "OK")
541        self.assertRunOK(
542            "put cf3_1 3 --create_if_missing --column_family=three",
543            "OK")
544        self.assertRunOK("get cf1_1 --column_family=default", "1")
545        self.assertRunOK("dump --column_family=two",
546                         "cf2_1 ==> 1\nKeys in range: 1")
547        self.assertRunOK("dump --column_family=two --try_load_options",
548                         "cf2_1 ==> 1\nKeys in range: 1")
549        self.assertRunOK("dump",
550                         "cf1_1 ==> 1\ncf1_3 ==> 3\nKeys in range: 2")
551        self.assertRunOK("get cf2_1 --column_family=two",
552                         "1")
553        self.assertRunOK("get cf3_1 --column_family=three",
554                         "3")
555        self.assertRunOK("drop_column_family three", "OK")
556        # non-existing column family.
557        self.assertRunFAIL("get cf3_1 --column_family=four")
558        self.assertRunFAIL("drop_column_family four")
559
560    def testIngestExternalSst(self):
561        print("Running testIngestExternalSst...")
562
563        # Dump, load, write external sst and ingest it in another db
564        dbPath = os.path.join(self.TMP_DIR, "db1")
565        self.assertRunOK(
566            "batchput --db=%s --create_if_missing x1 y1 x2 y2 x3 y3 x4 y4"
567            % dbPath,
568            "OK")
569        self.assertRunOK("scan --db=%s" % dbPath,
570                         "x1 : y1\nx2 : y2\nx3 : y3\nx4 : y4")
571        dumpFilePath = os.path.join(self.TMP_DIR, "dump1")
572        with open(dumpFilePath, 'w') as f:
573            f.write("x1 ==> y10\nx2 ==> y20\nx3 ==> y30\nx4 ==> y40")
574        externSstPath = os.path.join(self.TMP_DIR, "extern_data1.sst")
575        self.assertTrue(self.writeExternSst("--create_if_missing --db=%s"
576                            % dbPath,
577                        dumpFilePath,
578                        externSstPath))
579        # cannot ingest if allow_global_seqno is false
580        self.assertFalse(
581            self.ingestExternSst(
582                "--create_if_missing --allow_global_seqno=false --db=%s"
583                % dbPath,
584                externSstPath))
585        self.assertTrue(
586            self.ingestExternSst(
587                "--create_if_missing --allow_global_seqno --db=%s"
588                % dbPath,
589                externSstPath))
590        self.assertRunOKFull("scan --db=%s" % dbPath,
591                             "x1 : y10\nx2 : y20\nx3 : y30\nx4 : y40")
592
593if __name__ == "__main__":
594    unittest.main()
595