1 /*
2  * Copyright (c) 2016-present, Yann Collet, Facebook, Inc.
3  * All rights reserved.
4  *
5  * This source code is licensed under both the BSD-style license (found in the
6  * LICENSE file in the root directory of this source tree) and the GPLv2 (found
7  * in the COPYING file in the root directory of this source tree).
8  * You may select, at your option, one of the above-listed licenses.
9  */
10 
11 
12 /*-************************************
13 *  Tuning parameters
14 **************************************/
15 #ifndef ZSTDCLI_CLEVEL_DEFAULT
16 #  define ZSTDCLI_CLEVEL_DEFAULT 3
17 #endif
18 
19 #ifndef ZSTDCLI_CLEVEL_MAX
20 #  define ZSTDCLI_CLEVEL_MAX 19   /* without using --ultra */
21 #endif
22 
23 
24 
25 /*-************************************
26 *  Dependencies
27 **************************************/
28 #include "platform.h" /* IS_CONSOLE, PLATFORM_POSIX_VERSION */
29 #include "util.h"     /* UTIL_HAS_CREATEFILELIST, UTIL_createFileList */
30 #include <stdio.h>    /* fprintf(), stdin, stdout, stderr */
31 #include <string.h>   /* strcmp, strlen */
32 #include <errno.h>    /* errno */
33 #include "fileio.h"   /* stdinmark, stdoutmark, ZSTD_EXTENSION */
34 #ifndef ZSTD_NOBENCH
35 #  include "bench.h"  /* BMK_benchFiles, BMK_SetNbSeconds */
36 #endif
37 #ifndef ZSTD_NODICT
38 #  include "dibio.h"  /* ZDICT_cover_params_t, DiB_trainFromFiles() */
39 #endif
40 #define ZSTD_STATIC_LINKING_ONLY   /* ZSTD_maxCLevel */
41 #include "zstd.h"     /* ZSTD_VERSION_STRING */
42 
43 
44 /*-************************************
45 *  Constants
46 **************************************/
47 #define COMPRESSOR_NAME "zstd command line interface"
48 #ifndef ZSTD_VERSION
49 #  define ZSTD_VERSION "v" ZSTD_VERSION_STRING
50 #endif
51 #define AUTHOR "Yann Collet"
52 #define WELCOME_MESSAGE "*** %s %i-bits %s, by %s ***\n", COMPRESSOR_NAME, (int)(sizeof(size_t)*8), ZSTD_VERSION, AUTHOR
53 
54 #define ZSTD_ZSTDMT "zstdmt"
55 #define ZSTD_UNZSTD "unzstd"
56 #define ZSTD_CAT "zstdcat"
57 #define ZSTD_ZCAT "zcat"
58 #define ZSTD_GZ "gzip"
59 #define ZSTD_GUNZIP "gunzip"
60 #define ZSTD_GZCAT "gzcat"
61 #define ZSTD_LZMA "lzma"
62 #define ZSTD_UNLZMA "unlzma"
63 #define ZSTD_XZ "xz"
64 #define ZSTD_UNXZ "unxz"
65 #define ZSTD_LZ4 "lz4"
66 #define ZSTD_UNLZ4 "unlz4"
67 
68 #define KB *(1 <<10)
69 #define MB *(1 <<20)
70 #define GB *(1U<<30)
71 
72 #define DISPLAY_LEVEL_DEFAULT 2
73 
74 static const char*    g_defaultDictName = "dictionary";
75 static const unsigned g_defaultMaxDictSize = 110 KB;
76 static const int      g_defaultDictCLevel = 3;
77 static const unsigned g_defaultSelectivityLevel = 9;
78 static const unsigned g_defaultMaxWindowLog = 27;
79 #define OVERLAP_LOG_DEFAULT 9999
80 #define LDM_PARAM_DEFAULT 9999  /* Default for parameters where 0 is valid */
81 static U32 g_overlapLog = OVERLAP_LOG_DEFAULT;
82 static U32 g_ldmHashLog = 0;
83 static U32 g_ldmMinMatch = 0;
84 static U32 g_ldmHashEveryLog = LDM_PARAM_DEFAULT;
85 static U32 g_ldmBucketSizeLog = LDM_PARAM_DEFAULT;
86 
87 
88 /*-************************************
89 *  Display Macros
90 **************************************/
91 #define DISPLAY(...)         fprintf(g_displayOut, __VA_ARGS__)
92 #define DISPLAYLEVEL(l, ...) { if (g_displayLevel>=l) { DISPLAY(__VA_ARGS__); } }
93 static int g_displayLevel = DISPLAY_LEVEL_DEFAULT;   /* 0 : no display,  1: errors,  2 : + result + interaction + warnings,  3 : + progression,  4 : + information */
94 static FILE* g_displayOut;
95 
96 
97 /*-************************************
98 *  Command Line
99 **************************************/
usage(const char * programName)100 static int usage(const char* programName)
101 {
102     DISPLAY( "Usage : \n");
103     DISPLAY( "      %s [args] [FILE(s)] [-o file] \n", programName);
104     DISPLAY( "\n");
105     DISPLAY( "FILE    : a filename \n");
106     DISPLAY( "          with no FILE, or when FILE is - , read standard input\n");
107     DISPLAY( "Arguments : \n");
108 #ifndef ZSTD_NOCOMPRESS
109     DISPLAY( " -#     : # compression level (1-%d, default: %d) \n", ZSTDCLI_CLEVEL_MAX, ZSTDCLI_CLEVEL_DEFAULT);
110 #endif
111 #ifndef ZSTD_NODECOMPRESS
112     DISPLAY( " -d     : decompression \n");
113 #endif
114     DISPLAY( " -D file: use `file` as Dictionary \n");
115     DISPLAY( " -o file: result stored into `file` (only if 1 input file) \n");
116     DISPLAY( " -f     : overwrite output without prompting and (de)compress links \n");
117     DISPLAY( "--rm    : remove source file(s) after successful de/compression \n");
118     DISPLAY( " -k     : preserve source file(s) (default) \n");
119     DISPLAY( " -h/-H  : display help/long help and exit \n");
120     return 0;
121 }
122 
usage_advanced(const char * programName)123 static int usage_advanced(const char* programName)
124 {
125     DISPLAY(WELCOME_MESSAGE);
126     usage(programName);
127     DISPLAY( "\n");
128     DISPLAY( "Advanced arguments : \n");
129     DISPLAY( " -V     : display Version number and exit \n");
130     DISPLAY( " -v     : verbose mode; specify multiple times to increase verbosity\n");
131     DISPLAY( " -q     : suppress warnings; specify twice to suppress errors too\n");
132     DISPLAY( " -c     : force write to standard output, even if it is the console\n");
133     DISPLAY( " -l     : print information about zstd compressed files \n");
134 #ifndef ZSTD_NOCOMPRESS
135     DISPLAY( "--ultra : enable levels beyond %i, up to %i (requires more memory)\n", ZSTDCLI_CLEVEL_MAX, ZSTD_maxCLevel());
136     DISPLAY( "--long[=#]: enable long distance matching with given window log (default: %u)\n", g_defaultMaxWindowLog);
137     DISPLAY( "--fast[=#]: switch to ultra fast compression level (default: %u)\n", 1);
138 #ifdef ZSTD_MULTITHREAD
139     DISPLAY( " -T#    : spawns # compression threads (default: 1, 0==# cores) \n");
140     DISPLAY( " -B#    : select size of each job (default: 0==automatic) \n");
141 #endif
142     DISPLAY( "--no-dictID : don't write dictID into header (dictionary compression)\n");
143     DISPLAY( "--[no-]check : integrity check (default: enabled) \n");
144 #endif
145 #ifdef UTIL_HAS_CREATEFILELIST
146     DISPLAY( " -r     : operate recursively on directories \n");
147 #endif
148 #ifdef ZSTD_GZCOMPRESS
149     DISPLAY( "--format=gzip : compress files to the .gz format \n");
150 #endif
151 #ifdef ZSTD_LZMACOMPRESS
152     DISPLAY( "--format=xz : compress files to the .xz format \n");
153     DISPLAY( "--format=lzma : compress files to the .lzma format \n");
154 #endif
155 #ifdef ZSTD_LZ4COMPRESS
156     DISPLAY( "--format=lz4 : compress files to the .lz4 format \n");
157 #endif
158 #ifndef ZSTD_NODECOMPRESS
159     DISPLAY( "--test  : test compressed file integrity \n");
160 #if ZSTD_SPARSE_DEFAULT
161     DISPLAY( "--[no-]sparse : sparse mode (default: enabled on file, disabled on stdout)\n");
162 #else
163     DISPLAY( "--[no-]sparse : sparse mode (default: disabled)\n");
164 #endif
165 #endif
166     DISPLAY( " -M#    : Set a memory usage limit for decompression \n");
167     DISPLAY( "--      : All arguments after \"--\" are treated as files \n");
168 #ifndef ZSTD_NODICT
169     DISPLAY( "\n");
170     DISPLAY( "Dictionary builder : \n");
171     DISPLAY( "--train ## : create a dictionary from a training set of files \n");
172     DISPLAY( "--train-cover[=k=#,d=#,steps=#] : use the cover algorithm with optional args\n");
173     DISPLAY( "--train-legacy[=s=#] : use the legacy algorithm with selectivity (default: %u)\n", g_defaultSelectivityLevel);
174     DISPLAY( " -o file : `file` is dictionary name (default: %s) \n", g_defaultDictName);
175     DISPLAY( "--maxdict=# : limit dictionary to specified size (default: %u) \n", g_defaultMaxDictSize);
176     DISPLAY( "--dictID=# : force dictionary ID to specified value (default: random)\n");
177 #endif
178 #ifndef ZSTD_NOBENCH
179     DISPLAY( "\n");
180     DISPLAY( "Benchmark arguments : \n");
181     DISPLAY( " -b#    : benchmark file(s), using # compression level (default: %d) \n", ZSTDCLI_CLEVEL_DEFAULT);
182     DISPLAY( " -e#    : test all compression levels from -bX to # (default: 1)\n");
183     DISPLAY( " -i#    : minimum evaluation time in seconds (default: 3s) \n");
184     DISPLAY( " -B#    : cut file into independent blocks of size # (default: no block)\n");
185     DISPLAY( "--priority=rt : set process priority to real-time \n");
186 #endif
187     return 0;
188 }
189 
badusage(const char * programName)190 static int badusage(const char* programName)
191 {
192     DISPLAYLEVEL(1, "Incorrect parameters\n");
193     if (g_displayLevel >= 2) usage(programName);
194     return 1;
195 }
196 
waitEnter(void)197 static void waitEnter(void)
198 {
199     int unused;
200     DISPLAY("Press enter to continue...\n");
201     unused = getchar();
202     (void)unused;
203 }
204 
lastNameFromPath(const char * path)205 static const char* lastNameFromPath(const char* path)
206 {
207     const char* name = path;
208     if (strrchr(name, '/')) name = strrchr(name, '/') + 1;
209     if (strrchr(name, '\\')) name = strrchr(name, '\\') + 1; /* windows */
210     return name;
211 }
212 
213 /*! exeNameMatch() :
214     @return : a non-zero value if exeName matches test, excluding the extension
215    */
exeNameMatch(const char * exeName,const char * test)216 static int exeNameMatch(const char* exeName, const char* test)
217 {
218     return !strncmp(exeName, test, strlen(test)) &&
219         (exeName[strlen(test)] == '\0' || exeName[strlen(test)] == '.');
220 }
221 
222 /*! readU32FromChar() :
223  * @return : unsigned integer value read from input in `char` format.
224  *  allows and interprets K, KB, KiB, M, MB and MiB suffix.
225  *  Will also modify `*stringPtr`, advancing it to position where it stopped reading.
226  *  Note : function result can overflow if digit string > MAX_UINT */
readU32FromChar(const char ** stringPtr)227 static unsigned readU32FromChar(const char** stringPtr)
228 {
229     unsigned result = 0;
230     while ((**stringPtr >='0') && (**stringPtr <='9'))
231         result *= 10, result += **stringPtr - '0', (*stringPtr)++ ;
232     if ((**stringPtr=='K') || (**stringPtr=='M')) {
233         result <<= 10;
234         if (**stringPtr=='M') result <<= 10;
235         (*stringPtr)++ ;
236         if (**stringPtr=='i') (*stringPtr)++;
237         if (**stringPtr=='B') (*stringPtr)++;
238     }
239     return result;
240 }
241 
242 /** longCommandWArg() :
243  *  check if *stringPtr is the same as longCommand.
244  *  If yes, @return 1 and advances *stringPtr to the position which immediately follows longCommand.
245  * @return 0 and doesn't modify *stringPtr otherwise.
246  */
longCommandWArg(const char ** stringPtr,const char * longCommand)247 static unsigned longCommandWArg(const char** stringPtr, const char* longCommand)
248 {
249     size_t const comSize = strlen(longCommand);
250     int const result = !strncmp(*stringPtr, longCommand, comSize);
251     if (result) *stringPtr += comSize;
252     return result;
253 }
254 
255 
256 #ifndef ZSTD_NODICT
257 /**
258  * parseCoverParameters() :
259  * reads cover parameters from *stringPtr (e.g. "--train-cover=k=48,d=8,steps=32") into *params
260  * @return 1 means that cover parameters were correct
261  * @return 0 in case of malformed parameters
262  */
parseCoverParameters(const char * stringPtr,ZDICT_cover_params_t * params)263 static unsigned parseCoverParameters(const char* stringPtr, ZDICT_cover_params_t* params)
264 {
265     memset(params, 0, sizeof(*params));
266     for (; ;) {
267         if (longCommandWArg(&stringPtr, "k=")) { params->k = readU32FromChar(&stringPtr); if (stringPtr[0]==',') { stringPtr++; continue; } else break; }
268         if (longCommandWArg(&stringPtr, "d=")) { params->d = readU32FromChar(&stringPtr); if (stringPtr[0]==',') { stringPtr++; continue; } else break; }
269         if (longCommandWArg(&stringPtr, "steps=")) { params->steps = readU32FromChar(&stringPtr); if (stringPtr[0]==',') { stringPtr++; continue; } else break; }
270         return 0;
271     }
272     if (stringPtr[0] != 0) return 0;
273     DISPLAYLEVEL(4, "cover: k=%u\nd=%u\nsteps=%u\n", params->k, params->d, params->steps);
274     return 1;
275 }
276 
277 /**
278  * parseLegacyParameters() :
279  * reads legacy dictioanry builter parameters from *stringPtr (e.g. "--train-legacy=selectivity=8") into *selectivity
280  * @return 1 means that legacy dictionary builder parameters were correct
281  * @return 0 in case of malformed parameters
282  */
parseLegacyParameters(const char * stringPtr,unsigned * selectivity)283 static unsigned parseLegacyParameters(const char* stringPtr, unsigned* selectivity)
284 {
285     if (!longCommandWArg(&stringPtr, "s=") && !longCommandWArg(&stringPtr, "selectivity=")) { return 0; }
286     *selectivity = readU32FromChar(&stringPtr);
287     if (stringPtr[0] != 0) return 0;
288     DISPLAYLEVEL(4, "legacy: selectivity=%u\n", *selectivity);
289     return 1;
290 }
291 
defaultCoverParams(void)292 static ZDICT_cover_params_t defaultCoverParams(void)
293 {
294     ZDICT_cover_params_t params;
295     memset(&params, 0, sizeof(params));
296     params.d = 8;
297     params.steps = 4;
298     return params;
299 }
300 #endif
301 
302 
303 /** parseCompressionParameters() :
304  *  reads compression parameters from *stringPtr (e.g. "--zstd=wlog=23,clog=23,hlog=22,slog=6,slen=3,tlen=48,strat=6") into *params
305  *  @return 1 means that compression parameters were correct
306  *  @return 0 in case of malformed parameters
307  */
parseCompressionParameters(const char * stringPtr,ZSTD_compressionParameters * params)308 static unsigned parseCompressionParameters(const char* stringPtr, ZSTD_compressionParameters* params)
309 {
310     for ( ; ;) {
311         if (longCommandWArg(&stringPtr, "windowLog=") || longCommandWArg(&stringPtr, "wlog=")) { params->windowLog = readU32FromChar(&stringPtr); if (stringPtr[0]==',') { stringPtr++; continue; } else break; }
312         if (longCommandWArg(&stringPtr, "chainLog=") || longCommandWArg(&stringPtr, "clog=")) { params->chainLog = readU32FromChar(&stringPtr); if (stringPtr[0]==',') { stringPtr++; continue; } else break; }
313         if (longCommandWArg(&stringPtr, "hashLog=") || longCommandWArg(&stringPtr, "hlog=")) { params->hashLog = readU32FromChar(&stringPtr); if (stringPtr[0]==',') { stringPtr++; continue; } else break; }
314         if (longCommandWArg(&stringPtr, "searchLog=") || longCommandWArg(&stringPtr, "slog=")) { params->searchLog = readU32FromChar(&stringPtr); if (stringPtr[0]==',') { stringPtr++; continue; } else break; }
315         if (longCommandWArg(&stringPtr, "searchLength=") || longCommandWArg(&stringPtr, "slen=")) { params->searchLength = readU32FromChar(&stringPtr); if (stringPtr[0]==',') { stringPtr++; continue; } else break; }
316         if (longCommandWArg(&stringPtr, "targetLength=") || longCommandWArg(&stringPtr, "tlen=")) { params->targetLength = readU32FromChar(&stringPtr); if (stringPtr[0]==',') { stringPtr++; continue; } else break; }
317         if (longCommandWArg(&stringPtr, "strategy=") || longCommandWArg(&stringPtr, "strat=")) { params->strategy = (ZSTD_strategy)(readU32FromChar(&stringPtr)); if (stringPtr[0]==',') { stringPtr++; continue; } else break; }
318         if (longCommandWArg(&stringPtr, "overlapLog=") || longCommandWArg(&stringPtr, "ovlog=")) { g_overlapLog = readU32FromChar(&stringPtr); if (stringPtr[0]==',') { stringPtr++; continue; } else break; }
319         if (longCommandWArg(&stringPtr, "ldmHashLog=") || longCommandWArg(&stringPtr, "ldmhlog=")) { g_ldmHashLog = readU32FromChar(&stringPtr); if (stringPtr[0]==',') { stringPtr++; continue; } else break; }
320         if (longCommandWArg(&stringPtr, "ldmSearchLength=") || longCommandWArg(&stringPtr, "ldmslen=")) { g_ldmMinMatch = readU32FromChar(&stringPtr); if (stringPtr[0]==',') { stringPtr++; continue; } else break; }
321         if (longCommandWArg(&stringPtr, "ldmBucketSizeLog=") || longCommandWArg(&stringPtr, "ldmblog=")) { g_ldmBucketSizeLog = readU32FromChar(&stringPtr); if (stringPtr[0]==',') { stringPtr++; continue; } else break; }
322         if (longCommandWArg(&stringPtr, "ldmHashEveryLog=") || longCommandWArg(&stringPtr, "ldmhevery=")) { g_ldmHashEveryLog = readU32FromChar(&stringPtr); if (stringPtr[0]==',') { stringPtr++; continue; } else break; }
323         DISPLAYLEVEL(4, "invalid compression parameter \n");
324         return 0;
325     }
326 
327     DISPLAYLEVEL(4, "windowLog=%d, chainLog=%d, hashLog=%d, searchLog=%d \n", params->windowLog, params->chainLog, params->hashLog, params->searchLog);
328     DISPLAYLEVEL(4, "searchLength=%d, targetLength=%d, strategy=%d \n", params->searchLength, params->targetLength, params->strategy);
329     if (stringPtr[0] != 0) return 0; /* check the end of string */
330     return 1;
331 }
332 
printVersion(void)333 static void printVersion(void)
334 {
335     DISPLAY(WELCOME_MESSAGE);
336     /* format support */
337     DISPLAYLEVEL(3, "*** supports: zstd");
338 #if defined(ZSTD_LEGACY_SUPPORT) && (ZSTD_LEGACY_SUPPORT>0) && (ZSTD_LEGACY_SUPPORT<8)
339     DISPLAYLEVEL(3, ", zstd legacy v0.%d+", ZSTD_LEGACY_SUPPORT);
340 #endif
341 #ifdef ZSTD_GZCOMPRESS
342     DISPLAYLEVEL(3, ", gzip");
343 #endif
344 #ifdef ZSTD_LZ4COMPRESS
345     DISPLAYLEVEL(3, ", lz4");
346 #endif
347 #ifdef ZSTD_LZMACOMPRESS
348     DISPLAYLEVEL(3, ", lzma, xz ");
349 #endif
350     DISPLAYLEVEL(3, "\n");
351     /* posix support */
352 #ifdef _POSIX_C_SOURCE
353     DISPLAYLEVEL(4, "_POSIX_C_SOURCE defined: %ldL\n", (long) _POSIX_C_SOURCE);
354 #endif
355 #ifdef _POSIX_VERSION
356     DISPLAYLEVEL(4, "_POSIX_VERSION defined: %ldL \n", (long) _POSIX_VERSION);
357 #endif
358 #ifdef PLATFORM_POSIX_VERSION
359     DISPLAYLEVEL(4, "PLATFORM_POSIX_VERSION defined: %ldL\n", (long) PLATFORM_POSIX_VERSION);
360 #endif
361 }
362 
363 typedef enum { zom_compress, zom_decompress, zom_test, zom_bench, zom_train, zom_list } zstd_operation_mode;
364 
365 #define CLEAN_RETURN(i) { operationResult = (i); goto _end; }
366 
main(int argCount,const char * argv[])367 int main(int argCount, const char* argv[])
368 {
369     int argNb,
370         followLinks = 0,
371         forceStdout = 0,
372         lastCommand = 0,
373         ldmFlag = 0,
374         main_pause = 0,
375         nbWorkers = 0,
376         nextArgumentIsOutFileName = 0,
377         nextArgumentIsMaxDict = 0,
378         nextArgumentIsDictID = 0,
379         nextArgumentsAreFiles = 0,
380         nextEntryIsDictionary = 0,
381         operationResult = 0,
382         separateFiles = 0,
383         setRealTimePrio = 0,
384         singleThread = 0,
385         ultra=0;
386     unsigned bench_nbSeconds = 3;   /* would be better if this value was synchronized from bench */
387     size_t blockSize = 0;
388     zstd_operation_mode operation = zom_compress;
389     ZSTD_compressionParameters compressionParams;
390     int cLevel = ZSTDCLI_CLEVEL_DEFAULT;
391     int cLevelLast = -1000000000;
392     unsigned recursive = 0;
393     unsigned memLimit = 0;
394     const char** filenameTable = (const char**)malloc(argCount * sizeof(const char*));   /* argCount >= 1 */
395     unsigned filenameIdx = 0;
396     const char* programName = argv[0];
397     const char* outFileName = NULL;
398     const char* dictFileName = NULL;
399     const char* suffix = ZSTD_EXTENSION;
400     unsigned maxDictSize = g_defaultMaxDictSize;
401     unsigned dictID = 0;
402     int dictCLevel = g_defaultDictCLevel;
403     unsigned dictSelect = g_defaultSelectivityLevel;
404 #ifdef UTIL_HAS_CREATEFILELIST
405     const char** extendedFileList = NULL;
406     char* fileNamesBuf = NULL;
407     unsigned fileNamesNb;
408 #endif
409 #ifndef ZSTD_NODICT
410     ZDICT_cover_params_t coverParams = defaultCoverParams();
411     int cover = 1;
412 #endif
413 
414 
415     /* init */
416     (void)recursive; (void)cLevelLast;    /* not used when ZSTD_NOBENCH set */
417     (void)dictCLevel; (void)dictSelect; (void)dictID;  (void)maxDictSize; /* not used when ZSTD_NODICT set */
418     (void)ultra; (void)cLevel; (void)ldmFlag; /* not used when ZSTD_NOCOMPRESS set */
419     (void)memLimit;   /* not used when ZSTD_NODECOMPRESS set */
420     if (filenameTable==NULL) { DISPLAY("zstd: %s \n", strerror(errno)); exit(1); }
421     filenameTable[0] = stdinmark;
422     g_displayOut = stderr;
423     programName = lastNameFromPath(programName);
424 #ifdef ZSTD_MULTITHREAD
425     nbWorkers = 1;
426 #endif
427 
428     /* preset behaviors */
429     if (exeNameMatch(programName, ZSTD_ZSTDMT)) nbWorkers=0;
430     if (exeNameMatch(programName, ZSTD_UNZSTD)) operation=zom_decompress;
431     if (exeNameMatch(programName, ZSTD_CAT)) { operation=zom_decompress; forceStdout=1; FIO_overwriteMode(); outFileName=stdoutmark; g_displayLevel=1; }   /* supports multiple formats */
432     if (exeNameMatch(programName, ZSTD_ZCAT)) { operation=zom_decompress; forceStdout=1; FIO_overwriteMode(); outFileName=stdoutmark; g_displayLevel=1; }  /* behave like zcat, also supports multiple formats */
433     if (exeNameMatch(programName, ZSTD_GZ)) { suffix = GZ_EXTENSION; FIO_setCompressionType(FIO_gzipCompression); FIO_setRemoveSrcFile(1); }               /* behave like gzip */
434     if (exeNameMatch(programName, ZSTD_GUNZIP)) { operation=zom_decompress; FIO_setRemoveSrcFile(1); }                                                     /* behave like gunzip, also supports multiple formats */
435     if (exeNameMatch(programName, ZSTD_GZCAT)) { operation=zom_decompress; forceStdout=1; FIO_overwriteMode(); outFileName=stdoutmark; g_displayLevel=1; } /* behave like gzcat, also supports multiple formats */
436     if (exeNameMatch(programName, ZSTD_LZMA)) { suffix = LZMA_EXTENSION; FIO_setCompressionType(FIO_lzmaCompression); FIO_setRemoveSrcFile(1); }           /* behave like lzma */
437     if (exeNameMatch(programName, ZSTD_UNLZMA)) { operation=zom_decompress; FIO_setCompressionType(FIO_lzmaCompression); FIO_setRemoveSrcFile(1); }        /* behave like unlzma, also supports multiple formats */
438     if (exeNameMatch(programName, ZSTD_XZ)) { suffix = XZ_EXTENSION; FIO_setCompressionType(FIO_xzCompression); FIO_setRemoveSrcFile(1); }                 /* behave like xz */
439     if (exeNameMatch(programName, ZSTD_UNXZ)) { operation=zom_decompress; FIO_setCompressionType(FIO_xzCompression); FIO_setRemoveSrcFile(1); }            /* behave like unxz, also supports multiple formats */
440     if (exeNameMatch(programName, ZSTD_LZ4)) { suffix = LZ4_EXTENSION; FIO_setCompressionType(FIO_lz4Compression); }                                       /* behave like lz4 */
441     if (exeNameMatch(programName, ZSTD_UNLZ4)) { operation=zom_decompress; FIO_setCompressionType(FIO_lz4Compression); }                                   /* behave like unlz4, also supports multiple formats */
442     memset(&compressionParams, 0, sizeof(compressionParams));
443 
444     /* command switches */
445     for (argNb=1; argNb<argCount; argNb++) {
446         const char* argument = argv[argNb];
447         if(!argument) continue;   /* Protection if argument empty */
448 
449         if (nextArgumentsAreFiles==0) {
450             /* "-" means stdin/stdout */
451             if (!strcmp(argument, "-")){
452                 if (!filenameIdx) {
453                     filenameIdx=1, filenameTable[0]=stdinmark;
454                     outFileName=stdoutmark;
455                     g_displayLevel-=(g_displayLevel==2);
456                     continue;
457             }   }
458 
459             /* Decode commands (note : aggregated commands are allowed) */
460             if (argument[0]=='-') {
461 
462                 if (argument[1]=='-') {
463                     /* long commands (--long-word) */
464                     if (!strcmp(argument, "--")) { nextArgumentsAreFiles=1; continue; }   /* only file names allowed from now on */
465                     if (!strcmp(argument, "--list")) { operation=zom_list; continue; }
466                     if (!strcmp(argument, "--compress")) { operation=zom_compress; continue; }
467                     if (!strcmp(argument, "--decompress")) { operation=zom_decompress; continue; }
468                     if (!strcmp(argument, "--uncompress")) { operation=zom_decompress; continue; }
469                     if (!strcmp(argument, "--force")) { FIO_overwriteMode(); forceStdout=1; followLinks=1; continue; }
470                     if (!strcmp(argument, "--version")) { g_displayOut=stdout; DISPLAY(WELCOME_MESSAGE); CLEAN_RETURN(0); }
471                     if (!strcmp(argument, "--help")) { g_displayOut=stdout; CLEAN_RETURN(usage_advanced(programName)); }
472                     if (!strcmp(argument, "--verbose")) { g_displayLevel++; continue; }
473                     if (!strcmp(argument, "--quiet")) { g_displayLevel--; continue; }
474                     if (!strcmp(argument, "--stdout")) { forceStdout=1; outFileName=stdoutmark; g_displayLevel-=(g_displayLevel==2); continue; }
475                     if (!strcmp(argument, "--ultra")) { ultra=1; continue; }
476                     if (!strcmp(argument, "--check")) { FIO_setChecksumFlag(2); continue; }
477                     if (!strcmp(argument, "--no-check")) { FIO_setChecksumFlag(0); continue; }
478                     if (!strcmp(argument, "--sparse")) { FIO_setSparseWrite(2); continue; }
479                     if (!strcmp(argument, "--no-sparse")) { FIO_setSparseWrite(0); continue; }
480                     if (!strcmp(argument, "--test")) { operation=zom_test; continue; }
481                     if (!strcmp(argument, "--train")) { operation=zom_train; outFileName=g_defaultDictName; continue; }
482                     if (!strcmp(argument, "--maxdict")) { nextArgumentIsMaxDict=1; lastCommand=1; continue; }  /* kept available for compatibility with old syntax ; will be removed one day */
483                     if (!strcmp(argument, "--dictID")) { nextArgumentIsDictID=1; lastCommand=1; continue; }  /* kept available for compatibility with old syntax ; will be removed one day */
484                     if (!strcmp(argument, "--no-dictID")) { FIO_setDictIDFlag(0); continue; }
485                     if (!strcmp(argument, "--keep")) { FIO_setRemoveSrcFile(0); continue; }
486                     if (!strcmp(argument, "--rm")) { FIO_setRemoveSrcFile(1); continue; }
487                     if (!strcmp(argument, "--priority=rt")) { setRealTimePrio = 1; continue; }
488                     if (!strcmp(argument, "--single-thread")) { nbWorkers = 0; singleThread = 1; continue; }
489 #ifdef ZSTD_GZCOMPRESS
490                     if (!strcmp(argument, "--format=gzip")) { suffix = GZ_EXTENSION; FIO_setCompressionType(FIO_gzipCompression); continue; }
491 #endif
492 #ifdef ZSTD_LZMACOMPRESS
493                     if (!strcmp(argument, "--format=lzma")) { suffix = LZMA_EXTENSION; FIO_setCompressionType(FIO_lzmaCompression);  continue; }
494                     if (!strcmp(argument, "--format=xz")) { suffix = XZ_EXTENSION; FIO_setCompressionType(FIO_xzCompression);  continue; }
495 #endif
496 #ifdef ZSTD_LZ4COMPRESS
497                     if (!strcmp(argument, "--format=lz4")) { suffix = LZ4_EXTENSION; FIO_setCompressionType(FIO_lz4Compression);  continue; }
498 #endif
499 
500                     /* long commands with arguments */
501 #ifndef ZSTD_NODICT
502                     if (longCommandWArg(&argument, "--train-cover")) {
503                       operation = zom_train;
504                       outFileName = g_defaultDictName;
505                       cover = 1;
506                       /* Allow optional arguments following an = */
507                       if (*argument == 0) { memset(&coverParams, 0, sizeof(coverParams)); }
508                       else if (*argument++ != '=') { CLEAN_RETURN(badusage(programName)); }
509                       else if (!parseCoverParameters(argument, &coverParams)) { CLEAN_RETURN(badusage(programName)); }
510                       continue;
511                     }
512                     if (longCommandWArg(&argument, "--train-legacy")) {
513                       operation = zom_train;
514                       outFileName = g_defaultDictName;
515                       cover = 0;
516                       /* Allow optional arguments following an = */
517                       if (*argument == 0) { continue; }
518                       else if (*argument++ != '=') { CLEAN_RETURN(badusage(programName)); }
519                       else if (!parseLegacyParameters(argument, &dictSelect)) { CLEAN_RETURN(badusage(programName)); }
520                       continue;
521                     }
522 #endif
523                     if (longCommandWArg(&argument, "--threads=")) { nbWorkers = readU32FromChar(&argument); continue; }
524                     if (longCommandWArg(&argument, "--memlimit=")) { memLimit = readU32FromChar(&argument); continue; }
525                     if (longCommandWArg(&argument, "--memory=")) { memLimit = readU32FromChar(&argument); continue; }
526                     if (longCommandWArg(&argument, "--memlimit-decompress=")) { memLimit = readU32FromChar(&argument); continue; }
527                     if (longCommandWArg(&argument, "--block-size=")) { blockSize = readU32FromChar(&argument); continue; }
528                     if (longCommandWArg(&argument, "--maxdict=")) { maxDictSize = readU32FromChar(&argument); continue; }
529                     if (longCommandWArg(&argument, "--dictID=")) { dictID = readU32FromChar(&argument); continue; }
530                     if (longCommandWArg(&argument, "--zstd=")) { if (!parseCompressionParameters(argument, &compressionParams)) CLEAN_RETURN(badusage(programName)); continue; }
531                     if (longCommandWArg(&argument, "--long")) {
532                         unsigned ldmWindowLog = 0;
533                         ldmFlag = 1;
534                         /* Parse optional window log */
535                         if (*argument == '=') {
536                             ++argument;
537                             ldmWindowLog = readU32FromChar(&argument);
538                         } else if (*argument != 0) {
539                             /* Invalid character following --long */
540                             CLEAN_RETURN(badusage(programName));
541                         }
542                         /* Only set windowLog if not already set by --zstd */
543                         if (compressionParams.windowLog == 0)
544                             compressionParams.windowLog = ldmWindowLog;
545                         continue;
546                     }
547                     if (longCommandWArg(&argument, "--fast")) {
548                         /* Parse optional window log */
549                         if (*argument == '=') {
550                             U32 fastLevel;
551                             ++argument;
552                             fastLevel = readU32FromChar(&argument);
553                             if (fastLevel) cLevel = - (int)fastLevel;
554                         } else if (*argument != 0) {
555                             /* Invalid character following --fast */
556                             CLEAN_RETURN(badusage(programName));
557                         } else {
558                             cLevel = -1;  /* default for --fast */
559                         }
560                         continue;
561                     }
562                     /* fall-through, will trigger bad_usage() later on */
563                 }
564 
565                 argument++;
566                 while (argument[0]!=0) {
567                     if (lastCommand) {
568                         DISPLAY("error : command must be followed by argument \n");
569                         CLEAN_RETURN(1);
570                     }
571 #ifndef ZSTD_NOCOMPRESS
572                     /* compression Level */
573                     if ((*argument>='0') && (*argument<='9')) {
574                         dictCLevel = cLevel = readU32FromChar(&argument);
575                         continue;
576                     }
577 #endif
578 
579                     switch(argument[0])
580                     {
581                         /* Display help */
582                     case 'V': g_displayOut=stdout; printVersion(); CLEAN_RETURN(0);   /* Version Only */
583                     case 'H':
584                     case 'h': g_displayOut=stdout; CLEAN_RETURN(usage_advanced(programName));
585 
586                          /* Compress */
587                     case 'z': operation=zom_compress; argument++; break;
588 
589                          /* Decoding */
590                     case 'd':
591 #ifndef ZSTD_NOBENCH
592                             BMK_setDecodeOnlyMode(1);
593                             if (operation==zom_bench) { argument++; break; }  /* benchmark decode (hidden option) */
594 #endif
595                             operation=zom_decompress; argument++; break;
596 
597                         /* Force stdout, even if stdout==console */
598                     case 'c': forceStdout=1; outFileName=stdoutmark; argument++; break;
599 
600                         /* Use file content as dictionary */
601                     case 'D': nextEntryIsDictionary = 1; lastCommand = 1; argument++; break;
602 
603                         /* Overwrite */
604                     case 'f': FIO_overwriteMode(); forceStdout=1; followLinks=1; argument++; break;
605 
606                         /* Verbose mode */
607                     case 'v': g_displayLevel++; argument++; break;
608 
609                         /* Quiet mode */
610                     case 'q': g_displayLevel--; argument++; break;
611 
612                         /* keep source file (default) */
613                     case 'k': FIO_setRemoveSrcFile(0); argument++; break;
614 
615                         /* Checksum */
616                     case 'C': FIO_setChecksumFlag(2); argument++; break;
617 
618                         /* test compressed file */
619                     case 't': operation=zom_test; argument++; break;
620 
621                         /* destination file name */
622                     case 'o': nextArgumentIsOutFileName=1; lastCommand=1; argument++; break;
623 
624                         /* limit decompression memory */
625                     case 'M':
626                         argument++;
627                         memLimit = readU32FromChar(&argument);
628                         break;
629                     case 'l': operation=zom_list; argument++; break;
630 #ifdef UTIL_HAS_CREATEFILELIST
631                         /* recursive */
632                     case 'r': recursive=1; argument++; break;
633 #endif
634 
635 #ifndef ZSTD_NOBENCH
636                         /* Benchmark */
637                     case 'b':
638                         operation=zom_bench;
639                         argument++;
640                         break;
641 
642                         /* range bench (benchmark only) */
643                     case 'e':
644                         /* compression Level */
645                         argument++;
646                         cLevelLast = readU32FromChar(&argument);
647                         break;
648 
649                         /* Modify Nb Iterations (benchmark only) */
650                     case 'i':
651                         argument++;
652                         bench_nbSeconds = readU32FromChar(&argument);
653                         break;
654 
655                         /* cut input into blocks (benchmark only) */
656                     case 'B':
657                         argument++;
658                         blockSize = readU32FromChar(&argument);
659                         break;
660 
661                         /* benchmark files separately (hidden option) */
662                     case 'S':
663                         argument++;
664                         separateFiles = 1;
665                         break;
666 
667 #endif   /* ZSTD_NOBENCH */
668 
669                         /* nb of threads (hidden option) */
670                     case 'T':
671                         argument++;
672                         nbWorkers = readU32FromChar(&argument);
673                         break;
674 
675                         /* Dictionary Selection level */
676                     case 's':
677                         argument++;
678                         dictSelect = readU32FromChar(&argument);
679                         break;
680 
681                         /* Pause at the end (-p) or set an additional param (-p#) (hidden option) */
682                     case 'p': argument++;
683 #ifndef ZSTD_NOBENCH
684                         if ((*argument>='0') && (*argument<='9')) {
685                             BMK_setAdditionalParam(readU32FromChar(&argument));
686                         } else
687 #endif
688                             main_pause=1;
689                         break;
690                         /* unknown command */
691                     default : CLEAN_RETURN(badusage(programName));
692                     }
693                 }
694                 continue;
695             }   /* if (argument[0]=='-') */
696 
697             if (nextArgumentIsMaxDict) {  /* kept available for compatibility with old syntax ; will be removed one day */
698                 nextArgumentIsMaxDict = 0;
699                 lastCommand = 0;
700                 maxDictSize = readU32FromChar(&argument);
701                 continue;
702             }
703 
704             if (nextArgumentIsDictID) {  /* kept available for compatibility with old syntax ; will be removed one day */
705                 nextArgumentIsDictID = 0;
706                 lastCommand = 0;
707                 dictID = readU32FromChar(&argument);
708                 continue;
709             }
710 
711         }   /* if (nextArgumentIsAFile==0) */
712 
713         if (nextEntryIsDictionary) {
714             nextEntryIsDictionary = 0;
715             lastCommand = 0;
716             dictFileName = argument;
717             continue;
718         }
719 
720         if (nextArgumentIsOutFileName) {
721             nextArgumentIsOutFileName = 0;
722             lastCommand = 0;
723             outFileName = argument;
724             if (!strcmp(outFileName, "-")) outFileName = stdoutmark;
725             continue;
726         }
727 
728         /* add filename to list */
729         filenameTable[filenameIdx++] = argument;
730     }
731 
732     if (lastCommand) { /* forgotten argument */
733         DISPLAY("error : command must be followed by argument \n");
734         CLEAN_RETURN(1);
735     }
736 
737     /* Welcome message (if verbose) */
738     DISPLAYLEVEL(3, WELCOME_MESSAGE);
739 
740 #ifdef ZSTD_MULTITHREAD
741     if ((nbWorkers==0) && (!singleThread)) {
742         /* automatically set # workers based on # of reported cpus */
743         nbWorkers = UTIL_countPhysicalCores();
744         DISPLAYLEVEL(3, "Note: %d physical core(s) detected \n", nbWorkers);
745     }
746 #endif
747 
748     g_utilDisplayLevel = g_displayLevel;
749     if (!followLinks) {
750         unsigned u;
751         for (u=0, fileNamesNb=0; u<filenameIdx; u++) {
752             if (UTIL_isLink(filenameTable[u])) {
753                 DISPLAYLEVEL(2, "Warning : %s is a symbolic link, ignoring\n", filenameTable[u]);
754             } else {
755                 filenameTable[fileNamesNb++] = filenameTable[u];
756             }
757         }
758         filenameIdx = fileNamesNb;
759     }
760 #ifdef UTIL_HAS_CREATEFILELIST
761     if (recursive) {  /* at this stage, filenameTable is a list of paths, which can contain both files and directories */
762         extendedFileList = UTIL_createFileList(filenameTable, filenameIdx, &fileNamesBuf, &fileNamesNb, followLinks);
763         if (extendedFileList) {
764             unsigned u;
765             for (u=0; u<fileNamesNb; u++) DISPLAYLEVEL(4, "%u %s\n", u, extendedFileList[u]);
766             free((void*)filenameTable);
767             filenameTable = extendedFileList;
768             filenameIdx = fileNamesNb;
769         }
770     }
771 #endif
772 
773     if (operation == zom_list) {
774 #ifndef ZSTD_NODECOMPRESS
775         int const ret = FIO_listMultipleFiles(filenameIdx, filenameTable, g_displayLevel);
776         CLEAN_RETURN(ret);
777 #else
778         DISPLAY("file information is not supported \n");
779         CLEAN_RETURN(1);
780 #endif
781     }
782 
783     /* Check if benchmark is selected */
784     if (operation==zom_bench) {
785 #ifndef ZSTD_NOBENCH
786         BMK_setNotificationLevel(g_displayLevel);
787         BMK_setSeparateFiles(separateFiles);
788         BMK_setBlockSize(blockSize);
789         BMK_setNbWorkers(nbWorkers);
790         BMK_setRealTime(setRealTimePrio);
791         BMK_setNbSeconds(bench_nbSeconds);
792         BMK_setLdmFlag(ldmFlag);
793         BMK_setLdmMinMatch(g_ldmMinMatch);
794         BMK_setLdmHashLog(g_ldmHashLog);
795         if (g_ldmBucketSizeLog != LDM_PARAM_DEFAULT) {
796             BMK_setLdmBucketSizeLog(g_ldmBucketSizeLog);
797         }
798         if (g_ldmHashEveryLog != LDM_PARAM_DEFAULT) {
799             BMK_setLdmHashEveryLog(g_ldmHashEveryLog);
800         }
801         BMK_benchFiles(filenameTable, filenameIdx, dictFileName, cLevel, cLevelLast, &compressionParams);
802 #else
803         (void)bench_nbSeconds; (void)blockSize; (void)setRealTimePrio; (void)separateFiles;
804 #endif
805         goto _end;
806     }
807 
808     /* Check if dictionary builder is selected */
809     if (operation==zom_train) {
810 #ifndef ZSTD_NODICT
811         ZDICT_params_t zParams;
812         zParams.compressionLevel = dictCLevel;
813         zParams.notificationLevel = g_displayLevel;
814         zParams.dictID = dictID;
815         if (cover) {
816             int const optimize = !coverParams.k || !coverParams.d;
817             coverParams.nbThreads = nbWorkers;
818             coverParams.zParams = zParams;
819             operationResult = DiB_trainFromFiles(outFileName, maxDictSize, filenameTable, filenameIdx, blockSize, NULL, &coverParams, optimize);
820         } else {
821             ZDICT_legacy_params_t dictParams;
822             memset(&dictParams, 0, sizeof(dictParams));
823             dictParams.selectivityLevel = dictSelect;
824             dictParams.zParams = zParams;
825             operationResult = DiB_trainFromFiles(outFileName, maxDictSize, filenameTable, filenameIdx, blockSize, &dictParams, NULL, 0);
826         }
827 #endif
828         goto _end;
829     }
830 
831 #ifndef ZSTD_NODECOMPRESS
832     if (operation==zom_test) { outFileName=nulmark; FIO_setRemoveSrcFile(0); }  /* test mode */
833 #endif
834 
835     /* No input filename ==> use stdin and stdout */
836     filenameIdx += !filenameIdx;   /* filenameTable[0] is stdin by default */
837     if (!strcmp(filenameTable[0], stdinmark) && !outFileName)
838         outFileName = stdoutmark;  /* when input is stdin, default output is stdout */
839 
840     /* Check if input/output defined as console; trigger an error in this case */
841     if (!strcmp(filenameTable[0], stdinmark) && IS_CONSOLE(stdin) )
842         CLEAN_RETURN(badusage(programName));
843     if ( outFileName && !strcmp(outFileName, stdoutmark)
844       && IS_CONSOLE(stdout)
845       && !strcmp(filenameTable[0], stdinmark)
846       && !forceStdout
847       && operation!=zom_decompress )
848         CLEAN_RETURN(badusage(programName));
849 
850 #ifndef ZSTD_NOCOMPRESS
851     /* check compression level limits */
852     {   int const maxCLevel = ultra ? ZSTD_maxCLevel() : ZSTDCLI_CLEVEL_MAX;
853         if (cLevel > maxCLevel) {
854             DISPLAYLEVEL(2, "Warning : compression level higher than max, reduced to %i \n", maxCLevel);
855             cLevel = maxCLevel;
856     }   }
857 #endif
858 
859     /* No status message in pipe mode (stdin - stdout) or multi-files mode */
860     if (!strcmp(filenameTable[0], stdinmark) && outFileName && !strcmp(outFileName,stdoutmark) && (g_displayLevel==2)) g_displayLevel=1;
861     if ((filenameIdx>1) & (g_displayLevel==2)) g_displayLevel=1;
862 
863     /* IO Stream/File */
864     FIO_setNotificationLevel(g_displayLevel);
865     if (operation==zom_compress) {
866 #ifndef ZSTD_NOCOMPRESS
867         FIO_setNbWorkers(nbWorkers);
868         FIO_setBlockSize((U32)blockSize);
869         FIO_setLdmFlag(ldmFlag);
870         FIO_setLdmHashLog(g_ldmHashLog);
871         FIO_setLdmMinMatch(g_ldmMinMatch);
872         if (g_ldmBucketSizeLog != LDM_PARAM_DEFAULT) {
873             FIO_setLdmBucketSizeLog(g_ldmBucketSizeLog);
874         }
875         if (g_ldmHashEveryLog != LDM_PARAM_DEFAULT) {
876             FIO_setLdmHashEveryLog(g_ldmHashEveryLog);
877         }
878 
879         if (g_overlapLog!=OVERLAP_LOG_DEFAULT) FIO_setOverlapLog(g_overlapLog);
880         if ((filenameIdx==1) && outFileName)
881           operationResult = FIO_compressFilename(outFileName, filenameTable[0], dictFileName, cLevel, &compressionParams);
882         else
883           operationResult = FIO_compressMultipleFilenames(filenameTable, filenameIdx, outFileName, suffix, dictFileName, cLevel, &compressionParams);
884 #else
885         (void)suffix;
886         DISPLAY("Compression not supported\n");
887 #endif
888     } else {  /* decompression or test */
889 #ifndef ZSTD_NODECOMPRESS
890         if (memLimit == 0) {
891             if (compressionParams.windowLog == 0)
892                 memLimit = (U32)1 << g_defaultMaxWindowLog;
893             else {
894                 memLimit = (U32)1 << (compressionParams.windowLog & 31);
895             }
896         }
897         FIO_setMemLimit(memLimit);
898         if (filenameIdx==1 && outFileName)
899             operationResult = FIO_decompressFilename(outFileName, filenameTable[0], dictFileName);
900         else
901             operationResult = FIO_decompressMultipleFilenames(filenameTable, filenameIdx, outFileName, dictFileName);
902 #else
903         DISPLAY("Decompression not supported\n");
904 #endif
905     }
906 
907 _end:
908     if (main_pause) waitEnter();
909 #ifdef UTIL_HAS_CREATEFILELIST
910     if (extendedFileList)
911         UTIL_freeFileList(extendedFileList, fileNamesBuf);
912     else
913 #endif
914         free((void*)filenameTable);
915     return operationResult;
916 }
917