1 /* $NetBSD: tokenizer.c,v 1.22 2016/01/30 04:02:51 christos Exp $ */ 2 3 /*- 4 * Copyright (c) 1992, 1993 5 * The Regents of the University of California. All rights reserved. 6 * 7 * This code is derived from software contributed to Berkeley by 8 * Christos Zoulas of Cornell University. 9 * 10 * Redistribution and use in source and binary forms, with or without 11 * modification, are permitted provided that the following conditions 12 * are met: 13 * 1. Redistributions of source code must retain the above copyright 14 * notice, this list of conditions and the following disclaimer. 15 * 2. Redistributions in binary form must reproduce the above copyright 16 * notice, this list of conditions and the following disclaimer in the 17 * documentation and/or other materials provided with the distribution. 18 * 3. Neither the name of the University nor the names of its contributors 19 * may be used to endorse or promote products derived from this software 20 * without specific prior written permission. 21 * 22 * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND 23 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 24 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 25 * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE 26 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 27 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 28 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 29 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 30 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 31 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 32 * SUCH DAMAGE. 33 */ 34 35 #include "config.h" 36 #if !defined(lint) && !defined(SCCSID) 37 #if 0 38 static char sccsid[] = "@(#)tokenizer.c 8.1 (Berkeley) 6/4/93"; 39 #else 40 __RCSID("$NetBSD: tokenizer.c,v 1.22 2016/01/30 04:02:51 christos Exp $"); 41 #endif 42 #endif /* not lint && not SCCSID */ 43 #include <sys/cdefs.h> 44 __FBSDID("$FreeBSD$"); 45 46 /* We build this file twice, once as NARROW, once as WIDE. */ 47 /* 48 * tokenize.c: Bourne shell like tokenizer 49 */ 50 #include <string.h> 51 #include <stdlib.h> 52 #include "histedit.h" 53 #include "chartype.h" 54 55 typedef enum { 56 Q_none, Q_single, Q_double, Q_one, Q_doubleone 57 } quote_t; 58 59 #define TOK_KEEP 1 60 #define TOK_EAT 2 61 62 #define WINCR 20 63 #define AINCR 10 64 65 #define IFS STR("\t \n") 66 67 #define tok_malloc(a) malloc(a) 68 #define tok_free(a) free(a) 69 #define tok_realloc(a, b) realloc(a, b) 70 #define tok_strdup(a) Strdup(a) 71 72 73 struct TYPE(tokenizer) { 74 Char *ifs; /* In field separator */ 75 size_t argc, amax; /* Current and maximum number of args */ 76 Char **argv; /* Argument list */ 77 Char *wptr, *wmax; /* Space and limit on the word buffer */ 78 Char *wstart; /* Beginning of next word */ 79 Char *wspace; /* Space of word buffer */ 80 quote_t quote; /* Quoting state */ 81 int flags; /* flags; */ 82 }; 83 84 85 private void FUN(tok,finish)(TYPE(Tokenizer) *); 86 87 88 /* FUN(tok,finish)(): 89 * Finish a word in the tokenizer. 90 */ 91 private void 92 FUN(tok,finish)(TYPE(Tokenizer) *tok) 93 { 94 95 *tok->wptr = '\0'; 96 if ((tok->flags & TOK_KEEP) || tok->wptr != tok->wstart) { 97 tok->argv[tok->argc++] = tok->wstart; 98 tok->argv[tok->argc] = NULL; 99 tok->wstart = ++tok->wptr; 100 } 101 tok->flags &= ~TOK_KEEP; 102 } 103 104 105 /* FUN(tok,init)(): 106 * Initialize the tokenizer 107 */ 108 public TYPE(Tokenizer) * 109 FUN(tok,init)(const Char *ifs) 110 { 111 TYPE(Tokenizer) *tok = tok_malloc(sizeof(*tok)); 112 113 if (tok == NULL) 114 return NULL; 115 tok->ifs = tok_strdup(ifs ? ifs : IFS); 116 if (tok->ifs == NULL) { 117 tok_free(tok); 118 return NULL; 119 } 120 tok->argc = 0; 121 tok->amax = AINCR; 122 tok->argv = tok_malloc(sizeof(*tok->argv) * tok->amax); 123 if (tok->argv == NULL) { 124 tok_free(tok->ifs); 125 tok_free(tok); 126 return NULL; 127 } 128 tok->argv[0] = NULL; 129 tok->wspace = tok_malloc(WINCR * sizeof(*tok->wspace)); 130 if (tok->wspace == NULL) { 131 tok_free(tok->argv); 132 tok_free(tok->ifs); 133 tok_free(tok); 134 return NULL; 135 } 136 tok->wmax = tok->wspace + WINCR; 137 tok->wstart = tok->wspace; 138 tok->wptr = tok->wspace; 139 tok->flags = 0; 140 tok->quote = Q_none; 141 142 return tok; 143 } 144 145 146 /* FUN(tok,reset)(): 147 * Reset the tokenizer 148 */ 149 public void 150 FUN(tok,reset)(TYPE(Tokenizer) *tok) 151 { 152 153 tok->argc = 0; 154 tok->wstart = tok->wspace; 155 tok->wptr = tok->wspace; 156 tok->flags = 0; 157 tok->quote = Q_none; 158 } 159 160 161 /* FUN(tok,end)(): 162 * Clean up 163 */ 164 public void 165 FUN(tok,end)(TYPE(Tokenizer) *tok) 166 { 167 168 tok_free(tok->ifs); 169 tok_free(tok->wspace); 170 tok_free(tok->argv); 171 tok_free(tok); 172 } 173 174 175 176 /* FUN(tok,line)(): 177 * Bourne shell (sh(1)) like tokenizing 178 * Arguments: 179 * tok current tokenizer state (setup with FUN(tok,init)()) 180 * line line to parse 181 * Returns: 182 * -1 Internal error 183 * 3 Quoted return 184 * 2 Unmatched double quote 185 * 1 Unmatched single quote 186 * 0 Ok 187 * Modifies (if return value is 0): 188 * argc number of arguments 189 * argv argument array 190 * cursorc if !NULL, argv element containing cursor 191 * cursorv if !NULL, offset in argv[cursorc] of cursor 192 */ 193 public int 194 FUN(tok,line)(TYPE(Tokenizer) *tok, const TYPE(LineInfo) *line, 195 int *argc, const Char ***argv, int *cursorc, int *cursoro) 196 { 197 const Char *ptr; 198 int cc, co; 199 200 cc = co = -1; 201 ptr = line->buffer; 202 for (ptr = line->buffer; ;ptr++) { 203 if (ptr >= line->lastchar) 204 ptr = STR(""); 205 if (ptr == line->cursor) { 206 cc = (int)tok->argc; 207 co = (int)(tok->wptr - tok->wstart); 208 } 209 switch (*ptr) { 210 case '\'': 211 tok->flags |= TOK_KEEP; 212 tok->flags &= ~TOK_EAT; 213 switch (tok->quote) { 214 case Q_none: 215 tok->quote = Q_single; /* Enter single quote 216 * mode */ 217 break; 218 219 case Q_single: /* Exit single quote mode */ 220 tok->quote = Q_none; 221 break; 222 223 case Q_one: /* Quote this ' */ 224 tok->quote = Q_none; 225 *tok->wptr++ = *ptr; 226 break; 227 228 case Q_double: /* Stay in double quote mode */ 229 *tok->wptr++ = *ptr; 230 break; 231 232 case Q_doubleone: /* Quote this ' */ 233 tok->quote = Q_double; 234 *tok->wptr++ = *ptr; 235 break; 236 237 default: 238 return -1; 239 } 240 break; 241 242 case '"': 243 tok->flags &= ~TOK_EAT; 244 tok->flags |= TOK_KEEP; 245 switch (tok->quote) { 246 case Q_none: /* Enter double quote mode */ 247 tok->quote = Q_double; 248 break; 249 250 case Q_double: /* Exit double quote mode */ 251 tok->quote = Q_none; 252 break; 253 254 case Q_one: /* Quote this " */ 255 tok->quote = Q_none; 256 *tok->wptr++ = *ptr; 257 break; 258 259 case Q_single: /* Stay in single quote mode */ 260 *tok->wptr++ = *ptr; 261 break; 262 263 case Q_doubleone: /* Quote this " */ 264 tok->quote = Q_double; 265 *tok->wptr++ = *ptr; 266 break; 267 268 default: 269 return -1; 270 } 271 break; 272 273 case '\\': 274 tok->flags |= TOK_KEEP; 275 tok->flags &= ~TOK_EAT; 276 switch (tok->quote) { 277 case Q_none: /* Quote next character */ 278 tok->quote = Q_one; 279 break; 280 281 case Q_double: /* Quote next character */ 282 tok->quote = Q_doubleone; 283 break; 284 285 case Q_one: /* Quote this, restore state */ 286 *tok->wptr++ = *ptr; 287 tok->quote = Q_none; 288 break; 289 290 case Q_single: /* Stay in single quote mode */ 291 *tok->wptr++ = *ptr; 292 break; 293 294 case Q_doubleone: /* Quote this \ */ 295 tok->quote = Q_double; 296 *tok->wptr++ = *ptr; 297 break; 298 299 default: 300 return -1; 301 } 302 break; 303 304 case '\n': 305 tok->flags &= ~TOK_EAT; 306 switch (tok->quote) { 307 case Q_none: 308 goto tok_line_outok; 309 310 case Q_single: 311 case Q_double: 312 *tok->wptr++ = *ptr; /* Add the return */ 313 break; 314 315 case Q_doubleone: /* Back to double, eat the '\n' */ 316 tok->flags |= TOK_EAT; 317 tok->quote = Q_double; 318 break; 319 320 case Q_one: /* No quote, more eat the '\n' */ 321 tok->flags |= TOK_EAT; 322 tok->quote = Q_none; 323 break; 324 325 default: 326 return 0; 327 } 328 break; 329 330 case '\0': 331 switch (tok->quote) { 332 case Q_none: 333 /* Finish word and return */ 334 if (tok->flags & TOK_EAT) { 335 tok->flags &= ~TOK_EAT; 336 return 3; 337 } 338 goto tok_line_outok; 339 340 case Q_single: 341 return 1; 342 343 case Q_double: 344 return 2; 345 346 case Q_doubleone: 347 tok->quote = Q_double; 348 *tok->wptr++ = *ptr; 349 break; 350 351 case Q_one: 352 tok->quote = Q_none; 353 *tok->wptr++ = *ptr; 354 break; 355 356 default: 357 return -1; 358 } 359 break; 360 361 default: 362 tok->flags &= ~TOK_EAT; 363 switch (tok->quote) { 364 case Q_none: 365 if (Strchr(tok->ifs, *ptr) != NULL) 366 FUN(tok,finish)(tok); 367 else 368 *tok->wptr++ = *ptr; 369 break; 370 371 case Q_single: 372 case Q_double: 373 *tok->wptr++ = *ptr; 374 break; 375 376 377 case Q_doubleone: 378 *tok->wptr++ = '\\'; 379 tok->quote = Q_double; 380 *tok->wptr++ = *ptr; 381 break; 382 383 case Q_one: 384 tok->quote = Q_none; 385 *tok->wptr++ = *ptr; 386 break; 387 388 default: 389 return -1; 390 391 } 392 break; 393 } 394 395 if (tok->wptr >= tok->wmax - 4) { 396 size_t size = (size_t)(tok->wmax - tok->wspace + WINCR); 397 Char *s = tok_realloc(tok->wspace, 398 size * sizeof(*s)); 399 if (s == NULL) 400 return -1; 401 402 if (s != tok->wspace) { 403 size_t i; 404 for (i = 0; i < tok->argc; i++) { 405 tok->argv[i] = 406 (tok->argv[i] - tok->wspace) + s; 407 } 408 tok->wptr = (tok->wptr - tok->wspace) + s; 409 tok->wstart = (tok->wstart - tok->wspace) + s; 410 tok->wspace = s; 411 } 412 tok->wmax = s + size; 413 } 414 if (tok->argc >= tok->amax - 4) { 415 Char **p; 416 tok->amax += AINCR; 417 p = tok_realloc(tok->argv, tok->amax * sizeof(*p)); 418 if (p == NULL) 419 return -1; 420 tok->argv = p; 421 } 422 } 423 tok_line_outok: 424 if (cc == -1 && co == -1) { 425 cc = (int)tok->argc; 426 co = (int)(tok->wptr - tok->wstart); 427 } 428 if (cursorc != NULL) 429 *cursorc = cc; 430 if (cursoro != NULL) 431 *cursoro = co; 432 FUN(tok,finish)(tok); 433 *argv = (const Char **)tok->argv; 434 *argc = (int)tok->argc; 435 return 0; 436 } 437 438 /* FUN(tok,str)(): 439 * Simpler version of tok_line, taking a NUL terminated line 440 * and splitting into words, ignoring cursor state. 441 */ 442 public int 443 FUN(tok,str)(TYPE(Tokenizer) *tok, const Char *line, int *argc, 444 const Char ***argv) 445 { 446 TYPE(LineInfo) li; 447 448 memset(&li, 0, sizeof(li)); 449 li.buffer = line; 450 li.cursor = li.lastchar = Strchr(line, '\0'); 451 return FUN(tok,line)(tok, &li, argc, argv, NULL, NULL); 452 } 453