1 /* 2 * burl - buffer URL normalization 3 * 4 * Copyright(c) 2018 Glenn Strauss gstrauss()gluelogic.com All rights reserved 5 * License: BSD 3-clause (same as lighttpd) 6 */ 7 #include "first.h" 8 #include "burl.h" 9 10 #include <string.h> 11 12 #include "buffer.h" 13 #include "base64.h" 14 15 static const char hex_chars_uc[] = "0123456789ABCDEF"; 16 17 /* everything except: ! $ & ' ( ) * + , - . / 0-9 : ; = ? @ A-Z _ a-z ~ */ 18 static const char encoded_chars_http_uri_reqd[] = { 19 /* 20 0 1 2 3 4 5 6 7 8 9 A B C D E F 21 */ 22 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, /* 00 - 0F control chars */ 23 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, /* 10 - 1F */ 24 1, 0, 1, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, /* 20 - 2F space " # % */ 25 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, /* 30 - 3F < > */ 26 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, /* 40 - 4F */ 27 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 0, /* 50 - 5F [ \ ] ^ */ 28 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, /* 60 - 6F ` */ 29 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 0, 1, /* 70 - 7F { | } DEL */ 30 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, /* 80 - 8F */ 31 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, /* 90 - 9F */ 32 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, /* A0 - AF */ 33 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, /* B0 - BF */ 34 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, /* C0 - CF */ 35 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, /* D0 - DF */ 36 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, /* E0 - EF */ 37 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, /* F0 - FF */ 38 }; 39 40 41 /* c (char) and n (nibble) MUST be unsigned integer types */ 42 #define li_cton(c,n) \ 43 (((n) = (c) - '0') <= 9 || (((n) = ((c)&0xdf) - 'A') <= 5 ? ((n) += 10) : 0)) 44 45 /* b (byte) MUST be unsigned integer type 46 * https://en.wikipedia.org/wiki/UTF-8 47 * reject overlong encodings of 7-byte ASCII and invalid UTF-8 48 * (but does not detect other overlong multi-byte encodings) */ 49 #define li_utf8_invalid_byte(b) ((b) >= 0xF5 || ((b)|0x1) == 0xC1) 50 51 52 static int burl_is_unreserved (const int c) 53 { 54 return (light_isalnum(c) || c == '-' || c == '.' || c == '_' || c == '~'); 55 } 56 57 58 static int burl_normalize_basic_unreserved_fix (buffer *b, buffer *t, int i, int qs) 59 { 60 int j = i; 61 const int used = (int)buffer_clen(b); 62 const unsigned char * const s = (unsigned char *)b->ptr; 63 unsigned char * const p = 64 (unsigned char *)buffer_string_prepare_copy(t,i+(used-i)*3+1); 65 unsigned int n1, n2; 66 memcpy(p, s, (size_t)i); 67 for (; i < used; ++i, ++j) { 68 if (!encoded_chars_http_uri_reqd[s[i]]) { 69 if (s[i] == '?' && -1 == qs) qs = j; 70 p[j] = s[i]; 71 } 72 else if (s[i]=='%' && li_cton(s[i+1], n1) && li_cton(s[i+2], n2)) { 73 const unsigned int x = (n1 << 4) | n2; 74 if (burl_is_unreserved(x)) { 75 p[j] = x; 76 } 77 else { 78 p[j] = '%'; 79 p[++j] = hex_chars_uc[n1]; /*(s[i+1] & 0xdf)*/ 80 p[++j] = hex_chars_uc[n2]; /*(s[i+2] & 0xdf)*/ 81 if (li_utf8_invalid_byte(x)) qs = -2; 82 } 83 i+=2; 84 } 85 else if (s[i] == '#') break; /* ignore fragment */ 86 else { 87 p[j] = '%'; 88 p[++j] = hex_chars_uc[(s[i] >> 4) & 0xF]; 89 p[++j] = hex_chars_uc[s[i] & 0xF]; 90 if (li_utf8_invalid_byte(s[i])) qs = -2; 91 } 92 } 93 buffer_copy_string_len(b, (char *)p, (size_t)j); 94 return qs; 95 } 96 97 98 static int burl_normalize_basic_unreserved (buffer *b, buffer *t) 99 { 100 const unsigned char * const s = (unsigned char *)b->ptr; 101 const int used = (int)buffer_clen(b); 102 unsigned int n1, n2, x; 103 int qs = -1; 104 105 for (int i = 0; i < used; ++i) { 106 if (!encoded_chars_http_uri_reqd[s[i]]) { 107 if (s[i] == '?' && -1 == qs) qs = i; 108 } 109 else if (s[i]=='%' && li_cton(s[i+1], n1) && li_cton(s[i+2], n2) 110 && !burl_is_unreserved((x = (n1 << 4) | n2))) { 111 if (li_utf8_invalid_byte(x)) qs = -2; 112 if (s[i+1] >= 'a') b->ptr[i+1] &= 0xdf; /* uppercase hex */ 113 if (s[i+2] >= 'a') b->ptr[i+2] &= 0xdf; /* uppercase hex */ 114 i+=2; 115 } 116 else if (s[i] == '#') { /* ignore fragment */ 117 buffer_truncate(b, (size_t)i); 118 break; 119 } 120 else { 121 qs = burl_normalize_basic_unreserved_fix(b, t, i, qs); 122 break; 123 } 124 } 125 126 return qs; 127 } 128 129 130 static int burl_normalize_basic_required_fix (buffer *b, buffer *t, int i, int qs) 131 { 132 int j = i; 133 const int used = (int)buffer_clen(b); 134 const unsigned char * const s = (unsigned char *)b->ptr; 135 unsigned char * const p = 136 (unsigned char *)buffer_string_prepare_copy(t,i+(used-i)*3+1); 137 unsigned int n1, n2; 138 memcpy(p, s, (size_t)i); 139 for (; i < used; ++i, ++j) { 140 if (!encoded_chars_http_uri_reqd[s[i]]) { 141 if (s[i] == '?' && -1 == qs) qs = j; 142 p[j] = s[i]; 143 } 144 else if (s[i]=='%' && li_cton(s[i+1], n1) && li_cton(s[i+2], n2)) { 145 const unsigned int x = (n1 << 4) | n2; 146 if (!encoded_chars_http_uri_reqd[x] 147 && (qs < 0 148 ? (x != '/' && x != '?') 149 : (x != '&' && x != '=' && x != ';' && x != '+'))) { 150 p[j] = x; 151 } 152 else { 153 p[j] = '%'; 154 p[++j] = hex_chars_uc[n1]; /*(s[i+1] & 0xdf)*/ 155 p[++j] = hex_chars_uc[n2]; /*(s[i+2] & 0xdf)*/ 156 if (li_utf8_invalid_byte(x)) qs = -2; 157 } 158 i+=2; 159 } 160 else if (s[i] == '#') break; /* ignore fragment */ 161 else { 162 p[j] = '%'; 163 p[++j] = hex_chars_uc[(s[i] >> 4) & 0xF]; 164 p[++j] = hex_chars_uc[s[i] & 0xF]; 165 if (li_utf8_invalid_byte(s[i])) qs = -2; 166 } 167 } 168 buffer_copy_string_len(b, (char *)p, (size_t)j); 169 return qs; 170 } 171 172 173 static int burl_normalize_basic_required (buffer *b, buffer *t) 174 { 175 const unsigned char * const s = (unsigned char *)b->ptr; 176 const int used = (int)buffer_clen(b); 177 unsigned int n1, n2, x; 178 int qs = -1; 179 180 for (int i = 0; i < used; ++i) { 181 if (!encoded_chars_http_uri_reqd[s[i]]) { 182 if (s[i] == '?' && -1 == qs) qs = i; 183 } 184 else if (s[i]=='%' && li_cton(s[i+1], n1) && li_cton(s[i+2], n2) 185 && (encoded_chars_http_uri_reqd[(x = (n1 << 4) | n2)] 186 || (qs < 0 187 ? (x == '/' || x == '?') 188 : (x == '&' || x == '=' || x == ';' || x == '+')))) { 189 if (li_utf8_invalid_byte(x)) qs = -2; 190 if (s[i+1] >= 'a') b->ptr[i+1] &= 0xdf; /* uppercase hex */ 191 if (s[i+2] >= 'a') b->ptr[i+2] &= 0xdf; /* uppercase hex */ 192 i+=2; 193 } 194 else if (s[i] == '#') { /* ignore fragment */ 195 buffer_truncate(b, (size_t)i); 196 break; 197 } 198 else { 199 qs = burl_normalize_basic_required_fix(b, t, i, qs); 200 break; 201 } 202 } 203 204 return qs; 205 } 206 207 208 static int burl_contains_ctrls (const buffer *b) 209 { 210 const char * const s = b->ptr; 211 const int used = (int)buffer_clen(b); 212 for (int i = 0; i < used; ++i) { 213 if (s[i] == '%' && (s[i+1] < '2' || (s[i+1] == '7' && s[i+2] == 'F'))) 214 return 1; 215 } 216 return 0; 217 } 218 219 220 static void burl_normalize_qs20_to_plus_fix (buffer *b, int i) 221 { 222 char * const s = b->ptr; 223 const int used = (int)buffer_clen(b); 224 int j = i; 225 for (; i < used; ++i, ++j) { 226 s[j] = s[i]; 227 if (s[i] == '%' && s[i+1] == '2' && s[i+2] == '0') { 228 s[j] = '+'; 229 i+=2; 230 } 231 } 232 buffer_truncate(b, j); 233 } 234 235 236 static void burl_normalize_qs20_to_plus (buffer *b, int qs) 237 { 238 const char * const s = b->ptr; 239 const int used = qs < 0 ? 0 : (int)buffer_clen(b); 240 int i; 241 if (qs < 0) return; 242 for (i = qs+1; i < used; ++i) { 243 if (s[i] == '%' && s[i+1] == '2' && s[i+2] == '0') break; 244 } 245 if (i != used) burl_normalize_qs20_to_plus_fix(b, i); 246 } 247 248 249 static int burl_normalize_2F_to_slash_fix (buffer *b, int qs, int i) 250 { 251 char * const s = b->ptr; 252 const int blen = (int)buffer_clen(b); 253 const int used = qs < 0 ? blen : qs; 254 int j = i; 255 for (; i < used; ++i, ++j) { 256 s[j] = s[i]; 257 if (s[i] == '%' && s[i+1] == '2' && s[i+2] == 'F') { 258 s[j] = '/'; 259 i+=2; 260 } 261 } 262 if (qs >= 0) { 263 const int qslen = blen - qs; 264 memmove(s+j, s+qs, (size_t)qslen); 265 qs = j; 266 j += qslen; 267 } 268 buffer_truncate(b, j); 269 return qs; 270 } 271 272 273 static int burl_normalize_2F_to_slash (buffer *b, int qs, int flags) 274 { 275 /*("%2F" must already have been uppercased during normalization)*/ 276 const char * const s = b->ptr; 277 const int used = qs < 0 ? (int)buffer_clen(b) : qs; 278 for (int i = 0; i < used; ++i) { 279 if (s[i] == '%' && s[i+1] == '2' && s[i+2] == 'F') { 280 return (flags & HTTP_PARSEOPT_URL_NORMALIZE_PATH_2F_DECODE) 281 ? burl_normalize_2F_to_slash_fix(b, qs, i) 282 : -2; /*(flags & HTTP_PARSEOPT_URL_NORMALIZE_PATH_2F_REJECT)*/ 283 } 284 } 285 return qs; 286 } 287 288 289 static int burl_normalize_path (buffer *b, buffer *t, int qs, int flags) 290 { 291 const unsigned char * const s = (unsigned char *)b->ptr; 292 const int used = (int)buffer_clen(b); 293 int path_simplify = 0; 294 for (int i = 0, len = qs < 0 ? used : qs; i < len; ++i) { 295 if (s[i] == '.' && (s[i+1] != '.' || ++i) 296 && (s[i+1] == '/' || s[i+1] == '?' || s[i+1] == '\0')) { 297 path_simplify = 1; 298 break; 299 } 300 while (i < len && s[i] != '/') ++i; 301 if (s[i] == '/' && s[i+1] == '/') { /*(s[len] != '/')*/ 302 path_simplify = 1; 303 break; 304 } 305 } 306 307 if (path_simplify) { 308 if (flags & HTTP_PARSEOPT_URL_NORMALIZE_PATH_DOTSEG_REJECT) return -2; 309 if (qs >= 0) { 310 buffer_copy_string_len(t, b->ptr+qs, used - qs); 311 buffer_truncate(b, qs); 312 } 313 314 buffer_path_simplify(b); 315 316 if (qs >= 0) { 317 qs = (int)buffer_clen(b); 318 buffer_append_string_len(b, BUF_PTR_LEN(t)); 319 } 320 } 321 322 return qs; 323 } 324 325 326 int burl_normalize (buffer *b, buffer *t, int flags) 327 { 328 int qs; 329 330 #if defined(__WIN32) || defined(__CYGWIN__) 331 /* Windows and Cygwin treat '\\' as '/' if '\\' is present in path; 332 * convert to '/' for consistency before percent-encoding 333 * normalization which will convert '\\' to "%5C" in the URL. 334 * (Clients still should not be sending '\\' unencoded in requests.) */ 335 if (flags & HTTP_PARSEOPT_URL_NORMALIZE_PATH_BACKSLASH_TRANS) { 336 for (char *p = b->ptr; *p != '?' && *p != '\0'; ++p) { 337 if (*p == '\\') *p = '/'; 338 } 339 } 340 #endif 341 342 qs = (flags & HTTP_PARSEOPT_URL_NORMALIZE_REQUIRED) 343 ? burl_normalize_basic_required(b, t) 344 : burl_normalize_basic_unreserved(b, t); 345 if (-2 == qs) return -2; 346 347 if (flags & HTTP_PARSEOPT_URL_NORMALIZE_CTRLS_REJECT) { 348 if (burl_contains_ctrls(b)) return -2; 349 } 350 351 if (flags & (HTTP_PARSEOPT_URL_NORMALIZE_PATH_2F_DECODE 352 |HTTP_PARSEOPT_URL_NORMALIZE_PATH_2F_REJECT)) { 353 qs = burl_normalize_2F_to_slash(b, qs, flags); 354 if (-2 == qs) return -2; 355 } 356 357 if (flags & (HTTP_PARSEOPT_URL_NORMALIZE_PATH_DOTSEG_REMOVE 358 |HTTP_PARSEOPT_URL_NORMALIZE_PATH_DOTSEG_REJECT)) { 359 qs = burl_normalize_path(b, t, qs, flags); 360 if (-2 == qs) return -2; 361 } 362 363 if (flags & HTTP_PARSEOPT_URL_NORMALIZE_QUERY_20_PLUS) { 364 if (qs >= 0) burl_normalize_qs20_to_plus(b, qs); 365 } 366 367 return qs; 368 } 369 370 371 static void burl_append_encode_nde (buffer * const b, const char * const str, const size_t len) 372 { 373 /* percent-encodes everything except unreserved - . 0-9 A-Z _ a-z ~ 374 * unless already percent-encoded (does not double-encode) */ 375 /* Note: not checking for invalid UTF-8 */ 376 char * const p = buffer_string_prepare_append(b, len*3); 377 unsigned int n1, n2; 378 int j = 0; 379 for (unsigned int i = 0; i < len; ++i, ++j) { 380 if (str[i]=='%' && li_cton(str[i+1], n1) && li_cton(str[i+2], n2)) { 381 const unsigned int x = (n1 << 4) | n2; 382 if (burl_is_unreserved((int)x)) { 383 p[j] = (char)x; 384 } 385 else { /* leave UTF-8, control chars, and required chars encoded */ 386 p[j] = '%'; 387 p[++j] = str[i+1]; 388 p[++j] = str[i+2]; 389 } 390 i+=2; 391 } 392 else if (burl_is_unreserved(str[i])) { 393 p[j] = str[i]; 394 } 395 else { 396 p[j] = '%'; 397 p[++j] = hex_chars_uc[(str[i] >> 4) & 0xF]; 398 p[++j] = hex_chars_uc[str[i] & 0xF]; 399 } 400 } 401 buffer_commit(b, j); 402 } 403 404 405 static void burl_append_encode_psnde (buffer * const b, const char * const str, const size_t len) 406 { 407 /* percent-encodes everything except unreserved - . 0-9 A-Z _ a-z ~ plus / 408 * unless already percent-encoded (does not double-encode) */ 409 /* Note: not checking for invalid UTF-8 */ 410 char * const p = buffer_string_prepare_append(b, len*3); 411 unsigned int n1, n2; 412 int j = 0; 413 for (unsigned int i = 0; i < len; ++i, ++j) { 414 if (str[i]=='%' && li_cton(str[i+1], n1) && li_cton(str[i+2], n2)) { 415 const unsigned int x = (n1 << 4) | n2; 416 if (burl_is_unreserved((int)x)) { 417 p[j] = (char)x; 418 } 419 else { /* leave UTF-8, control chars, and required chars encoded */ 420 p[j] = '%'; 421 p[++j] = str[i+1]; 422 p[++j] = str[i+2]; 423 } 424 i+=2; 425 } 426 else if (burl_is_unreserved(str[i]) || str[i] == '/') { 427 p[j] = str[i]; 428 } 429 else { 430 p[j] = '%'; 431 p[++j] = hex_chars_uc[(str[i] >> 4) & 0xF]; 432 p[++j] = hex_chars_uc[str[i] & 0xF]; 433 } 434 } 435 buffer_commit(b, j); 436 } 437 438 439 static void burl_append_encode_all (buffer * const b, const char * const str, const size_t len) 440 { 441 /* percent-encodes everything except unreserved - . 0-9 A-Z _ a-z ~ 442 * Note: double-encodes any existing '%') */ 443 /* Note: not checking for invalid UTF-8 */ 444 char * const p = buffer_string_prepare_append(b, len*3); 445 int j = 0; 446 for (unsigned int i = 0; i < len; ++i, ++j) { 447 if (burl_is_unreserved(str[i])) { 448 p[j] = str[i]; 449 } 450 else { 451 p[j] = '%'; 452 p[++j] = hex_chars_uc[(str[i] >> 4) & 0xF]; 453 p[++j] = hex_chars_uc[str[i] & 0xF]; 454 } 455 } 456 buffer_commit(b, j); 457 } 458 459 460 static void burl_offset_tolower (buffer * const b, const size_t off) 461 { 462 /*(skips over all percent-encodings, including encoding of alpha chars)*/ 463 for (char *p = b->ptr+off; p[0]; ++p) { 464 if (light_isupper(p[0])) p[0] |= 0x20; 465 else if (p[0]=='%' && light_isxdigit(p[1]) && light_isxdigit(p[2])) 466 p+=2; 467 } 468 } 469 470 471 static void burl_offset_toupper (buffer * const b, const size_t off) 472 { 473 /*(skips over all percent-encodings, including encoding of alpha chars)*/ 474 for (char *p = b->ptr+off; p[0]; ++p) { 475 if (light_islower(p[0])) p[0] &= 0xdf; 476 else if (p[0]=='%' && light_isxdigit(p[1]) && light_isxdigit(p[2])) 477 p+=2; 478 } 479 } 480 481 482 void burl_append (buffer * const b, const char * const str, const size_t len, const int flags) 483 { 484 size_t off = 0; 485 486 if (0 == len) return; 487 488 if (0 == flags) { 489 buffer_append_string_len(b, str, len); 490 return; 491 } 492 493 if (flags & (BURL_TOUPPER|BURL_TOLOWER)) off = buffer_clen(b); 494 495 if (flags & BURL_ENCODE_NONE) { 496 buffer_append_string_len(b, str, len); 497 } 498 else if (flags & BURL_ENCODE_ALL) { 499 burl_append_encode_all(b, str, len); 500 } 501 else if (flags & BURL_ENCODE_NDE) { 502 burl_append_encode_nde(b, str, len); 503 } 504 else if (flags & BURL_ENCODE_PSNDE) { 505 burl_append_encode_psnde(b, str, len); 506 } 507 else if (flags & BURL_ENCODE_B64U) { 508 const unsigned char *s = (const unsigned char *)str; 509 buffer_append_base64_encode_no_padding(b, s, len, BASE64_URL); 510 } 511 else if (flags & BURL_DECODE_B64U) { 512 buffer_append_base64_decode(b, str, len, BASE64_URL); 513 } 514 515 /* note: not normalizing str, which could come from arbitrary header, 516 * so it is possible that alpha chars are percent-encoded upper/lowercase */ 517 if (flags & (BURL_TOLOWER|BURL_TOUPPER)) { 518 (flags & BURL_TOLOWER) 519 ? burl_offset_tolower(b, off) /*(flags & BURL_TOLOWER)*/ 520 : burl_offset_toupper(b, off); /*(flags & BURL_TOUPPER)*/ 521 } 522 } 523