1 /* $FreeBSD$ */
2 /* $NetBSD: citrus_iconv_std.c,v 1.16 2012/02/12 13:51:29 wiz Exp $ */
3
4 /*-
5 * SPDX-License-Identifier: BSD-2-Clause
6 *
7 * Copyright (c)2003 Citrus Project,
8 * All rights reserved.
9 *
10 * Redistribution and use in source and binary forms, with or without
11 * modification, are permitted provided that the following conditions
12 * are met:
13 * 1. Redistributions of source code must retain the above copyright
14 * notice, this list of conditions and the following disclaimer.
15 * 2. Redistributions in binary form must reproduce the above copyright
16 * notice, this list of conditions and the following disclaimer in the
17 * documentation and/or other materials provided with the distribution.
18 *
19 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
20 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
21 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
22 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
23 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
24 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
25 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
26 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
27 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
28 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
29 * SUCH DAMAGE.
30 */
31
32 #include <sys/cdefs.h>
33 #include <sys/endian.h>
34 #include <sys/queue.h>
35
36 #include <assert.h>
37 #include <errno.h>
38 #include <limits.h>
39 #include <stdbool.h>
40 #include <stdio.h>
41 #include <stdlib.h>
42 #include <string.h>
43
44 #include "citrus_namespace.h"
45 #include "citrus_types.h"
46 #include "citrus_module.h"
47 #include "citrus_region.h"
48 #include "citrus_mmap.h"
49 #include "citrus_hash.h"
50 #include "citrus_iconv.h"
51 #include "citrus_stdenc.h"
52 #include "citrus_mapper.h"
53 #include "citrus_csmapper.h"
54 #include "citrus_memstream.h"
55 #include "citrus_iconv_std.h"
56 #include "citrus_esdb.h"
57
58 /* ---------------------------------------------------------------------- */
59
60 _CITRUS_ICONV_DECLS(iconv_std);
61 _CITRUS_ICONV_DEF_OPS(iconv_std);
62
63
64 /* ---------------------------------------------------------------------- */
65
66 int
_citrus_iconv_std_iconv_getops(struct _citrus_iconv_ops * ops)67 _citrus_iconv_std_iconv_getops(struct _citrus_iconv_ops *ops)
68 {
69
70 memcpy(ops, &_citrus_iconv_std_iconv_ops,
71 sizeof(_citrus_iconv_std_iconv_ops));
72
73 return (0);
74 }
75
76 /* ---------------------------------------------------------------------- */
77
78 /*
79 * convenience routines for stdenc.
80 */
81 static __inline void
save_encoding_state(struct _citrus_iconv_std_encoding * se)82 save_encoding_state(struct _citrus_iconv_std_encoding *se)
83 {
84
85 if (se->se_ps)
86 memcpy(se->se_pssaved, se->se_ps,
87 _stdenc_get_state_size(se->se_handle));
88 }
89
90 static __inline void
restore_encoding_state(struct _citrus_iconv_std_encoding * se)91 restore_encoding_state(struct _citrus_iconv_std_encoding *se)
92 {
93
94 if (se->se_ps)
95 memcpy(se->se_ps, se->se_pssaved,
96 _stdenc_get_state_size(se->se_handle));
97 }
98
99 static __inline void
init_encoding_state(struct _citrus_iconv_std_encoding * se)100 init_encoding_state(struct _citrus_iconv_std_encoding *se)
101 {
102
103 if (se->se_ps)
104 _stdenc_init_state(se->se_handle, se->se_ps);
105 }
106
107 static __inline int
mbtocsx(struct _citrus_iconv_std_encoding * se,_csid_t * csid,_index_t * idx,char ** s,size_t n,size_t * nresult,struct iconv_hooks * hooks)108 mbtocsx(struct _citrus_iconv_std_encoding *se,
109 _csid_t *csid, _index_t *idx, char **s, size_t n, size_t *nresult,
110 struct iconv_hooks *hooks)
111 {
112
113 return (_stdenc_mbtocs(se->se_handle, csid, idx, s, n, se->se_ps,
114 nresult, hooks));
115 }
116
117 static __inline int
cstombx(struct _citrus_iconv_std_encoding * se,char * s,size_t n,_csid_t csid,_index_t idx,size_t * nresult,struct iconv_hooks * hooks)118 cstombx(struct _citrus_iconv_std_encoding *se,
119 char *s, size_t n, _csid_t csid, _index_t idx, size_t *nresult,
120 struct iconv_hooks *hooks)
121 {
122
123 return (_stdenc_cstomb(se->se_handle, s, n, csid, idx, se->se_ps,
124 nresult, hooks));
125 }
126
127 static __inline int
wctombx(struct _citrus_iconv_std_encoding * se,char * s,size_t n,_wc_t wc,size_t * nresult,struct iconv_hooks * hooks)128 wctombx(struct _citrus_iconv_std_encoding *se,
129 char *s, size_t n, _wc_t wc, size_t *nresult,
130 struct iconv_hooks *hooks)
131 {
132
133 return (_stdenc_wctomb(se->se_handle, s, n, wc, se->se_ps, nresult,
134 hooks));
135 }
136
137 static __inline int
put_state_resetx(struct _citrus_iconv_std_encoding * se,char * s,size_t n,size_t * nresult)138 put_state_resetx(struct _citrus_iconv_std_encoding *se, char *s, size_t n,
139 size_t *nresult)
140 {
141
142 return (_stdenc_put_state_reset(se->se_handle, s, n, se->se_ps, nresult));
143 }
144
145 static __inline int
get_state_desc_gen(struct _citrus_iconv_std_encoding * se,int * rstate)146 get_state_desc_gen(struct _citrus_iconv_std_encoding *se, int *rstate)
147 {
148 struct _stdenc_state_desc ssd;
149 int ret;
150
151 ret = _stdenc_get_state_desc(se->se_handle, se->se_ps,
152 _STDENC_SDID_GENERIC, &ssd);
153 if (!ret)
154 *rstate = ssd.u.generic.state;
155
156 return (ret);
157 }
158
159 /*
160 * init encoding context
161 */
162 static int
init_encoding(struct _citrus_iconv_std_encoding * se,struct _stdenc * cs,void * ps1,void * ps2)163 init_encoding(struct _citrus_iconv_std_encoding *se, struct _stdenc *cs,
164 void *ps1, void *ps2)
165 {
166 int ret = -1;
167
168 se->se_handle = cs;
169 se->se_ps = ps1;
170 se->se_pssaved = ps2;
171
172 if (se->se_ps)
173 ret = _stdenc_init_state(cs, se->se_ps);
174 if (!ret && se->se_pssaved)
175 ret = _stdenc_init_state(cs, se->se_pssaved);
176
177 return (ret);
178 }
179
180 static int
open_csmapper(struct _csmapper ** rcm,const char * src,const char * dst,unsigned long * rnorm)181 open_csmapper(struct _csmapper **rcm, const char *src, const char *dst,
182 unsigned long *rnorm)
183 {
184 struct _csmapper *cm;
185 int ret;
186
187 ret = _csmapper_open(&cm, src, dst, 0, rnorm);
188 if (ret)
189 return (ret);
190 if (_csmapper_get_src_max(cm) != 1 || _csmapper_get_dst_max(cm) != 1 ||
191 _csmapper_get_state_size(cm) != 0) {
192 _csmapper_close(cm);
193 return (EINVAL);
194 }
195
196 *rcm = cm;
197
198 return (0);
199 }
200
201 static void
close_dsts(struct _citrus_iconv_std_dst_list * dl)202 close_dsts(struct _citrus_iconv_std_dst_list *dl)
203 {
204 struct _citrus_iconv_std_dst *sd;
205
206 while ((sd = TAILQ_FIRST(dl)) != NULL) {
207 TAILQ_REMOVE(dl, sd, sd_entry);
208 _csmapper_close(sd->sd_mapper);
209 free(sd);
210 }
211 }
212
213 static int
open_dsts(struct _citrus_iconv_std_dst_list * dl,const struct _esdb_charset * ec,const struct _esdb * dbdst)214 open_dsts(struct _citrus_iconv_std_dst_list *dl,
215 const struct _esdb_charset *ec, const struct _esdb *dbdst)
216 {
217 struct _citrus_iconv_std_dst *sd, *sdtmp;
218 unsigned long norm;
219 int i, ret;
220
221 sd = malloc(sizeof(*sd));
222 if (sd == NULL)
223 return (errno);
224
225 for (i = 0; i < dbdst->db_num_charsets; i++) {
226 ret = open_csmapper(&sd->sd_mapper, ec->ec_csname,
227 dbdst->db_charsets[i].ec_csname, &norm);
228 if (ret == 0) {
229 sd->sd_csid = dbdst->db_charsets[i].ec_csid;
230 sd->sd_norm = norm;
231 /* insert this mapper by sorted order. */
232 TAILQ_FOREACH(sdtmp, dl, sd_entry) {
233 if (sdtmp->sd_norm > norm) {
234 TAILQ_INSERT_BEFORE(sdtmp, sd,
235 sd_entry);
236 sd = NULL;
237 break;
238 }
239 }
240 if (sd)
241 TAILQ_INSERT_TAIL(dl, sd, sd_entry);
242 sd = malloc(sizeof(*sd));
243 if (sd == NULL) {
244 ret = errno;
245 close_dsts(dl);
246 return (ret);
247 }
248 } else if (ret != ENOENT) {
249 close_dsts(dl);
250 free(sd);
251 return (ret);
252 }
253 }
254 free(sd);
255 return (0);
256 }
257
258 static void
close_srcs(struct _citrus_iconv_std_src_list * sl)259 close_srcs(struct _citrus_iconv_std_src_list *sl)
260 {
261 struct _citrus_iconv_std_src *ss;
262
263 while ((ss = TAILQ_FIRST(sl)) != NULL) {
264 TAILQ_REMOVE(sl, ss, ss_entry);
265 close_dsts(&ss->ss_dsts);
266 free(ss);
267 }
268 }
269
270 static int
open_srcs(struct _citrus_iconv_std_src_list * sl,const struct _esdb * dbsrc,const struct _esdb * dbdst)271 open_srcs(struct _citrus_iconv_std_src_list *sl,
272 const struct _esdb *dbsrc, const struct _esdb *dbdst)
273 {
274 struct _citrus_iconv_std_src *ss;
275 int count = 0, i, ret;
276
277 ss = malloc(sizeof(*ss));
278 if (ss == NULL)
279 return (errno);
280
281 TAILQ_INIT(&ss->ss_dsts);
282
283 for (i = 0; i < dbsrc->db_num_charsets; i++) {
284 ret = open_dsts(&ss->ss_dsts, &dbsrc->db_charsets[i], dbdst);
285 if (ret)
286 goto err;
287 if (!TAILQ_EMPTY(&ss->ss_dsts)) {
288 ss->ss_csid = dbsrc->db_charsets[i].ec_csid;
289 TAILQ_INSERT_TAIL(sl, ss, ss_entry);
290 ss = malloc(sizeof(*ss));
291 if (ss == NULL) {
292 ret = errno;
293 goto err;
294 }
295 count++;
296 TAILQ_INIT(&ss->ss_dsts);
297 }
298 }
299 free(ss);
300
301 return (count ? 0 : ENOENT);
302
303 err:
304 free(ss);
305 close_srcs(sl);
306 return (ret);
307 }
308
309 /* do convert a character */
310 #define E_NO_CORRESPONDING_CHAR ENOENT /* XXX */
311 static int
312 /*ARGSUSED*/
do_conv(const struct _citrus_iconv_std_shared * is,_csid_t * csid,_index_t * idx)313 do_conv(const struct _citrus_iconv_std_shared *is,
314 _csid_t *csid, _index_t *idx)
315 {
316 struct _citrus_iconv_std_dst *sd;
317 struct _citrus_iconv_std_src *ss;
318 _index_t tmpidx;
319 int ret;
320
321 TAILQ_FOREACH(ss, &is->is_srcs, ss_entry) {
322 if (ss->ss_csid == *csid) {
323 TAILQ_FOREACH(sd, &ss->ss_dsts, sd_entry) {
324 ret = _csmapper_convert(sd->sd_mapper,
325 &tmpidx, *idx, NULL);
326 switch (ret) {
327 case _MAPPER_CONVERT_SUCCESS:
328 *csid = sd->sd_csid;
329 *idx = tmpidx;
330 return (0);
331 case _MAPPER_CONVERT_NONIDENTICAL:
332 break;
333 case _MAPPER_CONVERT_SRC_MORE:
334 /*FALLTHROUGH*/
335 case _MAPPER_CONVERT_DST_MORE:
336 /*FALLTHROUGH*/
337 case _MAPPER_CONVERT_ILSEQ:
338 return (EILSEQ);
339 case _MAPPER_CONVERT_FATAL:
340 return (EINVAL);
341 }
342 }
343 break;
344 }
345 }
346
347 return (E_NO_CORRESPONDING_CHAR);
348 }
349 /* ---------------------------------------------------------------------- */
350
351 static int
352 /*ARGSUSED*/
_citrus_iconv_std_iconv_init_shared(struct _citrus_iconv_shared * ci,const char * __restrict src,const char * __restrict dst)353 _citrus_iconv_std_iconv_init_shared(struct _citrus_iconv_shared *ci,
354 const char * __restrict src, const char * __restrict dst)
355 {
356 struct _citrus_esdb esdbdst, esdbsrc;
357 struct _citrus_iconv_std_shared *is;
358 int ret;
359
360 is = malloc(sizeof(*is));
361 if (is == NULL) {
362 ret = errno;
363 goto err0;
364 }
365 ret = _citrus_esdb_open(&esdbsrc, src);
366 if (ret)
367 goto err1;
368 ret = _citrus_esdb_open(&esdbdst, dst);
369 if (ret)
370 goto err2;
371 ret = _stdenc_open(&is->is_src_encoding, esdbsrc.db_encname,
372 esdbsrc.db_variable, esdbsrc.db_len_variable);
373 if (ret)
374 goto err3;
375 ret = _stdenc_open(&is->is_dst_encoding, esdbdst.db_encname,
376 esdbdst.db_variable, esdbdst.db_len_variable);
377 if (ret)
378 goto err4;
379 is->is_use_invalid = esdbdst.db_use_invalid;
380 is->is_invalid = esdbdst.db_invalid;
381
382 TAILQ_INIT(&is->is_srcs);
383 ret = open_srcs(&is->is_srcs, &esdbsrc, &esdbdst);
384 if (ret)
385 goto err5;
386
387 _esdb_close(&esdbsrc);
388 _esdb_close(&esdbdst);
389 ci->ci_closure = is;
390
391 return (0);
392
393 err5:
394 _stdenc_close(is->is_dst_encoding);
395 err4:
396 _stdenc_close(is->is_src_encoding);
397 err3:
398 _esdb_close(&esdbdst);
399 err2:
400 _esdb_close(&esdbsrc);
401 err1:
402 free(is);
403 err0:
404 return (ret);
405 }
406
407 static void
_citrus_iconv_std_iconv_uninit_shared(struct _citrus_iconv_shared * ci)408 _citrus_iconv_std_iconv_uninit_shared(struct _citrus_iconv_shared *ci)
409 {
410 struct _citrus_iconv_std_shared *is = ci->ci_closure;
411
412 if (is == NULL)
413 return;
414
415 _stdenc_close(is->is_src_encoding);
416 _stdenc_close(is->is_dst_encoding);
417 close_srcs(&is->is_srcs);
418 free(is);
419 }
420
421 static int
_citrus_iconv_std_iconv_init_context(struct _citrus_iconv * cv)422 _citrus_iconv_std_iconv_init_context(struct _citrus_iconv *cv)
423 {
424 const struct _citrus_iconv_std_shared *is = cv->cv_shared->ci_closure;
425 struct _citrus_iconv_std_context *sc;
426 char *ptr;
427 size_t sz, szpsdst, szpssrc;
428
429 szpssrc = _stdenc_get_state_size(is->is_src_encoding);
430 szpsdst = _stdenc_get_state_size(is->is_dst_encoding);
431
432 sz = (szpssrc + szpsdst)*2 + sizeof(struct _citrus_iconv_std_context);
433 sc = malloc(sz);
434 if (sc == NULL)
435 return (errno);
436
437 ptr = (char *)&sc[1];
438 if (szpssrc > 0)
439 init_encoding(&sc->sc_src_encoding, is->is_src_encoding,
440 ptr, ptr+szpssrc);
441 else
442 init_encoding(&sc->sc_src_encoding, is->is_src_encoding,
443 NULL, NULL);
444 ptr += szpssrc*2;
445 if (szpsdst > 0)
446 init_encoding(&sc->sc_dst_encoding, is->is_dst_encoding,
447 ptr, ptr+szpsdst);
448 else
449 init_encoding(&sc->sc_dst_encoding, is->is_dst_encoding,
450 NULL, NULL);
451
452 cv->cv_closure = (void *)sc;
453
454 return (0);
455 }
456
457 static void
_citrus_iconv_std_iconv_uninit_context(struct _citrus_iconv * cv)458 _citrus_iconv_std_iconv_uninit_context(struct _citrus_iconv *cv)
459 {
460
461 free(cv->cv_closure);
462 }
463
464 static int
_citrus_iconv_std_iconv_convert(struct _citrus_iconv * __restrict cv,char * __restrict * __restrict in,size_t * __restrict inbytes,char * __restrict * __restrict out,size_t * __restrict outbytes,uint32_t flags,size_t * __restrict invalids)465 _citrus_iconv_std_iconv_convert(struct _citrus_iconv * __restrict cv,
466 char * __restrict * __restrict in, size_t * __restrict inbytes,
467 char * __restrict * __restrict out, size_t * __restrict outbytes,
468 uint32_t flags, size_t * __restrict invalids)
469 {
470 const struct _citrus_iconv_std_shared *is = cv->cv_shared->ci_closure;
471 struct _citrus_iconv_std_context *sc = cv->cv_closure;
472 _csid_t csid;
473 _index_t idx;
474 char *tmpin;
475 size_t inval, szrin, szrout;
476 int ret, state = 0;
477
478 inval = 0;
479 if (in == NULL || *in == NULL) {
480 /* special cases */
481 if (out != NULL && *out != NULL) {
482 /* init output state and store the shift sequence */
483 save_encoding_state(&sc->sc_src_encoding);
484 save_encoding_state(&sc->sc_dst_encoding);
485 szrout = 0;
486
487 ret = put_state_resetx(&sc->sc_dst_encoding,
488 *out, *outbytes, &szrout);
489 if (ret)
490 goto err;
491
492 if (szrout == (size_t)-2) {
493 /* too small to store the character */
494 ret = EINVAL;
495 goto err;
496 }
497 *out += szrout;
498 *outbytes -= szrout;
499 } else
500 /* otherwise, discard the shift sequence */
501 init_encoding_state(&sc->sc_dst_encoding);
502 init_encoding_state(&sc->sc_src_encoding);
503 *invalids = 0;
504 return (0);
505 }
506
507 /* normal case */
508 for (;;) {
509 if (*inbytes == 0) {
510 ret = get_state_desc_gen(&sc->sc_src_encoding, &state);
511 if (state == _STDENC_SDGEN_INITIAL ||
512 state == _STDENC_SDGEN_STABLE)
513 break;
514 }
515
516 /* save the encoding states for the error recovery */
517 save_encoding_state(&sc->sc_src_encoding);
518 save_encoding_state(&sc->sc_dst_encoding);
519
520 /* mb -> csid/index */
521 tmpin = *in;
522 szrin = szrout = 0;
523 ret = mbtocsx(&sc->sc_src_encoding, &csid, &idx, &tmpin,
524 *inbytes, &szrin, cv->cv_shared->ci_hooks);
525 if (ret)
526 goto err;
527
528 if (szrin == (size_t)-2) {
529 /* incompleted character */
530 ret = get_state_desc_gen(&sc->sc_src_encoding, &state);
531 if (ret) {
532 ret = EINVAL;
533 goto err;
534 }
535 switch (state) {
536 case _STDENC_SDGEN_INITIAL:
537 case _STDENC_SDGEN_STABLE:
538 /* fetch shift sequences only. */
539 goto next;
540 }
541 ret = EINVAL;
542 goto err;
543 }
544 /* convert the character */
545 ret = do_conv(is, &csid, &idx);
546 if (ret) {
547 if (ret == E_NO_CORRESPONDING_CHAR) {
548 /*
549 * GNU iconv returns EILSEQ when no
550 * corresponding character in the output.
551 * Some software depends on this behavior
552 * though this is against POSIX specification.
553 */
554 if (cv->cv_shared->ci_ilseq_invalid != 0) {
555 ret = EILSEQ;
556 goto err;
557 }
558 inval++;
559 szrout = 0;
560 if ((((flags & _CITRUS_ICONV_F_HIDE_INVALID) == 0) &&
561 !cv->cv_shared->ci_discard_ilseq) &&
562 is->is_use_invalid) {
563 ret = wctombx(&sc->sc_dst_encoding,
564 *out, *outbytes, is->is_invalid,
565 &szrout, cv->cv_shared->ci_hooks);
566 if (ret)
567 goto err;
568 }
569 goto next;
570 } else
571 goto err;
572 }
573 /* csid/index -> mb */
574 ret = cstombx(&sc->sc_dst_encoding,
575 *out, *outbytes, csid, idx, &szrout,
576 cv->cv_shared->ci_hooks);
577 if (ret)
578 goto err;
579 next:
580 *inbytes -= tmpin-*in; /* szrin is insufficient on \0. */
581 *in = tmpin;
582 *outbytes -= szrout;
583 *out += szrout;
584 }
585 *invalids = inval;
586
587 return (0);
588
589 err:
590 restore_encoding_state(&sc->sc_src_encoding);
591 restore_encoding_state(&sc->sc_dst_encoding);
592 *invalids = inval;
593
594 return (ret);
595 }
596