1 /*-
2 * SPDX-License-Identifier: BSD-2-Clause
3 *
4 * Copyright (c) 2009-2010 The FreeBSD Foundation
5 *
6 * This software was developed by Pawel Jakub Dawidek under sponsorship from
7 * the FreeBSD Foundation.
8 *
9 * Redistribution and use in source and binary forms, with or without
10 * modification, are permitted provided that the following conditions
11 * are met:
12 * 1. Redistributions of source code must retain the above copyright
13 * notice, this list of conditions and the following disclaimer.
14 * 2. Redistributions in binary form must reproduce the above copyright
15 * notice, this list of conditions and the following disclaimer in the
16 * documentation and/or other materials provided with the distribution.
17 *
18 * THIS SOFTWARE IS PROVIDED BY THE AUTHORS AND CONTRIBUTORS ``AS IS'' AND
19 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
20 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
21 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHORS OR CONTRIBUTORS BE LIABLE
22 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
23 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
24 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
25 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
26 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
27 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
28 * SUCH DAMAGE.
29 */
30
31 #include <sys/cdefs.h>
32 #include <sys/types.h>
33 #include <sys/wait.h>
34
35 #include <errno.h>
36 #include <pthread.h>
37 #include <signal.h>
38 #include <stdio.h>
39 #include <string.h>
40 #include <unistd.h>
41
42 #include "hast.h"
43 #include "hastd.h"
44 #include "hast_checksum.h"
45 #include "hast_compression.h"
46 #include "hast_proto.h"
47 #include "hooks.h"
48 #include "nv.h"
49 #include "pjdlog.h"
50 #include "proto.h"
51 #include "subr.h"
52
53 #include "control.h"
54
55 void
child_cleanup(struct hast_resource * res)56 child_cleanup(struct hast_resource *res)
57 {
58
59 proto_close(res->hr_ctrl);
60 res->hr_ctrl = NULL;
61 if (res->hr_event != NULL) {
62 proto_close(res->hr_event);
63 res->hr_event = NULL;
64 }
65 if (res->hr_conn != NULL) {
66 proto_close(res->hr_conn);
67 res->hr_conn = NULL;
68 }
69 res->hr_workerpid = 0;
70 }
71
72 static void
control_set_role_common(struct hastd_config * cfg,struct nv * nvout,uint8_t role,struct hast_resource * res,const char * name,unsigned int no)73 control_set_role_common(struct hastd_config *cfg, struct nv *nvout,
74 uint8_t role, struct hast_resource *res, const char *name, unsigned int no)
75 {
76 int oldrole;
77
78 /* Name is always needed. */
79 if (name != NULL)
80 nv_add_string(nvout, name, "resource%u", no);
81
82 if (res == NULL) {
83 PJDLOG_ASSERT(cfg != NULL);
84 PJDLOG_ASSERT(name != NULL);
85
86 TAILQ_FOREACH(res, &cfg->hc_resources, hr_next) {
87 if (strcmp(res->hr_name, name) == 0)
88 break;
89 }
90 if (res == NULL) {
91 nv_add_int16(nvout, EHAST_NOENTRY, "error%u", no);
92 return;
93 }
94 }
95 PJDLOG_ASSERT(res != NULL);
96
97 /* Send previous role back. */
98 nv_add_string(nvout, role2str(res->hr_role), "role%u", no);
99
100 /* Nothing changed, return here. */
101 if (role == res->hr_role)
102 return;
103
104 pjdlog_prefix_set("[%s] (%s) ", res->hr_name, role2str(res->hr_role));
105 pjdlog_info("Role changed to %s.", role2str(role));
106
107 /* Change role to the new one. */
108 oldrole = res->hr_role;
109 res->hr_role = role;
110 pjdlog_prefix_set("[%s] (%s) ", res->hr_name, role2str(res->hr_role));
111
112 /*
113 * If previous role was primary or secondary we have to kill process
114 * doing that work.
115 */
116 if (res->hr_workerpid != 0) {
117 if (kill(res->hr_workerpid, SIGTERM) == -1) {
118 pjdlog_errno(LOG_WARNING,
119 "Unable to kill worker process %u",
120 (unsigned int)res->hr_workerpid);
121 } else if (waitpid(res->hr_workerpid, NULL, 0) !=
122 res->hr_workerpid) {
123 pjdlog_errno(LOG_WARNING,
124 "Error while waiting for worker process %u",
125 (unsigned int)res->hr_workerpid);
126 } else {
127 pjdlog_debug(1, "Worker process %u stopped.",
128 (unsigned int)res->hr_workerpid);
129 }
130 child_cleanup(res);
131 }
132
133 /* Start worker process if we are changing to primary. */
134 if (role == HAST_ROLE_PRIMARY)
135 hastd_primary(res);
136 pjdlog_prefix_set("%s", "");
137 hook_exec(res->hr_exec, "role", res->hr_name, role2str(oldrole),
138 role2str(res->hr_role), NULL);
139 }
140
141 void
control_set_role(struct hast_resource * res,uint8_t role)142 control_set_role(struct hast_resource *res, uint8_t role)
143 {
144
145 control_set_role_common(NULL, NULL, role, res, NULL, 0);
146 }
147
148 static void
control_status_worker(struct hast_resource * res,struct nv * nvout,unsigned int no)149 control_status_worker(struct hast_resource *res, struct nv *nvout,
150 unsigned int no)
151 {
152 struct nv *cnvin, *cnvout;
153 const char *str;
154 int error;
155
156 cnvin = NULL;
157
158 /*
159 * Prepare and send command to worker process.
160 */
161 cnvout = nv_alloc();
162 nv_add_uint8(cnvout, CONTROL_STATUS, "cmd");
163 error = nv_error(cnvout);
164 if (error != 0) {
165 pjdlog_common(LOG_ERR, 0, error,
166 "Unable to prepare control header");
167 goto end;
168 }
169 if (hast_proto_send(res, res->hr_ctrl, cnvout, NULL, 0) == -1) {
170 error = errno;
171 pjdlog_errno(LOG_ERR, "Unable to send control header");
172 goto end;
173 }
174
175 /*
176 * Receive response.
177 */
178 if (hast_proto_recv_hdr(res->hr_ctrl, &cnvin) == -1) {
179 error = errno;
180 pjdlog_errno(LOG_ERR, "Unable to receive control header");
181 goto end;
182 }
183
184 error = nv_get_int16(cnvin, "error");
185 if (error != 0)
186 goto end;
187
188 if ((str = nv_get_string(cnvin, "status")) == NULL) {
189 error = ENOENT;
190 pjdlog_errno(LOG_ERR, "Field 'status' is missing.");
191 goto end;
192 }
193 nv_add_string(nvout, str, "status%u", no);
194 nv_add_uint64(nvout, nv_get_uint64(cnvin, "dirty"), "dirty%u", no);
195 nv_add_uint32(nvout, nv_get_uint32(cnvin, "extentsize"),
196 "extentsize%u", no);
197 nv_add_uint32(nvout, nv_get_uint32(cnvin, "keepdirty"),
198 "keepdirty%u", no);
199 nv_add_uint64(nvout, nv_get_uint64(cnvin, "stat_read"),
200 "stat_read%u", no);
201 nv_add_uint64(nvout, nv_get_uint64(cnvin, "stat_write"),
202 "stat_write%u", no);
203 nv_add_uint64(nvout, nv_get_uint64(cnvin, "stat_delete"),
204 "stat_delete%u", no);
205 nv_add_uint64(nvout, nv_get_uint64(cnvin, "stat_flush"),
206 "stat_flush%u", no);
207 nv_add_uint64(nvout, nv_get_uint64(cnvin, "stat_activemap_update"),
208 "stat_activemap_update%u", no);
209 nv_add_uint64(nvout, nv_get_uint64(cnvin, "stat_read_error"),
210 "stat_read_error%u", no);
211 nv_add_uint64(nvout, nv_get_uint64(cnvin, "stat_write_error"),
212 "stat_write_error%u", no);
213 nv_add_uint64(nvout, nv_get_uint64(cnvin, "stat_delete_error"),
214 "stat_delete_error%u", no);
215 nv_add_uint64(nvout, nv_get_uint64(cnvin, "stat_flush_error"),
216 "stat_flush_error%u", no);
217 nv_add_uint64(nvout, nv_get_uint64(cnvin, "idle_queue_size"),
218 "idle_queue_size%u", no);
219 nv_add_uint64(nvout, nv_get_uint64(cnvin, "local_queue_size"),
220 "local_queue_size%u", no);
221 nv_add_uint64(nvout, nv_get_uint64(cnvin, "send_queue_size"),
222 "send_queue_size%u", no);
223 nv_add_uint64(nvout, nv_get_uint64(cnvin, "recv_queue_size"),
224 "recv_queue_size%u", no);
225 nv_add_uint64(nvout, nv_get_uint64(cnvin, "done_queue_size"),
226 "done_queue_size%u", no);
227 end:
228 if (cnvin != NULL)
229 nv_free(cnvin);
230 if (cnvout != NULL)
231 nv_free(cnvout);
232 if (error != 0)
233 nv_add_int16(nvout, error, "error");
234 }
235
236 static void
control_status(struct hastd_config * cfg,struct nv * nvout,struct hast_resource * res,const char * name,unsigned int no)237 control_status(struct hastd_config *cfg, struct nv *nvout,
238 struct hast_resource *res, const char *name, unsigned int no)
239 {
240
241 PJDLOG_ASSERT(cfg != NULL);
242 PJDLOG_ASSERT(nvout != NULL);
243 PJDLOG_ASSERT(name != NULL);
244
245 /* Name is always needed. */
246 nv_add_string(nvout, name, "resource%u", no);
247
248 if (res == NULL) {
249 TAILQ_FOREACH(res, &cfg->hc_resources, hr_next) {
250 if (strcmp(res->hr_name, name) == 0)
251 break;
252 }
253 if (res == NULL) {
254 nv_add_int16(nvout, EHAST_NOENTRY, "error%u", no);
255 return;
256 }
257 }
258 PJDLOG_ASSERT(res != NULL);
259 nv_add_string(nvout, res->hr_provname, "provname%u", no);
260 nv_add_string(nvout, res->hr_localpath, "localpath%u", no);
261 nv_add_string(nvout, res->hr_remoteaddr, "remoteaddr%u", no);
262 if (res->hr_sourceaddr[0] != '\0')
263 nv_add_string(nvout, res->hr_sourceaddr, "sourceaddr%u", no);
264 switch (res->hr_replication) {
265 case HAST_REPLICATION_FULLSYNC:
266 nv_add_string(nvout, "fullsync", "replication%u", no);
267 break;
268 case HAST_REPLICATION_MEMSYNC:
269 nv_add_string(nvout, "memsync", "replication%u", no);
270 break;
271 case HAST_REPLICATION_ASYNC:
272 nv_add_string(nvout, "async", "replication%u", no);
273 break;
274 default:
275 nv_add_string(nvout, "unknown", "replication%u", no);
276 break;
277 }
278 nv_add_string(nvout, checksum_name(res->hr_checksum),
279 "checksum%u", no);
280 nv_add_string(nvout, compression_name(res->hr_compression),
281 "compression%u", no);
282 nv_add_string(nvout, role2str(res->hr_role), "role%u", no);
283 nv_add_int32(nvout, res->hr_workerpid, "workerpid%u", no);
284
285 switch (res->hr_role) {
286 case HAST_ROLE_PRIMARY:
287 PJDLOG_ASSERT(res->hr_workerpid != 0);
288 /* FALLTHROUGH */
289 case HAST_ROLE_SECONDARY:
290 if (res->hr_workerpid != 0)
291 break;
292 /* FALLTHROUGH */
293 default:
294 return;
295 }
296
297 /*
298 * If we are here, it means that we have a worker process, which we
299 * want to ask some questions.
300 */
301 control_status_worker(res, nvout, no);
302 }
303
304 void
control_handle(struct hastd_config * cfg)305 control_handle(struct hastd_config *cfg)
306 {
307 struct proto_conn *conn;
308 struct nv *nvin, *nvout;
309 unsigned int ii;
310 const char *str;
311 uint8_t cmd, role;
312 int error;
313
314 if (proto_accept(cfg->hc_controlconn, &conn) == -1) {
315 pjdlog_errno(LOG_ERR, "Unable to accept control connection");
316 return;
317 }
318
319 cfg->hc_controlin = conn;
320 nvin = nvout = NULL;
321 role = HAST_ROLE_UNDEF;
322
323 if (hast_proto_recv_hdr(conn, &nvin) == -1) {
324 pjdlog_errno(LOG_ERR, "Unable to receive control header");
325 nvin = NULL;
326 goto close;
327 }
328
329 /* Obtain command code. 0 means that nv_get_uint8() failed. */
330 cmd = nv_get_uint8(nvin, "cmd");
331 if (cmd == 0) {
332 pjdlog_error("Control header is missing 'cmd' field.");
333 goto close;
334 }
335
336 /* Allocate outgoing nv structure. */
337 nvout = nv_alloc();
338 if (nvout == NULL) {
339 pjdlog_error("Unable to allocate header for control response.");
340 goto close;
341 }
342
343 error = 0;
344
345 str = nv_get_string(nvin, "resource0");
346 if (str == NULL) {
347 pjdlog_error("Control header is missing 'resource0' field.");
348 error = EHAST_INVALID;
349 goto fail;
350 }
351 if (cmd == HASTCTL_CMD_SETROLE) {
352 role = nv_get_uint8(nvin, "role");
353 switch (role) {
354 case HAST_ROLE_INIT:
355 case HAST_ROLE_PRIMARY:
356 case HAST_ROLE_SECONDARY:
357 break;
358 default:
359 pjdlog_error("Invalid role received (%hhu).", role);
360 error = EHAST_INVALID;
361 goto fail;
362 }
363 }
364 if (strcmp(str, "all") == 0) {
365 struct hast_resource *res;
366
367 /* All configured resources. */
368
369 ii = 0;
370 TAILQ_FOREACH(res, &cfg->hc_resources, hr_next) {
371 switch (cmd) {
372 case HASTCTL_CMD_SETROLE:
373 control_set_role_common(cfg, nvout, role, res,
374 res->hr_name, ii++);
375 break;
376 case HASTCTL_CMD_STATUS:
377 control_status(cfg, nvout, res, res->hr_name,
378 ii++);
379 break;
380 default:
381 pjdlog_error("Invalid command received (%hhu).",
382 cmd);
383 error = EHAST_UNIMPLEMENTED;
384 goto fail;
385 }
386 }
387 } else {
388 /* Only selected resources. */
389
390 for (ii = 0; ; ii++) {
391 str = nv_get_string(nvin, "resource%u", ii);
392 if (str == NULL)
393 break;
394 switch (cmd) {
395 case HASTCTL_CMD_SETROLE:
396 control_set_role_common(cfg, nvout, role, NULL,
397 str, ii);
398 break;
399 case HASTCTL_CMD_STATUS:
400 control_status(cfg, nvout, NULL, str, ii);
401 break;
402 default:
403 pjdlog_error("Invalid command received (%hhu).",
404 cmd);
405 error = EHAST_UNIMPLEMENTED;
406 goto fail;
407 }
408 }
409 }
410 if (nv_error(nvout) != 0)
411 goto close;
412 fail:
413 if (error != 0)
414 nv_add_int16(nvout, error, "error");
415
416 if (hast_proto_send(NULL, conn, nvout, NULL, 0) == -1)
417 pjdlog_errno(LOG_ERR, "Unable to send control response");
418 close:
419 if (nvin != NULL)
420 nv_free(nvin);
421 if (nvout != NULL)
422 nv_free(nvout);
423 proto_close(conn);
424 cfg->hc_controlin = NULL;
425 }
426
427 /*
428 * Thread handles control requests from the parent.
429 */
430 void *
ctrl_thread(void * arg)431 ctrl_thread(void *arg)
432 {
433 struct hast_resource *res = arg;
434 struct nv *nvin, *nvout;
435 uint8_t cmd;
436
437 for (;;) {
438 if (hast_proto_recv_hdr(res->hr_ctrl, &nvin) == -1) {
439 if (sigexit_received)
440 pthread_exit(NULL);
441 pjdlog_errno(LOG_ERR,
442 "Unable to receive control message");
443 kill(getpid(), SIGTERM);
444 pthread_exit(NULL);
445 }
446 cmd = nv_get_uint8(nvin, "cmd");
447 if (cmd == 0) {
448 pjdlog_error("Control message is missing 'cmd' field.");
449 nv_free(nvin);
450 continue;
451 }
452 nvout = nv_alloc();
453 switch (cmd) {
454 case CONTROL_STATUS:
455 if (res->hr_remotein != NULL &&
456 res->hr_remoteout != NULL) {
457 nv_add_string(nvout, "complete", "status");
458 } else {
459 nv_add_string(nvout, "degraded", "status");
460 }
461 nv_add_uint32(nvout, (uint32_t)res->hr_extentsize,
462 "extentsize");
463 if (res->hr_role == HAST_ROLE_PRIMARY) {
464 nv_add_uint32(nvout,
465 (uint32_t)res->hr_keepdirty, "keepdirty");
466 nv_add_uint64(nvout,
467 (uint64_t)(activemap_ndirty(res->hr_amp) *
468 res->hr_extentsize), "dirty");
469 } else {
470 nv_add_uint32(nvout, (uint32_t)0, "keepdirty");
471 nv_add_uint64(nvout, (uint64_t)0, "dirty");
472 }
473 nv_add_uint64(nvout, res->hr_stat_read, "stat_read");
474 nv_add_uint64(nvout, res->hr_stat_write, "stat_write");
475 nv_add_uint64(nvout, res->hr_stat_delete,
476 "stat_delete");
477 nv_add_uint64(nvout, res->hr_stat_flush, "stat_flush");
478 nv_add_uint64(nvout, res->hr_stat_activemap_update,
479 "stat_activemap_update");
480 nv_add_uint64(nvout, res->hr_stat_read_error,
481 "stat_read_error");
482 nv_add_uint64(nvout, res->hr_stat_write_error +
483 res->hr_stat_activemap_write_error,
484 "stat_write_error");
485 nv_add_uint64(nvout, res->hr_stat_delete_error,
486 "stat_delete_error");
487 nv_add_uint64(nvout, res->hr_stat_flush_error +
488 res->hr_stat_activemap_flush_error,
489 "stat_flush_error");
490 res->output_status_aux(nvout);
491 nv_add_int16(nvout, 0, "error");
492 break;
493 case CONTROL_RELOAD:
494 /*
495 * When parent receives SIGHUP and discovers that
496 * something related to us has changes, it sends reload
497 * message to us.
498 */
499 PJDLOG_ASSERT(res->hr_role == HAST_ROLE_PRIMARY);
500 primary_config_reload(res, nvin);
501 nv_add_int16(nvout, 0, "error");
502 break;
503 default:
504 nv_add_int16(nvout, EINVAL, "error");
505 break;
506 }
507 nv_free(nvin);
508 if (nv_error(nvout) != 0) {
509 pjdlog_error("Unable to create answer on control message.");
510 nv_free(nvout);
511 continue;
512 }
513 if (hast_proto_send(NULL, res->hr_ctrl, nvout, NULL, 0) == -1) {
514 pjdlog_errno(LOG_ERR,
515 "Unable to send reply to control message");
516 }
517 nv_free(nvout);
518 }
519 /* NOTREACHED */
520 return (NULL);
521 }
522