1 // SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB 2 /* 3 * Copyright (c) 2019 Mellanox Technologies. All rights reserved. 4 */ 5 #include <rdma/ib_verbs.h> 6 #include <rdma/rdma_counter.h> 7 8 #include "core_priv.h" 9 #include "restrack.h" 10 11 #define ALL_AUTO_MODE_MASKS (RDMA_COUNTER_MASK_QP_TYPE | RDMA_COUNTER_MASK_PID) 12 13 static int __counter_set_mode(struct rdma_port_counter *port_counter, 14 enum rdma_nl_counter_mode new_mode, 15 enum rdma_nl_counter_mask new_mask) 16 { 17 if (new_mode == RDMA_COUNTER_MODE_AUTO) { 18 if (new_mask & (~ALL_AUTO_MODE_MASKS)) 19 return -EINVAL; 20 if (port_counter->num_counters) 21 return -EBUSY; 22 } 23 24 port_counter->mode.mode = new_mode; 25 port_counter->mode.mask = new_mask; 26 return 0; 27 } 28 29 /* 30 * rdma_counter_set_auto_mode() - Turn on/off per-port auto mode 31 * 32 * @dev: Device to operate 33 * @port: Port to use 34 * @mask: Mask to configure 35 * @extack: Message to the user 36 * 37 * Return 0 on success. If counter mode wasn't changed then it is considered 38 * as success as well. 39 * Return -EBUSY when changing to auto mode while there are bounded counters. 40 * 41 */ 42 int rdma_counter_set_auto_mode(struct ib_device *dev, u32 port, 43 enum rdma_nl_counter_mask mask, 44 struct netlink_ext_ack *extack) 45 { 46 struct rdma_port_counter *port_counter; 47 enum rdma_nl_counter_mode mode; 48 int ret; 49 50 port_counter = &dev->port_data[port].port_counter; 51 if (!port_counter->hstats) 52 return -EOPNOTSUPP; 53 54 mutex_lock(&port_counter->lock); 55 if (mask) 56 mode = RDMA_COUNTER_MODE_AUTO; 57 else 58 mode = (port_counter->num_counters) ? RDMA_COUNTER_MODE_MANUAL : 59 RDMA_COUNTER_MODE_NONE; 60 61 if (port_counter->mode.mode == mode && 62 port_counter->mode.mask == mask) { 63 ret = 0; 64 goto out; 65 } 66 67 ret = __counter_set_mode(port_counter, mode, mask); 68 69 out: 70 mutex_unlock(&port_counter->lock); 71 if (ret == -EBUSY) 72 NL_SET_ERR_MSG( 73 extack, 74 "Modifying auto mode is not allowed when there is a bound QP"); 75 return ret; 76 } 77 78 static void auto_mode_init_counter(struct rdma_counter *counter, 79 const struct ib_qp *qp, 80 enum rdma_nl_counter_mask new_mask) 81 { 82 struct auto_mode_param *param = &counter->mode.param; 83 84 counter->mode.mode = RDMA_COUNTER_MODE_AUTO; 85 counter->mode.mask = new_mask; 86 87 if (new_mask & RDMA_COUNTER_MASK_QP_TYPE) 88 param->qp_type = qp->qp_type; 89 } 90 91 static int __rdma_counter_bind_qp(struct rdma_counter *counter, 92 struct ib_qp *qp) 93 { 94 int ret; 95 96 if (qp->counter) 97 return -EINVAL; 98 99 if (!qp->device->ops.counter_bind_qp) 100 return -EOPNOTSUPP; 101 102 mutex_lock(&counter->lock); 103 ret = qp->device->ops.counter_bind_qp(counter, qp); 104 mutex_unlock(&counter->lock); 105 106 return ret; 107 } 108 109 int rdma_counter_modify(struct ib_device *dev, u32 port, 110 unsigned int index, bool enable) 111 { 112 struct rdma_hw_stats *stats; 113 int ret = 0; 114 115 if (!dev->ops.modify_hw_stat) 116 return -EOPNOTSUPP; 117 118 stats = ib_get_hw_stats_port(dev, port); 119 if (!stats || index >= stats->num_counters || 120 !(stats->descs[index].flags & IB_STAT_FLAG_OPTIONAL)) 121 return -EINVAL; 122 123 mutex_lock(&stats->lock); 124 125 if (enable != test_bit(index, stats->is_disabled)) 126 goto out; 127 128 ret = dev->ops.modify_hw_stat(dev, port, index, enable); 129 if (ret) 130 goto out; 131 132 if (enable) 133 clear_bit(index, stats->is_disabled); 134 else 135 set_bit(index, stats->is_disabled); 136 out: 137 mutex_unlock(&stats->lock); 138 return ret; 139 } 140 141 static struct rdma_counter *alloc_and_bind(struct ib_device *dev, u32 port, 142 struct ib_qp *qp, 143 enum rdma_nl_counter_mode mode) 144 { 145 struct rdma_port_counter *port_counter; 146 struct rdma_counter *counter; 147 int ret; 148 149 if (!dev->ops.counter_dealloc || !dev->ops.counter_alloc_stats) 150 return NULL; 151 152 counter = rdma_zalloc_drv_obj(dev, rdma_counter); 153 if (!counter) 154 return NULL; 155 156 counter->device = dev; 157 counter->port = port; 158 159 dev->ops.counter_init(counter); 160 161 rdma_restrack_new(&counter->res, RDMA_RESTRACK_COUNTER); 162 counter->stats = dev->ops.counter_alloc_stats(counter); 163 if (!counter->stats) 164 goto err_stats; 165 166 port_counter = &dev->port_data[port].port_counter; 167 mutex_lock(&port_counter->lock); 168 switch (mode) { 169 case RDMA_COUNTER_MODE_MANUAL: 170 ret = __counter_set_mode(port_counter, RDMA_COUNTER_MODE_MANUAL, 171 0); 172 if (ret) { 173 mutex_unlock(&port_counter->lock); 174 goto err_mode; 175 } 176 break; 177 case RDMA_COUNTER_MODE_AUTO: 178 auto_mode_init_counter(counter, qp, port_counter->mode.mask); 179 break; 180 default: 181 ret = -EOPNOTSUPP; 182 mutex_unlock(&port_counter->lock); 183 goto err_mode; 184 } 185 186 port_counter->num_counters++; 187 mutex_unlock(&port_counter->lock); 188 189 counter->mode.mode = mode; 190 kref_init(&counter->kref); 191 mutex_init(&counter->lock); 192 193 ret = __rdma_counter_bind_qp(counter, qp); 194 if (ret) 195 goto err_mode; 196 197 rdma_restrack_parent_name(&counter->res, &qp->res); 198 rdma_restrack_add(&counter->res); 199 return counter; 200 201 err_mode: 202 rdma_free_hw_stats_struct(counter->stats); 203 err_stats: 204 rdma_restrack_put(&counter->res); 205 kfree(counter); 206 return NULL; 207 } 208 209 static void rdma_counter_free(struct rdma_counter *counter) 210 { 211 struct rdma_port_counter *port_counter; 212 213 port_counter = &counter->device->port_data[counter->port].port_counter; 214 mutex_lock(&port_counter->lock); 215 port_counter->num_counters--; 216 if (!port_counter->num_counters && 217 (port_counter->mode.mode == RDMA_COUNTER_MODE_MANUAL)) 218 __counter_set_mode(port_counter, RDMA_COUNTER_MODE_NONE, 0); 219 220 mutex_unlock(&port_counter->lock); 221 222 rdma_restrack_del(&counter->res); 223 rdma_free_hw_stats_struct(counter->stats); 224 kfree(counter); 225 } 226 227 static bool auto_mode_match(struct ib_qp *qp, struct rdma_counter *counter, 228 enum rdma_nl_counter_mask auto_mask) 229 { 230 struct auto_mode_param *param = &counter->mode.param; 231 bool match = true; 232 233 if (auto_mask & RDMA_COUNTER_MASK_QP_TYPE) 234 match &= (param->qp_type == qp->qp_type); 235 236 if (auto_mask & RDMA_COUNTER_MASK_PID) 237 match &= (task_pid_nr(counter->res.task) == 238 task_pid_nr(qp->res.task)); 239 240 return match; 241 } 242 243 static int __rdma_counter_unbind_qp(struct ib_qp *qp) 244 { 245 struct rdma_counter *counter = qp->counter; 246 int ret; 247 248 if (!qp->device->ops.counter_unbind_qp) 249 return -EOPNOTSUPP; 250 251 mutex_lock(&counter->lock); 252 ret = qp->device->ops.counter_unbind_qp(qp); 253 mutex_unlock(&counter->lock); 254 255 return ret; 256 } 257 258 static void counter_history_stat_update(struct rdma_counter *counter) 259 { 260 struct ib_device *dev = counter->device; 261 struct rdma_port_counter *port_counter; 262 int i; 263 264 port_counter = &dev->port_data[counter->port].port_counter; 265 if (!port_counter->hstats) 266 return; 267 268 rdma_counter_query_stats(counter); 269 270 for (i = 0; i < counter->stats->num_counters; i++) 271 port_counter->hstats->value[i] += counter->stats->value[i]; 272 } 273 274 /* 275 * rdma_get_counter_auto_mode - Find the counter that @qp should be bound 276 * with in auto mode 277 * 278 * Return: The counter (with ref-count increased) if found 279 */ 280 static struct rdma_counter *rdma_get_counter_auto_mode(struct ib_qp *qp, 281 u32 port) 282 { 283 struct rdma_port_counter *port_counter; 284 struct rdma_counter *counter = NULL; 285 struct ib_device *dev = qp->device; 286 struct rdma_restrack_entry *res; 287 struct rdma_restrack_root *rt; 288 unsigned long id = 0; 289 290 port_counter = &dev->port_data[port].port_counter; 291 rt = &dev->res[RDMA_RESTRACK_COUNTER]; 292 xa_lock(&rt->xa); 293 xa_for_each(&rt->xa, id, res) { 294 counter = container_of(res, struct rdma_counter, res); 295 if ((counter->device != qp->device) || (counter->port != port)) 296 goto next; 297 298 if (auto_mode_match(qp, counter, port_counter->mode.mask)) 299 break; 300 next: 301 counter = NULL; 302 } 303 304 if (counter && !kref_get_unless_zero(&counter->kref)) 305 counter = NULL; 306 307 xa_unlock(&rt->xa); 308 return counter; 309 } 310 311 static void counter_release(struct kref *kref) 312 { 313 struct rdma_counter *counter; 314 315 counter = container_of(kref, struct rdma_counter, kref); 316 counter_history_stat_update(counter); 317 counter->device->ops.counter_dealloc(counter); 318 rdma_counter_free(counter); 319 } 320 321 /* 322 * rdma_counter_bind_qp_auto - Check and bind the QP to a counter base on 323 * the auto-mode rule 324 */ 325 int rdma_counter_bind_qp_auto(struct ib_qp *qp, u32 port) 326 { 327 struct rdma_port_counter *port_counter; 328 struct ib_device *dev = qp->device; 329 struct rdma_counter *counter; 330 int ret; 331 332 if (!rdma_restrack_is_tracked(&qp->res) || rdma_is_kernel_res(&qp->res)) 333 return 0; 334 335 if (!rdma_is_port_valid(dev, port)) 336 return -EINVAL; 337 338 port_counter = &dev->port_data[port].port_counter; 339 if (port_counter->mode.mode != RDMA_COUNTER_MODE_AUTO) 340 return 0; 341 342 counter = rdma_get_counter_auto_mode(qp, port); 343 if (counter) { 344 ret = __rdma_counter_bind_qp(counter, qp); 345 if (ret) { 346 kref_put(&counter->kref, counter_release); 347 return ret; 348 } 349 } else { 350 counter = alloc_and_bind(dev, port, qp, RDMA_COUNTER_MODE_AUTO); 351 if (!counter) 352 return -ENOMEM; 353 } 354 355 return 0; 356 } 357 358 /* 359 * rdma_counter_unbind_qp - Unbind a qp from a counter 360 * @force: 361 * true - Decrease the counter ref-count anyway (e.g., qp destroy) 362 */ 363 int rdma_counter_unbind_qp(struct ib_qp *qp, bool force) 364 { 365 struct rdma_counter *counter = qp->counter; 366 int ret; 367 368 if (!counter) 369 return -EINVAL; 370 371 ret = __rdma_counter_unbind_qp(qp); 372 if (ret && !force) 373 return ret; 374 375 kref_put(&counter->kref, counter_release); 376 return 0; 377 } 378 379 int rdma_counter_query_stats(struct rdma_counter *counter) 380 { 381 struct ib_device *dev = counter->device; 382 int ret; 383 384 if (!dev->ops.counter_update_stats) 385 return -EINVAL; 386 387 mutex_lock(&counter->lock); 388 ret = dev->ops.counter_update_stats(counter); 389 mutex_unlock(&counter->lock); 390 391 return ret; 392 } 393 394 static u64 get_running_counters_hwstat_sum(struct ib_device *dev, 395 u32 port, u32 index) 396 { 397 struct rdma_restrack_entry *res; 398 struct rdma_restrack_root *rt; 399 struct rdma_counter *counter; 400 unsigned long id = 0; 401 u64 sum = 0; 402 403 rt = &dev->res[RDMA_RESTRACK_COUNTER]; 404 xa_lock(&rt->xa); 405 xa_for_each(&rt->xa, id, res) { 406 if (!rdma_restrack_get(res)) 407 continue; 408 409 xa_unlock(&rt->xa); 410 411 counter = container_of(res, struct rdma_counter, res); 412 if ((counter->device != dev) || (counter->port != port) || 413 rdma_counter_query_stats(counter)) 414 goto next; 415 416 sum += counter->stats->value[index]; 417 418 next: 419 xa_lock(&rt->xa); 420 rdma_restrack_put(res); 421 } 422 423 xa_unlock(&rt->xa); 424 return sum; 425 } 426 427 /* 428 * rdma_counter_get_hwstat_value() - Get the sum value of all counters on a 429 * specific port, including the running ones and history data 430 */ 431 u64 rdma_counter_get_hwstat_value(struct ib_device *dev, u32 port, u32 index) 432 { 433 struct rdma_port_counter *port_counter; 434 u64 sum; 435 436 port_counter = &dev->port_data[port].port_counter; 437 if (!port_counter->hstats) 438 return 0; 439 440 sum = get_running_counters_hwstat_sum(dev, port, index); 441 sum += port_counter->hstats->value[index]; 442 443 return sum; 444 } 445 446 static struct ib_qp *rdma_counter_get_qp(struct ib_device *dev, u32 qp_num) 447 { 448 struct rdma_restrack_entry *res = NULL; 449 struct ib_qp *qp = NULL; 450 451 res = rdma_restrack_get_byid(dev, RDMA_RESTRACK_QP, qp_num); 452 if (IS_ERR(res)) 453 return NULL; 454 455 qp = container_of(res, struct ib_qp, res); 456 if (qp->qp_type == IB_QPT_RAW_PACKET && !capable(CAP_NET_RAW)) 457 goto err; 458 459 return qp; 460 461 err: 462 rdma_restrack_put(res); 463 return NULL; 464 } 465 466 static struct rdma_counter *rdma_get_counter_by_id(struct ib_device *dev, 467 u32 counter_id) 468 { 469 struct rdma_restrack_entry *res; 470 struct rdma_counter *counter; 471 472 res = rdma_restrack_get_byid(dev, RDMA_RESTRACK_COUNTER, counter_id); 473 if (IS_ERR(res)) 474 return NULL; 475 476 counter = container_of(res, struct rdma_counter, res); 477 kref_get(&counter->kref); 478 rdma_restrack_put(res); 479 480 return counter; 481 } 482 483 /* 484 * rdma_counter_bind_qpn() - Bind QP @qp_num to counter @counter_id 485 */ 486 int rdma_counter_bind_qpn(struct ib_device *dev, u32 port, 487 u32 qp_num, u32 counter_id) 488 { 489 struct rdma_port_counter *port_counter; 490 struct rdma_counter *counter; 491 struct ib_qp *qp; 492 int ret; 493 494 port_counter = &dev->port_data[port].port_counter; 495 if (port_counter->mode.mode == RDMA_COUNTER_MODE_AUTO) 496 return -EINVAL; 497 498 qp = rdma_counter_get_qp(dev, qp_num); 499 if (!qp) 500 return -ENOENT; 501 502 counter = rdma_get_counter_by_id(dev, counter_id); 503 if (!counter) { 504 ret = -ENOENT; 505 goto err; 506 } 507 508 if (rdma_is_kernel_res(&counter->res) != rdma_is_kernel_res(&qp->res)) { 509 ret = -EINVAL; 510 goto err_task; 511 } 512 513 if ((counter->device != qp->device) || (counter->port != qp->port)) { 514 ret = -EINVAL; 515 goto err_task; 516 } 517 518 ret = __rdma_counter_bind_qp(counter, qp); 519 if (ret) 520 goto err_task; 521 522 rdma_restrack_put(&qp->res); 523 return 0; 524 525 err_task: 526 kref_put(&counter->kref, counter_release); 527 err: 528 rdma_restrack_put(&qp->res); 529 return ret; 530 } 531 532 /* 533 * rdma_counter_bind_qpn_alloc() - Alloc a counter and bind QP @qp_num to it 534 * The id of new counter is returned in @counter_id 535 */ 536 int rdma_counter_bind_qpn_alloc(struct ib_device *dev, u32 port, 537 u32 qp_num, u32 *counter_id) 538 { 539 struct rdma_port_counter *port_counter; 540 struct rdma_counter *counter; 541 struct ib_qp *qp; 542 int ret; 543 544 if (!rdma_is_port_valid(dev, port)) 545 return -EINVAL; 546 547 port_counter = &dev->port_data[port].port_counter; 548 if (!port_counter->hstats) 549 return -EOPNOTSUPP; 550 551 if (port_counter->mode.mode == RDMA_COUNTER_MODE_AUTO) 552 return -EINVAL; 553 554 qp = rdma_counter_get_qp(dev, qp_num); 555 if (!qp) 556 return -ENOENT; 557 558 if (rdma_is_port_valid(dev, qp->port) && (qp->port != port)) { 559 ret = -EINVAL; 560 goto err; 561 } 562 563 counter = alloc_and_bind(dev, port, qp, RDMA_COUNTER_MODE_MANUAL); 564 if (!counter) { 565 ret = -ENOMEM; 566 goto err; 567 } 568 569 if (counter_id) 570 *counter_id = counter->id; 571 572 rdma_restrack_put(&qp->res); 573 return 0; 574 575 err: 576 rdma_restrack_put(&qp->res); 577 return ret; 578 } 579 580 /* 581 * rdma_counter_unbind_qpn() - Unbind QP @qp_num from a counter 582 */ 583 int rdma_counter_unbind_qpn(struct ib_device *dev, u32 port, 584 u32 qp_num, u32 counter_id) 585 { 586 struct rdma_port_counter *port_counter; 587 struct ib_qp *qp; 588 int ret; 589 590 if (!rdma_is_port_valid(dev, port)) 591 return -EINVAL; 592 593 qp = rdma_counter_get_qp(dev, qp_num); 594 if (!qp) 595 return -ENOENT; 596 597 if (rdma_is_port_valid(dev, qp->port) && (qp->port != port)) { 598 ret = -EINVAL; 599 goto out; 600 } 601 602 port_counter = &dev->port_data[port].port_counter; 603 if (!qp->counter || qp->counter->id != counter_id || 604 port_counter->mode.mode != RDMA_COUNTER_MODE_MANUAL) { 605 ret = -EINVAL; 606 goto out; 607 } 608 609 ret = rdma_counter_unbind_qp(qp, false); 610 611 out: 612 rdma_restrack_put(&qp->res); 613 return ret; 614 } 615 616 int rdma_counter_get_mode(struct ib_device *dev, u32 port, 617 enum rdma_nl_counter_mode *mode, 618 enum rdma_nl_counter_mask *mask) 619 { 620 struct rdma_port_counter *port_counter; 621 622 port_counter = &dev->port_data[port].port_counter; 623 *mode = port_counter->mode.mode; 624 *mask = port_counter->mode.mask; 625 626 return 0; 627 } 628 629 void rdma_counter_init(struct ib_device *dev) 630 { 631 struct rdma_port_counter *port_counter; 632 u32 port, i; 633 634 if (!dev->port_data) 635 return; 636 637 rdma_for_each_port(dev, port) { 638 port_counter = &dev->port_data[port].port_counter; 639 port_counter->mode.mode = RDMA_COUNTER_MODE_NONE; 640 mutex_init(&port_counter->lock); 641 642 if (!dev->ops.alloc_hw_port_stats) 643 continue; 644 645 port_counter->hstats = dev->ops.alloc_hw_port_stats(dev, port); 646 if (!port_counter->hstats) 647 goto fail; 648 } 649 650 return; 651 652 fail: 653 for (i = port; i >= rdma_start_port(dev); i--) { 654 port_counter = &dev->port_data[port].port_counter; 655 rdma_free_hw_stats_struct(port_counter->hstats); 656 port_counter->hstats = NULL; 657 mutex_destroy(&port_counter->lock); 658 } 659 } 660 661 void rdma_counter_release(struct ib_device *dev) 662 { 663 struct rdma_port_counter *port_counter; 664 u32 port; 665 666 rdma_for_each_port(dev, port) { 667 port_counter = &dev->port_data[port].port_counter; 668 rdma_free_hw_stats_struct(port_counter->hstats); 669 mutex_destroy(&port_counter->lock); 670 } 671 } 672