1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * Loopback bridge driver for the Greybus loopback module. 4 * 5 * Copyright 2014 Google Inc. 6 * Copyright 2014 Linaro Ltd. 7 */ 8 9 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt 10 11 #include <linux/kernel.h> 12 #include <linux/module.h> 13 #include <linux/mutex.h> 14 #include <linux/slab.h> 15 #include <linux/kthread.h> 16 #include <linux/delay.h> 17 #include <linux/random.h> 18 #include <linux/sizes.h> 19 #include <linux/cdev.h> 20 #include <linux/fs.h> 21 #include <linux/kfifo.h> 22 #include <linux/debugfs.h> 23 #include <linux/list_sort.h> 24 #include <linux/spinlock.h> 25 #include <linux/workqueue.h> 26 #include <linux/atomic.h> 27 #include <linux/pm_runtime.h> 28 29 #include <asm/div64.h> 30 31 #include "greybus.h" 32 #include "connection.h" 33 34 #define NSEC_PER_DAY 86400000000000ULL 35 36 struct gb_loopback_stats { 37 u32 min; 38 u32 max; 39 u64 sum; 40 u32 count; 41 }; 42 43 struct gb_loopback_device { 44 struct dentry *root; 45 u32 count; 46 size_t size_max; 47 48 /* We need to take a lock in atomic context */ 49 spinlock_t lock; 50 struct list_head list; 51 struct list_head list_op_async; 52 wait_queue_head_t wq; 53 }; 54 55 static struct gb_loopback_device gb_dev; 56 57 struct gb_loopback_async_operation { 58 struct gb_loopback *gb; 59 struct gb_operation *operation; 60 ktime_t ts; 61 int (*completion)(struct gb_loopback_async_operation *op_async); 62 }; 63 64 struct gb_loopback { 65 struct gb_connection *connection; 66 67 struct dentry *file; 68 struct kfifo kfifo_lat; 69 struct mutex mutex; 70 struct task_struct *task; 71 struct list_head entry; 72 struct device *dev; 73 wait_queue_head_t wq; 74 wait_queue_head_t wq_completion; 75 atomic_t outstanding_operations; 76 77 /* Per connection stats */ 78 ktime_t ts; 79 struct gb_loopback_stats latency; 80 struct gb_loopback_stats throughput; 81 struct gb_loopback_stats requests_per_second; 82 struct gb_loopback_stats apbridge_unipro_latency; 83 struct gb_loopback_stats gbphy_firmware_latency; 84 85 int type; 86 int async; 87 int id; 88 u32 size; 89 u32 iteration_max; 90 u32 iteration_count; 91 int us_wait; 92 u32 error; 93 u32 requests_completed; 94 u32 requests_timedout; 95 u32 timeout; 96 u32 jiffy_timeout; 97 u32 timeout_min; 98 u32 timeout_max; 99 u32 outstanding_operations_max; 100 u64 elapsed_nsecs; 101 u32 apbridge_latency_ts; 102 u32 gbphy_latency_ts; 103 104 u32 send_count; 105 }; 106 107 static struct class loopback_class = { 108 .name = "gb_loopback", 109 .owner = THIS_MODULE, 110 }; 111 static DEFINE_IDA(loopback_ida); 112 113 /* Min/max values in jiffies */ 114 #define GB_LOOPBACK_TIMEOUT_MIN 1 115 #define GB_LOOPBACK_TIMEOUT_MAX 10000 116 117 #define GB_LOOPBACK_FIFO_DEFAULT 8192 118 119 static unsigned int kfifo_depth = GB_LOOPBACK_FIFO_DEFAULT; 120 module_param(kfifo_depth, uint, 0444); 121 122 /* Maximum size of any one send data buffer we support */ 123 #define MAX_PACKET_SIZE (PAGE_SIZE * 2) 124 125 #define GB_LOOPBACK_US_WAIT_MAX 1000000 126 127 /* interface sysfs attributes */ 128 #define gb_loopback_ro_attr(field) \ 129 static ssize_t field##_show(struct device *dev, \ 130 struct device_attribute *attr, \ 131 char *buf) \ 132 { \ 133 struct gb_loopback *gb = dev_get_drvdata(dev); \ 134 return sprintf(buf, "%u\n", gb->field); \ 135 } \ 136 static DEVICE_ATTR_RO(field) 137 138 #define gb_loopback_ro_stats_attr(name, field, type) \ 139 static ssize_t name##_##field##_show(struct device *dev, \ 140 struct device_attribute *attr, \ 141 char *buf) \ 142 { \ 143 struct gb_loopback *gb = dev_get_drvdata(dev); \ 144 /* Report 0 for min and max if no transfer successed */ \ 145 if (!gb->requests_completed) \ 146 return sprintf(buf, "0\n"); \ 147 return sprintf(buf, "%"#type"\n", gb->name.field); \ 148 } \ 149 static DEVICE_ATTR_RO(name##_##field) 150 151 #define gb_loopback_ro_avg_attr(name) \ 152 static ssize_t name##_avg_show(struct device *dev, \ 153 struct device_attribute *attr, \ 154 char *buf) \ 155 { \ 156 struct gb_loopback_stats *stats; \ 157 struct gb_loopback *gb; \ 158 u64 avg, rem; \ 159 u32 count; \ 160 gb = dev_get_drvdata(dev); \ 161 stats = &gb->name; \ 162 count = stats->count ? stats->count : 1; \ 163 avg = stats->sum + count / 2000000; /* round closest */ \ 164 rem = do_div(avg, count); \ 165 rem *= 1000000; \ 166 do_div(rem, count); \ 167 return sprintf(buf, "%llu.%06u\n", avg, (u32)rem); \ 168 } \ 169 static DEVICE_ATTR_RO(name##_avg) 170 171 #define gb_loopback_stats_attrs(field) \ 172 gb_loopback_ro_stats_attr(field, min, u); \ 173 gb_loopback_ro_stats_attr(field, max, u); \ 174 gb_loopback_ro_avg_attr(field) 175 176 #define gb_loopback_attr(field, type) \ 177 static ssize_t field##_show(struct device *dev, \ 178 struct device_attribute *attr, \ 179 char *buf) \ 180 { \ 181 struct gb_loopback *gb = dev_get_drvdata(dev); \ 182 return sprintf(buf, "%"#type"\n", gb->field); \ 183 } \ 184 static ssize_t field##_store(struct device *dev, \ 185 struct device_attribute *attr, \ 186 const char *buf, \ 187 size_t len) \ 188 { \ 189 int ret; \ 190 struct gb_loopback *gb = dev_get_drvdata(dev); \ 191 mutex_lock(&gb->mutex); \ 192 ret = sscanf(buf, "%"#type, &gb->field); \ 193 if (ret != 1) \ 194 len = -EINVAL; \ 195 else \ 196 gb_loopback_check_attr(gb, bundle); \ 197 mutex_unlock(&gb->mutex); \ 198 return len; \ 199 } \ 200 static DEVICE_ATTR_RW(field) 201 202 #define gb_dev_loopback_ro_attr(field, conn) \ 203 static ssize_t field##_show(struct device *dev, \ 204 struct device_attribute *attr, \ 205 char *buf) \ 206 { \ 207 struct gb_loopback *gb = dev_get_drvdata(dev); \ 208 return sprintf(buf, "%u\n", gb->field); \ 209 } \ 210 static DEVICE_ATTR_RO(field) 211 212 #define gb_dev_loopback_rw_attr(field, type) \ 213 static ssize_t field##_show(struct device *dev, \ 214 struct device_attribute *attr, \ 215 char *buf) \ 216 { \ 217 struct gb_loopback *gb = dev_get_drvdata(dev); \ 218 return sprintf(buf, "%"#type"\n", gb->field); \ 219 } \ 220 static ssize_t field##_store(struct device *dev, \ 221 struct device_attribute *attr, \ 222 const char *buf, \ 223 size_t len) \ 224 { \ 225 int ret; \ 226 struct gb_loopback *gb = dev_get_drvdata(dev); \ 227 mutex_lock(&gb->mutex); \ 228 ret = sscanf(buf, "%"#type, &gb->field); \ 229 if (ret != 1) \ 230 len = -EINVAL; \ 231 else \ 232 gb_loopback_check_attr(gb); \ 233 mutex_unlock(&gb->mutex); \ 234 return len; \ 235 } \ 236 static DEVICE_ATTR_RW(field) 237 238 static void gb_loopback_reset_stats(struct gb_loopback *gb); 239 static void gb_loopback_check_attr(struct gb_loopback *gb) 240 { 241 if (gb->us_wait > GB_LOOPBACK_US_WAIT_MAX) 242 gb->us_wait = GB_LOOPBACK_US_WAIT_MAX; 243 if (gb->size > gb_dev.size_max) 244 gb->size = gb_dev.size_max; 245 gb->requests_timedout = 0; 246 gb->requests_completed = 0; 247 gb->iteration_count = 0; 248 gb->send_count = 0; 249 gb->error = 0; 250 251 if (kfifo_depth < gb->iteration_max) { 252 dev_warn(gb->dev, 253 "cannot log bytes %u kfifo_depth %u\n", 254 gb->iteration_max, kfifo_depth); 255 } 256 kfifo_reset_out(&gb->kfifo_lat); 257 258 switch (gb->type) { 259 case GB_LOOPBACK_TYPE_PING: 260 case GB_LOOPBACK_TYPE_TRANSFER: 261 case GB_LOOPBACK_TYPE_SINK: 262 gb->jiffy_timeout = usecs_to_jiffies(gb->timeout); 263 if (!gb->jiffy_timeout) 264 gb->jiffy_timeout = GB_LOOPBACK_TIMEOUT_MIN; 265 else if (gb->jiffy_timeout > GB_LOOPBACK_TIMEOUT_MAX) 266 gb->jiffy_timeout = GB_LOOPBACK_TIMEOUT_MAX; 267 gb_loopback_reset_stats(gb); 268 wake_up(&gb->wq); 269 break; 270 default: 271 gb->type = 0; 272 break; 273 } 274 } 275 276 /* Time to send and receive one message */ 277 gb_loopback_stats_attrs(latency); 278 /* Number of requests sent per second on this cport */ 279 gb_loopback_stats_attrs(requests_per_second); 280 /* Quantity of data sent and received on this cport */ 281 gb_loopback_stats_attrs(throughput); 282 /* Latency across the UniPro link from APBridge's perspective */ 283 gb_loopback_stats_attrs(apbridge_unipro_latency); 284 /* Firmware induced overhead in the GPBridge */ 285 gb_loopback_stats_attrs(gbphy_firmware_latency); 286 287 /* Number of errors encountered during loop */ 288 gb_loopback_ro_attr(error); 289 /* Number of requests successfully completed async */ 290 gb_loopback_ro_attr(requests_completed); 291 /* Number of requests timed out async */ 292 gb_loopback_ro_attr(requests_timedout); 293 /* Timeout minimum in useconds */ 294 gb_loopback_ro_attr(timeout_min); 295 /* Timeout minimum in useconds */ 296 gb_loopback_ro_attr(timeout_max); 297 298 /* 299 * Type of loopback message to send based on protocol type definitions 300 * 0 => Don't send message 301 * 2 => Send ping message continuously (message without payload) 302 * 3 => Send transfer message continuously (message with payload, 303 * payload returned in response) 304 * 4 => Send a sink message (message with payload, no payload in response) 305 */ 306 gb_dev_loopback_rw_attr(type, d); 307 /* Size of transfer message payload: 0-4096 bytes */ 308 gb_dev_loopback_rw_attr(size, u); 309 /* Time to wait between two messages: 0-1000 ms */ 310 gb_dev_loopback_rw_attr(us_wait, d); 311 /* Maximum iterations for a given operation: 1-(2^32-1), 0 implies infinite */ 312 gb_dev_loopback_rw_attr(iteration_max, u); 313 /* The current index of the for (i = 0; i < iteration_max; i++) loop */ 314 gb_dev_loopback_ro_attr(iteration_count, false); 315 /* A flag to indicate synchronous or asynchronous operations */ 316 gb_dev_loopback_rw_attr(async, u); 317 /* Timeout of an individual asynchronous request */ 318 gb_dev_loopback_rw_attr(timeout, u); 319 /* Maximum number of in-flight operations before back-off */ 320 gb_dev_loopback_rw_attr(outstanding_operations_max, u); 321 322 static struct attribute *loopback_attrs[] = { 323 &dev_attr_latency_min.attr, 324 &dev_attr_latency_max.attr, 325 &dev_attr_latency_avg.attr, 326 &dev_attr_requests_per_second_min.attr, 327 &dev_attr_requests_per_second_max.attr, 328 &dev_attr_requests_per_second_avg.attr, 329 &dev_attr_throughput_min.attr, 330 &dev_attr_throughput_max.attr, 331 &dev_attr_throughput_avg.attr, 332 &dev_attr_apbridge_unipro_latency_min.attr, 333 &dev_attr_apbridge_unipro_latency_max.attr, 334 &dev_attr_apbridge_unipro_latency_avg.attr, 335 &dev_attr_gbphy_firmware_latency_min.attr, 336 &dev_attr_gbphy_firmware_latency_max.attr, 337 &dev_attr_gbphy_firmware_latency_avg.attr, 338 &dev_attr_type.attr, 339 &dev_attr_size.attr, 340 &dev_attr_us_wait.attr, 341 &dev_attr_iteration_count.attr, 342 &dev_attr_iteration_max.attr, 343 &dev_attr_async.attr, 344 &dev_attr_error.attr, 345 &dev_attr_requests_completed.attr, 346 &dev_attr_requests_timedout.attr, 347 &dev_attr_timeout.attr, 348 &dev_attr_outstanding_operations_max.attr, 349 &dev_attr_timeout_min.attr, 350 &dev_attr_timeout_max.attr, 351 NULL, 352 }; 353 ATTRIBUTE_GROUPS(loopback); 354 355 static void gb_loopback_calculate_stats(struct gb_loopback *gb, bool error); 356 357 static u32 gb_loopback_nsec_to_usec_latency(u64 elapsed_nsecs) 358 { 359 do_div(elapsed_nsecs, NSEC_PER_USEC); 360 return elapsed_nsecs; 361 } 362 363 static u64 __gb_loopback_calc_latency(u64 t1, u64 t2) 364 { 365 if (t2 > t1) 366 return t2 - t1; 367 else 368 return NSEC_PER_DAY - t2 + t1; 369 } 370 371 static u64 gb_loopback_calc_latency(ktime_t ts, ktime_t te) 372 { 373 return __gb_loopback_calc_latency(ktime_to_ns(ts), ktime_to_ns(te)); 374 } 375 376 static int gb_loopback_operation_sync(struct gb_loopback *gb, int type, 377 void *request, int request_size, 378 void *response, int response_size) 379 { 380 struct gb_operation *operation; 381 ktime_t ts, te; 382 int ret; 383 384 ts = ktime_get(); 385 operation = gb_operation_create(gb->connection, type, request_size, 386 response_size, GFP_KERNEL); 387 if (!operation) 388 return -ENOMEM; 389 390 if (request_size) 391 memcpy(operation->request->payload, request, request_size); 392 393 ret = gb_operation_request_send_sync(operation); 394 if (ret) { 395 dev_err(&gb->connection->bundle->dev, 396 "synchronous operation failed: %d\n", ret); 397 goto out_put_operation; 398 } else { 399 if (response_size == operation->response->payload_size) { 400 memcpy(response, operation->response->payload, 401 response_size); 402 } else { 403 dev_err(&gb->connection->bundle->dev, 404 "response size %zu expected %d\n", 405 operation->response->payload_size, 406 response_size); 407 ret = -EINVAL; 408 goto out_put_operation; 409 } 410 } 411 412 te = ktime_get(); 413 414 /* Calculate the total time the message took */ 415 gb->elapsed_nsecs = gb_loopback_calc_latency(ts, te); 416 417 out_put_operation: 418 gb_operation_put(operation); 419 420 return ret; 421 } 422 423 static void gb_loopback_async_wait_all(struct gb_loopback *gb) 424 { 425 wait_event(gb->wq_completion, 426 !atomic_read(&gb->outstanding_operations)); 427 } 428 429 static void gb_loopback_async_operation_callback(struct gb_operation *operation) 430 { 431 struct gb_loopback_async_operation *op_async; 432 struct gb_loopback *gb; 433 ktime_t te; 434 int result; 435 436 te = ktime_get(); 437 result = gb_operation_result(operation); 438 op_async = gb_operation_get_data(operation); 439 gb = op_async->gb; 440 441 mutex_lock(&gb->mutex); 442 443 if (!result && op_async->completion) 444 result = op_async->completion(op_async); 445 446 if (!result) { 447 gb->elapsed_nsecs = gb_loopback_calc_latency(op_async->ts, te); 448 } else { 449 gb->error++; 450 if (result == -ETIMEDOUT) 451 gb->requests_timedout++; 452 } 453 454 gb->iteration_count++; 455 gb_loopback_calculate_stats(gb, result); 456 457 mutex_unlock(&gb->mutex); 458 459 dev_dbg(&gb->connection->bundle->dev, "complete operation %d\n", 460 operation->id); 461 462 /* Wake up waiters */ 463 atomic_dec(&op_async->gb->outstanding_operations); 464 wake_up(&gb->wq_completion); 465 466 /* Release resources */ 467 gb_operation_put(operation); 468 kfree(op_async); 469 } 470 471 static int gb_loopback_async_operation(struct gb_loopback *gb, int type, 472 void *request, int request_size, 473 int response_size, 474 void *completion) 475 { 476 struct gb_loopback_async_operation *op_async; 477 struct gb_operation *operation; 478 int ret; 479 480 op_async = kzalloc(sizeof(*op_async), GFP_KERNEL); 481 if (!op_async) 482 return -ENOMEM; 483 484 operation = gb_operation_create(gb->connection, type, request_size, 485 response_size, GFP_KERNEL); 486 if (!operation) { 487 kfree(op_async); 488 return -ENOMEM; 489 } 490 491 if (request_size) 492 memcpy(operation->request->payload, request, request_size); 493 494 gb_operation_set_data(operation, op_async); 495 496 op_async->gb = gb; 497 op_async->operation = operation; 498 op_async->completion = completion; 499 500 op_async->ts = ktime_get(); 501 502 atomic_inc(&gb->outstanding_operations); 503 ret = gb_operation_request_send(operation, 504 gb_loopback_async_operation_callback, 505 jiffies_to_msecs(gb->jiffy_timeout), 506 GFP_KERNEL); 507 if (ret) { 508 atomic_dec(&gb->outstanding_operations); 509 gb_operation_put(operation); 510 kfree(op_async); 511 } 512 return ret; 513 } 514 515 static int gb_loopback_sync_sink(struct gb_loopback *gb, u32 len) 516 { 517 struct gb_loopback_transfer_request *request; 518 int retval; 519 520 request = kmalloc(len + sizeof(*request), GFP_KERNEL); 521 if (!request) 522 return -ENOMEM; 523 524 request->len = cpu_to_le32(len); 525 retval = gb_loopback_operation_sync(gb, GB_LOOPBACK_TYPE_SINK, 526 request, len + sizeof(*request), 527 NULL, 0); 528 kfree(request); 529 return retval; 530 } 531 532 static int gb_loopback_sync_transfer(struct gb_loopback *gb, u32 len) 533 { 534 struct gb_loopback_transfer_request *request; 535 struct gb_loopback_transfer_response *response; 536 int retval; 537 538 gb->apbridge_latency_ts = 0; 539 gb->gbphy_latency_ts = 0; 540 541 request = kmalloc(len + sizeof(*request), GFP_KERNEL); 542 if (!request) 543 return -ENOMEM; 544 response = kmalloc(len + sizeof(*response), GFP_KERNEL); 545 if (!response) { 546 kfree(request); 547 return -ENOMEM; 548 } 549 550 memset(request->data, 0x5A, len); 551 552 request->len = cpu_to_le32(len); 553 retval = gb_loopback_operation_sync(gb, GB_LOOPBACK_TYPE_TRANSFER, 554 request, len + sizeof(*request), 555 response, len + sizeof(*response)); 556 if (retval) 557 goto gb_error; 558 559 if (memcmp(request->data, response->data, len)) { 560 dev_err(&gb->connection->bundle->dev, 561 "Loopback Data doesn't match\n"); 562 retval = -EREMOTEIO; 563 } 564 gb->apbridge_latency_ts = (u32)__le32_to_cpu(response->reserved0); 565 gb->gbphy_latency_ts = (u32)__le32_to_cpu(response->reserved1); 566 567 gb_error: 568 kfree(request); 569 kfree(response); 570 571 return retval; 572 } 573 574 static int gb_loopback_sync_ping(struct gb_loopback *gb) 575 { 576 return gb_loopback_operation_sync(gb, GB_LOOPBACK_TYPE_PING, 577 NULL, 0, NULL, 0); 578 } 579 580 static int gb_loopback_async_sink(struct gb_loopback *gb, u32 len) 581 { 582 struct gb_loopback_transfer_request *request; 583 int retval; 584 585 request = kmalloc(len + sizeof(*request), GFP_KERNEL); 586 if (!request) 587 return -ENOMEM; 588 589 request->len = cpu_to_le32(len); 590 retval = gb_loopback_async_operation(gb, GB_LOOPBACK_TYPE_SINK, 591 request, len + sizeof(*request), 592 0, NULL); 593 kfree(request); 594 return retval; 595 } 596 597 static int gb_loopback_async_transfer_complete( 598 struct gb_loopback_async_operation *op_async) 599 { 600 struct gb_loopback *gb; 601 struct gb_operation *operation; 602 struct gb_loopback_transfer_request *request; 603 struct gb_loopback_transfer_response *response; 604 size_t len; 605 int retval = 0; 606 607 gb = op_async->gb; 608 operation = op_async->operation; 609 request = operation->request->payload; 610 response = operation->response->payload; 611 len = le32_to_cpu(request->len); 612 613 if (memcmp(request->data, response->data, len)) { 614 dev_err(&gb->connection->bundle->dev, 615 "Loopback Data doesn't match operation id %d\n", 616 operation->id); 617 retval = -EREMOTEIO; 618 } else { 619 gb->apbridge_latency_ts = 620 (u32)__le32_to_cpu(response->reserved0); 621 gb->gbphy_latency_ts = 622 (u32)__le32_to_cpu(response->reserved1); 623 } 624 625 return retval; 626 } 627 628 static int gb_loopback_async_transfer(struct gb_loopback *gb, u32 len) 629 { 630 struct gb_loopback_transfer_request *request; 631 int retval, response_len; 632 633 request = kmalloc(len + sizeof(*request), GFP_KERNEL); 634 if (!request) 635 return -ENOMEM; 636 637 memset(request->data, 0x5A, len); 638 639 request->len = cpu_to_le32(len); 640 response_len = sizeof(struct gb_loopback_transfer_response); 641 retval = gb_loopback_async_operation(gb, GB_LOOPBACK_TYPE_TRANSFER, 642 request, len + sizeof(*request), 643 len + response_len, 644 gb_loopback_async_transfer_complete); 645 if (retval) 646 goto gb_error; 647 648 gb_error: 649 kfree(request); 650 return retval; 651 } 652 653 static int gb_loopback_async_ping(struct gb_loopback *gb) 654 { 655 return gb_loopback_async_operation(gb, GB_LOOPBACK_TYPE_PING, 656 NULL, 0, 0, NULL); 657 } 658 659 static int gb_loopback_request_handler(struct gb_operation *operation) 660 { 661 struct gb_connection *connection = operation->connection; 662 struct gb_loopback_transfer_request *request; 663 struct gb_loopback_transfer_response *response; 664 struct device *dev = &connection->bundle->dev; 665 size_t len; 666 667 /* By convention, the AP initiates the version operation */ 668 switch (operation->type) { 669 case GB_LOOPBACK_TYPE_PING: 670 case GB_LOOPBACK_TYPE_SINK: 671 return 0; 672 case GB_LOOPBACK_TYPE_TRANSFER: 673 if (operation->request->payload_size < sizeof(*request)) { 674 dev_err(dev, "transfer request too small (%zu < %zu)\n", 675 operation->request->payload_size, 676 sizeof(*request)); 677 return -EINVAL; /* -EMSGSIZE */ 678 } 679 request = operation->request->payload; 680 len = le32_to_cpu(request->len); 681 if (len > gb_dev.size_max) { 682 dev_err(dev, "transfer request too large (%zu > %zu)\n", 683 len, gb_dev.size_max); 684 return -EINVAL; 685 } 686 687 if (!gb_operation_response_alloc(operation, 688 len + sizeof(*response), GFP_KERNEL)) { 689 dev_err(dev, "error allocating response\n"); 690 return -ENOMEM; 691 } 692 response = operation->response->payload; 693 response->len = cpu_to_le32(len); 694 if (len) 695 memcpy(response->data, request->data, len); 696 697 return 0; 698 default: 699 dev_err(dev, "unsupported request: %u\n", operation->type); 700 return -EINVAL; 701 } 702 } 703 704 static void gb_loopback_reset_stats(struct gb_loopback *gb) 705 { 706 struct gb_loopback_stats reset = { 707 .min = U32_MAX, 708 }; 709 710 /* Reset per-connection stats */ 711 memcpy(&gb->latency, &reset, 712 sizeof(struct gb_loopback_stats)); 713 memcpy(&gb->throughput, &reset, 714 sizeof(struct gb_loopback_stats)); 715 memcpy(&gb->requests_per_second, &reset, 716 sizeof(struct gb_loopback_stats)); 717 memcpy(&gb->apbridge_unipro_latency, &reset, 718 sizeof(struct gb_loopback_stats)); 719 memcpy(&gb->gbphy_firmware_latency, &reset, 720 sizeof(struct gb_loopback_stats)); 721 722 /* Should be initialized at least once per transaction set */ 723 gb->apbridge_latency_ts = 0; 724 gb->gbphy_latency_ts = 0; 725 gb->ts = ktime_set(0, 0); 726 } 727 728 static void gb_loopback_update_stats(struct gb_loopback_stats *stats, u32 val) 729 { 730 if (stats->min > val) 731 stats->min = val; 732 if (stats->max < val) 733 stats->max = val; 734 stats->sum += val; 735 stats->count++; 736 } 737 738 static void gb_loopback_update_stats_window(struct gb_loopback_stats *stats, 739 u64 val, u32 count) 740 { 741 stats->sum += val; 742 stats->count += count; 743 744 do_div(val, count); 745 if (stats->min > val) 746 stats->min = val; 747 if (stats->max < val) 748 stats->max = val; 749 } 750 751 static void gb_loopback_requests_update(struct gb_loopback *gb, u32 latency) 752 { 753 u64 req = gb->requests_completed * USEC_PER_SEC; 754 755 gb_loopback_update_stats_window(&gb->requests_per_second, req, latency); 756 } 757 758 static void gb_loopback_throughput_update(struct gb_loopback *gb, u32 latency) 759 { 760 u64 aggregate_size = sizeof(struct gb_operation_msg_hdr) * 2; 761 762 switch (gb->type) { 763 case GB_LOOPBACK_TYPE_PING: 764 break; 765 case GB_LOOPBACK_TYPE_SINK: 766 aggregate_size += sizeof(struct gb_loopback_transfer_request) + 767 gb->size; 768 break; 769 case GB_LOOPBACK_TYPE_TRANSFER: 770 aggregate_size += sizeof(struct gb_loopback_transfer_request) + 771 sizeof(struct gb_loopback_transfer_response) + 772 gb->size * 2; 773 break; 774 default: 775 return; 776 } 777 778 aggregate_size *= gb->requests_completed; 779 aggregate_size *= USEC_PER_SEC; 780 gb_loopback_update_stats_window(&gb->throughput, aggregate_size, 781 latency); 782 } 783 784 static void gb_loopback_calculate_latency_stats(struct gb_loopback *gb) 785 { 786 u32 lat; 787 788 /* Express latency in terms of microseconds */ 789 lat = gb_loopback_nsec_to_usec_latency(gb->elapsed_nsecs); 790 791 /* Log latency stastic */ 792 gb_loopback_update_stats(&gb->latency, lat); 793 794 /* Raw latency log on a per thread basis */ 795 kfifo_in(&gb->kfifo_lat, (unsigned char *)&lat, sizeof(lat)); 796 797 /* Log the firmware supplied latency values */ 798 gb_loopback_update_stats(&gb->apbridge_unipro_latency, 799 gb->apbridge_latency_ts); 800 gb_loopback_update_stats(&gb->gbphy_firmware_latency, 801 gb->gbphy_latency_ts); 802 } 803 804 static void gb_loopback_calculate_stats(struct gb_loopback *gb, bool error) 805 { 806 u64 nlat; 807 u32 lat; 808 ktime_t te; 809 810 if (!error) { 811 gb->requests_completed++; 812 gb_loopback_calculate_latency_stats(gb); 813 } 814 815 te = ktime_get(); 816 nlat = gb_loopback_calc_latency(gb->ts, te); 817 if (nlat >= NSEC_PER_SEC || gb->iteration_count == gb->iteration_max) { 818 lat = gb_loopback_nsec_to_usec_latency(nlat); 819 820 gb_loopback_throughput_update(gb, lat); 821 gb_loopback_requests_update(gb, lat); 822 823 if (gb->iteration_count != gb->iteration_max) { 824 gb->ts = te; 825 gb->requests_completed = 0; 826 } 827 } 828 } 829 830 static void gb_loopback_async_wait_to_send(struct gb_loopback *gb) 831 { 832 if (!(gb->async && gb->outstanding_operations_max)) 833 return; 834 wait_event_interruptible(gb->wq_completion, 835 (atomic_read(&gb->outstanding_operations) < 836 gb->outstanding_operations_max) || 837 kthread_should_stop()); 838 } 839 840 static int gb_loopback_fn(void *data) 841 { 842 int error = 0; 843 int us_wait = 0; 844 int type; 845 int ret; 846 u32 size; 847 848 struct gb_loopback *gb = data; 849 struct gb_bundle *bundle = gb->connection->bundle; 850 851 ret = gb_pm_runtime_get_sync(bundle); 852 if (ret) 853 return ret; 854 855 while (1) { 856 if (!gb->type) { 857 gb_pm_runtime_put_autosuspend(bundle); 858 wait_event_interruptible(gb->wq, gb->type || 859 kthread_should_stop()); 860 ret = gb_pm_runtime_get_sync(bundle); 861 if (ret) 862 return ret; 863 } 864 865 if (kthread_should_stop()) 866 break; 867 868 /* Limit the maximum number of in-flight async operations */ 869 gb_loopback_async_wait_to_send(gb); 870 if (kthread_should_stop()) 871 break; 872 873 mutex_lock(&gb->mutex); 874 875 /* Optionally terminate */ 876 if (gb->send_count == gb->iteration_max) { 877 mutex_unlock(&gb->mutex); 878 879 /* Wait for synchronous and asynchronus completion */ 880 gb_loopback_async_wait_all(gb); 881 882 /* Mark complete unless user-space has poked us */ 883 mutex_lock(&gb->mutex); 884 if (gb->iteration_count == gb->iteration_max) { 885 gb->type = 0; 886 gb->send_count = 0; 887 sysfs_notify(&gb->dev->kobj, NULL, 888 "iteration_count"); 889 dev_dbg(&bundle->dev, "load test complete\n"); 890 } else { 891 dev_dbg(&bundle->dev, 892 "continuing on with new test set\n"); 893 } 894 mutex_unlock(&gb->mutex); 895 continue; 896 } 897 size = gb->size; 898 us_wait = gb->us_wait; 899 type = gb->type; 900 if (ktime_to_ns(gb->ts) == 0) 901 gb->ts = ktime_get(); 902 903 /* Else operations to perform */ 904 if (gb->async) { 905 if (type == GB_LOOPBACK_TYPE_PING) 906 error = gb_loopback_async_ping(gb); 907 else if (type == GB_LOOPBACK_TYPE_TRANSFER) 908 error = gb_loopback_async_transfer(gb, size); 909 else if (type == GB_LOOPBACK_TYPE_SINK) 910 error = gb_loopback_async_sink(gb, size); 911 912 if (error) { 913 gb->error++; 914 gb->iteration_count++; 915 } 916 } else { 917 /* We are effectively single threaded here */ 918 if (type == GB_LOOPBACK_TYPE_PING) 919 error = gb_loopback_sync_ping(gb); 920 else if (type == GB_LOOPBACK_TYPE_TRANSFER) 921 error = gb_loopback_sync_transfer(gb, size); 922 else if (type == GB_LOOPBACK_TYPE_SINK) 923 error = gb_loopback_sync_sink(gb, size); 924 925 if (error) 926 gb->error++; 927 gb->iteration_count++; 928 gb_loopback_calculate_stats(gb, !!error); 929 } 930 gb->send_count++; 931 mutex_unlock(&gb->mutex); 932 933 if (us_wait) { 934 if (us_wait < 20000) 935 usleep_range(us_wait, us_wait + 100); 936 else 937 msleep(us_wait / 1000); 938 } 939 } 940 941 gb_pm_runtime_put_autosuspend(bundle); 942 943 return 0; 944 } 945 946 static int gb_loopback_dbgfs_latency_show_common(struct seq_file *s, 947 struct kfifo *kfifo, 948 struct mutex *mutex) 949 { 950 u32 latency; 951 int retval; 952 953 if (kfifo_len(kfifo) == 0) { 954 retval = -EAGAIN; 955 goto done; 956 } 957 958 mutex_lock(mutex); 959 retval = kfifo_out(kfifo, &latency, sizeof(latency)); 960 if (retval > 0) { 961 seq_printf(s, "%u", latency); 962 retval = 0; 963 } 964 mutex_unlock(mutex); 965 done: 966 return retval; 967 } 968 969 static int gb_loopback_dbgfs_latency_show(struct seq_file *s, void *unused) 970 { 971 struct gb_loopback *gb = s->private; 972 973 return gb_loopback_dbgfs_latency_show_common(s, &gb->kfifo_lat, 974 &gb->mutex); 975 } 976 977 static int gb_loopback_latency_open(struct inode *inode, struct file *file) 978 { 979 return single_open(file, gb_loopback_dbgfs_latency_show, 980 inode->i_private); 981 } 982 983 static const struct file_operations gb_loopback_debugfs_latency_ops = { 984 .open = gb_loopback_latency_open, 985 .read = seq_read, 986 .llseek = seq_lseek, 987 .release = single_release, 988 }; 989 990 static int gb_loopback_bus_id_compare(void *priv, struct list_head *lha, 991 struct list_head *lhb) 992 { 993 struct gb_loopback *a = list_entry(lha, struct gb_loopback, entry); 994 struct gb_loopback *b = list_entry(lhb, struct gb_loopback, entry); 995 struct gb_connection *ca = a->connection; 996 struct gb_connection *cb = b->connection; 997 998 if (ca->bundle->intf->interface_id < cb->bundle->intf->interface_id) 999 return -1; 1000 if (cb->bundle->intf->interface_id < ca->bundle->intf->interface_id) 1001 return 1; 1002 if (ca->bundle->id < cb->bundle->id) 1003 return -1; 1004 if (cb->bundle->id < ca->bundle->id) 1005 return 1; 1006 if (ca->intf_cport_id < cb->intf_cport_id) 1007 return -1; 1008 else if (cb->intf_cport_id < ca->intf_cport_id) 1009 return 1; 1010 1011 return 0; 1012 } 1013 1014 static void gb_loopback_insert_id(struct gb_loopback *gb) 1015 { 1016 /* perform an insertion sort */ 1017 list_add_tail(&gb->entry, &gb_dev.list); 1018 list_sort(NULL, &gb_dev.list, gb_loopback_bus_id_compare); 1019 } 1020 1021 #define DEBUGFS_NAMELEN 32 1022 1023 static int gb_loopback_probe(struct gb_bundle *bundle, 1024 const struct greybus_bundle_id *id) 1025 { 1026 struct greybus_descriptor_cport *cport_desc; 1027 struct gb_connection *connection; 1028 struct gb_loopback *gb; 1029 struct device *dev; 1030 int retval; 1031 char name[DEBUGFS_NAMELEN]; 1032 unsigned long flags; 1033 1034 if (bundle->num_cports != 1) 1035 return -ENODEV; 1036 1037 cport_desc = &bundle->cport_desc[0]; 1038 if (cport_desc->protocol_id != GREYBUS_PROTOCOL_LOOPBACK) 1039 return -ENODEV; 1040 1041 gb = kzalloc(sizeof(*gb), GFP_KERNEL); 1042 if (!gb) 1043 return -ENOMEM; 1044 1045 connection = gb_connection_create(bundle, le16_to_cpu(cport_desc->id), 1046 gb_loopback_request_handler); 1047 if (IS_ERR(connection)) { 1048 retval = PTR_ERR(connection); 1049 goto out_kzalloc; 1050 } 1051 1052 gb->connection = connection; 1053 greybus_set_drvdata(bundle, gb); 1054 1055 init_waitqueue_head(&gb->wq); 1056 init_waitqueue_head(&gb->wq_completion); 1057 atomic_set(&gb->outstanding_operations, 0); 1058 gb_loopback_reset_stats(gb); 1059 1060 /* Reported values to user-space for min/max timeouts */ 1061 gb->timeout_min = jiffies_to_usecs(GB_LOOPBACK_TIMEOUT_MIN); 1062 gb->timeout_max = jiffies_to_usecs(GB_LOOPBACK_TIMEOUT_MAX); 1063 1064 if (!gb_dev.count) { 1065 /* Calculate maximum payload */ 1066 gb_dev.size_max = gb_operation_get_payload_size_max(connection); 1067 if (gb_dev.size_max <= 1068 sizeof(struct gb_loopback_transfer_request)) { 1069 retval = -EINVAL; 1070 goto out_connection_destroy; 1071 } 1072 gb_dev.size_max -= sizeof(struct gb_loopback_transfer_request); 1073 } 1074 1075 /* Create per-connection sysfs and debugfs data-points */ 1076 snprintf(name, sizeof(name), "raw_latency_%s", 1077 dev_name(&connection->bundle->dev)); 1078 gb->file = debugfs_create_file(name, S_IFREG | 0444, gb_dev.root, gb, 1079 &gb_loopback_debugfs_latency_ops); 1080 1081 gb->id = ida_simple_get(&loopback_ida, 0, 0, GFP_KERNEL); 1082 if (gb->id < 0) { 1083 retval = gb->id; 1084 goto out_debugfs_remove; 1085 } 1086 1087 retval = gb_connection_enable(connection); 1088 if (retval) 1089 goto out_ida_remove; 1090 1091 dev = device_create_with_groups(&loopback_class, 1092 &connection->bundle->dev, 1093 MKDEV(0, 0), gb, loopback_groups, 1094 "gb_loopback%d", gb->id); 1095 if (IS_ERR(dev)) { 1096 retval = PTR_ERR(dev); 1097 goto out_connection_disable; 1098 } 1099 gb->dev = dev; 1100 1101 /* Allocate kfifo */ 1102 if (kfifo_alloc(&gb->kfifo_lat, kfifo_depth * sizeof(u32), 1103 GFP_KERNEL)) { 1104 retval = -ENOMEM; 1105 goto out_conn; 1106 } 1107 /* Fork worker thread */ 1108 mutex_init(&gb->mutex); 1109 gb->task = kthread_run(gb_loopback_fn, gb, "gb_loopback"); 1110 if (IS_ERR(gb->task)) { 1111 retval = PTR_ERR(gb->task); 1112 goto out_kfifo; 1113 } 1114 1115 spin_lock_irqsave(&gb_dev.lock, flags); 1116 gb_loopback_insert_id(gb); 1117 gb_dev.count++; 1118 spin_unlock_irqrestore(&gb_dev.lock, flags); 1119 1120 gb_connection_latency_tag_enable(connection); 1121 1122 gb_pm_runtime_put_autosuspend(bundle); 1123 1124 return 0; 1125 1126 out_kfifo: 1127 kfifo_free(&gb->kfifo_lat); 1128 out_conn: 1129 device_unregister(dev); 1130 out_connection_disable: 1131 gb_connection_disable(connection); 1132 out_ida_remove: 1133 ida_simple_remove(&loopback_ida, gb->id); 1134 out_debugfs_remove: 1135 debugfs_remove(gb->file); 1136 out_connection_destroy: 1137 gb_connection_destroy(connection); 1138 out_kzalloc: 1139 kfree(gb); 1140 1141 return retval; 1142 } 1143 1144 static void gb_loopback_disconnect(struct gb_bundle *bundle) 1145 { 1146 struct gb_loopback *gb = greybus_get_drvdata(bundle); 1147 unsigned long flags; 1148 int ret; 1149 1150 ret = gb_pm_runtime_get_sync(bundle); 1151 if (ret) 1152 gb_pm_runtime_get_noresume(bundle); 1153 1154 gb_connection_disable(gb->connection); 1155 1156 if (!IS_ERR_OR_NULL(gb->task)) 1157 kthread_stop(gb->task); 1158 1159 kfifo_free(&gb->kfifo_lat); 1160 gb_connection_latency_tag_disable(gb->connection); 1161 debugfs_remove(gb->file); 1162 1163 /* 1164 * FIXME: gb_loopback_async_wait_all() is redundant now, as connection 1165 * is disabled at the beginning and so we can't have any more 1166 * incoming/outgoing requests. 1167 */ 1168 gb_loopback_async_wait_all(gb); 1169 1170 spin_lock_irqsave(&gb_dev.lock, flags); 1171 gb_dev.count--; 1172 list_del(&gb->entry); 1173 spin_unlock_irqrestore(&gb_dev.lock, flags); 1174 1175 device_unregister(gb->dev); 1176 ida_simple_remove(&loopback_ida, gb->id); 1177 1178 gb_connection_destroy(gb->connection); 1179 kfree(gb); 1180 } 1181 1182 static const struct greybus_bundle_id gb_loopback_id_table[] = { 1183 { GREYBUS_DEVICE_CLASS(GREYBUS_CLASS_LOOPBACK) }, 1184 { } 1185 }; 1186 MODULE_DEVICE_TABLE(greybus, gb_loopback_id_table); 1187 1188 static struct greybus_driver gb_loopback_driver = { 1189 .name = "loopback", 1190 .probe = gb_loopback_probe, 1191 .disconnect = gb_loopback_disconnect, 1192 .id_table = gb_loopback_id_table, 1193 }; 1194 1195 static int loopback_init(void) 1196 { 1197 int retval; 1198 1199 INIT_LIST_HEAD(&gb_dev.list); 1200 INIT_LIST_HEAD(&gb_dev.list_op_async); 1201 spin_lock_init(&gb_dev.lock); 1202 gb_dev.root = debugfs_create_dir("gb_loopback", NULL); 1203 1204 retval = class_register(&loopback_class); 1205 if (retval) 1206 goto err; 1207 1208 retval = greybus_register(&gb_loopback_driver); 1209 if (retval) 1210 goto err_unregister; 1211 1212 return 0; 1213 1214 err_unregister: 1215 class_unregister(&loopback_class); 1216 err: 1217 debugfs_remove_recursive(gb_dev.root); 1218 return retval; 1219 } 1220 module_init(loopback_init); 1221 1222 static void __exit loopback_exit(void) 1223 { 1224 debugfs_remove_recursive(gb_dev.root); 1225 greybus_deregister(&gb_loopback_driver); 1226 class_unregister(&loopback_class); 1227 ida_destroy(&loopback_ida); 1228 } 1229 module_exit(loopback_exit); 1230 1231 MODULE_LICENSE("GPL v2"); 1232