1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * Copyright (c) 2016-2017, Linaro Ltd 4 */ 5 6 #include <linux/idr.h> 7 #include <linux/interrupt.h> 8 #include <linux/io.h> 9 #include <linux/list.h> 10 #include <linux/mfd/syscon.h> 11 #include <linux/module.h> 12 #include <linux/of.h> 13 #include <linux/of_address.h> 14 #include <linux/platform_device.h> 15 #include <linux/regmap.h> 16 #include <linux/rpmsg.h> 17 #include <linux/sizes.h> 18 #include <linux/slab.h> 19 #include <linux/wait.h> 20 #include <linux/workqueue.h> 21 #include <linux/mailbox_client.h> 22 23 #include "rpmsg_internal.h" 24 #include "qcom_glink_native.h" 25 26 #define GLINK_NAME_SIZE 32 27 #define GLINK_VERSION_1 1 28 29 #define RPM_GLINK_CID_MIN 1 30 #define RPM_GLINK_CID_MAX 65536 31 32 struct glink_msg { 33 __le16 cmd; 34 __le16 param1; 35 __le32 param2; 36 u8 data[]; 37 } __packed; 38 39 /** 40 * struct glink_defer_cmd - deferred incoming control message 41 * @node: list node 42 * @msg: message header 43 * @data: payload of the message 44 * 45 * Copy of a received control message, to be added to @rx_queue and processed 46 * by @rx_work of @qcom_glink. 47 */ 48 struct glink_defer_cmd { 49 struct list_head node; 50 51 struct glink_msg msg; 52 u8 data[]; 53 }; 54 55 /** 56 * struct glink_core_rx_intent - RX intent 57 * RX intent 58 * 59 * @data: pointer to the data (may be NULL for zero-copy) 60 * @id: remote or local intent ID 61 * @size: size of the original intent (do not modify) 62 * @reuse: To mark if the intent can be reused after first use 63 * @in_use: To mark if intent is already in use for the channel 64 * @offset: next write offset (initially 0) 65 * @node: list node 66 */ 67 struct glink_core_rx_intent { 68 void *data; 69 u32 id; 70 size_t size; 71 bool reuse; 72 bool in_use; 73 u32 offset; 74 75 struct list_head node; 76 }; 77 78 /** 79 * struct qcom_glink - driver context, relates to one remote subsystem 80 * @dev: reference to the associated struct device 81 * @rx_pipe: pipe object for receive FIFO 82 * @tx_pipe: pipe object for transmit FIFO 83 * @rx_work: worker for handling received control messages 84 * @rx_lock: protects the @rx_queue 85 * @rx_queue: queue of received control messages to be processed in @rx_work 86 * @tx_lock: synchronizes operations on the tx fifo 87 * @idr_lock: synchronizes @lcids and @rcids modifications 88 * @lcids: idr of all channels with a known local channel id 89 * @rcids: idr of all channels with a known remote channel id 90 * @features: remote features 91 * @intentless: flag to indicate that there is no intent 92 * @tx_avail_notify: Waitqueue for pending tx tasks 93 * @sent_read_notify: flag to check cmd sent or not 94 * @abort_tx: flag indicating that all tx attempts should fail 95 */ 96 struct qcom_glink { 97 struct device *dev; 98 99 struct qcom_glink_pipe *rx_pipe; 100 struct qcom_glink_pipe *tx_pipe; 101 102 struct work_struct rx_work; 103 spinlock_t rx_lock; 104 struct list_head rx_queue; 105 106 spinlock_t tx_lock; 107 108 spinlock_t idr_lock; 109 struct idr lcids; 110 struct idr rcids; 111 unsigned long features; 112 113 bool intentless; 114 wait_queue_head_t tx_avail_notify; 115 bool sent_read_notify; 116 117 bool abort_tx; 118 }; 119 120 enum { 121 GLINK_STATE_CLOSED, 122 GLINK_STATE_OPENING, 123 GLINK_STATE_OPEN, 124 GLINK_STATE_CLOSING, 125 }; 126 127 /** 128 * struct glink_channel - internal representation of a channel 129 * @rpdev: rpdev reference, only used for primary endpoints 130 * @ept: rpmsg endpoint this channel is associated with 131 * @glink: qcom_glink context handle 132 * @refcount: refcount for the channel object 133 * @recv_lock: guard for @ept.cb 134 * @name: unique channel name/identifier 135 * @lcid: channel id, in local space 136 * @rcid: channel id, in remote space 137 * @intent_lock: lock for protection of @liids, @riids 138 * @liids: idr of all local intents 139 * @riids: idr of all remote intents 140 * @intent_work: worker responsible for transmitting rx_done packets 141 * @done_intents: list of intents that needs to be announced rx_done 142 * @buf: receive buffer, for gathering fragments 143 * @buf_offset: write offset in @buf 144 * @buf_size: size of current @buf 145 * @open_ack: completed once remote has acked the open-request 146 * @open_req: completed once open-request has been received 147 * @intent_req_lock: Synchronises multiple intent requests 148 * @intent_req_result: Result of intent request 149 * @intent_req_wq: wait queue for intent_req signalling 150 */ 151 struct glink_channel { 152 struct rpmsg_endpoint ept; 153 154 struct rpmsg_device *rpdev; 155 struct qcom_glink *glink; 156 157 struct kref refcount; 158 159 spinlock_t recv_lock; 160 161 char *name; 162 unsigned int lcid; 163 unsigned int rcid; 164 165 spinlock_t intent_lock; 166 struct idr liids; 167 struct idr riids; 168 struct work_struct intent_work; 169 struct list_head done_intents; 170 171 struct glink_core_rx_intent *buf; 172 int buf_offset; 173 int buf_size; 174 175 struct completion open_ack; 176 struct completion open_req; 177 178 struct mutex intent_req_lock; 179 int intent_req_result; 180 wait_queue_head_t intent_req_wq; 181 }; 182 183 #define to_glink_channel(_ept) container_of(_ept, struct glink_channel, ept) 184 185 static const struct rpmsg_endpoint_ops glink_endpoint_ops; 186 187 #define GLINK_CMD_VERSION 0 188 #define GLINK_CMD_VERSION_ACK 1 189 #define GLINK_CMD_OPEN 2 190 #define GLINK_CMD_CLOSE 3 191 #define GLINK_CMD_OPEN_ACK 4 192 #define GLINK_CMD_INTENT 5 193 #define GLINK_CMD_RX_DONE 6 194 #define GLINK_CMD_RX_INTENT_REQ 7 195 #define GLINK_CMD_RX_INTENT_REQ_ACK 8 196 #define GLINK_CMD_TX_DATA 9 197 #define GLINK_CMD_CLOSE_ACK 11 198 #define GLINK_CMD_TX_DATA_CONT 12 199 #define GLINK_CMD_READ_NOTIF 13 200 #define GLINK_CMD_RX_DONE_W_REUSE 14 201 202 #define GLINK_FEATURE_INTENTLESS BIT(1) 203 204 static void qcom_glink_rx_done_work(struct work_struct *work); 205 206 static struct glink_channel *qcom_glink_alloc_channel(struct qcom_glink *glink, 207 const char *name) 208 { 209 struct glink_channel *channel; 210 211 channel = kzalloc(sizeof(*channel), GFP_KERNEL); 212 if (!channel) 213 return ERR_PTR(-ENOMEM); 214 215 /* Setup glink internal glink_channel data */ 216 spin_lock_init(&channel->recv_lock); 217 spin_lock_init(&channel->intent_lock); 218 mutex_init(&channel->intent_req_lock); 219 220 channel->glink = glink; 221 channel->name = kstrdup(name, GFP_KERNEL); 222 223 init_completion(&channel->open_req); 224 init_completion(&channel->open_ack); 225 init_waitqueue_head(&channel->intent_req_wq); 226 227 INIT_LIST_HEAD(&channel->done_intents); 228 INIT_WORK(&channel->intent_work, qcom_glink_rx_done_work); 229 230 idr_init(&channel->liids); 231 idr_init(&channel->riids); 232 kref_init(&channel->refcount); 233 234 return channel; 235 } 236 237 static void qcom_glink_channel_release(struct kref *ref) 238 { 239 struct glink_channel *channel = container_of(ref, struct glink_channel, 240 refcount); 241 struct glink_core_rx_intent *intent; 242 struct glink_core_rx_intent *tmp; 243 unsigned long flags; 244 int iid; 245 246 /* cancel pending rx_done work */ 247 cancel_work_sync(&channel->intent_work); 248 249 spin_lock_irqsave(&channel->intent_lock, flags); 250 /* Free all non-reuse intents pending rx_done work */ 251 list_for_each_entry_safe(intent, tmp, &channel->done_intents, node) { 252 if (!intent->reuse) { 253 kfree(intent->data); 254 kfree(intent); 255 } 256 } 257 258 idr_for_each_entry(&channel->liids, tmp, iid) { 259 kfree(tmp->data); 260 kfree(tmp); 261 } 262 idr_destroy(&channel->liids); 263 264 idr_for_each_entry(&channel->riids, tmp, iid) 265 kfree(tmp); 266 idr_destroy(&channel->riids); 267 spin_unlock_irqrestore(&channel->intent_lock, flags); 268 269 kfree(channel->name); 270 kfree(channel); 271 } 272 273 static size_t qcom_glink_rx_avail(struct qcom_glink *glink) 274 { 275 return glink->rx_pipe->avail(glink->rx_pipe); 276 } 277 278 static void qcom_glink_rx_peek(struct qcom_glink *glink, 279 void *data, unsigned int offset, size_t count) 280 { 281 glink->rx_pipe->peek(glink->rx_pipe, data, offset, count); 282 } 283 284 static void qcom_glink_rx_advance(struct qcom_glink *glink, size_t count) 285 { 286 glink->rx_pipe->advance(glink->rx_pipe, count); 287 } 288 289 static size_t qcom_glink_tx_avail(struct qcom_glink *glink) 290 { 291 return glink->tx_pipe->avail(glink->tx_pipe); 292 } 293 294 static void qcom_glink_tx_write(struct qcom_glink *glink, 295 const void *hdr, size_t hlen, 296 const void *data, size_t dlen) 297 { 298 glink->tx_pipe->write(glink->tx_pipe, hdr, hlen, data, dlen); 299 } 300 301 static void qcom_glink_tx_kick(struct qcom_glink *glink) 302 { 303 glink->tx_pipe->kick(glink->tx_pipe); 304 } 305 306 static void qcom_glink_send_read_notify(struct qcom_glink *glink) 307 { 308 struct glink_msg msg; 309 310 msg.cmd = cpu_to_le16(GLINK_CMD_READ_NOTIF); 311 msg.param1 = 0; 312 msg.param2 = 0; 313 314 qcom_glink_tx_write(glink, &msg, sizeof(msg), NULL, 0); 315 316 qcom_glink_tx_kick(glink); 317 } 318 319 static int qcom_glink_tx(struct qcom_glink *glink, 320 const void *hdr, size_t hlen, 321 const void *data, size_t dlen, bool wait) 322 { 323 unsigned int tlen = hlen + dlen; 324 unsigned long flags; 325 int ret = 0; 326 327 /* Reject packets that are too big */ 328 if (tlen >= glink->tx_pipe->length) 329 return -EINVAL; 330 331 spin_lock_irqsave(&glink->tx_lock, flags); 332 333 if (glink->abort_tx) { 334 ret = -EIO; 335 goto out; 336 } 337 338 while (qcom_glink_tx_avail(glink) < tlen) { 339 if (!wait) { 340 ret = -EAGAIN; 341 goto out; 342 } 343 344 if (glink->abort_tx) { 345 ret = -EIO; 346 goto out; 347 } 348 349 if (!glink->sent_read_notify) { 350 glink->sent_read_notify = true; 351 qcom_glink_send_read_notify(glink); 352 } 353 354 /* Wait without holding the tx_lock */ 355 spin_unlock_irqrestore(&glink->tx_lock, flags); 356 357 wait_event_timeout(glink->tx_avail_notify, 358 qcom_glink_tx_avail(glink) >= tlen, 10 * HZ); 359 360 spin_lock_irqsave(&glink->tx_lock, flags); 361 362 if (qcom_glink_tx_avail(glink) >= tlen) 363 glink->sent_read_notify = false; 364 } 365 366 qcom_glink_tx_write(glink, hdr, hlen, data, dlen); 367 qcom_glink_tx_kick(glink); 368 369 out: 370 spin_unlock_irqrestore(&glink->tx_lock, flags); 371 372 return ret; 373 } 374 375 static int qcom_glink_send_version(struct qcom_glink *glink) 376 { 377 struct glink_msg msg; 378 379 msg.cmd = cpu_to_le16(GLINK_CMD_VERSION); 380 msg.param1 = cpu_to_le16(GLINK_VERSION_1); 381 msg.param2 = cpu_to_le32(glink->features); 382 383 return qcom_glink_tx(glink, &msg, sizeof(msg), NULL, 0, true); 384 } 385 386 static void qcom_glink_send_version_ack(struct qcom_glink *glink) 387 { 388 struct glink_msg msg; 389 390 msg.cmd = cpu_to_le16(GLINK_CMD_VERSION_ACK); 391 msg.param1 = cpu_to_le16(GLINK_VERSION_1); 392 msg.param2 = cpu_to_le32(glink->features); 393 394 qcom_glink_tx(glink, &msg, sizeof(msg), NULL, 0, true); 395 } 396 397 static void qcom_glink_send_open_ack(struct qcom_glink *glink, 398 struct glink_channel *channel) 399 { 400 struct glink_msg msg; 401 402 msg.cmd = cpu_to_le16(GLINK_CMD_OPEN_ACK); 403 msg.param1 = cpu_to_le16(channel->rcid); 404 msg.param2 = cpu_to_le32(0); 405 406 qcom_glink_tx(glink, &msg, sizeof(msg), NULL, 0, true); 407 } 408 409 static void qcom_glink_handle_intent_req_ack(struct qcom_glink *glink, 410 unsigned int cid, bool granted) 411 { 412 struct glink_channel *channel; 413 unsigned long flags; 414 415 spin_lock_irqsave(&glink->idr_lock, flags); 416 channel = idr_find(&glink->rcids, cid); 417 spin_unlock_irqrestore(&glink->idr_lock, flags); 418 if (!channel) { 419 dev_err(glink->dev, "unable to find channel\n"); 420 return; 421 } 422 423 channel->intent_req_result = granted; 424 wake_up_all(&channel->intent_req_wq); 425 } 426 427 static void qcom_glink_intent_req_abort(struct glink_channel *channel) 428 { 429 channel->intent_req_result = 0; 430 wake_up_all(&channel->intent_req_wq); 431 } 432 433 /** 434 * qcom_glink_send_open_req() - send a GLINK_CMD_OPEN request to the remote 435 * @glink: Ptr to the glink edge 436 * @channel: Ptr to the channel that the open req is sent 437 * 438 * Allocates a local channel id and sends a GLINK_CMD_OPEN message to the remote. 439 * Will return with refcount held, regardless of outcome. 440 * 441 * Return: 0 on success, negative errno otherwise. 442 */ 443 static int qcom_glink_send_open_req(struct qcom_glink *glink, 444 struct glink_channel *channel) 445 { 446 struct { 447 struct glink_msg msg; 448 u8 name[GLINK_NAME_SIZE]; 449 } __packed req; 450 int name_len = strlen(channel->name) + 1; 451 int req_len = ALIGN(sizeof(req.msg) + name_len, 8); 452 int ret; 453 unsigned long flags; 454 455 kref_get(&channel->refcount); 456 457 spin_lock_irqsave(&glink->idr_lock, flags); 458 ret = idr_alloc_cyclic(&glink->lcids, channel, 459 RPM_GLINK_CID_MIN, RPM_GLINK_CID_MAX, 460 GFP_ATOMIC); 461 spin_unlock_irqrestore(&glink->idr_lock, flags); 462 if (ret < 0) 463 return ret; 464 465 channel->lcid = ret; 466 467 req.msg.cmd = cpu_to_le16(GLINK_CMD_OPEN); 468 req.msg.param1 = cpu_to_le16(channel->lcid); 469 req.msg.param2 = cpu_to_le32(name_len); 470 strcpy(req.name, channel->name); 471 472 ret = qcom_glink_tx(glink, &req, req_len, NULL, 0, true); 473 if (ret) 474 goto remove_idr; 475 476 return 0; 477 478 remove_idr: 479 spin_lock_irqsave(&glink->idr_lock, flags); 480 idr_remove(&glink->lcids, channel->lcid); 481 channel->lcid = 0; 482 spin_unlock_irqrestore(&glink->idr_lock, flags); 483 484 return ret; 485 } 486 487 static void qcom_glink_send_close_req(struct qcom_glink *glink, 488 struct glink_channel *channel) 489 { 490 struct glink_msg req; 491 492 req.cmd = cpu_to_le16(GLINK_CMD_CLOSE); 493 req.param1 = cpu_to_le16(channel->lcid); 494 req.param2 = 0; 495 496 qcom_glink_tx(glink, &req, sizeof(req), NULL, 0, true); 497 } 498 499 static void qcom_glink_send_close_ack(struct qcom_glink *glink, 500 unsigned int rcid) 501 { 502 struct glink_msg req; 503 504 req.cmd = cpu_to_le16(GLINK_CMD_CLOSE_ACK); 505 req.param1 = cpu_to_le16(rcid); 506 req.param2 = 0; 507 508 qcom_glink_tx(glink, &req, sizeof(req), NULL, 0, true); 509 } 510 511 static void qcom_glink_rx_done_work(struct work_struct *work) 512 { 513 struct glink_channel *channel = container_of(work, struct glink_channel, 514 intent_work); 515 struct qcom_glink *glink = channel->glink; 516 struct glink_core_rx_intent *intent, *tmp; 517 struct { 518 u16 id; 519 u16 lcid; 520 u32 liid; 521 } __packed cmd; 522 523 unsigned int cid = channel->lcid; 524 unsigned int iid; 525 bool reuse; 526 unsigned long flags; 527 528 spin_lock_irqsave(&channel->intent_lock, flags); 529 list_for_each_entry_safe(intent, tmp, &channel->done_intents, node) { 530 list_del(&intent->node); 531 spin_unlock_irqrestore(&channel->intent_lock, flags); 532 iid = intent->id; 533 reuse = intent->reuse; 534 535 cmd.id = reuse ? GLINK_CMD_RX_DONE_W_REUSE : GLINK_CMD_RX_DONE; 536 cmd.lcid = cid; 537 cmd.liid = iid; 538 539 qcom_glink_tx(glink, &cmd, sizeof(cmd), NULL, 0, true); 540 if (!reuse) { 541 kfree(intent->data); 542 kfree(intent); 543 } 544 spin_lock_irqsave(&channel->intent_lock, flags); 545 } 546 spin_unlock_irqrestore(&channel->intent_lock, flags); 547 } 548 549 static void qcom_glink_rx_done(struct qcom_glink *glink, 550 struct glink_channel *channel, 551 struct glink_core_rx_intent *intent) 552 { 553 /* We don't send RX_DONE to intentless systems */ 554 if (glink->intentless) { 555 kfree(intent->data); 556 kfree(intent); 557 return; 558 } 559 560 /* Take it off the tree of receive intents */ 561 if (!intent->reuse) { 562 spin_lock(&channel->intent_lock); 563 idr_remove(&channel->liids, intent->id); 564 spin_unlock(&channel->intent_lock); 565 } 566 567 /* Schedule the sending of a rx_done indication */ 568 spin_lock(&channel->intent_lock); 569 list_add_tail(&intent->node, &channel->done_intents); 570 spin_unlock(&channel->intent_lock); 571 572 schedule_work(&channel->intent_work); 573 } 574 575 /** 576 * qcom_glink_receive_version() - receive version/features from remote system 577 * 578 * @glink: pointer to transport interface 579 * @version: remote version 580 * @features: remote features 581 * 582 * This function is called in response to a remote-initiated version/feature 583 * negotiation sequence. 584 */ 585 static void qcom_glink_receive_version(struct qcom_glink *glink, 586 u32 version, 587 u32 features) 588 { 589 switch (version) { 590 case 0: 591 break; 592 case GLINK_VERSION_1: 593 glink->features &= features; 594 fallthrough; 595 default: 596 qcom_glink_send_version_ack(glink); 597 break; 598 } 599 } 600 601 /** 602 * qcom_glink_receive_version_ack() - receive negotiation ack from remote system 603 * 604 * @glink: pointer to transport interface 605 * @version: remote version response 606 * @features: remote features response 607 * 608 * This function is called in response to a local-initiated version/feature 609 * negotiation sequence and is the counter-offer from the remote side based 610 * upon the initial version and feature set requested. 611 */ 612 static void qcom_glink_receive_version_ack(struct qcom_glink *glink, 613 u32 version, 614 u32 features) 615 { 616 switch (version) { 617 case 0: 618 /* Version negotiation failed */ 619 break; 620 case GLINK_VERSION_1: 621 if (features == glink->features) 622 break; 623 624 glink->features &= features; 625 fallthrough; 626 default: 627 qcom_glink_send_version(glink); 628 break; 629 } 630 } 631 632 /** 633 * qcom_glink_send_intent_req_ack() - convert an rx intent request ack cmd to 634 * wire format and transmit 635 * @glink: The transport to transmit on. 636 * @channel: The glink channel 637 * @granted: The request response to encode. 638 * 639 * Return: 0 on success or standard Linux error code. 640 */ 641 static int qcom_glink_send_intent_req_ack(struct qcom_glink *glink, 642 struct glink_channel *channel, 643 bool granted) 644 { 645 struct glink_msg msg; 646 647 msg.cmd = cpu_to_le16(GLINK_CMD_RX_INTENT_REQ_ACK); 648 msg.param1 = cpu_to_le16(channel->lcid); 649 msg.param2 = cpu_to_le32(granted); 650 651 qcom_glink_tx(glink, &msg, sizeof(msg), NULL, 0, true); 652 653 return 0; 654 } 655 656 /** 657 * qcom_glink_advertise_intent - convert an rx intent cmd to wire format and 658 * transmit 659 * @glink: The transport to transmit on. 660 * @channel: The local channel 661 * @intent: The intent to pass on to remote. 662 * 663 * Return: 0 on success or standard Linux error code. 664 */ 665 static int qcom_glink_advertise_intent(struct qcom_glink *glink, 666 struct glink_channel *channel, 667 struct glink_core_rx_intent *intent) 668 { 669 struct command { 670 __le16 id; 671 __le16 lcid; 672 __le32 count; 673 __le32 size; 674 __le32 liid; 675 } __packed; 676 struct command cmd; 677 678 cmd.id = cpu_to_le16(GLINK_CMD_INTENT); 679 cmd.lcid = cpu_to_le16(channel->lcid); 680 cmd.count = cpu_to_le32(1); 681 cmd.size = cpu_to_le32(intent->size); 682 cmd.liid = cpu_to_le32(intent->id); 683 684 qcom_glink_tx(glink, &cmd, sizeof(cmd), NULL, 0, true); 685 686 return 0; 687 } 688 689 static struct glink_core_rx_intent * 690 qcom_glink_alloc_intent(struct qcom_glink *glink, 691 struct glink_channel *channel, 692 size_t size, 693 bool reuseable) 694 { 695 struct glink_core_rx_intent *intent; 696 int ret; 697 unsigned long flags; 698 699 intent = kzalloc(sizeof(*intent), GFP_KERNEL); 700 if (!intent) 701 return NULL; 702 703 intent->data = kzalloc(size, GFP_KERNEL); 704 if (!intent->data) 705 goto free_intent; 706 707 spin_lock_irqsave(&channel->intent_lock, flags); 708 ret = idr_alloc_cyclic(&channel->liids, intent, 1, -1, GFP_ATOMIC); 709 if (ret < 0) { 710 spin_unlock_irqrestore(&channel->intent_lock, flags); 711 goto free_data; 712 } 713 spin_unlock_irqrestore(&channel->intent_lock, flags); 714 715 intent->id = ret; 716 intent->size = size; 717 intent->reuse = reuseable; 718 719 return intent; 720 721 free_data: 722 kfree(intent->data); 723 free_intent: 724 kfree(intent); 725 return NULL; 726 } 727 728 static void qcom_glink_handle_rx_done(struct qcom_glink *glink, 729 u32 cid, uint32_t iid, 730 bool reuse) 731 { 732 struct glink_core_rx_intent *intent; 733 struct glink_channel *channel; 734 unsigned long flags; 735 736 spin_lock_irqsave(&glink->idr_lock, flags); 737 channel = idr_find(&glink->rcids, cid); 738 spin_unlock_irqrestore(&glink->idr_lock, flags); 739 if (!channel) { 740 dev_err(glink->dev, "invalid channel id received\n"); 741 return; 742 } 743 744 spin_lock_irqsave(&channel->intent_lock, flags); 745 intent = idr_find(&channel->riids, iid); 746 747 if (!intent) { 748 spin_unlock_irqrestore(&channel->intent_lock, flags); 749 dev_err(glink->dev, "invalid intent id received\n"); 750 return; 751 } 752 753 intent->in_use = false; 754 755 if (!reuse) { 756 idr_remove(&channel->riids, intent->id); 757 kfree(intent); 758 } 759 spin_unlock_irqrestore(&channel->intent_lock, flags); 760 } 761 762 /** 763 * qcom_glink_handle_intent_req() - Receive a request for rx_intent 764 * from remote side 765 * @glink: Pointer to the transport interface 766 * @cid: Remote channel ID 767 * @size: size of the intent 768 * 769 * The function searches for the local channel to which the request for 770 * rx_intent has arrived and allocates and notifies the remote back 771 */ 772 static void qcom_glink_handle_intent_req(struct qcom_glink *glink, 773 u32 cid, size_t size) 774 { 775 struct glink_core_rx_intent *intent; 776 struct glink_channel *channel; 777 unsigned long flags; 778 779 spin_lock_irqsave(&glink->idr_lock, flags); 780 channel = idr_find(&glink->rcids, cid); 781 spin_unlock_irqrestore(&glink->idr_lock, flags); 782 783 if (!channel) { 784 pr_err("%s channel not found for cid %d\n", __func__, cid); 785 return; 786 } 787 788 intent = qcom_glink_alloc_intent(glink, channel, size, false); 789 if (intent) 790 qcom_glink_advertise_intent(glink, channel, intent); 791 792 qcom_glink_send_intent_req_ack(glink, channel, !!intent); 793 } 794 795 static int qcom_glink_rx_defer(struct qcom_glink *glink, size_t extra) 796 { 797 struct glink_defer_cmd *dcmd; 798 799 extra = ALIGN(extra, 8); 800 801 if (qcom_glink_rx_avail(glink) < sizeof(struct glink_msg) + extra) { 802 dev_dbg(glink->dev, "Insufficient data in rx fifo"); 803 return -ENXIO; 804 } 805 806 dcmd = kzalloc(struct_size(dcmd, data, extra), GFP_ATOMIC); 807 if (!dcmd) 808 return -ENOMEM; 809 810 INIT_LIST_HEAD(&dcmd->node); 811 812 qcom_glink_rx_peek(glink, &dcmd->msg, 0, sizeof(dcmd->msg) + extra); 813 814 spin_lock(&glink->rx_lock); 815 list_add_tail(&dcmd->node, &glink->rx_queue); 816 spin_unlock(&glink->rx_lock); 817 818 schedule_work(&glink->rx_work); 819 qcom_glink_rx_advance(glink, sizeof(dcmd->msg) + extra); 820 821 return 0; 822 } 823 824 static int qcom_glink_rx_data(struct qcom_glink *glink, size_t avail) 825 { 826 struct glink_core_rx_intent *intent; 827 struct glink_channel *channel; 828 struct { 829 struct glink_msg msg; 830 __le32 chunk_size; 831 __le32 left_size; 832 } __packed hdr; 833 unsigned int chunk_size; 834 unsigned int left_size; 835 unsigned int rcid; 836 unsigned int liid; 837 int ret = 0; 838 unsigned long flags; 839 840 if (avail < sizeof(hdr)) { 841 dev_dbg(glink->dev, "Not enough data in fifo\n"); 842 return -EAGAIN; 843 } 844 845 qcom_glink_rx_peek(glink, &hdr, 0, sizeof(hdr)); 846 chunk_size = le32_to_cpu(hdr.chunk_size); 847 left_size = le32_to_cpu(hdr.left_size); 848 849 if (avail < sizeof(hdr) + chunk_size) { 850 dev_dbg(glink->dev, "Payload not yet in fifo\n"); 851 return -EAGAIN; 852 } 853 854 rcid = le16_to_cpu(hdr.msg.param1); 855 spin_lock_irqsave(&glink->idr_lock, flags); 856 channel = idr_find(&glink->rcids, rcid); 857 spin_unlock_irqrestore(&glink->idr_lock, flags); 858 if (!channel) { 859 dev_dbg(glink->dev, "Data on non-existing channel\n"); 860 861 /* Drop the message */ 862 goto advance_rx; 863 } 864 865 if (glink->intentless) { 866 /* Might have an ongoing, fragmented, message to append */ 867 if (!channel->buf) { 868 intent = kzalloc(sizeof(*intent), GFP_ATOMIC); 869 if (!intent) 870 return -ENOMEM; 871 872 intent->data = kmalloc(chunk_size + left_size, 873 GFP_ATOMIC); 874 if (!intent->data) { 875 kfree(intent); 876 return -ENOMEM; 877 } 878 879 intent->id = 0xbabababa; 880 intent->size = chunk_size + left_size; 881 intent->offset = 0; 882 883 channel->buf = intent; 884 } else { 885 intent = channel->buf; 886 } 887 } else { 888 liid = le32_to_cpu(hdr.msg.param2); 889 890 spin_lock_irqsave(&channel->intent_lock, flags); 891 intent = idr_find(&channel->liids, liid); 892 spin_unlock_irqrestore(&channel->intent_lock, flags); 893 894 if (!intent) { 895 dev_err(glink->dev, 896 "no intent found for channel %s intent %d", 897 channel->name, liid); 898 ret = -ENOENT; 899 goto advance_rx; 900 } 901 } 902 903 if (intent->size - intent->offset < chunk_size) { 904 dev_err(glink->dev, "Insufficient space in intent\n"); 905 906 /* The packet header lied, drop payload */ 907 goto advance_rx; 908 } 909 910 qcom_glink_rx_peek(glink, intent->data + intent->offset, 911 sizeof(hdr), chunk_size); 912 intent->offset += chunk_size; 913 914 /* Handle message when no fragments remain to be received */ 915 if (!left_size) { 916 spin_lock(&channel->recv_lock); 917 if (channel->ept.cb) { 918 channel->ept.cb(channel->ept.rpdev, 919 intent->data, 920 intent->offset, 921 channel->ept.priv, 922 RPMSG_ADDR_ANY); 923 } 924 spin_unlock(&channel->recv_lock); 925 926 intent->offset = 0; 927 channel->buf = NULL; 928 929 qcom_glink_rx_done(glink, channel, intent); 930 } 931 932 advance_rx: 933 qcom_glink_rx_advance(glink, ALIGN(sizeof(hdr) + chunk_size, 8)); 934 935 return ret; 936 } 937 938 static void qcom_glink_handle_intent(struct qcom_glink *glink, 939 unsigned int cid, 940 unsigned int count, 941 size_t avail) 942 { 943 struct glink_core_rx_intent *intent; 944 struct glink_channel *channel; 945 struct intent_pair { 946 __le32 size; 947 __le32 iid; 948 }; 949 950 struct { 951 struct glink_msg msg; 952 struct intent_pair intents[]; 953 } __packed * msg; 954 955 const size_t msglen = struct_size(msg, intents, count); 956 int ret; 957 int i; 958 unsigned long flags; 959 960 if (avail < msglen) { 961 dev_dbg(glink->dev, "Not enough data in fifo\n"); 962 return; 963 } 964 965 spin_lock_irqsave(&glink->idr_lock, flags); 966 channel = idr_find(&glink->rcids, cid); 967 spin_unlock_irqrestore(&glink->idr_lock, flags); 968 if (!channel) { 969 dev_err(glink->dev, "intents for non-existing channel\n"); 970 qcom_glink_rx_advance(glink, ALIGN(msglen, 8)); 971 return; 972 } 973 974 msg = kmalloc(msglen, GFP_ATOMIC); 975 if (!msg) 976 return; 977 978 qcom_glink_rx_peek(glink, msg, 0, msglen); 979 980 for (i = 0; i < count; ++i) { 981 intent = kzalloc(sizeof(*intent), GFP_ATOMIC); 982 if (!intent) 983 break; 984 985 intent->id = le32_to_cpu(msg->intents[i].iid); 986 intent->size = le32_to_cpu(msg->intents[i].size); 987 988 spin_lock_irqsave(&channel->intent_lock, flags); 989 ret = idr_alloc(&channel->riids, intent, 990 intent->id, intent->id + 1, GFP_ATOMIC); 991 spin_unlock_irqrestore(&channel->intent_lock, flags); 992 993 if (ret < 0) 994 dev_err(glink->dev, "failed to store remote intent\n"); 995 } 996 997 kfree(msg); 998 qcom_glink_rx_advance(glink, ALIGN(msglen, 8)); 999 } 1000 1001 static int qcom_glink_rx_open_ack(struct qcom_glink *glink, unsigned int lcid) 1002 { 1003 struct glink_channel *channel; 1004 1005 spin_lock(&glink->idr_lock); 1006 channel = idr_find(&glink->lcids, lcid); 1007 spin_unlock(&glink->idr_lock); 1008 if (!channel) { 1009 dev_err(glink->dev, "Invalid open ack packet\n"); 1010 return -EINVAL; 1011 } 1012 1013 complete_all(&channel->open_ack); 1014 1015 return 0; 1016 } 1017 1018 void qcom_glink_native_rx(struct qcom_glink *glink) 1019 { 1020 struct glink_msg msg; 1021 unsigned int param1; 1022 unsigned int param2; 1023 unsigned int avail; 1024 unsigned int cmd; 1025 int ret = 0; 1026 1027 /* To wakeup any blocking writers */ 1028 wake_up_all(&glink->tx_avail_notify); 1029 1030 for (;;) { 1031 avail = qcom_glink_rx_avail(glink); 1032 if (avail < sizeof(msg)) 1033 break; 1034 1035 qcom_glink_rx_peek(glink, &msg, 0, sizeof(msg)); 1036 1037 cmd = le16_to_cpu(msg.cmd); 1038 param1 = le16_to_cpu(msg.param1); 1039 param2 = le32_to_cpu(msg.param2); 1040 1041 switch (cmd) { 1042 case GLINK_CMD_VERSION: 1043 case GLINK_CMD_VERSION_ACK: 1044 case GLINK_CMD_CLOSE: 1045 case GLINK_CMD_CLOSE_ACK: 1046 case GLINK_CMD_RX_INTENT_REQ: 1047 ret = qcom_glink_rx_defer(glink, 0); 1048 break; 1049 case GLINK_CMD_OPEN_ACK: 1050 ret = qcom_glink_rx_open_ack(glink, param1); 1051 qcom_glink_rx_advance(glink, ALIGN(sizeof(msg), 8)); 1052 break; 1053 case GLINK_CMD_OPEN: 1054 ret = qcom_glink_rx_defer(glink, param2); 1055 break; 1056 case GLINK_CMD_TX_DATA: 1057 case GLINK_CMD_TX_DATA_CONT: 1058 ret = qcom_glink_rx_data(glink, avail); 1059 break; 1060 case GLINK_CMD_READ_NOTIF: 1061 qcom_glink_rx_advance(glink, ALIGN(sizeof(msg), 8)); 1062 qcom_glink_tx_kick(glink); 1063 break; 1064 case GLINK_CMD_INTENT: 1065 qcom_glink_handle_intent(glink, param1, param2, avail); 1066 break; 1067 case GLINK_CMD_RX_DONE: 1068 qcom_glink_handle_rx_done(glink, param1, param2, false); 1069 qcom_glink_rx_advance(glink, ALIGN(sizeof(msg), 8)); 1070 break; 1071 case GLINK_CMD_RX_DONE_W_REUSE: 1072 qcom_glink_handle_rx_done(glink, param1, param2, true); 1073 qcom_glink_rx_advance(glink, ALIGN(sizeof(msg), 8)); 1074 break; 1075 case GLINK_CMD_RX_INTENT_REQ_ACK: 1076 qcom_glink_handle_intent_req_ack(glink, param1, param2); 1077 qcom_glink_rx_advance(glink, ALIGN(sizeof(msg), 8)); 1078 break; 1079 default: 1080 dev_err(glink->dev, "unhandled rx cmd: %d\n", cmd); 1081 ret = -EINVAL; 1082 break; 1083 } 1084 1085 if (ret) 1086 break; 1087 } 1088 } 1089 EXPORT_SYMBOL(qcom_glink_native_rx); 1090 1091 /* Locally initiated rpmsg_create_ept */ 1092 static struct glink_channel *qcom_glink_create_local(struct qcom_glink *glink, 1093 const char *name) 1094 { 1095 struct glink_channel *channel; 1096 int ret; 1097 unsigned long flags; 1098 1099 channel = qcom_glink_alloc_channel(glink, name); 1100 if (IS_ERR(channel)) 1101 return ERR_CAST(channel); 1102 1103 ret = qcom_glink_send_open_req(glink, channel); 1104 if (ret) 1105 goto release_channel; 1106 1107 ret = wait_for_completion_timeout(&channel->open_ack, 5 * HZ); 1108 if (!ret) 1109 goto err_timeout; 1110 1111 ret = wait_for_completion_timeout(&channel->open_req, 5 * HZ); 1112 if (!ret) 1113 goto err_timeout; 1114 1115 qcom_glink_send_open_ack(glink, channel); 1116 1117 return channel; 1118 1119 err_timeout: 1120 /* qcom_glink_send_open_req() did register the channel in lcids*/ 1121 spin_lock_irqsave(&glink->idr_lock, flags); 1122 idr_remove(&glink->lcids, channel->lcid); 1123 spin_unlock_irqrestore(&glink->idr_lock, flags); 1124 1125 release_channel: 1126 /* Release qcom_glink_send_open_req() reference */ 1127 kref_put(&channel->refcount, qcom_glink_channel_release); 1128 /* Release qcom_glink_alloc_channel() reference */ 1129 kref_put(&channel->refcount, qcom_glink_channel_release); 1130 1131 return ERR_PTR(-ETIMEDOUT); 1132 } 1133 1134 /* Remote initiated rpmsg_create_ept */ 1135 static int qcom_glink_create_remote(struct qcom_glink *glink, 1136 struct glink_channel *channel) 1137 { 1138 int ret; 1139 1140 qcom_glink_send_open_ack(glink, channel); 1141 1142 ret = qcom_glink_send_open_req(glink, channel); 1143 if (ret) 1144 goto close_link; 1145 1146 ret = wait_for_completion_timeout(&channel->open_ack, 5 * HZ); 1147 if (!ret) { 1148 ret = -ETIMEDOUT; 1149 goto close_link; 1150 } 1151 1152 return 0; 1153 1154 close_link: 1155 /* 1156 * Send a close request to "undo" our open-ack. The close-ack will 1157 * release qcom_glink_send_open_req() reference and the last reference 1158 * will be relesed after receiving remote_close or transport unregister 1159 * by calling qcom_glink_native_remove(). 1160 */ 1161 qcom_glink_send_close_req(glink, channel); 1162 1163 return ret; 1164 } 1165 1166 static struct rpmsg_endpoint *qcom_glink_create_ept(struct rpmsg_device *rpdev, 1167 rpmsg_rx_cb_t cb, 1168 void *priv, 1169 struct rpmsg_channel_info 1170 chinfo) 1171 { 1172 struct glink_channel *parent = to_glink_channel(rpdev->ept); 1173 struct glink_channel *channel; 1174 struct qcom_glink *glink = parent->glink; 1175 struct rpmsg_endpoint *ept; 1176 const char *name = chinfo.name; 1177 int cid; 1178 int ret; 1179 unsigned long flags; 1180 1181 spin_lock_irqsave(&glink->idr_lock, flags); 1182 idr_for_each_entry(&glink->rcids, channel, cid) { 1183 if (!strcmp(channel->name, name)) 1184 break; 1185 } 1186 spin_unlock_irqrestore(&glink->idr_lock, flags); 1187 1188 if (!channel) { 1189 channel = qcom_glink_create_local(glink, name); 1190 if (IS_ERR(channel)) 1191 return NULL; 1192 } else { 1193 ret = qcom_glink_create_remote(glink, channel); 1194 if (ret) 1195 return NULL; 1196 } 1197 1198 ept = &channel->ept; 1199 ept->rpdev = rpdev; 1200 ept->cb = cb; 1201 ept->priv = priv; 1202 ept->ops = &glink_endpoint_ops; 1203 1204 return ept; 1205 } 1206 1207 static int qcom_glink_announce_create(struct rpmsg_device *rpdev) 1208 { 1209 struct glink_channel *channel = to_glink_channel(rpdev->ept); 1210 struct device_node *np = rpdev->dev.of_node; 1211 struct qcom_glink *glink = channel->glink; 1212 struct glink_core_rx_intent *intent; 1213 const struct property *prop = NULL; 1214 __be32 defaults[] = { cpu_to_be32(SZ_1K), cpu_to_be32(5) }; 1215 int num_intents; 1216 int num_groups = 1; 1217 __be32 *val = defaults; 1218 int size; 1219 1220 if (glink->intentless || !completion_done(&channel->open_ack)) 1221 return 0; 1222 1223 prop = of_find_property(np, "qcom,intents", NULL); 1224 if (prop) { 1225 val = prop->value; 1226 num_groups = prop->length / sizeof(u32) / 2; 1227 } 1228 1229 /* Channel is now open, advertise base set of intents */ 1230 while (num_groups--) { 1231 size = be32_to_cpup(val++); 1232 num_intents = be32_to_cpup(val++); 1233 while (num_intents--) { 1234 intent = qcom_glink_alloc_intent(glink, channel, size, 1235 true); 1236 if (!intent) 1237 break; 1238 1239 qcom_glink_advertise_intent(glink, channel, intent); 1240 } 1241 } 1242 return 0; 1243 } 1244 1245 static void qcom_glink_destroy_ept(struct rpmsg_endpoint *ept) 1246 { 1247 struct glink_channel *channel = to_glink_channel(ept); 1248 struct qcom_glink *glink = channel->glink; 1249 unsigned long flags; 1250 1251 spin_lock_irqsave(&channel->recv_lock, flags); 1252 channel->ept.cb = NULL; 1253 spin_unlock_irqrestore(&channel->recv_lock, flags); 1254 1255 /* Decouple the potential rpdev from the channel */ 1256 channel->rpdev = NULL; 1257 1258 qcom_glink_send_close_req(glink, channel); 1259 } 1260 1261 static int qcom_glink_request_intent(struct qcom_glink *glink, 1262 struct glink_channel *channel, 1263 size_t size) 1264 { 1265 struct { 1266 u16 id; 1267 u16 cid; 1268 u32 size; 1269 } __packed cmd; 1270 1271 int ret; 1272 1273 mutex_lock(&channel->intent_req_lock); 1274 1275 WRITE_ONCE(channel->intent_req_result, -1); 1276 1277 cmd.id = GLINK_CMD_RX_INTENT_REQ; 1278 cmd.cid = channel->lcid; 1279 cmd.size = size; 1280 1281 ret = qcom_glink_tx(glink, &cmd, sizeof(cmd), NULL, 0, true); 1282 if (ret) 1283 goto unlock; 1284 1285 ret = wait_event_timeout(channel->intent_req_wq, 1286 READ_ONCE(channel->intent_req_result) >= 0, 1287 10 * HZ); 1288 if (!ret) { 1289 dev_err(glink->dev, "intent request timed out\n"); 1290 ret = -ETIMEDOUT; 1291 } else { 1292 ret = READ_ONCE(channel->intent_req_result) ? 0 : -ECANCELED; 1293 } 1294 1295 unlock: 1296 mutex_unlock(&channel->intent_req_lock); 1297 return ret; 1298 } 1299 1300 static int __qcom_glink_send(struct glink_channel *channel, 1301 void *data, int len, bool wait) 1302 { 1303 struct qcom_glink *glink = channel->glink; 1304 struct glink_core_rx_intent *intent = NULL; 1305 struct glink_core_rx_intent *tmp; 1306 int iid = 0; 1307 struct { 1308 struct glink_msg msg; 1309 __le32 chunk_size; 1310 __le32 left_size; 1311 } __packed req; 1312 int ret; 1313 unsigned long flags; 1314 int chunk_size = len; 1315 int left_size = 0; 1316 1317 if (!glink->intentless) { 1318 while (!intent) { 1319 spin_lock_irqsave(&channel->intent_lock, flags); 1320 idr_for_each_entry(&channel->riids, tmp, iid) { 1321 if (tmp->size >= len && !tmp->in_use) { 1322 if (!intent) 1323 intent = tmp; 1324 else if (intent->size > tmp->size) 1325 intent = tmp; 1326 if (intent->size == len) 1327 break; 1328 } 1329 } 1330 if (intent) 1331 intent->in_use = true; 1332 spin_unlock_irqrestore(&channel->intent_lock, flags); 1333 1334 /* We found an available intent */ 1335 if (intent) 1336 break; 1337 1338 if (!wait) 1339 return -EBUSY; 1340 1341 ret = qcom_glink_request_intent(glink, channel, len); 1342 if (ret < 0) 1343 return ret; 1344 } 1345 1346 iid = intent->id; 1347 } 1348 1349 if (wait && chunk_size > SZ_8K) { 1350 chunk_size = SZ_8K; 1351 left_size = len - chunk_size; 1352 } 1353 req.msg.cmd = cpu_to_le16(GLINK_CMD_TX_DATA); 1354 req.msg.param1 = cpu_to_le16(channel->lcid); 1355 req.msg.param2 = cpu_to_le32(iid); 1356 req.chunk_size = cpu_to_le32(chunk_size); 1357 req.left_size = cpu_to_le32(left_size); 1358 1359 ret = qcom_glink_tx(glink, &req, sizeof(req), data, chunk_size, wait); 1360 1361 /* Mark intent available if we failed */ 1362 if (ret && intent) { 1363 intent->in_use = false; 1364 return ret; 1365 } 1366 1367 while (left_size > 0) { 1368 data = (void *)((char *)data + chunk_size); 1369 chunk_size = left_size; 1370 if (chunk_size > SZ_8K) 1371 chunk_size = SZ_8K; 1372 left_size -= chunk_size; 1373 1374 req.msg.cmd = cpu_to_le16(GLINK_CMD_TX_DATA_CONT); 1375 req.msg.param1 = cpu_to_le16(channel->lcid); 1376 req.msg.param2 = cpu_to_le32(iid); 1377 req.chunk_size = cpu_to_le32(chunk_size); 1378 req.left_size = cpu_to_le32(left_size); 1379 1380 ret = qcom_glink_tx(glink, &req, sizeof(req), data, 1381 chunk_size, wait); 1382 1383 /* Mark intent available if we failed */ 1384 if (ret && intent) { 1385 intent->in_use = false; 1386 break; 1387 } 1388 } 1389 return ret; 1390 } 1391 1392 static int qcom_glink_send(struct rpmsg_endpoint *ept, void *data, int len) 1393 { 1394 struct glink_channel *channel = to_glink_channel(ept); 1395 1396 return __qcom_glink_send(channel, data, len, true); 1397 } 1398 1399 static int qcom_glink_trysend(struct rpmsg_endpoint *ept, void *data, int len) 1400 { 1401 struct glink_channel *channel = to_glink_channel(ept); 1402 1403 return __qcom_glink_send(channel, data, len, false); 1404 } 1405 1406 static int qcom_glink_sendto(struct rpmsg_endpoint *ept, void *data, int len, u32 dst) 1407 { 1408 struct glink_channel *channel = to_glink_channel(ept); 1409 1410 return __qcom_glink_send(channel, data, len, true); 1411 } 1412 1413 static int qcom_glink_trysendto(struct rpmsg_endpoint *ept, void *data, int len, u32 dst) 1414 { 1415 struct glink_channel *channel = to_glink_channel(ept); 1416 1417 return __qcom_glink_send(channel, data, len, false); 1418 } 1419 1420 /* 1421 * Finds the device_node for the glink child interested in this channel. 1422 */ 1423 static struct device_node *qcom_glink_match_channel(struct device_node *node, 1424 const char *channel) 1425 { 1426 struct device_node *child; 1427 const char *name; 1428 const char *key; 1429 int ret; 1430 1431 for_each_available_child_of_node(node, child) { 1432 key = "qcom,glink-channels"; 1433 ret = of_property_read_string(child, key, &name); 1434 if (ret) 1435 continue; 1436 1437 if (strcmp(name, channel) == 0) 1438 return child; 1439 } 1440 1441 return NULL; 1442 } 1443 1444 static const struct rpmsg_device_ops glink_device_ops = { 1445 .create_ept = qcom_glink_create_ept, 1446 .announce_create = qcom_glink_announce_create, 1447 }; 1448 1449 static const struct rpmsg_endpoint_ops glink_endpoint_ops = { 1450 .destroy_ept = qcom_glink_destroy_ept, 1451 .send = qcom_glink_send, 1452 .sendto = qcom_glink_sendto, 1453 .trysend = qcom_glink_trysend, 1454 .trysendto = qcom_glink_trysendto, 1455 }; 1456 1457 static void qcom_glink_rpdev_release(struct device *dev) 1458 { 1459 struct rpmsg_device *rpdev = to_rpmsg_device(dev); 1460 1461 kfree(rpdev->driver_override); 1462 kfree(rpdev); 1463 } 1464 1465 static int qcom_glink_rx_open(struct qcom_glink *glink, unsigned int rcid, 1466 char *name) 1467 { 1468 struct glink_channel *channel; 1469 struct rpmsg_device *rpdev; 1470 bool create_device = false; 1471 struct device_node *node; 1472 int lcid; 1473 int ret; 1474 unsigned long flags; 1475 1476 spin_lock_irqsave(&glink->idr_lock, flags); 1477 idr_for_each_entry(&glink->lcids, channel, lcid) { 1478 if (!strcmp(channel->name, name)) 1479 break; 1480 } 1481 spin_unlock_irqrestore(&glink->idr_lock, flags); 1482 1483 if (!channel) { 1484 channel = qcom_glink_alloc_channel(glink, name); 1485 if (IS_ERR(channel)) 1486 return PTR_ERR(channel); 1487 1488 /* The opening dance was initiated by the remote */ 1489 create_device = true; 1490 } 1491 1492 spin_lock_irqsave(&glink->idr_lock, flags); 1493 ret = idr_alloc(&glink->rcids, channel, rcid, rcid + 1, GFP_ATOMIC); 1494 if (ret < 0) { 1495 dev_err(glink->dev, "Unable to insert channel into rcid list\n"); 1496 spin_unlock_irqrestore(&glink->idr_lock, flags); 1497 goto free_channel; 1498 } 1499 channel->rcid = ret; 1500 spin_unlock_irqrestore(&glink->idr_lock, flags); 1501 1502 complete_all(&channel->open_req); 1503 1504 if (create_device) { 1505 rpdev = kzalloc(sizeof(*rpdev), GFP_KERNEL); 1506 if (!rpdev) { 1507 ret = -ENOMEM; 1508 goto rcid_remove; 1509 } 1510 1511 rpdev->ept = &channel->ept; 1512 strscpy_pad(rpdev->id.name, name, RPMSG_NAME_SIZE); 1513 rpdev->src = RPMSG_ADDR_ANY; 1514 rpdev->dst = RPMSG_ADDR_ANY; 1515 rpdev->ops = &glink_device_ops; 1516 1517 node = qcom_glink_match_channel(glink->dev->of_node, name); 1518 rpdev->dev.of_node = node; 1519 rpdev->dev.parent = glink->dev; 1520 rpdev->dev.release = qcom_glink_rpdev_release; 1521 1522 ret = rpmsg_register_device(rpdev); 1523 if (ret) 1524 goto rcid_remove; 1525 1526 channel->rpdev = rpdev; 1527 } 1528 1529 return 0; 1530 1531 rcid_remove: 1532 spin_lock_irqsave(&glink->idr_lock, flags); 1533 idr_remove(&glink->rcids, channel->rcid); 1534 channel->rcid = 0; 1535 spin_unlock_irqrestore(&glink->idr_lock, flags); 1536 free_channel: 1537 /* Release the reference, iff we took it */ 1538 if (create_device) 1539 kref_put(&channel->refcount, qcom_glink_channel_release); 1540 1541 return ret; 1542 } 1543 1544 static void qcom_glink_rx_close(struct qcom_glink *glink, unsigned int rcid) 1545 { 1546 struct rpmsg_channel_info chinfo; 1547 struct glink_channel *channel; 1548 unsigned long flags; 1549 1550 spin_lock_irqsave(&glink->idr_lock, flags); 1551 channel = idr_find(&glink->rcids, rcid); 1552 spin_unlock_irqrestore(&glink->idr_lock, flags); 1553 if (WARN(!channel, "close request on unknown channel\n")) 1554 return; 1555 1556 /* cancel pending rx_done work */ 1557 cancel_work_sync(&channel->intent_work); 1558 1559 if (channel->rpdev) { 1560 strscpy_pad(chinfo.name, channel->name, sizeof(chinfo.name)); 1561 chinfo.src = RPMSG_ADDR_ANY; 1562 chinfo.dst = RPMSG_ADDR_ANY; 1563 1564 rpmsg_unregister_device(glink->dev, &chinfo); 1565 } 1566 channel->rpdev = NULL; 1567 1568 qcom_glink_send_close_ack(glink, channel->rcid); 1569 1570 spin_lock_irqsave(&glink->idr_lock, flags); 1571 idr_remove(&glink->rcids, channel->rcid); 1572 channel->rcid = 0; 1573 spin_unlock_irqrestore(&glink->idr_lock, flags); 1574 1575 kref_put(&channel->refcount, qcom_glink_channel_release); 1576 } 1577 1578 static void qcom_glink_rx_close_ack(struct qcom_glink *glink, unsigned int lcid) 1579 { 1580 struct rpmsg_channel_info chinfo; 1581 struct glink_channel *channel; 1582 unsigned long flags; 1583 1584 /* To wakeup any blocking writers */ 1585 wake_up_all(&glink->tx_avail_notify); 1586 1587 spin_lock_irqsave(&glink->idr_lock, flags); 1588 channel = idr_find(&glink->lcids, lcid); 1589 if (WARN(!channel, "close ack on unknown channel\n")) { 1590 spin_unlock_irqrestore(&glink->idr_lock, flags); 1591 return; 1592 } 1593 1594 idr_remove(&glink->lcids, channel->lcid); 1595 channel->lcid = 0; 1596 spin_unlock_irqrestore(&glink->idr_lock, flags); 1597 1598 /* Decouple the potential rpdev from the channel */ 1599 if (channel->rpdev) { 1600 strscpy(chinfo.name, channel->name, sizeof(chinfo.name)); 1601 chinfo.src = RPMSG_ADDR_ANY; 1602 chinfo.dst = RPMSG_ADDR_ANY; 1603 1604 rpmsg_unregister_device(glink->dev, &chinfo); 1605 } 1606 channel->rpdev = NULL; 1607 1608 kref_put(&channel->refcount, qcom_glink_channel_release); 1609 } 1610 1611 static void qcom_glink_work(struct work_struct *work) 1612 { 1613 struct qcom_glink *glink = container_of(work, struct qcom_glink, 1614 rx_work); 1615 struct glink_defer_cmd *dcmd; 1616 struct glink_msg *msg; 1617 unsigned long flags; 1618 unsigned int param1; 1619 unsigned int param2; 1620 unsigned int cmd; 1621 1622 for (;;) { 1623 spin_lock_irqsave(&glink->rx_lock, flags); 1624 if (list_empty(&glink->rx_queue)) { 1625 spin_unlock_irqrestore(&glink->rx_lock, flags); 1626 break; 1627 } 1628 dcmd = list_first_entry(&glink->rx_queue, 1629 struct glink_defer_cmd, node); 1630 list_del(&dcmd->node); 1631 spin_unlock_irqrestore(&glink->rx_lock, flags); 1632 1633 msg = &dcmd->msg; 1634 cmd = le16_to_cpu(msg->cmd); 1635 param1 = le16_to_cpu(msg->param1); 1636 param2 = le32_to_cpu(msg->param2); 1637 1638 switch (cmd) { 1639 case GLINK_CMD_VERSION: 1640 qcom_glink_receive_version(glink, param1, param2); 1641 break; 1642 case GLINK_CMD_VERSION_ACK: 1643 qcom_glink_receive_version_ack(glink, param1, param2); 1644 break; 1645 case GLINK_CMD_OPEN: 1646 qcom_glink_rx_open(glink, param1, msg->data); 1647 break; 1648 case GLINK_CMD_CLOSE: 1649 qcom_glink_rx_close(glink, param1); 1650 break; 1651 case GLINK_CMD_CLOSE_ACK: 1652 qcom_glink_rx_close_ack(glink, param1); 1653 break; 1654 case GLINK_CMD_RX_INTENT_REQ: 1655 qcom_glink_handle_intent_req(glink, param1, param2); 1656 break; 1657 default: 1658 WARN(1, "Unknown defer object %d\n", cmd); 1659 break; 1660 } 1661 1662 kfree(dcmd); 1663 } 1664 } 1665 1666 static void qcom_glink_cancel_rx_work(struct qcom_glink *glink) 1667 { 1668 struct glink_defer_cmd *dcmd; 1669 struct glink_defer_cmd *tmp; 1670 1671 /* cancel any pending deferred rx_work */ 1672 cancel_work_sync(&glink->rx_work); 1673 1674 list_for_each_entry_safe(dcmd, tmp, &glink->rx_queue, node) 1675 kfree(dcmd); 1676 } 1677 1678 static ssize_t rpmsg_name_show(struct device *dev, 1679 struct device_attribute *attr, char *buf) 1680 { 1681 int ret = 0; 1682 const char *name; 1683 1684 ret = of_property_read_string(dev->of_node, "label", &name); 1685 if (ret < 0) 1686 name = dev->of_node->name; 1687 1688 return sysfs_emit(buf, "%s\n", name); 1689 } 1690 static DEVICE_ATTR_RO(rpmsg_name); 1691 1692 static struct attribute *qcom_glink_attrs[] = { 1693 &dev_attr_rpmsg_name.attr, 1694 NULL 1695 }; 1696 ATTRIBUTE_GROUPS(qcom_glink); 1697 1698 static void qcom_glink_device_release(struct device *dev) 1699 { 1700 struct rpmsg_device *rpdev = to_rpmsg_device(dev); 1701 struct glink_channel *channel = to_glink_channel(rpdev->ept); 1702 1703 /* Release qcom_glink_alloc_channel() reference */ 1704 kref_put(&channel->refcount, qcom_glink_channel_release); 1705 kfree(rpdev->driver_override); 1706 kfree(rpdev); 1707 } 1708 1709 static int qcom_glink_create_chrdev(struct qcom_glink *glink) 1710 { 1711 struct rpmsg_device *rpdev; 1712 struct glink_channel *channel; 1713 1714 rpdev = kzalloc(sizeof(*rpdev), GFP_KERNEL); 1715 if (!rpdev) 1716 return -ENOMEM; 1717 1718 channel = qcom_glink_alloc_channel(glink, "rpmsg_chrdev"); 1719 if (IS_ERR(channel)) { 1720 kfree(rpdev); 1721 return PTR_ERR(channel); 1722 } 1723 channel->rpdev = rpdev; 1724 1725 rpdev->ept = &channel->ept; 1726 rpdev->ops = &glink_device_ops; 1727 rpdev->dev.parent = glink->dev; 1728 rpdev->dev.release = qcom_glink_device_release; 1729 1730 return rpmsg_ctrldev_register_device(rpdev); 1731 } 1732 1733 struct qcom_glink *qcom_glink_native_probe(struct device *dev, 1734 unsigned long features, 1735 struct qcom_glink_pipe *rx, 1736 struct qcom_glink_pipe *tx, 1737 bool intentless) 1738 { 1739 int ret; 1740 struct qcom_glink *glink; 1741 1742 glink = devm_kzalloc(dev, sizeof(*glink), GFP_KERNEL); 1743 if (!glink) 1744 return ERR_PTR(-ENOMEM); 1745 1746 glink->dev = dev; 1747 glink->tx_pipe = tx; 1748 glink->rx_pipe = rx; 1749 1750 glink->features = features; 1751 glink->intentless = intentless; 1752 1753 spin_lock_init(&glink->tx_lock); 1754 spin_lock_init(&glink->rx_lock); 1755 INIT_LIST_HEAD(&glink->rx_queue); 1756 INIT_WORK(&glink->rx_work, qcom_glink_work); 1757 init_waitqueue_head(&glink->tx_avail_notify); 1758 1759 spin_lock_init(&glink->idr_lock); 1760 idr_init(&glink->lcids); 1761 idr_init(&glink->rcids); 1762 1763 glink->dev->groups = qcom_glink_groups; 1764 1765 ret = device_add_groups(dev, qcom_glink_groups); 1766 if (ret) 1767 dev_err(dev, "failed to add groups\n"); 1768 1769 ret = qcom_glink_send_version(glink); 1770 if (ret) 1771 return ERR_PTR(ret); 1772 1773 ret = qcom_glink_create_chrdev(glink); 1774 if (ret) 1775 dev_err(glink->dev, "failed to register chrdev\n"); 1776 1777 return glink; 1778 } 1779 EXPORT_SYMBOL_GPL(qcom_glink_native_probe); 1780 1781 static int qcom_glink_remove_device(struct device *dev, void *data) 1782 { 1783 device_unregister(dev); 1784 1785 return 0; 1786 } 1787 1788 void qcom_glink_native_remove(struct qcom_glink *glink) 1789 { 1790 struct glink_channel *channel; 1791 unsigned long flags; 1792 int cid; 1793 int ret; 1794 1795 qcom_glink_cancel_rx_work(glink); 1796 1797 /* Fail all attempts at sending messages */ 1798 spin_lock_irqsave(&glink->tx_lock, flags); 1799 glink->abort_tx = true; 1800 wake_up_all(&glink->tx_avail_notify); 1801 spin_unlock_irqrestore(&glink->tx_lock, flags); 1802 1803 /* Abort any senders waiting for intent requests */ 1804 spin_lock_irqsave(&glink->idr_lock, flags); 1805 idr_for_each_entry(&glink->lcids, channel, cid) 1806 qcom_glink_intent_req_abort(channel); 1807 spin_unlock_irqrestore(&glink->idr_lock, flags); 1808 1809 ret = device_for_each_child(glink->dev, NULL, qcom_glink_remove_device); 1810 if (ret) 1811 dev_warn(glink->dev, "Can't remove GLINK devices: %d\n", ret); 1812 1813 /* Release any defunct local channels, waiting for close-ack */ 1814 idr_for_each_entry(&glink->lcids, channel, cid) 1815 kref_put(&channel->refcount, qcom_glink_channel_release); 1816 1817 /* Release any defunct local channels, waiting for close-req */ 1818 idr_for_each_entry(&glink->rcids, channel, cid) 1819 kref_put(&channel->refcount, qcom_glink_channel_release); 1820 1821 idr_destroy(&glink->lcids); 1822 idr_destroy(&glink->rcids); 1823 } 1824 EXPORT_SYMBOL_GPL(qcom_glink_native_remove); 1825 1826 MODULE_DESCRIPTION("Qualcomm GLINK driver"); 1827 MODULE_LICENSE("GPL v2"); 1828