1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * Copyright (c) 2016-2017, Linaro Ltd 4 */ 5 6 #include <linux/idr.h> 7 #include <linux/interrupt.h> 8 #include <linux/io.h> 9 #include <linux/list.h> 10 #include <linux/mfd/syscon.h> 11 #include <linux/module.h> 12 #include <linux/of.h> 13 #include <linux/of_address.h> 14 #include <linux/platform_device.h> 15 #include <linux/regmap.h> 16 #include <linux/rpmsg.h> 17 #include <linux/sizes.h> 18 #include <linux/slab.h> 19 #include <linux/workqueue.h> 20 #include <linux/mailbox_client.h> 21 22 #include "rpmsg_internal.h" 23 #include "qcom_glink_native.h" 24 25 #define GLINK_NAME_SIZE 32 26 #define GLINK_VERSION_1 1 27 28 #define RPM_GLINK_CID_MIN 1 29 #define RPM_GLINK_CID_MAX 65536 30 31 struct glink_msg { 32 __le16 cmd; 33 __le16 param1; 34 __le32 param2; 35 u8 data[]; 36 } __packed; 37 38 /** 39 * struct glink_defer_cmd - deferred incoming control message 40 * @node: list node 41 * @msg: message header 42 * @data: payload of the message 43 * 44 * Copy of a received control message, to be added to @rx_queue and processed 45 * by @rx_work of @qcom_glink. 46 */ 47 struct glink_defer_cmd { 48 struct list_head node; 49 50 struct glink_msg msg; 51 u8 data[]; 52 }; 53 54 /** 55 * struct glink_core_rx_intent - RX intent 56 * RX intent 57 * 58 * @data: pointer to the data (may be NULL for zero-copy) 59 * @id: remote or local intent ID 60 * @size: size of the original intent (do not modify) 61 * @reuse: To mark if the intent can be reused after first use 62 * @in_use: To mark if intent is already in use for the channel 63 * @offset: next write offset (initially 0) 64 * @node: list node 65 */ 66 struct glink_core_rx_intent { 67 void *data; 68 u32 id; 69 size_t size; 70 bool reuse; 71 bool in_use; 72 u32 offset; 73 74 struct list_head node; 75 }; 76 77 /** 78 * struct qcom_glink - driver context, relates to one remote subsystem 79 * @dev: reference to the associated struct device 80 * @rx_pipe: pipe object for receive FIFO 81 * @tx_pipe: pipe object for transmit FIFO 82 * @rx_work: worker for handling received control messages 83 * @rx_lock: protects the @rx_queue 84 * @rx_queue: queue of received control messages to be processed in @rx_work 85 * @tx_lock: synchronizes operations on the tx fifo 86 * @idr_lock: synchronizes @lcids and @rcids modifications 87 * @lcids: idr of all channels with a known local channel id 88 * @rcids: idr of all channels with a known remote channel id 89 * @features: remote features 90 * @intentless: flag to indicate that there is no intent 91 * @tx_avail_notify: Waitqueue for pending tx tasks 92 * @sent_read_notify: flag to check cmd sent or not 93 * @abort_tx: flag indicating that all tx attempts should fail 94 */ 95 struct qcom_glink { 96 struct device *dev; 97 98 struct qcom_glink_pipe *rx_pipe; 99 struct qcom_glink_pipe *tx_pipe; 100 101 struct work_struct rx_work; 102 spinlock_t rx_lock; 103 struct list_head rx_queue; 104 105 spinlock_t tx_lock; 106 107 spinlock_t idr_lock; 108 struct idr lcids; 109 struct idr rcids; 110 unsigned long features; 111 112 bool intentless; 113 wait_queue_head_t tx_avail_notify; 114 bool sent_read_notify; 115 116 bool abort_tx; 117 }; 118 119 enum { 120 GLINK_STATE_CLOSED, 121 GLINK_STATE_OPENING, 122 GLINK_STATE_OPEN, 123 GLINK_STATE_CLOSING, 124 }; 125 126 /** 127 * struct glink_channel - internal representation of a channel 128 * @rpdev: rpdev reference, only used for primary endpoints 129 * @ept: rpmsg endpoint this channel is associated with 130 * @glink: qcom_glink context handle 131 * @refcount: refcount for the channel object 132 * @recv_lock: guard for @ept.cb 133 * @name: unique channel name/identifier 134 * @lcid: channel id, in local space 135 * @rcid: channel id, in remote space 136 * @intent_lock: lock for protection of @liids, @riids 137 * @liids: idr of all local intents 138 * @riids: idr of all remote intents 139 * @intent_work: worker responsible for transmitting rx_done packets 140 * @done_intents: list of intents that needs to be announced rx_done 141 * @buf: receive buffer, for gathering fragments 142 * @buf_offset: write offset in @buf 143 * @buf_size: size of current @buf 144 * @open_ack: completed once remote has acked the open-request 145 * @open_req: completed once open-request has been received 146 * @intent_req_lock: Synchronises multiple intent requests 147 * @intent_req_result: Result of intent request 148 * @intent_req_comp: Completion for intent_req signalling 149 */ 150 struct glink_channel { 151 struct rpmsg_endpoint ept; 152 153 struct rpmsg_device *rpdev; 154 struct qcom_glink *glink; 155 156 struct kref refcount; 157 158 spinlock_t recv_lock; 159 160 char *name; 161 unsigned int lcid; 162 unsigned int rcid; 163 164 spinlock_t intent_lock; 165 struct idr liids; 166 struct idr riids; 167 struct work_struct intent_work; 168 struct list_head done_intents; 169 170 struct glink_core_rx_intent *buf; 171 int buf_offset; 172 int buf_size; 173 174 struct completion open_ack; 175 struct completion open_req; 176 177 struct mutex intent_req_lock; 178 bool intent_req_result; 179 struct completion intent_req_comp; 180 }; 181 182 #define to_glink_channel(_ept) container_of(_ept, struct glink_channel, ept) 183 184 static const struct rpmsg_endpoint_ops glink_endpoint_ops; 185 186 #define RPM_CMD_VERSION 0 187 #define RPM_CMD_VERSION_ACK 1 188 #define RPM_CMD_OPEN 2 189 #define RPM_CMD_CLOSE 3 190 #define RPM_CMD_OPEN_ACK 4 191 #define RPM_CMD_INTENT 5 192 #define RPM_CMD_RX_DONE 6 193 #define RPM_CMD_RX_INTENT_REQ 7 194 #define RPM_CMD_RX_INTENT_REQ_ACK 8 195 #define RPM_CMD_TX_DATA 9 196 #define RPM_CMD_CLOSE_ACK 11 197 #define RPM_CMD_TX_DATA_CONT 12 198 #define RPM_CMD_READ_NOTIF 13 199 #define RPM_CMD_RX_DONE_W_REUSE 14 200 201 #define GLINK_FEATURE_INTENTLESS BIT(1) 202 203 static void qcom_glink_rx_done_work(struct work_struct *work); 204 205 static struct glink_channel *qcom_glink_alloc_channel(struct qcom_glink *glink, 206 const char *name) 207 { 208 struct glink_channel *channel; 209 210 channel = kzalloc(sizeof(*channel), GFP_KERNEL); 211 if (!channel) 212 return ERR_PTR(-ENOMEM); 213 214 /* Setup glink internal glink_channel data */ 215 spin_lock_init(&channel->recv_lock); 216 spin_lock_init(&channel->intent_lock); 217 mutex_init(&channel->intent_req_lock); 218 219 channel->glink = glink; 220 channel->name = kstrdup(name, GFP_KERNEL); 221 222 init_completion(&channel->open_req); 223 init_completion(&channel->open_ack); 224 init_completion(&channel->intent_req_comp); 225 226 INIT_LIST_HEAD(&channel->done_intents); 227 INIT_WORK(&channel->intent_work, qcom_glink_rx_done_work); 228 229 idr_init(&channel->liids); 230 idr_init(&channel->riids); 231 kref_init(&channel->refcount); 232 233 return channel; 234 } 235 236 static void qcom_glink_channel_release(struct kref *ref) 237 { 238 struct glink_channel *channel = container_of(ref, struct glink_channel, 239 refcount); 240 struct glink_core_rx_intent *intent; 241 struct glink_core_rx_intent *tmp; 242 unsigned long flags; 243 int iid; 244 245 /* cancel pending rx_done work */ 246 cancel_work_sync(&channel->intent_work); 247 248 spin_lock_irqsave(&channel->intent_lock, flags); 249 /* Free all non-reuse intents pending rx_done work */ 250 list_for_each_entry_safe(intent, tmp, &channel->done_intents, node) { 251 if (!intent->reuse) { 252 kfree(intent->data); 253 kfree(intent); 254 } 255 } 256 257 idr_for_each_entry(&channel->liids, tmp, iid) { 258 kfree(tmp->data); 259 kfree(tmp); 260 } 261 idr_destroy(&channel->liids); 262 263 idr_for_each_entry(&channel->riids, tmp, iid) 264 kfree(tmp); 265 idr_destroy(&channel->riids); 266 spin_unlock_irqrestore(&channel->intent_lock, flags); 267 268 kfree(channel->name); 269 kfree(channel); 270 } 271 272 static size_t qcom_glink_rx_avail(struct qcom_glink *glink) 273 { 274 return glink->rx_pipe->avail(glink->rx_pipe); 275 } 276 277 static void qcom_glink_rx_peak(struct qcom_glink *glink, 278 void *data, unsigned int offset, size_t count) 279 { 280 glink->rx_pipe->peak(glink->rx_pipe, data, offset, count); 281 } 282 283 static void qcom_glink_rx_advance(struct qcom_glink *glink, size_t count) 284 { 285 glink->rx_pipe->advance(glink->rx_pipe, count); 286 } 287 288 static size_t qcom_glink_tx_avail(struct qcom_glink *glink) 289 { 290 return glink->tx_pipe->avail(glink->tx_pipe); 291 } 292 293 static void qcom_glink_tx_write(struct qcom_glink *glink, 294 const void *hdr, size_t hlen, 295 const void *data, size_t dlen) 296 { 297 glink->tx_pipe->write(glink->tx_pipe, hdr, hlen, data, dlen); 298 } 299 300 static void qcom_glink_tx_kick(struct qcom_glink *glink) 301 { 302 glink->tx_pipe->kick(glink->tx_pipe); 303 } 304 305 static void qcom_glink_send_read_notify(struct qcom_glink *glink) 306 { 307 struct glink_msg msg; 308 309 msg.cmd = cpu_to_le16(RPM_CMD_READ_NOTIF); 310 msg.param1 = 0; 311 msg.param2 = 0; 312 313 qcom_glink_tx_write(glink, &msg, sizeof(msg), NULL, 0); 314 315 qcom_glink_tx_kick(glink); 316 } 317 318 static int qcom_glink_tx(struct qcom_glink *glink, 319 const void *hdr, size_t hlen, 320 const void *data, size_t dlen, bool wait) 321 { 322 unsigned int tlen = hlen + dlen; 323 unsigned long flags; 324 int ret = 0; 325 326 /* Reject packets that are too big */ 327 if (tlen >= glink->tx_pipe->length) 328 return -EINVAL; 329 330 spin_lock_irqsave(&glink->tx_lock, flags); 331 332 if (glink->abort_tx) { 333 ret = -EIO; 334 goto out; 335 } 336 337 while (qcom_glink_tx_avail(glink) < tlen) { 338 if (!wait) { 339 ret = -EAGAIN; 340 goto out; 341 } 342 343 if (glink->abort_tx) { 344 ret = -EIO; 345 goto out; 346 } 347 348 if (!glink->sent_read_notify) { 349 glink->sent_read_notify = true; 350 qcom_glink_send_read_notify(glink); 351 } 352 353 /* Wait without holding the tx_lock */ 354 spin_unlock_irqrestore(&glink->tx_lock, flags); 355 356 wait_event_timeout(glink->tx_avail_notify, 357 qcom_glink_tx_avail(glink) >= tlen, 10 * HZ); 358 359 spin_lock_irqsave(&glink->tx_lock, flags); 360 361 if (qcom_glink_tx_avail(glink) >= tlen) 362 glink->sent_read_notify = false; 363 } 364 365 qcom_glink_tx_write(glink, hdr, hlen, data, dlen); 366 qcom_glink_tx_kick(glink); 367 368 out: 369 spin_unlock_irqrestore(&glink->tx_lock, flags); 370 371 return ret; 372 } 373 374 static int qcom_glink_send_version(struct qcom_glink *glink) 375 { 376 struct glink_msg msg; 377 378 msg.cmd = cpu_to_le16(RPM_CMD_VERSION); 379 msg.param1 = cpu_to_le16(GLINK_VERSION_1); 380 msg.param2 = cpu_to_le32(glink->features); 381 382 return qcom_glink_tx(glink, &msg, sizeof(msg), NULL, 0, true); 383 } 384 385 static void qcom_glink_send_version_ack(struct qcom_glink *glink) 386 { 387 struct glink_msg msg; 388 389 msg.cmd = cpu_to_le16(RPM_CMD_VERSION_ACK); 390 msg.param1 = cpu_to_le16(GLINK_VERSION_1); 391 msg.param2 = cpu_to_le32(glink->features); 392 393 qcom_glink_tx(glink, &msg, sizeof(msg), NULL, 0, true); 394 } 395 396 static void qcom_glink_send_open_ack(struct qcom_glink *glink, 397 struct glink_channel *channel) 398 { 399 struct glink_msg msg; 400 401 msg.cmd = cpu_to_le16(RPM_CMD_OPEN_ACK); 402 msg.param1 = cpu_to_le16(channel->rcid); 403 msg.param2 = cpu_to_le32(0); 404 405 qcom_glink_tx(glink, &msg, sizeof(msg), NULL, 0, true); 406 } 407 408 static void qcom_glink_handle_intent_req_ack(struct qcom_glink *glink, 409 unsigned int cid, bool granted) 410 { 411 struct glink_channel *channel; 412 unsigned long flags; 413 414 spin_lock_irqsave(&glink->idr_lock, flags); 415 channel = idr_find(&glink->rcids, cid); 416 spin_unlock_irqrestore(&glink->idr_lock, flags); 417 if (!channel) { 418 dev_err(glink->dev, "unable to find channel\n"); 419 return; 420 } 421 422 channel->intent_req_result = granted; 423 complete(&channel->intent_req_comp); 424 } 425 426 /** 427 * qcom_glink_send_open_req() - send a RPM_CMD_OPEN request to the remote 428 * @glink: Ptr to the glink edge 429 * @channel: Ptr to the channel that the open req is sent 430 * 431 * Allocates a local channel id and sends a RPM_CMD_OPEN message to the remote. 432 * Will return with refcount held, regardless of outcome. 433 * 434 * Return: 0 on success, negative errno otherwise. 435 */ 436 static int qcom_glink_send_open_req(struct qcom_glink *glink, 437 struct glink_channel *channel) 438 { 439 struct { 440 struct glink_msg msg; 441 u8 name[GLINK_NAME_SIZE]; 442 } __packed req; 443 int name_len = strlen(channel->name) + 1; 444 int req_len = ALIGN(sizeof(req.msg) + name_len, 8); 445 int ret; 446 unsigned long flags; 447 448 kref_get(&channel->refcount); 449 450 spin_lock_irqsave(&glink->idr_lock, flags); 451 ret = idr_alloc_cyclic(&glink->lcids, channel, 452 RPM_GLINK_CID_MIN, RPM_GLINK_CID_MAX, 453 GFP_ATOMIC); 454 spin_unlock_irqrestore(&glink->idr_lock, flags); 455 if (ret < 0) 456 return ret; 457 458 channel->lcid = ret; 459 460 req.msg.cmd = cpu_to_le16(RPM_CMD_OPEN); 461 req.msg.param1 = cpu_to_le16(channel->lcid); 462 req.msg.param2 = cpu_to_le32(name_len); 463 strcpy(req.name, channel->name); 464 465 ret = qcom_glink_tx(glink, &req, req_len, NULL, 0, true); 466 if (ret) 467 goto remove_idr; 468 469 return 0; 470 471 remove_idr: 472 spin_lock_irqsave(&glink->idr_lock, flags); 473 idr_remove(&glink->lcids, channel->lcid); 474 channel->lcid = 0; 475 spin_unlock_irqrestore(&glink->idr_lock, flags); 476 477 return ret; 478 } 479 480 static void qcom_glink_send_close_req(struct qcom_glink *glink, 481 struct glink_channel *channel) 482 { 483 struct glink_msg req; 484 485 req.cmd = cpu_to_le16(RPM_CMD_CLOSE); 486 req.param1 = cpu_to_le16(channel->lcid); 487 req.param2 = 0; 488 489 qcom_glink_tx(glink, &req, sizeof(req), NULL, 0, true); 490 } 491 492 static void qcom_glink_send_close_ack(struct qcom_glink *glink, 493 unsigned int rcid) 494 { 495 struct glink_msg req; 496 497 req.cmd = cpu_to_le16(RPM_CMD_CLOSE_ACK); 498 req.param1 = cpu_to_le16(rcid); 499 req.param2 = 0; 500 501 qcom_glink_tx(glink, &req, sizeof(req), NULL, 0, true); 502 } 503 504 static void qcom_glink_rx_done_work(struct work_struct *work) 505 { 506 struct glink_channel *channel = container_of(work, struct glink_channel, 507 intent_work); 508 struct qcom_glink *glink = channel->glink; 509 struct glink_core_rx_intent *intent, *tmp; 510 struct { 511 u16 id; 512 u16 lcid; 513 u32 liid; 514 } __packed cmd; 515 516 unsigned int cid = channel->lcid; 517 unsigned int iid; 518 bool reuse; 519 unsigned long flags; 520 521 spin_lock_irqsave(&channel->intent_lock, flags); 522 list_for_each_entry_safe(intent, tmp, &channel->done_intents, node) { 523 list_del(&intent->node); 524 spin_unlock_irqrestore(&channel->intent_lock, flags); 525 iid = intent->id; 526 reuse = intent->reuse; 527 528 cmd.id = reuse ? RPM_CMD_RX_DONE_W_REUSE : RPM_CMD_RX_DONE; 529 cmd.lcid = cid; 530 cmd.liid = iid; 531 532 qcom_glink_tx(glink, &cmd, sizeof(cmd), NULL, 0, true); 533 if (!reuse) { 534 kfree(intent->data); 535 kfree(intent); 536 } 537 spin_lock_irqsave(&channel->intent_lock, flags); 538 } 539 spin_unlock_irqrestore(&channel->intent_lock, flags); 540 } 541 542 static void qcom_glink_rx_done(struct qcom_glink *glink, 543 struct glink_channel *channel, 544 struct glink_core_rx_intent *intent) 545 { 546 /* We don't send RX_DONE to intentless systems */ 547 if (glink->intentless) { 548 kfree(intent->data); 549 kfree(intent); 550 return; 551 } 552 553 /* Take it off the tree of receive intents */ 554 if (!intent->reuse) { 555 spin_lock(&channel->intent_lock); 556 idr_remove(&channel->liids, intent->id); 557 spin_unlock(&channel->intent_lock); 558 } 559 560 /* Schedule the sending of a rx_done indication */ 561 spin_lock(&channel->intent_lock); 562 list_add_tail(&intent->node, &channel->done_intents); 563 spin_unlock(&channel->intent_lock); 564 565 schedule_work(&channel->intent_work); 566 } 567 568 /** 569 * qcom_glink_receive_version() - receive version/features from remote system 570 * 571 * @glink: pointer to transport interface 572 * @version: remote version 573 * @features: remote features 574 * 575 * This function is called in response to a remote-initiated version/feature 576 * negotiation sequence. 577 */ 578 static void qcom_glink_receive_version(struct qcom_glink *glink, 579 u32 version, 580 u32 features) 581 { 582 switch (version) { 583 case 0: 584 break; 585 case GLINK_VERSION_1: 586 glink->features &= features; 587 fallthrough; 588 default: 589 qcom_glink_send_version_ack(glink); 590 break; 591 } 592 } 593 594 /** 595 * qcom_glink_receive_version_ack() - receive negotiation ack from remote system 596 * 597 * @glink: pointer to transport interface 598 * @version: remote version response 599 * @features: remote features response 600 * 601 * This function is called in response to a local-initiated version/feature 602 * negotiation sequence and is the counter-offer from the remote side based 603 * upon the initial version and feature set requested. 604 */ 605 static void qcom_glink_receive_version_ack(struct qcom_glink *glink, 606 u32 version, 607 u32 features) 608 { 609 switch (version) { 610 case 0: 611 /* Version negotiation failed */ 612 break; 613 case GLINK_VERSION_1: 614 if (features == glink->features) 615 break; 616 617 glink->features &= features; 618 fallthrough; 619 default: 620 qcom_glink_send_version(glink); 621 break; 622 } 623 } 624 625 /** 626 * qcom_glink_send_intent_req_ack() - convert an rx intent request ack cmd to 627 * wire format and transmit 628 * @glink: The transport to transmit on. 629 * @channel: The glink channel 630 * @granted: The request response to encode. 631 * 632 * Return: 0 on success or standard Linux error code. 633 */ 634 static int qcom_glink_send_intent_req_ack(struct qcom_glink *glink, 635 struct glink_channel *channel, 636 bool granted) 637 { 638 struct glink_msg msg; 639 640 msg.cmd = cpu_to_le16(RPM_CMD_RX_INTENT_REQ_ACK); 641 msg.param1 = cpu_to_le16(channel->lcid); 642 msg.param2 = cpu_to_le32(granted); 643 644 qcom_glink_tx(glink, &msg, sizeof(msg), NULL, 0, true); 645 646 return 0; 647 } 648 649 /** 650 * qcom_glink_advertise_intent - convert an rx intent cmd to wire format and 651 * transmit 652 * @glink: The transport to transmit on. 653 * @channel: The local channel 654 * @intent: The intent to pass on to remote. 655 * 656 * Return: 0 on success or standard Linux error code. 657 */ 658 static int qcom_glink_advertise_intent(struct qcom_glink *glink, 659 struct glink_channel *channel, 660 struct glink_core_rx_intent *intent) 661 { 662 struct command { 663 __le16 id; 664 __le16 lcid; 665 __le32 count; 666 __le32 size; 667 __le32 liid; 668 } __packed; 669 struct command cmd; 670 671 cmd.id = cpu_to_le16(RPM_CMD_INTENT); 672 cmd.lcid = cpu_to_le16(channel->lcid); 673 cmd.count = cpu_to_le32(1); 674 cmd.size = cpu_to_le32(intent->size); 675 cmd.liid = cpu_to_le32(intent->id); 676 677 qcom_glink_tx(glink, &cmd, sizeof(cmd), NULL, 0, true); 678 679 return 0; 680 } 681 682 static struct glink_core_rx_intent * 683 qcom_glink_alloc_intent(struct qcom_glink *glink, 684 struct glink_channel *channel, 685 size_t size, 686 bool reuseable) 687 { 688 struct glink_core_rx_intent *intent; 689 int ret; 690 unsigned long flags; 691 692 intent = kzalloc(sizeof(*intent), GFP_KERNEL); 693 if (!intent) 694 return NULL; 695 696 intent->data = kzalloc(size, GFP_KERNEL); 697 if (!intent->data) 698 goto free_intent; 699 700 spin_lock_irqsave(&channel->intent_lock, flags); 701 ret = idr_alloc_cyclic(&channel->liids, intent, 1, -1, GFP_ATOMIC); 702 if (ret < 0) { 703 spin_unlock_irqrestore(&channel->intent_lock, flags); 704 goto free_data; 705 } 706 spin_unlock_irqrestore(&channel->intent_lock, flags); 707 708 intent->id = ret; 709 intent->size = size; 710 intent->reuse = reuseable; 711 712 return intent; 713 714 free_data: 715 kfree(intent->data); 716 free_intent: 717 kfree(intent); 718 return NULL; 719 } 720 721 static void qcom_glink_handle_rx_done(struct qcom_glink *glink, 722 u32 cid, uint32_t iid, 723 bool reuse) 724 { 725 struct glink_core_rx_intent *intent; 726 struct glink_channel *channel; 727 unsigned long flags; 728 729 spin_lock_irqsave(&glink->idr_lock, flags); 730 channel = idr_find(&glink->rcids, cid); 731 spin_unlock_irqrestore(&glink->idr_lock, flags); 732 if (!channel) { 733 dev_err(glink->dev, "invalid channel id received\n"); 734 return; 735 } 736 737 spin_lock_irqsave(&channel->intent_lock, flags); 738 intent = idr_find(&channel->riids, iid); 739 740 if (!intent) { 741 spin_unlock_irqrestore(&channel->intent_lock, flags); 742 dev_err(glink->dev, "invalid intent id received\n"); 743 return; 744 } 745 746 intent->in_use = false; 747 748 if (!reuse) { 749 idr_remove(&channel->riids, intent->id); 750 kfree(intent); 751 } 752 spin_unlock_irqrestore(&channel->intent_lock, flags); 753 } 754 755 /** 756 * qcom_glink_handle_intent_req() - Receive a request for rx_intent 757 * from remote side 758 * @glink: Pointer to the transport interface 759 * @cid: Remote channel ID 760 * @size: size of the intent 761 * 762 * The function searches for the local channel to which the request for 763 * rx_intent has arrived and allocates and notifies the remote back 764 */ 765 static void qcom_glink_handle_intent_req(struct qcom_glink *glink, 766 u32 cid, size_t size) 767 { 768 struct glink_core_rx_intent *intent; 769 struct glink_channel *channel; 770 unsigned long flags; 771 772 spin_lock_irqsave(&glink->idr_lock, flags); 773 channel = idr_find(&glink->rcids, cid); 774 spin_unlock_irqrestore(&glink->idr_lock, flags); 775 776 if (!channel) { 777 pr_err("%s channel not found for cid %d\n", __func__, cid); 778 return; 779 } 780 781 intent = qcom_glink_alloc_intent(glink, channel, size, false); 782 if (intent) 783 qcom_glink_advertise_intent(glink, channel, intent); 784 785 qcom_glink_send_intent_req_ack(glink, channel, !!intent); 786 } 787 788 static int qcom_glink_rx_defer(struct qcom_glink *glink, size_t extra) 789 { 790 struct glink_defer_cmd *dcmd; 791 792 extra = ALIGN(extra, 8); 793 794 if (qcom_glink_rx_avail(glink) < sizeof(struct glink_msg) + extra) { 795 dev_dbg(glink->dev, "Insufficient data in rx fifo"); 796 return -ENXIO; 797 } 798 799 dcmd = kzalloc(struct_size(dcmd, data, extra), GFP_ATOMIC); 800 if (!dcmd) 801 return -ENOMEM; 802 803 INIT_LIST_HEAD(&dcmd->node); 804 805 qcom_glink_rx_peak(glink, &dcmd->msg, 0, sizeof(dcmd->msg) + extra); 806 807 spin_lock(&glink->rx_lock); 808 list_add_tail(&dcmd->node, &glink->rx_queue); 809 spin_unlock(&glink->rx_lock); 810 811 schedule_work(&glink->rx_work); 812 qcom_glink_rx_advance(glink, sizeof(dcmd->msg) + extra); 813 814 return 0; 815 } 816 817 static int qcom_glink_rx_data(struct qcom_glink *glink, size_t avail) 818 { 819 struct glink_core_rx_intent *intent; 820 struct glink_channel *channel; 821 struct { 822 struct glink_msg msg; 823 __le32 chunk_size; 824 __le32 left_size; 825 } __packed hdr; 826 unsigned int chunk_size; 827 unsigned int left_size; 828 unsigned int rcid; 829 unsigned int liid; 830 int ret = 0; 831 unsigned long flags; 832 833 if (avail < sizeof(hdr)) { 834 dev_dbg(glink->dev, "Not enough data in fifo\n"); 835 return -EAGAIN; 836 } 837 838 qcom_glink_rx_peak(glink, &hdr, 0, sizeof(hdr)); 839 chunk_size = le32_to_cpu(hdr.chunk_size); 840 left_size = le32_to_cpu(hdr.left_size); 841 842 if (avail < sizeof(hdr) + chunk_size) { 843 dev_dbg(glink->dev, "Payload not yet in fifo\n"); 844 return -EAGAIN; 845 } 846 847 rcid = le16_to_cpu(hdr.msg.param1); 848 spin_lock_irqsave(&glink->idr_lock, flags); 849 channel = idr_find(&glink->rcids, rcid); 850 spin_unlock_irqrestore(&glink->idr_lock, flags); 851 if (!channel) { 852 dev_dbg(glink->dev, "Data on non-existing channel\n"); 853 854 /* Drop the message */ 855 goto advance_rx; 856 } 857 858 if (glink->intentless) { 859 /* Might have an ongoing, fragmented, message to append */ 860 if (!channel->buf) { 861 intent = kzalloc(sizeof(*intent), GFP_ATOMIC); 862 if (!intent) 863 return -ENOMEM; 864 865 intent->data = kmalloc(chunk_size + left_size, 866 GFP_ATOMIC); 867 if (!intent->data) { 868 kfree(intent); 869 return -ENOMEM; 870 } 871 872 intent->id = 0xbabababa; 873 intent->size = chunk_size + left_size; 874 intent->offset = 0; 875 876 channel->buf = intent; 877 } else { 878 intent = channel->buf; 879 } 880 } else { 881 liid = le32_to_cpu(hdr.msg.param2); 882 883 spin_lock_irqsave(&channel->intent_lock, flags); 884 intent = idr_find(&channel->liids, liid); 885 spin_unlock_irqrestore(&channel->intent_lock, flags); 886 887 if (!intent) { 888 dev_err(glink->dev, 889 "no intent found for channel %s intent %d", 890 channel->name, liid); 891 ret = -ENOENT; 892 goto advance_rx; 893 } 894 } 895 896 if (intent->size - intent->offset < chunk_size) { 897 dev_err(glink->dev, "Insufficient space in intent\n"); 898 899 /* The packet header lied, drop payload */ 900 goto advance_rx; 901 } 902 903 qcom_glink_rx_peak(glink, intent->data + intent->offset, 904 sizeof(hdr), chunk_size); 905 intent->offset += chunk_size; 906 907 /* Handle message when no fragments remain to be received */ 908 if (!left_size) { 909 spin_lock(&channel->recv_lock); 910 if (channel->ept.cb) { 911 channel->ept.cb(channel->ept.rpdev, 912 intent->data, 913 intent->offset, 914 channel->ept.priv, 915 RPMSG_ADDR_ANY); 916 } 917 spin_unlock(&channel->recv_lock); 918 919 intent->offset = 0; 920 channel->buf = NULL; 921 922 qcom_glink_rx_done(glink, channel, intent); 923 } 924 925 advance_rx: 926 qcom_glink_rx_advance(glink, ALIGN(sizeof(hdr) + chunk_size, 8)); 927 928 return ret; 929 } 930 931 static void qcom_glink_handle_intent(struct qcom_glink *glink, 932 unsigned int cid, 933 unsigned int count, 934 size_t avail) 935 { 936 struct glink_core_rx_intent *intent; 937 struct glink_channel *channel; 938 struct intent_pair { 939 __le32 size; 940 __le32 iid; 941 }; 942 943 struct { 944 struct glink_msg msg; 945 struct intent_pair intents[]; 946 } __packed * msg; 947 948 const size_t msglen = struct_size(msg, intents, count); 949 int ret; 950 int i; 951 unsigned long flags; 952 953 if (avail < msglen) { 954 dev_dbg(glink->dev, "Not enough data in fifo\n"); 955 return; 956 } 957 958 spin_lock_irqsave(&glink->idr_lock, flags); 959 channel = idr_find(&glink->rcids, cid); 960 spin_unlock_irqrestore(&glink->idr_lock, flags); 961 if (!channel) { 962 dev_err(glink->dev, "intents for non-existing channel\n"); 963 return; 964 } 965 966 msg = kmalloc(msglen, GFP_ATOMIC); 967 if (!msg) 968 return; 969 970 qcom_glink_rx_peak(glink, msg, 0, msglen); 971 972 for (i = 0; i < count; ++i) { 973 intent = kzalloc(sizeof(*intent), GFP_ATOMIC); 974 if (!intent) 975 break; 976 977 intent->id = le32_to_cpu(msg->intents[i].iid); 978 intent->size = le32_to_cpu(msg->intents[i].size); 979 980 spin_lock_irqsave(&channel->intent_lock, flags); 981 ret = idr_alloc(&channel->riids, intent, 982 intent->id, intent->id + 1, GFP_ATOMIC); 983 spin_unlock_irqrestore(&channel->intent_lock, flags); 984 985 if (ret < 0) 986 dev_err(glink->dev, "failed to store remote intent\n"); 987 } 988 989 kfree(msg); 990 qcom_glink_rx_advance(glink, ALIGN(msglen, 8)); 991 } 992 993 static int qcom_glink_rx_open_ack(struct qcom_glink *glink, unsigned int lcid) 994 { 995 struct glink_channel *channel; 996 997 spin_lock(&glink->idr_lock); 998 channel = idr_find(&glink->lcids, lcid); 999 spin_unlock(&glink->idr_lock); 1000 if (!channel) { 1001 dev_err(glink->dev, "Invalid open ack packet\n"); 1002 return -EINVAL; 1003 } 1004 1005 complete_all(&channel->open_ack); 1006 1007 return 0; 1008 } 1009 1010 void qcom_glink_native_rx(struct qcom_glink *glink) 1011 { 1012 struct glink_msg msg; 1013 unsigned int param1; 1014 unsigned int param2; 1015 unsigned int avail; 1016 unsigned int cmd; 1017 int ret = 0; 1018 1019 /* To wakeup any blocking writers */ 1020 wake_up_all(&glink->tx_avail_notify); 1021 1022 for (;;) { 1023 avail = qcom_glink_rx_avail(glink); 1024 if (avail < sizeof(msg)) 1025 break; 1026 1027 qcom_glink_rx_peak(glink, &msg, 0, sizeof(msg)); 1028 1029 cmd = le16_to_cpu(msg.cmd); 1030 param1 = le16_to_cpu(msg.param1); 1031 param2 = le32_to_cpu(msg.param2); 1032 1033 switch (cmd) { 1034 case RPM_CMD_VERSION: 1035 case RPM_CMD_VERSION_ACK: 1036 case RPM_CMD_CLOSE: 1037 case RPM_CMD_CLOSE_ACK: 1038 case RPM_CMD_RX_INTENT_REQ: 1039 ret = qcom_glink_rx_defer(glink, 0); 1040 break; 1041 case RPM_CMD_OPEN_ACK: 1042 ret = qcom_glink_rx_open_ack(glink, param1); 1043 qcom_glink_rx_advance(glink, ALIGN(sizeof(msg), 8)); 1044 break; 1045 case RPM_CMD_OPEN: 1046 ret = qcom_glink_rx_defer(glink, param2); 1047 break; 1048 case RPM_CMD_TX_DATA: 1049 case RPM_CMD_TX_DATA_CONT: 1050 ret = qcom_glink_rx_data(glink, avail); 1051 break; 1052 case RPM_CMD_READ_NOTIF: 1053 qcom_glink_rx_advance(glink, ALIGN(sizeof(msg), 8)); 1054 qcom_glink_tx_kick(glink); 1055 break; 1056 case RPM_CMD_INTENT: 1057 qcom_glink_handle_intent(glink, param1, param2, avail); 1058 break; 1059 case RPM_CMD_RX_DONE: 1060 qcom_glink_handle_rx_done(glink, param1, param2, false); 1061 qcom_glink_rx_advance(glink, ALIGN(sizeof(msg), 8)); 1062 break; 1063 case RPM_CMD_RX_DONE_W_REUSE: 1064 qcom_glink_handle_rx_done(glink, param1, param2, true); 1065 qcom_glink_rx_advance(glink, ALIGN(sizeof(msg), 8)); 1066 break; 1067 case RPM_CMD_RX_INTENT_REQ_ACK: 1068 qcom_glink_handle_intent_req_ack(glink, param1, param2); 1069 qcom_glink_rx_advance(glink, ALIGN(sizeof(msg), 8)); 1070 break; 1071 default: 1072 dev_err(glink->dev, "unhandled rx cmd: %d\n", cmd); 1073 ret = -EINVAL; 1074 break; 1075 } 1076 1077 if (ret) 1078 break; 1079 } 1080 } 1081 EXPORT_SYMBOL(qcom_glink_native_rx); 1082 1083 /* Locally initiated rpmsg_create_ept */ 1084 static struct glink_channel *qcom_glink_create_local(struct qcom_glink *glink, 1085 const char *name) 1086 { 1087 struct glink_channel *channel; 1088 int ret; 1089 unsigned long flags; 1090 1091 channel = qcom_glink_alloc_channel(glink, name); 1092 if (IS_ERR(channel)) 1093 return ERR_CAST(channel); 1094 1095 ret = qcom_glink_send_open_req(glink, channel); 1096 if (ret) 1097 goto release_channel; 1098 1099 ret = wait_for_completion_timeout(&channel->open_ack, 5 * HZ); 1100 if (!ret) 1101 goto err_timeout; 1102 1103 ret = wait_for_completion_timeout(&channel->open_req, 5 * HZ); 1104 if (!ret) 1105 goto err_timeout; 1106 1107 qcom_glink_send_open_ack(glink, channel); 1108 1109 return channel; 1110 1111 err_timeout: 1112 /* qcom_glink_send_open_req() did register the channel in lcids*/ 1113 spin_lock_irqsave(&glink->idr_lock, flags); 1114 idr_remove(&glink->lcids, channel->lcid); 1115 spin_unlock_irqrestore(&glink->idr_lock, flags); 1116 1117 release_channel: 1118 /* Release qcom_glink_send_open_req() reference */ 1119 kref_put(&channel->refcount, qcom_glink_channel_release); 1120 /* Release qcom_glink_alloc_channel() reference */ 1121 kref_put(&channel->refcount, qcom_glink_channel_release); 1122 1123 return ERR_PTR(-ETIMEDOUT); 1124 } 1125 1126 /* Remote initiated rpmsg_create_ept */ 1127 static int qcom_glink_create_remote(struct qcom_glink *glink, 1128 struct glink_channel *channel) 1129 { 1130 int ret; 1131 1132 qcom_glink_send_open_ack(glink, channel); 1133 1134 ret = qcom_glink_send_open_req(glink, channel); 1135 if (ret) 1136 goto close_link; 1137 1138 ret = wait_for_completion_timeout(&channel->open_ack, 5 * HZ); 1139 if (!ret) { 1140 ret = -ETIMEDOUT; 1141 goto close_link; 1142 } 1143 1144 return 0; 1145 1146 close_link: 1147 /* 1148 * Send a close request to "undo" our open-ack. The close-ack will 1149 * release qcom_glink_send_open_req() reference and the last reference 1150 * will be relesed after receiving remote_close or transport unregister 1151 * by calling qcom_glink_native_remove(). 1152 */ 1153 qcom_glink_send_close_req(glink, channel); 1154 1155 return ret; 1156 } 1157 1158 static struct rpmsg_endpoint *qcom_glink_create_ept(struct rpmsg_device *rpdev, 1159 rpmsg_rx_cb_t cb, 1160 void *priv, 1161 struct rpmsg_channel_info 1162 chinfo) 1163 { 1164 struct glink_channel *parent = to_glink_channel(rpdev->ept); 1165 struct glink_channel *channel; 1166 struct qcom_glink *glink = parent->glink; 1167 struct rpmsg_endpoint *ept; 1168 const char *name = chinfo.name; 1169 int cid; 1170 int ret; 1171 unsigned long flags; 1172 1173 spin_lock_irqsave(&glink->idr_lock, flags); 1174 idr_for_each_entry(&glink->rcids, channel, cid) { 1175 if (!strcmp(channel->name, name)) 1176 break; 1177 } 1178 spin_unlock_irqrestore(&glink->idr_lock, flags); 1179 1180 if (!channel) { 1181 channel = qcom_glink_create_local(glink, name); 1182 if (IS_ERR(channel)) 1183 return NULL; 1184 } else { 1185 ret = qcom_glink_create_remote(glink, channel); 1186 if (ret) 1187 return NULL; 1188 } 1189 1190 ept = &channel->ept; 1191 ept->rpdev = rpdev; 1192 ept->cb = cb; 1193 ept->priv = priv; 1194 ept->ops = &glink_endpoint_ops; 1195 1196 return ept; 1197 } 1198 1199 static int qcom_glink_announce_create(struct rpmsg_device *rpdev) 1200 { 1201 struct glink_channel *channel = to_glink_channel(rpdev->ept); 1202 struct device_node *np = rpdev->dev.of_node; 1203 struct qcom_glink *glink = channel->glink; 1204 struct glink_core_rx_intent *intent; 1205 const struct property *prop = NULL; 1206 __be32 defaults[] = { cpu_to_be32(SZ_1K), cpu_to_be32(5) }; 1207 int num_intents; 1208 int num_groups = 1; 1209 __be32 *val = defaults; 1210 int size; 1211 1212 if (glink->intentless || !completion_done(&channel->open_ack)) 1213 return 0; 1214 1215 prop = of_find_property(np, "qcom,intents", NULL); 1216 if (prop) { 1217 val = prop->value; 1218 num_groups = prop->length / sizeof(u32) / 2; 1219 } 1220 1221 /* Channel is now open, advertise base set of intents */ 1222 while (num_groups--) { 1223 size = be32_to_cpup(val++); 1224 num_intents = be32_to_cpup(val++); 1225 while (num_intents--) { 1226 intent = qcom_glink_alloc_intent(glink, channel, size, 1227 true); 1228 if (!intent) 1229 break; 1230 1231 qcom_glink_advertise_intent(glink, channel, intent); 1232 } 1233 } 1234 return 0; 1235 } 1236 1237 static void qcom_glink_destroy_ept(struct rpmsg_endpoint *ept) 1238 { 1239 struct glink_channel *channel = to_glink_channel(ept); 1240 struct qcom_glink *glink = channel->glink; 1241 unsigned long flags; 1242 1243 spin_lock_irqsave(&channel->recv_lock, flags); 1244 channel->ept.cb = NULL; 1245 spin_unlock_irqrestore(&channel->recv_lock, flags); 1246 1247 /* Decouple the potential rpdev from the channel */ 1248 channel->rpdev = NULL; 1249 1250 qcom_glink_send_close_req(glink, channel); 1251 } 1252 1253 static int qcom_glink_request_intent(struct qcom_glink *glink, 1254 struct glink_channel *channel, 1255 size_t size) 1256 { 1257 struct { 1258 u16 id; 1259 u16 cid; 1260 u32 size; 1261 } __packed cmd; 1262 1263 int ret; 1264 1265 mutex_lock(&channel->intent_req_lock); 1266 1267 reinit_completion(&channel->intent_req_comp); 1268 1269 cmd.id = RPM_CMD_RX_INTENT_REQ; 1270 cmd.cid = channel->lcid; 1271 cmd.size = size; 1272 1273 ret = qcom_glink_tx(glink, &cmd, sizeof(cmd), NULL, 0, true); 1274 if (ret) 1275 goto unlock; 1276 1277 ret = wait_for_completion_timeout(&channel->intent_req_comp, 10 * HZ); 1278 if (!ret) { 1279 dev_err(glink->dev, "intent request timed out\n"); 1280 ret = -ETIMEDOUT; 1281 } else { 1282 ret = channel->intent_req_result ? 0 : -ECANCELED; 1283 } 1284 1285 unlock: 1286 mutex_unlock(&channel->intent_req_lock); 1287 return ret; 1288 } 1289 1290 static int __qcom_glink_send(struct glink_channel *channel, 1291 void *data, int len, bool wait) 1292 { 1293 struct qcom_glink *glink = channel->glink; 1294 struct glink_core_rx_intent *intent = NULL; 1295 struct glink_core_rx_intent *tmp; 1296 int iid = 0; 1297 struct { 1298 struct glink_msg msg; 1299 __le32 chunk_size; 1300 __le32 left_size; 1301 } __packed req; 1302 int ret; 1303 unsigned long flags; 1304 int chunk_size = len; 1305 int left_size = 0; 1306 1307 if (!glink->intentless) { 1308 while (!intent) { 1309 spin_lock_irqsave(&channel->intent_lock, flags); 1310 idr_for_each_entry(&channel->riids, tmp, iid) { 1311 if (tmp->size >= len && !tmp->in_use) { 1312 if (!intent) 1313 intent = tmp; 1314 else if (intent->size > tmp->size) 1315 intent = tmp; 1316 if (intent->size == len) 1317 break; 1318 } 1319 } 1320 if (intent) 1321 intent->in_use = true; 1322 spin_unlock_irqrestore(&channel->intent_lock, flags); 1323 1324 /* We found an available intent */ 1325 if (intent) 1326 break; 1327 1328 if (!wait) 1329 return -EBUSY; 1330 1331 ret = qcom_glink_request_intent(glink, channel, len); 1332 if (ret < 0) 1333 return ret; 1334 } 1335 1336 iid = intent->id; 1337 } 1338 1339 if (wait && chunk_size > SZ_8K) { 1340 chunk_size = SZ_8K; 1341 left_size = len - chunk_size; 1342 } 1343 req.msg.cmd = cpu_to_le16(RPM_CMD_TX_DATA); 1344 req.msg.param1 = cpu_to_le16(channel->lcid); 1345 req.msg.param2 = cpu_to_le32(iid); 1346 req.chunk_size = cpu_to_le32(chunk_size); 1347 req.left_size = cpu_to_le32(left_size); 1348 1349 ret = qcom_glink_tx(glink, &req, sizeof(req), data, chunk_size, wait); 1350 1351 /* Mark intent available if we failed */ 1352 if (ret && intent) { 1353 intent->in_use = false; 1354 return ret; 1355 } 1356 1357 while (left_size > 0) { 1358 data = (void *)((char *)data + chunk_size); 1359 chunk_size = left_size; 1360 if (chunk_size > SZ_8K) 1361 chunk_size = SZ_8K; 1362 left_size -= chunk_size; 1363 1364 req.msg.cmd = cpu_to_le16(RPM_CMD_TX_DATA_CONT); 1365 req.msg.param1 = cpu_to_le16(channel->lcid); 1366 req.msg.param2 = cpu_to_le32(iid); 1367 req.chunk_size = cpu_to_le32(chunk_size); 1368 req.left_size = cpu_to_le32(left_size); 1369 1370 ret = qcom_glink_tx(glink, &req, sizeof(req), data, 1371 chunk_size, wait); 1372 1373 /* Mark intent available if we failed */ 1374 if (ret && intent) { 1375 intent->in_use = false; 1376 break; 1377 } 1378 } 1379 return ret; 1380 } 1381 1382 static int qcom_glink_send(struct rpmsg_endpoint *ept, void *data, int len) 1383 { 1384 struct glink_channel *channel = to_glink_channel(ept); 1385 1386 return __qcom_glink_send(channel, data, len, true); 1387 } 1388 1389 static int qcom_glink_trysend(struct rpmsg_endpoint *ept, void *data, int len) 1390 { 1391 struct glink_channel *channel = to_glink_channel(ept); 1392 1393 return __qcom_glink_send(channel, data, len, false); 1394 } 1395 1396 static int qcom_glink_sendto(struct rpmsg_endpoint *ept, void *data, int len, u32 dst) 1397 { 1398 struct glink_channel *channel = to_glink_channel(ept); 1399 1400 return __qcom_glink_send(channel, data, len, true); 1401 } 1402 1403 static int qcom_glink_trysendto(struct rpmsg_endpoint *ept, void *data, int len, u32 dst) 1404 { 1405 struct glink_channel *channel = to_glink_channel(ept); 1406 1407 return __qcom_glink_send(channel, data, len, false); 1408 } 1409 1410 /* 1411 * Finds the device_node for the glink child interested in this channel. 1412 */ 1413 static struct device_node *qcom_glink_match_channel(struct device_node *node, 1414 const char *channel) 1415 { 1416 struct device_node *child; 1417 const char *name; 1418 const char *key; 1419 int ret; 1420 1421 for_each_available_child_of_node(node, child) { 1422 key = "qcom,glink-channels"; 1423 ret = of_property_read_string(child, key, &name); 1424 if (ret) 1425 continue; 1426 1427 if (strcmp(name, channel) == 0) 1428 return child; 1429 } 1430 1431 return NULL; 1432 } 1433 1434 static const struct rpmsg_device_ops glink_device_ops = { 1435 .create_ept = qcom_glink_create_ept, 1436 .announce_create = qcom_glink_announce_create, 1437 }; 1438 1439 static const struct rpmsg_endpoint_ops glink_endpoint_ops = { 1440 .destroy_ept = qcom_glink_destroy_ept, 1441 .send = qcom_glink_send, 1442 .sendto = qcom_glink_sendto, 1443 .trysend = qcom_glink_trysend, 1444 .trysendto = qcom_glink_trysendto, 1445 }; 1446 1447 static void qcom_glink_rpdev_release(struct device *dev) 1448 { 1449 struct rpmsg_device *rpdev = to_rpmsg_device(dev); 1450 1451 kfree(rpdev); 1452 } 1453 1454 static int qcom_glink_rx_open(struct qcom_glink *glink, unsigned int rcid, 1455 char *name) 1456 { 1457 struct glink_channel *channel; 1458 struct rpmsg_device *rpdev; 1459 bool create_device = false; 1460 struct device_node *node; 1461 int lcid; 1462 int ret; 1463 unsigned long flags; 1464 1465 spin_lock_irqsave(&glink->idr_lock, flags); 1466 idr_for_each_entry(&glink->lcids, channel, lcid) { 1467 if (!strcmp(channel->name, name)) 1468 break; 1469 } 1470 spin_unlock_irqrestore(&glink->idr_lock, flags); 1471 1472 if (!channel) { 1473 channel = qcom_glink_alloc_channel(glink, name); 1474 if (IS_ERR(channel)) 1475 return PTR_ERR(channel); 1476 1477 /* The opening dance was initiated by the remote */ 1478 create_device = true; 1479 } 1480 1481 spin_lock_irqsave(&glink->idr_lock, flags); 1482 ret = idr_alloc(&glink->rcids, channel, rcid, rcid + 1, GFP_ATOMIC); 1483 if (ret < 0) { 1484 dev_err(glink->dev, "Unable to insert channel into rcid list\n"); 1485 spin_unlock_irqrestore(&glink->idr_lock, flags); 1486 goto free_channel; 1487 } 1488 channel->rcid = ret; 1489 spin_unlock_irqrestore(&glink->idr_lock, flags); 1490 1491 complete_all(&channel->open_req); 1492 1493 if (create_device) { 1494 rpdev = kzalloc(sizeof(*rpdev), GFP_KERNEL); 1495 if (!rpdev) { 1496 ret = -ENOMEM; 1497 goto rcid_remove; 1498 } 1499 1500 rpdev->ept = &channel->ept; 1501 strscpy_pad(rpdev->id.name, name, RPMSG_NAME_SIZE); 1502 rpdev->src = RPMSG_ADDR_ANY; 1503 rpdev->dst = RPMSG_ADDR_ANY; 1504 rpdev->ops = &glink_device_ops; 1505 1506 node = qcom_glink_match_channel(glink->dev->of_node, name); 1507 rpdev->dev.of_node = node; 1508 rpdev->dev.parent = glink->dev; 1509 rpdev->dev.release = qcom_glink_rpdev_release; 1510 1511 ret = rpmsg_register_device(rpdev); 1512 if (ret) 1513 goto rcid_remove; 1514 1515 channel->rpdev = rpdev; 1516 } 1517 1518 return 0; 1519 1520 rcid_remove: 1521 spin_lock_irqsave(&glink->idr_lock, flags); 1522 idr_remove(&glink->rcids, channel->rcid); 1523 channel->rcid = 0; 1524 spin_unlock_irqrestore(&glink->idr_lock, flags); 1525 free_channel: 1526 /* Release the reference, iff we took it */ 1527 if (create_device) 1528 kref_put(&channel->refcount, qcom_glink_channel_release); 1529 1530 return ret; 1531 } 1532 1533 static void qcom_glink_rx_close(struct qcom_glink *glink, unsigned int rcid) 1534 { 1535 struct rpmsg_channel_info chinfo; 1536 struct glink_channel *channel; 1537 unsigned long flags; 1538 1539 spin_lock_irqsave(&glink->idr_lock, flags); 1540 channel = idr_find(&glink->rcids, rcid); 1541 spin_unlock_irqrestore(&glink->idr_lock, flags); 1542 if (WARN(!channel, "close request on unknown channel\n")) 1543 return; 1544 1545 /* cancel pending rx_done work */ 1546 cancel_work_sync(&channel->intent_work); 1547 1548 if (channel->rpdev) { 1549 strscpy_pad(chinfo.name, channel->name, sizeof(chinfo.name)); 1550 chinfo.src = RPMSG_ADDR_ANY; 1551 chinfo.dst = RPMSG_ADDR_ANY; 1552 1553 rpmsg_unregister_device(glink->dev, &chinfo); 1554 } 1555 channel->rpdev = NULL; 1556 1557 qcom_glink_send_close_ack(glink, channel->rcid); 1558 1559 spin_lock_irqsave(&glink->idr_lock, flags); 1560 idr_remove(&glink->rcids, channel->rcid); 1561 channel->rcid = 0; 1562 spin_unlock_irqrestore(&glink->idr_lock, flags); 1563 1564 kref_put(&channel->refcount, qcom_glink_channel_release); 1565 } 1566 1567 static void qcom_glink_rx_close_ack(struct qcom_glink *glink, unsigned int lcid) 1568 { 1569 struct rpmsg_channel_info chinfo; 1570 struct glink_channel *channel; 1571 unsigned long flags; 1572 1573 /* To wakeup any blocking writers */ 1574 wake_up_all(&glink->tx_avail_notify); 1575 1576 spin_lock_irqsave(&glink->idr_lock, flags); 1577 channel = idr_find(&glink->lcids, lcid); 1578 if (WARN(!channel, "close ack on unknown channel\n")) { 1579 spin_unlock_irqrestore(&glink->idr_lock, flags); 1580 return; 1581 } 1582 1583 idr_remove(&glink->lcids, channel->lcid); 1584 channel->lcid = 0; 1585 spin_unlock_irqrestore(&glink->idr_lock, flags); 1586 1587 /* Decouple the potential rpdev from the channel */ 1588 if (channel->rpdev) { 1589 strscpy(chinfo.name, channel->name, sizeof(chinfo.name)); 1590 chinfo.src = RPMSG_ADDR_ANY; 1591 chinfo.dst = RPMSG_ADDR_ANY; 1592 1593 rpmsg_unregister_device(glink->dev, &chinfo); 1594 } 1595 channel->rpdev = NULL; 1596 1597 kref_put(&channel->refcount, qcom_glink_channel_release); 1598 } 1599 1600 static void qcom_glink_work(struct work_struct *work) 1601 { 1602 struct qcom_glink *glink = container_of(work, struct qcom_glink, 1603 rx_work); 1604 struct glink_defer_cmd *dcmd; 1605 struct glink_msg *msg; 1606 unsigned long flags; 1607 unsigned int param1; 1608 unsigned int param2; 1609 unsigned int cmd; 1610 1611 for (;;) { 1612 spin_lock_irqsave(&glink->rx_lock, flags); 1613 if (list_empty(&glink->rx_queue)) { 1614 spin_unlock_irqrestore(&glink->rx_lock, flags); 1615 break; 1616 } 1617 dcmd = list_first_entry(&glink->rx_queue, 1618 struct glink_defer_cmd, node); 1619 list_del(&dcmd->node); 1620 spin_unlock_irqrestore(&glink->rx_lock, flags); 1621 1622 msg = &dcmd->msg; 1623 cmd = le16_to_cpu(msg->cmd); 1624 param1 = le16_to_cpu(msg->param1); 1625 param2 = le32_to_cpu(msg->param2); 1626 1627 switch (cmd) { 1628 case RPM_CMD_VERSION: 1629 qcom_glink_receive_version(glink, param1, param2); 1630 break; 1631 case RPM_CMD_VERSION_ACK: 1632 qcom_glink_receive_version_ack(glink, param1, param2); 1633 break; 1634 case RPM_CMD_OPEN: 1635 qcom_glink_rx_open(glink, param1, msg->data); 1636 break; 1637 case RPM_CMD_CLOSE: 1638 qcom_glink_rx_close(glink, param1); 1639 break; 1640 case RPM_CMD_CLOSE_ACK: 1641 qcom_glink_rx_close_ack(glink, param1); 1642 break; 1643 case RPM_CMD_RX_INTENT_REQ: 1644 qcom_glink_handle_intent_req(glink, param1, param2); 1645 break; 1646 default: 1647 WARN(1, "Unknown defer object %d\n", cmd); 1648 break; 1649 } 1650 1651 kfree(dcmd); 1652 } 1653 } 1654 1655 static void qcom_glink_cancel_rx_work(struct qcom_glink *glink) 1656 { 1657 struct glink_defer_cmd *dcmd; 1658 struct glink_defer_cmd *tmp; 1659 1660 /* cancel any pending deferred rx_work */ 1661 cancel_work_sync(&glink->rx_work); 1662 1663 list_for_each_entry_safe(dcmd, tmp, &glink->rx_queue, node) 1664 kfree(dcmd); 1665 } 1666 1667 static ssize_t rpmsg_name_show(struct device *dev, 1668 struct device_attribute *attr, char *buf) 1669 { 1670 int ret = 0; 1671 const char *name; 1672 1673 ret = of_property_read_string(dev->of_node, "label", &name); 1674 if (ret < 0) 1675 name = dev->of_node->name; 1676 1677 return sysfs_emit(buf, "%s\n", name); 1678 } 1679 static DEVICE_ATTR_RO(rpmsg_name); 1680 1681 static struct attribute *qcom_glink_attrs[] = { 1682 &dev_attr_rpmsg_name.attr, 1683 NULL 1684 }; 1685 ATTRIBUTE_GROUPS(qcom_glink); 1686 1687 static void qcom_glink_device_release(struct device *dev) 1688 { 1689 struct rpmsg_device *rpdev = to_rpmsg_device(dev); 1690 struct glink_channel *channel = to_glink_channel(rpdev->ept); 1691 1692 /* Release qcom_glink_alloc_channel() reference */ 1693 kref_put(&channel->refcount, qcom_glink_channel_release); 1694 kfree(rpdev); 1695 } 1696 1697 static int qcom_glink_create_chrdev(struct qcom_glink *glink) 1698 { 1699 struct rpmsg_device *rpdev; 1700 struct glink_channel *channel; 1701 1702 rpdev = kzalloc(sizeof(*rpdev), GFP_KERNEL); 1703 if (!rpdev) 1704 return -ENOMEM; 1705 1706 channel = qcom_glink_alloc_channel(glink, "rpmsg_chrdev"); 1707 if (IS_ERR(channel)) { 1708 kfree(rpdev); 1709 return PTR_ERR(channel); 1710 } 1711 channel->rpdev = rpdev; 1712 1713 rpdev->ept = &channel->ept; 1714 rpdev->ops = &glink_device_ops; 1715 rpdev->dev.parent = glink->dev; 1716 rpdev->dev.release = qcom_glink_device_release; 1717 1718 return rpmsg_ctrldev_register_device(rpdev); 1719 } 1720 1721 struct qcom_glink *qcom_glink_native_probe(struct device *dev, 1722 unsigned long features, 1723 struct qcom_glink_pipe *rx, 1724 struct qcom_glink_pipe *tx, 1725 bool intentless) 1726 { 1727 int ret; 1728 struct qcom_glink *glink; 1729 1730 glink = devm_kzalloc(dev, sizeof(*glink), GFP_KERNEL); 1731 if (!glink) 1732 return ERR_PTR(-ENOMEM); 1733 1734 glink->dev = dev; 1735 glink->tx_pipe = tx; 1736 glink->rx_pipe = rx; 1737 1738 glink->features = features; 1739 glink->intentless = intentless; 1740 1741 spin_lock_init(&glink->tx_lock); 1742 spin_lock_init(&glink->rx_lock); 1743 INIT_LIST_HEAD(&glink->rx_queue); 1744 INIT_WORK(&glink->rx_work, qcom_glink_work); 1745 init_waitqueue_head(&glink->tx_avail_notify); 1746 1747 spin_lock_init(&glink->idr_lock); 1748 idr_init(&glink->lcids); 1749 idr_init(&glink->rcids); 1750 1751 glink->dev->groups = qcom_glink_groups; 1752 1753 ret = device_add_groups(dev, qcom_glink_groups); 1754 if (ret) 1755 dev_err(dev, "failed to add groups\n"); 1756 1757 ret = qcom_glink_send_version(glink); 1758 if (ret) 1759 return ERR_PTR(ret); 1760 1761 ret = qcom_glink_create_chrdev(glink); 1762 if (ret) 1763 dev_err(glink->dev, "failed to register chrdev\n"); 1764 1765 return glink; 1766 } 1767 EXPORT_SYMBOL_GPL(qcom_glink_native_probe); 1768 1769 static int qcom_glink_remove_device(struct device *dev, void *data) 1770 { 1771 device_unregister(dev); 1772 1773 return 0; 1774 } 1775 1776 void qcom_glink_native_remove(struct qcom_glink *glink) 1777 { 1778 struct glink_channel *channel; 1779 unsigned long flags; 1780 int cid; 1781 int ret; 1782 1783 qcom_glink_cancel_rx_work(glink); 1784 1785 /* Fail all attempts at sending messages */ 1786 spin_lock_irqsave(&glink->tx_lock, flags); 1787 glink->abort_tx = true; 1788 wake_up_all(&glink->tx_avail_notify); 1789 spin_unlock_irqrestore(&glink->tx_lock, flags); 1790 1791 ret = device_for_each_child(glink->dev, NULL, qcom_glink_remove_device); 1792 if (ret) 1793 dev_warn(glink->dev, "Can't remove GLINK devices: %d\n", ret); 1794 1795 /* Release any defunct local channels, waiting for close-ack */ 1796 idr_for_each_entry(&glink->lcids, channel, cid) 1797 kref_put(&channel->refcount, qcom_glink_channel_release); 1798 1799 /* Release any defunct local channels, waiting for close-req */ 1800 idr_for_each_entry(&glink->rcids, channel, cid) 1801 kref_put(&channel->refcount, qcom_glink_channel_release); 1802 1803 idr_destroy(&glink->lcids); 1804 idr_destroy(&glink->rcids); 1805 } 1806 EXPORT_SYMBOL_GPL(qcom_glink_native_remove); 1807 1808 MODULE_DESCRIPTION("Qualcomm GLINK driver"); 1809 MODULE_LICENSE("GPL v2"); 1810