1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * Copyright (c) 2016-2017, Linaro Ltd 4 */ 5 6 #include <linux/idr.h> 7 #include <linux/interrupt.h> 8 #include <linux/io.h> 9 #include <linux/list.h> 10 #include <linux/mfd/syscon.h> 11 #include <linux/module.h> 12 #include <linux/of.h> 13 #include <linux/of_address.h> 14 #include <linux/platform_device.h> 15 #include <linux/regmap.h> 16 #include <linux/rpmsg.h> 17 #include <linux/sizes.h> 18 #include <linux/slab.h> 19 #include <linux/wait.h> 20 #include <linux/workqueue.h> 21 #include <linux/mailbox_client.h> 22 23 #include "rpmsg_internal.h" 24 #include "qcom_glink_native.h" 25 26 #define GLINK_NAME_SIZE 32 27 #define GLINK_VERSION_1 1 28 29 #define RPM_GLINK_CID_MIN 1 30 #define RPM_GLINK_CID_MAX 65536 31 32 struct glink_msg { 33 __le16 cmd; 34 __le16 param1; 35 __le32 param2; 36 u8 data[]; 37 } __packed; 38 39 /** 40 * struct glink_defer_cmd - deferred incoming control message 41 * @node: list node 42 * @msg: message header 43 * @data: payload of the message 44 * 45 * Copy of a received control message, to be added to @rx_queue and processed 46 * by @rx_work of @qcom_glink. 47 */ 48 struct glink_defer_cmd { 49 struct list_head node; 50 51 struct glink_msg msg; 52 u8 data[]; 53 }; 54 55 /** 56 * struct glink_core_rx_intent - RX intent 57 * RX intent 58 * 59 * @data: pointer to the data (may be NULL for zero-copy) 60 * @id: remote or local intent ID 61 * @size: size of the original intent (do not modify) 62 * @reuse: To mark if the intent can be reused after first use 63 * @in_use: To mark if intent is already in use for the channel 64 * @offset: next write offset (initially 0) 65 * @node: list node 66 */ 67 struct glink_core_rx_intent { 68 void *data; 69 u32 id; 70 size_t size; 71 bool reuse; 72 bool in_use; 73 u32 offset; 74 75 struct list_head node; 76 }; 77 78 /** 79 * struct qcom_glink - driver context, relates to one remote subsystem 80 * @dev: reference to the associated struct device 81 * @rx_pipe: pipe object for receive FIFO 82 * @tx_pipe: pipe object for transmit FIFO 83 * @rx_work: worker for handling received control messages 84 * @rx_lock: protects the @rx_queue 85 * @rx_queue: queue of received control messages to be processed in @rx_work 86 * @tx_lock: synchronizes operations on the tx fifo 87 * @idr_lock: synchronizes @lcids and @rcids modifications 88 * @lcids: idr of all channels with a known local channel id 89 * @rcids: idr of all channels with a known remote channel id 90 * @features: remote features 91 * @intentless: flag to indicate that there is no intent 92 * @tx_avail_notify: Waitqueue for pending tx tasks 93 * @sent_read_notify: flag to check cmd sent or not 94 * @abort_tx: flag indicating that all tx attempts should fail 95 */ 96 struct qcom_glink { 97 struct device *dev; 98 99 struct qcom_glink_pipe *rx_pipe; 100 struct qcom_glink_pipe *tx_pipe; 101 102 struct work_struct rx_work; 103 spinlock_t rx_lock; 104 struct list_head rx_queue; 105 106 spinlock_t tx_lock; 107 108 spinlock_t idr_lock; 109 struct idr lcids; 110 struct idr rcids; 111 unsigned long features; 112 113 bool intentless; 114 wait_queue_head_t tx_avail_notify; 115 bool sent_read_notify; 116 117 bool abort_tx; 118 }; 119 120 enum { 121 GLINK_STATE_CLOSED, 122 GLINK_STATE_OPENING, 123 GLINK_STATE_OPEN, 124 GLINK_STATE_CLOSING, 125 }; 126 127 /** 128 * struct glink_channel - internal representation of a channel 129 * @rpdev: rpdev reference, only used for primary endpoints 130 * @ept: rpmsg endpoint this channel is associated with 131 * @glink: qcom_glink context handle 132 * @refcount: refcount for the channel object 133 * @recv_lock: guard for @ept.cb 134 * @name: unique channel name/identifier 135 * @lcid: channel id, in local space 136 * @rcid: channel id, in remote space 137 * @intent_lock: lock for protection of @liids, @riids 138 * @liids: idr of all local intents 139 * @riids: idr of all remote intents 140 * @intent_work: worker responsible for transmitting rx_done packets 141 * @done_intents: list of intents that needs to be announced rx_done 142 * @buf: receive buffer, for gathering fragments 143 * @buf_offset: write offset in @buf 144 * @buf_size: size of current @buf 145 * @open_ack: completed once remote has acked the open-request 146 * @open_req: completed once open-request has been received 147 * @intent_req_lock: Synchronises multiple intent requests 148 * @intent_req_result: Result of intent request 149 * @intent_received: flag indicating that an intent has been received 150 * @intent_req_wq: wait queue for intent_req signalling 151 */ 152 struct glink_channel { 153 struct rpmsg_endpoint ept; 154 155 struct rpmsg_device *rpdev; 156 struct qcom_glink *glink; 157 158 struct kref refcount; 159 160 spinlock_t recv_lock; 161 162 char *name; 163 unsigned int lcid; 164 unsigned int rcid; 165 166 spinlock_t intent_lock; 167 struct idr liids; 168 struct idr riids; 169 struct work_struct intent_work; 170 struct list_head done_intents; 171 172 struct glink_core_rx_intent *buf; 173 int buf_offset; 174 int buf_size; 175 176 struct completion open_ack; 177 struct completion open_req; 178 179 struct mutex intent_req_lock; 180 int intent_req_result; 181 bool intent_received; 182 wait_queue_head_t intent_req_wq; 183 }; 184 185 #define to_glink_channel(_ept) container_of(_ept, struct glink_channel, ept) 186 187 static const struct rpmsg_endpoint_ops glink_endpoint_ops; 188 189 #define GLINK_CMD_VERSION 0 190 #define GLINK_CMD_VERSION_ACK 1 191 #define GLINK_CMD_OPEN 2 192 #define GLINK_CMD_CLOSE 3 193 #define GLINK_CMD_OPEN_ACK 4 194 #define GLINK_CMD_INTENT 5 195 #define GLINK_CMD_RX_DONE 6 196 #define GLINK_CMD_RX_INTENT_REQ 7 197 #define GLINK_CMD_RX_INTENT_REQ_ACK 8 198 #define GLINK_CMD_TX_DATA 9 199 #define GLINK_CMD_CLOSE_ACK 11 200 #define GLINK_CMD_TX_DATA_CONT 12 201 #define GLINK_CMD_READ_NOTIF 13 202 #define GLINK_CMD_RX_DONE_W_REUSE 14 203 #define GLINK_CMD_SIGNALS 15 204 205 #define GLINK_FEATURE_INTENTLESS BIT(1) 206 207 #define NATIVE_DTR_SIG NATIVE_DSR_SIG 208 #define NATIVE_DSR_SIG BIT(31) 209 #define NATIVE_RTS_SIG NATIVE_CTS_SIG 210 #define NATIVE_CTS_SIG BIT(30) 211 212 static void qcom_glink_rx_done_work(struct work_struct *work); 213 214 static struct glink_channel *qcom_glink_alloc_channel(struct qcom_glink *glink, 215 const char *name) 216 { 217 struct glink_channel *channel; 218 219 channel = kzalloc(sizeof(*channel), GFP_KERNEL); 220 if (!channel) 221 return ERR_PTR(-ENOMEM); 222 223 /* Setup glink internal glink_channel data */ 224 spin_lock_init(&channel->recv_lock); 225 spin_lock_init(&channel->intent_lock); 226 mutex_init(&channel->intent_req_lock); 227 228 channel->glink = glink; 229 channel->name = kstrdup(name, GFP_KERNEL); 230 if (!channel->name) { 231 kfree(channel); 232 return ERR_PTR(-ENOMEM); 233 } 234 235 init_completion(&channel->open_req); 236 init_completion(&channel->open_ack); 237 init_waitqueue_head(&channel->intent_req_wq); 238 239 INIT_LIST_HEAD(&channel->done_intents); 240 INIT_WORK(&channel->intent_work, qcom_glink_rx_done_work); 241 242 idr_init(&channel->liids); 243 idr_init(&channel->riids); 244 kref_init(&channel->refcount); 245 246 return channel; 247 } 248 249 static void qcom_glink_channel_release(struct kref *ref) 250 { 251 struct glink_channel *channel = container_of(ref, struct glink_channel, 252 refcount); 253 struct glink_core_rx_intent *intent; 254 struct glink_core_rx_intent *tmp; 255 unsigned long flags; 256 int iid; 257 258 /* cancel pending rx_done work */ 259 cancel_work_sync(&channel->intent_work); 260 261 spin_lock_irqsave(&channel->intent_lock, flags); 262 /* Free all non-reuse intents pending rx_done work */ 263 list_for_each_entry_safe(intent, tmp, &channel->done_intents, node) { 264 if (!intent->reuse) { 265 kfree(intent->data); 266 kfree(intent); 267 } 268 } 269 270 idr_for_each_entry(&channel->liids, tmp, iid) { 271 kfree(tmp->data); 272 kfree(tmp); 273 } 274 idr_destroy(&channel->liids); 275 276 idr_for_each_entry(&channel->riids, tmp, iid) 277 kfree(tmp); 278 idr_destroy(&channel->riids); 279 spin_unlock_irqrestore(&channel->intent_lock, flags); 280 281 kfree(channel->name); 282 kfree(channel); 283 } 284 285 static size_t qcom_glink_rx_avail(struct qcom_glink *glink) 286 { 287 return glink->rx_pipe->avail(glink->rx_pipe); 288 } 289 290 static void qcom_glink_rx_peek(struct qcom_glink *glink, 291 void *data, unsigned int offset, size_t count) 292 { 293 glink->rx_pipe->peek(glink->rx_pipe, data, offset, count); 294 } 295 296 static void qcom_glink_rx_advance(struct qcom_glink *glink, size_t count) 297 { 298 glink->rx_pipe->advance(glink->rx_pipe, count); 299 } 300 301 static size_t qcom_glink_tx_avail(struct qcom_glink *glink) 302 { 303 return glink->tx_pipe->avail(glink->tx_pipe); 304 } 305 306 static void qcom_glink_tx_write(struct qcom_glink *glink, 307 const void *hdr, size_t hlen, 308 const void *data, size_t dlen) 309 { 310 glink->tx_pipe->write(glink->tx_pipe, hdr, hlen, data, dlen); 311 } 312 313 static void qcom_glink_tx_kick(struct qcom_glink *glink) 314 { 315 glink->tx_pipe->kick(glink->tx_pipe); 316 } 317 318 static void qcom_glink_send_read_notify(struct qcom_glink *glink) 319 { 320 struct glink_msg msg; 321 322 msg.cmd = cpu_to_le16(GLINK_CMD_READ_NOTIF); 323 msg.param1 = 0; 324 msg.param2 = 0; 325 326 qcom_glink_tx_write(glink, &msg, sizeof(msg), NULL, 0); 327 328 qcom_glink_tx_kick(glink); 329 } 330 331 static int qcom_glink_tx(struct qcom_glink *glink, 332 const void *hdr, size_t hlen, 333 const void *data, size_t dlen, bool wait) 334 { 335 unsigned int tlen = hlen + dlen; 336 unsigned long flags; 337 int ret = 0; 338 339 /* Reject packets that are too big */ 340 if (tlen >= glink->tx_pipe->length) 341 return -EINVAL; 342 343 spin_lock_irqsave(&glink->tx_lock, flags); 344 345 if (glink->abort_tx) { 346 ret = -EIO; 347 goto out; 348 } 349 350 while (qcom_glink_tx_avail(glink) < tlen) { 351 if (!wait) { 352 ret = -EAGAIN; 353 goto out; 354 } 355 356 if (glink->abort_tx) { 357 ret = -EIO; 358 goto out; 359 } 360 361 if (!glink->sent_read_notify) { 362 glink->sent_read_notify = true; 363 qcom_glink_send_read_notify(glink); 364 } 365 366 /* Wait without holding the tx_lock */ 367 spin_unlock_irqrestore(&glink->tx_lock, flags); 368 369 wait_event_timeout(glink->tx_avail_notify, 370 qcom_glink_tx_avail(glink) >= tlen, 10 * HZ); 371 372 spin_lock_irqsave(&glink->tx_lock, flags); 373 374 if (qcom_glink_tx_avail(glink) >= tlen) 375 glink->sent_read_notify = false; 376 } 377 378 qcom_glink_tx_write(glink, hdr, hlen, data, dlen); 379 qcom_glink_tx_kick(glink); 380 381 out: 382 spin_unlock_irqrestore(&glink->tx_lock, flags); 383 384 return ret; 385 } 386 387 static int qcom_glink_send_version(struct qcom_glink *glink) 388 { 389 struct glink_msg msg; 390 391 msg.cmd = cpu_to_le16(GLINK_CMD_VERSION); 392 msg.param1 = cpu_to_le16(GLINK_VERSION_1); 393 msg.param2 = cpu_to_le32(glink->features); 394 395 return qcom_glink_tx(glink, &msg, sizeof(msg), NULL, 0, true); 396 } 397 398 static void qcom_glink_send_version_ack(struct qcom_glink *glink) 399 { 400 struct glink_msg msg; 401 402 msg.cmd = cpu_to_le16(GLINK_CMD_VERSION_ACK); 403 msg.param1 = cpu_to_le16(GLINK_VERSION_1); 404 msg.param2 = cpu_to_le32(glink->features); 405 406 qcom_glink_tx(glink, &msg, sizeof(msg), NULL, 0, true); 407 } 408 409 static void qcom_glink_send_open_ack(struct qcom_glink *glink, 410 struct glink_channel *channel) 411 { 412 struct glink_msg msg; 413 414 msg.cmd = cpu_to_le16(GLINK_CMD_OPEN_ACK); 415 msg.param1 = cpu_to_le16(channel->rcid); 416 msg.param2 = cpu_to_le32(0); 417 418 qcom_glink_tx(glink, &msg, sizeof(msg), NULL, 0, true); 419 } 420 421 static void qcom_glink_handle_intent_req_ack(struct qcom_glink *glink, 422 unsigned int cid, bool granted) 423 { 424 struct glink_channel *channel; 425 unsigned long flags; 426 427 qcom_glink_rx_advance(glink, ALIGN(sizeof(struct glink_msg), 8)); 428 429 spin_lock_irqsave(&glink->idr_lock, flags); 430 channel = idr_find(&glink->rcids, cid); 431 spin_unlock_irqrestore(&glink->idr_lock, flags); 432 if (!channel) { 433 dev_err(glink->dev, "unable to find channel\n"); 434 return; 435 } 436 437 WRITE_ONCE(channel->intent_req_result, granted); 438 wake_up_all(&channel->intent_req_wq); 439 } 440 441 static void qcom_glink_intent_req_abort(struct glink_channel *channel) 442 { 443 WRITE_ONCE(channel->intent_req_result, 0); 444 wake_up_all(&channel->intent_req_wq); 445 } 446 447 /** 448 * qcom_glink_send_open_req() - send a GLINK_CMD_OPEN request to the remote 449 * @glink: Ptr to the glink edge 450 * @channel: Ptr to the channel that the open req is sent 451 * 452 * Allocates a local channel id and sends a GLINK_CMD_OPEN message to the remote. 453 * Will return with refcount held, regardless of outcome. 454 * 455 * Return: 0 on success, negative errno otherwise. 456 */ 457 static int qcom_glink_send_open_req(struct qcom_glink *glink, 458 struct glink_channel *channel) 459 { 460 struct { 461 struct glink_msg msg; 462 u8 name[GLINK_NAME_SIZE]; 463 } __packed req; 464 int name_len = strlen(channel->name) + 1; 465 int req_len = ALIGN(sizeof(req.msg) + name_len, 8); 466 int ret; 467 unsigned long flags; 468 469 kref_get(&channel->refcount); 470 471 spin_lock_irqsave(&glink->idr_lock, flags); 472 ret = idr_alloc_cyclic(&glink->lcids, channel, 473 RPM_GLINK_CID_MIN, RPM_GLINK_CID_MAX, 474 GFP_ATOMIC); 475 spin_unlock_irqrestore(&glink->idr_lock, flags); 476 if (ret < 0) 477 return ret; 478 479 channel->lcid = ret; 480 481 req.msg.cmd = cpu_to_le16(GLINK_CMD_OPEN); 482 req.msg.param1 = cpu_to_le16(channel->lcid); 483 req.msg.param2 = cpu_to_le32(name_len); 484 strcpy(req.name, channel->name); 485 486 ret = qcom_glink_tx(glink, &req, req_len, NULL, 0, true); 487 if (ret) 488 goto remove_idr; 489 490 return 0; 491 492 remove_idr: 493 spin_lock_irqsave(&glink->idr_lock, flags); 494 idr_remove(&glink->lcids, channel->lcid); 495 channel->lcid = 0; 496 spin_unlock_irqrestore(&glink->idr_lock, flags); 497 498 return ret; 499 } 500 501 static void qcom_glink_send_close_req(struct qcom_glink *glink, 502 struct glink_channel *channel) 503 { 504 struct glink_msg req; 505 506 req.cmd = cpu_to_le16(GLINK_CMD_CLOSE); 507 req.param1 = cpu_to_le16(channel->lcid); 508 req.param2 = 0; 509 510 qcom_glink_tx(glink, &req, sizeof(req), NULL, 0, true); 511 } 512 513 static void qcom_glink_send_close_ack(struct qcom_glink *glink, 514 struct glink_channel *channel) 515 { 516 struct glink_msg req; 517 518 req.cmd = cpu_to_le16(GLINK_CMD_CLOSE_ACK); 519 req.param1 = cpu_to_le16(channel->rcid); 520 req.param2 = 0; 521 522 qcom_glink_tx(glink, &req, sizeof(req), NULL, 0, true); 523 } 524 525 static void qcom_glink_rx_done_work(struct work_struct *work) 526 { 527 struct glink_channel *channel = container_of(work, struct glink_channel, 528 intent_work); 529 struct qcom_glink *glink = channel->glink; 530 struct glink_core_rx_intent *intent, *tmp; 531 struct { 532 u16 id; 533 u16 lcid; 534 u32 liid; 535 } __packed cmd; 536 537 unsigned int cid = channel->lcid; 538 unsigned int iid; 539 bool reuse; 540 unsigned long flags; 541 542 spin_lock_irqsave(&channel->intent_lock, flags); 543 list_for_each_entry_safe(intent, tmp, &channel->done_intents, node) { 544 list_del(&intent->node); 545 spin_unlock_irqrestore(&channel->intent_lock, flags); 546 iid = intent->id; 547 reuse = intent->reuse; 548 549 cmd.id = reuse ? GLINK_CMD_RX_DONE_W_REUSE : GLINK_CMD_RX_DONE; 550 cmd.lcid = cid; 551 cmd.liid = iid; 552 553 qcom_glink_tx(glink, &cmd, sizeof(cmd), NULL, 0, true); 554 if (!reuse) { 555 kfree(intent->data); 556 kfree(intent); 557 } 558 spin_lock_irqsave(&channel->intent_lock, flags); 559 } 560 spin_unlock_irqrestore(&channel->intent_lock, flags); 561 } 562 563 static void qcom_glink_rx_done(struct qcom_glink *glink, 564 struct glink_channel *channel, 565 struct glink_core_rx_intent *intent) 566 { 567 /* We don't send RX_DONE to intentless systems */ 568 if (glink->intentless) { 569 kfree(intent->data); 570 kfree(intent); 571 return; 572 } 573 574 /* Take it off the tree of receive intents */ 575 if (!intent->reuse) { 576 spin_lock(&channel->intent_lock); 577 idr_remove(&channel->liids, intent->id); 578 spin_unlock(&channel->intent_lock); 579 } 580 581 /* Schedule the sending of a rx_done indication */ 582 spin_lock(&channel->intent_lock); 583 list_add_tail(&intent->node, &channel->done_intents); 584 spin_unlock(&channel->intent_lock); 585 586 schedule_work(&channel->intent_work); 587 } 588 589 /** 590 * qcom_glink_receive_version() - receive version/features from remote system 591 * 592 * @glink: pointer to transport interface 593 * @version: remote version 594 * @features: remote features 595 * 596 * This function is called in response to a remote-initiated version/feature 597 * negotiation sequence. 598 */ 599 static void qcom_glink_receive_version(struct qcom_glink *glink, 600 u32 version, 601 u32 features) 602 { 603 switch (version) { 604 case 0: 605 break; 606 case GLINK_VERSION_1: 607 glink->features &= features; 608 fallthrough; 609 default: 610 qcom_glink_send_version_ack(glink); 611 break; 612 } 613 } 614 615 /** 616 * qcom_glink_receive_version_ack() - receive negotiation ack from remote system 617 * 618 * @glink: pointer to transport interface 619 * @version: remote version response 620 * @features: remote features response 621 * 622 * This function is called in response to a local-initiated version/feature 623 * negotiation sequence and is the counter-offer from the remote side based 624 * upon the initial version and feature set requested. 625 */ 626 static void qcom_glink_receive_version_ack(struct qcom_glink *glink, 627 u32 version, 628 u32 features) 629 { 630 switch (version) { 631 case 0: 632 /* Version negotiation failed */ 633 break; 634 case GLINK_VERSION_1: 635 if (features == glink->features) 636 break; 637 638 glink->features &= features; 639 fallthrough; 640 default: 641 qcom_glink_send_version(glink); 642 break; 643 } 644 } 645 646 /** 647 * qcom_glink_send_intent_req_ack() - convert an rx intent request ack cmd to 648 * wire format and transmit 649 * @glink: The transport to transmit on. 650 * @channel: The glink channel 651 * @granted: The request response to encode. 652 * 653 * Return: 0 on success or standard Linux error code. 654 */ 655 static int qcom_glink_send_intent_req_ack(struct qcom_glink *glink, 656 struct glink_channel *channel, 657 bool granted) 658 { 659 struct glink_msg msg; 660 661 msg.cmd = cpu_to_le16(GLINK_CMD_RX_INTENT_REQ_ACK); 662 msg.param1 = cpu_to_le16(channel->lcid); 663 msg.param2 = cpu_to_le32(granted); 664 665 qcom_glink_tx(glink, &msg, sizeof(msg), NULL, 0, true); 666 667 return 0; 668 } 669 670 /** 671 * qcom_glink_advertise_intent - convert an rx intent cmd to wire format and 672 * transmit 673 * @glink: The transport to transmit on. 674 * @channel: The local channel 675 * @intent: The intent to pass on to remote. 676 * 677 * Return: 0 on success or standard Linux error code. 678 */ 679 static int qcom_glink_advertise_intent(struct qcom_glink *glink, 680 struct glink_channel *channel, 681 struct glink_core_rx_intent *intent) 682 { 683 struct command { 684 __le16 id; 685 __le16 lcid; 686 __le32 count; 687 __le32 size; 688 __le32 liid; 689 } __packed; 690 struct command cmd; 691 692 cmd.id = cpu_to_le16(GLINK_CMD_INTENT); 693 cmd.lcid = cpu_to_le16(channel->lcid); 694 cmd.count = cpu_to_le32(1); 695 cmd.size = cpu_to_le32(intent->size); 696 cmd.liid = cpu_to_le32(intent->id); 697 698 qcom_glink_tx(glink, &cmd, sizeof(cmd), NULL, 0, true); 699 700 return 0; 701 } 702 703 static struct glink_core_rx_intent * 704 qcom_glink_alloc_intent(struct qcom_glink *glink, 705 struct glink_channel *channel, 706 size_t size, 707 bool reuseable) 708 { 709 struct glink_core_rx_intent *intent; 710 int ret; 711 unsigned long flags; 712 713 intent = kzalloc(sizeof(*intent), GFP_KERNEL); 714 if (!intent) 715 return NULL; 716 717 intent->data = kzalloc(size, GFP_KERNEL); 718 if (!intent->data) 719 goto free_intent; 720 721 spin_lock_irqsave(&channel->intent_lock, flags); 722 ret = idr_alloc_cyclic(&channel->liids, intent, 1, -1, GFP_ATOMIC); 723 if (ret < 0) { 724 spin_unlock_irqrestore(&channel->intent_lock, flags); 725 goto free_data; 726 } 727 spin_unlock_irqrestore(&channel->intent_lock, flags); 728 729 intent->id = ret; 730 intent->size = size; 731 intent->reuse = reuseable; 732 733 return intent; 734 735 free_data: 736 kfree(intent->data); 737 free_intent: 738 kfree(intent); 739 return NULL; 740 } 741 742 static void qcom_glink_handle_rx_done(struct qcom_glink *glink, 743 u32 cid, uint32_t iid, 744 bool reuse) 745 { 746 struct glink_core_rx_intent *intent; 747 struct glink_channel *channel; 748 unsigned long flags; 749 750 qcom_glink_rx_advance(glink, ALIGN(sizeof(struct glink_msg), 8)); 751 752 spin_lock_irqsave(&glink->idr_lock, flags); 753 channel = idr_find(&glink->rcids, cid); 754 spin_unlock_irqrestore(&glink->idr_lock, flags); 755 if (!channel) { 756 dev_err(glink->dev, "invalid channel id received\n"); 757 return; 758 } 759 760 spin_lock_irqsave(&channel->intent_lock, flags); 761 intent = idr_find(&channel->riids, iid); 762 763 if (!intent) { 764 spin_unlock_irqrestore(&channel->intent_lock, flags); 765 dev_err(glink->dev, "invalid intent id received\n"); 766 return; 767 } 768 769 intent->in_use = false; 770 771 if (!reuse) { 772 idr_remove(&channel->riids, intent->id); 773 kfree(intent); 774 } 775 spin_unlock_irqrestore(&channel->intent_lock, flags); 776 777 if (reuse) { 778 WRITE_ONCE(channel->intent_received, true); 779 wake_up_all(&channel->intent_req_wq); 780 } 781 } 782 783 /** 784 * qcom_glink_handle_intent_req() - Receive a request for rx_intent 785 * from remote side 786 * @glink: Pointer to the transport interface 787 * @cid: Remote channel ID 788 * @size: size of the intent 789 * 790 * The function searches for the local channel to which the request for 791 * rx_intent has arrived and allocates and notifies the remote back 792 */ 793 static void qcom_glink_handle_intent_req(struct qcom_glink *glink, 794 u32 cid, size_t size) 795 { 796 struct glink_core_rx_intent *intent; 797 struct glink_channel *channel; 798 unsigned long flags; 799 800 spin_lock_irqsave(&glink->idr_lock, flags); 801 channel = idr_find(&glink->rcids, cid); 802 spin_unlock_irqrestore(&glink->idr_lock, flags); 803 804 if (!channel) { 805 pr_err("%s channel not found for cid %d\n", __func__, cid); 806 return; 807 } 808 809 intent = qcom_glink_alloc_intent(glink, channel, size, false); 810 if (intent) 811 qcom_glink_advertise_intent(glink, channel, intent); 812 813 qcom_glink_send_intent_req_ack(glink, channel, !!intent); 814 } 815 816 static int qcom_glink_rx_defer(struct qcom_glink *glink, size_t extra) 817 { 818 struct glink_defer_cmd *dcmd; 819 820 extra = ALIGN(extra, 8); 821 822 if (qcom_glink_rx_avail(glink) < sizeof(struct glink_msg) + extra) { 823 dev_dbg(glink->dev, "Insufficient data in rx fifo"); 824 return -ENXIO; 825 } 826 827 dcmd = kzalloc(struct_size(dcmd, data, extra), GFP_ATOMIC); 828 if (!dcmd) 829 return -ENOMEM; 830 831 INIT_LIST_HEAD(&dcmd->node); 832 833 qcom_glink_rx_peek(glink, &dcmd->msg, 0, sizeof(dcmd->msg) + extra); 834 835 spin_lock(&glink->rx_lock); 836 list_add_tail(&dcmd->node, &glink->rx_queue); 837 spin_unlock(&glink->rx_lock); 838 839 schedule_work(&glink->rx_work); 840 qcom_glink_rx_advance(glink, sizeof(dcmd->msg) + extra); 841 842 return 0; 843 } 844 845 static int qcom_glink_rx_data(struct qcom_glink *glink, size_t avail) 846 { 847 struct glink_core_rx_intent *intent; 848 struct glink_channel *channel; 849 struct { 850 struct glink_msg msg; 851 __le32 chunk_size; 852 __le32 left_size; 853 } __packed hdr; 854 unsigned int chunk_size; 855 unsigned int left_size; 856 unsigned int rcid; 857 unsigned int liid; 858 int ret = 0; 859 unsigned long flags; 860 861 if (avail < sizeof(hdr)) { 862 dev_dbg(glink->dev, "Not enough data in fifo\n"); 863 return -EAGAIN; 864 } 865 866 qcom_glink_rx_peek(glink, &hdr, 0, sizeof(hdr)); 867 chunk_size = le32_to_cpu(hdr.chunk_size); 868 left_size = le32_to_cpu(hdr.left_size); 869 870 if (avail < sizeof(hdr) + chunk_size) { 871 dev_dbg(glink->dev, "Payload not yet in fifo\n"); 872 return -EAGAIN; 873 } 874 875 rcid = le16_to_cpu(hdr.msg.param1); 876 spin_lock_irqsave(&glink->idr_lock, flags); 877 channel = idr_find(&glink->rcids, rcid); 878 spin_unlock_irqrestore(&glink->idr_lock, flags); 879 if (!channel) { 880 dev_dbg(glink->dev, "Data on non-existing channel\n"); 881 882 /* Drop the message */ 883 goto advance_rx; 884 } 885 886 if (glink->intentless) { 887 /* Might have an ongoing, fragmented, message to append */ 888 if (!channel->buf) { 889 intent = kzalloc(sizeof(*intent), GFP_ATOMIC); 890 if (!intent) 891 return -ENOMEM; 892 893 intent->data = kmalloc(chunk_size + left_size, 894 GFP_ATOMIC); 895 if (!intent->data) { 896 kfree(intent); 897 return -ENOMEM; 898 } 899 900 intent->id = 0xbabababa; 901 intent->size = chunk_size + left_size; 902 intent->offset = 0; 903 904 channel->buf = intent; 905 } else { 906 intent = channel->buf; 907 } 908 } else { 909 liid = le32_to_cpu(hdr.msg.param2); 910 911 spin_lock_irqsave(&channel->intent_lock, flags); 912 intent = idr_find(&channel->liids, liid); 913 spin_unlock_irqrestore(&channel->intent_lock, flags); 914 915 if (!intent) { 916 dev_err(glink->dev, 917 "no intent found for channel %s intent %d", 918 channel->name, liid); 919 ret = -ENOENT; 920 goto advance_rx; 921 } 922 } 923 924 if (intent->size - intent->offset < chunk_size) { 925 dev_err(glink->dev, "Insufficient space in intent\n"); 926 927 /* The packet header lied, drop payload */ 928 goto advance_rx; 929 } 930 931 qcom_glink_rx_peek(glink, intent->data + intent->offset, 932 sizeof(hdr), chunk_size); 933 intent->offset += chunk_size; 934 935 /* Handle message when no fragments remain to be received */ 936 if (!left_size) { 937 spin_lock(&channel->recv_lock); 938 if (channel->ept.cb) { 939 channel->ept.cb(channel->ept.rpdev, 940 intent->data, 941 intent->offset, 942 channel->ept.priv, 943 RPMSG_ADDR_ANY); 944 } 945 spin_unlock(&channel->recv_lock); 946 947 intent->offset = 0; 948 channel->buf = NULL; 949 950 qcom_glink_rx_done(glink, channel, intent); 951 } 952 953 advance_rx: 954 qcom_glink_rx_advance(glink, ALIGN(sizeof(hdr) + chunk_size, 8)); 955 956 return ret; 957 } 958 959 static void qcom_glink_rx_read_notif(struct qcom_glink *glink) 960 { 961 qcom_glink_rx_advance(glink, ALIGN(sizeof(struct glink_msg), 8)); 962 qcom_glink_tx_kick(glink); 963 } 964 965 static void qcom_glink_handle_intent(struct qcom_glink *glink, 966 unsigned int cid, 967 unsigned int count, 968 size_t avail) 969 { 970 struct glink_core_rx_intent *intent; 971 struct glink_channel *channel; 972 struct intent_pair { 973 __le32 size; 974 __le32 iid; 975 }; 976 977 struct { 978 struct glink_msg msg; 979 struct intent_pair intents[]; 980 } __packed * msg; 981 982 const size_t msglen = struct_size(msg, intents, count); 983 int ret; 984 int i; 985 unsigned long flags; 986 987 if (avail < msglen) { 988 dev_dbg(glink->dev, "Not enough data in fifo\n"); 989 return; 990 } 991 992 spin_lock_irqsave(&glink->idr_lock, flags); 993 channel = idr_find(&glink->rcids, cid); 994 spin_unlock_irqrestore(&glink->idr_lock, flags); 995 if (!channel) { 996 dev_err(glink->dev, "intents for non-existing channel\n"); 997 qcom_glink_rx_advance(glink, ALIGN(msglen, 8)); 998 return; 999 } 1000 1001 msg = kmalloc(msglen, GFP_ATOMIC); 1002 if (!msg) 1003 return; 1004 1005 qcom_glink_rx_peek(glink, msg, 0, msglen); 1006 1007 for (i = 0; i < count; ++i) { 1008 intent = kzalloc(sizeof(*intent), GFP_ATOMIC); 1009 if (!intent) 1010 break; 1011 1012 intent->id = le32_to_cpu(msg->intents[i].iid); 1013 intent->size = le32_to_cpu(msg->intents[i].size); 1014 1015 spin_lock_irqsave(&channel->intent_lock, flags); 1016 ret = idr_alloc(&channel->riids, intent, 1017 intent->id, intent->id + 1, GFP_ATOMIC); 1018 spin_unlock_irqrestore(&channel->intent_lock, flags); 1019 1020 if (ret < 0) 1021 dev_err(glink->dev, "failed to store remote intent\n"); 1022 } 1023 1024 WRITE_ONCE(channel->intent_received, true); 1025 wake_up_all(&channel->intent_req_wq); 1026 1027 kfree(msg); 1028 qcom_glink_rx_advance(glink, ALIGN(msglen, 8)); 1029 } 1030 1031 static int qcom_glink_rx_open_ack(struct qcom_glink *glink, unsigned int lcid) 1032 { 1033 struct glink_channel *channel; 1034 1035 qcom_glink_rx_advance(glink, ALIGN(sizeof(struct glink_msg), 8)); 1036 1037 spin_lock(&glink->idr_lock); 1038 channel = idr_find(&glink->lcids, lcid); 1039 spin_unlock(&glink->idr_lock); 1040 if (!channel) { 1041 dev_err(glink->dev, "Invalid open ack packet\n"); 1042 return -EINVAL; 1043 } 1044 1045 complete_all(&channel->open_ack); 1046 1047 return 0; 1048 } 1049 1050 /** 1051 * qcom_glink_set_flow_control() - convert a signal cmd to wire format and transmit 1052 * @ept: Rpmsg endpoint for channel. 1053 * @pause: Pause transmission 1054 * @dst: destination address of the endpoint 1055 * 1056 * Return: 0 on success or standard Linux error code. 1057 */ 1058 static int qcom_glink_set_flow_control(struct rpmsg_endpoint *ept, bool pause, u32 dst) 1059 { 1060 struct glink_channel *channel = to_glink_channel(ept); 1061 struct qcom_glink *glink = channel->glink; 1062 struct glink_msg msg; 1063 u32 sigs = 0; 1064 1065 if (pause) 1066 sigs |= NATIVE_DTR_SIG | NATIVE_RTS_SIG; 1067 1068 msg.cmd = cpu_to_le16(GLINK_CMD_SIGNALS); 1069 msg.param1 = cpu_to_le16(channel->lcid); 1070 msg.param2 = cpu_to_le32(sigs); 1071 1072 return qcom_glink_tx(glink, &msg, sizeof(msg), NULL, 0, true); 1073 } 1074 1075 static void qcom_glink_handle_signals(struct qcom_glink *glink, 1076 unsigned int rcid, unsigned int sigs) 1077 { 1078 struct glink_channel *channel; 1079 unsigned long flags; 1080 bool enable; 1081 1082 qcom_glink_rx_advance(glink, ALIGN(sizeof(struct glink_msg), 8)); 1083 1084 spin_lock_irqsave(&glink->idr_lock, flags); 1085 channel = idr_find(&glink->rcids, rcid); 1086 spin_unlock_irqrestore(&glink->idr_lock, flags); 1087 if (!channel) { 1088 dev_err(glink->dev, "signal for non-existing channel\n"); 1089 return; 1090 } 1091 1092 enable = sigs & NATIVE_DSR_SIG || sigs & NATIVE_CTS_SIG; 1093 1094 if (channel->ept.flow_cb) 1095 channel->ept.flow_cb(channel->ept.rpdev, channel->ept.priv, enable); 1096 } 1097 1098 void qcom_glink_native_rx(struct qcom_glink *glink) 1099 { 1100 struct glink_msg msg; 1101 unsigned int param1; 1102 unsigned int param2; 1103 unsigned int avail; 1104 unsigned int cmd; 1105 int ret = 0; 1106 1107 /* To wakeup any blocking writers */ 1108 wake_up_all(&glink->tx_avail_notify); 1109 1110 for (;;) { 1111 avail = qcom_glink_rx_avail(glink); 1112 if (avail < sizeof(msg)) 1113 break; 1114 1115 qcom_glink_rx_peek(glink, &msg, 0, sizeof(msg)); 1116 1117 cmd = le16_to_cpu(msg.cmd); 1118 param1 = le16_to_cpu(msg.param1); 1119 param2 = le32_to_cpu(msg.param2); 1120 1121 switch (cmd) { 1122 case GLINK_CMD_VERSION: 1123 case GLINK_CMD_VERSION_ACK: 1124 case GLINK_CMD_CLOSE: 1125 case GLINK_CMD_CLOSE_ACK: 1126 case GLINK_CMD_RX_INTENT_REQ: 1127 ret = qcom_glink_rx_defer(glink, 0); 1128 break; 1129 case GLINK_CMD_OPEN_ACK: 1130 ret = qcom_glink_rx_open_ack(glink, param1); 1131 break; 1132 case GLINK_CMD_OPEN: 1133 ret = qcom_glink_rx_defer(glink, param2); 1134 break; 1135 case GLINK_CMD_TX_DATA: 1136 case GLINK_CMD_TX_DATA_CONT: 1137 ret = qcom_glink_rx_data(glink, avail); 1138 break; 1139 case GLINK_CMD_READ_NOTIF: 1140 qcom_glink_rx_read_notif(glink); 1141 break; 1142 case GLINK_CMD_INTENT: 1143 qcom_glink_handle_intent(glink, param1, param2, avail); 1144 break; 1145 case GLINK_CMD_RX_DONE: 1146 qcom_glink_handle_rx_done(glink, param1, param2, false); 1147 break; 1148 case GLINK_CMD_RX_DONE_W_REUSE: 1149 qcom_glink_handle_rx_done(glink, param1, param2, true); 1150 break; 1151 case GLINK_CMD_RX_INTENT_REQ_ACK: 1152 qcom_glink_handle_intent_req_ack(glink, param1, param2); 1153 break; 1154 case GLINK_CMD_SIGNALS: 1155 qcom_glink_handle_signals(glink, param1, param2); 1156 break; 1157 default: 1158 dev_err(glink->dev, "unhandled rx cmd: %d\n", cmd); 1159 ret = -EINVAL; 1160 break; 1161 } 1162 1163 if (ret) 1164 break; 1165 } 1166 } 1167 EXPORT_SYMBOL(qcom_glink_native_rx); 1168 1169 /* Locally initiated rpmsg_create_ept */ 1170 static struct glink_channel *qcom_glink_create_local(struct qcom_glink *glink, 1171 const char *name) 1172 { 1173 struct glink_channel *channel; 1174 int ret; 1175 unsigned long flags; 1176 1177 channel = qcom_glink_alloc_channel(glink, name); 1178 if (IS_ERR(channel)) 1179 return ERR_CAST(channel); 1180 1181 ret = qcom_glink_send_open_req(glink, channel); 1182 if (ret) 1183 goto release_channel; 1184 1185 ret = wait_for_completion_timeout(&channel->open_ack, 5 * HZ); 1186 if (!ret) 1187 goto err_timeout; 1188 1189 ret = wait_for_completion_timeout(&channel->open_req, 5 * HZ); 1190 if (!ret) 1191 goto err_timeout; 1192 1193 qcom_glink_send_open_ack(glink, channel); 1194 1195 return channel; 1196 1197 err_timeout: 1198 /* qcom_glink_send_open_req() did register the channel in lcids*/ 1199 spin_lock_irqsave(&glink->idr_lock, flags); 1200 idr_remove(&glink->lcids, channel->lcid); 1201 spin_unlock_irqrestore(&glink->idr_lock, flags); 1202 1203 release_channel: 1204 /* Release qcom_glink_send_open_req() reference */ 1205 kref_put(&channel->refcount, qcom_glink_channel_release); 1206 /* Release qcom_glink_alloc_channel() reference */ 1207 kref_put(&channel->refcount, qcom_glink_channel_release); 1208 1209 return ERR_PTR(-ETIMEDOUT); 1210 } 1211 1212 /* Remote initiated rpmsg_create_ept */ 1213 static int qcom_glink_create_remote(struct qcom_glink *glink, 1214 struct glink_channel *channel) 1215 { 1216 int ret; 1217 1218 qcom_glink_send_open_ack(glink, channel); 1219 1220 ret = qcom_glink_send_open_req(glink, channel); 1221 if (ret) 1222 goto close_link; 1223 1224 ret = wait_for_completion_timeout(&channel->open_ack, 5 * HZ); 1225 if (!ret) { 1226 ret = -ETIMEDOUT; 1227 goto close_link; 1228 } 1229 1230 return 0; 1231 1232 close_link: 1233 /* 1234 * Send a close request to "undo" our open-ack. The close-ack will 1235 * release qcom_glink_send_open_req() reference and the last reference 1236 * will be relesed after receiving remote_close or transport unregister 1237 * by calling qcom_glink_native_remove(). 1238 */ 1239 qcom_glink_send_close_req(glink, channel); 1240 1241 return ret; 1242 } 1243 1244 static struct rpmsg_endpoint *qcom_glink_create_ept(struct rpmsg_device *rpdev, 1245 rpmsg_rx_cb_t cb, 1246 void *priv, 1247 struct rpmsg_channel_info 1248 chinfo) 1249 { 1250 struct glink_channel *parent = to_glink_channel(rpdev->ept); 1251 struct glink_channel *channel; 1252 struct qcom_glink *glink = parent->glink; 1253 struct rpmsg_endpoint *ept; 1254 const char *name = chinfo.name; 1255 int cid; 1256 int ret; 1257 unsigned long flags; 1258 1259 spin_lock_irqsave(&glink->idr_lock, flags); 1260 idr_for_each_entry(&glink->rcids, channel, cid) { 1261 if (!strcmp(channel->name, name)) 1262 break; 1263 } 1264 spin_unlock_irqrestore(&glink->idr_lock, flags); 1265 1266 if (!channel) { 1267 channel = qcom_glink_create_local(glink, name); 1268 if (IS_ERR(channel)) 1269 return NULL; 1270 } else { 1271 ret = qcom_glink_create_remote(glink, channel); 1272 if (ret) 1273 return NULL; 1274 } 1275 1276 ept = &channel->ept; 1277 ept->rpdev = rpdev; 1278 ept->cb = cb; 1279 ept->priv = priv; 1280 ept->ops = &glink_endpoint_ops; 1281 1282 return ept; 1283 } 1284 1285 static int qcom_glink_announce_create(struct rpmsg_device *rpdev) 1286 { 1287 struct glink_channel *channel = to_glink_channel(rpdev->ept); 1288 struct device_node *np = rpdev->dev.of_node; 1289 struct qcom_glink *glink = channel->glink; 1290 struct glink_core_rx_intent *intent; 1291 const struct property *prop = NULL; 1292 __be32 defaults[] = { cpu_to_be32(SZ_1K), cpu_to_be32(5) }; 1293 int num_intents; 1294 int num_groups = 1; 1295 __be32 *val = defaults; 1296 int size; 1297 1298 if (glink->intentless || !completion_done(&channel->open_ack)) 1299 return 0; 1300 1301 prop = of_find_property(np, "qcom,intents", NULL); 1302 if (prop) { 1303 val = prop->value; 1304 num_groups = prop->length / sizeof(u32) / 2; 1305 } 1306 1307 /* Channel is now open, advertise base set of intents */ 1308 while (num_groups--) { 1309 size = be32_to_cpup(val++); 1310 num_intents = be32_to_cpup(val++); 1311 while (num_intents--) { 1312 intent = qcom_glink_alloc_intent(glink, channel, size, 1313 true); 1314 if (!intent) 1315 break; 1316 1317 qcom_glink_advertise_intent(glink, channel, intent); 1318 } 1319 } 1320 return 0; 1321 } 1322 1323 static void qcom_glink_destroy_ept(struct rpmsg_endpoint *ept) 1324 { 1325 struct glink_channel *channel = to_glink_channel(ept); 1326 struct qcom_glink *glink = channel->glink; 1327 unsigned long flags; 1328 1329 spin_lock_irqsave(&channel->recv_lock, flags); 1330 channel->ept.cb = NULL; 1331 spin_unlock_irqrestore(&channel->recv_lock, flags); 1332 1333 /* Decouple the potential rpdev from the channel */ 1334 channel->rpdev = NULL; 1335 1336 qcom_glink_send_close_req(glink, channel); 1337 } 1338 1339 static int qcom_glink_request_intent(struct qcom_glink *glink, 1340 struct glink_channel *channel, 1341 size_t size) 1342 { 1343 struct { 1344 u16 id; 1345 u16 cid; 1346 u32 size; 1347 } __packed cmd; 1348 1349 int ret; 1350 1351 mutex_lock(&channel->intent_req_lock); 1352 1353 WRITE_ONCE(channel->intent_req_result, -1); 1354 WRITE_ONCE(channel->intent_received, false); 1355 1356 cmd.id = GLINK_CMD_RX_INTENT_REQ; 1357 cmd.cid = channel->lcid; 1358 cmd.size = size; 1359 1360 ret = qcom_glink_tx(glink, &cmd, sizeof(cmd), NULL, 0, true); 1361 if (ret) 1362 goto unlock; 1363 1364 ret = wait_event_timeout(channel->intent_req_wq, 1365 READ_ONCE(channel->intent_req_result) >= 0 && 1366 READ_ONCE(channel->intent_received), 1367 10 * HZ); 1368 if (!ret) { 1369 dev_err(glink->dev, "intent request timed out\n"); 1370 ret = -ETIMEDOUT; 1371 } else { 1372 ret = READ_ONCE(channel->intent_req_result) ? 0 : -ECANCELED; 1373 } 1374 1375 unlock: 1376 mutex_unlock(&channel->intent_req_lock); 1377 return ret; 1378 } 1379 1380 static int __qcom_glink_send(struct glink_channel *channel, 1381 void *data, int len, bool wait) 1382 { 1383 struct qcom_glink *glink = channel->glink; 1384 struct glink_core_rx_intent *intent = NULL; 1385 struct glink_core_rx_intent *tmp; 1386 int iid = 0; 1387 struct { 1388 struct glink_msg msg; 1389 __le32 chunk_size; 1390 __le32 left_size; 1391 } __packed req; 1392 int ret; 1393 unsigned long flags; 1394 int chunk_size = len; 1395 size_t offset = 0; 1396 1397 if (!glink->intentless) { 1398 while (!intent) { 1399 spin_lock_irqsave(&channel->intent_lock, flags); 1400 idr_for_each_entry(&channel->riids, tmp, iid) { 1401 if (tmp->size >= len && !tmp->in_use) { 1402 if (!intent) 1403 intent = tmp; 1404 else if (intent->size > tmp->size) 1405 intent = tmp; 1406 if (intent->size == len) 1407 break; 1408 } 1409 } 1410 if (intent) 1411 intent->in_use = true; 1412 spin_unlock_irqrestore(&channel->intent_lock, flags); 1413 1414 /* We found an available intent */ 1415 if (intent) 1416 break; 1417 1418 if (!wait) 1419 return -EBUSY; 1420 1421 ret = qcom_glink_request_intent(glink, channel, len); 1422 if (ret < 0) 1423 return ret; 1424 } 1425 1426 iid = intent->id; 1427 } 1428 1429 while (offset < len) { 1430 chunk_size = len - offset; 1431 if (chunk_size > SZ_8K && wait) 1432 chunk_size = SZ_8K; 1433 1434 req.msg.cmd = cpu_to_le16(offset == 0 ? GLINK_CMD_TX_DATA : GLINK_CMD_TX_DATA_CONT); 1435 req.msg.param1 = cpu_to_le16(channel->lcid); 1436 req.msg.param2 = cpu_to_le32(iid); 1437 req.chunk_size = cpu_to_le32(chunk_size); 1438 req.left_size = cpu_to_le32(len - offset - chunk_size); 1439 1440 ret = qcom_glink_tx(glink, &req, sizeof(req), data + offset, chunk_size, wait); 1441 if (ret) { 1442 /* Mark intent available if we failed */ 1443 if (intent) 1444 intent->in_use = false; 1445 return ret; 1446 } 1447 1448 offset += chunk_size; 1449 } 1450 1451 return 0; 1452 } 1453 1454 static int qcom_glink_send(struct rpmsg_endpoint *ept, void *data, int len) 1455 { 1456 struct glink_channel *channel = to_glink_channel(ept); 1457 1458 return __qcom_glink_send(channel, data, len, true); 1459 } 1460 1461 static int qcom_glink_trysend(struct rpmsg_endpoint *ept, void *data, int len) 1462 { 1463 struct glink_channel *channel = to_glink_channel(ept); 1464 1465 return __qcom_glink_send(channel, data, len, false); 1466 } 1467 1468 static int qcom_glink_sendto(struct rpmsg_endpoint *ept, void *data, int len, u32 dst) 1469 { 1470 struct glink_channel *channel = to_glink_channel(ept); 1471 1472 return __qcom_glink_send(channel, data, len, true); 1473 } 1474 1475 static int qcom_glink_trysendto(struct rpmsg_endpoint *ept, void *data, int len, u32 dst) 1476 { 1477 struct glink_channel *channel = to_glink_channel(ept); 1478 1479 return __qcom_glink_send(channel, data, len, false); 1480 } 1481 1482 /* 1483 * Finds the device_node for the glink child interested in this channel. 1484 */ 1485 static struct device_node *qcom_glink_match_channel(struct device_node *node, 1486 const char *channel) 1487 { 1488 struct device_node *child; 1489 const char *name; 1490 const char *key; 1491 int ret; 1492 1493 for_each_available_child_of_node(node, child) { 1494 key = "qcom,glink-channels"; 1495 ret = of_property_read_string(child, key, &name); 1496 if (ret) 1497 continue; 1498 1499 if (strcmp(name, channel) == 0) 1500 return child; 1501 } 1502 1503 return NULL; 1504 } 1505 1506 static const struct rpmsg_device_ops glink_device_ops = { 1507 .create_ept = qcom_glink_create_ept, 1508 .announce_create = qcom_glink_announce_create, 1509 }; 1510 1511 static const struct rpmsg_endpoint_ops glink_endpoint_ops = { 1512 .destroy_ept = qcom_glink_destroy_ept, 1513 .send = qcom_glink_send, 1514 .sendto = qcom_glink_sendto, 1515 .trysend = qcom_glink_trysend, 1516 .trysendto = qcom_glink_trysendto, 1517 .set_flow_control = qcom_glink_set_flow_control, 1518 }; 1519 1520 static void qcom_glink_rpdev_release(struct device *dev) 1521 { 1522 struct rpmsg_device *rpdev = to_rpmsg_device(dev); 1523 1524 kfree(rpdev->driver_override); 1525 kfree(rpdev); 1526 } 1527 1528 static int qcom_glink_rx_open(struct qcom_glink *glink, unsigned int rcid, 1529 char *name) 1530 { 1531 struct glink_channel *channel; 1532 struct rpmsg_device *rpdev; 1533 bool create_device = false; 1534 struct device_node *node; 1535 int lcid; 1536 int ret; 1537 unsigned long flags; 1538 1539 spin_lock_irqsave(&glink->idr_lock, flags); 1540 idr_for_each_entry(&glink->lcids, channel, lcid) { 1541 if (!strcmp(channel->name, name)) 1542 break; 1543 } 1544 spin_unlock_irqrestore(&glink->idr_lock, flags); 1545 1546 if (!channel) { 1547 channel = qcom_glink_alloc_channel(glink, name); 1548 if (IS_ERR(channel)) 1549 return PTR_ERR(channel); 1550 1551 /* The opening dance was initiated by the remote */ 1552 create_device = true; 1553 } 1554 1555 spin_lock_irqsave(&glink->idr_lock, flags); 1556 ret = idr_alloc(&glink->rcids, channel, rcid, rcid + 1, GFP_ATOMIC); 1557 if (ret < 0) { 1558 dev_err(glink->dev, "Unable to insert channel into rcid list\n"); 1559 spin_unlock_irqrestore(&glink->idr_lock, flags); 1560 goto free_channel; 1561 } 1562 channel->rcid = ret; 1563 spin_unlock_irqrestore(&glink->idr_lock, flags); 1564 1565 complete_all(&channel->open_req); 1566 1567 if (create_device) { 1568 rpdev = kzalloc(sizeof(*rpdev), GFP_KERNEL); 1569 if (!rpdev) { 1570 ret = -ENOMEM; 1571 goto rcid_remove; 1572 } 1573 1574 rpdev->ept = &channel->ept; 1575 strscpy_pad(rpdev->id.name, name, RPMSG_NAME_SIZE); 1576 rpdev->src = RPMSG_ADDR_ANY; 1577 rpdev->dst = RPMSG_ADDR_ANY; 1578 rpdev->ops = &glink_device_ops; 1579 1580 node = qcom_glink_match_channel(glink->dev->of_node, name); 1581 rpdev->dev.of_node = node; 1582 rpdev->dev.parent = glink->dev; 1583 rpdev->dev.release = qcom_glink_rpdev_release; 1584 1585 ret = rpmsg_register_device(rpdev); 1586 if (ret) 1587 goto rcid_remove; 1588 1589 channel->rpdev = rpdev; 1590 } 1591 1592 return 0; 1593 1594 rcid_remove: 1595 spin_lock_irqsave(&glink->idr_lock, flags); 1596 idr_remove(&glink->rcids, channel->rcid); 1597 channel->rcid = 0; 1598 spin_unlock_irqrestore(&glink->idr_lock, flags); 1599 free_channel: 1600 /* Release the reference, iff we took it */ 1601 if (create_device) 1602 kref_put(&channel->refcount, qcom_glink_channel_release); 1603 1604 return ret; 1605 } 1606 1607 static void qcom_glink_rx_close(struct qcom_glink *glink, unsigned int rcid) 1608 { 1609 struct rpmsg_channel_info chinfo; 1610 struct glink_channel *channel; 1611 unsigned long flags; 1612 1613 spin_lock_irqsave(&glink->idr_lock, flags); 1614 channel = idr_find(&glink->rcids, rcid); 1615 spin_unlock_irqrestore(&glink->idr_lock, flags); 1616 if (WARN(!channel, "close request on unknown channel\n")) 1617 return; 1618 1619 /* cancel pending rx_done work */ 1620 cancel_work_sync(&channel->intent_work); 1621 1622 if (channel->rpdev) { 1623 strscpy_pad(chinfo.name, channel->name, sizeof(chinfo.name)); 1624 chinfo.src = RPMSG_ADDR_ANY; 1625 chinfo.dst = RPMSG_ADDR_ANY; 1626 1627 rpmsg_unregister_device(glink->dev, &chinfo); 1628 } 1629 channel->rpdev = NULL; 1630 1631 qcom_glink_send_close_ack(glink, channel); 1632 1633 spin_lock_irqsave(&glink->idr_lock, flags); 1634 idr_remove(&glink->rcids, channel->rcid); 1635 channel->rcid = 0; 1636 spin_unlock_irqrestore(&glink->idr_lock, flags); 1637 1638 kref_put(&channel->refcount, qcom_glink_channel_release); 1639 } 1640 1641 static void qcom_glink_rx_close_ack(struct qcom_glink *glink, unsigned int lcid) 1642 { 1643 struct rpmsg_channel_info chinfo; 1644 struct glink_channel *channel; 1645 unsigned long flags; 1646 1647 /* To wakeup any blocking writers */ 1648 wake_up_all(&glink->tx_avail_notify); 1649 1650 spin_lock_irqsave(&glink->idr_lock, flags); 1651 channel = idr_find(&glink->lcids, lcid); 1652 if (WARN(!channel, "close ack on unknown channel\n")) { 1653 spin_unlock_irqrestore(&glink->idr_lock, flags); 1654 return; 1655 } 1656 1657 idr_remove(&glink->lcids, channel->lcid); 1658 channel->lcid = 0; 1659 spin_unlock_irqrestore(&glink->idr_lock, flags); 1660 1661 /* Decouple the potential rpdev from the channel */ 1662 if (channel->rpdev) { 1663 strscpy(chinfo.name, channel->name, sizeof(chinfo.name)); 1664 chinfo.src = RPMSG_ADDR_ANY; 1665 chinfo.dst = RPMSG_ADDR_ANY; 1666 1667 rpmsg_unregister_device(glink->dev, &chinfo); 1668 } 1669 channel->rpdev = NULL; 1670 1671 kref_put(&channel->refcount, qcom_glink_channel_release); 1672 } 1673 1674 static void qcom_glink_work(struct work_struct *work) 1675 { 1676 struct qcom_glink *glink = container_of(work, struct qcom_glink, 1677 rx_work); 1678 struct glink_defer_cmd *dcmd; 1679 struct glink_msg *msg; 1680 unsigned long flags; 1681 unsigned int param1; 1682 unsigned int param2; 1683 unsigned int cmd; 1684 1685 for (;;) { 1686 spin_lock_irqsave(&glink->rx_lock, flags); 1687 if (list_empty(&glink->rx_queue)) { 1688 spin_unlock_irqrestore(&glink->rx_lock, flags); 1689 break; 1690 } 1691 dcmd = list_first_entry(&glink->rx_queue, 1692 struct glink_defer_cmd, node); 1693 list_del(&dcmd->node); 1694 spin_unlock_irqrestore(&glink->rx_lock, flags); 1695 1696 msg = &dcmd->msg; 1697 cmd = le16_to_cpu(msg->cmd); 1698 param1 = le16_to_cpu(msg->param1); 1699 param2 = le32_to_cpu(msg->param2); 1700 1701 switch (cmd) { 1702 case GLINK_CMD_VERSION: 1703 qcom_glink_receive_version(glink, param1, param2); 1704 break; 1705 case GLINK_CMD_VERSION_ACK: 1706 qcom_glink_receive_version_ack(glink, param1, param2); 1707 break; 1708 case GLINK_CMD_OPEN: 1709 qcom_glink_rx_open(glink, param1, msg->data); 1710 break; 1711 case GLINK_CMD_CLOSE: 1712 qcom_glink_rx_close(glink, param1); 1713 break; 1714 case GLINK_CMD_CLOSE_ACK: 1715 qcom_glink_rx_close_ack(glink, param1); 1716 break; 1717 case GLINK_CMD_RX_INTENT_REQ: 1718 qcom_glink_handle_intent_req(glink, param1, param2); 1719 break; 1720 default: 1721 WARN(1, "Unknown defer object %d\n", cmd); 1722 break; 1723 } 1724 1725 kfree(dcmd); 1726 } 1727 } 1728 1729 static void qcom_glink_cancel_rx_work(struct qcom_glink *glink) 1730 { 1731 struct glink_defer_cmd *dcmd; 1732 struct glink_defer_cmd *tmp; 1733 1734 /* cancel any pending deferred rx_work */ 1735 cancel_work_sync(&glink->rx_work); 1736 1737 list_for_each_entry_safe(dcmd, tmp, &glink->rx_queue, node) 1738 kfree(dcmd); 1739 } 1740 1741 static ssize_t rpmsg_name_show(struct device *dev, 1742 struct device_attribute *attr, char *buf) 1743 { 1744 int ret = 0; 1745 const char *name; 1746 1747 ret = of_property_read_string(dev->of_node, "label", &name); 1748 if (ret < 0) 1749 name = dev->of_node->name; 1750 1751 return sysfs_emit(buf, "%s\n", name); 1752 } 1753 static DEVICE_ATTR_RO(rpmsg_name); 1754 1755 static struct attribute *qcom_glink_attrs[] = { 1756 &dev_attr_rpmsg_name.attr, 1757 NULL 1758 }; 1759 ATTRIBUTE_GROUPS(qcom_glink); 1760 1761 static void qcom_glink_device_release(struct device *dev) 1762 { 1763 struct rpmsg_device *rpdev = to_rpmsg_device(dev); 1764 struct glink_channel *channel = to_glink_channel(rpdev->ept); 1765 1766 /* Release qcom_glink_alloc_channel() reference */ 1767 kref_put(&channel->refcount, qcom_glink_channel_release); 1768 kfree(rpdev->driver_override); 1769 kfree(rpdev); 1770 } 1771 1772 static int qcom_glink_create_chrdev(struct qcom_glink *glink) 1773 { 1774 struct rpmsg_device *rpdev; 1775 struct glink_channel *channel; 1776 1777 rpdev = kzalloc(sizeof(*rpdev), GFP_KERNEL); 1778 if (!rpdev) 1779 return -ENOMEM; 1780 1781 channel = qcom_glink_alloc_channel(glink, "rpmsg_chrdev"); 1782 if (IS_ERR(channel)) { 1783 kfree(rpdev); 1784 return PTR_ERR(channel); 1785 } 1786 channel->rpdev = rpdev; 1787 1788 rpdev->ept = &channel->ept; 1789 rpdev->ops = &glink_device_ops; 1790 rpdev->dev.parent = glink->dev; 1791 rpdev->dev.release = qcom_glink_device_release; 1792 1793 return rpmsg_ctrldev_register_device(rpdev); 1794 } 1795 1796 struct qcom_glink *qcom_glink_native_probe(struct device *dev, 1797 unsigned long features, 1798 struct qcom_glink_pipe *rx, 1799 struct qcom_glink_pipe *tx, 1800 bool intentless) 1801 { 1802 int ret; 1803 struct qcom_glink *glink; 1804 1805 glink = devm_kzalloc(dev, sizeof(*glink), GFP_KERNEL); 1806 if (!glink) 1807 return ERR_PTR(-ENOMEM); 1808 1809 glink->dev = dev; 1810 glink->tx_pipe = tx; 1811 glink->rx_pipe = rx; 1812 1813 glink->features = features; 1814 glink->intentless = intentless; 1815 1816 spin_lock_init(&glink->tx_lock); 1817 spin_lock_init(&glink->rx_lock); 1818 INIT_LIST_HEAD(&glink->rx_queue); 1819 INIT_WORK(&glink->rx_work, qcom_glink_work); 1820 init_waitqueue_head(&glink->tx_avail_notify); 1821 1822 spin_lock_init(&glink->idr_lock); 1823 idr_init(&glink->lcids); 1824 idr_init(&glink->rcids); 1825 1826 glink->dev->groups = qcom_glink_groups; 1827 1828 ret = device_add_groups(dev, qcom_glink_groups); 1829 if (ret) 1830 dev_err(dev, "failed to add groups\n"); 1831 1832 ret = qcom_glink_send_version(glink); 1833 if (ret) 1834 return ERR_PTR(ret); 1835 1836 ret = qcom_glink_create_chrdev(glink); 1837 if (ret) 1838 dev_err(glink->dev, "failed to register chrdev\n"); 1839 1840 return glink; 1841 } 1842 EXPORT_SYMBOL_GPL(qcom_glink_native_probe); 1843 1844 static int qcom_glink_remove_device(struct device *dev, void *data) 1845 { 1846 device_unregister(dev); 1847 1848 return 0; 1849 } 1850 1851 void qcom_glink_native_remove(struct qcom_glink *glink) 1852 { 1853 struct glink_channel *channel; 1854 unsigned long flags; 1855 int cid; 1856 int ret; 1857 1858 qcom_glink_cancel_rx_work(glink); 1859 1860 /* Fail all attempts at sending messages */ 1861 spin_lock_irqsave(&glink->tx_lock, flags); 1862 glink->abort_tx = true; 1863 wake_up_all(&glink->tx_avail_notify); 1864 spin_unlock_irqrestore(&glink->tx_lock, flags); 1865 1866 /* Abort any senders waiting for intent requests */ 1867 spin_lock_irqsave(&glink->idr_lock, flags); 1868 idr_for_each_entry(&glink->lcids, channel, cid) 1869 qcom_glink_intent_req_abort(channel); 1870 spin_unlock_irqrestore(&glink->idr_lock, flags); 1871 1872 ret = device_for_each_child(glink->dev, NULL, qcom_glink_remove_device); 1873 if (ret) 1874 dev_warn(glink->dev, "Can't remove GLINK devices: %d\n", ret); 1875 1876 /* Release any defunct local channels, waiting for close-ack */ 1877 idr_for_each_entry(&glink->lcids, channel, cid) 1878 kref_put(&channel->refcount, qcom_glink_channel_release); 1879 1880 /* Release any defunct local channels, waiting for close-req */ 1881 idr_for_each_entry(&glink->rcids, channel, cid) 1882 kref_put(&channel->refcount, qcom_glink_channel_release); 1883 1884 idr_destroy(&glink->lcids); 1885 idr_destroy(&glink->rcids); 1886 } 1887 EXPORT_SYMBOL_GPL(qcom_glink_native_remove); 1888 1889 MODULE_DESCRIPTION("Qualcomm GLINK driver"); 1890 MODULE_LICENSE("GPL v2"); 1891