1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * Copyright (c) 2016-2017, Linaro Ltd 4 */ 5 6 #include <linux/idr.h> 7 #include <linux/interrupt.h> 8 #include <linux/io.h> 9 #include <linux/list.h> 10 #include <linux/mfd/syscon.h> 11 #include <linux/module.h> 12 #include <linux/of.h> 13 #include <linux/of_address.h> 14 #include <linux/platform_device.h> 15 #include <linux/regmap.h> 16 #include <linux/rpmsg.h> 17 #include <linux/sizes.h> 18 #include <linux/slab.h> 19 #include <linux/workqueue.h> 20 #include <linux/mailbox_client.h> 21 22 #include "rpmsg_internal.h" 23 #include "qcom_glink_native.h" 24 25 #define GLINK_NAME_SIZE 32 26 #define GLINK_VERSION_1 1 27 28 #define RPM_GLINK_CID_MIN 1 29 #define RPM_GLINK_CID_MAX 65536 30 31 struct glink_msg { 32 __le16 cmd; 33 __le16 param1; 34 __le32 param2; 35 u8 data[]; 36 } __packed; 37 38 /** 39 * struct glink_defer_cmd - deferred incoming control message 40 * @node: list node 41 * @msg: message header 42 * @data: payload of the message 43 * 44 * Copy of a received control message, to be added to @rx_queue and processed 45 * by @rx_work of @qcom_glink. 46 */ 47 struct glink_defer_cmd { 48 struct list_head node; 49 50 struct glink_msg msg; 51 u8 data[]; 52 }; 53 54 /** 55 * struct glink_core_rx_intent - RX intent 56 * RX intent 57 * 58 * @data: pointer to the data (may be NULL for zero-copy) 59 * @id: remote or local intent ID 60 * @size: size of the original intent (do not modify) 61 * @reuse: To mark if the intent can be reused after first use 62 * @in_use: To mark if intent is already in use for the channel 63 * @offset: next write offset (initially 0) 64 * @node: list node 65 */ 66 struct glink_core_rx_intent { 67 void *data; 68 u32 id; 69 size_t size; 70 bool reuse; 71 bool in_use; 72 u32 offset; 73 74 struct list_head node; 75 }; 76 77 /** 78 * struct qcom_glink - driver context, relates to one remote subsystem 79 * @dev: reference to the associated struct device 80 * @rx_pipe: pipe object for receive FIFO 81 * @tx_pipe: pipe object for transmit FIFO 82 * @rx_work: worker for handling received control messages 83 * @rx_lock: protects the @rx_queue 84 * @rx_queue: queue of received control messages to be processed in @rx_work 85 * @tx_lock: synchronizes operations on the tx fifo 86 * @idr_lock: synchronizes @lcids and @rcids modifications 87 * @lcids: idr of all channels with a known local channel id 88 * @rcids: idr of all channels with a known remote channel id 89 * @features: remote features 90 * @intentless: flag to indicate that there is no intent 91 * @tx_avail_notify: Waitqueue for pending tx tasks 92 * @sent_read_notify: flag to check cmd sent or not 93 */ 94 struct qcom_glink { 95 struct device *dev; 96 97 struct qcom_glink_pipe *rx_pipe; 98 struct qcom_glink_pipe *tx_pipe; 99 100 struct work_struct rx_work; 101 spinlock_t rx_lock; 102 struct list_head rx_queue; 103 104 spinlock_t tx_lock; 105 106 spinlock_t idr_lock; 107 struct idr lcids; 108 struct idr rcids; 109 unsigned long features; 110 111 bool intentless; 112 wait_queue_head_t tx_avail_notify; 113 bool sent_read_notify; 114 }; 115 116 enum { 117 GLINK_STATE_CLOSED, 118 GLINK_STATE_OPENING, 119 GLINK_STATE_OPEN, 120 GLINK_STATE_CLOSING, 121 }; 122 123 /** 124 * struct glink_channel - internal representation of a channel 125 * @rpdev: rpdev reference, only used for primary endpoints 126 * @ept: rpmsg endpoint this channel is associated with 127 * @glink: qcom_glink context handle 128 * @refcount: refcount for the channel object 129 * @recv_lock: guard for @ept.cb 130 * @name: unique channel name/identifier 131 * @lcid: channel id, in local space 132 * @rcid: channel id, in remote space 133 * @intent_lock: lock for protection of @liids, @riids 134 * @liids: idr of all local intents 135 * @riids: idr of all remote intents 136 * @intent_work: worker responsible for transmitting rx_done packets 137 * @done_intents: list of intents that needs to be announced rx_done 138 * @buf: receive buffer, for gathering fragments 139 * @buf_offset: write offset in @buf 140 * @buf_size: size of current @buf 141 * @open_ack: completed once remote has acked the open-request 142 * @open_req: completed once open-request has been received 143 * @intent_req_lock: Synchronises multiple intent requests 144 * @intent_req_result: Result of intent request 145 * @intent_req_comp: Completion for intent_req signalling 146 */ 147 struct glink_channel { 148 struct rpmsg_endpoint ept; 149 150 struct rpmsg_device *rpdev; 151 struct qcom_glink *glink; 152 153 struct kref refcount; 154 155 spinlock_t recv_lock; 156 157 char *name; 158 unsigned int lcid; 159 unsigned int rcid; 160 161 spinlock_t intent_lock; 162 struct idr liids; 163 struct idr riids; 164 struct work_struct intent_work; 165 struct list_head done_intents; 166 167 struct glink_core_rx_intent *buf; 168 int buf_offset; 169 int buf_size; 170 171 struct completion open_ack; 172 struct completion open_req; 173 174 struct mutex intent_req_lock; 175 bool intent_req_result; 176 struct completion intent_req_comp; 177 }; 178 179 #define to_glink_channel(_ept) container_of(_ept, struct glink_channel, ept) 180 181 static const struct rpmsg_endpoint_ops glink_endpoint_ops; 182 183 #define RPM_CMD_VERSION 0 184 #define RPM_CMD_VERSION_ACK 1 185 #define RPM_CMD_OPEN 2 186 #define RPM_CMD_CLOSE 3 187 #define RPM_CMD_OPEN_ACK 4 188 #define RPM_CMD_INTENT 5 189 #define RPM_CMD_RX_DONE 6 190 #define RPM_CMD_RX_INTENT_REQ 7 191 #define RPM_CMD_RX_INTENT_REQ_ACK 8 192 #define RPM_CMD_TX_DATA 9 193 #define RPM_CMD_CLOSE_ACK 11 194 #define RPM_CMD_TX_DATA_CONT 12 195 #define RPM_CMD_READ_NOTIF 13 196 #define RPM_CMD_RX_DONE_W_REUSE 14 197 198 #define GLINK_FEATURE_INTENTLESS BIT(1) 199 200 static void qcom_glink_rx_done_work(struct work_struct *work); 201 202 static struct glink_channel *qcom_glink_alloc_channel(struct qcom_glink *glink, 203 const char *name) 204 { 205 struct glink_channel *channel; 206 207 channel = kzalloc(sizeof(*channel), GFP_KERNEL); 208 if (!channel) 209 return ERR_PTR(-ENOMEM); 210 211 /* Setup glink internal glink_channel data */ 212 spin_lock_init(&channel->recv_lock); 213 spin_lock_init(&channel->intent_lock); 214 mutex_init(&channel->intent_req_lock); 215 216 channel->glink = glink; 217 channel->name = kstrdup(name, GFP_KERNEL); 218 219 init_completion(&channel->open_req); 220 init_completion(&channel->open_ack); 221 init_completion(&channel->intent_req_comp); 222 223 INIT_LIST_HEAD(&channel->done_intents); 224 INIT_WORK(&channel->intent_work, qcom_glink_rx_done_work); 225 226 idr_init(&channel->liids); 227 idr_init(&channel->riids); 228 kref_init(&channel->refcount); 229 230 return channel; 231 } 232 233 static void qcom_glink_channel_release(struct kref *ref) 234 { 235 struct glink_channel *channel = container_of(ref, struct glink_channel, 236 refcount); 237 struct glink_core_rx_intent *intent; 238 struct glink_core_rx_intent *tmp; 239 unsigned long flags; 240 int iid; 241 242 /* cancel pending rx_done work */ 243 cancel_work_sync(&channel->intent_work); 244 245 spin_lock_irqsave(&channel->intent_lock, flags); 246 /* Free all non-reuse intents pending rx_done work */ 247 list_for_each_entry_safe(intent, tmp, &channel->done_intents, node) { 248 if (!intent->reuse) { 249 kfree(intent->data); 250 kfree(intent); 251 } 252 } 253 254 idr_for_each_entry(&channel->liids, tmp, iid) { 255 kfree(tmp->data); 256 kfree(tmp); 257 } 258 idr_destroy(&channel->liids); 259 260 idr_for_each_entry(&channel->riids, tmp, iid) 261 kfree(tmp); 262 idr_destroy(&channel->riids); 263 spin_unlock_irqrestore(&channel->intent_lock, flags); 264 265 kfree(channel->name); 266 kfree(channel); 267 } 268 269 static size_t qcom_glink_rx_avail(struct qcom_glink *glink) 270 { 271 return glink->rx_pipe->avail(glink->rx_pipe); 272 } 273 274 static void qcom_glink_rx_peak(struct qcom_glink *glink, 275 void *data, unsigned int offset, size_t count) 276 { 277 glink->rx_pipe->peak(glink->rx_pipe, data, offset, count); 278 } 279 280 static void qcom_glink_rx_advance(struct qcom_glink *glink, size_t count) 281 { 282 glink->rx_pipe->advance(glink->rx_pipe, count); 283 } 284 285 static size_t qcom_glink_tx_avail(struct qcom_glink *glink) 286 { 287 return glink->tx_pipe->avail(glink->tx_pipe); 288 } 289 290 static void qcom_glink_tx_write(struct qcom_glink *glink, 291 const void *hdr, size_t hlen, 292 const void *data, size_t dlen) 293 { 294 glink->tx_pipe->write(glink->tx_pipe, hdr, hlen, data, dlen); 295 } 296 297 static void qcom_glink_tx_kick(struct qcom_glink *glink) 298 { 299 glink->tx_pipe->kick(glink->tx_pipe); 300 } 301 302 static void qcom_glink_send_read_notify(struct qcom_glink *glink) 303 { 304 struct glink_msg msg; 305 306 msg.cmd = cpu_to_le16(RPM_CMD_READ_NOTIF); 307 msg.param1 = 0; 308 msg.param2 = 0; 309 310 qcom_glink_tx_write(glink, &msg, sizeof(msg), NULL, 0); 311 312 qcom_glink_tx_kick(glink); 313 } 314 315 static int qcom_glink_tx(struct qcom_glink *glink, 316 const void *hdr, size_t hlen, 317 const void *data, size_t dlen, bool wait) 318 { 319 unsigned int tlen = hlen + dlen; 320 unsigned long flags; 321 int ret = 0; 322 323 /* Reject packets that are too big */ 324 if (tlen >= glink->tx_pipe->length) 325 return -EINVAL; 326 327 spin_lock_irqsave(&glink->tx_lock, flags); 328 329 while (qcom_glink_tx_avail(glink) < tlen) { 330 if (!wait) { 331 ret = -EAGAIN; 332 goto out; 333 } 334 335 if (!glink->sent_read_notify) { 336 glink->sent_read_notify = true; 337 qcom_glink_send_read_notify(glink); 338 } 339 340 /* Wait without holding the tx_lock */ 341 spin_unlock_irqrestore(&glink->tx_lock, flags); 342 343 wait_event_timeout(glink->tx_avail_notify, 344 qcom_glink_tx_avail(glink) >= tlen, 10 * HZ); 345 346 spin_lock_irqsave(&glink->tx_lock, flags); 347 348 if (qcom_glink_tx_avail(glink) >= tlen) 349 glink->sent_read_notify = false; 350 } 351 352 qcom_glink_tx_write(glink, hdr, hlen, data, dlen); 353 qcom_glink_tx_kick(glink); 354 355 out: 356 spin_unlock_irqrestore(&glink->tx_lock, flags); 357 358 return ret; 359 } 360 361 static int qcom_glink_send_version(struct qcom_glink *glink) 362 { 363 struct glink_msg msg; 364 365 msg.cmd = cpu_to_le16(RPM_CMD_VERSION); 366 msg.param1 = cpu_to_le16(GLINK_VERSION_1); 367 msg.param2 = cpu_to_le32(glink->features); 368 369 return qcom_glink_tx(glink, &msg, sizeof(msg), NULL, 0, true); 370 } 371 372 static void qcom_glink_send_version_ack(struct qcom_glink *glink) 373 { 374 struct glink_msg msg; 375 376 msg.cmd = cpu_to_le16(RPM_CMD_VERSION_ACK); 377 msg.param1 = cpu_to_le16(GLINK_VERSION_1); 378 msg.param2 = cpu_to_le32(glink->features); 379 380 qcom_glink_tx(glink, &msg, sizeof(msg), NULL, 0, true); 381 } 382 383 static void qcom_glink_send_open_ack(struct qcom_glink *glink, 384 struct glink_channel *channel) 385 { 386 struct glink_msg msg; 387 388 msg.cmd = cpu_to_le16(RPM_CMD_OPEN_ACK); 389 msg.param1 = cpu_to_le16(channel->rcid); 390 msg.param2 = cpu_to_le32(0); 391 392 qcom_glink_tx(glink, &msg, sizeof(msg), NULL, 0, true); 393 } 394 395 static void qcom_glink_handle_intent_req_ack(struct qcom_glink *glink, 396 unsigned int cid, bool granted) 397 { 398 struct glink_channel *channel; 399 unsigned long flags; 400 401 spin_lock_irqsave(&glink->idr_lock, flags); 402 channel = idr_find(&glink->rcids, cid); 403 spin_unlock_irqrestore(&glink->idr_lock, flags); 404 if (!channel) { 405 dev_err(glink->dev, "unable to find channel\n"); 406 return; 407 } 408 409 channel->intent_req_result = granted; 410 complete(&channel->intent_req_comp); 411 } 412 413 /** 414 * qcom_glink_send_open_req() - send a RPM_CMD_OPEN request to the remote 415 * @glink: Ptr to the glink edge 416 * @channel: Ptr to the channel that the open req is sent 417 * 418 * Allocates a local channel id and sends a RPM_CMD_OPEN message to the remote. 419 * Will return with refcount held, regardless of outcome. 420 * 421 * Return: 0 on success, negative errno otherwise. 422 */ 423 static int qcom_glink_send_open_req(struct qcom_glink *glink, 424 struct glink_channel *channel) 425 { 426 struct { 427 struct glink_msg msg; 428 u8 name[GLINK_NAME_SIZE]; 429 } __packed req; 430 int name_len = strlen(channel->name) + 1; 431 int req_len = ALIGN(sizeof(req.msg) + name_len, 8); 432 int ret; 433 unsigned long flags; 434 435 kref_get(&channel->refcount); 436 437 spin_lock_irqsave(&glink->idr_lock, flags); 438 ret = idr_alloc_cyclic(&glink->lcids, channel, 439 RPM_GLINK_CID_MIN, RPM_GLINK_CID_MAX, 440 GFP_ATOMIC); 441 spin_unlock_irqrestore(&glink->idr_lock, flags); 442 if (ret < 0) 443 return ret; 444 445 channel->lcid = ret; 446 447 req.msg.cmd = cpu_to_le16(RPM_CMD_OPEN); 448 req.msg.param1 = cpu_to_le16(channel->lcid); 449 req.msg.param2 = cpu_to_le32(name_len); 450 strcpy(req.name, channel->name); 451 452 ret = qcom_glink_tx(glink, &req, req_len, NULL, 0, true); 453 if (ret) 454 goto remove_idr; 455 456 return 0; 457 458 remove_idr: 459 spin_lock_irqsave(&glink->idr_lock, flags); 460 idr_remove(&glink->lcids, channel->lcid); 461 channel->lcid = 0; 462 spin_unlock_irqrestore(&glink->idr_lock, flags); 463 464 return ret; 465 } 466 467 static void qcom_glink_send_close_req(struct qcom_glink *glink, 468 struct glink_channel *channel) 469 { 470 struct glink_msg req; 471 472 req.cmd = cpu_to_le16(RPM_CMD_CLOSE); 473 req.param1 = cpu_to_le16(channel->lcid); 474 req.param2 = 0; 475 476 qcom_glink_tx(glink, &req, sizeof(req), NULL, 0, true); 477 } 478 479 static void qcom_glink_send_close_ack(struct qcom_glink *glink, 480 unsigned int rcid) 481 { 482 struct glink_msg req; 483 484 req.cmd = cpu_to_le16(RPM_CMD_CLOSE_ACK); 485 req.param1 = cpu_to_le16(rcid); 486 req.param2 = 0; 487 488 qcom_glink_tx(glink, &req, sizeof(req), NULL, 0, true); 489 } 490 491 static void qcom_glink_rx_done_work(struct work_struct *work) 492 { 493 struct glink_channel *channel = container_of(work, struct glink_channel, 494 intent_work); 495 struct qcom_glink *glink = channel->glink; 496 struct glink_core_rx_intent *intent, *tmp; 497 struct { 498 u16 id; 499 u16 lcid; 500 u32 liid; 501 } __packed cmd; 502 503 unsigned int cid = channel->lcid; 504 unsigned int iid; 505 bool reuse; 506 unsigned long flags; 507 508 spin_lock_irqsave(&channel->intent_lock, flags); 509 list_for_each_entry_safe(intent, tmp, &channel->done_intents, node) { 510 list_del(&intent->node); 511 spin_unlock_irqrestore(&channel->intent_lock, flags); 512 iid = intent->id; 513 reuse = intent->reuse; 514 515 cmd.id = reuse ? RPM_CMD_RX_DONE_W_REUSE : RPM_CMD_RX_DONE; 516 cmd.lcid = cid; 517 cmd.liid = iid; 518 519 qcom_glink_tx(glink, &cmd, sizeof(cmd), NULL, 0, true); 520 if (!reuse) { 521 kfree(intent->data); 522 kfree(intent); 523 } 524 spin_lock_irqsave(&channel->intent_lock, flags); 525 } 526 spin_unlock_irqrestore(&channel->intent_lock, flags); 527 } 528 529 static void qcom_glink_rx_done(struct qcom_glink *glink, 530 struct glink_channel *channel, 531 struct glink_core_rx_intent *intent) 532 { 533 /* We don't send RX_DONE to intentless systems */ 534 if (glink->intentless) { 535 kfree(intent->data); 536 kfree(intent); 537 return; 538 } 539 540 /* Take it off the tree of receive intents */ 541 if (!intent->reuse) { 542 spin_lock(&channel->intent_lock); 543 idr_remove(&channel->liids, intent->id); 544 spin_unlock(&channel->intent_lock); 545 } 546 547 /* Schedule the sending of a rx_done indication */ 548 spin_lock(&channel->intent_lock); 549 list_add_tail(&intent->node, &channel->done_intents); 550 spin_unlock(&channel->intent_lock); 551 552 schedule_work(&channel->intent_work); 553 } 554 555 /** 556 * qcom_glink_receive_version() - receive version/features from remote system 557 * 558 * @glink: pointer to transport interface 559 * @version: remote version 560 * @features: remote features 561 * 562 * This function is called in response to a remote-initiated version/feature 563 * negotiation sequence. 564 */ 565 static void qcom_glink_receive_version(struct qcom_glink *glink, 566 u32 version, 567 u32 features) 568 { 569 switch (version) { 570 case 0: 571 break; 572 case GLINK_VERSION_1: 573 glink->features &= features; 574 fallthrough; 575 default: 576 qcom_glink_send_version_ack(glink); 577 break; 578 } 579 } 580 581 /** 582 * qcom_glink_receive_version_ack() - receive negotiation ack from remote system 583 * 584 * @glink: pointer to transport interface 585 * @version: remote version response 586 * @features: remote features response 587 * 588 * This function is called in response to a local-initiated version/feature 589 * negotiation sequence and is the counter-offer from the remote side based 590 * upon the initial version and feature set requested. 591 */ 592 static void qcom_glink_receive_version_ack(struct qcom_glink *glink, 593 u32 version, 594 u32 features) 595 { 596 switch (version) { 597 case 0: 598 /* Version negotiation failed */ 599 break; 600 case GLINK_VERSION_1: 601 if (features == glink->features) 602 break; 603 604 glink->features &= features; 605 fallthrough; 606 default: 607 qcom_glink_send_version(glink); 608 break; 609 } 610 } 611 612 /** 613 * qcom_glink_send_intent_req_ack() - convert an rx intent request ack cmd to 614 * wire format and transmit 615 * @glink: The transport to transmit on. 616 * @channel: The glink channel 617 * @granted: The request response to encode. 618 * 619 * Return: 0 on success or standard Linux error code. 620 */ 621 static int qcom_glink_send_intent_req_ack(struct qcom_glink *glink, 622 struct glink_channel *channel, 623 bool granted) 624 { 625 struct glink_msg msg; 626 627 msg.cmd = cpu_to_le16(RPM_CMD_RX_INTENT_REQ_ACK); 628 msg.param1 = cpu_to_le16(channel->lcid); 629 msg.param2 = cpu_to_le32(granted); 630 631 qcom_glink_tx(glink, &msg, sizeof(msg), NULL, 0, true); 632 633 return 0; 634 } 635 636 /** 637 * qcom_glink_advertise_intent - convert an rx intent cmd to wire format and 638 * transmit 639 * @glink: The transport to transmit on. 640 * @channel: The local channel 641 * @intent: The intent to pass on to remote. 642 * 643 * Return: 0 on success or standard Linux error code. 644 */ 645 static int qcom_glink_advertise_intent(struct qcom_glink *glink, 646 struct glink_channel *channel, 647 struct glink_core_rx_intent *intent) 648 { 649 struct command { 650 __le16 id; 651 __le16 lcid; 652 __le32 count; 653 __le32 size; 654 __le32 liid; 655 } __packed; 656 struct command cmd; 657 658 cmd.id = cpu_to_le16(RPM_CMD_INTENT); 659 cmd.lcid = cpu_to_le16(channel->lcid); 660 cmd.count = cpu_to_le32(1); 661 cmd.size = cpu_to_le32(intent->size); 662 cmd.liid = cpu_to_le32(intent->id); 663 664 qcom_glink_tx(glink, &cmd, sizeof(cmd), NULL, 0, true); 665 666 return 0; 667 } 668 669 static struct glink_core_rx_intent * 670 qcom_glink_alloc_intent(struct qcom_glink *glink, 671 struct glink_channel *channel, 672 size_t size, 673 bool reuseable) 674 { 675 struct glink_core_rx_intent *intent; 676 int ret; 677 unsigned long flags; 678 679 intent = kzalloc(sizeof(*intent), GFP_KERNEL); 680 if (!intent) 681 return NULL; 682 683 intent->data = kzalloc(size, GFP_KERNEL); 684 if (!intent->data) 685 goto free_intent; 686 687 spin_lock_irqsave(&channel->intent_lock, flags); 688 ret = idr_alloc_cyclic(&channel->liids, intent, 1, -1, GFP_ATOMIC); 689 if (ret < 0) { 690 spin_unlock_irqrestore(&channel->intent_lock, flags); 691 goto free_data; 692 } 693 spin_unlock_irqrestore(&channel->intent_lock, flags); 694 695 intent->id = ret; 696 intent->size = size; 697 intent->reuse = reuseable; 698 699 return intent; 700 701 free_data: 702 kfree(intent->data); 703 free_intent: 704 kfree(intent); 705 return NULL; 706 } 707 708 static void qcom_glink_handle_rx_done(struct qcom_glink *glink, 709 u32 cid, uint32_t iid, 710 bool reuse) 711 { 712 struct glink_core_rx_intent *intent; 713 struct glink_channel *channel; 714 unsigned long flags; 715 716 spin_lock_irqsave(&glink->idr_lock, flags); 717 channel = idr_find(&glink->rcids, cid); 718 spin_unlock_irqrestore(&glink->idr_lock, flags); 719 if (!channel) { 720 dev_err(glink->dev, "invalid channel id received\n"); 721 return; 722 } 723 724 spin_lock_irqsave(&channel->intent_lock, flags); 725 intent = idr_find(&channel->riids, iid); 726 727 if (!intent) { 728 spin_unlock_irqrestore(&channel->intent_lock, flags); 729 dev_err(glink->dev, "invalid intent id received\n"); 730 return; 731 } 732 733 intent->in_use = false; 734 735 if (!reuse) { 736 idr_remove(&channel->riids, intent->id); 737 kfree(intent); 738 } 739 spin_unlock_irqrestore(&channel->intent_lock, flags); 740 } 741 742 /** 743 * qcom_glink_handle_intent_req() - Receive a request for rx_intent 744 * from remote side 745 * @glink: Pointer to the transport interface 746 * @cid: Remote channel ID 747 * @size: size of the intent 748 * 749 * The function searches for the local channel to which the request for 750 * rx_intent has arrived and allocates and notifies the remote back 751 */ 752 static void qcom_glink_handle_intent_req(struct qcom_glink *glink, 753 u32 cid, size_t size) 754 { 755 struct glink_core_rx_intent *intent; 756 struct glink_channel *channel; 757 unsigned long flags; 758 759 spin_lock_irqsave(&glink->idr_lock, flags); 760 channel = idr_find(&glink->rcids, cid); 761 spin_unlock_irqrestore(&glink->idr_lock, flags); 762 763 if (!channel) { 764 pr_err("%s channel not found for cid %d\n", __func__, cid); 765 return; 766 } 767 768 intent = qcom_glink_alloc_intent(glink, channel, size, false); 769 if (intent) 770 qcom_glink_advertise_intent(glink, channel, intent); 771 772 qcom_glink_send_intent_req_ack(glink, channel, !!intent); 773 } 774 775 static int qcom_glink_rx_defer(struct qcom_glink *glink, size_t extra) 776 { 777 struct glink_defer_cmd *dcmd; 778 779 extra = ALIGN(extra, 8); 780 781 if (qcom_glink_rx_avail(glink) < sizeof(struct glink_msg) + extra) { 782 dev_dbg(glink->dev, "Insufficient data in rx fifo"); 783 return -ENXIO; 784 } 785 786 dcmd = kzalloc(struct_size(dcmd, data, extra), GFP_ATOMIC); 787 if (!dcmd) 788 return -ENOMEM; 789 790 INIT_LIST_HEAD(&dcmd->node); 791 792 qcom_glink_rx_peak(glink, &dcmd->msg, 0, sizeof(dcmd->msg) + extra); 793 794 spin_lock(&glink->rx_lock); 795 list_add_tail(&dcmd->node, &glink->rx_queue); 796 spin_unlock(&glink->rx_lock); 797 798 schedule_work(&glink->rx_work); 799 qcom_glink_rx_advance(glink, sizeof(dcmd->msg) + extra); 800 801 return 0; 802 } 803 804 static int qcom_glink_rx_data(struct qcom_glink *glink, size_t avail) 805 { 806 struct glink_core_rx_intent *intent; 807 struct glink_channel *channel; 808 struct { 809 struct glink_msg msg; 810 __le32 chunk_size; 811 __le32 left_size; 812 } __packed hdr; 813 unsigned int chunk_size; 814 unsigned int left_size; 815 unsigned int rcid; 816 unsigned int liid; 817 int ret = 0; 818 unsigned long flags; 819 820 if (avail < sizeof(hdr)) { 821 dev_dbg(glink->dev, "Not enough data in fifo\n"); 822 return -EAGAIN; 823 } 824 825 qcom_glink_rx_peak(glink, &hdr, 0, sizeof(hdr)); 826 chunk_size = le32_to_cpu(hdr.chunk_size); 827 left_size = le32_to_cpu(hdr.left_size); 828 829 if (avail < sizeof(hdr) + chunk_size) { 830 dev_dbg(glink->dev, "Payload not yet in fifo\n"); 831 return -EAGAIN; 832 } 833 834 rcid = le16_to_cpu(hdr.msg.param1); 835 spin_lock_irqsave(&glink->idr_lock, flags); 836 channel = idr_find(&glink->rcids, rcid); 837 spin_unlock_irqrestore(&glink->idr_lock, flags); 838 if (!channel) { 839 dev_dbg(glink->dev, "Data on non-existing channel\n"); 840 841 /* Drop the message */ 842 goto advance_rx; 843 } 844 845 if (glink->intentless) { 846 /* Might have an ongoing, fragmented, message to append */ 847 if (!channel->buf) { 848 intent = kzalloc(sizeof(*intent), GFP_ATOMIC); 849 if (!intent) 850 return -ENOMEM; 851 852 intent->data = kmalloc(chunk_size + left_size, 853 GFP_ATOMIC); 854 if (!intent->data) { 855 kfree(intent); 856 return -ENOMEM; 857 } 858 859 intent->id = 0xbabababa; 860 intent->size = chunk_size + left_size; 861 intent->offset = 0; 862 863 channel->buf = intent; 864 } else { 865 intent = channel->buf; 866 } 867 } else { 868 liid = le32_to_cpu(hdr.msg.param2); 869 870 spin_lock_irqsave(&channel->intent_lock, flags); 871 intent = idr_find(&channel->liids, liid); 872 spin_unlock_irqrestore(&channel->intent_lock, flags); 873 874 if (!intent) { 875 dev_err(glink->dev, 876 "no intent found for channel %s intent %d", 877 channel->name, liid); 878 ret = -ENOENT; 879 goto advance_rx; 880 } 881 } 882 883 if (intent->size - intent->offset < chunk_size) { 884 dev_err(glink->dev, "Insufficient space in intent\n"); 885 886 /* The packet header lied, drop payload */ 887 goto advance_rx; 888 } 889 890 qcom_glink_rx_peak(glink, intent->data + intent->offset, 891 sizeof(hdr), chunk_size); 892 intent->offset += chunk_size; 893 894 /* Handle message when no fragments remain to be received */ 895 if (!left_size) { 896 spin_lock(&channel->recv_lock); 897 if (channel->ept.cb) { 898 channel->ept.cb(channel->ept.rpdev, 899 intent->data, 900 intent->offset, 901 channel->ept.priv, 902 RPMSG_ADDR_ANY); 903 } 904 spin_unlock(&channel->recv_lock); 905 906 intent->offset = 0; 907 channel->buf = NULL; 908 909 qcom_glink_rx_done(glink, channel, intent); 910 } 911 912 advance_rx: 913 qcom_glink_rx_advance(glink, ALIGN(sizeof(hdr) + chunk_size, 8)); 914 915 return ret; 916 } 917 918 static void qcom_glink_handle_intent(struct qcom_glink *glink, 919 unsigned int cid, 920 unsigned int count, 921 size_t avail) 922 { 923 struct glink_core_rx_intent *intent; 924 struct glink_channel *channel; 925 struct intent_pair { 926 __le32 size; 927 __le32 iid; 928 }; 929 930 struct { 931 struct glink_msg msg; 932 struct intent_pair intents[]; 933 } __packed * msg; 934 935 const size_t msglen = struct_size(msg, intents, count); 936 int ret; 937 int i; 938 unsigned long flags; 939 940 if (avail < msglen) { 941 dev_dbg(glink->dev, "Not enough data in fifo\n"); 942 return; 943 } 944 945 spin_lock_irqsave(&glink->idr_lock, flags); 946 channel = idr_find(&glink->rcids, cid); 947 spin_unlock_irqrestore(&glink->idr_lock, flags); 948 if (!channel) { 949 dev_err(glink->dev, "intents for non-existing channel\n"); 950 return; 951 } 952 953 msg = kmalloc(msglen, GFP_ATOMIC); 954 if (!msg) 955 return; 956 957 qcom_glink_rx_peak(glink, msg, 0, msglen); 958 959 for (i = 0; i < count; ++i) { 960 intent = kzalloc(sizeof(*intent), GFP_ATOMIC); 961 if (!intent) 962 break; 963 964 intent->id = le32_to_cpu(msg->intents[i].iid); 965 intent->size = le32_to_cpu(msg->intents[i].size); 966 967 spin_lock_irqsave(&channel->intent_lock, flags); 968 ret = idr_alloc(&channel->riids, intent, 969 intent->id, intent->id + 1, GFP_ATOMIC); 970 spin_unlock_irqrestore(&channel->intent_lock, flags); 971 972 if (ret < 0) 973 dev_err(glink->dev, "failed to store remote intent\n"); 974 } 975 976 kfree(msg); 977 qcom_glink_rx_advance(glink, ALIGN(msglen, 8)); 978 } 979 980 static int qcom_glink_rx_open_ack(struct qcom_glink *glink, unsigned int lcid) 981 { 982 struct glink_channel *channel; 983 984 spin_lock(&glink->idr_lock); 985 channel = idr_find(&glink->lcids, lcid); 986 spin_unlock(&glink->idr_lock); 987 if (!channel) { 988 dev_err(glink->dev, "Invalid open ack packet\n"); 989 return -EINVAL; 990 } 991 992 complete_all(&channel->open_ack); 993 994 return 0; 995 } 996 997 void qcom_glink_native_rx(struct qcom_glink *glink) 998 { 999 struct glink_msg msg; 1000 unsigned int param1; 1001 unsigned int param2; 1002 unsigned int avail; 1003 unsigned int cmd; 1004 int ret = 0; 1005 1006 /* To wakeup any blocking writers */ 1007 wake_up_all(&glink->tx_avail_notify); 1008 1009 for (;;) { 1010 avail = qcom_glink_rx_avail(glink); 1011 if (avail < sizeof(msg)) 1012 break; 1013 1014 qcom_glink_rx_peak(glink, &msg, 0, sizeof(msg)); 1015 1016 cmd = le16_to_cpu(msg.cmd); 1017 param1 = le16_to_cpu(msg.param1); 1018 param2 = le32_to_cpu(msg.param2); 1019 1020 switch (cmd) { 1021 case RPM_CMD_VERSION: 1022 case RPM_CMD_VERSION_ACK: 1023 case RPM_CMD_CLOSE: 1024 case RPM_CMD_CLOSE_ACK: 1025 case RPM_CMD_RX_INTENT_REQ: 1026 ret = qcom_glink_rx_defer(glink, 0); 1027 break; 1028 case RPM_CMD_OPEN_ACK: 1029 ret = qcom_glink_rx_open_ack(glink, param1); 1030 qcom_glink_rx_advance(glink, ALIGN(sizeof(msg), 8)); 1031 break; 1032 case RPM_CMD_OPEN: 1033 ret = qcom_glink_rx_defer(glink, param2); 1034 break; 1035 case RPM_CMD_TX_DATA: 1036 case RPM_CMD_TX_DATA_CONT: 1037 ret = qcom_glink_rx_data(glink, avail); 1038 break; 1039 case RPM_CMD_READ_NOTIF: 1040 qcom_glink_rx_advance(glink, ALIGN(sizeof(msg), 8)); 1041 qcom_glink_tx_kick(glink); 1042 break; 1043 case RPM_CMD_INTENT: 1044 qcom_glink_handle_intent(glink, param1, param2, avail); 1045 break; 1046 case RPM_CMD_RX_DONE: 1047 qcom_glink_handle_rx_done(glink, param1, param2, false); 1048 qcom_glink_rx_advance(glink, ALIGN(sizeof(msg), 8)); 1049 break; 1050 case RPM_CMD_RX_DONE_W_REUSE: 1051 qcom_glink_handle_rx_done(glink, param1, param2, true); 1052 qcom_glink_rx_advance(glink, ALIGN(sizeof(msg), 8)); 1053 break; 1054 case RPM_CMD_RX_INTENT_REQ_ACK: 1055 qcom_glink_handle_intent_req_ack(glink, param1, param2); 1056 qcom_glink_rx_advance(glink, ALIGN(sizeof(msg), 8)); 1057 break; 1058 default: 1059 dev_err(glink->dev, "unhandled rx cmd: %d\n", cmd); 1060 ret = -EINVAL; 1061 break; 1062 } 1063 1064 if (ret) 1065 break; 1066 } 1067 } 1068 EXPORT_SYMBOL(qcom_glink_native_rx); 1069 1070 /* Locally initiated rpmsg_create_ept */ 1071 static struct glink_channel *qcom_glink_create_local(struct qcom_glink *glink, 1072 const char *name) 1073 { 1074 struct glink_channel *channel; 1075 int ret; 1076 unsigned long flags; 1077 1078 channel = qcom_glink_alloc_channel(glink, name); 1079 if (IS_ERR(channel)) 1080 return ERR_CAST(channel); 1081 1082 ret = qcom_glink_send_open_req(glink, channel); 1083 if (ret) 1084 goto release_channel; 1085 1086 ret = wait_for_completion_timeout(&channel->open_ack, 5 * HZ); 1087 if (!ret) 1088 goto err_timeout; 1089 1090 ret = wait_for_completion_timeout(&channel->open_req, 5 * HZ); 1091 if (!ret) 1092 goto err_timeout; 1093 1094 qcom_glink_send_open_ack(glink, channel); 1095 1096 return channel; 1097 1098 err_timeout: 1099 /* qcom_glink_send_open_req() did register the channel in lcids*/ 1100 spin_lock_irqsave(&glink->idr_lock, flags); 1101 idr_remove(&glink->lcids, channel->lcid); 1102 spin_unlock_irqrestore(&glink->idr_lock, flags); 1103 1104 release_channel: 1105 /* Release qcom_glink_send_open_req() reference */ 1106 kref_put(&channel->refcount, qcom_glink_channel_release); 1107 /* Release qcom_glink_alloc_channel() reference */ 1108 kref_put(&channel->refcount, qcom_glink_channel_release); 1109 1110 return ERR_PTR(-ETIMEDOUT); 1111 } 1112 1113 /* Remote initiated rpmsg_create_ept */ 1114 static int qcom_glink_create_remote(struct qcom_glink *glink, 1115 struct glink_channel *channel) 1116 { 1117 int ret; 1118 1119 qcom_glink_send_open_ack(glink, channel); 1120 1121 ret = qcom_glink_send_open_req(glink, channel); 1122 if (ret) 1123 goto close_link; 1124 1125 ret = wait_for_completion_timeout(&channel->open_ack, 5 * HZ); 1126 if (!ret) { 1127 ret = -ETIMEDOUT; 1128 goto close_link; 1129 } 1130 1131 return 0; 1132 1133 close_link: 1134 /* 1135 * Send a close request to "undo" our open-ack. The close-ack will 1136 * release qcom_glink_send_open_req() reference and the last reference 1137 * will be relesed after receiving remote_close or transport unregister 1138 * by calling qcom_glink_native_remove(). 1139 */ 1140 qcom_glink_send_close_req(glink, channel); 1141 1142 return ret; 1143 } 1144 1145 static struct rpmsg_endpoint *qcom_glink_create_ept(struct rpmsg_device *rpdev, 1146 rpmsg_rx_cb_t cb, 1147 void *priv, 1148 struct rpmsg_channel_info 1149 chinfo) 1150 { 1151 struct glink_channel *parent = to_glink_channel(rpdev->ept); 1152 struct glink_channel *channel; 1153 struct qcom_glink *glink = parent->glink; 1154 struct rpmsg_endpoint *ept; 1155 const char *name = chinfo.name; 1156 int cid; 1157 int ret; 1158 unsigned long flags; 1159 1160 spin_lock_irqsave(&glink->idr_lock, flags); 1161 idr_for_each_entry(&glink->rcids, channel, cid) { 1162 if (!strcmp(channel->name, name)) 1163 break; 1164 } 1165 spin_unlock_irqrestore(&glink->idr_lock, flags); 1166 1167 if (!channel) { 1168 channel = qcom_glink_create_local(glink, name); 1169 if (IS_ERR(channel)) 1170 return NULL; 1171 } else { 1172 ret = qcom_glink_create_remote(glink, channel); 1173 if (ret) 1174 return NULL; 1175 } 1176 1177 ept = &channel->ept; 1178 ept->rpdev = rpdev; 1179 ept->cb = cb; 1180 ept->priv = priv; 1181 ept->ops = &glink_endpoint_ops; 1182 1183 return ept; 1184 } 1185 1186 static int qcom_glink_announce_create(struct rpmsg_device *rpdev) 1187 { 1188 struct glink_channel *channel = to_glink_channel(rpdev->ept); 1189 struct device_node *np = rpdev->dev.of_node; 1190 struct qcom_glink *glink = channel->glink; 1191 struct glink_core_rx_intent *intent; 1192 const struct property *prop = NULL; 1193 __be32 defaults[] = { cpu_to_be32(SZ_1K), cpu_to_be32(5) }; 1194 int num_intents; 1195 int num_groups = 1; 1196 __be32 *val = defaults; 1197 int size; 1198 1199 if (glink->intentless || !completion_done(&channel->open_ack)) 1200 return 0; 1201 1202 prop = of_find_property(np, "qcom,intents", NULL); 1203 if (prop) { 1204 val = prop->value; 1205 num_groups = prop->length / sizeof(u32) / 2; 1206 } 1207 1208 /* Channel is now open, advertise base set of intents */ 1209 while (num_groups--) { 1210 size = be32_to_cpup(val++); 1211 num_intents = be32_to_cpup(val++); 1212 while (num_intents--) { 1213 intent = qcom_glink_alloc_intent(glink, channel, size, 1214 true); 1215 if (!intent) 1216 break; 1217 1218 qcom_glink_advertise_intent(glink, channel, intent); 1219 } 1220 } 1221 return 0; 1222 } 1223 1224 static void qcom_glink_destroy_ept(struct rpmsg_endpoint *ept) 1225 { 1226 struct glink_channel *channel = to_glink_channel(ept); 1227 struct qcom_glink *glink = channel->glink; 1228 unsigned long flags; 1229 1230 spin_lock_irqsave(&channel->recv_lock, flags); 1231 channel->ept.cb = NULL; 1232 spin_unlock_irqrestore(&channel->recv_lock, flags); 1233 1234 /* Decouple the potential rpdev from the channel */ 1235 channel->rpdev = NULL; 1236 1237 qcom_glink_send_close_req(glink, channel); 1238 } 1239 1240 static int qcom_glink_request_intent(struct qcom_glink *glink, 1241 struct glink_channel *channel, 1242 size_t size) 1243 { 1244 struct { 1245 u16 id; 1246 u16 cid; 1247 u32 size; 1248 } __packed cmd; 1249 1250 int ret; 1251 1252 mutex_lock(&channel->intent_req_lock); 1253 1254 reinit_completion(&channel->intent_req_comp); 1255 1256 cmd.id = RPM_CMD_RX_INTENT_REQ; 1257 cmd.cid = channel->lcid; 1258 cmd.size = size; 1259 1260 ret = qcom_glink_tx(glink, &cmd, sizeof(cmd), NULL, 0, true); 1261 if (ret) 1262 goto unlock; 1263 1264 ret = wait_for_completion_timeout(&channel->intent_req_comp, 10 * HZ); 1265 if (!ret) { 1266 dev_err(glink->dev, "intent request timed out\n"); 1267 ret = -ETIMEDOUT; 1268 } else { 1269 ret = channel->intent_req_result ? 0 : -ECANCELED; 1270 } 1271 1272 unlock: 1273 mutex_unlock(&channel->intent_req_lock); 1274 return ret; 1275 } 1276 1277 static int __qcom_glink_send(struct glink_channel *channel, 1278 void *data, int len, bool wait) 1279 { 1280 struct qcom_glink *glink = channel->glink; 1281 struct glink_core_rx_intent *intent = NULL; 1282 struct glink_core_rx_intent *tmp; 1283 int iid = 0; 1284 struct { 1285 struct glink_msg msg; 1286 __le32 chunk_size; 1287 __le32 left_size; 1288 } __packed req; 1289 int ret; 1290 unsigned long flags; 1291 int chunk_size = len; 1292 int left_size = 0; 1293 1294 if (!glink->intentless) { 1295 while (!intent) { 1296 spin_lock_irqsave(&channel->intent_lock, flags); 1297 idr_for_each_entry(&channel->riids, tmp, iid) { 1298 if (tmp->size >= len && !tmp->in_use) { 1299 if (!intent) 1300 intent = tmp; 1301 else if (intent->size > tmp->size) 1302 intent = tmp; 1303 if (intent->size == len) 1304 break; 1305 } 1306 } 1307 if (intent) 1308 intent->in_use = true; 1309 spin_unlock_irqrestore(&channel->intent_lock, flags); 1310 1311 /* We found an available intent */ 1312 if (intent) 1313 break; 1314 1315 if (!wait) 1316 return -EBUSY; 1317 1318 ret = qcom_glink_request_intent(glink, channel, len); 1319 if (ret < 0) 1320 return ret; 1321 } 1322 1323 iid = intent->id; 1324 } 1325 1326 if (wait && chunk_size > SZ_8K) { 1327 chunk_size = SZ_8K; 1328 left_size = len - chunk_size; 1329 } 1330 req.msg.cmd = cpu_to_le16(RPM_CMD_TX_DATA); 1331 req.msg.param1 = cpu_to_le16(channel->lcid); 1332 req.msg.param2 = cpu_to_le32(iid); 1333 req.chunk_size = cpu_to_le32(chunk_size); 1334 req.left_size = cpu_to_le32(left_size); 1335 1336 ret = qcom_glink_tx(glink, &req, sizeof(req), data, chunk_size, wait); 1337 1338 /* Mark intent available if we failed */ 1339 if (ret && intent) { 1340 intent->in_use = false; 1341 return ret; 1342 } 1343 1344 while (left_size > 0) { 1345 data = (void *)((char *)data + chunk_size); 1346 chunk_size = left_size; 1347 if (chunk_size > SZ_8K) 1348 chunk_size = SZ_8K; 1349 left_size -= chunk_size; 1350 1351 req.msg.cmd = cpu_to_le16(RPM_CMD_TX_DATA_CONT); 1352 req.msg.param1 = cpu_to_le16(channel->lcid); 1353 req.msg.param2 = cpu_to_le32(iid); 1354 req.chunk_size = cpu_to_le32(chunk_size); 1355 req.left_size = cpu_to_le32(left_size); 1356 1357 ret = qcom_glink_tx(glink, &req, sizeof(req), data, 1358 chunk_size, wait); 1359 1360 /* Mark intent available if we failed */ 1361 if (ret && intent) { 1362 intent->in_use = false; 1363 break; 1364 } 1365 } 1366 return ret; 1367 } 1368 1369 static int qcom_glink_send(struct rpmsg_endpoint *ept, void *data, int len) 1370 { 1371 struct glink_channel *channel = to_glink_channel(ept); 1372 1373 return __qcom_glink_send(channel, data, len, true); 1374 } 1375 1376 static int qcom_glink_trysend(struct rpmsg_endpoint *ept, void *data, int len) 1377 { 1378 struct glink_channel *channel = to_glink_channel(ept); 1379 1380 return __qcom_glink_send(channel, data, len, false); 1381 } 1382 1383 static int qcom_glink_sendto(struct rpmsg_endpoint *ept, void *data, int len, u32 dst) 1384 { 1385 struct glink_channel *channel = to_glink_channel(ept); 1386 1387 return __qcom_glink_send(channel, data, len, true); 1388 } 1389 1390 static int qcom_glink_trysendto(struct rpmsg_endpoint *ept, void *data, int len, u32 dst) 1391 { 1392 struct glink_channel *channel = to_glink_channel(ept); 1393 1394 return __qcom_glink_send(channel, data, len, false); 1395 } 1396 1397 /* 1398 * Finds the device_node for the glink child interested in this channel. 1399 */ 1400 static struct device_node *qcom_glink_match_channel(struct device_node *node, 1401 const char *channel) 1402 { 1403 struct device_node *child; 1404 const char *name; 1405 const char *key; 1406 int ret; 1407 1408 for_each_available_child_of_node(node, child) { 1409 key = "qcom,glink-channels"; 1410 ret = of_property_read_string(child, key, &name); 1411 if (ret) 1412 continue; 1413 1414 if (strcmp(name, channel) == 0) 1415 return child; 1416 } 1417 1418 return NULL; 1419 } 1420 1421 static const struct rpmsg_device_ops glink_device_ops = { 1422 .create_ept = qcom_glink_create_ept, 1423 .announce_create = qcom_glink_announce_create, 1424 }; 1425 1426 static const struct rpmsg_endpoint_ops glink_endpoint_ops = { 1427 .destroy_ept = qcom_glink_destroy_ept, 1428 .send = qcom_glink_send, 1429 .sendto = qcom_glink_sendto, 1430 .trysend = qcom_glink_trysend, 1431 .trysendto = qcom_glink_trysendto, 1432 }; 1433 1434 static void qcom_glink_rpdev_release(struct device *dev) 1435 { 1436 struct rpmsg_device *rpdev = to_rpmsg_device(dev); 1437 1438 kfree(rpdev); 1439 } 1440 1441 static int qcom_glink_rx_open(struct qcom_glink *glink, unsigned int rcid, 1442 char *name) 1443 { 1444 struct glink_channel *channel; 1445 struct rpmsg_device *rpdev; 1446 bool create_device = false; 1447 struct device_node *node; 1448 int lcid; 1449 int ret; 1450 unsigned long flags; 1451 1452 spin_lock_irqsave(&glink->idr_lock, flags); 1453 idr_for_each_entry(&glink->lcids, channel, lcid) { 1454 if (!strcmp(channel->name, name)) 1455 break; 1456 } 1457 spin_unlock_irqrestore(&glink->idr_lock, flags); 1458 1459 if (!channel) { 1460 channel = qcom_glink_alloc_channel(glink, name); 1461 if (IS_ERR(channel)) 1462 return PTR_ERR(channel); 1463 1464 /* The opening dance was initiated by the remote */ 1465 create_device = true; 1466 } 1467 1468 spin_lock_irqsave(&glink->idr_lock, flags); 1469 ret = idr_alloc(&glink->rcids, channel, rcid, rcid + 1, GFP_ATOMIC); 1470 if (ret < 0) { 1471 dev_err(glink->dev, "Unable to insert channel into rcid list\n"); 1472 spin_unlock_irqrestore(&glink->idr_lock, flags); 1473 goto free_channel; 1474 } 1475 channel->rcid = ret; 1476 spin_unlock_irqrestore(&glink->idr_lock, flags); 1477 1478 complete_all(&channel->open_req); 1479 1480 if (create_device) { 1481 rpdev = kzalloc(sizeof(*rpdev), GFP_KERNEL); 1482 if (!rpdev) { 1483 ret = -ENOMEM; 1484 goto rcid_remove; 1485 } 1486 1487 rpdev->ept = &channel->ept; 1488 strscpy_pad(rpdev->id.name, name, RPMSG_NAME_SIZE); 1489 rpdev->src = RPMSG_ADDR_ANY; 1490 rpdev->dst = RPMSG_ADDR_ANY; 1491 rpdev->ops = &glink_device_ops; 1492 1493 node = qcom_glink_match_channel(glink->dev->of_node, name); 1494 rpdev->dev.of_node = node; 1495 rpdev->dev.parent = glink->dev; 1496 rpdev->dev.release = qcom_glink_rpdev_release; 1497 1498 ret = rpmsg_register_device(rpdev); 1499 if (ret) 1500 goto rcid_remove; 1501 1502 channel->rpdev = rpdev; 1503 } 1504 1505 return 0; 1506 1507 rcid_remove: 1508 spin_lock_irqsave(&glink->idr_lock, flags); 1509 idr_remove(&glink->rcids, channel->rcid); 1510 channel->rcid = 0; 1511 spin_unlock_irqrestore(&glink->idr_lock, flags); 1512 free_channel: 1513 /* Release the reference, iff we took it */ 1514 if (create_device) 1515 kref_put(&channel->refcount, qcom_glink_channel_release); 1516 1517 return ret; 1518 } 1519 1520 static void qcom_glink_rx_close(struct qcom_glink *glink, unsigned int rcid) 1521 { 1522 struct rpmsg_channel_info chinfo; 1523 struct glink_channel *channel; 1524 unsigned long flags; 1525 1526 spin_lock_irqsave(&glink->idr_lock, flags); 1527 channel = idr_find(&glink->rcids, rcid); 1528 spin_unlock_irqrestore(&glink->idr_lock, flags); 1529 if (WARN(!channel, "close request on unknown channel\n")) 1530 return; 1531 1532 /* cancel pending rx_done work */ 1533 cancel_work_sync(&channel->intent_work); 1534 1535 if (channel->rpdev) { 1536 strscpy_pad(chinfo.name, channel->name, sizeof(chinfo.name)); 1537 chinfo.src = RPMSG_ADDR_ANY; 1538 chinfo.dst = RPMSG_ADDR_ANY; 1539 1540 rpmsg_unregister_device(glink->dev, &chinfo); 1541 } 1542 channel->rpdev = NULL; 1543 1544 qcom_glink_send_close_ack(glink, channel->rcid); 1545 1546 spin_lock_irqsave(&glink->idr_lock, flags); 1547 idr_remove(&glink->rcids, channel->rcid); 1548 channel->rcid = 0; 1549 spin_unlock_irqrestore(&glink->idr_lock, flags); 1550 1551 kref_put(&channel->refcount, qcom_glink_channel_release); 1552 } 1553 1554 static void qcom_glink_rx_close_ack(struct qcom_glink *glink, unsigned int lcid) 1555 { 1556 struct rpmsg_channel_info chinfo; 1557 struct glink_channel *channel; 1558 unsigned long flags; 1559 1560 /* To wakeup any blocking writers */ 1561 wake_up_all(&glink->tx_avail_notify); 1562 1563 spin_lock_irqsave(&glink->idr_lock, flags); 1564 channel = idr_find(&glink->lcids, lcid); 1565 if (WARN(!channel, "close ack on unknown channel\n")) { 1566 spin_unlock_irqrestore(&glink->idr_lock, flags); 1567 return; 1568 } 1569 1570 idr_remove(&glink->lcids, channel->lcid); 1571 channel->lcid = 0; 1572 spin_unlock_irqrestore(&glink->idr_lock, flags); 1573 1574 /* Decouple the potential rpdev from the channel */ 1575 if (channel->rpdev) { 1576 strscpy(chinfo.name, channel->name, sizeof(chinfo.name)); 1577 chinfo.src = RPMSG_ADDR_ANY; 1578 chinfo.dst = RPMSG_ADDR_ANY; 1579 1580 rpmsg_unregister_device(glink->dev, &chinfo); 1581 } 1582 channel->rpdev = NULL; 1583 1584 kref_put(&channel->refcount, qcom_glink_channel_release); 1585 } 1586 1587 static void qcom_glink_work(struct work_struct *work) 1588 { 1589 struct qcom_glink *glink = container_of(work, struct qcom_glink, 1590 rx_work); 1591 struct glink_defer_cmd *dcmd; 1592 struct glink_msg *msg; 1593 unsigned long flags; 1594 unsigned int param1; 1595 unsigned int param2; 1596 unsigned int cmd; 1597 1598 for (;;) { 1599 spin_lock_irqsave(&glink->rx_lock, flags); 1600 if (list_empty(&glink->rx_queue)) { 1601 spin_unlock_irqrestore(&glink->rx_lock, flags); 1602 break; 1603 } 1604 dcmd = list_first_entry(&glink->rx_queue, 1605 struct glink_defer_cmd, node); 1606 list_del(&dcmd->node); 1607 spin_unlock_irqrestore(&glink->rx_lock, flags); 1608 1609 msg = &dcmd->msg; 1610 cmd = le16_to_cpu(msg->cmd); 1611 param1 = le16_to_cpu(msg->param1); 1612 param2 = le32_to_cpu(msg->param2); 1613 1614 switch (cmd) { 1615 case RPM_CMD_VERSION: 1616 qcom_glink_receive_version(glink, param1, param2); 1617 break; 1618 case RPM_CMD_VERSION_ACK: 1619 qcom_glink_receive_version_ack(glink, param1, param2); 1620 break; 1621 case RPM_CMD_OPEN: 1622 qcom_glink_rx_open(glink, param1, msg->data); 1623 break; 1624 case RPM_CMD_CLOSE: 1625 qcom_glink_rx_close(glink, param1); 1626 break; 1627 case RPM_CMD_CLOSE_ACK: 1628 qcom_glink_rx_close_ack(glink, param1); 1629 break; 1630 case RPM_CMD_RX_INTENT_REQ: 1631 qcom_glink_handle_intent_req(glink, param1, param2); 1632 break; 1633 default: 1634 WARN(1, "Unknown defer object %d\n", cmd); 1635 break; 1636 } 1637 1638 kfree(dcmd); 1639 } 1640 } 1641 1642 static void qcom_glink_cancel_rx_work(struct qcom_glink *glink) 1643 { 1644 struct glink_defer_cmd *dcmd; 1645 struct glink_defer_cmd *tmp; 1646 1647 /* cancel any pending deferred rx_work */ 1648 cancel_work_sync(&glink->rx_work); 1649 1650 list_for_each_entry_safe(dcmd, tmp, &glink->rx_queue, node) 1651 kfree(dcmd); 1652 } 1653 1654 static ssize_t rpmsg_name_show(struct device *dev, 1655 struct device_attribute *attr, char *buf) 1656 { 1657 int ret = 0; 1658 const char *name; 1659 1660 ret = of_property_read_string(dev->of_node, "label", &name); 1661 if (ret < 0) 1662 name = dev->of_node->name; 1663 1664 return sysfs_emit(buf, "%s\n", name); 1665 } 1666 static DEVICE_ATTR_RO(rpmsg_name); 1667 1668 static struct attribute *qcom_glink_attrs[] = { 1669 &dev_attr_rpmsg_name.attr, 1670 NULL 1671 }; 1672 ATTRIBUTE_GROUPS(qcom_glink); 1673 1674 static void qcom_glink_device_release(struct device *dev) 1675 { 1676 struct rpmsg_device *rpdev = to_rpmsg_device(dev); 1677 struct glink_channel *channel = to_glink_channel(rpdev->ept); 1678 1679 /* Release qcom_glink_alloc_channel() reference */ 1680 kref_put(&channel->refcount, qcom_glink_channel_release); 1681 kfree(rpdev); 1682 } 1683 1684 static int qcom_glink_create_chrdev(struct qcom_glink *glink) 1685 { 1686 struct rpmsg_device *rpdev; 1687 struct glink_channel *channel; 1688 1689 rpdev = kzalloc(sizeof(*rpdev), GFP_KERNEL); 1690 if (!rpdev) 1691 return -ENOMEM; 1692 1693 channel = qcom_glink_alloc_channel(glink, "rpmsg_chrdev"); 1694 if (IS_ERR(channel)) { 1695 kfree(rpdev); 1696 return PTR_ERR(channel); 1697 } 1698 channel->rpdev = rpdev; 1699 1700 rpdev->ept = &channel->ept; 1701 rpdev->ops = &glink_device_ops; 1702 rpdev->dev.parent = glink->dev; 1703 rpdev->dev.release = qcom_glink_device_release; 1704 1705 return rpmsg_ctrldev_register_device(rpdev); 1706 } 1707 1708 struct qcom_glink *qcom_glink_native_probe(struct device *dev, 1709 unsigned long features, 1710 struct qcom_glink_pipe *rx, 1711 struct qcom_glink_pipe *tx, 1712 bool intentless) 1713 { 1714 int ret; 1715 struct qcom_glink *glink; 1716 1717 glink = devm_kzalloc(dev, sizeof(*glink), GFP_KERNEL); 1718 if (!glink) 1719 return ERR_PTR(-ENOMEM); 1720 1721 glink->dev = dev; 1722 glink->tx_pipe = tx; 1723 glink->rx_pipe = rx; 1724 1725 glink->features = features; 1726 glink->intentless = intentless; 1727 1728 spin_lock_init(&glink->tx_lock); 1729 spin_lock_init(&glink->rx_lock); 1730 INIT_LIST_HEAD(&glink->rx_queue); 1731 INIT_WORK(&glink->rx_work, qcom_glink_work); 1732 init_waitqueue_head(&glink->tx_avail_notify); 1733 1734 spin_lock_init(&glink->idr_lock); 1735 idr_init(&glink->lcids); 1736 idr_init(&glink->rcids); 1737 1738 glink->dev->groups = qcom_glink_groups; 1739 1740 ret = device_add_groups(dev, qcom_glink_groups); 1741 if (ret) 1742 dev_err(dev, "failed to add groups\n"); 1743 1744 ret = qcom_glink_send_version(glink); 1745 if (ret) 1746 return ERR_PTR(ret); 1747 1748 ret = qcom_glink_create_chrdev(glink); 1749 if (ret) 1750 dev_err(glink->dev, "failed to register chrdev\n"); 1751 1752 return glink; 1753 } 1754 EXPORT_SYMBOL_GPL(qcom_glink_native_probe); 1755 1756 static int qcom_glink_remove_device(struct device *dev, void *data) 1757 { 1758 device_unregister(dev); 1759 1760 return 0; 1761 } 1762 1763 void qcom_glink_native_remove(struct qcom_glink *glink) 1764 { 1765 struct glink_channel *channel; 1766 int cid; 1767 int ret; 1768 1769 qcom_glink_cancel_rx_work(glink); 1770 1771 ret = device_for_each_child(glink->dev, NULL, qcom_glink_remove_device); 1772 if (ret) 1773 dev_warn(glink->dev, "Can't remove GLINK devices: %d\n", ret); 1774 1775 /* Release any defunct local channels, waiting for close-ack */ 1776 idr_for_each_entry(&glink->lcids, channel, cid) 1777 kref_put(&channel->refcount, qcom_glink_channel_release); 1778 1779 /* Release any defunct local channels, waiting for close-req */ 1780 idr_for_each_entry(&glink->rcids, channel, cid) 1781 kref_put(&channel->refcount, qcom_glink_channel_release); 1782 1783 idr_destroy(&glink->lcids); 1784 idr_destroy(&glink->rcids); 1785 } 1786 EXPORT_SYMBOL_GPL(qcom_glink_native_remove); 1787 1788 MODULE_DESCRIPTION("Qualcomm GLINK driver"); 1789 MODULE_LICENSE("GPL v2"); 1790