?  search:
summary | log | graphiclog1 | graphiclog2 | commit | commitdiff | tree | refs | edit | fork
blame | history | raw | HEAD
Merge tag 'trace-printf-v6.13' of git://git.kernel.org/pub/scm/linux/kernel/git/trace...
[drm/drm-misc.git] / drivers / rpmsg / qcom_glink_native.c
bloba2f9d85c7156dca5794b740fae98f98661d10b11
1 // SPDX-License-Identifier: GPL-2.0
2 /*
3 * Copyright (c) 2016-2017, Linaro Ltd
4 */
5
6 #include <linux/idr.h>
7 #include <linux/interrupt.h>
8 #include <linux/io.h>
9 #include <linux/list.h>
10 #include <linux/mfd/syscon.h>
11 #include <linux/module.h>
12 #include <linux/of.h>
13 #include <linux/of_address.h>
14 #include <linux/platform_device.h>
15 #include <linux/regmap.h>
16 #include <linux/rpmsg.h>
17 #include <linux/sizes.h>
18 #include <linux/slab.h>
19 #include <linux/wait.h>
20 #include <linux/workqueue.h>
21 #include <linux/mailbox_client.h>
22
23 #include "rpmsg_internal.h"
24 #include "qcom_glink_native.h"
25
26 #define CREATE_TRACE_POINTS
27 #include "qcom_glink_trace.h"
28
29 #define GLINK_NAME_SIZE 32
30 #define GLINK_VERSION_1 1
31
32 #define RPM_GLINK_CID_MIN 1
33 #define RPM_GLINK_CID_MAX 65536
34
35 struct glink_msg {
36 /* New members MUST be added within the __struct_group() macro below. */
37 __struct_group(glink_msg_hdr, hdr, __packed,
38 __le16 cmd;
39 __le16 param1;
40 __le32 param2;
41 );
42 u8 data[];
43 } __packed;
44 static_assert(offsetof(struct glink_msg, data) == sizeof(struct glink_msg_hdr),
45 "struct member likely outside of __struct_group()");
46
47 /**
48 * struct glink_defer_cmd - deferred incoming control message
49 * @node: list node
50 * @msg: message header
51 * @data: payload of the message
52 *
53 * Copy of a received control message, to be added to @rx_queue and processed
54 * by @rx_work of @qcom_glink.
55 */
56 struct glink_defer_cmd {
57 struct list_head node;
58
59 struct glink_msg_hdr msg;
60 u8 data[];
61 };
62
63 /**
64 * struct glink_core_rx_intent - RX intent
65 * RX intent
66 *
67 * @data: pointer to the data (may be NULL for zero-copy)
68 * @id: remote or local intent ID
69 * @size: size of the original intent (do not modify)
70 * @reuse: To mark if the intent can be reused after first use
71 * @in_use: To mark if intent is already in use for the channel
72 * @offset: next write offset (initially 0)
73 * @node: list node
74 */
75 struct glink_core_rx_intent {
76 void *data;
77 u32 id;
78 size_t size;
79 bool reuse;
80 bool in_use;
81 u32 offset;
82
83 struct list_head node;
84 };
85
86 /**
87 * struct qcom_glink - driver context, relates to one remote subsystem
88 * @dev: reference to the associated struct device
89 * @label: identifier of the glink edge
90 * @rx_pipe: pipe object for receive FIFO
91 * @tx_pipe: pipe object for transmit FIFO
92 * @rx_work: worker for handling received control messages
93 * @rx_lock: protects the @rx_queue
94 * @rx_queue: queue of received control messages to be processed in @rx_work
95 * @tx_lock: synchronizes operations on the tx fifo
96 * @idr_lock: synchronizes @lcids and @rcids modifications
97 * @lcids: idr of all channels with a known local channel id
98 * @rcids: idr of all channels with a known remote channel id
99 * @features: remote features
100 * @intentless: flag to indicate that there is no intent
101 * @tx_avail_notify: Waitqueue for pending tx tasks
102 * @sent_read_notify: flag to check cmd sent or not
103 * @abort_tx: flag indicating that all tx attempts should fail
104 */
105 struct qcom_glink {
106 struct device *dev;
107
108 const char *label;
109
110 struct qcom_glink_pipe *rx_pipe;
111 struct qcom_glink_pipe *tx_pipe;
112
113 struct work_struct rx_work;
114 spinlock_t rx_lock;
115 struct list_head rx_queue;
116
117 spinlock_t tx_lock;
118
119 spinlock_t idr_lock;
120 struct idr lcids;
121 struct idr rcids;
122 unsigned long features;
123
124 bool intentless;
125 wait_queue_head_t tx_avail_notify;
126 bool sent_read_notify;
127
128 bool abort_tx;
129 };
130
131 enum {
132 GLINK_STATE_CLOSED,
133 GLINK_STATE_OPENING,
134 GLINK_STATE_OPEN,
135 GLINK_STATE_CLOSING,
136 };
137
138 /**
139 * struct glink_channel - internal representation of a channel
140 * @rpdev: rpdev reference, only used for primary endpoints
141 * @ept: rpmsg endpoint this channel is associated with
142 * @glink: qcom_glink context handle
143 * @refcount: refcount for the channel object
144 * @recv_lock: guard for @ept.cb
145 * @name: unique channel name/identifier
146 * @lcid: channel id, in local space
147 * @rcid: channel id, in remote space
148 * @intent_lock: lock for protection of @liids, @riids
149 * @liids: idr of all local intents
150 * @riids: idr of all remote intents
151 * @intent_work: worker responsible for transmitting rx_done packets
152 * @done_intents: list of intents that needs to be announced rx_done
153 * @buf: receive buffer, for gathering fragments
154 * @buf_offset: write offset in @buf
155 * @buf_size: size of current @buf
156 * @open_ack: completed once remote has acked the open-request
157 * @open_req: completed once open-request has been received
158 * @intent_req_lock: Synchronises multiple intent requests
159 * @intent_req_result: Result of intent request
160 * @intent_received: flag indicating that an intent has been received
161 * @intent_req_wq: wait queue for intent_req signalling
162 */
163 struct glink_channel {
164 struct rpmsg_endpoint ept;
165
166 struct rpmsg_device *rpdev;
167 struct qcom_glink *glink;
168
169 struct kref refcount;
170
171 spinlock_t recv_lock;
172
173 char *name;
174 unsigned int lcid;
175 unsigned int rcid;
176
177 spinlock_t intent_lock;
178 struct idr liids;
179 struct idr riids;
180 struct work_struct intent_work;
181 struct list_head done_intents;
182
183 struct glink_core_rx_intent *buf;
184 int buf_offset;
185 int buf_size;
186
187 struct completion open_ack;
188 struct completion open_req;
189
190 struct mutex intent_req_lock;
191 int intent_req_result;
192 bool intent_received;
193 wait_queue_head_t intent_req_wq;
194 };
195
196 #define to_glink_channel(_ept) container_of(_ept, struct glink_channel, ept)
197
198 static const struct rpmsg_endpoint_ops glink_endpoint_ops;
199
200 #define GLINK_CMD_VERSION 0
201 #define GLINK_CMD_VERSION_ACK 1
202 #define GLINK_CMD_OPEN 2
203 #define GLINK_CMD_CLOSE 3
204 #define GLINK_CMD_OPEN_ACK 4
205 #define GLINK_CMD_INTENT 5
206 #define GLINK_CMD_RX_DONE 6
207 #define GLINK_CMD_RX_INTENT_REQ 7
208 #define GLINK_CMD_RX_INTENT_REQ_ACK 8
209 #define GLINK_CMD_TX_DATA 9
210 #define GLINK_CMD_CLOSE_ACK 11
211 #define GLINK_CMD_TX_DATA_CONT 12
212 #define GLINK_CMD_READ_NOTIF 13
213 #define GLINK_CMD_RX_DONE_W_REUSE 14
214 #define GLINK_CMD_SIGNALS 15
215
216 #define GLINK_FEATURE_INTENTLESS BIT(1)
217
218 #define NATIVE_DTR_SIG NATIVE_DSR_SIG
219 #define NATIVE_DSR_SIG BIT(31)
220 #define NATIVE_RTS_SIG NATIVE_CTS_SIG
221 #define NATIVE_CTS_SIG BIT(30)
222
223 static void qcom_glink_rx_done_work(struct work_struct *work);
224
225 static struct glink_channel *qcom_glink_alloc_channel(struct qcom_glink *glink,
226 const char *name)
227 {
228 struct glink_channel *channel;
229
230 channel = kzalloc(sizeof(*channel), GFP_KERNEL);
231 if (!channel)
232 return ERR_PTR(-ENOMEM);
233
234 /* Setup glink internal glink_channel data */
235 spin_lock_init(&channel->recv_lock);
236 spin_lock_init(&channel->intent_lock);
237 mutex_init(&channel->intent_req_lock);
238
239 channel->glink = glink;
240 channel->name = kstrdup(name, GFP_KERNEL);
241 if (!channel->name) {
242 kfree(channel);
243 return ERR_PTR(-ENOMEM);
244 }
245
246 init_completion(&channel->open_req);
247 init_completion(&channel->open_ack);
248 init_waitqueue_head(&channel->intent_req_wq);
249
250 INIT_LIST_HEAD(&channel->done_intents);
251 INIT_WORK(&channel->intent_work, qcom_glink_rx_done_work);
252
253 idr_init(&channel->liids);
254 idr_init(&channel->riids);
255 kref_init(&channel->refcount);
256
257 return channel;
258 }
259
260 static void qcom_glink_channel_release(struct kref *ref)
261 {
262 struct glink_channel *channel = container_of(ref, struct glink_channel,
263 refcount);
264 struct glink_core_rx_intent *intent;
265 struct glink_core_rx_intent *tmp;
266 unsigned long flags;
267 int iid;
268
269 /* cancel pending rx_done work */
270 cancel_work_sync(&channel->intent_work);
271
272 spin_lock_irqsave(&channel->intent_lock, flags);
273 /* Free all non-reuse intents pending rx_done work */
274 list_for_each_entry_safe(intent, tmp, &channel->done_intents, node) {
275 if (!intent->reuse) {
276 kfree(intent->data);
277 kfree(intent);
278 }
279 }
280
281 idr_for_each_entry(&channel->liids, tmp, iid) {
282 kfree(tmp->data);
283 kfree(tmp);
284 }
285 idr_destroy(&channel->liids);
286
287 idr_for_each_entry(&channel->riids, tmp, iid)
288 kfree(tmp);
289 idr_destroy(&channel->riids);
290 spin_unlock_irqrestore(&channel->intent_lock, flags);
291
292 kfree(channel->name);
293 kfree(channel);
294 }
295
296 static size_t qcom_glink_rx_avail(struct qcom_glink *glink)
297 {
298 return glink->rx_pipe->avail(glink->rx_pipe);
299 }
300
301 static void qcom_glink_rx_peek(struct qcom_glink *glink,
302 void *data, unsigned int offset, size_t count)
303 {
304 glink->rx_pipe->peek(glink->rx_pipe, data, offset, count);
305 }
306
307 static void qcom_glink_rx_advance(struct qcom_glink *glink, size_t count)
308 {
309 glink->rx_pipe->advance(glink->rx_pipe, count);
310 }
311
312 static size_t qcom_glink_tx_avail(struct qcom_glink *glink)
313 {
314 return glink->tx_pipe->avail(glink->tx_pipe);
315 }
316
317 static void qcom_glink_tx_write(struct qcom_glink *glink,
318 const void *hdr, size_t hlen,
319 const void *data, size_t dlen)
320 {
321 glink->tx_pipe->write(glink->tx_pipe, hdr, hlen, data, dlen);
322 }
323
324 static void qcom_glink_tx_kick(struct qcom_glink *glink)
325 {
326 glink->tx_pipe->kick(glink->tx_pipe);
327 }
328
329 static void qcom_glink_send_read_notify(struct qcom_glink *glink)
330 {
331 struct glink_msg msg;
332
333 msg.cmd = cpu_to_le16(GLINK_CMD_READ_NOTIF);
334 msg.param1 = 0;
335 msg.param2 = 0;
336
337 qcom_glink_tx_write(glink, &msg, sizeof(msg), NULL, 0);
338
339 qcom_glink_tx_kick(glink);
340 }
341
342 static int qcom_glink_tx(struct qcom_glink *glink,
343 const void *hdr, size_t hlen,
344 const void *data, size_t dlen, bool wait)
345 {
346 unsigned int tlen = hlen + dlen;
347 unsigned long flags;
348 int ret = 0;
349
350 /* Reject packets that are too big */
351 if (tlen >= glink->tx_pipe->length)
352 return -EINVAL;
353
354 spin_lock_irqsave(&glink->tx_lock, flags);
355
356 if (glink->abort_tx) {
357 ret = -EIO;
358 goto out;
359 }
360
361 while (qcom_glink_tx_avail(glink) < tlen) {
362 if (!wait) {
363 ret = -EAGAIN;
364 goto out;
365 }
366
367 if (glink->abort_tx) {
368 ret = -EIO;
369 goto out;
370 }
371
372 if (!glink->sent_read_notify) {
373 glink->sent_read_notify = true;
374 qcom_glink_send_read_notify(glink);
375 }
376
377 /* Wait without holding the tx_lock */
378 spin_unlock_irqrestore(&glink->tx_lock, flags);
379
380 wait_event_timeout(glink->tx_avail_notify,
381 qcom_glink_tx_avail(glink) >= tlen, 10 * HZ);
382
383 spin_lock_irqsave(&glink->tx_lock, flags);
384
385 if (qcom_glink_tx_avail(glink) >= tlen)
386 glink->sent_read_notify = false;
387 }
388
389 qcom_glink_tx_write(glink, hdr, hlen, data, dlen);
390 qcom_glink_tx_kick(glink);
391
392 out:
393 spin_unlock_irqrestore(&glink->tx_lock, flags);
394
395 return ret;
396 }
397
398 static int qcom_glink_send_version(struct qcom_glink *glink)
399 {
400 struct glink_msg msg;
401
402 msg.cmd = cpu_to_le16(GLINK_CMD_VERSION);
403 msg.param1 = cpu_to_le16(GLINK_VERSION_1);
404 msg.param2 = cpu_to_le32(glink->features);
405
406 trace_qcom_glink_cmd_version_tx(glink->label, GLINK_VERSION_1, glink->features);
407
408 return qcom_glink_tx(glink, &msg, sizeof(msg), NULL, 0, true);
409 }
410
411 static void qcom_glink_send_version_ack(struct qcom_glink *glink)
412 {
413 struct glink_msg msg;
414
415 msg.cmd = cpu_to_le16(GLINK_CMD_VERSION_ACK);
416 msg.param1 = cpu_to_le16(GLINK_VERSION_1);
417 msg.param2 = cpu_to_le32(glink->features);
418
419 trace_qcom_glink_cmd_version_ack_tx(glink->label, msg.param1, msg.param2);
420
421 qcom_glink_tx(glink, &msg, sizeof(msg), NULL, 0, true);
422 }
423
424 static void qcom_glink_send_open_ack(struct qcom_glink *glink,
425 struct glink_channel *channel)
426 {
427 struct glink_msg msg;
428
429 msg.cmd = cpu_to_le16(GLINK_CMD_OPEN_ACK);
430 msg.param1 = cpu_to_le16(channel->rcid);
431 msg.param2 = cpu_to_le32(0);
432
433 trace_qcom_glink_cmd_open_ack_tx(glink->label, channel->name,
434 channel->lcid, channel->rcid);
435
436 qcom_glink_tx(glink, &msg, sizeof(msg), NULL, 0, true);
437 }
438
439 static void qcom_glink_handle_intent_req_ack(struct qcom_glink *glink,
440 unsigned int cid, bool granted)
441 {
442 struct glink_channel *channel;
443 unsigned long flags;
444
445 qcom_glink_rx_advance(glink, ALIGN(sizeof(struct glink_msg), 8));
446
447 spin_lock_irqsave(&glink->idr_lock, flags);
448 channel = idr_find(&glink->rcids, cid);
449 spin_unlock_irqrestore(&glink->idr_lock, flags);
450
451 trace_qcom_glink_cmd_rx_intent_req_ack_rx(glink->label,
452 channel ? channel->name : NULL,
453 channel ? channel->lcid : 0,
454 cid, granted);
455 if (!channel) {
456 dev_err(glink->dev, "unable to find channel\n");
457 return;
458 }
459
460 WRITE_ONCE(channel->intent_req_result, granted);
461 wake_up_all(&channel->intent_req_wq);
462 }
463
464 static void qcom_glink_intent_req_abort(struct glink_channel *channel)
465 {
466 WRITE_ONCE(channel->intent_req_result, 0);
467 wake_up_all(&channel->intent_req_wq);
468 }
469
470 /**
471 * qcom_glink_send_open_req() - send a GLINK_CMD_OPEN request to the remote
472 * @glink: Ptr to the glink edge
473 * @channel: Ptr to the channel that the open req is sent
474 *
475 * Allocates a local channel id and sends a GLINK_CMD_OPEN message to the remote.
476 * Will return with refcount held, regardless of outcome.
477 *
478 * Return: 0 on success, negative errno otherwise.
479 */
480 static int qcom_glink_send_open_req(struct qcom_glink *glink,
481 struct glink_channel *channel)
482 {
483 DEFINE_RAW_FLEX(struct glink_msg, req, data, GLINK_NAME_SIZE);
484 int name_len = strlen(channel->name) + 1;
485 int req_len = ALIGN(sizeof(*req) + name_len, 8);
486 int ret;
487 unsigned long flags;
488
489 kref_get(&channel->refcount);
490
491 spin_lock_irqsave(&glink->idr_lock, flags);
492 ret = idr_alloc_cyclic(&glink->lcids, channel,
493 RPM_GLINK_CID_MIN, RPM_GLINK_CID_MAX,
494 GFP_ATOMIC);
495 spin_unlock_irqrestore(&glink->idr_lock, flags);
496 if (ret < 0)
497 return ret;
498
499 channel->lcid = ret;
500
501 req->cmd = cpu_to_le16(GLINK_CMD_OPEN);
502 req->param1 = cpu_to_le16(channel->lcid);
503 req->param2 = cpu_to_le32(name_len);
504 strcpy(req->data, channel->name);
505
506 trace_qcom_glink_cmd_open_tx(glink->label, channel->name,
507 channel->lcid, channel->rcid);
508
509 ret = qcom_glink_tx(glink, req, req_len, NULL, 0, true);
510 if (ret)
511 goto remove_idr;
512
513 return 0;
514
515 remove_idr:
516 spin_lock_irqsave(&glink->idr_lock, flags);
517 idr_remove(&glink->lcids, channel->lcid);
518 channel->lcid = 0;
519 spin_unlock_irqrestore(&glink->idr_lock, flags);
520
521 return ret;
522 }
523
524 static void qcom_glink_send_close_req(struct qcom_glink *glink,
525 struct glink_channel *channel)
526 {
527 struct glink_msg req;
528
529 req.cmd = cpu_to_le16(GLINK_CMD_CLOSE);
530 req.param1 = cpu_to_le16(channel->lcid);
531 req.param2 = 0;
532
533 trace_qcom_glink_cmd_close_tx(glink->label, channel->name,
534 channel->lcid, channel->rcid);
535
536 qcom_glink_tx(glink, &req, sizeof(req), NULL, 0, true);
537 }
538
539 static void qcom_glink_send_close_ack(struct qcom_glink *glink,
540 struct glink_channel *channel)
541 {
542 struct glink_msg req;
543
544 req.cmd = cpu_to_le16(GLINK_CMD_CLOSE_ACK);
545 req.param1 = cpu_to_le16(channel->rcid);
546 req.param2 = 0;
547
548 trace_qcom_glink_cmd_close_ack_tx(glink->label, channel->name,
549 channel->lcid, channel->rcid);
550
551 qcom_glink_tx(glink, &req, sizeof(req), NULL, 0, true);
552 }
553
554 static void qcom_glink_rx_done_work(struct work_struct *work)
555 {
556 struct glink_channel *channel = container_of(work, struct glink_channel,
557 intent_work);
558 struct qcom_glink *glink = channel->glink;
559 struct glink_core_rx_intent *intent, *tmp;
560 struct {
561 u16 id;
562 u16 lcid;
563 u32 liid;
564 } __packed cmd;
565
566 unsigned int cid = channel->lcid;
567 unsigned int iid;
568 bool reuse;
569 unsigned long flags;
570
571 spin_lock_irqsave(&channel->intent_lock, flags);
572 list_for_each_entry_safe(intent, tmp, &channel->done_intents, node) {
573 list_del(&intent->node);
574 spin_unlock_irqrestore(&channel->intent_lock, flags);
575 iid = intent->id;
576 reuse = intent->reuse;
577
578 cmd.id = reuse ? GLINK_CMD_RX_DONE_W_REUSE : GLINK_CMD_RX_DONE;
579 cmd.lcid = cid;
580 cmd.liid = iid;
581
582 trace_qcom_glink_cmd_rx_done_tx(glink->label, channel->name,
583 channel->lcid, channel->rcid, cmd.liid, reuse);
584
585 qcom_glink_tx(glink, &cmd, sizeof(cmd), NULL, 0, true);
586 if (!reuse) {
587 kfree(intent->data);
588 kfree(intent);
589 }
590 spin_lock_irqsave(&channel->intent_lock, flags);
591 }
592 spin_unlock_irqrestore(&channel->intent_lock, flags);
593 }
594
595 static void qcom_glink_rx_done(struct qcom_glink *glink,
596 struct glink_channel *channel,
597 struct glink_core_rx_intent *intent)
598 {
599 /* We don't send RX_DONE to intentless systems */
600 if (glink->intentless) {
601 kfree(intent->data);
602 kfree(intent);
603 return;
604 }
605
606 /* Take it off the tree of receive intents */
607 if (!intent->reuse) {
608 spin_lock(&channel->intent_lock);
609 idr_remove(&channel->liids, intent->id);
610 spin_unlock(&channel->intent_lock);
611 }
612
613 /* Schedule the sending of a rx_done indication */
614 spin_lock(&channel->intent_lock);
615 list_add_tail(&intent->node, &channel->done_intents);
616 spin_unlock(&channel->intent_lock);
617
618 schedule_work(&channel->intent_work);
619 }
620
621 /**
622 * qcom_glink_receive_version() - receive version/features from remote system
623 *
624 * @glink: pointer to transport interface
625 * @version: remote version
626 * @features: remote features
627 *
628 * This function is called in response to a remote-initiated version/feature
629 * negotiation sequence.
630 */
631 static void qcom_glink_receive_version(struct qcom_glink *glink,
632 u32 version,
633 u32 features)
634 {
635 trace_qcom_glink_cmd_version_rx(glink->label, version, features);
636
637 switch (version) {
638 case 0:
639 break;
640 case GLINK_VERSION_1:
641 glink->features &= features;
642 fallthrough;
643 default:
644 qcom_glink_send_version_ack(glink);
645 break;
646 }
647 }
648
649 /**
650 * qcom_glink_receive_version_ack() - receive negotiation ack from remote system
651 *
652 * @glink: pointer to transport interface
653 * @version: remote version response
654 * @features: remote features response
655 *
656 * This function is called in response to a local-initiated version/feature
657 * negotiation sequence and is the counter-offer from the remote side based
658 * upon the initial version and feature set requested.
659 */
660 static void qcom_glink_receive_version_ack(struct qcom_glink *glink,
661 u32 version,
662 u32 features)
663 {
664 trace_qcom_glink_cmd_version_ack_rx(glink->label, version, features);
665
666 switch (version) {
667 case 0:
668 /* Version negotiation failed */
669 break;
670 case GLINK_VERSION_1:
671 if (features == glink->features)
672 break;
673
674 glink->features &= features;
675 fallthrough;
676 default:
677 qcom_glink_send_version(glink);
678 break;
679 }
680 }
681
682 /**
683 * qcom_glink_send_intent_req_ack() - convert an rx intent request ack cmd to
684 * wire format and transmit
685 * @glink: The transport to transmit on.
686 * @channel: The glink channel
687 * @granted: The request response to encode.
688 *
689 * Return: 0 on success or standard Linux error code.
690 */
691 static int qcom_glink_send_intent_req_ack(struct qcom_glink *glink,
692 struct glink_channel *channel,
693 bool granted)
694 {
695 struct glink_msg msg;
696
697 trace_qcom_glink_cmd_rx_intent_req_ack_tx(glink->label, channel->name,
698 channel->lcid, channel->rcid,
699 granted);
700
701 msg.cmd = cpu_to_le16(GLINK_CMD_RX_INTENT_REQ_ACK);
702 msg.param1 = cpu_to_le16(channel->lcid);
703 msg.param2 = cpu_to_le32(granted);
704
705 qcom_glink_tx(glink, &msg, sizeof(msg), NULL, 0, true);
706
707 return 0;
708 }
709
710 /**
711 * qcom_glink_advertise_intent - convert an rx intent cmd to wire format and
712 * transmit
713 * @glink: The transport to transmit on.
714 * @channel: The local channel
715 * @intent: The intent to pass on to remote.
716 *
717 * Return: 0 on success or standard Linux error code.
718 */
719 static int qcom_glink_advertise_intent(struct qcom_glink *glink,
720 struct glink_channel *channel,
721 struct glink_core_rx_intent *intent)
722 {
723 struct command {
724 __le16 id;
725 __le16 lcid;
726 __le32 count;
727 __le32 size;
728 __le32 liid;
729 } __packed;
730 struct command cmd;
731
732 cmd.id = cpu_to_le16(GLINK_CMD_INTENT);
733 cmd.lcid = cpu_to_le16(channel->lcid);
734 cmd.count = cpu_to_le32(1);
735 cmd.size = cpu_to_le32(intent->size);
736 cmd.liid = cpu_to_le32(intent->id);
737
738 trace_qcom_glink_cmd_intent_tx(glink->label, channel->name,
739 channel->lcid, channel->rcid,
740 cmd.count, cmd.size, cmd.liid);
741
742 qcom_glink_tx(glink, &cmd, sizeof(cmd), NULL, 0, true);
743
744 return 0;
745 }
746
747 static struct glink_core_rx_intent *
748 qcom_glink_alloc_intent(struct qcom_glink *glink,
749 struct glink_channel *channel,
750 size_t size,
751 bool reuseable)
752 {
753 struct glink_core_rx_intent *intent;
754 int ret;
755 unsigned long flags;
756
757 intent = kzalloc(sizeof(*intent), GFP_KERNEL);
758 if (!intent)
759 return NULL;
760
761 intent->data = kzalloc(size, GFP_KERNEL);
762 if (!intent->data)
763 goto free_intent;
764
765 spin_lock_irqsave(&channel->intent_lock, flags);
766 ret = idr_alloc_cyclic(&channel->liids, intent, 1, -1, GFP_ATOMIC);
767 if (ret < 0) {
768 spin_unlock_irqrestore(&channel->intent_lock, flags);
769 goto free_data;
770 }
771 spin_unlock_irqrestore(&channel->intent_lock, flags);
772
773 intent->id = ret;
774 intent->size = size;
775 intent->reuse = reuseable;
776
777 return intent;
778
779 free_data:
780 kfree(intent->data);
781 free_intent:
782 kfree(intent);
783 return NULL;
784 }
785
786 static void qcom_glink_handle_rx_done(struct qcom_glink *glink,
787 u32 cid, uint32_t iid,
788 bool reuse)
789 {
790 struct glink_core_rx_intent *intent;
791 struct glink_channel *channel;
792 unsigned long flags;
793
794 qcom_glink_rx_advance(glink, ALIGN(sizeof(struct glink_msg), 8));
795
796 spin_lock_irqsave(&glink->idr_lock, flags);
797 channel = idr_find(&glink->rcids, cid);
798 spin_unlock_irqrestore(&glink->idr_lock, flags);
799
800 trace_qcom_glink_cmd_rx_done_rx(glink->label, channel ? channel->name : NULL,
801 channel ? channel->lcid : 0, cid, iid, reuse);
802 if (!channel) {
803 dev_err(glink->dev, "invalid channel id received\n");
804 return;
805 }
806
807 spin_lock_irqsave(&channel->intent_lock, flags);
808 intent = idr_find(&channel->riids, iid);
809
810 if (!intent) {
811 spin_unlock_irqrestore(&channel->intent_lock, flags);
812 dev_err(glink->dev, "invalid intent id received\n");
813 return;
814 }
815
816 intent->in_use = false;
817
818 if (!reuse) {
819 idr_remove(&channel->riids, intent->id);
820 kfree(intent);
821 }
822 spin_unlock_irqrestore(&channel->intent_lock, flags);
823
824 if (reuse) {
825 WRITE_ONCE(channel->intent_received, true);
826 wake_up_all(&channel->intent_req_wq);
827 }
828 }
829
830 /**
831 * qcom_glink_handle_intent_req() - Receive a request for rx_intent
832 * from remote side
833 * @glink: Pointer to the transport interface
834 * @cid: Remote channel ID
835 * @size: size of the intent
836 *
837 * The function searches for the local channel to which the request for
838 * rx_intent has arrived and allocates and notifies the remote back
839 */
840 static void qcom_glink_handle_intent_req(struct qcom_glink *glink,
841 u32 cid, size_t size)
842 {
843 struct glink_core_rx_intent *intent;
844 struct glink_channel *channel;
845 unsigned long flags;
846
847 spin_lock_irqsave(&glink->idr_lock, flags);
848 channel = idr_find(&glink->rcids, cid);
849 spin_unlock_irqrestore(&glink->idr_lock, flags);
850
851 trace_qcom_glink_cmd_rx_intent_req_rx(glink->label,
852 channel ? channel->name : NULL,
853 channel ? channel->lcid : 0,
854 cid, size);
855 if (!channel) {
856 pr_err("%s channel not found for cid %d\n", __func__, cid);
857 return;
858 }
859
860 intent = qcom_glink_alloc_intent(glink, channel, size, false);
861 if (intent)
862 qcom_glink_advertise_intent(glink, channel, intent);
863
864 qcom_glink_send_intent_req_ack(glink, channel, !!intent);
865 }
866
867 static int qcom_glink_rx_defer(struct qcom_glink *glink, size_t extra)
868 {
869 struct glink_defer_cmd *dcmd;
870
871 extra = ALIGN(extra, 8);
872
873 if (qcom_glink_rx_avail(glink) < sizeof(struct glink_msg) + extra) {
874 dev_dbg(glink->dev, "Insufficient data in rx fifo");
875 return -ENXIO;
876 }
877
878 dcmd = kzalloc(struct_size(dcmd, data, extra), GFP_ATOMIC);
879 if (!dcmd)
880 return -ENOMEM;
881
882 INIT_LIST_HEAD(&dcmd->node);
883
884 qcom_glink_rx_peek(glink,
885 container_of(&dcmd->msg, struct glink_msg, hdr), 0,
886 sizeof(dcmd->msg) + extra);
887
888 spin_lock(&glink->rx_lock);
889 list_add_tail(&dcmd->node, &glink->rx_queue);
890 spin_unlock(&glink->rx_lock);
891
892 schedule_work(&glink->rx_work);
893 qcom_glink_rx_advance(glink, sizeof(dcmd->msg) + extra);
894
895 return 0;
896 }
897
898 static int qcom_glink_rx_data(struct qcom_glink *glink, size_t avail)
899 {
900 struct glink_core_rx_intent *intent;
901 struct glink_channel *channel;
902 struct {
903 struct glink_msg_hdr msg;
904 __le32 chunk_size;
905 __le32 left_size;
906 } __packed hdr;
907 unsigned int chunk_size;
908 unsigned int left_size;
909 unsigned int rcid;
910 unsigned int liid;
911 int ret = 0;
912 unsigned long flags;
913
914 if (avail < sizeof(hdr)) {
915 dev_dbg(glink->dev, "Not enough data in fifo\n");
916 return -EAGAIN;
917 }
918
919 qcom_glink_rx_peek(glink, &hdr, 0, sizeof(hdr));
920 chunk_size = le32_to_cpu(hdr.chunk_size);
921 left_size = le32_to_cpu(hdr.left_size);
922
923 if (avail < sizeof(hdr) + chunk_size) {
924 dev_dbg(glink->dev, "Payload not yet in fifo\n");
925 return -EAGAIN;
926 }
927
928 rcid = le16_to_cpu(hdr.msg.param1);
929 liid = le32_to_cpu(hdr.msg.param2);
930 spin_lock_irqsave(&glink->idr_lock, flags);
931 channel = idr_find(&glink->rcids, rcid);
932 spin_unlock_irqrestore(&glink->idr_lock, flags);
933
934 trace_qcom_glink_cmd_tx_data_rx(glink->label, channel ? channel->name : NULL,
935 channel ? channel->lcid : 0, rcid,
936 liid, chunk_size, left_size,
937 hdr.msg.cmd == GLINK_CMD_TX_DATA_CONT);
938 if (!channel) {
939 dev_dbg(glink->dev, "Data on non-existing channel\n");
940
941 /* Drop the message */
942 goto advance_rx;
943 }
944
945 if (glink->intentless) {
946 /* Might have an ongoing, fragmented, message to append */
947 if (!channel->buf) {
948 intent = kzalloc(sizeof(*intent), GFP_ATOMIC);
949 if (!intent)
950 return -ENOMEM;
951
952 intent->data = kmalloc(chunk_size + left_size,
953 GFP_ATOMIC);
954 if (!intent->data) {
955 kfree(intent);
956 return -ENOMEM;
957 }
958
959 intent->id = 0xbabababa;
960 intent->size = chunk_size + left_size;
961 intent->offset = 0;
962
963 channel->buf = intent;
964 } else {
965 intent = channel->buf;
966 }
967 } else {
968 spin_lock_irqsave(&channel->intent_lock, flags);
969 intent = idr_find(&channel->liids, liid);
970 spin_unlock_irqrestore(&channel->intent_lock, flags);
971
972 if (!intent) {
973 dev_err(glink->dev,
974 "no intent found for channel %s intent %d",
975 channel->name, liid);
976 ret = -ENOENT;
977 goto advance_rx;
978 }
979 }
980
981 if (intent->size - intent->offset < chunk_size) {
982 dev_err(glink->dev, "Insufficient space in intent\n");
983
984 /* The packet header lied, drop payload */
985 goto advance_rx;
986 }
987
988 qcom_glink_rx_peek(glink, intent->data + intent->offset,
989 sizeof(hdr), chunk_size);
990 intent->offset += chunk_size;
991
992 /* Handle message when no fragments remain to be received */
993 if (!left_size) {
994 spin_lock(&channel->recv_lock);
995 if (channel->ept.cb) {
996 channel->ept.cb(channel->ept.rpdev,
997 intent->data,
998 intent->offset,
999 channel->ept.priv,
1000 RPMSG_ADDR_ANY);
1001 }
1002 spin_unlock(&channel->recv_lock);
1003
1004 intent->offset = 0;
1005 channel->buf = NULL;
1006
1007 qcom_glink_rx_done(glink, channel, intent);
1008 }
1009
1010 advance_rx:
1011 qcom_glink_rx_advance(glink, ALIGN(sizeof(hdr) + chunk_size, 8));
1012
1013 return ret;
1014 }
1015
1016 static void qcom_glink_rx_read_notif(struct qcom_glink *glink)
1017 {
1018 trace_qcom_glink_cmd_read_notif_rx(glink->label);
1019
1020 qcom_glink_rx_advance(glink, ALIGN(sizeof(struct glink_msg), 8));
1021 qcom_glink_tx_kick(glink);
1022 }
1023
1024 static void qcom_glink_handle_intent(struct qcom_glink *glink,
1025 unsigned int cid,
1026 unsigned int count,
1027 size_t avail)
1028 {
1029 struct glink_core_rx_intent *intent;
1030 struct glink_channel *channel;
1031 struct intent_pair {
1032 __le32 size;
1033 __le32 iid;
1034 };
1035
1036 struct {
1037 struct glink_msg_hdr msg;
1038 struct intent_pair intents[];
1039 } __packed * msg;
1040
1041 const size_t msglen = struct_size(msg, intents, count);
1042 int ret;
1043 int i;
1044 unsigned long flags;
1045
1046 if (avail < msglen) {
1047 dev_dbg(glink->dev, "Not enough data in fifo\n");
1048 return;
1049 }
1050
1051 spin_lock_irqsave(&glink->idr_lock, flags);
1052 channel = idr_find(&glink->rcids, cid);
1053 spin_unlock_irqrestore(&glink->idr_lock, flags);
1054 if (!channel) {
1055 trace_qcom_glink_cmd_intent_rx(glink->label, NULL, 0, cid, count, 0, 0);
1056 dev_err(glink->dev, "intents for non-existing channel\n");
1057 qcom_glink_rx_advance(glink, ALIGN(msglen, 8));
1058 return;
1059 }
1060
1061 msg = kmalloc(msglen, GFP_ATOMIC);
1062 if (!msg)
1063 return;
1064
1065 qcom_glink_rx_peek(glink, msg, 0, msglen);
1066
1067 trace_qcom_glink_cmd_intent_rx(glink->label, channel->name,
1068 channel->lcid, cid, count,
1069 count > 0 ? msg->intents[0].size : 0,
1070 count > 0 ? msg->intents[0].iid : 0);
1071
1072 for (i = 0; i < count; ++i) {
1073 intent = kzalloc(sizeof(*intent), GFP_ATOMIC);
1074 if (!intent)
1075 break;
1076
1077 intent->id = le32_to_cpu(msg->intents[i].iid);
1078 intent->size = le32_to_cpu(msg->intents[i].size);
1079
1080 spin_lock_irqsave(&channel->intent_lock, flags);
1081 ret = idr_alloc(&channel->riids, intent,
1082 intent->id, intent->id + 1, GFP_ATOMIC);
1083 spin_unlock_irqrestore(&channel->intent_lock, flags);
1084
1085 if (ret < 0)
1086 dev_err(glink->dev, "failed to store remote intent\n");
1087 }
1088
1089 WRITE_ONCE(channel->intent_received, true);
1090 wake_up_all(&channel->intent_req_wq);
1091
1092 kfree(msg);
1093 qcom_glink_rx_advance(glink, ALIGN(msglen, 8));
1094 }
1095
1096 static int qcom_glink_rx_open_ack(struct qcom_glink *glink, unsigned int lcid)
1097 {
1098 struct glink_channel *channel;
1099
1100 qcom_glink_rx_advance(glink, ALIGN(sizeof(struct glink_msg), 8));
1101
1102 spin_lock(&glink->idr_lock);
1103 channel = idr_find(&glink->lcids, lcid);
1104 spin_unlock(&glink->idr_lock);
1105
1106 trace_qcom_glink_cmd_open_ack_rx(glink->label, channel ? channel->name : NULL,
1107 lcid, channel ? channel->rcid : 0);
1108 if (!channel) {
1109 dev_err(glink->dev, "Invalid open ack packet\n");
1110 return -EINVAL;
1111 }
1112
1113 complete_all(&channel->open_ack);
1114
1115 return 0;
1116 }
1117
1118 /**
1119 * qcom_glink_set_flow_control() - convert a signal cmd to wire format and transmit
1120 * @ept: Rpmsg endpoint for channel.
1121 * @pause: Pause transmission
1122 * @dst: destination address of the endpoint
1123 *
1124 * Return: 0 on success or standard Linux error code.
1125 */
1126 static int qcom_glink_set_flow_control(struct rpmsg_endpoint *ept, bool pause, u32 dst)
1127 {
1128 struct glink_channel *channel = to_glink_channel(ept);
1129 struct qcom_glink *glink = channel->glink;
1130 struct glink_msg msg;
1131 u32 sigs = 0;
1132
1133 if (pause)
1134 sigs |= NATIVE_DTR_SIG | NATIVE_RTS_SIG;
1135
1136 msg.cmd = cpu_to_le16(GLINK_CMD_SIGNALS);
1137 msg.param1 = cpu_to_le16(channel->lcid);
1138 msg.param2 = cpu_to_le32(sigs);
1139
1140 trace_qcom_glink_cmd_signal_tx(glink->label, channel->name,
1141 channel->lcid, channel->rcid, sigs);
1142
1143 return qcom_glink_tx(glink, &msg, sizeof(msg), NULL, 0, true);
1144 }
1145
1146 static void qcom_glink_handle_signals(struct qcom_glink *glink,
1147 unsigned int rcid, unsigned int sigs)
1148 {
1149 struct glink_channel *channel;
1150 unsigned long flags;
1151 bool enable;
1152
1153 qcom_glink_rx_advance(glink, ALIGN(sizeof(struct glink_msg), 8));
1154
1155 spin_lock_irqsave(&glink->idr_lock, flags);
1156 channel = idr_find(&glink->rcids, rcid);
1157 spin_unlock_irqrestore(&glink->idr_lock, flags);
1158
1159 trace_qcom_glink_cmd_signal_rx(glink->label, channel ? channel->name : NULL,
1160 channel ? channel->lcid : 0, rcid, sigs);
1161 if (!channel) {
1162 dev_err(glink->dev, "signal for non-existing channel\n");
1163 return;
1164 }
1165
1166 enable = sigs & NATIVE_DSR_SIG || sigs & NATIVE_CTS_SIG;
1167
1168 if (channel->ept.flow_cb)
1169 channel->ept.flow_cb(channel->ept.rpdev, channel->ept.priv, enable);
1170 }
1171
1172 void qcom_glink_native_rx(struct qcom_glink *glink)
1173 {
1174 struct glink_msg msg;
1175 unsigned int param1;
1176 unsigned int param2;
1177 unsigned int avail;
1178 unsigned int cmd;
1179 int ret = 0;
1180
1181 /* To wakeup any blocking writers */
1182 wake_up_all(&glink->tx_avail_notify);
1183
1184 for (;;) {
1185 avail = qcom_glink_rx_avail(glink);
1186 if (avail < sizeof(msg))
1187 break;
1188
1189 qcom_glink_rx_peek(glink, &msg, 0, sizeof(msg));
1190
1191 cmd = le16_to_cpu(msg.cmd);
1192 param1 = le16_to_cpu(msg.param1);
1193 param2 = le32_to_cpu(msg.param2);
1194
1195 switch (cmd) {
1196 case GLINK_CMD_VERSION:
1197 case GLINK_CMD_VERSION_ACK:
1198 case GLINK_CMD_CLOSE:
1199 case GLINK_CMD_CLOSE_ACK:
1200 case GLINK_CMD_RX_INTENT_REQ:
1201 ret = qcom_glink_rx_defer(glink, 0);
1202 break;
1203 case GLINK_CMD_OPEN_ACK:
1204 ret = qcom_glink_rx_open_ack(glink, param1);
1205 break;
1206 case GLINK_CMD_OPEN:
1207 /* upper 16 bits of param2 are the "prio" field */
1208 ret = qcom_glink_rx_defer(glink, param2 & 0xffff);
1209 break;
1210 case GLINK_CMD_TX_DATA:
1211 case GLINK_CMD_TX_DATA_CONT:
1212 ret = qcom_glink_rx_data(glink, avail);
1213 break;
1214 case GLINK_CMD_READ_NOTIF:
1215 qcom_glink_rx_read_notif(glink);
1216 break;
1217 case GLINK_CMD_INTENT:
1218 qcom_glink_handle_intent(glink, param1, param2, avail);
1219 break;
1220 case GLINK_CMD_RX_DONE:
1221 qcom_glink_handle_rx_done(glink, param1, param2, false);
1222 break;
1223 case GLINK_CMD_RX_DONE_W_REUSE:
1224 qcom_glink_handle_rx_done(glink, param1, param2, true);
1225 break;
1226 case GLINK_CMD_RX_INTENT_REQ_ACK:
1227 qcom_glink_handle_intent_req_ack(glink, param1, param2);
1228 break;
1229 case GLINK_CMD_SIGNALS:
1230 qcom_glink_handle_signals(glink, param1, param2);
1231 break;
1232 default:
1233 dev_err(glink->dev, "unhandled rx cmd: %d\n", cmd);
1234 ret = -EINVAL;
1235 break;
1236 }
1237
1238 if (ret)
1239 break;
1240 }
1241 }
1242 EXPORT_SYMBOL(qcom_glink_native_rx);
1243
1244 /* Locally initiated rpmsg_create_ept */
1245 static struct glink_channel *qcom_glink_create_local(struct qcom_glink *glink,
1246 const char *name)
1247 {
1248 struct glink_channel *channel;
1249 int ret;
1250 unsigned long flags;
1251
1252 channel = qcom_glink_alloc_channel(glink, name);
1253 if (IS_ERR(channel))
1254 return ERR_CAST(channel);
1255
1256 ret = qcom_glink_send_open_req(glink, channel);
1257 if (ret)
1258 goto release_channel;
1259
1260 ret = wait_for_completion_timeout(&channel->open_ack, 5 * HZ);
1261 if (!ret)
1262 goto err_timeout;
1263
1264 ret = wait_for_completion_timeout(&channel->open_req, 5 * HZ);
1265 if (!ret)
1266 goto err_timeout;
1267
1268 qcom_glink_send_open_ack(glink, channel);
1269
1270 return channel;
1271
1272 err_timeout:
1273 /* qcom_glink_send_open_req() did register the channel in lcids*/
1274 spin_lock_irqsave(&glink->idr_lock, flags);
1275 idr_remove(&glink->lcids, channel->lcid);
1276 spin_unlock_irqrestore(&glink->idr_lock, flags);
1277
1278 release_channel:
1279 /* Release qcom_glink_send_open_req() reference */
1280 kref_put(&channel->refcount, qcom_glink_channel_release);
1281 /* Release qcom_glink_alloc_channel() reference */
1282 kref_put(&channel->refcount, qcom_glink_channel_release);
1283
1284 return ERR_PTR(-ETIMEDOUT);
1285 }
1286
1287 /* Remote initiated rpmsg_create_ept */
1288 static int qcom_glink_create_remote(struct qcom_glink *glink,
1289 struct glink_channel *channel)
1290 {
1291 int ret;
1292
1293 qcom_glink_send_open_ack(glink, channel);
1294
1295 ret = qcom_glink_send_open_req(glink, channel);
1296 if (ret)
1297 goto close_link;
1298
1299 ret = wait_for_completion_timeout(&channel->open_ack, 5 * HZ);
1300 if (!ret) {
1301 ret = -ETIMEDOUT;
1302 goto close_link;
1303 }
1304
1305 return 0;
1306
1307 close_link:
1308 /*
1309 * Send a close request to "undo" our open-ack. The close-ack will
1310 * release qcom_glink_send_open_req() reference and the last reference
1311 * will be relesed after receiving remote_close or transport unregister
1312 * by calling qcom_glink_native_remove().
1313 */
1314 qcom_glink_send_close_req(glink, channel);
1315
1316 return ret;
1317 }
1318
1319 static struct rpmsg_endpoint *qcom_glink_create_ept(struct rpmsg_device *rpdev,
1320 rpmsg_rx_cb_t cb,
1321 void *priv,
1322 struct rpmsg_channel_info
1323 chinfo)
1324 {
1325 struct glink_channel *parent = to_glink_channel(rpdev->ept);
1326 struct glink_channel *channel;
1327 struct qcom_glink *glink = parent->glink;
1328 struct rpmsg_endpoint *ept;
1329 const char *name = chinfo.name;
1330 int cid;
1331 int ret;
1332 unsigned long flags;
1333
1334 spin_lock_irqsave(&glink->idr_lock, flags);
1335 idr_for_each_entry(&glink->rcids, channel, cid) {
1336 if (!strcmp(channel->name, name))
1337 break;
1338 }
1339 spin_unlock_irqrestore(&glink->idr_lock, flags);
1340
1341 if (!channel) {
1342 channel = qcom_glink_create_local(glink, name);
1343 if (IS_ERR(channel))
1344 return NULL;
1345 } else {
1346 ret = qcom_glink_create_remote(glink, channel);
1347 if (ret)
1348 return NULL;
1349 }
1350
1351 ept = &channel->ept;
1352 ept->rpdev = rpdev;
1353 ept->cb = cb;
1354 ept->priv = priv;
1355 ept->ops = &glink_endpoint_ops;
1356
1357 return ept;
1358 }
1359
1360 static int qcom_glink_announce_create(struct rpmsg_device *rpdev)
1361 {
1362 struct glink_channel *channel = to_glink_channel(rpdev->ept);
1363 struct device_node *np = rpdev->dev.of_node;
1364 struct qcom_glink *glink = channel->glink;
1365 struct glink_core_rx_intent *intent;
1366 const struct property *prop = NULL;
1367 __be32 defaults[] = { cpu_to_be32(SZ_1K), cpu_to_be32(5) };
1368 int num_intents;
1369 int num_groups = 1;
1370 __be32 *val = defaults;
1371 int size;
1372
1373 if (glink->intentless || !completion_done(&channel->open_ack))
1374 return 0;
1375
1376 prop = of_find_property(np, "qcom,intents", NULL);
1377 if (prop) {
1378 val = prop->value;
1379 num_groups = prop->length / sizeof(u32) / 2;
1380 }
1381
1382 /* Channel is now open, advertise base set of intents */
1383 while (num_groups--) {
1384 size = be32_to_cpup(val++);
1385 num_intents = be32_to_cpup(val++);
1386 while (num_intents--) {
1387 intent = qcom_glink_alloc_intent(glink, channel, size,
1388 true);
1389 if (!intent)
1390 break;
1391
1392 qcom_glink_advertise_intent(glink, channel, intent);
1393 }
1394 }
1395 return 0;
1396 }
1397
1398 static void qcom_glink_destroy_ept(struct rpmsg_endpoint *ept)
1399 {
1400 struct glink_channel *channel = to_glink_channel(ept);
1401 struct qcom_glink *glink = channel->glink;
1402 unsigned long flags;
1403
1404 spin_lock_irqsave(&channel->recv_lock, flags);
1405 channel->ept.cb = NULL;
1406 spin_unlock_irqrestore(&channel->recv_lock, flags);
1407
1408 /* Decouple the potential rpdev from the channel */
1409 channel->rpdev = NULL;
1410
1411 qcom_glink_send_close_req(glink, channel);
1412 }
1413
1414 static int qcom_glink_request_intent(struct qcom_glink *glink,
1415 struct glink_channel *channel,
1416 size_t size)
1417 {
1418 struct {
1419 u16 id;
1420 u16 cid;
1421 u32 size;
1422 } __packed cmd;
1423
1424 int ret;
1425
1426 mutex_lock(&channel->intent_req_lock);
1427
1428 WRITE_ONCE(channel->intent_req_result, -1);
1429 WRITE_ONCE(channel->intent_received, false);
1430
1431 cmd.id = GLINK_CMD_RX_INTENT_REQ;
1432 cmd.cid = channel->lcid;
1433 cmd.size = size;
1434
1435 trace_qcom_glink_cmd_rx_intent_req_tx(glink->label, channel->name,
1436 channel->lcid, channel->rcid,
1437 cmd.size);
1438
1439 ret = qcom_glink_tx(glink, &cmd, sizeof(cmd), NULL, 0, true);
1440 if (ret)
1441 goto unlock;
1442
1443 ret = wait_event_timeout(channel->intent_req_wq,
1444 READ_ONCE(channel->intent_req_result) == 0 ||
1445 (READ_ONCE(channel->intent_req_result) > 0 &&
1446 READ_ONCE(channel->intent_received)) ||
1447 glink->abort_tx,
1448 10 * HZ);
1449 if (!ret) {
1450 dev_err(glink->dev, "intent request timed out\n");
1451 ret = -ETIMEDOUT;
1452 } else if (glink->abort_tx) {
1453 ret = -ECANCELED;
1454 } else {
1455 ret = READ_ONCE(channel->intent_req_result) ? 0 : -EAGAIN;
1456 }
1457
1458 unlock:
1459 mutex_unlock(&channel->intent_req_lock);
1460 return ret;
1461 }
1462
1463 static int __qcom_glink_send(struct glink_channel *channel,
1464 void *data, int len, bool wait)
1465 {
1466 struct qcom_glink *glink = channel->glink;
1467 struct glink_core_rx_intent *intent = NULL;
1468 struct glink_core_rx_intent *tmp;
1469 int iid = 0;
1470 struct {
1471 struct glink_msg_hdr msg;
1472 __le32 chunk_size;
1473 __le32 left_size;
1474 } __packed req;
1475 int ret;
1476 unsigned long flags;
1477 int chunk_size = len;
1478 size_t offset = 0;
1479
1480 if (!glink->intentless) {
1481 while (!intent) {
1482 spin_lock_irqsave(&channel->intent_lock, flags);
1483 idr_for_each_entry(&channel->riids, tmp, iid) {
1484 if (tmp->size >= len && !tmp->in_use) {
1485 if (!intent)
1486 intent = tmp;
1487 else if (intent->size > tmp->size)
1488 intent = tmp;
1489 if (intent->size == len)
1490 break;
1491 }
1492 }
1493 if (intent)
1494 intent->in_use = true;
1495 spin_unlock_irqrestore(&channel->intent_lock, flags);
1496
1497 /* We found an available intent */
1498 if (intent)
1499 break;
1500
1501 if (!wait)
1502 return -EBUSY;
1503
1504 ret = qcom_glink_request_intent(glink, channel, len);
1505 if (ret < 0)
1506 return ret;
1507 }
1508
1509 iid = intent->id;
1510 }
1511
1512 while (offset < len) {
1513 chunk_size = len - offset;
1514 if (chunk_size > SZ_8K && wait)
1515 chunk_size = SZ_8K;
1516
1517 req.msg.cmd = cpu_to_le16(offset == 0 ? GLINK_CMD_TX_DATA : GLINK_CMD_TX_DATA_CONT);
1518 req.msg.param1 = cpu_to_le16(channel->lcid);
1519 req.msg.param2 = cpu_to_le32(iid);
1520 req.chunk_size = cpu_to_le32(chunk_size);
1521 req.left_size = cpu_to_le32(len - offset - chunk_size);
1522
1523 trace_qcom_glink_cmd_tx_data_tx(glink->label, channel->name,
1524 channel->lcid, channel->rcid,
1525 iid, chunk_size,
1526 len - offset - chunk_size,
1527 offset > 0);
1528
1529 ret = qcom_glink_tx(glink, &req, sizeof(req), data + offset, chunk_size, wait);
1530 if (ret) {
1531 /* Mark intent available if we failed */
1532 if (intent)
1533 intent->in_use = false;
1534 return ret;
1535 }
1536
1537 offset += chunk_size;
1538 }
1539
1540 return 0;
1541 }
1542
1543 static int qcom_glink_send(struct rpmsg_endpoint *ept, void *data, int len)
1544 {
1545 struct glink_channel *channel = to_glink_channel(ept);
1546
1547 return __qcom_glink_send(channel, data, len, true);
1548 }
1549
1550 static int qcom_glink_trysend(struct rpmsg_endpoint *ept, void *data, int len)
1551 {
1552 struct glink_channel *channel = to_glink_channel(ept);
1553
1554 return __qcom_glink_send(channel, data, len, false);
1555 }
1556
1557 static int qcom_glink_sendto(struct rpmsg_endpoint *ept, void *data, int len, u32 dst)
1558 {
1559 struct glink_channel *channel = to_glink_channel(ept);
1560
1561 return __qcom_glink_send(channel, data, len, true);
1562 }
1563
1564 static int qcom_glink_trysendto(struct rpmsg_endpoint *ept, void *data, int len, u32 dst)
1565 {
1566 struct glink_channel *channel = to_glink_channel(ept);
1567
1568 return __qcom_glink_send(channel, data, len, false);
1569 }
1570
1571 /*
1572 * Finds the device_node for the glink child interested in this channel.
1573 */
1574 static struct device_node *qcom_glink_match_channel(struct device_node *node,
1575 const char *channel)
1576 {
1577 struct device_node *child;
1578 const char *name;
1579 const char *key;
1580 int ret;
1581
1582 for_each_available_child_of_node(node, child) {
1583 key = "qcom,glink-channels";
1584 ret = of_property_read_string(child, key, &name);
1585 if (ret)
1586 continue;
1587
1588 if (strcmp(name, channel) == 0)
1589 return child;
1590 }
1591
1592 return NULL;
1593 }
1594
1595 static const struct rpmsg_device_ops glink_device_ops = {
1596 .create_ept = qcom_glink_create_ept,
1597 .announce_create = qcom_glink_announce_create,
1598 };
1599
1600 static const struct rpmsg_endpoint_ops glink_endpoint_ops = {
1601 .destroy_ept = qcom_glink_destroy_ept,
1602 .send = qcom_glink_send,
1603 .sendto = qcom_glink_sendto,
1604 .trysend = qcom_glink_trysend,
1605 .trysendto = qcom_glink_trysendto,
1606 .set_flow_control = qcom_glink_set_flow_control,
1607 };
1608
1609 static void qcom_glink_rpdev_release(struct device *dev)
1610 {
1611 struct rpmsg_device *rpdev = to_rpmsg_device(dev);
1612
1613 kfree(rpdev->driver_override);
1614 kfree(rpdev);
1615 }
1616
1617 static int qcom_glink_rx_open(struct qcom_glink *glink, unsigned int rcid,
1618 char *name)
1619 {
1620 struct glink_channel *channel;
1621 struct rpmsg_device *rpdev;
1622 bool create_device = false;
1623 struct device_node *node;
1624 int lcid;
1625 int ret;
1626 unsigned long flags;
1627
1628 spin_lock_irqsave(&glink->idr_lock, flags);
1629 idr_for_each_entry(&glink->lcids, channel, lcid) {
1630 if (!strcmp(channel->name, name))
1631 break;
1632 }
1633 spin_unlock_irqrestore(&glink->idr_lock, flags);
1634
1635 if (!channel) {
1636 channel = qcom_glink_alloc_channel(glink, name);
1637 if (IS_ERR(channel))
1638 return PTR_ERR(channel);
1639
1640 /* The opening dance was initiated by the remote */
1641 create_device = true;
1642 }
1643
1644 trace_qcom_glink_cmd_open_rx(glink->label, name, channel->lcid, rcid);
1645
1646 spin_lock_irqsave(&glink->idr_lock, flags);
1647 ret = idr_alloc(&glink->rcids, channel, rcid, rcid + 1, GFP_ATOMIC);
1648 if (ret < 0) {
1649 dev_err(glink->dev, "Unable to insert channel into rcid list\n");
1650 spin_unlock_irqrestore(&glink->idr_lock, flags);
1651 goto free_channel;
1652 }
1653 channel->rcid = ret;
1654 spin_unlock_irqrestore(&glink->idr_lock, flags);
1655
1656 complete_all(&channel->open_req);
1657
1658 if (create_device) {
1659 rpdev = kzalloc(sizeof(*rpdev), GFP_KERNEL);
1660 if (!rpdev) {
1661 ret = -ENOMEM;
1662 goto rcid_remove;
1663 }
1664
1665 rpdev->ept = &channel->ept;
1666 strscpy_pad(rpdev->id.name, name, RPMSG_NAME_SIZE);
1667 rpdev->src = RPMSG_ADDR_ANY;
1668 rpdev->dst = RPMSG_ADDR_ANY;
1669 rpdev->ops = &glink_device_ops;
1670
1671 node = qcom_glink_match_channel(glink->dev->of_node, name);
1672 rpdev->dev.of_node = node;
1673 rpdev->dev.parent = glink->dev;
1674 rpdev->dev.release = qcom_glink_rpdev_release;
1675
1676 ret = rpmsg_register_device(rpdev);
1677 if (ret)
1678 goto rcid_remove;
1679
1680 channel->rpdev = rpdev;
1681 }
1682
1683 return 0;
1684
1685 rcid_remove:
1686 spin_lock_irqsave(&glink->idr_lock, flags);
1687 idr_remove(&glink->rcids, channel->rcid);
1688 channel->rcid = 0;
1689 spin_unlock_irqrestore(&glink->idr_lock, flags);
1690 free_channel:
1691 /* Release the reference, iff we took it */
1692 if (create_device)
1693 kref_put(&channel->refcount, qcom_glink_channel_release);
1694
1695 return ret;
1696 }
1697
1698 static void qcom_glink_rx_close(struct qcom_glink *glink, unsigned int rcid)
1699 {
1700 struct rpmsg_channel_info chinfo;
1701 struct glink_channel *channel;
1702 unsigned long flags;
1703
1704 spin_lock_irqsave(&glink->idr_lock, flags);
1705 channel = idr_find(&glink->rcids, rcid);
1706 spin_unlock_irqrestore(&glink->idr_lock, flags);
1707
1708 trace_qcom_glink_cmd_close_rx(glink->label, channel ? channel->name : NULL,
1709 channel ? channel->lcid : 0, rcid);
1710 if (WARN(!channel, "close request on unknown channel\n"))
1711 return;
1712
1713 /* cancel pending rx_done work */
1714 cancel_work_sync(&channel->intent_work);
1715
1716 if (channel->rpdev) {
1717 strscpy_pad(chinfo.name, channel->name, sizeof(chinfo.name));
1718 chinfo.src = RPMSG_ADDR_ANY;
1719 chinfo.dst = RPMSG_ADDR_ANY;
1720
1721 rpmsg_unregister_device(glink->dev, &chinfo);
1722 }
1723 channel->rpdev = NULL;
1724
1725 qcom_glink_send_close_ack(glink, channel);
1726
1727 spin_lock_irqsave(&glink->idr_lock, flags);
1728 idr_remove(&glink->rcids, channel->rcid);
1729 channel->rcid = 0;
1730 spin_unlock_irqrestore(&glink->idr_lock, flags);
1731
1732 kref_put(&channel->refcount, qcom_glink_channel_release);
1733 }
1734
1735 static void qcom_glink_rx_close_ack(struct qcom_glink *glink, unsigned int lcid)
1736 {
1737 struct rpmsg_channel_info chinfo;
1738 struct glink_channel *channel;
1739 unsigned long flags;
1740
1741 /* To wakeup any blocking writers */
1742 wake_up_all(&glink->tx_avail_notify);
1743
1744 spin_lock_irqsave(&glink->idr_lock, flags);
1745 channel = idr_find(&glink->lcids, lcid);
1746
1747 trace_qcom_glink_cmd_close_ack_rx(glink->label, channel ? channel->name : NULL,
1748 lcid, channel ? channel->rcid : 0);
1749 if (WARN(!channel, "close ack on unknown channel\n")) {
1750 spin_unlock_irqrestore(&glink->idr_lock, flags);
1751 return;
1752 }
1753
1754 idr_remove(&glink->lcids, channel->lcid);
1755 channel->lcid = 0;
1756 spin_unlock_irqrestore(&glink->idr_lock, flags);
1757
1758 /* Decouple the potential rpdev from the channel */
1759 if (channel->rpdev) {
1760 strscpy(chinfo.name, channel->name, sizeof(chinfo.name));
1761 chinfo.src = RPMSG_ADDR_ANY;
1762 chinfo.dst = RPMSG_ADDR_ANY;
1763
1764 rpmsg_unregister_device(glink->dev, &chinfo);
1765 }
1766 channel->rpdev = NULL;
1767
1768 kref_put(&channel->refcount, qcom_glink_channel_release);
1769 }
1770
1771 static void qcom_glink_work(struct work_struct *work)
1772 {
1773 struct qcom_glink *glink = container_of(work, struct qcom_glink,
1774 rx_work);
1775 struct glink_defer_cmd *dcmd;
1776 struct glink_msg *msg;
1777 unsigned long flags;
1778 unsigned int param1;
1779 unsigned int param2;
1780 unsigned int cmd;
1781
1782 for (;;) {
1783 spin_lock_irqsave(&glink->rx_lock, flags);
1784 if (list_empty(&glink->rx_queue)) {
1785 spin_unlock_irqrestore(&glink->rx_lock, flags);
1786 break;
1787 }
1788 dcmd = list_first_entry(&glink->rx_queue,
1789 struct glink_defer_cmd, node);
1790 list_del(&dcmd->node);
1791 spin_unlock_irqrestore(&glink->rx_lock, flags);
1792
1793 msg = container_of(&dcmd->msg, struct glink_msg, hdr);
1794 cmd = le16_to_cpu(msg->cmd);
1795 param1 = le16_to_cpu(msg->param1);
1796 param2 = le32_to_cpu(msg->param2);
1797
1798 switch (cmd) {
1799 case GLINK_CMD_VERSION:
1800 qcom_glink_receive_version(glink, param1, param2);
1801 break;
1802 case GLINK_CMD_VERSION_ACK:
1803 qcom_glink_receive_version_ack(glink, param1, param2);
1804 break;
1805 case GLINK_CMD_OPEN:
1806 qcom_glink_rx_open(glink, param1, msg->data);
1807 break;
1808 case GLINK_CMD_CLOSE:
1809 qcom_glink_rx_close(glink, param1);
1810 break;
1811 case GLINK_CMD_CLOSE_ACK:
1812 qcom_glink_rx_close_ack(glink, param1);
1813 break;
1814 case GLINK_CMD_RX_INTENT_REQ:
1815 qcom_glink_handle_intent_req(glink, param1, param2);
1816 break;
1817 default:
1818 WARN(1, "Unknown defer object %d\n", cmd);
1819 break;
1820 }
1821
1822 kfree(dcmd);
1823 }
1824 }
1825
1826 static void qcom_glink_cancel_rx_work(struct qcom_glink *glink)
1827 {
1828 struct glink_defer_cmd *dcmd;
1829 struct glink_defer_cmd *tmp;
1830
1831 /* cancel any pending deferred rx_work */
1832 cancel_work_sync(&glink->rx_work);
1833
1834 list_for_each_entry_safe(dcmd, tmp, &glink->rx_queue, node)
1835 kfree(dcmd);
1836 }
1837
1838 static ssize_t rpmsg_name_show(struct device *dev,
1839 struct device_attribute *attr, char *buf)
1840 {
1841 int ret = 0;
1842 const char *name;
1843
1844 ret = of_property_read_string(dev->of_node, "label", &name);
1845 if (ret < 0)
1846 name = dev->of_node->name;
1847
1848 return sysfs_emit(buf, "%s\n", name);
1849 }
1850 static DEVICE_ATTR_RO(rpmsg_name);
1851
1852 static struct attribute *qcom_glink_attrs[] = {
1853 &dev_attr_rpmsg_name.attr,
1854 NULL
1855 };
1856 ATTRIBUTE_GROUPS(qcom_glink);
1857
1858 static void qcom_glink_device_release(struct device *dev)
1859 {
1860 struct rpmsg_device *rpdev = to_rpmsg_device(dev);
1861 struct glink_channel *channel = to_glink_channel(rpdev->ept);
1862
1863 /* Release qcom_glink_alloc_channel() reference */
1864 kref_put(&channel->refcount, qcom_glink_channel_release);
1865 kfree(rpdev->driver_override);
1866 kfree(rpdev);
1867 }
1868
1869 static int qcom_glink_create_chrdev(struct qcom_glink *glink)
1870 {
1871 struct rpmsg_device *rpdev;
1872 struct glink_channel *channel;
1873
1874 rpdev = kzalloc(sizeof(*rpdev), GFP_KERNEL);
1875 if (!rpdev)
1876 return -ENOMEM;
1877
1878 channel = qcom_glink_alloc_channel(glink, "rpmsg_chrdev");
1879 if (IS_ERR(channel)) {
1880 kfree(rpdev);
1881 return PTR_ERR(channel);
1882 }
1883 channel->rpdev = rpdev;
1884
1885 rpdev->ept = &channel->ept;
1886 rpdev->ops = &glink_device_ops;
1887 rpdev->dev.parent = glink->dev;
1888 rpdev->dev.release = qcom_glink_device_release;
1889
1890 return rpmsg_ctrldev_register_device(rpdev);
1891 }
1892
1893 struct qcom_glink *qcom_glink_native_probe(struct device *dev,
1894 unsigned long features,
1895 struct qcom_glink_pipe *rx,
1896 struct qcom_glink_pipe *tx,
1897 bool intentless)
1898 {
1899 int ret;
1900 struct qcom_glink *glink;
1901
1902 glink = devm_kzalloc(dev, sizeof(*glink), GFP_KERNEL);
1903 if (!glink)
1904 return ERR_PTR(-ENOMEM);
1905
1906 glink->dev = dev;
1907 glink->tx_pipe = tx;
1908 glink->rx_pipe = rx;
1909
1910 glink->features = features;
1911 glink->intentless = intentless;
1912
1913 spin_lock_init(&glink->tx_lock);
1914 spin_lock_init(&glink->rx_lock);
1915 INIT_LIST_HEAD(&glink->rx_queue);
1916 INIT_WORK(&glink->rx_work, qcom_glink_work);
1917 init_waitqueue_head(&glink->tx_avail_notify);
1918
1919 spin_lock_init(&glink->idr_lock);
1920 idr_init(&glink->lcids);
1921 idr_init(&glink->rcids);
1922
1923 ret = of_property_read_string(dev->of_node, "label", &glink->label);
1924 if (ret < 0)
1925 glink->label = dev->of_node->name;
1926
1927 glink->dev->groups = qcom_glink_groups;
1928
1929 ret = device_add_groups(dev, qcom_glink_groups);
1930 if (ret)
1931 dev_err(dev, "failed to add groups\n");
1932
1933 ret = qcom_glink_send_version(glink);
1934 if (ret)
1935 return ERR_PTR(ret);
1936
1937 ret = qcom_glink_create_chrdev(glink);
1938 if (ret)
1939 dev_err(glink->dev, "failed to register chrdev\n");
1940
1941 return glink;
1942 }
1943 EXPORT_SYMBOL_GPL(qcom_glink_native_probe);
1944
1945 static int qcom_glink_remove_device(struct device *dev, void *data)
1946 {
1947 device_unregister(dev);
1948
1949 return 0;
1950 }
1951
1952 void qcom_glink_native_remove(struct qcom_glink *glink)
1953 {
1954 struct glink_channel *channel;
1955 unsigned long flags;
1956 int cid;
1957 int ret;
1958
1959 qcom_glink_cancel_rx_work(glink);
1960
1961 /* Fail all attempts at sending messages */
1962 spin_lock_irqsave(&glink->tx_lock, flags);
1963 glink->abort_tx = true;
1964 wake_up_all(&glink->tx_avail_notify);
1965 spin_unlock_irqrestore(&glink->tx_lock, flags);
1966
1967 /* Abort any senders waiting for intent requests */
1968 spin_lock_irqsave(&glink->idr_lock, flags);
1969 idr_for_each_entry(&glink->lcids, channel, cid)
1970 qcom_glink_intent_req_abort(channel);
1971 spin_unlock_irqrestore(&glink->idr_lock, flags);
1972
1973 ret = device_for_each_child(glink->dev, NULL, qcom_glink_remove_device);
1974 if (ret)
1975 dev_warn(glink->dev, "Can't remove GLINK devices: %d\n", ret);
1976
1977 /* Release any defunct local channels, waiting for close-ack */
1978 idr_for_each_entry(&glink->lcids, channel, cid)
1979 kref_put(&channel->refcount, qcom_glink_channel_release);
1980
1981 /* Release any defunct local channels, waiting for close-req */
1982 idr_for_each_entry(&glink->rcids, channel, cid)
1983 kref_put(&channel->refcount, qcom_glink_channel_release);
1984
1985 idr_destroy(&glink->lcids);
1986 idr_destroy(&glink->rcids);
1987 }
1988 EXPORT_SYMBOL_GPL(qcom_glink_native_remove);
1989
1990 MODULE_DESCRIPTION("Qualcomm GLINK driver");
1991 MODULE_LICENSE("GPL v2");
Kernel DRM miscellaneous fixes and cross-tree changes