?  search:
summary | log | graphiclog1 | graphiclog2 | commit | commitdiff | tree | refs | edit | fork
blame | history | raw | HEAD
io_uring: ensure finish_wait() is always called in __io_uring_task_cancel()
[linux/fpc-iii.git] / drivers / thunderbolt / icm.c
blob8b7f941a9bb7f104f79f205cd236053e34cdc293
1 // SPDX-License-Identifier: GPL-2.0
2 /*
3 * Internal Thunderbolt Connection Manager. This is a firmware running on
4 * the Thunderbolt host controller performing most of the low-level
5 * handling.
6 *
7 * Copyright (C) 2017, Intel Corporation
8 * Authors: Michael Jamet <michael.jamet@intel.com>
9 * Mika Westerberg <mika.westerberg@linux.intel.com>
10 */
11
12 #include <linux/delay.h>
13 #include <linux/mutex.h>
14 #include <linux/moduleparam.h>
15 #include <linux/pci.h>
16 #include <linux/pm_runtime.h>
17 #include <linux/platform_data/x86/apple.h>
18 #include <linux/sizes.h>
19 #include <linux/slab.h>
20 #include <linux/workqueue.h>
21
22 #include "ctl.h"
23 #include "nhi_regs.h"
24 #include "tb.h"
25
26 #define PCIE2CIO_CMD 0x30
27 #define PCIE2CIO_CMD_TIMEOUT BIT(31)
28 #define PCIE2CIO_CMD_START BIT(30)
29 #define PCIE2CIO_CMD_WRITE BIT(21)
30 #define PCIE2CIO_CMD_CS_MASK GENMASK(20, 19)
31 #define PCIE2CIO_CMD_CS_SHIFT 19
32 #define PCIE2CIO_CMD_PORT_MASK GENMASK(18, 13)
33 #define PCIE2CIO_CMD_PORT_SHIFT 13
34
35 #define PCIE2CIO_WRDATA 0x34
36 #define PCIE2CIO_RDDATA 0x38
37
38 #define PHY_PORT_CS1 0x37
39 #define PHY_PORT_CS1_LINK_DISABLE BIT(14)
40 #define PHY_PORT_CS1_LINK_STATE_MASK GENMASK(29, 26)
41 #define PHY_PORT_CS1_LINK_STATE_SHIFT 26
42
43 #define ICM_TIMEOUT 5000 /* ms */
44 #define ICM_APPROVE_TIMEOUT 10000 /* ms */
45 #define ICM_MAX_LINK 4
46
47 static bool start_icm;
48 module_param(start_icm, bool, 0444);
49 MODULE_PARM_DESC(start_icm, "start ICM firmware if it is not running (default: false)");
50
51 /**
52 * struct usb4_switch_nvm_auth - Holds USB4 NVM_AUTH status
53 * @reply: Reply from ICM firmware is placed here
54 * @request: Request that is sent to ICM firmware
55 * @icm: Pointer to ICM private data
56 */
57 struct usb4_switch_nvm_auth {
58 struct icm_usb4_switch_op_response reply;
59 struct icm_usb4_switch_op request;
60 struct icm *icm;
61 };
62
63 /**
64 * struct icm - Internal connection manager private data
65 * @request_lock: Makes sure only one message is send to ICM at time
66 * @rescan_work: Work used to rescan the surviving switches after resume
67 * @upstream_port: Pointer to the PCIe upstream port this host
68 * controller is connected. This is only set for systems
69 * where ICM needs to be started manually
70 * @vnd_cap: Vendor defined capability where PCIe2CIO mailbox resides
71 * (only set when @upstream_port is not %NULL)
72 * @safe_mode: ICM is in safe mode
73 * @max_boot_acl: Maximum number of preboot ACL entries (%0 if not supported)
74 * @rpm: Does the controller support runtime PM (RTD3)
75 * @can_upgrade_nvm: Can the NVM firmware be upgrade on this controller
76 * @proto_version: Firmware protocol version
77 * @last_nvm_auth: Last USB4 router NVM_AUTH result (or %NULL if not set)
78 * @veto: Is RTD3 veto in effect
79 * @is_supported: Checks if we can support ICM on this controller
80 * @cio_reset: Trigger CIO reset
81 * @get_mode: Read and return the ICM firmware mode (optional)
82 * @get_route: Find a route string for given switch
83 * @save_devices: Ask ICM to save devices to ACL when suspending (optional)
84 * @driver_ready: Send driver ready message to ICM
85 * @set_uuid: Set UUID for the root switch (optional)
86 * @device_connected: Handle device connected ICM message
87 * @device_disconnected: Handle device disconnected ICM message
88 * @xdomain_connected - Handle XDomain connected ICM message
89 * @xdomain_disconnected - Handle XDomain disconnected ICM message
90 * @rtd3_veto: Handle RTD3 veto notification ICM message
91 */
92 struct icm {
93 struct mutex request_lock;
94 struct delayed_work rescan_work;
95 struct pci_dev *upstream_port;
96 int vnd_cap;
97 bool safe_mode;
98 size_t max_boot_acl;
99 bool rpm;
100 bool can_upgrade_nvm;
101 u8 proto_version;
102 struct usb4_switch_nvm_auth *last_nvm_auth;
103 bool veto;
104 bool (*is_supported)(struct tb *tb);
105 int (*cio_reset)(struct tb *tb);
106 int (*get_mode)(struct tb *tb);
107 int (*get_route)(struct tb *tb, u8 link, u8 depth, u64 *route);
108 void (*save_devices)(struct tb *tb);
109 int (*driver_ready)(struct tb *tb,
110 enum tb_security_level *security_level,
111 u8 *proto_version, size_t *nboot_acl, bool *rpm);
112 void (*set_uuid)(struct tb *tb);
113 void (*device_connected)(struct tb *tb,
114 const struct icm_pkg_header *hdr);
115 void (*device_disconnected)(struct tb *tb,
116 const struct icm_pkg_header *hdr);
117 void (*xdomain_connected)(struct tb *tb,
118 const struct icm_pkg_header *hdr);
119 void (*xdomain_disconnected)(struct tb *tb,
120 const struct icm_pkg_header *hdr);
121 void (*rtd3_veto)(struct tb *tb, const struct icm_pkg_header *hdr);
122 };
123
124 struct icm_notification {
125 struct work_struct work;
126 struct icm_pkg_header *pkg;
127 struct tb *tb;
128 };
129
130 struct ep_name_entry {
131 u8 len;
132 u8 type;
133 u8 data[];
134 };
135
136 #define EP_NAME_INTEL_VSS 0x10
137
138 /* Intel Vendor specific structure */
139 struct intel_vss {
140 u16 vendor;
141 u16 model;
142 u8 mc;
143 u8 flags;
144 u16 pci_devid;
145 u32 nvm_version;
146 };
147
148 #define INTEL_VSS_FLAGS_RTD3 BIT(0)
149
150 static const struct intel_vss *parse_intel_vss(const void *ep_name, size_t size)
151 {
152 const void *end = ep_name + size;
153
154 while (ep_name < end) {
155 const struct ep_name_entry *ep = ep_name;
156
157 if (!ep->len)
158 break;
159 if (ep_name + ep->len > end)
160 break;
161
162 if (ep->type == EP_NAME_INTEL_VSS)
163 return (const struct intel_vss *)ep->data;
164
165 ep_name += ep->len;
166 }
167
168 return NULL;
169 }
170
171 static bool intel_vss_is_rtd3(const void *ep_name, size_t size)
172 {
173 const struct intel_vss *vss;
174
175 vss = parse_intel_vss(ep_name, size);
176 if (vss)
177 return !!(vss->flags & INTEL_VSS_FLAGS_RTD3);
178
179 return false;
180 }
181
182 static inline struct tb *icm_to_tb(struct icm *icm)
183 {
184 return ((void *)icm - sizeof(struct tb));
185 }
186
187 static inline u8 phy_port_from_route(u64 route, u8 depth)
188 {
189 u8 link;
190
191 link = depth ? route >> ((depth - 1) * 8) : route;
192 return tb_phy_port_from_link(link);
193 }
194
195 static inline u8 dual_link_from_link(u8 link)
196 {
197 return link ? ((link - 1) ^ 0x01) + 1 : 0;
198 }
199
200 static inline u64 get_route(u32 route_hi, u32 route_lo)
201 {
202 return (u64)route_hi << 32 | route_lo;
203 }
204
205 static inline u64 get_parent_route(u64 route)
206 {
207 int depth = tb_route_length(route);
208 return depth ? route & ~(0xffULL << (depth - 1) * TB_ROUTE_SHIFT) : 0;
209 }
210
211 static int pci2cio_wait_completion(struct icm *icm, unsigned long timeout_msec)
212 {
213 unsigned long end = jiffies + msecs_to_jiffies(timeout_msec);
214 u32 cmd;
215
216 do {
217 pci_read_config_dword(icm->upstream_port,
218 icm->vnd_cap + PCIE2CIO_CMD, &cmd);
219 if (!(cmd & PCIE2CIO_CMD_START)) {
220 if (cmd & PCIE2CIO_CMD_TIMEOUT)
221 break;
222 return 0;
223 }
224
225 msleep(50);
226 } while (time_before(jiffies, end));
227
228 return -ETIMEDOUT;
229 }
230
231 static int pcie2cio_read(struct icm *icm, enum tb_cfg_space cs,
232 unsigned int port, unsigned int index, u32 *data)
233 {
234 struct pci_dev *pdev = icm->upstream_port;
235 int ret, vnd_cap = icm->vnd_cap;
236 u32 cmd;
237
238 cmd = index;
239 cmd |= (port << PCIE2CIO_CMD_PORT_SHIFT) & PCIE2CIO_CMD_PORT_MASK;
240 cmd |= (cs << PCIE2CIO_CMD_CS_SHIFT) & PCIE2CIO_CMD_CS_MASK;
241 cmd |= PCIE2CIO_CMD_START;
242 pci_write_config_dword(pdev, vnd_cap + PCIE2CIO_CMD, cmd);
243
244 ret = pci2cio_wait_completion(icm, 5000);
245 if (ret)
246 return ret;
247
248 pci_read_config_dword(pdev, vnd_cap + PCIE2CIO_RDDATA, data);
249 return 0;
250 }
251
252 static int pcie2cio_write(struct icm *icm, enum tb_cfg_space cs,
253 unsigned int port, unsigned int index, u32 data)
254 {
255 struct pci_dev *pdev = icm->upstream_port;
256 int vnd_cap = icm->vnd_cap;
257 u32 cmd;
258
259 pci_write_config_dword(pdev, vnd_cap + PCIE2CIO_WRDATA, data);
260
261 cmd = index;
262 cmd |= (port << PCIE2CIO_CMD_PORT_SHIFT) & PCIE2CIO_CMD_PORT_MASK;
263 cmd |= (cs << PCIE2CIO_CMD_CS_SHIFT) & PCIE2CIO_CMD_CS_MASK;
264 cmd |= PCIE2CIO_CMD_WRITE | PCIE2CIO_CMD_START;
265 pci_write_config_dword(pdev, vnd_cap + PCIE2CIO_CMD, cmd);
266
267 return pci2cio_wait_completion(icm, 5000);
268 }
269
270 static bool icm_match(const struct tb_cfg_request *req,
271 const struct ctl_pkg *pkg)
272 {
273 const struct icm_pkg_header *res_hdr = pkg->buffer;
274 const struct icm_pkg_header *req_hdr = req->request;
275
276 if (pkg->frame.eof != req->response_type)
277 return false;
278 if (res_hdr->code != req_hdr->code)
279 return false;
280
281 return true;
282 }
283
284 static bool icm_copy(struct tb_cfg_request *req, const struct ctl_pkg *pkg)
285 {
286 const struct icm_pkg_header *hdr = pkg->buffer;
287
288 if (hdr->packet_id < req->npackets) {
289 size_t offset = hdr->packet_id * req->response_size;
290
291 memcpy(req->response + offset, pkg->buffer, req->response_size);
292 }
293
294 return hdr->packet_id == hdr->total_packets - 1;
295 }
296
297 static int icm_request(struct tb *tb, const void *request, size_t request_size,
298 void *response, size_t response_size, size_t npackets,
299 unsigned int timeout_msec)
300 {
301 struct icm *icm = tb_priv(tb);
302 int retries = 3;
303
304 do {
305 struct tb_cfg_request *req;
306 struct tb_cfg_result res;
307
308 req = tb_cfg_request_alloc();
309 if (!req)
310 return -ENOMEM;
311
312 req->match = icm_match;
313 req->copy = icm_copy;
314 req->request = request;
315 req->request_size = request_size;
316 req->request_type = TB_CFG_PKG_ICM_CMD;
317 req->response = response;
318 req->npackets = npackets;
319 req->response_size = response_size;
320 req->response_type = TB_CFG_PKG_ICM_RESP;
321
322 mutex_lock(&icm->request_lock);
323 res = tb_cfg_request_sync(tb->ctl, req, timeout_msec);
324 mutex_unlock(&icm->request_lock);
325
326 tb_cfg_request_put(req);
327
328 if (res.err != -ETIMEDOUT)
329 return res.err == 1 ? -EIO : res.err;
330
331 usleep_range(20, 50);
332 } while (retries--);
333
334 return -ETIMEDOUT;
335 }
336
337 /*
338 * If rescan is queued to run (we are resuming), postpone it to give the
339 * firmware some more time to send device connected notifications for next
340 * devices in the chain.
341 */
342 static void icm_postpone_rescan(struct tb *tb)
343 {
344 struct icm *icm = tb_priv(tb);
345
346 if (delayed_work_pending(&icm->rescan_work))
347 mod_delayed_work(tb->wq, &icm->rescan_work,
348 msecs_to_jiffies(500));
349 }
350
351 static void icm_veto_begin(struct tb *tb)
352 {
353 struct icm *icm = tb_priv(tb);
354
355 if (!icm->veto) {
356 icm->veto = true;
357 /* Keep the domain powered while veto is in effect */
358 pm_runtime_get(&tb->dev);
359 }
360 }
361
362 static void icm_veto_end(struct tb *tb)
363 {
364 struct icm *icm = tb_priv(tb);
365
366 if (icm->veto) {
367 icm->veto = false;
368 /* Allow the domain suspend now */
369 pm_runtime_mark_last_busy(&tb->dev);
370 pm_runtime_put_autosuspend(&tb->dev);
371 }
372 }
373
374 static bool icm_firmware_running(const struct tb_nhi *nhi)
375 {
376 u32 val;
377
378 val = ioread32(nhi->iobase + REG_FW_STS);
379 return !!(val & REG_FW_STS_ICM_EN);
380 }
381
382 static bool icm_fr_is_supported(struct tb *tb)
383 {
384 return !x86_apple_machine;
385 }
386
387 static inline int icm_fr_get_switch_index(u32 port)
388 {
389 int index;
390
391 if ((port & ICM_PORT_TYPE_MASK) != TB_TYPE_PORT)
392 return 0;
393
394 index = port >> ICM_PORT_INDEX_SHIFT;
395 return index != 0xff ? index : 0;
396 }
397
398 static int icm_fr_get_route(struct tb *tb, u8 link, u8 depth, u64 *route)
399 {
400 struct icm_fr_pkg_get_topology_response *switches, *sw;
401 struct icm_fr_pkg_get_topology request = {
402 .hdr = { .code = ICM_GET_TOPOLOGY },
403 };
404 size_t npackets = ICM_GET_TOPOLOGY_PACKETS;
405 int ret, index;
406 u8 i;
407
408 switches = kcalloc(npackets, sizeof(*switches), GFP_KERNEL);
409 if (!switches)
410 return -ENOMEM;
411
412 ret = icm_request(tb, &request, sizeof(request), switches,
413 sizeof(*switches), npackets, ICM_TIMEOUT);
414 if (ret)
415 goto err_free;
416
417 sw = &switches[0];
418 index = icm_fr_get_switch_index(sw->ports[link]);
419 if (!index) {
420 ret = -ENODEV;
421 goto err_free;
422 }
423
424 sw = &switches[index];
425 for (i = 1; i < depth; i++) {
426 unsigned int j;
427
428 if (!(sw->first_data & ICM_SWITCH_USED)) {
429 ret = -ENODEV;
430 goto err_free;
431 }
432
433 for (j = 0; j < ARRAY_SIZE(sw->ports); j++) {
434 index = icm_fr_get_switch_index(sw->ports[j]);
435 if (index > sw->switch_index) {
436 sw = &switches[index];
437 break;
438 }
439 }
440 }
441
442 *route = get_route(sw->route_hi, sw->route_lo);
443
444 err_free:
445 kfree(switches);
446 return ret;
447 }
448
449 static void icm_fr_save_devices(struct tb *tb)
450 {
451 nhi_mailbox_cmd(tb->nhi, NHI_MAILBOX_SAVE_DEVS, 0);
452 }
453
454 static int
455 icm_fr_driver_ready(struct tb *tb, enum tb_security_level *security_level,
456 u8 *proto_version, size_t *nboot_acl, bool *rpm)
457 {
458 struct icm_fr_pkg_driver_ready_response reply;
459 struct icm_pkg_driver_ready request = {
460 .hdr.code = ICM_DRIVER_READY,
461 };
462 int ret;
463
464 memset(&reply, 0, sizeof(reply));
465 ret = icm_request(tb, &request, sizeof(request), &reply, sizeof(reply),
466 1, ICM_TIMEOUT);
467 if (ret)
468 return ret;
469
470 if (security_level)
471 *security_level = reply.security_level & ICM_FR_SLEVEL_MASK;
472
473 return 0;
474 }
475
476 static int icm_fr_approve_switch(struct tb *tb, struct tb_switch *sw)
477 {
478 struct icm_fr_pkg_approve_device request;
479 struct icm_fr_pkg_approve_device reply;
480 int ret;
481
482 memset(&request, 0, sizeof(request));
483 memcpy(&request.ep_uuid, sw->uuid, sizeof(request.ep_uuid));
484 request.hdr.code = ICM_APPROVE_DEVICE;
485 request.connection_id = sw->connection_id;
486 request.connection_key = sw->connection_key;
487
488 memset(&reply, 0, sizeof(reply));
489 /* Use larger timeout as establishing tunnels can take some time */
490 ret = icm_request(tb, &request, sizeof(request), &reply, sizeof(reply),
491 1, ICM_APPROVE_TIMEOUT);
492 if (ret)
493 return ret;
494
495 if (reply.hdr.flags & ICM_FLAGS_ERROR) {
496 tb_warn(tb, "PCIe tunnel creation failed\n");
497 return -EIO;
498 }
499
500 return 0;
501 }
502
503 static int icm_fr_add_switch_key(struct tb *tb, struct tb_switch *sw)
504 {
505 struct icm_fr_pkg_add_device_key request;
506 struct icm_fr_pkg_add_device_key_response reply;
507 int ret;
508
509 memset(&request, 0, sizeof(request));
510 memcpy(&request.ep_uuid, sw->uuid, sizeof(request.ep_uuid));
511 request.hdr.code = ICM_ADD_DEVICE_KEY;
512 request.connection_id = sw->connection_id;
513 request.connection_key = sw->connection_key;
514 memcpy(request.key, sw->key, TB_SWITCH_KEY_SIZE);
515
516 memset(&reply, 0, sizeof(reply));
517 ret = icm_request(tb, &request, sizeof(request), &reply, sizeof(reply),
518 1, ICM_TIMEOUT);
519 if (ret)
520 return ret;
521
522 if (reply.hdr.flags & ICM_FLAGS_ERROR) {
523 tb_warn(tb, "Adding key to switch failed\n");
524 return -EIO;
525 }
526
527 return 0;
528 }
529
530 static int icm_fr_challenge_switch_key(struct tb *tb, struct tb_switch *sw,
531 const u8 *challenge, u8 *response)
532 {
533 struct icm_fr_pkg_challenge_device request;
534 struct icm_fr_pkg_challenge_device_response reply;
535 int ret;
536
537 memset(&request, 0, sizeof(request));
538 memcpy(&request.ep_uuid, sw->uuid, sizeof(request.ep_uuid));
539 request.hdr.code = ICM_CHALLENGE_DEVICE;
540 request.connection_id = sw->connection_id;
541 request.connection_key = sw->connection_key;
542 memcpy(request.challenge, challenge, TB_SWITCH_KEY_SIZE);
543
544 memset(&reply, 0, sizeof(reply));
545 ret = icm_request(tb, &request, sizeof(request), &reply, sizeof(reply),
546 1, ICM_TIMEOUT);
547 if (ret)
548 return ret;
549
550 if (reply.hdr.flags & ICM_FLAGS_ERROR)
551 return -EKEYREJECTED;
552 if (reply.hdr.flags & ICM_FLAGS_NO_KEY)
553 return -ENOKEY;
554
555 memcpy(response, reply.response, TB_SWITCH_KEY_SIZE);
556
557 return 0;
558 }
559
560 static int icm_fr_approve_xdomain_paths(struct tb *tb, struct tb_xdomain *xd)
561 {
562 struct icm_fr_pkg_approve_xdomain_response reply;
563 struct icm_fr_pkg_approve_xdomain request;
564 int ret;
565
566 memset(&request, 0, sizeof(request));
567 request.hdr.code = ICM_APPROVE_XDOMAIN;
568 request.link_info = xd->depth << ICM_LINK_INFO_DEPTH_SHIFT | xd->link;
569 memcpy(&request.remote_uuid, xd->remote_uuid, sizeof(*xd->remote_uuid));
570
571 request.transmit_path = xd->transmit_path;
572 request.transmit_ring = xd->transmit_ring;
573 request.receive_path = xd->receive_path;
574 request.receive_ring = xd->receive_ring;
575
576 memset(&reply, 0, sizeof(reply));
577 ret = icm_request(tb, &request, sizeof(request), &reply, sizeof(reply),
578 1, ICM_TIMEOUT);
579 if (ret)
580 return ret;
581
582 if (reply.hdr.flags & ICM_FLAGS_ERROR)
583 return -EIO;
584
585 return 0;
586 }
587
588 static int icm_fr_disconnect_xdomain_paths(struct tb *tb, struct tb_xdomain *xd)
589 {
590 u8 phy_port;
591 u8 cmd;
592
593 phy_port = tb_phy_port_from_link(xd->link);
594 if (phy_port == 0)
595 cmd = NHI_MAILBOX_DISCONNECT_PA;
596 else
597 cmd = NHI_MAILBOX_DISCONNECT_PB;
598
599 nhi_mailbox_cmd(tb->nhi, cmd, 1);
600 usleep_range(10, 50);
601 nhi_mailbox_cmd(tb->nhi, cmd, 2);
602 return 0;
603 }
604
605 static struct tb_switch *alloc_switch(struct tb_switch *parent_sw, u64 route,
606 const uuid_t *uuid)
607 {
608 struct tb *tb = parent_sw->tb;
609 struct tb_switch *sw;
610
611 sw = tb_switch_alloc(tb, &parent_sw->dev, route);
612 if (IS_ERR(sw)) {
613 tb_warn(tb, "failed to allocate switch at %llx\n", route);
614 return sw;
615 }
616
617 sw->uuid = kmemdup(uuid, sizeof(*uuid), GFP_KERNEL);
618 if (!sw->uuid) {
619 tb_switch_put(sw);
620 return ERR_PTR(-ENOMEM);
621 }
622
623 init_completion(&sw->rpm_complete);
624 return sw;
625 }
626
627 static int add_switch(struct tb_switch *parent_sw, struct tb_switch *sw)
628 {
629 u64 route = tb_route(sw);
630 int ret;
631
632 /* Link the two switches now */
633 tb_port_at(route, parent_sw)->remote = tb_upstream_port(sw);
634 tb_upstream_port(sw)->remote = tb_port_at(route, parent_sw);
635
636 ret = tb_switch_add(sw);
637 if (ret)
638 tb_port_at(tb_route(sw), parent_sw)->remote = NULL;
639
640 return ret;
641 }
642
643 static void update_switch(struct tb_switch *parent_sw, struct tb_switch *sw,
644 u64 route, u8 connection_id, u8 connection_key,
645 u8 link, u8 depth, bool boot)
646 {
647 /* Disconnect from parent */
648 tb_port_at(tb_route(sw), parent_sw)->remote = NULL;
649 /* Re-connect via updated port*/
650 tb_port_at(route, parent_sw)->remote = tb_upstream_port(sw);
651
652 /* Update with the new addressing information */
653 sw->config.route_hi = upper_32_bits(route);
654 sw->config.route_lo = lower_32_bits(route);
655 sw->connection_id = connection_id;
656 sw->connection_key = connection_key;
657 sw->link = link;
658 sw->depth = depth;
659 sw->boot = boot;
660
661 /* This switch still exists */
662 sw->is_unplugged = false;
663
664 /* Runtime resume is now complete */
665 complete(&sw->rpm_complete);
666 }
667
668 static void remove_switch(struct tb_switch *sw)
669 {
670 struct tb_switch *parent_sw;
671
672 parent_sw = tb_to_switch(sw->dev.parent);
673 tb_port_at(tb_route(sw), parent_sw)->remote = NULL;
674 tb_switch_remove(sw);
675 }
676
677 static void add_xdomain(struct tb_switch *sw, u64 route,
678 const uuid_t *local_uuid, const uuid_t *remote_uuid,
679 u8 link, u8 depth)
680 {
681 struct tb_xdomain *xd;
682
683 pm_runtime_get_sync(&sw->dev);
684
685 xd = tb_xdomain_alloc(sw->tb, &sw->dev, route, local_uuid, remote_uuid);
686 if (!xd)
687 goto out;
688
689 xd->link = link;
690 xd->depth = depth;
691
692 tb_port_at(route, sw)->xdomain = xd;
693
694 tb_xdomain_add(xd);
695
696 out:
697 pm_runtime_mark_last_busy(&sw->dev);
698 pm_runtime_put_autosuspend(&sw->dev);
699 }
700
701 static void update_xdomain(struct tb_xdomain *xd, u64 route, u8 link)
702 {
703 xd->link = link;
704 xd->route = route;
705 xd->is_unplugged = false;
706 }
707
708 static void remove_xdomain(struct tb_xdomain *xd)
709 {
710 struct tb_switch *sw;
711
712 sw = tb_to_switch(xd->dev.parent);
713 tb_port_at(xd->route, sw)->xdomain = NULL;
714 tb_xdomain_remove(xd);
715 }
716
717 static void
718 icm_fr_device_connected(struct tb *tb, const struct icm_pkg_header *hdr)
719 {
720 const struct icm_fr_event_device_connected *pkg =
721 (const struct icm_fr_event_device_connected *)hdr;
722 enum tb_security_level security_level;
723 struct tb_switch *sw, *parent_sw;
724 bool boot, dual_lane, speed_gen3;
725 struct icm *icm = tb_priv(tb);
726 bool authorized = false;
727 struct tb_xdomain *xd;
728 u8 link, depth;
729 u64 route;
730 int ret;
731
732 icm_postpone_rescan(tb);
733
734 link = pkg->link_info & ICM_LINK_INFO_LINK_MASK;
735 depth = (pkg->link_info & ICM_LINK_INFO_DEPTH_MASK) >>
736 ICM_LINK_INFO_DEPTH_SHIFT;
737 authorized = pkg->link_info & ICM_LINK_INFO_APPROVED;
738 security_level = (pkg->hdr.flags & ICM_FLAGS_SLEVEL_MASK) >>
739 ICM_FLAGS_SLEVEL_SHIFT;
740 boot = pkg->link_info & ICM_LINK_INFO_BOOT;
741 dual_lane = pkg->hdr.flags & ICM_FLAGS_DUAL_LANE;
742 speed_gen3 = pkg->hdr.flags & ICM_FLAGS_SPEED_GEN3;
743
744 if (pkg->link_info & ICM_LINK_INFO_REJECTED) {
745 tb_info(tb, "switch at %u.%u was rejected by ICM firmware because topology limit exceeded\n",
746 link, depth);
747 return;
748 }
749
750 sw = tb_switch_find_by_uuid(tb, &pkg->ep_uuid);
751 if (sw) {
752 u8 phy_port, sw_phy_port;
753
754 parent_sw = tb_to_switch(sw->dev.parent);
755 sw_phy_port = tb_phy_port_from_link(sw->link);
756 phy_port = tb_phy_port_from_link(link);
757
758 /*
759 * On resume ICM will send us connected events for the
760 * devices that still are present. However, that
761 * information might have changed for example by the
762 * fact that a switch on a dual-link connection might
763 * have been enumerated using the other link now. Make
764 * sure our book keeping matches that.
765 */
766 if (sw->depth == depth && sw_phy_port == phy_port &&
767 !!sw->authorized == authorized) {
768 /*
769 * It was enumerated through another link so update
770 * route string accordingly.
771 */
772 if (sw->link != link) {
773 ret = icm->get_route(tb, link, depth, &route);
774 if (ret) {
775 tb_err(tb, "failed to update route string for switch at %u.%u\n",
776 link, depth);
777 tb_switch_put(sw);
778 return;
779 }
780 } else {
781 route = tb_route(sw);
782 }
783
784 update_switch(parent_sw, sw, route, pkg->connection_id,
785 pkg->connection_key, link, depth, boot);
786 tb_switch_put(sw);
787 return;
788 }
789
790 /*
791 * User connected the same switch to another physical
792 * port or to another part of the topology. Remove the
793 * existing switch now before adding the new one.
794 */
795 remove_switch(sw);
796 tb_switch_put(sw);
797 }
798
799 /*
800 * If the switch was not found by UUID, look for a switch on
801 * same physical port (taking possible link aggregation into
802 * account) and depth. If we found one it is definitely a stale
803 * one so remove it first.
804 */
805 sw = tb_switch_find_by_link_depth(tb, link, depth);
806 if (!sw) {
807 u8 dual_link;
808
809 dual_link = dual_link_from_link(link);
810 if (dual_link)
811 sw = tb_switch_find_by_link_depth(tb, dual_link, depth);
812 }
813 if (sw) {
814 remove_switch(sw);
815 tb_switch_put(sw);
816 }
817
818 /* Remove existing XDomain connection if found */
819 xd = tb_xdomain_find_by_link_depth(tb, link, depth);
820 if (xd) {
821 remove_xdomain(xd);
822 tb_xdomain_put(xd);
823 }
824
825 parent_sw = tb_switch_find_by_link_depth(tb, link, depth - 1);
826 if (!parent_sw) {
827 tb_err(tb, "failed to find parent switch for %u.%u\n",
828 link, depth);
829 return;
830 }
831
832 ret = icm->get_route(tb, link, depth, &route);
833 if (ret) {
834 tb_err(tb, "failed to find route string for switch at %u.%u\n",
835 link, depth);
836 tb_switch_put(parent_sw);
837 return;
838 }
839
840 pm_runtime_get_sync(&parent_sw->dev);
841
842 sw = alloc_switch(parent_sw, route, &pkg->ep_uuid);
843 if (!IS_ERR(sw)) {
844 sw->connection_id = pkg->connection_id;
845 sw->connection_key = pkg->connection_key;
846 sw->link = link;
847 sw->depth = depth;
848 sw->authorized = authorized;
849 sw->security_level = security_level;
850 sw->boot = boot;
851 sw->link_speed = speed_gen3 ? 20 : 10;
852 sw->link_width = dual_lane ? 2 : 1;
853 sw->rpm = intel_vss_is_rtd3(pkg->ep_name, sizeof(pkg->ep_name));
854
855 if (add_switch(parent_sw, sw))
856 tb_switch_put(sw);
857 }
858
859 pm_runtime_mark_last_busy(&parent_sw->dev);
860 pm_runtime_put_autosuspend(&parent_sw->dev);
861
862 tb_switch_put(parent_sw);
863 }
864
865 static void
866 icm_fr_device_disconnected(struct tb *tb, const struct icm_pkg_header *hdr)
867 {
868 const struct icm_fr_event_device_disconnected *pkg =
869 (const struct icm_fr_event_device_disconnected *)hdr;
870 struct tb_switch *sw;
871 u8 link, depth;
872
873 link = pkg->link_info & ICM_LINK_INFO_LINK_MASK;
874 depth = (pkg->link_info & ICM_LINK_INFO_DEPTH_MASK) >>
875 ICM_LINK_INFO_DEPTH_SHIFT;
876
877 if (link > ICM_MAX_LINK || depth > TB_SWITCH_MAX_DEPTH) {
878 tb_warn(tb, "invalid topology %u.%u, ignoring\n", link, depth);
879 return;
880 }
881
882 sw = tb_switch_find_by_link_depth(tb, link, depth);
883 if (!sw) {
884 tb_warn(tb, "no switch exists at %u.%u, ignoring\n", link,
885 depth);
886 return;
887 }
888
889 pm_runtime_get_sync(sw->dev.parent);
890
891 remove_switch(sw);
892
893 pm_runtime_mark_last_busy(sw->dev.parent);
894 pm_runtime_put_autosuspend(sw->dev.parent);
895
896 tb_switch_put(sw);
897 }
898
899 static void
900 icm_fr_xdomain_connected(struct tb *tb, const struct icm_pkg_header *hdr)
901 {
902 const struct icm_fr_event_xdomain_connected *pkg =
903 (const struct icm_fr_event_xdomain_connected *)hdr;
904 struct tb_xdomain *xd;
905 struct tb_switch *sw;
906 u8 link, depth;
907 u64 route;
908
909 link = pkg->link_info & ICM_LINK_INFO_LINK_MASK;
910 depth = (pkg->link_info & ICM_LINK_INFO_DEPTH_MASK) >>
911 ICM_LINK_INFO_DEPTH_SHIFT;
912
913 if (link > ICM_MAX_LINK || depth > TB_SWITCH_MAX_DEPTH) {
914 tb_warn(tb, "invalid topology %u.%u, ignoring\n", link, depth);
915 return;
916 }
917
918 route = get_route(pkg->local_route_hi, pkg->local_route_lo);
919
920 xd = tb_xdomain_find_by_uuid(tb, &pkg->remote_uuid);
921 if (xd) {
922 u8 xd_phy_port, phy_port;
923
924 xd_phy_port = phy_port_from_route(xd->route, xd->depth);
925 phy_port = phy_port_from_route(route, depth);
926
927 if (xd->depth == depth && xd_phy_port == phy_port) {
928 update_xdomain(xd, route, link);
929 tb_xdomain_put(xd);
930 return;
931 }
932
933 /*
934 * If we find an existing XDomain connection remove it
935 * now. We need to go through login handshake and
936 * everything anyway to be able to re-establish the
937 * connection.
938 */
939 remove_xdomain(xd);
940 tb_xdomain_put(xd);
941 }
942
943 /*
944 * Look if there already exists an XDomain in the same place
945 * than the new one and in that case remove it because it is
946 * most likely another host that got disconnected.
947 */
948 xd = tb_xdomain_find_by_link_depth(tb, link, depth);
949 if (!xd) {
950 u8 dual_link;
951
952 dual_link = dual_link_from_link(link);
953 if (dual_link)
954 xd = tb_xdomain_find_by_link_depth(tb, dual_link,
955 depth);
956 }
957 if (xd) {
958 remove_xdomain(xd);
959 tb_xdomain_put(xd);
960 }
961
962 /*
963 * If the user disconnected a switch during suspend and
964 * connected another host to the same port, remove the switch
965 * first.
966 */
967 sw = tb_switch_find_by_route(tb, route);
968 if (sw) {
969 remove_switch(sw);
970 tb_switch_put(sw);
971 }
972
973 sw = tb_switch_find_by_link_depth(tb, link, depth);
974 if (!sw) {
975 tb_warn(tb, "no switch exists at %u.%u, ignoring\n", link,
976 depth);
977 return;
978 }
979
980 add_xdomain(sw, route, &pkg->local_uuid, &pkg->remote_uuid, link,
981 depth);
982 tb_switch_put(sw);
983 }
984
985 static void
986 icm_fr_xdomain_disconnected(struct tb *tb, const struct icm_pkg_header *hdr)
987 {
988 const struct icm_fr_event_xdomain_disconnected *pkg =
989 (const struct icm_fr_event_xdomain_disconnected *)hdr;
990 struct tb_xdomain *xd;
991
992 /*
993 * If the connection is through one or multiple devices, the
994 * XDomain device is removed along with them so it is fine if we
995 * cannot find it here.
996 */
997 xd = tb_xdomain_find_by_uuid(tb, &pkg->remote_uuid);
998 if (xd) {
999 remove_xdomain(xd);
1000 tb_xdomain_put(xd);
1001 }
1002 }
1003
1004 static int icm_tr_cio_reset(struct tb *tb)
1005 {
1006 return pcie2cio_write(tb_priv(tb), TB_CFG_SWITCH, 0, 0x777, BIT(1));
1007 }
1008
1009 static int
1010 icm_tr_driver_ready(struct tb *tb, enum tb_security_level *security_level,
1011 u8 *proto_version, size_t *nboot_acl, bool *rpm)
1012 {
1013 struct icm_tr_pkg_driver_ready_response reply;
1014 struct icm_pkg_driver_ready request = {
1015 .hdr.code = ICM_DRIVER_READY,
1016 };
1017 int ret;
1018
1019 memset(&reply, 0, sizeof(reply));
1020 ret = icm_request(tb, &request, sizeof(request), &reply, sizeof(reply),
1021 1, 20000);
1022 if (ret)
1023 return ret;
1024
1025 if (security_level)
1026 *security_level = reply.info & ICM_TR_INFO_SLEVEL_MASK;
1027 if (proto_version)
1028 *proto_version = (reply.info & ICM_TR_INFO_PROTO_VERSION_MASK) >>
1029 ICM_TR_INFO_PROTO_VERSION_SHIFT;
1030 if (nboot_acl)
1031 *nboot_acl = (reply.info & ICM_TR_INFO_BOOT_ACL_MASK) >>
1032 ICM_TR_INFO_BOOT_ACL_SHIFT;
1033 if (rpm)
1034 *rpm = !!(reply.hdr.flags & ICM_TR_FLAGS_RTD3);
1035
1036 return 0;
1037 }
1038
1039 static int icm_tr_approve_switch(struct tb *tb, struct tb_switch *sw)
1040 {
1041 struct icm_tr_pkg_approve_device request;
1042 struct icm_tr_pkg_approve_device reply;
1043 int ret;
1044
1045 memset(&request, 0, sizeof(request));
1046 memcpy(&request.ep_uuid, sw->uuid, sizeof(request.ep_uuid));
1047 request.hdr.code = ICM_APPROVE_DEVICE;
1048 request.route_lo = sw->config.route_lo;
1049 request.route_hi = sw->config.route_hi;
1050 request.connection_id = sw->connection_id;
1051
1052 memset(&reply, 0, sizeof(reply));
1053 ret = icm_request(tb, &request, sizeof(request), &reply, sizeof(reply),
1054 1, ICM_APPROVE_TIMEOUT);
1055 if (ret)
1056 return ret;
1057
1058 if (reply.hdr.flags & ICM_FLAGS_ERROR) {
1059 tb_warn(tb, "PCIe tunnel creation failed\n");
1060 return -EIO;
1061 }
1062
1063 return 0;
1064 }
1065
1066 static int icm_tr_add_switch_key(struct tb *tb, struct tb_switch *sw)
1067 {
1068 struct icm_tr_pkg_add_device_key_response reply;
1069 struct icm_tr_pkg_add_device_key request;
1070 int ret;
1071
1072 memset(&request, 0, sizeof(request));
1073 memcpy(&request.ep_uuid, sw->uuid, sizeof(request.ep_uuid));
1074 request.hdr.code = ICM_ADD_DEVICE_KEY;
1075 request.route_lo = sw->config.route_lo;
1076 request.route_hi = sw->config.route_hi;
1077 request.connection_id = sw->connection_id;
1078 memcpy(request.key, sw->key, TB_SWITCH_KEY_SIZE);
1079
1080 memset(&reply, 0, sizeof(reply));
1081 ret = icm_request(tb, &request, sizeof(request), &reply, sizeof(reply),
1082 1, ICM_TIMEOUT);
1083 if (ret)
1084 return ret;
1085
1086 if (reply.hdr.flags & ICM_FLAGS_ERROR) {
1087 tb_warn(tb, "Adding key to switch failed\n");
1088 return -EIO;
1089 }
1090
1091 return 0;
1092 }
1093
1094 static int icm_tr_challenge_switch_key(struct tb *tb, struct tb_switch *sw,
1095 const u8 *challenge, u8 *response)
1096 {
1097 struct icm_tr_pkg_challenge_device_response reply;
1098 struct icm_tr_pkg_challenge_device request;
1099 int ret;
1100
1101 memset(&request, 0, sizeof(request));
1102 memcpy(&request.ep_uuid, sw->uuid, sizeof(request.ep_uuid));
1103 request.hdr.code = ICM_CHALLENGE_DEVICE;
1104 request.route_lo = sw->config.route_lo;
1105 request.route_hi = sw->config.route_hi;
1106 request.connection_id = sw->connection_id;
1107 memcpy(request.challenge, challenge, TB_SWITCH_KEY_SIZE);
1108
1109 memset(&reply, 0, sizeof(reply));
1110 ret = icm_request(tb, &request, sizeof(request), &reply, sizeof(reply),
1111 1, ICM_TIMEOUT);
1112 if (ret)
1113 return ret;
1114
1115 if (reply.hdr.flags & ICM_FLAGS_ERROR)
1116 return -EKEYREJECTED;
1117 if (reply.hdr.flags & ICM_FLAGS_NO_KEY)
1118 return -ENOKEY;
1119
1120 memcpy(response, reply.response, TB_SWITCH_KEY_SIZE);
1121
1122 return 0;
1123 }
1124
1125 static int icm_tr_approve_xdomain_paths(struct tb *tb, struct tb_xdomain *xd)
1126 {
1127 struct icm_tr_pkg_approve_xdomain_response reply;
1128 struct icm_tr_pkg_approve_xdomain request;
1129 int ret;
1130
1131 memset(&request, 0, sizeof(request));
1132 request.hdr.code = ICM_APPROVE_XDOMAIN;
1133 request.route_hi = upper_32_bits(xd->route);
1134 request.route_lo = lower_32_bits(xd->route);
1135 request.transmit_path = xd->transmit_path;
1136 request.transmit_ring = xd->transmit_ring;
1137 request.receive_path = xd->receive_path;
1138 request.receive_ring = xd->receive_ring;
1139 memcpy(&request.remote_uuid, xd->remote_uuid, sizeof(*xd->remote_uuid));
1140
1141 memset(&reply, 0, sizeof(reply));
1142 ret = icm_request(tb, &request, sizeof(request), &reply, sizeof(reply),
1143 1, ICM_TIMEOUT);
1144 if (ret)
1145 return ret;
1146
1147 if (reply.hdr.flags & ICM_FLAGS_ERROR)
1148 return -EIO;
1149
1150 return 0;
1151 }
1152
1153 static int icm_tr_xdomain_tear_down(struct tb *tb, struct tb_xdomain *xd,
1154 int stage)
1155 {
1156 struct icm_tr_pkg_disconnect_xdomain_response reply;
1157 struct icm_tr_pkg_disconnect_xdomain request;
1158 int ret;
1159
1160 memset(&request, 0, sizeof(request));
1161 request.hdr.code = ICM_DISCONNECT_XDOMAIN;
1162 request.stage = stage;
1163 request.route_hi = upper_32_bits(xd->route);
1164 request.route_lo = lower_32_bits(xd->route);
1165 memcpy(&request.remote_uuid, xd->remote_uuid, sizeof(*xd->remote_uuid));
1166
1167 memset(&reply, 0, sizeof(reply));
1168 ret = icm_request(tb, &request, sizeof(request), &reply, sizeof(reply),
1169 1, ICM_TIMEOUT);
1170 if (ret)
1171 return ret;
1172
1173 if (reply.hdr.flags & ICM_FLAGS_ERROR)
1174 return -EIO;
1175
1176 return 0;
1177 }
1178
1179 static int icm_tr_disconnect_xdomain_paths(struct tb *tb, struct tb_xdomain *xd)
1180 {
1181 int ret;
1182
1183 ret = icm_tr_xdomain_tear_down(tb, xd, 1);
1184 if (ret)
1185 return ret;
1186
1187 usleep_range(10, 50);
1188 return icm_tr_xdomain_tear_down(tb, xd, 2);
1189 }
1190
1191 static void
1192 __icm_tr_device_connected(struct tb *tb, const struct icm_pkg_header *hdr,
1193 bool force_rtd3)
1194 {
1195 const struct icm_tr_event_device_connected *pkg =
1196 (const struct icm_tr_event_device_connected *)hdr;
1197 bool authorized, boot, dual_lane, speed_gen3;
1198 enum tb_security_level security_level;
1199 struct tb_switch *sw, *parent_sw;
1200 struct tb_xdomain *xd;
1201 u64 route;
1202
1203 icm_postpone_rescan(tb);
1204
1205 /*
1206 * Currently we don't use the QoS information coming with the
1207 * device connected message so simply just ignore that extra
1208 * packet for now.
1209 */
1210 if (pkg->hdr.packet_id)
1211 return;
1212
1213 route = get_route(pkg->route_hi, pkg->route_lo);
1214 authorized = pkg->link_info & ICM_LINK_INFO_APPROVED;
1215 security_level = (pkg->hdr.flags & ICM_FLAGS_SLEVEL_MASK) >>
1216 ICM_FLAGS_SLEVEL_SHIFT;
1217 boot = pkg->link_info & ICM_LINK_INFO_BOOT;
1218 dual_lane = pkg->hdr.flags & ICM_FLAGS_DUAL_LANE;
1219 speed_gen3 = pkg->hdr.flags & ICM_FLAGS_SPEED_GEN3;
1220
1221 if (pkg->link_info & ICM_LINK_INFO_REJECTED) {
1222 tb_info(tb, "switch at %llx was rejected by ICM firmware because topology limit exceeded\n",
1223 route);
1224 return;
1225 }
1226
1227 sw = tb_switch_find_by_uuid(tb, &pkg->ep_uuid);
1228 if (sw) {
1229 /* Update the switch if it is still in the same place */
1230 if (tb_route(sw) == route && !!sw->authorized == authorized) {
1231 parent_sw = tb_to_switch(sw->dev.parent);
1232 update_switch(parent_sw, sw, route, pkg->connection_id,
1233 0, 0, 0, boot);
1234 tb_switch_put(sw);
1235 return;
1236 }
1237
1238 remove_switch(sw);
1239 tb_switch_put(sw);
1240 }
1241
1242 /* Another switch with the same address */
1243 sw = tb_switch_find_by_route(tb, route);
1244 if (sw) {
1245 remove_switch(sw);
1246 tb_switch_put(sw);
1247 }
1248
1249 /* XDomain connection with the same address */
1250 xd = tb_xdomain_find_by_route(tb, route);
1251 if (xd) {
1252 remove_xdomain(xd);
1253 tb_xdomain_put(xd);
1254 }
1255
1256 parent_sw = tb_switch_find_by_route(tb, get_parent_route(route));
1257 if (!parent_sw) {
1258 tb_err(tb, "failed to find parent switch for %llx\n", route);
1259 return;
1260 }
1261
1262 pm_runtime_get_sync(&parent_sw->dev);
1263
1264 sw = alloc_switch(parent_sw, route, &pkg->ep_uuid);
1265 if (!IS_ERR(sw)) {
1266 sw->connection_id = pkg->connection_id;
1267 sw->authorized = authorized;
1268 sw->security_level = security_level;
1269 sw->boot = boot;
1270 sw->link_speed = speed_gen3 ? 20 : 10;
1271 sw->link_width = dual_lane ? 2 : 1;
1272 sw->rpm = force_rtd3;
1273 if (!sw->rpm)
1274 sw->rpm = intel_vss_is_rtd3(pkg->ep_name,
1275 sizeof(pkg->ep_name));
1276
1277 if (add_switch(parent_sw, sw))
1278 tb_switch_put(sw);
1279 }
1280
1281 pm_runtime_mark_last_busy(&parent_sw->dev);
1282 pm_runtime_put_autosuspend(&parent_sw->dev);
1283
1284 tb_switch_put(parent_sw);
1285 }
1286
1287 static void
1288 icm_tr_device_connected(struct tb *tb, const struct icm_pkg_header *hdr)
1289 {
1290 __icm_tr_device_connected(tb, hdr, false);
1291 }
1292
1293 static void
1294 icm_tr_device_disconnected(struct tb *tb, const struct icm_pkg_header *hdr)
1295 {
1296 const struct icm_tr_event_device_disconnected *pkg =
1297 (const struct icm_tr_event_device_disconnected *)hdr;
1298 struct tb_switch *sw;
1299 u64 route;
1300
1301 route = get_route(pkg->route_hi, pkg->route_lo);
1302
1303 sw = tb_switch_find_by_route(tb, route);
1304 if (!sw) {
1305 tb_warn(tb, "no switch exists at %llx, ignoring\n", route);
1306 return;
1307 }
1308 pm_runtime_get_sync(sw->dev.parent);
1309
1310 remove_switch(sw);
1311
1312 pm_runtime_mark_last_busy(sw->dev.parent);
1313 pm_runtime_put_autosuspend(sw->dev.parent);
1314
1315 tb_switch_put(sw);
1316 }
1317
1318 static void
1319 icm_tr_xdomain_connected(struct tb *tb, const struct icm_pkg_header *hdr)
1320 {
1321 const struct icm_tr_event_xdomain_connected *pkg =
1322 (const struct icm_tr_event_xdomain_connected *)hdr;
1323 struct tb_xdomain *xd;
1324 struct tb_switch *sw;
1325 u64 route;
1326
1327 if (!tb->root_switch)
1328 return;
1329
1330 route = get_route(pkg->local_route_hi, pkg->local_route_lo);
1331
1332 xd = tb_xdomain_find_by_uuid(tb, &pkg->remote_uuid);
1333 if (xd) {
1334 if (xd->route == route) {
1335 update_xdomain(xd, route, 0);
1336 tb_xdomain_put(xd);
1337 return;
1338 }
1339
1340 remove_xdomain(xd);
1341 tb_xdomain_put(xd);
1342 }
1343
1344 /* An existing xdomain with the same address */
1345 xd = tb_xdomain_find_by_route(tb, route);
1346 if (xd) {
1347 remove_xdomain(xd);
1348 tb_xdomain_put(xd);
1349 }
1350
1351 /*
1352 * If the user disconnected a switch during suspend and
1353 * connected another host to the same port, remove the switch
1354 * first.
1355 */
1356 sw = tb_switch_find_by_route(tb, route);
1357 if (sw) {
1358 remove_switch(sw);
1359 tb_switch_put(sw);
1360 }
1361
1362 sw = tb_switch_find_by_route(tb, get_parent_route(route));
1363 if (!sw) {
1364 tb_warn(tb, "no switch exists at %llx, ignoring\n", route);
1365 return;
1366 }
1367
1368 add_xdomain(sw, route, &pkg->local_uuid, &pkg->remote_uuid, 0, 0);
1369 tb_switch_put(sw);
1370 }
1371
1372 static void
1373 icm_tr_xdomain_disconnected(struct tb *tb, const struct icm_pkg_header *hdr)
1374 {
1375 const struct icm_tr_event_xdomain_disconnected *pkg =
1376 (const struct icm_tr_event_xdomain_disconnected *)hdr;
1377 struct tb_xdomain *xd;
1378 u64 route;
1379
1380 route = get_route(pkg->route_hi, pkg->route_lo);
1381
1382 xd = tb_xdomain_find_by_route(tb, route);
1383 if (xd) {
1384 remove_xdomain(xd);
1385 tb_xdomain_put(xd);
1386 }
1387 }
1388
1389 static struct pci_dev *get_upstream_port(struct pci_dev *pdev)
1390 {
1391 struct pci_dev *parent;
1392
1393 parent = pci_upstream_bridge(pdev);
1394 while (parent) {
1395 if (!pci_is_pcie(parent))
1396 return NULL;
1397 if (pci_pcie_type(parent) == PCI_EXP_TYPE_UPSTREAM)
1398 break;
1399 parent = pci_upstream_bridge(parent);
1400 }
1401
1402 if (!parent)
1403 return NULL;
1404
1405 switch (parent->device) {
1406 case PCI_DEVICE_ID_INTEL_ALPINE_RIDGE_2C_BRIDGE:
1407 case PCI_DEVICE_ID_INTEL_ALPINE_RIDGE_4C_BRIDGE:
1408 case PCI_DEVICE_ID_INTEL_ALPINE_RIDGE_LP_BRIDGE:
1409 case PCI_DEVICE_ID_INTEL_ALPINE_RIDGE_C_4C_BRIDGE:
1410 case PCI_DEVICE_ID_INTEL_ALPINE_RIDGE_C_2C_BRIDGE:
1411 case PCI_DEVICE_ID_INTEL_TITAN_RIDGE_2C_BRIDGE:
1412 case PCI_DEVICE_ID_INTEL_TITAN_RIDGE_4C_BRIDGE:
1413 return parent;
1414 }
1415
1416 return NULL;
1417 }
1418
1419 static bool icm_ar_is_supported(struct tb *tb)
1420 {
1421 struct pci_dev *upstream_port;
1422 struct icm *icm = tb_priv(tb);
1423
1424 /*
1425 * Starting from Alpine Ridge we can use ICM on Apple machines
1426 * as well. We just need to reset and re-enable it first.
1427 * However, only start it if explicitly asked by the user.
1428 */
1429 if (icm_firmware_running(tb->nhi))
1430 return true;
1431 if (!start_icm)
1432 return false;
1433
1434 /*
1435 * Find the upstream PCIe port in case we need to do reset
1436 * through its vendor specific registers.
1437 */
1438 upstream_port = get_upstream_port(tb->nhi->pdev);
1439 if (upstream_port) {
1440 int cap;
1441
1442 cap = pci_find_ext_capability(upstream_port,
1443 PCI_EXT_CAP_ID_VNDR);
1444 if (cap > 0) {
1445 icm->upstream_port = upstream_port;
1446 icm->vnd_cap = cap;
1447
1448 return true;
1449 }
1450 }
1451
1452 return false;
1453 }
1454
1455 static int icm_ar_cio_reset(struct tb *tb)
1456 {
1457 return pcie2cio_write(tb_priv(tb), TB_CFG_SWITCH, 0, 0x50, BIT(9));
1458 }
1459
1460 static int icm_ar_get_mode(struct tb *tb)
1461 {
1462 struct tb_nhi *nhi = tb->nhi;
1463 int retries = 60;
1464 u32 val;
1465
1466 do {
1467 val = ioread32(nhi->iobase + REG_FW_STS);
1468 if (val & REG_FW_STS_NVM_AUTH_DONE)
1469 break;
1470 msleep(50);
1471 } while (--retries);
1472
1473 if (!retries) {
1474 dev_err(&nhi->pdev->dev, "ICM firmware not authenticated\n");
1475 return -ENODEV;
1476 }
1477
1478 return nhi_mailbox_mode(nhi);
1479 }
1480
1481 static int
1482 icm_ar_driver_ready(struct tb *tb, enum tb_security_level *security_level,
1483 u8 *proto_version, size_t *nboot_acl, bool *rpm)
1484 {
1485 struct icm_ar_pkg_driver_ready_response reply;
1486 struct icm_pkg_driver_ready request = {
1487 .hdr.code = ICM_DRIVER_READY,
1488 };
1489 int ret;
1490
1491 memset(&reply, 0, sizeof(reply));
1492 ret = icm_request(tb, &request, sizeof(request), &reply, sizeof(reply),
1493 1, ICM_TIMEOUT);
1494 if (ret)
1495 return ret;
1496
1497 if (security_level)
1498 *security_level = reply.info & ICM_AR_INFO_SLEVEL_MASK;
1499 if (nboot_acl && (reply.info & ICM_AR_INFO_BOOT_ACL_SUPPORTED))
1500 *nboot_acl = (reply.info & ICM_AR_INFO_BOOT_ACL_MASK) >>
1501 ICM_AR_INFO_BOOT_ACL_SHIFT;
1502 if (rpm)
1503 *rpm = !!(reply.hdr.flags & ICM_AR_FLAGS_RTD3);
1504
1505 return 0;
1506 }
1507
1508 static int icm_ar_get_route(struct tb *tb, u8 link, u8 depth, u64 *route)
1509 {
1510 struct icm_ar_pkg_get_route_response reply;
1511 struct icm_ar_pkg_get_route request = {
1512 .hdr = { .code = ICM_GET_ROUTE },
1513 .link_info = depth << ICM_LINK_INFO_DEPTH_SHIFT | link,
1514 };
1515 int ret;
1516
1517 memset(&reply, 0, sizeof(reply));
1518 ret = icm_request(tb, &request, sizeof(request), &reply, sizeof(reply),
1519 1, ICM_TIMEOUT);
1520 if (ret)
1521 return ret;
1522
1523 if (reply.hdr.flags & ICM_FLAGS_ERROR)
1524 return -EIO;
1525
1526 *route = get_route(reply.route_hi, reply.route_lo);
1527 return 0;
1528 }
1529
1530 static int icm_ar_get_boot_acl(struct tb *tb, uuid_t *uuids, size_t nuuids)
1531 {
1532 struct icm_ar_pkg_preboot_acl_response reply;
1533 struct icm_ar_pkg_preboot_acl request = {
1534 .hdr = { .code = ICM_PREBOOT_ACL },
1535 };
1536 int ret, i;
1537
1538 memset(&reply, 0, sizeof(reply));
1539 ret = icm_request(tb, &request, sizeof(request), &reply, sizeof(reply),
1540 1, ICM_TIMEOUT);
1541 if (ret)
1542 return ret;
1543
1544 if (reply.hdr.flags & ICM_FLAGS_ERROR)
1545 return -EIO;
1546
1547 for (i = 0; i < nuuids; i++) {
1548 u32 *uuid = (u32 *)&uuids[i];
1549
1550 uuid[0] = reply.acl[i].uuid_lo;
1551 uuid[1] = reply.acl[i].uuid_hi;
1552
1553 if (uuid[0] == 0xffffffff && uuid[1] == 0xffffffff) {
1554 /* Map empty entries to null UUID */
1555 uuid[0] = 0;
1556 uuid[1] = 0;
1557 } else if (uuid[0] != 0 || uuid[1] != 0) {
1558 /* Upper two DWs are always one's */
1559 uuid[2] = 0xffffffff;
1560 uuid[3] = 0xffffffff;
1561 }
1562 }
1563
1564 return ret;
1565 }
1566
1567 static int icm_ar_set_boot_acl(struct tb *tb, const uuid_t *uuids,
1568 size_t nuuids)
1569 {
1570 struct icm_ar_pkg_preboot_acl_response reply;
1571 struct icm_ar_pkg_preboot_acl request = {
1572 .hdr = {
1573 .code = ICM_PREBOOT_ACL,
1574 .flags = ICM_FLAGS_WRITE,
1575 },
1576 };
1577 int ret, i;
1578
1579 for (i = 0; i < nuuids; i++) {
1580 const u32 *uuid = (const u32 *)&uuids[i];
1581
1582 if (uuid_is_null(&uuids[i])) {
1583 /*
1584 * Map null UUID to the empty (all one) entries
1585 * for ICM.
1586 */
1587 request.acl[i].uuid_lo = 0xffffffff;
1588 request.acl[i].uuid_hi = 0xffffffff;
1589 } else {
1590 /* Two high DWs need to be set to all one */
1591 if (uuid[2] != 0xffffffff || uuid[3] != 0xffffffff)
1592 return -EINVAL;
1593
1594 request.acl[i].uuid_lo = uuid[0];
1595 request.acl[i].uuid_hi = uuid[1];
1596 }
1597 }
1598
1599 memset(&reply, 0, sizeof(reply));
1600 ret = icm_request(tb, &request, sizeof(request), &reply, sizeof(reply),
1601 1, ICM_TIMEOUT);
1602 if (ret)
1603 return ret;
1604
1605 if (reply.hdr.flags & ICM_FLAGS_ERROR)
1606 return -EIO;
1607
1608 return 0;
1609 }
1610
1611 static int
1612 icm_icl_driver_ready(struct tb *tb, enum tb_security_level *security_level,
1613 u8 *proto_version, size_t *nboot_acl, bool *rpm)
1614 {
1615 struct icm_tr_pkg_driver_ready_response reply;
1616 struct icm_pkg_driver_ready request = {
1617 .hdr.code = ICM_DRIVER_READY,
1618 };
1619 int ret;
1620
1621 memset(&reply, 0, sizeof(reply));
1622 ret = icm_request(tb, &request, sizeof(request), &reply, sizeof(reply),
1623 1, 20000);
1624 if (ret)
1625 return ret;
1626
1627 if (proto_version)
1628 *proto_version = (reply.info & ICM_TR_INFO_PROTO_VERSION_MASK) >>
1629 ICM_TR_INFO_PROTO_VERSION_SHIFT;
1630
1631 /* Ice Lake always supports RTD3 */
1632 if (rpm)
1633 *rpm = true;
1634
1635 return 0;
1636 }
1637
1638 static void icm_icl_set_uuid(struct tb *tb)
1639 {
1640 struct tb_nhi *nhi = tb->nhi;
1641 u32 uuid[4];
1642
1643 pci_read_config_dword(nhi->pdev, VS_CAP_10, &uuid[0]);
1644 pci_read_config_dword(nhi->pdev, VS_CAP_11, &uuid[1]);
1645 uuid[2] = 0xffffffff;
1646 uuid[3] = 0xffffffff;
1647
1648 tb->root_switch->uuid = kmemdup(uuid, sizeof(uuid), GFP_KERNEL);
1649 }
1650
1651 static void
1652 icm_icl_device_connected(struct tb *tb, const struct icm_pkg_header *hdr)
1653 {
1654 __icm_tr_device_connected(tb, hdr, true);
1655 }
1656
1657 static void icm_icl_rtd3_veto(struct tb *tb, const struct icm_pkg_header *hdr)
1658 {
1659 const struct icm_icl_event_rtd3_veto *pkg =
1660 (const struct icm_icl_event_rtd3_veto *)hdr;
1661
1662 tb_dbg(tb, "ICM rtd3 veto=0x%08x\n", pkg->veto_reason);
1663
1664 if (pkg->veto_reason)
1665 icm_veto_begin(tb);
1666 else
1667 icm_veto_end(tb);
1668 }
1669
1670 static bool icm_tgl_is_supported(struct tb *tb)
1671 {
1672 u32 val;
1673
1674 /*
1675 * If the firmware is not running use software CM. This platform
1676 * should fully support both.
1677 */
1678 val = ioread32(tb->nhi->iobase + REG_FW_STS);
1679 return !!(val & REG_FW_STS_NVM_AUTH_DONE);
1680 }
1681
1682 static void icm_handle_notification(struct work_struct *work)
1683 {
1684 struct icm_notification *n = container_of(work, typeof(*n), work);
1685 struct tb *tb = n->tb;
1686 struct icm *icm = tb_priv(tb);
1687
1688 mutex_lock(&tb->lock);
1689
1690 /*
1691 * When the domain is stopped we flush its workqueue but before
1692 * that the root switch is removed. In that case we should treat
1693 * the queued events as being canceled.
1694 */
1695 if (tb->root_switch) {
1696 switch (n->pkg->code) {
1697 case ICM_EVENT_DEVICE_CONNECTED:
1698 icm->device_connected(tb, n->pkg);
1699 break;
1700 case ICM_EVENT_DEVICE_DISCONNECTED:
1701 icm->device_disconnected(tb, n->pkg);
1702 break;
1703 case ICM_EVENT_XDOMAIN_CONNECTED:
1704 icm->xdomain_connected(tb, n->pkg);
1705 break;
1706 case ICM_EVENT_XDOMAIN_DISCONNECTED:
1707 icm->xdomain_disconnected(tb, n->pkg);
1708 break;
1709 case ICM_EVENT_RTD3_VETO:
1710 icm->rtd3_veto(tb, n->pkg);
1711 break;
1712 }
1713 }
1714
1715 mutex_unlock(&tb->lock);
1716
1717 kfree(n->pkg);
1718 kfree(n);
1719 }
1720
1721 static void icm_handle_event(struct tb *tb, enum tb_cfg_pkg_type type,
1722 const void *buf, size_t size)
1723 {
1724 struct icm_notification *n;
1725
1726 n = kmalloc(sizeof(*n), GFP_KERNEL);
1727 if (!n)
1728 return;
1729
1730 INIT_WORK(&n->work, icm_handle_notification);
1731 n->pkg = kmemdup(buf, size, GFP_KERNEL);
1732 n->tb = tb;
1733
1734 queue_work(tb->wq, &n->work);
1735 }
1736
1737 static int
1738 __icm_driver_ready(struct tb *tb, enum tb_security_level *security_level,
1739 u8 *proto_version, size_t *nboot_acl, bool *rpm)
1740 {
1741 struct icm *icm = tb_priv(tb);
1742 unsigned int retries = 50;
1743 int ret;
1744
1745 ret = icm->driver_ready(tb, security_level, proto_version, nboot_acl,
1746 rpm);
1747 if (ret) {
1748 tb_err(tb, "failed to send driver ready to ICM\n");
1749 return ret;
1750 }
1751
1752 /*
1753 * Hold on here until the switch config space is accessible so
1754 * that we can read root switch config successfully.
1755 */
1756 do {
1757 struct tb_cfg_result res;
1758 u32 tmp;
1759
1760 res = tb_cfg_read_raw(tb->ctl, &tmp, 0, 0, TB_CFG_SWITCH,
1761 0, 1, 100);
1762 if (!res.err)
1763 return 0;
1764
1765 msleep(50);
1766 } while (--retries);
1767
1768 tb_err(tb, "failed to read root switch config space, giving up\n");
1769 return -ETIMEDOUT;
1770 }
1771
1772 static int icm_firmware_reset(struct tb *tb, struct tb_nhi *nhi)
1773 {
1774 struct icm *icm = tb_priv(tb);
1775 u32 val;
1776
1777 if (!icm->upstream_port)
1778 return -ENODEV;
1779
1780 /* Put ARC to wait for CIO reset event to happen */
1781 val = ioread32(nhi->iobase + REG_FW_STS);
1782 val |= REG_FW_STS_CIO_RESET_REQ;
1783 iowrite32(val, nhi->iobase + REG_FW_STS);
1784
1785 /* Re-start ARC */
1786 val = ioread32(nhi->iobase + REG_FW_STS);
1787 val |= REG_FW_STS_ICM_EN_INVERT;
1788 val |= REG_FW_STS_ICM_EN_CPU;
1789 iowrite32(val, nhi->iobase + REG_FW_STS);
1790
1791 /* Trigger CIO reset now */
1792 return icm->cio_reset(tb);
1793 }
1794
1795 static int icm_firmware_start(struct tb *tb, struct tb_nhi *nhi)
1796 {
1797 unsigned int retries = 10;
1798 int ret;
1799 u32 val;
1800
1801 /* Check if the ICM firmware is already running */
1802 if (icm_firmware_running(nhi))
1803 return 0;
1804
1805 dev_dbg(&nhi->pdev->dev, "starting ICM firmware\n");
1806
1807 ret = icm_firmware_reset(tb, nhi);
1808 if (ret)
1809 return ret;
1810
1811 /* Wait until the ICM firmware tells us it is up and running */
1812 do {
1813 /* Check that the ICM firmware is running */
1814 val = ioread32(nhi->iobase + REG_FW_STS);
1815 if (val & REG_FW_STS_NVM_AUTH_DONE)
1816 return 0;
1817
1818 msleep(300);
1819 } while (--retries);
1820
1821 return -ETIMEDOUT;
1822 }
1823
1824 static int icm_reset_phy_port(struct tb *tb, int phy_port)
1825 {
1826 struct icm *icm = tb_priv(tb);
1827 u32 state0, state1;
1828 int port0, port1;
1829 u32 val0, val1;
1830 int ret;
1831
1832 if (!icm->upstream_port)
1833 return 0;
1834
1835 if (phy_port) {
1836 port0 = 3;
1837 port1 = 4;
1838 } else {
1839 port0 = 1;
1840 port1 = 2;
1841 }
1842
1843 /*
1844 * Read link status of both null ports belonging to a single
1845 * physical port.
1846 */
1847 ret = pcie2cio_read(icm, TB_CFG_PORT, port0, PHY_PORT_CS1, &val0);
1848 if (ret)
1849 return ret;
1850 ret = pcie2cio_read(icm, TB_CFG_PORT, port1, PHY_PORT_CS1, &val1);
1851 if (ret)
1852 return ret;
1853
1854 state0 = val0 & PHY_PORT_CS1_LINK_STATE_MASK;
1855 state0 >>= PHY_PORT_CS1_LINK_STATE_SHIFT;
1856 state1 = val1 & PHY_PORT_CS1_LINK_STATE_MASK;
1857 state1 >>= PHY_PORT_CS1_LINK_STATE_SHIFT;
1858
1859 /* If they are both up we need to reset them now */
1860 if (state0 != TB_PORT_UP || state1 != TB_PORT_UP)
1861 return 0;
1862
1863 val0 |= PHY_PORT_CS1_LINK_DISABLE;
1864 ret = pcie2cio_write(icm, TB_CFG_PORT, port0, PHY_PORT_CS1, val0);
1865 if (ret)
1866 return ret;
1867
1868 val1 |= PHY_PORT_CS1_LINK_DISABLE;
1869 ret = pcie2cio_write(icm, TB_CFG_PORT, port1, PHY_PORT_CS1, val1);
1870 if (ret)
1871 return ret;
1872
1873 /* Wait a bit and then re-enable both ports */
1874 usleep_range(10, 100);
1875
1876 ret = pcie2cio_read(icm, TB_CFG_PORT, port0, PHY_PORT_CS1, &val0);
1877 if (ret)
1878 return ret;
1879 ret = pcie2cio_read(icm, TB_CFG_PORT, port1, PHY_PORT_CS1, &val1);
1880 if (ret)
1881 return ret;
1882
1883 val0 &= ~PHY_PORT_CS1_LINK_DISABLE;
1884 ret = pcie2cio_write(icm, TB_CFG_PORT, port0, PHY_PORT_CS1, val0);
1885 if (ret)
1886 return ret;
1887
1888 val1 &= ~PHY_PORT_CS1_LINK_DISABLE;
1889 return pcie2cio_write(icm, TB_CFG_PORT, port1, PHY_PORT_CS1, val1);
1890 }
1891
1892 static int icm_firmware_init(struct tb *tb)
1893 {
1894 struct icm *icm = tb_priv(tb);
1895 struct tb_nhi *nhi = tb->nhi;
1896 int ret;
1897
1898 ret = icm_firmware_start(tb, nhi);
1899 if (ret) {
1900 dev_err(&nhi->pdev->dev, "could not start ICM firmware\n");
1901 return ret;
1902 }
1903
1904 if (icm->get_mode) {
1905 ret = icm->get_mode(tb);
1906
1907 switch (ret) {
1908 case NHI_FW_SAFE_MODE:
1909 icm->safe_mode = true;
1910 break;
1911
1912 case NHI_FW_CM_MODE:
1913 /* Ask ICM to accept all Thunderbolt devices */
1914 nhi_mailbox_cmd(nhi, NHI_MAILBOX_ALLOW_ALL_DEVS, 0);
1915 break;
1916
1917 default:
1918 if (ret < 0)
1919 return ret;
1920
1921 tb_err(tb, "ICM firmware is in wrong mode: %u\n", ret);
1922 return -ENODEV;
1923 }
1924 }
1925
1926 /*
1927 * Reset both physical ports if there is anything connected to
1928 * them already.
1929 */
1930 ret = icm_reset_phy_port(tb, 0);
1931 if (ret)
1932 dev_warn(&nhi->pdev->dev, "failed to reset links on port0\n");
1933 ret = icm_reset_phy_port(tb, 1);
1934 if (ret)
1935 dev_warn(&nhi->pdev->dev, "failed to reset links on port1\n");
1936
1937 return 0;
1938 }
1939
1940 static int icm_driver_ready(struct tb *tb)
1941 {
1942 struct icm *icm = tb_priv(tb);
1943 int ret;
1944
1945 ret = icm_firmware_init(tb);
1946 if (ret)
1947 return ret;
1948
1949 if (icm->safe_mode) {
1950 tb_info(tb, "Thunderbolt host controller is in safe mode.\n");
1951 tb_info(tb, "You need to update NVM firmware of the controller before it can be used.\n");
1952 tb_info(tb, "For latest updates check https://thunderbolttechnology.net/updates.\n");
1953 return 0;
1954 }
1955
1956 ret = __icm_driver_ready(tb, &tb->security_level, &icm->proto_version,
1957 &tb->nboot_acl, &icm->rpm);
1958 if (ret)
1959 return ret;
1960
1961 /*
1962 * Make sure the number of supported preboot ACL matches what we
1963 * expect or disable the whole feature.
1964 */
1965 if (tb->nboot_acl > icm->max_boot_acl)
1966 tb->nboot_acl = 0;
1967
1968 if (icm->proto_version >= 3)
1969 tb_dbg(tb, "USB4 proxy operations supported\n");
1970
1971 return 0;
1972 }
1973
1974 static int icm_suspend(struct tb *tb)
1975 {
1976 struct icm *icm = tb_priv(tb);
1977
1978 if (icm->save_devices)
1979 icm->save_devices(tb);
1980
1981 nhi_mailbox_cmd(tb->nhi, NHI_MAILBOX_DRV_UNLOADS, 0);
1982 return 0;
1983 }
1984
1985 /*
1986 * Mark all switches (except root switch) below this one unplugged. ICM
1987 * firmware will send us an updated list of switches after we have send
1988 * it driver ready command. If a switch is not in that list it will be
1989 * removed when we perform rescan.
1990 */
1991 static void icm_unplug_children(struct tb_switch *sw)
1992 {
1993 struct tb_port *port;
1994
1995 if (tb_route(sw))
1996 sw->is_unplugged = true;
1997
1998 tb_switch_for_each_port(sw, port) {
1999 if (port->xdomain)
2000 port->xdomain->is_unplugged = true;
2001 else if (tb_port_has_remote(port))
2002 icm_unplug_children(port->remote->sw);
2003 }
2004 }
2005
2006 static int complete_rpm(struct device *dev, void *data)
2007 {
2008 struct tb_switch *sw = tb_to_switch(dev);
2009
2010 if (sw)
2011 complete(&sw->rpm_complete);
2012 return 0;
2013 }
2014
2015 static void remove_unplugged_switch(struct tb_switch *sw)
2016 {
2017 struct device *parent = get_device(sw->dev.parent);
2018
2019 pm_runtime_get_sync(parent);
2020
2021 /*
2022 * Signal this and switches below for rpm_complete because
2023 * tb_switch_remove() calls pm_runtime_get_sync() that then waits
2024 * for it.
2025 */
2026 complete_rpm(&sw->dev, NULL);
2027 bus_for_each_dev(&tb_bus_type, &sw->dev, NULL, complete_rpm);
2028 tb_switch_remove(sw);
2029
2030 pm_runtime_mark_last_busy(parent);
2031 pm_runtime_put_autosuspend(parent);
2032
2033 put_device(parent);
2034 }
2035
2036 static void icm_free_unplugged_children(struct tb_switch *sw)
2037 {
2038 struct tb_port *port;
2039
2040 tb_switch_for_each_port(sw, port) {
2041 if (port->xdomain && port->xdomain->is_unplugged) {
2042 tb_xdomain_remove(port->xdomain);
2043 port->xdomain = NULL;
2044 } else if (tb_port_has_remote(port)) {
2045 if (port->remote->sw->is_unplugged) {
2046 remove_unplugged_switch(port->remote->sw);
2047 port->remote = NULL;
2048 } else {
2049 icm_free_unplugged_children(port->remote->sw);
2050 }
2051 }
2052 }
2053 }
2054
2055 static void icm_rescan_work(struct work_struct *work)
2056 {
2057 struct icm *icm = container_of(work, struct icm, rescan_work.work);
2058 struct tb *tb = icm_to_tb(icm);
2059
2060 mutex_lock(&tb->lock);
2061 if (tb->root_switch)
2062 icm_free_unplugged_children(tb->root_switch);
2063 mutex_unlock(&tb->lock);
2064 }
2065
2066 static void icm_complete(struct tb *tb)
2067 {
2068 struct icm *icm = tb_priv(tb);
2069
2070 if (tb->nhi->going_away)
2071 return;
2072
2073 /*
2074 * If RTD3 was vetoed before we entered system suspend allow it
2075 * again now before driver ready is sent. Firmware sends a new RTD3
2076 * veto if it is still the case after we have sent it driver ready
2077 * command.
2078 */
2079 icm_veto_end(tb);
2080 icm_unplug_children(tb->root_switch);
2081
2082 /*
2083 * Now all existing children should be resumed, start events
2084 * from ICM to get updated status.
2085 */
2086 __icm_driver_ready(tb, NULL, NULL, NULL, NULL);
2087
2088 /*
2089 * We do not get notifications of devices that have been
2090 * unplugged during suspend so schedule rescan to clean them up
2091 * if any.
2092 */
2093 queue_delayed_work(tb->wq, &icm->rescan_work, msecs_to_jiffies(500));
2094 }
2095
2096 static int icm_runtime_suspend(struct tb *tb)
2097 {
2098 nhi_mailbox_cmd(tb->nhi, NHI_MAILBOX_DRV_UNLOADS, 0);
2099 return 0;
2100 }
2101
2102 static int icm_runtime_suspend_switch(struct tb_switch *sw)
2103 {
2104 if (tb_route(sw))
2105 reinit_completion(&sw->rpm_complete);
2106 return 0;
2107 }
2108
2109 static int icm_runtime_resume_switch(struct tb_switch *sw)
2110 {
2111 if (tb_route(sw)) {
2112 if (!wait_for_completion_timeout(&sw->rpm_complete,
2113 msecs_to_jiffies(500))) {
2114 dev_dbg(&sw->dev, "runtime resuming timed out\n");
2115 }
2116 }
2117 return 0;
2118 }
2119
2120 static int icm_runtime_resume(struct tb *tb)
2121 {
2122 /*
2123 * We can reuse the same resume functionality than with system
2124 * suspend.
2125 */
2126 icm_complete(tb);
2127 return 0;
2128 }
2129
2130 static int icm_start(struct tb *tb)
2131 {
2132 struct icm *icm = tb_priv(tb);
2133 int ret;
2134
2135 if (icm->safe_mode)
2136 tb->root_switch = tb_switch_alloc_safe_mode(tb, &tb->dev, 0);
2137 else
2138 tb->root_switch = tb_switch_alloc(tb, &tb->dev, 0);
2139 if (IS_ERR(tb->root_switch))
2140 return PTR_ERR(tb->root_switch);
2141
2142 tb->root_switch->no_nvm_upgrade = !icm->can_upgrade_nvm;
2143 tb->root_switch->rpm = icm->rpm;
2144
2145 if (icm->set_uuid)
2146 icm->set_uuid(tb);
2147
2148 ret = tb_switch_add(tb->root_switch);
2149 if (ret) {
2150 tb_switch_put(tb->root_switch);
2151 tb->root_switch = NULL;
2152 }
2153
2154 return ret;
2155 }
2156
2157 static void icm_stop(struct tb *tb)
2158 {
2159 struct icm *icm = tb_priv(tb);
2160
2161 cancel_delayed_work(&icm->rescan_work);
2162 tb_switch_remove(tb->root_switch);
2163 tb->root_switch = NULL;
2164 nhi_mailbox_cmd(tb->nhi, NHI_MAILBOX_DRV_UNLOADS, 0);
2165 kfree(icm->last_nvm_auth);
2166 icm->last_nvm_auth = NULL;
2167 }
2168
2169 static int icm_disconnect_pcie_paths(struct tb *tb)
2170 {
2171 return nhi_mailbox_cmd(tb->nhi, NHI_MAILBOX_DISCONNECT_PCIE_PATHS, 0);
2172 }
2173
2174 static void icm_usb4_switch_nvm_auth_complete(void *data)
2175 {
2176 struct usb4_switch_nvm_auth *auth = data;
2177 struct icm *icm = auth->icm;
2178 struct tb *tb = icm_to_tb(icm);
2179
2180 tb_dbg(tb, "NVM_AUTH response for %llx flags %#x status %#x\n",
2181 get_route(auth->reply.route_hi, auth->reply.route_lo),
2182 auth->reply.hdr.flags, auth->reply.status);
2183
2184 mutex_lock(&tb->lock);
2185 if (WARN_ON(icm->last_nvm_auth))
2186 kfree(icm->last_nvm_auth);
2187 icm->last_nvm_auth = auth;
2188 mutex_unlock(&tb->lock);
2189 }
2190
2191 static int icm_usb4_switch_nvm_authenticate(struct tb *tb, u64 route)
2192 {
2193 struct usb4_switch_nvm_auth *auth;
2194 struct icm *icm = tb_priv(tb);
2195 struct tb_cfg_request *req;
2196 int ret;
2197
2198 auth = kzalloc(sizeof(*auth), GFP_KERNEL);
2199 if (!auth)
2200 return -ENOMEM;
2201
2202 auth->icm = icm;
2203 auth->request.hdr.code = ICM_USB4_SWITCH_OP;
2204 auth->request.route_hi = upper_32_bits(route);
2205 auth->request.route_lo = lower_32_bits(route);
2206 auth->request.opcode = USB4_SWITCH_OP_NVM_AUTH;
2207
2208 req = tb_cfg_request_alloc();
2209 if (!req) {
2210 ret = -ENOMEM;
2211 goto err_free_auth;
2212 }
2213
2214 req->match = icm_match;
2215 req->copy = icm_copy;
2216 req->request = &auth->request;
2217 req->request_size = sizeof(auth->request);
2218 req->request_type = TB_CFG_PKG_ICM_CMD;
2219 req->response = &auth->reply;
2220 req->npackets = 1;
2221 req->response_size = sizeof(auth->reply);
2222 req->response_type = TB_CFG_PKG_ICM_RESP;
2223
2224 tb_dbg(tb, "NVM_AUTH request for %llx\n", route);
2225
2226 mutex_lock(&icm->request_lock);
2227 ret = tb_cfg_request(tb->ctl, req, icm_usb4_switch_nvm_auth_complete,
2228 auth);
2229 mutex_unlock(&icm->request_lock);
2230
2231 tb_cfg_request_put(req);
2232 if (ret)
2233 goto err_free_auth;
2234 return 0;
2235
2236 err_free_auth:
2237 kfree(auth);
2238 return ret;
2239 }
2240
2241 static int icm_usb4_switch_op(struct tb_switch *sw, u16 opcode, u32 *metadata,
2242 u8 *status, const void *tx_data, size_t tx_data_len,
2243 void *rx_data, size_t rx_data_len)
2244 {
2245 struct icm_usb4_switch_op_response reply;
2246 struct icm_usb4_switch_op request;
2247 struct tb *tb = sw->tb;
2248 struct icm *icm = tb_priv(tb);
2249 u64 route = tb_route(sw);
2250 int ret;
2251
2252 /*
2253 * USB4 router operation proxy is supported in firmware if the
2254 * protocol version is 3 or higher.
2255 */
2256 if (icm->proto_version < 3)
2257 return -EOPNOTSUPP;
2258
2259 /*
2260 * NVM_AUTH is a special USB4 proxy operation that does not
2261 * return immediately so handle it separately.
2262 */
2263 if (opcode == USB4_SWITCH_OP_NVM_AUTH)
2264 return icm_usb4_switch_nvm_authenticate(tb, route);
2265
2266 memset(&request, 0, sizeof(request));
2267 request.hdr.code = ICM_USB4_SWITCH_OP;
2268 request.route_hi = upper_32_bits(route);
2269 request.route_lo = lower_32_bits(route);
2270 request.opcode = opcode;
2271 if (metadata)
2272 request.metadata = *metadata;
2273
2274 if (tx_data_len) {
2275 request.data_len_valid |= ICM_USB4_SWITCH_DATA_VALID;
2276 if (tx_data_len < ARRAY_SIZE(request.data))
2277 request.data_len_valid =
2278 tx_data_len & ICM_USB4_SWITCH_DATA_LEN_MASK;
2279 memcpy(request.data, tx_data, tx_data_len * sizeof(u32));
2280 }
2281
2282 memset(&reply, 0, sizeof(reply));
2283 ret = icm_request(tb, &request, sizeof(request), &reply, sizeof(reply),
2284 1, ICM_TIMEOUT);
2285 if (ret)
2286 return ret;
2287
2288 if (reply.hdr.flags & ICM_FLAGS_ERROR)
2289 return -EIO;
2290
2291 if (status)
2292 *status = reply.status;
2293
2294 if (metadata)
2295 *metadata = reply.metadata;
2296
2297 if (rx_data_len)
2298 memcpy(rx_data, reply.data, rx_data_len * sizeof(u32));
2299
2300 return 0;
2301 }
2302
2303 static int icm_usb4_switch_nvm_authenticate_status(struct tb_switch *sw,
2304 u32 *status)
2305 {
2306 struct usb4_switch_nvm_auth *auth;
2307 struct tb *tb = sw->tb;
2308 struct icm *icm = tb_priv(tb);
2309 int ret = 0;
2310
2311 if (icm->proto_version < 3)
2312 return -EOPNOTSUPP;
2313
2314 auth = icm->last_nvm_auth;
2315 icm->last_nvm_auth = NULL;
2316
2317 if (auth && auth->reply.route_hi == sw->config.route_hi &&
2318 auth->reply.route_lo == sw->config.route_lo) {
2319 tb_dbg(tb, "NVM_AUTH found for %llx flags 0x%#x status %#x\n",
2320 tb_route(sw), auth->reply.hdr.flags, auth->reply.status);
2321 if (auth->reply.hdr.flags & ICM_FLAGS_ERROR)
2322 ret = -EIO;
2323 else
2324 *status = auth->reply.status;
2325 } else {
2326 *status = 0;
2327 }
2328
2329 kfree(auth);
2330 return ret;
2331 }
2332
2333 /* Falcon Ridge */
2334 static const struct tb_cm_ops icm_fr_ops = {
2335 .driver_ready = icm_driver_ready,
2336 .start = icm_start,
2337 .stop = icm_stop,
2338 .suspend = icm_suspend,
2339 .complete = icm_complete,
2340 .handle_event = icm_handle_event,
2341 .approve_switch = icm_fr_approve_switch,
2342 .add_switch_key = icm_fr_add_switch_key,
2343 .challenge_switch_key = icm_fr_challenge_switch_key,
2344 .disconnect_pcie_paths = icm_disconnect_pcie_paths,
2345 .approve_xdomain_paths = icm_fr_approve_xdomain_paths,
2346 .disconnect_xdomain_paths = icm_fr_disconnect_xdomain_paths,
2347 };
2348
2349 /* Alpine Ridge */
2350 static const struct tb_cm_ops icm_ar_ops = {
2351 .driver_ready = icm_driver_ready,
2352 .start = icm_start,
2353 .stop = icm_stop,
2354 .suspend = icm_suspend,
2355 .complete = icm_complete,
2356 .runtime_suspend = icm_runtime_suspend,
2357 .runtime_resume = icm_runtime_resume,
2358 .runtime_suspend_switch = icm_runtime_suspend_switch,
2359 .runtime_resume_switch = icm_runtime_resume_switch,
2360 .handle_event = icm_handle_event,
2361 .get_boot_acl = icm_ar_get_boot_acl,
2362 .set_boot_acl = icm_ar_set_boot_acl,
2363 .approve_switch = icm_fr_approve_switch,
2364 .add_switch_key = icm_fr_add_switch_key,
2365 .challenge_switch_key = icm_fr_challenge_switch_key,
2366 .disconnect_pcie_paths = icm_disconnect_pcie_paths,
2367 .approve_xdomain_paths = icm_fr_approve_xdomain_paths,
2368 .disconnect_xdomain_paths = icm_fr_disconnect_xdomain_paths,
2369 };
2370
2371 /* Titan Ridge */
2372 static const struct tb_cm_ops icm_tr_ops = {
2373 .driver_ready = icm_driver_ready,
2374 .start = icm_start,
2375 .stop = icm_stop,
2376 .suspend = icm_suspend,
2377 .complete = icm_complete,
2378 .runtime_suspend = icm_runtime_suspend,
2379 .runtime_resume = icm_runtime_resume,
2380 .runtime_suspend_switch = icm_runtime_suspend_switch,
2381 .runtime_resume_switch = icm_runtime_resume_switch,
2382 .handle_event = icm_handle_event,
2383 .get_boot_acl = icm_ar_get_boot_acl,
2384 .set_boot_acl = icm_ar_set_boot_acl,
2385 .approve_switch = icm_tr_approve_switch,
2386 .add_switch_key = icm_tr_add_switch_key,
2387 .challenge_switch_key = icm_tr_challenge_switch_key,
2388 .disconnect_pcie_paths = icm_disconnect_pcie_paths,
2389 .approve_xdomain_paths = icm_tr_approve_xdomain_paths,
2390 .disconnect_xdomain_paths = icm_tr_disconnect_xdomain_paths,
2391 .usb4_switch_op = icm_usb4_switch_op,
2392 .usb4_switch_nvm_authenticate_status =
2393 icm_usb4_switch_nvm_authenticate_status,
2394 };
2395
2396 /* Ice Lake */
2397 static const struct tb_cm_ops icm_icl_ops = {
2398 .driver_ready = icm_driver_ready,
2399 .start = icm_start,
2400 .stop = icm_stop,
2401 .complete = icm_complete,
2402 .runtime_suspend = icm_runtime_suspend,
2403 .runtime_resume = icm_runtime_resume,
2404 .handle_event = icm_handle_event,
2405 .approve_xdomain_paths = icm_tr_approve_xdomain_paths,
2406 .disconnect_xdomain_paths = icm_tr_disconnect_xdomain_paths,
2407 .usb4_switch_op = icm_usb4_switch_op,
2408 .usb4_switch_nvm_authenticate_status =
2409 icm_usb4_switch_nvm_authenticate_status,
2410 };
2411
2412 struct tb *icm_probe(struct tb_nhi *nhi)
2413 {
2414 struct icm *icm;
2415 struct tb *tb;
2416
2417 tb = tb_domain_alloc(nhi, sizeof(struct icm));
2418 if (!tb)
2419 return NULL;
2420
2421 icm = tb_priv(tb);
2422 INIT_DELAYED_WORK(&icm->rescan_work, icm_rescan_work);
2423 mutex_init(&icm->request_lock);
2424
2425 switch (nhi->pdev->device) {
2426 case PCI_DEVICE_ID_INTEL_FALCON_RIDGE_2C_NHI:
2427 case PCI_DEVICE_ID_INTEL_FALCON_RIDGE_4C_NHI:
2428 icm->can_upgrade_nvm = true;
2429 icm->is_supported = icm_fr_is_supported;
2430 icm->get_route = icm_fr_get_route;
2431 icm->save_devices = icm_fr_save_devices;
2432 icm->driver_ready = icm_fr_driver_ready;
2433 icm->device_connected = icm_fr_device_connected;
2434 icm->device_disconnected = icm_fr_device_disconnected;
2435 icm->xdomain_connected = icm_fr_xdomain_connected;
2436 icm->xdomain_disconnected = icm_fr_xdomain_disconnected;
2437 tb->cm_ops = &icm_fr_ops;
2438 break;
2439
2440 case PCI_DEVICE_ID_INTEL_ALPINE_RIDGE_2C_NHI:
2441 case PCI_DEVICE_ID_INTEL_ALPINE_RIDGE_4C_NHI:
2442 case PCI_DEVICE_ID_INTEL_ALPINE_RIDGE_LP_NHI:
2443 case PCI_DEVICE_ID_INTEL_ALPINE_RIDGE_C_4C_NHI:
2444 case PCI_DEVICE_ID_INTEL_ALPINE_RIDGE_C_2C_NHI:
2445 icm->max_boot_acl = ICM_AR_PREBOOT_ACL_ENTRIES;
2446 /*
2447 * NVM upgrade has not been tested on Apple systems and
2448 * they don't provide images publicly either. To be on
2449 * the safe side prevent root switch NVM upgrade on Macs
2450 * for now.
2451 */
2452 icm->can_upgrade_nvm = !x86_apple_machine;
2453 icm->is_supported = icm_ar_is_supported;
2454 icm->cio_reset = icm_ar_cio_reset;
2455 icm->get_mode = icm_ar_get_mode;
2456 icm->get_route = icm_ar_get_route;
2457 icm->save_devices = icm_fr_save_devices;
2458 icm->driver_ready = icm_ar_driver_ready;
2459 icm->device_connected = icm_fr_device_connected;
2460 icm->device_disconnected = icm_fr_device_disconnected;
2461 icm->xdomain_connected = icm_fr_xdomain_connected;
2462 icm->xdomain_disconnected = icm_fr_xdomain_disconnected;
2463 tb->cm_ops = &icm_ar_ops;
2464 break;
2465
2466 case PCI_DEVICE_ID_INTEL_TITAN_RIDGE_2C_NHI:
2467 case PCI_DEVICE_ID_INTEL_TITAN_RIDGE_4C_NHI:
2468 icm->max_boot_acl = ICM_AR_PREBOOT_ACL_ENTRIES;
2469 icm->can_upgrade_nvm = !x86_apple_machine;
2470 icm->is_supported = icm_ar_is_supported;
2471 icm->cio_reset = icm_tr_cio_reset;
2472 icm->get_mode = icm_ar_get_mode;
2473 icm->driver_ready = icm_tr_driver_ready;
2474 icm->device_connected = icm_tr_device_connected;
2475 icm->device_disconnected = icm_tr_device_disconnected;
2476 icm->xdomain_connected = icm_tr_xdomain_connected;
2477 icm->xdomain_disconnected = icm_tr_xdomain_disconnected;
2478 tb->cm_ops = &icm_tr_ops;
2479 break;
2480
2481 case PCI_DEVICE_ID_INTEL_ICL_NHI0:
2482 case PCI_DEVICE_ID_INTEL_ICL_NHI1:
2483 icm->is_supported = icm_fr_is_supported;
2484 icm->driver_ready = icm_icl_driver_ready;
2485 icm->set_uuid = icm_icl_set_uuid;
2486 icm->device_connected = icm_icl_device_connected;
2487 icm->device_disconnected = icm_tr_device_disconnected;
2488 icm->xdomain_connected = icm_tr_xdomain_connected;
2489 icm->xdomain_disconnected = icm_tr_xdomain_disconnected;
2490 icm->rtd3_veto = icm_icl_rtd3_veto;
2491 tb->cm_ops = &icm_icl_ops;
2492 break;
2493
2494 case PCI_DEVICE_ID_INTEL_TGL_NHI0:
2495 case PCI_DEVICE_ID_INTEL_TGL_NHI1:
2496 case PCI_DEVICE_ID_INTEL_TGL_H_NHI0:
2497 case PCI_DEVICE_ID_INTEL_TGL_H_NHI1:
2498 icm->is_supported = icm_tgl_is_supported;
2499 icm->driver_ready = icm_icl_driver_ready;
2500 icm->set_uuid = icm_icl_set_uuid;
2501 icm->device_connected = icm_icl_device_connected;
2502 icm->device_disconnected = icm_tr_device_disconnected;
2503 icm->xdomain_connected = icm_tr_xdomain_connected;
2504 icm->xdomain_disconnected = icm_tr_xdomain_disconnected;
2505 icm->rtd3_veto = icm_icl_rtd3_veto;
2506 tb->cm_ops = &icm_icl_ops;
2507 break;
2508
2509 case PCI_DEVICE_ID_INTEL_MAPLE_RIDGE_4C_NHI:
2510 icm->is_supported = icm_tgl_is_supported;
2511 icm->get_mode = icm_ar_get_mode;
2512 icm->driver_ready = icm_tr_driver_ready;
2513 icm->device_connected = icm_tr_device_connected;
2514 icm->device_disconnected = icm_tr_device_disconnected;
2515 icm->xdomain_connected = icm_tr_xdomain_connected;
2516 icm->xdomain_disconnected = icm_tr_xdomain_disconnected;
2517 tb->cm_ops = &icm_tr_ops;
2518 break;
2519 }
2520
2521 if (!icm->is_supported || !icm->is_supported(tb)) {
2522 dev_dbg(&nhi->pdev->dev, "ICM not supported on this controller\n");
2523 tb_domain_put(tb);
2524 return NULL;
2525 }
2526
2527 tb_dbg(tb, "using firmware connection manager\n");
2528
2529 return tb;
2530 }
Linux with added FPC-III support