1 // SPDX-License-Identifier: GPL-2.0
3 * Interconnect framework core driver
5 * Copyright (c) 2017-2019, Linaro Ltd.
6 * Author: Georgi Djakov <georgi.djakov@linaro.org>
9 #include <linux/debugfs.h>
10 #include <linux/device.h>
11 #include <linux/idr.h>
12 #include <linux/init.h>
13 #include <linux/interconnect.h>
14 #include <linux/interconnect-provider.h>
15 #include <linux/list.h>
16 #include <linux/module.h>
17 #include <linux/mutex.h>
18 #include <linux/slab.h>
20 #include <linux/overflow.h>
24 #define CREATE_TRACE_POINTS
27 static DEFINE_IDR(icc_idr
);
28 static LIST_HEAD(icc_providers
);
29 static DEFINE_MUTEX(icc_lock
);
30 static struct dentry
*icc_debugfs_dir
;
32 static void icc_summary_show_one(struct seq_file
*s
, struct icc_node
*n
)
37 seq_printf(s
, "%-42s %12u %12u\n",
38 n
->name
, n
->avg_bw
, n
->peak_bw
);
41 static int icc_summary_show(struct seq_file
*s
, void *data
)
43 struct icc_provider
*provider
;
45 seq_puts(s
, " node tag avg peak\n");
46 seq_puts(s
, "--------------------------------------------------------------------\n");
48 mutex_lock(&icc_lock
);
50 list_for_each_entry(provider
, &icc_providers
, provider_list
) {
53 list_for_each_entry(n
, &provider
->nodes
, node_list
) {
56 icc_summary_show_one(s
, n
);
57 hlist_for_each_entry(r
, &n
->req_list
, req_node
) {
61 seq_printf(s
, " %-27s %12u %12u %12u\n",
62 dev_name(r
->dev
), r
->tag
, r
->avg_bw
,
68 mutex_unlock(&icc_lock
);
72 DEFINE_SHOW_ATTRIBUTE(icc_summary
);
74 static void icc_graph_show_link(struct seq_file
*s
, int level
,
75 struct icc_node
*n
, struct icc_node
*m
)
77 seq_printf(s
, "%s\"%d:%s\" -> \"%d:%s\"\n",
78 level
== 2 ? "\t\t" : "\t",
79 n
->id
, n
->name
, m
->id
, m
->name
);
82 static void icc_graph_show_node(struct seq_file
*s
, struct icc_node
*n
)
84 seq_printf(s
, "\t\t\"%d:%s\" [label=\"%d:%s",
85 n
->id
, n
->name
, n
->id
, n
->name
);
86 seq_printf(s
, "\n\t\t\t|avg_bw=%ukBps", n
->avg_bw
);
87 seq_printf(s
, "\n\t\t\t|peak_bw=%ukBps", n
->peak_bw
);
91 static int icc_graph_show(struct seq_file
*s
, void *data
)
93 struct icc_provider
*provider
;
95 int cluster_index
= 0;
98 seq_puts(s
, "digraph {\n\trankdir = LR\n\tnode [shape = record]\n");
99 mutex_lock(&icc_lock
);
101 /* draw providers as cluster subgraphs */
103 list_for_each_entry(provider
, &icc_providers
, provider_list
) {
104 seq_printf(s
, "\tsubgraph cluster_%d {\n", ++cluster_index
);
106 seq_printf(s
, "\t\tlabel = \"%s\"\n",
107 dev_name(provider
->dev
));
110 list_for_each_entry(n
, &provider
->nodes
, node_list
)
111 icc_graph_show_node(s
, n
);
113 /* draw internal links */
114 list_for_each_entry(n
, &provider
->nodes
, node_list
)
115 for (i
= 0; i
< n
->num_links
; ++i
)
116 if (n
->provider
== n
->links
[i
]->provider
)
117 icc_graph_show_link(s
, 2, n
,
120 seq_puts(s
, "\t}\n");
123 /* draw external links */
124 list_for_each_entry(provider
, &icc_providers
, provider_list
)
125 list_for_each_entry(n
, &provider
->nodes
, node_list
)
126 for (i
= 0; i
< n
->num_links
; ++i
)
127 if (n
->provider
!= n
->links
[i
]->provider
)
128 icc_graph_show_link(s
, 1, n
,
131 mutex_unlock(&icc_lock
);
136 DEFINE_SHOW_ATTRIBUTE(icc_graph
);
138 static struct icc_node
*node_find(const int id
)
140 return idr_find(&icc_idr
, id
);
143 static struct icc_path
*path_init(struct device
*dev
, struct icc_node
*dst
,
146 struct icc_node
*node
= dst
;
147 struct icc_path
*path
;
150 path
= kzalloc(struct_size(path
, reqs
, num_nodes
), GFP_KERNEL
);
152 return ERR_PTR(-ENOMEM
);
154 path
->num_nodes
= num_nodes
;
156 for (i
= num_nodes
- 1; i
>= 0; i
--) {
157 node
->provider
->users
++;
158 hlist_add_head(&path
->reqs
[i
].req_node
, &node
->req_list
);
159 path
->reqs
[i
].node
= node
;
160 path
->reqs
[i
].dev
= dev
;
161 /* reference to previous node was saved during path traversal */
162 node
= node
->reverse
;
168 static struct icc_path
*path_find(struct device
*dev
, struct icc_node
*src
,
169 struct icc_node
*dst
)
171 struct icc_path
*path
= ERR_PTR(-EPROBE_DEFER
);
172 struct icc_node
*n
, *node
= NULL
;
173 struct list_head traverse_list
;
174 struct list_head edge_list
;
175 struct list_head visited_list
;
179 INIT_LIST_HEAD(&traverse_list
);
180 INIT_LIST_HEAD(&edge_list
);
181 INIT_LIST_HEAD(&visited_list
);
183 list_add(&src
->search_list
, &traverse_list
);
187 list_for_each_entry_safe(node
, n
, &traverse_list
, search_list
) {
190 list_splice_init(&edge_list
, &visited_list
);
191 list_splice_init(&traverse_list
, &visited_list
);
194 for (i
= 0; i
< node
->num_links
; i
++) {
195 struct icc_node
*tmp
= node
->links
[i
];
198 path
= ERR_PTR(-ENOENT
);
202 if (tmp
->is_traversed
)
205 tmp
->is_traversed
= true;
207 list_add_tail(&tmp
->search_list
, &edge_list
);
214 list_splice_init(&traverse_list
, &visited_list
);
215 list_splice_init(&edge_list
, &traverse_list
);
217 /* count the hops including the source */
220 } while (!list_empty(&traverse_list
));
224 /* reset the traversed state */
225 list_for_each_entry_reverse(n
, &visited_list
, search_list
)
226 n
->is_traversed
= false;
229 path
= path_init(dev
, dst
, depth
);
235 * We want the path to honor all bandwidth requests, so the average and peak
236 * bandwidth requirements from each consumer are aggregated at each node.
237 * The aggregation is platform specific, so each platform can customize it by
238 * implementing its own aggregate() function.
241 static int aggregate_requests(struct icc_node
*node
)
243 struct icc_provider
*p
= node
->provider
;
249 if (p
->pre_aggregate
)
250 p
->pre_aggregate(node
);
252 hlist_for_each_entry(r
, &node
->req_list
, req_node
)
253 p
->aggregate(node
, r
->tag
, r
->avg_bw
, r
->peak_bw
,
254 &node
->avg_bw
, &node
->peak_bw
);
259 static int apply_constraints(struct icc_path
*path
)
261 struct icc_node
*next
, *prev
= NULL
;
265 for (i
= 0; i
< path
->num_nodes
; i
++) {
266 next
= path
->reqs
[i
].node
;
269 * Both endpoints should be valid master-slave pairs of the
270 * same interconnect provider that will be configured.
272 if (!prev
|| next
->provider
!= prev
->provider
) {
277 /* set the constraints */
278 ret
= next
->provider
->set(prev
, next
);
288 int icc_std_aggregate(struct icc_node
*node
, u32 tag
, u32 avg_bw
,
289 u32 peak_bw
, u32
*agg_avg
, u32
*agg_peak
)
292 *agg_peak
= max(*agg_peak
, peak_bw
);
296 EXPORT_SYMBOL_GPL(icc_std_aggregate
);
298 /* of_icc_xlate_onecell() - Translate function using a single index.
299 * @spec: OF phandle args to map into an interconnect node.
300 * @data: private data (pointer to struct icc_onecell_data)
302 * This is a generic translate function that can be used to model simple
303 * interconnect providers that have one device tree node and provide
304 * multiple interconnect nodes. A single cell is used as an index into
305 * an array of icc nodes specified in the icc_onecell_data struct when
306 * registering the provider.
308 struct icc_node
*of_icc_xlate_onecell(struct of_phandle_args
*spec
,
311 struct icc_onecell_data
*icc_data
= data
;
312 unsigned int idx
= spec
->args
[0];
314 if (idx
>= icc_data
->num_nodes
) {
315 pr_err("%s: invalid index %u\n", __func__
, idx
);
316 return ERR_PTR(-EINVAL
);
319 return icc_data
->nodes
[idx
];
321 EXPORT_SYMBOL_GPL(of_icc_xlate_onecell
);
324 * of_icc_get_from_provider() - Look-up interconnect node
325 * @spec: OF phandle args to use for look-up
327 * Looks for interconnect provider under the node specified by @spec and if
328 * found, uses xlate function of the provider to map phandle args to node.
330 * Returns a valid pointer to struct icc_node on success or ERR_PTR()
333 static struct icc_node
*of_icc_get_from_provider(struct of_phandle_args
*spec
)
335 struct icc_node
*node
= ERR_PTR(-EPROBE_DEFER
);
336 struct icc_provider
*provider
;
338 if (!spec
|| spec
->args_count
!= 1)
339 return ERR_PTR(-EINVAL
);
341 mutex_lock(&icc_lock
);
342 list_for_each_entry(provider
, &icc_providers
, provider_list
) {
343 if (provider
->dev
->of_node
== spec
->np
)
344 node
= provider
->xlate(spec
, provider
->data
);
348 mutex_unlock(&icc_lock
);
354 * of_icc_get() - get a path handle from a DT node based on name
355 * @dev: device pointer for the consumer device
356 * @name: interconnect path name
358 * This function will search for a path between two endpoints and return an
359 * icc_path handle on success. Use icc_put() to release constraints when they
360 * are not needed anymore.
361 * If the interconnect API is disabled, NULL is returned and the consumer
362 * drivers will still build. Drivers are free to handle this specifically,
363 * but they don't have to.
365 * Return: icc_path pointer on success or ERR_PTR() on error. NULL is returned
366 * when the API is disabled or the "interconnects" DT property is missing.
368 struct icc_path
*of_icc_get(struct device
*dev
, const char *name
)
370 struct icc_path
*path
= ERR_PTR(-EPROBE_DEFER
);
371 struct icc_node
*src_node
, *dst_node
;
372 struct device_node
*np
= NULL
;
373 struct of_phandle_args src_args
, dst_args
;
377 if (!dev
|| !dev
->of_node
)
378 return ERR_PTR(-ENODEV
);
383 * When the consumer DT node do not have "interconnects" property
384 * return a NULL path to skip setting constraints.
386 if (!of_find_property(np
, "interconnects", NULL
))
390 * We use a combination of phandle and specifier for endpoint. For now
391 * lets support only global ids and extend this in the future if needed
392 * without breaking DT compatibility.
395 idx
= of_property_match_string(np
, "interconnect-names", name
);
400 ret
= of_parse_phandle_with_args(np
, "interconnects",
401 "#interconnect-cells", idx
* 2,
406 of_node_put(src_args
.np
);
408 ret
= of_parse_phandle_with_args(np
, "interconnects",
409 "#interconnect-cells", idx
* 2 + 1,
414 of_node_put(dst_args
.np
);
416 src_node
= of_icc_get_from_provider(&src_args
);
418 if (IS_ERR(src_node
)) {
419 if (PTR_ERR(src_node
) != -EPROBE_DEFER
)
420 dev_err(dev
, "error finding src node: %ld\n",
422 return ERR_CAST(src_node
);
425 dst_node
= of_icc_get_from_provider(&dst_args
);
427 if (IS_ERR(dst_node
)) {
428 if (PTR_ERR(dst_node
) != -EPROBE_DEFER
)
429 dev_err(dev
, "error finding dst node: %ld\n",
431 return ERR_CAST(dst_node
);
434 mutex_lock(&icc_lock
);
435 path
= path_find(dev
, src_node
, dst_node
);
436 mutex_unlock(&icc_lock
);
438 dev_err(dev
, "%s: invalid path=%ld\n", __func__
, PTR_ERR(path
));
443 path
->name
= kstrdup_const(name
, GFP_KERNEL
);
445 path
->name
= kasprintf(GFP_KERNEL
, "%s-%s",
446 src_node
->name
, dst_node
->name
);
450 EXPORT_SYMBOL_GPL(of_icc_get
);
453 * icc_set_tag() - set an optional tag on a path
454 * @path: the path we want to tag
455 * @tag: the tag value
457 * This function allows consumers to append a tag to the requests associated
458 * with a path, so that a different aggregation could be done based on this tag.
460 void icc_set_tag(struct icc_path
*path
, u32 tag
)
467 mutex_lock(&icc_lock
);
469 for (i
= 0; i
< path
->num_nodes
; i
++)
470 path
->reqs
[i
].tag
= tag
;
472 mutex_unlock(&icc_lock
);
474 EXPORT_SYMBOL_GPL(icc_set_tag
);
477 * icc_set_bw() - set bandwidth constraints on an interconnect path
478 * @path: reference to the path returned by icc_get()
479 * @avg_bw: average bandwidth in kilobytes per second
480 * @peak_bw: peak bandwidth in kilobytes per second
482 * This function is used by an interconnect consumer to express its own needs
483 * in terms of bandwidth for a previously requested path between two endpoints.
484 * The requests are aggregated and each node is updated accordingly. The entire
485 * path is locked by a mutex to ensure that the set() is completed.
486 * The @path can be NULL when the "interconnects" DT properties is missing,
487 * which will mean that no constraints will be set.
489 * Returns 0 on success, or an appropriate error code otherwise.
491 int icc_set_bw(struct icc_path
*path
, u32 avg_bw
, u32 peak_bw
)
493 struct icc_node
*node
;
494 u32 old_avg
, old_peak
;
501 if (WARN_ON(IS_ERR(path
) || !path
->num_nodes
))
504 mutex_lock(&icc_lock
);
506 old_avg
= path
->reqs
[0].avg_bw
;
507 old_peak
= path
->reqs
[0].peak_bw
;
509 for (i
= 0; i
< path
->num_nodes
; i
++) {
510 node
= path
->reqs
[i
].node
;
512 /* update the consumer request for this path */
513 path
->reqs
[i
].avg_bw
= avg_bw
;
514 path
->reqs
[i
].peak_bw
= peak_bw
;
516 /* aggregate requests for this node */
517 aggregate_requests(node
);
519 trace_icc_set_bw(path
, node
, i
, avg_bw
, peak_bw
);
522 ret
= apply_constraints(path
);
524 pr_debug("interconnect: error applying constraints (%d)\n",
527 for (i
= 0; i
< path
->num_nodes
; i
++) {
528 node
= path
->reqs
[i
].node
;
529 path
->reqs
[i
].avg_bw
= old_avg
;
530 path
->reqs
[i
].peak_bw
= old_peak
;
531 aggregate_requests(node
);
533 apply_constraints(path
);
536 mutex_unlock(&icc_lock
);
538 trace_icc_set_bw_end(path
, ret
);
542 EXPORT_SYMBOL_GPL(icc_set_bw
);
545 * icc_get() - return a handle for path between two endpoints
546 * @dev: the device requesting the path
547 * @src_id: source device port id
548 * @dst_id: destination device port id
550 * This function will search for a path between two endpoints and return an
551 * icc_path handle on success. Use icc_put() to release
552 * constraints when they are not needed anymore.
553 * If the interconnect API is disabled, NULL is returned and the consumer
554 * drivers will still build. Drivers are free to handle this specifically,
555 * but they don't have to.
557 * Return: icc_path pointer on success, ERR_PTR() on error or NULL if the
558 * interconnect API is disabled.
560 struct icc_path
*icc_get(struct device
*dev
, const int src_id
, const int dst_id
)
562 struct icc_node
*src
, *dst
;
563 struct icc_path
*path
= ERR_PTR(-EPROBE_DEFER
);
565 mutex_lock(&icc_lock
);
567 src
= node_find(src_id
);
571 dst
= node_find(dst_id
);
575 path
= path_find(dev
, src
, dst
);
577 dev_err(dev
, "%s: invalid path=%ld\n", __func__
, PTR_ERR(path
));
581 path
->name
= kasprintf(GFP_KERNEL
, "%s-%s", src
->name
, dst
->name
);
583 mutex_unlock(&icc_lock
);
586 EXPORT_SYMBOL_GPL(icc_get
);
589 * icc_put() - release the reference to the icc_path
590 * @path: interconnect path
592 * Use this function to release the constraints on a path when the path is
593 * no longer needed. The constraints will be re-aggregated.
595 void icc_put(struct icc_path
*path
)
597 struct icc_node
*node
;
601 if (!path
|| WARN_ON(IS_ERR(path
)))
604 ret
= icc_set_bw(path
, 0, 0);
606 pr_err("%s: error (%d)\n", __func__
, ret
);
608 mutex_lock(&icc_lock
);
609 for (i
= 0; i
< path
->num_nodes
; i
++) {
610 node
= path
->reqs
[i
].node
;
611 hlist_del(&path
->reqs
[i
].req_node
);
612 if (!WARN_ON(!node
->provider
->users
))
613 node
->provider
->users
--;
615 mutex_unlock(&icc_lock
);
617 kfree_const(path
->name
);
620 EXPORT_SYMBOL_GPL(icc_put
);
622 static struct icc_node
*icc_node_create_nolock(int id
)
624 struct icc_node
*node
;
626 /* check if node already exists */
627 node
= node_find(id
);
631 node
= kzalloc(sizeof(*node
), GFP_KERNEL
);
633 return ERR_PTR(-ENOMEM
);
635 id
= idr_alloc(&icc_idr
, node
, id
, id
+ 1, GFP_KERNEL
);
637 WARN(1, "%s: couldn't get idr\n", __func__
);
648 * icc_node_create() - create a node
651 * Return: icc_node pointer on success, or ERR_PTR() on error
653 struct icc_node
*icc_node_create(int id
)
655 struct icc_node
*node
;
657 mutex_lock(&icc_lock
);
659 node
= icc_node_create_nolock(id
);
661 mutex_unlock(&icc_lock
);
665 EXPORT_SYMBOL_GPL(icc_node_create
);
668 * icc_node_destroy() - destroy a node
671 void icc_node_destroy(int id
)
673 struct icc_node
*node
;
675 mutex_lock(&icc_lock
);
677 node
= node_find(id
);
679 idr_remove(&icc_idr
, node
->id
);
680 WARN_ON(!hlist_empty(&node
->req_list
));
683 mutex_unlock(&icc_lock
);
687 EXPORT_SYMBOL_GPL(icc_node_destroy
);
690 * icc_link_create() - create a link between two nodes
691 * @node: source node id
692 * @dst_id: destination node id
694 * Create a link between two nodes. The nodes might belong to different
695 * interconnect providers and the @dst_id node might not exist (if the
696 * provider driver has not probed yet). So just create the @dst_id node
697 * and when the actual provider driver is probed, the rest of the node
700 * Return: 0 on success, or an error code otherwise
702 int icc_link_create(struct icc_node
*node
, const int dst_id
)
704 struct icc_node
*dst
;
705 struct icc_node
**new;
711 mutex_lock(&icc_lock
);
713 dst
= node_find(dst_id
);
715 dst
= icc_node_create_nolock(dst_id
);
723 new = krealloc(node
->links
,
724 (node
->num_links
+ 1) * sizeof(*node
->links
),
732 node
->links
[node
->num_links
++] = dst
;
735 mutex_unlock(&icc_lock
);
739 EXPORT_SYMBOL_GPL(icc_link_create
);
742 * icc_link_destroy() - destroy a link between two nodes
743 * @src: pointer to source node
744 * @dst: pointer to destination node
746 * Return: 0 on success, or an error code otherwise
748 int icc_link_destroy(struct icc_node
*src
, struct icc_node
*dst
)
750 struct icc_node
**new;
754 if (IS_ERR_OR_NULL(src
))
757 if (IS_ERR_OR_NULL(dst
))
760 mutex_lock(&icc_lock
);
762 for (slot
= 0; slot
< src
->num_links
; slot
++)
763 if (src
->links
[slot
] == dst
)
766 if (WARN_ON(slot
== src
->num_links
)) {
771 src
->links
[slot
] = src
->links
[--src
->num_links
];
773 new = krealloc(src
->links
, src
->num_links
* sizeof(*src
->links
),
779 mutex_unlock(&icc_lock
);
783 EXPORT_SYMBOL_GPL(icc_link_destroy
);
786 * icc_node_add() - add interconnect node to interconnect provider
787 * @node: pointer to the interconnect node
788 * @provider: pointer to the interconnect provider
790 void icc_node_add(struct icc_node
*node
, struct icc_provider
*provider
)
792 mutex_lock(&icc_lock
);
794 node
->provider
= provider
;
795 list_add_tail(&node
->node_list
, &provider
->nodes
);
797 mutex_unlock(&icc_lock
);
799 EXPORT_SYMBOL_GPL(icc_node_add
);
802 * icc_node_del() - delete interconnect node from interconnect provider
803 * @node: pointer to the interconnect node
805 void icc_node_del(struct icc_node
*node
)
807 mutex_lock(&icc_lock
);
809 list_del(&node
->node_list
);
811 mutex_unlock(&icc_lock
);
813 EXPORT_SYMBOL_GPL(icc_node_del
);
816 * icc_nodes_remove() - remove all previously added nodes from provider
817 * @provider: the interconnect provider we are removing nodes from
819 * Return: 0 on success, or an error code otherwise
821 int icc_nodes_remove(struct icc_provider
*provider
)
823 struct icc_node
*n
, *tmp
;
825 if (WARN_ON(IS_ERR_OR_NULL(provider
)))
828 list_for_each_entry_safe_reverse(n
, tmp
, &provider
->nodes
, node_list
) {
830 icc_node_destroy(n
->id
);
835 EXPORT_SYMBOL_GPL(icc_nodes_remove
);
838 * icc_provider_add() - add a new interconnect provider
839 * @provider: the interconnect provider that will be added into topology
841 * Return: 0 on success, or an error code otherwise
843 int icc_provider_add(struct icc_provider
*provider
)
845 if (WARN_ON(!provider
->set
))
847 if (WARN_ON(!provider
->xlate
))
850 mutex_lock(&icc_lock
);
852 INIT_LIST_HEAD(&provider
->nodes
);
853 list_add_tail(&provider
->provider_list
, &icc_providers
);
855 mutex_unlock(&icc_lock
);
857 dev_dbg(provider
->dev
, "interconnect provider added to topology\n");
861 EXPORT_SYMBOL_GPL(icc_provider_add
);
864 * icc_provider_del() - delete previously added interconnect provider
865 * @provider: the interconnect provider that will be removed from topology
867 * Return: 0 on success, or an error code otherwise
869 int icc_provider_del(struct icc_provider
*provider
)
871 mutex_lock(&icc_lock
);
872 if (provider
->users
) {
873 pr_warn("interconnect provider still has %d users\n",
875 mutex_unlock(&icc_lock
);
879 if (!list_empty(&provider
->nodes
)) {
880 pr_warn("interconnect provider still has nodes\n");
881 mutex_unlock(&icc_lock
);
885 list_del(&provider
->provider_list
);
886 mutex_unlock(&icc_lock
);
890 EXPORT_SYMBOL_GPL(icc_provider_del
);
892 static int __init
icc_init(void)
894 icc_debugfs_dir
= debugfs_create_dir("interconnect", NULL
);
895 debugfs_create_file("interconnect_summary", 0444,
896 icc_debugfs_dir
, NULL
, &icc_summary_fops
);
897 debugfs_create_file("interconnect_graph", 0444,
898 icc_debugfs_dir
, NULL
, &icc_graph_fops
);
902 static void __exit
icc_exit(void)
904 debugfs_remove_recursive(icc_debugfs_dir
);
906 module_init(icc_init
);
907 module_exit(icc_exit
);
909 MODULE_AUTHOR("Georgi Djakov <georgi.djakov@linaro.org>");
910 MODULE_DESCRIPTION("Interconnect Driver Core");
911 MODULE_LICENSE("GPL v2");