1 // SPDX-License-Identifier: GPL-2.0
3 * Interconnect framework core driver
5 * Copyright (c) 2017-2019, Linaro Ltd.
6 * Author: Georgi Djakov <georgi.djakov@linaro.org>
9 #include <linux/debugfs.h>
10 #include <linux/device.h>
11 #include <linux/idr.h>
12 #include <linux/init.h>
13 #include <linux/interconnect.h>
14 #include <linux/interconnect-provider.h>
15 #include <linux/list.h>
16 #include <linux/module.h>
17 #include <linux/mutex.h>
18 #include <linux/slab.h>
20 #include <linux/overflow.h>
24 #define CREATE_TRACE_POINTS
27 static DEFINE_IDR(icc_idr
);
28 static LIST_HEAD(icc_providers
);
29 static DEFINE_MUTEX(icc_lock
);
30 static struct dentry
*icc_debugfs_dir
;
32 static void icc_summary_show_one(struct seq_file
*s
, struct icc_node
*n
)
37 seq_printf(s
, "%-42s %12u %12u\n",
38 n
->name
, n
->avg_bw
, n
->peak_bw
);
41 static int icc_summary_show(struct seq_file
*s
, void *data
)
43 struct icc_provider
*provider
;
45 seq_puts(s
, " node tag avg peak\n");
46 seq_puts(s
, "--------------------------------------------------------------------\n");
48 mutex_lock(&icc_lock
);
50 list_for_each_entry(provider
, &icc_providers
, provider_list
) {
53 list_for_each_entry(n
, &provider
->nodes
, node_list
) {
56 icc_summary_show_one(s
, n
);
57 hlist_for_each_entry(r
, &n
->req_list
, req_node
) {
61 seq_printf(s
, " %-27s %12u %12u %12u\n",
62 dev_name(r
->dev
), r
->tag
, r
->avg_bw
,
68 mutex_unlock(&icc_lock
);
72 DEFINE_SHOW_ATTRIBUTE(icc_summary
);
74 static void icc_graph_show_link(struct seq_file
*s
, int level
,
75 struct icc_node
*n
, struct icc_node
*m
)
77 seq_printf(s
, "%s\"%d:%s\" -> \"%d:%s\"\n",
78 level
== 2 ? "\t\t" : "\t",
79 n
->id
, n
->name
, m
->id
, m
->name
);
82 static void icc_graph_show_node(struct seq_file
*s
, struct icc_node
*n
)
84 seq_printf(s
, "\t\t\"%d:%s\" [label=\"%d:%s",
85 n
->id
, n
->name
, n
->id
, n
->name
);
86 seq_printf(s
, "\n\t\t\t|avg_bw=%ukBps", n
->avg_bw
);
87 seq_printf(s
, "\n\t\t\t|peak_bw=%ukBps", n
->peak_bw
);
91 static int icc_graph_show(struct seq_file
*s
, void *data
)
93 struct icc_provider
*provider
;
95 int cluster_index
= 0;
98 seq_puts(s
, "digraph {\n\trankdir = LR\n\tnode [shape = record]\n");
99 mutex_lock(&icc_lock
);
101 /* draw providers as cluster subgraphs */
103 list_for_each_entry(provider
, &icc_providers
, provider_list
) {
104 seq_printf(s
, "\tsubgraph cluster_%d {\n", ++cluster_index
);
106 seq_printf(s
, "\t\tlabel = \"%s\"\n",
107 dev_name(provider
->dev
));
110 list_for_each_entry(n
, &provider
->nodes
, node_list
)
111 icc_graph_show_node(s
, n
);
113 /* draw internal links */
114 list_for_each_entry(n
, &provider
->nodes
, node_list
)
115 for (i
= 0; i
< n
->num_links
; ++i
)
116 if (n
->provider
== n
->links
[i
]->provider
)
117 icc_graph_show_link(s
, 2, n
,
120 seq_puts(s
, "\t}\n");
123 /* draw external links */
124 list_for_each_entry(provider
, &icc_providers
, provider_list
)
125 list_for_each_entry(n
, &provider
->nodes
, node_list
)
126 for (i
= 0; i
< n
->num_links
; ++i
)
127 if (n
->provider
!= n
->links
[i
]->provider
)
128 icc_graph_show_link(s
, 1, n
,
131 mutex_unlock(&icc_lock
);
136 DEFINE_SHOW_ATTRIBUTE(icc_graph
);
138 static struct icc_node
*node_find(const int id
)
140 return idr_find(&icc_idr
, id
);
143 static struct icc_path
*path_init(struct device
*dev
, struct icc_node
*dst
,
146 struct icc_node
*node
= dst
;
147 struct icc_path
*path
;
150 path
= kzalloc(struct_size(path
, reqs
, num_nodes
), GFP_KERNEL
);
152 return ERR_PTR(-ENOMEM
);
154 path
->num_nodes
= num_nodes
;
156 for (i
= num_nodes
- 1; i
>= 0; i
--) {
157 node
->provider
->users
++;
158 hlist_add_head(&path
->reqs
[i
].req_node
, &node
->req_list
);
159 path
->reqs
[i
].node
= node
;
160 path
->reqs
[i
].dev
= dev
;
161 /* reference to previous node was saved during path traversal */
162 node
= node
->reverse
;
168 static struct icc_path
*path_find(struct device
*dev
, struct icc_node
*src
,
169 struct icc_node
*dst
)
171 struct icc_path
*path
= ERR_PTR(-EPROBE_DEFER
);
172 struct icc_node
*n
, *node
= NULL
;
173 struct list_head traverse_list
;
174 struct list_head edge_list
;
175 struct list_head visited_list
;
179 INIT_LIST_HEAD(&traverse_list
);
180 INIT_LIST_HEAD(&edge_list
);
181 INIT_LIST_HEAD(&visited_list
);
183 list_add(&src
->search_list
, &traverse_list
);
187 list_for_each_entry_safe(node
, n
, &traverse_list
, search_list
) {
190 list_splice_init(&edge_list
, &visited_list
);
191 list_splice_init(&traverse_list
, &visited_list
);
194 for (i
= 0; i
< node
->num_links
; i
++) {
195 struct icc_node
*tmp
= node
->links
[i
];
198 path
= ERR_PTR(-ENOENT
);
202 if (tmp
->is_traversed
)
205 tmp
->is_traversed
= true;
207 list_add_tail(&tmp
->search_list
, &edge_list
);
214 list_splice_init(&traverse_list
, &visited_list
);
215 list_splice_init(&edge_list
, &traverse_list
);
217 /* count the hops including the source */
220 } while (!list_empty(&traverse_list
));
224 /* reset the traversed state */
225 list_for_each_entry_reverse(n
, &visited_list
, search_list
)
226 n
->is_traversed
= false;
229 path
= path_init(dev
, dst
, depth
);
235 * We want the path to honor all bandwidth requests, so the average and peak
236 * bandwidth requirements from each consumer are aggregated at each node.
237 * The aggregation is platform specific, so each platform can customize it by
238 * implementing its own aggregate() function.
241 static int aggregate_requests(struct icc_node
*node
)
243 struct icc_provider
*p
= node
->provider
;
249 if (p
->pre_aggregate
)
250 p
->pre_aggregate(node
);
252 hlist_for_each_entry(r
, &node
->req_list
, req_node
)
253 p
->aggregate(node
, r
->tag
, r
->avg_bw
, r
->peak_bw
,
254 &node
->avg_bw
, &node
->peak_bw
);
259 static int apply_constraints(struct icc_path
*path
)
261 struct icc_node
*next
, *prev
= NULL
;
265 for (i
= 0; i
< path
->num_nodes
; i
++) {
266 next
= path
->reqs
[i
].node
;
269 * Both endpoints should be valid master-slave pairs of the
270 * same interconnect provider that will be configured.
272 if (!prev
|| next
->provider
!= prev
->provider
) {
277 /* set the constraints */
278 ret
= next
->provider
->set(prev
, next
);
288 int icc_std_aggregate(struct icc_node
*node
, u32 tag
, u32 avg_bw
,
289 u32 peak_bw
, u32
*agg_avg
, u32
*agg_peak
)
292 *agg_peak
= max(*agg_peak
, peak_bw
);
296 EXPORT_SYMBOL_GPL(icc_std_aggregate
);
298 /* of_icc_xlate_onecell() - Translate function using a single index.
299 * @spec: OF phandle args to map into an interconnect node.
300 * @data: private data (pointer to struct icc_onecell_data)
302 * This is a generic translate function that can be used to model simple
303 * interconnect providers that have one device tree node and provide
304 * multiple interconnect nodes. A single cell is used as an index into
305 * an array of icc nodes specified in the icc_onecell_data struct when
306 * registering the provider.
308 struct icc_node
*of_icc_xlate_onecell(struct of_phandle_args
*spec
,
311 struct icc_onecell_data
*icc_data
= data
;
312 unsigned int idx
= spec
->args
[0];
314 if (idx
>= icc_data
->num_nodes
) {
315 pr_err("%s: invalid index %u\n", __func__
, idx
);
316 return ERR_PTR(-EINVAL
);
319 return icc_data
->nodes
[idx
];
321 EXPORT_SYMBOL_GPL(of_icc_xlate_onecell
);
324 * of_icc_get_from_provider() - Look-up interconnect node
325 * @spec: OF phandle args to use for look-up
327 * Looks for interconnect provider under the node specified by @spec and if
328 * found, uses xlate function of the provider to map phandle args to node.
330 * Returns a valid pointer to struct icc_node on success or ERR_PTR()
333 static struct icc_node
*of_icc_get_from_provider(struct of_phandle_args
*spec
)
335 struct icc_node
*node
= ERR_PTR(-EPROBE_DEFER
);
336 struct icc_provider
*provider
;
338 if (!spec
|| spec
->args_count
!= 1)
339 return ERR_PTR(-EINVAL
);
341 mutex_lock(&icc_lock
);
342 list_for_each_entry(provider
, &icc_providers
, provider_list
) {
343 if (provider
->dev
->of_node
== spec
->np
)
344 node
= provider
->xlate(spec
, provider
->data
);
348 mutex_unlock(&icc_lock
);
354 * of_icc_get() - get a path handle from a DT node based on name
355 * @dev: device pointer for the consumer device
356 * @name: interconnect path name
358 * This function will search for a path between two endpoints and return an
359 * icc_path handle on success. Use icc_put() to release constraints when they
360 * are not needed anymore.
361 * If the interconnect API is disabled, NULL is returned and the consumer
362 * drivers will still build. Drivers are free to handle this specifically,
363 * but they don't have to.
365 * Return: icc_path pointer on success or ERR_PTR() on error. NULL is returned
366 * when the API is disabled or the "interconnects" DT property is missing.
368 struct icc_path
*of_icc_get(struct device
*dev
, const char *name
)
370 struct icc_path
*path
= ERR_PTR(-EPROBE_DEFER
);
371 struct icc_node
*src_node
, *dst_node
;
372 struct device_node
*np
= NULL
;
373 struct of_phandle_args src_args
, dst_args
;
377 if (!dev
|| !dev
->of_node
)
378 return ERR_PTR(-ENODEV
);
383 * When the consumer DT node do not have "interconnects" property
384 * return a NULL path to skip setting constraints.
386 if (!of_find_property(np
, "interconnects", NULL
))
390 * We use a combination of phandle and specifier for endpoint. For now
391 * lets support only global ids and extend this in the future if needed
392 * without breaking DT compatibility.
395 idx
= of_property_match_string(np
, "interconnect-names", name
);
400 ret
= of_parse_phandle_with_args(np
, "interconnects",
401 "#interconnect-cells", idx
* 2,
406 of_node_put(src_args
.np
);
408 ret
= of_parse_phandle_with_args(np
, "interconnects",
409 "#interconnect-cells", idx
* 2 + 1,
414 of_node_put(dst_args
.np
);
416 src_node
= of_icc_get_from_provider(&src_args
);
418 if (IS_ERR(src_node
)) {
419 if (PTR_ERR(src_node
) != -EPROBE_DEFER
)
420 dev_err(dev
, "error finding src node: %ld\n",
422 return ERR_CAST(src_node
);
425 dst_node
= of_icc_get_from_provider(&dst_args
);
427 if (IS_ERR(dst_node
)) {
428 if (PTR_ERR(dst_node
) != -EPROBE_DEFER
)
429 dev_err(dev
, "error finding dst node: %ld\n",
431 return ERR_CAST(dst_node
);
434 mutex_lock(&icc_lock
);
435 path
= path_find(dev
, src_node
, dst_node
);
436 mutex_unlock(&icc_lock
);
438 dev_err(dev
, "%s: invalid path=%ld\n", __func__
, PTR_ERR(path
));
443 path
->name
= kstrdup_const(name
, GFP_KERNEL
);
445 path
->name
= kasprintf(GFP_KERNEL
, "%s-%s",
446 src_node
->name
, dst_node
->name
);
450 return ERR_PTR(-ENOMEM
);
455 EXPORT_SYMBOL_GPL(of_icc_get
);
458 * icc_set_tag() - set an optional tag on a path
459 * @path: the path we want to tag
460 * @tag: the tag value
462 * This function allows consumers to append a tag to the requests associated
463 * with a path, so that a different aggregation could be done based on this tag.
465 void icc_set_tag(struct icc_path
*path
, u32 tag
)
472 mutex_lock(&icc_lock
);
474 for (i
= 0; i
< path
->num_nodes
; i
++)
475 path
->reqs
[i
].tag
= tag
;
477 mutex_unlock(&icc_lock
);
479 EXPORT_SYMBOL_GPL(icc_set_tag
);
482 * icc_set_bw() - set bandwidth constraints on an interconnect path
483 * @path: reference to the path returned by icc_get()
484 * @avg_bw: average bandwidth in kilobytes per second
485 * @peak_bw: peak bandwidth in kilobytes per second
487 * This function is used by an interconnect consumer to express its own needs
488 * in terms of bandwidth for a previously requested path between two endpoints.
489 * The requests are aggregated and each node is updated accordingly. The entire
490 * path is locked by a mutex to ensure that the set() is completed.
491 * The @path can be NULL when the "interconnects" DT properties is missing,
492 * which will mean that no constraints will be set.
494 * Returns 0 on success, or an appropriate error code otherwise.
496 int icc_set_bw(struct icc_path
*path
, u32 avg_bw
, u32 peak_bw
)
498 struct icc_node
*node
;
499 u32 old_avg
, old_peak
;
506 if (WARN_ON(IS_ERR(path
) || !path
->num_nodes
))
509 mutex_lock(&icc_lock
);
511 old_avg
= path
->reqs
[0].avg_bw
;
512 old_peak
= path
->reqs
[0].peak_bw
;
514 for (i
= 0; i
< path
->num_nodes
; i
++) {
515 node
= path
->reqs
[i
].node
;
517 /* update the consumer request for this path */
518 path
->reqs
[i
].avg_bw
= avg_bw
;
519 path
->reqs
[i
].peak_bw
= peak_bw
;
521 /* aggregate requests for this node */
522 aggregate_requests(node
);
524 trace_icc_set_bw(path
, node
, i
, avg_bw
, peak_bw
);
527 ret
= apply_constraints(path
);
529 pr_debug("interconnect: error applying constraints (%d)\n",
532 for (i
= 0; i
< path
->num_nodes
; i
++) {
533 node
= path
->reqs
[i
].node
;
534 path
->reqs
[i
].avg_bw
= old_avg
;
535 path
->reqs
[i
].peak_bw
= old_peak
;
536 aggregate_requests(node
);
538 apply_constraints(path
);
541 mutex_unlock(&icc_lock
);
543 trace_icc_set_bw_end(path
, ret
);
547 EXPORT_SYMBOL_GPL(icc_set_bw
);
550 * icc_get() - return a handle for path between two endpoints
551 * @dev: the device requesting the path
552 * @src_id: source device port id
553 * @dst_id: destination device port id
555 * This function will search for a path between two endpoints and return an
556 * icc_path handle on success. Use icc_put() to release
557 * constraints when they are not needed anymore.
558 * If the interconnect API is disabled, NULL is returned and the consumer
559 * drivers will still build. Drivers are free to handle this specifically,
560 * but they don't have to.
562 * Return: icc_path pointer on success, ERR_PTR() on error or NULL if the
563 * interconnect API is disabled.
565 struct icc_path
*icc_get(struct device
*dev
, const int src_id
, const int dst_id
)
567 struct icc_node
*src
, *dst
;
568 struct icc_path
*path
= ERR_PTR(-EPROBE_DEFER
);
570 mutex_lock(&icc_lock
);
572 src
= node_find(src_id
);
576 dst
= node_find(dst_id
);
580 path
= path_find(dev
, src
, dst
);
582 dev_err(dev
, "%s: invalid path=%ld\n", __func__
, PTR_ERR(path
));
586 path
->name
= kasprintf(GFP_KERNEL
, "%s-%s", src
->name
, dst
->name
);
589 path
= ERR_PTR(-ENOMEM
);
592 mutex_unlock(&icc_lock
);
595 EXPORT_SYMBOL_GPL(icc_get
);
598 * icc_put() - release the reference to the icc_path
599 * @path: interconnect path
601 * Use this function to release the constraints on a path when the path is
602 * no longer needed. The constraints will be re-aggregated.
604 void icc_put(struct icc_path
*path
)
606 struct icc_node
*node
;
610 if (!path
|| WARN_ON(IS_ERR(path
)))
613 ret
= icc_set_bw(path
, 0, 0);
615 pr_err("%s: error (%d)\n", __func__
, ret
);
617 mutex_lock(&icc_lock
);
618 for (i
= 0; i
< path
->num_nodes
; i
++) {
619 node
= path
->reqs
[i
].node
;
620 hlist_del(&path
->reqs
[i
].req_node
);
621 if (!WARN_ON(!node
->provider
->users
))
622 node
->provider
->users
--;
624 mutex_unlock(&icc_lock
);
626 kfree_const(path
->name
);
629 EXPORT_SYMBOL_GPL(icc_put
);
631 static struct icc_node
*icc_node_create_nolock(int id
)
633 struct icc_node
*node
;
635 /* check if node already exists */
636 node
= node_find(id
);
640 node
= kzalloc(sizeof(*node
), GFP_KERNEL
);
642 return ERR_PTR(-ENOMEM
);
644 id
= idr_alloc(&icc_idr
, node
, id
, id
+ 1, GFP_KERNEL
);
646 WARN(1, "%s: couldn't get idr\n", __func__
);
657 * icc_node_create() - create a node
660 * Return: icc_node pointer on success, or ERR_PTR() on error
662 struct icc_node
*icc_node_create(int id
)
664 struct icc_node
*node
;
666 mutex_lock(&icc_lock
);
668 node
= icc_node_create_nolock(id
);
670 mutex_unlock(&icc_lock
);
674 EXPORT_SYMBOL_GPL(icc_node_create
);
677 * icc_node_destroy() - destroy a node
680 void icc_node_destroy(int id
)
682 struct icc_node
*node
;
684 mutex_lock(&icc_lock
);
686 node
= node_find(id
);
688 idr_remove(&icc_idr
, node
->id
);
689 WARN_ON(!hlist_empty(&node
->req_list
));
692 mutex_unlock(&icc_lock
);
696 EXPORT_SYMBOL_GPL(icc_node_destroy
);
699 * icc_link_create() - create a link between two nodes
700 * @node: source node id
701 * @dst_id: destination node id
703 * Create a link between two nodes. The nodes might belong to different
704 * interconnect providers and the @dst_id node might not exist (if the
705 * provider driver has not probed yet). So just create the @dst_id node
706 * and when the actual provider driver is probed, the rest of the node
709 * Return: 0 on success, or an error code otherwise
711 int icc_link_create(struct icc_node
*node
, const int dst_id
)
713 struct icc_node
*dst
;
714 struct icc_node
**new;
720 mutex_lock(&icc_lock
);
722 dst
= node_find(dst_id
);
724 dst
= icc_node_create_nolock(dst_id
);
732 new = krealloc(node
->links
,
733 (node
->num_links
+ 1) * sizeof(*node
->links
),
741 node
->links
[node
->num_links
++] = dst
;
744 mutex_unlock(&icc_lock
);
748 EXPORT_SYMBOL_GPL(icc_link_create
);
751 * icc_link_destroy() - destroy a link between two nodes
752 * @src: pointer to source node
753 * @dst: pointer to destination node
755 * Return: 0 on success, or an error code otherwise
757 int icc_link_destroy(struct icc_node
*src
, struct icc_node
*dst
)
759 struct icc_node
**new;
763 if (IS_ERR_OR_NULL(src
))
766 if (IS_ERR_OR_NULL(dst
))
769 mutex_lock(&icc_lock
);
771 for (slot
= 0; slot
< src
->num_links
; slot
++)
772 if (src
->links
[slot
] == dst
)
775 if (WARN_ON(slot
== src
->num_links
)) {
780 src
->links
[slot
] = src
->links
[--src
->num_links
];
782 new = krealloc(src
->links
, src
->num_links
* sizeof(*src
->links
),
788 mutex_unlock(&icc_lock
);
792 EXPORT_SYMBOL_GPL(icc_link_destroy
);
795 * icc_node_add() - add interconnect node to interconnect provider
796 * @node: pointer to the interconnect node
797 * @provider: pointer to the interconnect provider
799 void icc_node_add(struct icc_node
*node
, struct icc_provider
*provider
)
801 mutex_lock(&icc_lock
);
803 node
->provider
= provider
;
804 list_add_tail(&node
->node_list
, &provider
->nodes
);
806 mutex_unlock(&icc_lock
);
808 EXPORT_SYMBOL_GPL(icc_node_add
);
811 * icc_node_del() - delete interconnect node from interconnect provider
812 * @node: pointer to the interconnect node
814 void icc_node_del(struct icc_node
*node
)
816 mutex_lock(&icc_lock
);
818 list_del(&node
->node_list
);
820 mutex_unlock(&icc_lock
);
822 EXPORT_SYMBOL_GPL(icc_node_del
);
825 * icc_nodes_remove() - remove all previously added nodes from provider
826 * @provider: the interconnect provider we are removing nodes from
828 * Return: 0 on success, or an error code otherwise
830 int icc_nodes_remove(struct icc_provider
*provider
)
832 struct icc_node
*n
, *tmp
;
834 if (WARN_ON(IS_ERR_OR_NULL(provider
)))
837 list_for_each_entry_safe_reverse(n
, tmp
, &provider
->nodes
, node_list
) {
839 icc_node_destroy(n
->id
);
844 EXPORT_SYMBOL_GPL(icc_nodes_remove
);
847 * icc_provider_add() - add a new interconnect provider
848 * @provider: the interconnect provider that will be added into topology
850 * Return: 0 on success, or an error code otherwise
852 int icc_provider_add(struct icc_provider
*provider
)
854 if (WARN_ON(!provider
->set
))
856 if (WARN_ON(!provider
->xlate
))
859 mutex_lock(&icc_lock
);
861 INIT_LIST_HEAD(&provider
->nodes
);
862 list_add_tail(&provider
->provider_list
, &icc_providers
);
864 mutex_unlock(&icc_lock
);
866 dev_dbg(provider
->dev
, "interconnect provider added to topology\n");
870 EXPORT_SYMBOL_GPL(icc_provider_add
);
873 * icc_provider_del() - delete previously added interconnect provider
874 * @provider: the interconnect provider that will be removed from topology
876 * Return: 0 on success, or an error code otherwise
878 int icc_provider_del(struct icc_provider
*provider
)
880 mutex_lock(&icc_lock
);
881 if (provider
->users
) {
882 pr_warn("interconnect provider still has %d users\n",
884 mutex_unlock(&icc_lock
);
888 if (!list_empty(&provider
->nodes
)) {
889 pr_warn("interconnect provider still has nodes\n");
890 mutex_unlock(&icc_lock
);
894 list_del(&provider
->provider_list
);
895 mutex_unlock(&icc_lock
);
899 EXPORT_SYMBOL_GPL(icc_provider_del
);
901 static int __init
icc_init(void)
903 icc_debugfs_dir
= debugfs_create_dir("interconnect", NULL
);
904 debugfs_create_file("interconnect_summary", 0444,
905 icc_debugfs_dir
, NULL
, &icc_summary_fops
);
906 debugfs_create_file("interconnect_graph", 0444,
907 icc_debugfs_dir
, NULL
, &icc_graph_fops
);
911 static void __exit
icc_exit(void)
913 debugfs_remove_recursive(icc_debugfs_dir
);
915 module_init(icc_init
);
916 module_exit(icc_exit
);
918 MODULE_AUTHOR("Georgi Djakov <georgi.djakov@linaro.org>");
919 MODULE_DESCRIPTION("Interconnect Driver Core");
920 MODULE_LICENSE("GPL v2");