1 /* The industrial I/O core
3 * Copyright (c) 2008 Jonathan Cameron
5 * This program is free software; you can redistribute it and/or modify it
6 * under the terms of the GNU General Public License version 2 as published by
7 * the Free Software Foundation.
9 * Handling of buffer allocation / resizing.
12 * Things to look at here.
13 * - Better memory allocation techniques?
14 * - Alternative access techniques?
16 #include <linux/kernel.h>
17 #include <linux/export.h>
18 #include <linux/device.h>
20 #include <linux/cdev.h>
21 #include <linux/slab.h>
22 #include <linux/poll.h>
23 #include <linux/sched/signal.h>
25 #include <linux/iio/iio.h>
27 #include <linux/iio/sysfs.h>
28 #include <linux/iio/buffer.h>
29 #include <linux/iio/buffer_impl.h>
31 static const char * const iio_endian_prefix
[] = {
36 static bool iio_buffer_is_active(struct iio_buffer
*buf
)
38 return !list_empty(&buf
->buffer_list
);
41 static size_t iio_buffer_data_available(struct iio_buffer
*buf
)
43 return buf
->access
->data_available(buf
);
46 static int iio_buffer_flush_hwfifo(struct iio_dev
*indio_dev
,
47 struct iio_buffer
*buf
, size_t required
)
49 if (!indio_dev
->info
->hwfifo_flush_to_buffer
)
52 return indio_dev
->info
->hwfifo_flush_to_buffer(indio_dev
, required
);
55 static bool iio_buffer_ready(struct iio_dev
*indio_dev
, struct iio_buffer
*buf
,
56 size_t to_wait
, int to_flush
)
61 /* wakeup if the device was unregistered */
65 /* drain the buffer if it was disabled */
66 if (!iio_buffer_is_active(buf
)) {
67 to_wait
= min_t(size_t, to_wait
, 1);
71 avail
= iio_buffer_data_available(buf
);
73 if (avail
>= to_wait
) {
74 /* force a flush for non-blocking reads */
75 if (!to_wait
&& avail
< to_flush
)
76 iio_buffer_flush_hwfifo(indio_dev
, buf
,
82 flushed
= iio_buffer_flush_hwfifo(indio_dev
, buf
,
87 if (avail
+ flushed
>= to_wait
)
94 * iio_buffer_read_first_n_outer() - chrdev read for buffer access
95 * @filp: File structure pointer for the char device
96 * @buf: Destination buffer for iio buffer read
97 * @n: First n bytes to read
98 * @f_ps: Long offset provided by the user as a seek position
100 * This function relies on all buffer implementations having an
101 * iio_buffer as their first element.
103 * Return: negative values corresponding to error codes or ret != 0
104 * for ending the reading activity
106 ssize_t
iio_buffer_read_first_n_outer(struct file
*filp
, char __user
*buf
,
107 size_t n
, loff_t
*f_ps
)
109 struct iio_dev
*indio_dev
= filp
->private_data
;
110 struct iio_buffer
*rb
= indio_dev
->buffer
;
111 DEFINE_WAIT_FUNC(wait
, woken_wake_function
);
116 if (!indio_dev
->info
)
119 if (!rb
|| !rb
->access
->read_first_n
)
122 datum_size
= rb
->bytes_per_datum
;
125 * If datum_size is 0 there will never be anything to read from the
126 * buffer, so signal end of file now.
131 if (filp
->f_flags
& O_NONBLOCK
)
134 to_wait
= min_t(size_t, n
/ datum_size
, rb
->watermark
);
136 add_wait_queue(&rb
->pollq
, &wait
);
138 if (!indio_dev
->info
) {
143 if (!iio_buffer_ready(indio_dev
, rb
, to_wait
, n
/ datum_size
)) {
144 if (signal_pending(current
)) {
149 wait_woken(&wait
, TASK_INTERRUPTIBLE
,
150 MAX_SCHEDULE_TIMEOUT
);
154 ret
= rb
->access
->read_first_n(rb
, n
, buf
);
155 if (ret
== 0 && (filp
->f_flags
& O_NONBLOCK
))
158 remove_wait_queue(&rb
->pollq
, &wait
);
164 * iio_buffer_poll() - poll the buffer to find out if it has data
165 * @filp: File structure pointer for device access
166 * @wait: Poll table structure pointer for which the driver adds
169 * Return: (EPOLLIN | EPOLLRDNORM) if data is available for reading
170 * or 0 for other cases
172 __poll_t
iio_buffer_poll(struct file
*filp
,
173 struct poll_table_struct
*wait
)
175 struct iio_dev
*indio_dev
= filp
->private_data
;
176 struct iio_buffer
*rb
= indio_dev
->buffer
;
178 if (!indio_dev
->info
|| rb
== NULL
)
181 poll_wait(filp
, &rb
->pollq
, wait
);
182 if (iio_buffer_ready(indio_dev
, rb
, rb
->watermark
, 0))
183 return EPOLLIN
| EPOLLRDNORM
;
188 * iio_buffer_wakeup_poll - Wakes up the buffer waitqueue
189 * @indio_dev: The IIO device
191 * Wakes up the event waitqueue used for poll(). Should usually
192 * be called when the device is unregistered.
194 void iio_buffer_wakeup_poll(struct iio_dev
*indio_dev
)
196 if (!indio_dev
->buffer
)
199 wake_up(&indio_dev
->buffer
->pollq
);
202 void iio_buffer_init(struct iio_buffer
*buffer
)
204 INIT_LIST_HEAD(&buffer
->demux_list
);
205 INIT_LIST_HEAD(&buffer
->buffer_list
);
206 init_waitqueue_head(&buffer
->pollq
);
207 kref_init(&buffer
->ref
);
208 if (!buffer
->watermark
)
209 buffer
->watermark
= 1;
211 EXPORT_SYMBOL(iio_buffer_init
);
214 * iio_buffer_set_attrs - Set buffer specific attributes
215 * @buffer: The buffer for which we are setting attributes
216 * @attrs: Pointer to a null terminated list of pointers to attributes
218 void iio_buffer_set_attrs(struct iio_buffer
*buffer
,
219 const struct attribute
**attrs
)
221 buffer
->attrs
= attrs
;
223 EXPORT_SYMBOL_GPL(iio_buffer_set_attrs
);
225 static ssize_t
iio_show_scan_index(struct device
*dev
,
226 struct device_attribute
*attr
,
229 return sprintf(buf
, "%u\n", to_iio_dev_attr(attr
)->c
->scan_index
);
232 static ssize_t
iio_show_fixed_type(struct device
*dev
,
233 struct device_attribute
*attr
,
236 struct iio_dev_attr
*this_attr
= to_iio_dev_attr(attr
);
237 u8 type
= this_attr
->c
->scan_type
.endianness
;
239 if (type
== IIO_CPU
) {
240 #ifdef __LITTLE_ENDIAN
246 if (this_attr
->c
->scan_type
.repeat
> 1)
247 return sprintf(buf
, "%s:%c%d/%dX%d>>%u\n",
248 iio_endian_prefix
[type
],
249 this_attr
->c
->scan_type
.sign
,
250 this_attr
->c
->scan_type
.realbits
,
251 this_attr
->c
->scan_type
.storagebits
,
252 this_attr
->c
->scan_type
.repeat
,
253 this_attr
->c
->scan_type
.shift
);
255 return sprintf(buf
, "%s:%c%d/%d>>%u\n",
256 iio_endian_prefix
[type
],
257 this_attr
->c
->scan_type
.sign
,
258 this_attr
->c
->scan_type
.realbits
,
259 this_attr
->c
->scan_type
.storagebits
,
260 this_attr
->c
->scan_type
.shift
);
263 static ssize_t
iio_scan_el_show(struct device
*dev
,
264 struct device_attribute
*attr
,
268 struct iio_dev
*indio_dev
= dev_to_iio_dev(dev
);
270 /* Ensure ret is 0 or 1. */
271 ret
= !!test_bit(to_iio_dev_attr(attr
)->address
,
272 indio_dev
->buffer
->scan_mask
);
274 return sprintf(buf
, "%d\n", ret
);
277 /* Note NULL used as error indicator as it doesn't make sense. */
278 static const unsigned long *iio_scan_mask_match(const unsigned long *av_masks
,
279 unsigned int masklength
,
280 const unsigned long *mask
,
283 if (bitmap_empty(mask
, masklength
))
287 if (bitmap_equal(mask
, av_masks
, masklength
))
290 if (bitmap_subset(mask
, av_masks
, masklength
))
293 av_masks
+= BITS_TO_LONGS(masklength
);
298 static bool iio_validate_scan_mask(struct iio_dev
*indio_dev
,
299 const unsigned long *mask
)
301 if (!indio_dev
->setup_ops
->validate_scan_mask
)
304 return indio_dev
->setup_ops
->validate_scan_mask(indio_dev
, mask
);
308 * iio_scan_mask_set() - set particular bit in the scan mask
309 * @indio_dev: the iio device
310 * @buffer: the buffer whose scan mask we are interested in
311 * @bit: the bit to be set.
313 * Note that at this point we have no way of knowing what other
314 * buffers might request, hence this code only verifies that the
315 * individual buffers request is plausible.
317 static int iio_scan_mask_set(struct iio_dev
*indio_dev
,
318 struct iio_buffer
*buffer
, int bit
)
320 const unsigned long *mask
;
321 unsigned long *trialmask
;
323 trialmask
= kcalloc(BITS_TO_LONGS(indio_dev
->masklength
),
324 sizeof(*trialmask
), GFP_KERNEL
);
325 if (trialmask
== NULL
)
327 if (!indio_dev
->masklength
) {
328 WARN(1, "Trying to set scanmask prior to registering buffer\n");
329 goto err_invalid_mask
;
331 bitmap_copy(trialmask
, buffer
->scan_mask
, indio_dev
->masklength
);
332 set_bit(bit
, trialmask
);
334 if (!iio_validate_scan_mask(indio_dev
, trialmask
))
335 goto err_invalid_mask
;
337 if (indio_dev
->available_scan_masks
) {
338 mask
= iio_scan_mask_match(indio_dev
->available_scan_masks
,
339 indio_dev
->masklength
,
342 goto err_invalid_mask
;
344 bitmap_copy(buffer
->scan_mask
, trialmask
, indio_dev
->masklength
);
355 static int iio_scan_mask_clear(struct iio_buffer
*buffer
, int bit
)
357 clear_bit(bit
, buffer
->scan_mask
);
361 static int iio_scan_mask_query(struct iio_dev
*indio_dev
,
362 struct iio_buffer
*buffer
, int bit
)
364 if (bit
> indio_dev
->masklength
)
367 if (!buffer
->scan_mask
)
370 /* Ensure return value is 0 or 1. */
371 return !!test_bit(bit
, buffer
->scan_mask
);
374 static ssize_t
iio_scan_el_store(struct device
*dev
,
375 struct device_attribute
*attr
,
381 struct iio_dev
*indio_dev
= dev_to_iio_dev(dev
);
382 struct iio_buffer
*buffer
= indio_dev
->buffer
;
383 struct iio_dev_attr
*this_attr
= to_iio_dev_attr(attr
);
385 ret
= strtobool(buf
, &state
);
388 mutex_lock(&indio_dev
->mlock
);
389 if (iio_buffer_is_active(indio_dev
->buffer
)) {
393 ret
= iio_scan_mask_query(indio_dev
, buffer
, this_attr
->address
);
397 ret
= iio_scan_mask_clear(buffer
, this_attr
->address
);
400 } else if (state
&& !ret
) {
401 ret
= iio_scan_mask_set(indio_dev
, buffer
, this_attr
->address
);
407 mutex_unlock(&indio_dev
->mlock
);
409 return ret
< 0 ? ret
: len
;
413 static ssize_t
iio_scan_el_ts_show(struct device
*dev
,
414 struct device_attribute
*attr
,
417 struct iio_dev
*indio_dev
= dev_to_iio_dev(dev
);
418 return sprintf(buf
, "%d\n", indio_dev
->buffer
->scan_timestamp
);
421 static ssize_t
iio_scan_el_ts_store(struct device
*dev
,
422 struct device_attribute
*attr
,
427 struct iio_dev
*indio_dev
= dev_to_iio_dev(dev
);
430 ret
= strtobool(buf
, &state
);
434 mutex_lock(&indio_dev
->mlock
);
435 if (iio_buffer_is_active(indio_dev
->buffer
)) {
439 indio_dev
->buffer
->scan_timestamp
= state
;
441 mutex_unlock(&indio_dev
->mlock
);
443 return ret
? ret
: len
;
446 static int iio_buffer_add_channel_sysfs(struct iio_dev
*indio_dev
,
447 const struct iio_chan_spec
*chan
)
449 int ret
, attrcount
= 0;
450 struct iio_buffer
*buffer
= indio_dev
->buffer
;
452 ret
= __iio_add_chan_devattr("index",
454 &iio_show_scan_index
,
459 &buffer
->scan_el_dev_attr_list
);
463 ret
= __iio_add_chan_devattr("type",
465 &iio_show_fixed_type
,
470 &buffer
->scan_el_dev_attr_list
);
474 if (chan
->type
!= IIO_TIMESTAMP
)
475 ret
= __iio_add_chan_devattr("en",
482 &buffer
->scan_el_dev_attr_list
);
484 ret
= __iio_add_chan_devattr("en",
486 &iio_scan_el_ts_show
,
487 &iio_scan_el_ts_store
,
491 &buffer
->scan_el_dev_attr_list
);
499 static ssize_t
iio_buffer_read_length(struct device
*dev
,
500 struct device_attribute
*attr
,
503 struct iio_dev
*indio_dev
= dev_to_iio_dev(dev
);
504 struct iio_buffer
*buffer
= indio_dev
->buffer
;
506 return sprintf(buf
, "%d\n", buffer
->length
);
509 static ssize_t
iio_buffer_write_length(struct device
*dev
,
510 struct device_attribute
*attr
,
511 const char *buf
, size_t len
)
513 struct iio_dev
*indio_dev
= dev_to_iio_dev(dev
);
514 struct iio_buffer
*buffer
= indio_dev
->buffer
;
518 ret
= kstrtouint(buf
, 10, &val
);
522 if (val
== buffer
->length
)
525 mutex_lock(&indio_dev
->mlock
);
526 if (iio_buffer_is_active(indio_dev
->buffer
)) {
529 buffer
->access
->set_length(buffer
, val
);
534 if (buffer
->length
&& buffer
->length
< buffer
->watermark
)
535 buffer
->watermark
= buffer
->length
;
537 mutex_unlock(&indio_dev
->mlock
);
539 return ret
? ret
: len
;
542 static ssize_t
iio_buffer_show_enable(struct device
*dev
,
543 struct device_attribute
*attr
,
546 struct iio_dev
*indio_dev
= dev_to_iio_dev(dev
);
547 return sprintf(buf
, "%d\n", iio_buffer_is_active(indio_dev
->buffer
));
550 static unsigned int iio_storage_bytes_for_si(struct iio_dev
*indio_dev
,
551 unsigned int scan_index
)
553 const struct iio_chan_spec
*ch
;
556 ch
= iio_find_channel_from_si(indio_dev
, scan_index
);
557 bytes
= ch
->scan_type
.storagebits
/ 8;
558 if (ch
->scan_type
.repeat
> 1)
559 bytes
*= ch
->scan_type
.repeat
;
563 static unsigned int iio_storage_bytes_for_timestamp(struct iio_dev
*indio_dev
)
565 return iio_storage_bytes_for_si(indio_dev
,
566 indio_dev
->scan_index_timestamp
);
569 static int iio_compute_scan_bytes(struct iio_dev
*indio_dev
,
570 const unsigned long *mask
, bool timestamp
)
573 int length
, i
, largest
= 0;
575 /* How much space will the demuxed element take? */
576 for_each_set_bit(i
, mask
,
577 indio_dev
->masklength
) {
578 length
= iio_storage_bytes_for_si(indio_dev
, i
);
579 bytes
= ALIGN(bytes
, length
);
581 largest
= max(largest
, length
);
585 length
= iio_storage_bytes_for_timestamp(indio_dev
);
586 bytes
= ALIGN(bytes
, length
);
588 largest
= max(largest
, length
);
591 bytes
= ALIGN(bytes
, largest
);
595 static void iio_buffer_activate(struct iio_dev
*indio_dev
,
596 struct iio_buffer
*buffer
)
598 iio_buffer_get(buffer
);
599 list_add(&buffer
->buffer_list
, &indio_dev
->buffer_list
);
602 static void iio_buffer_deactivate(struct iio_buffer
*buffer
)
604 list_del_init(&buffer
->buffer_list
);
605 wake_up_interruptible(&buffer
->pollq
);
606 iio_buffer_put(buffer
);
609 static void iio_buffer_deactivate_all(struct iio_dev
*indio_dev
)
611 struct iio_buffer
*buffer
, *_buffer
;
613 list_for_each_entry_safe(buffer
, _buffer
,
614 &indio_dev
->buffer_list
, buffer_list
)
615 iio_buffer_deactivate(buffer
);
618 static int iio_buffer_enable(struct iio_buffer
*buffer
,
619 struct iio_dev
*indio_dev
)
621 if (!buffer
->access
->enable
)
623 return buffer
->access
->enable(buffer
, indio_dev
);
626 static int iio_buffer_disable(struct iio_buffer
*buffer
,
627 struct iio_dev
*indio_dev
)
629 if (!buffer
->access
->disable
)
631 return buffer
->access
->disable(buffer
, indio_dev
);
634 static void iio_buffer_update_bytes_per_datum(struct iio_dev
*indio_dev
,
635 struct iio_buffer
*buffer
)
639 if (!buffer
->access
->set_bytes_per_datum
)
642 bytes
= iio_compute_scan_bytes(indio_dev
, buffer
->scan_mask
,
643 buffer
->scan_timestamp
);
645 buffer
->access
->set_bytes_per_datum(buffer
, bytes
);
648 static int iio_buffer_request_update(struct iio_dev
*indio_dev
,
649 struct iio_buffer
*buffer
)
653 iio_buffer_update_bytes_per_datum(indio_dev
, buffer
);
654 if (buffer
->access
->request_update
) {
655 ret
= buffer
->access
->request_update(buffer
);
657 dev_dbg(&indio_dev
->dev
,
658 "Buffer not started: buffer parameter update failed (%d)\n",
667 static void iio_free_scan_mask(struct iio_dev
*indio_dev
,
668 const unsigned long *mask
)
670 /* If the mask is dynamically allocated free it, otherwise do nothing */
671 if (!indio_dev
->available_scan_masks
)
675 struct iio_device_config
{
677 unsigned int watermark
;
678 const unsigned long *scan_mask
;
679 unsigned int scan_bytes
;
683 static int iio_verify_update(struct iio_dev
*indio_dev
,
684 struct iio_buffer
*insert_buffer
, struct iio_buffer
*remove_buffer
,
685 struct iio_device_config
*config
)
687 unsigned long *compound_mask
;
688 const unsigned long *scan_mask
;
689 bool strict_scanmask
= false;
690 struct iio_buffer
*buffer
;
694 memset(config
, 0, sizeof(*config
));
695 config
->watermark
= ~0;
698 * If there is just one buffer and we are removing it there is nothing
701 if (remove_buffer
&& !insert_buffer
&&
702 list_is_singular(&indio_dev
->buffer_list
))
705 modes
= indio_dev
->modes
;
707 list_for_each_entry(buffer
, &indio_dev
->buffer_list
, buffer_list
) {
708 if (buffer
== remove_buffer
)
710 modes
&= buffer
->access
->modes
;
711 config
->watermark
= min(config
->watermark
, buffer
->watermark
);
715 modes
&= insert_buffer
->access
->modes
;
716 config
->watermark
= min(config
->watermark
,
717 insert_buffer
->watermark
);
720 /* Definitely possible for devices to support both of these. */
721 if ((modes
& INDIO_BUFFER_TRIGGERED
) && indio_dev
->trig
) {
722 config
->mode
= INDIO_BUFFER_TRIGGERED
;
723 } else if (modes
& INDIO_BUFFER_HARDWARE
) {
725 * Keep things simple for now and only allow a single buffer to
726 * be connected in hardware mode.
728 if (insert_buffer
&& !list_empty(&indio_dev
->buffer_list
))
730 config
->mode
= INDIO_BUFFER_HARDWARE
;
731 strict_scanmask
= true;
732 } else if (modes
& INDIO_BUFFER_SOFTWARE
) {
733 config
->mode
= INDIO_BUFFER_SOFTWARE
;
735 /* Can only occur on first buffer */
736 if (indio_dev
->modes
& INDIO_BUFFER_TRIGGERED
)
737 dev_dbg(&indio_dev
->dev
, "Buffer not started: no trigger\n");
741 /* What scan mask do we actually have? */
742 compound_mask
= kcalloc(BITS_TO_LONGS(indio_dev
->masklength
),
743 sizeof(long), GFP_KERNEL
);
744 if (compound_mask
== NULL
)
747 scan_timestamp
= false;
749 list_for_each_entry(buffer
, &indio_dev
->buffer_list
, buffer_list
) {
750 if (buffer
== remove_buffer
)
752 bitmap_or(compound_mask
, compound_mask
, buffer
->scan_mask
,
753 indio_dev
->masklength
);
754 scan_timestamp
|= buffer
->scan_timestamp
;
758 bitmap_or(compound_mask
, compound_mask
,
759 insert_buffer
->scan_mask
, indio_dev
->masklength
);
760 scan_timestamp
|= insert_buffer
->scan_timestamp
;
763 if (indio_dev
->available_scan_masks
) {
764 scan_mask
= iio_scan_mask_match(indio_dev
->available_scan_masks
,
765 indio_dev
->masklength
,
768 kfree(compound_mask
);
769 if (scan_mask
== NULL
)
772 scan_mask
= compound_mask
;
775 config
->scan_bytes
= iio_compute_scan_bytes(indio_dev
,
776 scan_mask
, scan_timestamp
);
777 config
->scan_mask
= scan_mask
;
778 config
->scan_timestamp
= scan_timestamp
;
784 * struct iio_demux_table - table describing demux memcpy ops
785 * @from: index to copy from
786 * @to: index to copy to
787 * @length: how many bytes to copy
788 * @l: list head used for management
790 struct iio_demux_table
{
797 static void iio_buffer_demux_free(struct iio_buffer
*buffer
)
799 struct iio_demux_table
*p
, *q
;
800 list_for_each_entry_safe(p
, q
, &buffer
->demux_list
, l
) {
806 static int iio_buffer_add_demux(struct iio_buffer
*buffer
,
807 struct iio_demux_table
**p
, unsigned int in_loc
, unsigned int out_loc
,
811 if (*p
&& (*p
)->from
+ (*p
)->length
== in_loc
&&
812 (*p
)->to
+ (*p
)->length
== out_loc
) {
813 (*p
)->length
+= length
;
815 *p
= kmalloc(sizeof(**p
), GFP_KERNEL
);
820 (*p
)->length
= length
;
821 list_add_tail(&(*p
)->l
, &buffer
->demux_list
);
827 static int iio_buffer_update_demux(struct iio_dev
*indio_dev
,
828 struct iio_buffer
*buffer
)
830 int ret
, in_ind
= -1, out_ind
, length
;
831 unsigned in_loc
= 0, out_loc
= 0;
832 struct iio_demux_table
*p
= NULL
;
834 /* Clear out any old demux */
835 iio_buffer_demux_free(buffer
);
836 kfree(buffer
->demux_bounce
);
837 buffer
->demux_bounce
= NULL
;
839 /* First work out which scan mode we will actually have */
840 if (bitmap_equal(indio_dev
->active_scan_mask
,
842 indio_dev
->masklength
))
845 /* Now we have the two masks, work from least sig and build up sizes */
846 for_each_set_bit(out_ind
,
848 indio_dev
->masklength
) {
849 in_ind
= find_next_bit(indio_dev
->active_scan_mask
,
850 indio_dev
->masklength
,
852 while (in_ind
!= out_ind
) {
853 in_ind
= find_next_bit(indio_dev
->active_scan_mask
,
854 indio_dev
->masklength
,
856 length
= iio_storage_bytes_for_si(indio_dev
, in_ind
);
857 /* Make sure we are aligned */
858 in_loc
= roundup(in_loc
, length
) + length
;
860 length
= iio_storage_bytes_for_si(indio_dev
, in_ind
);
861 out_loc
= roundup(out_loc
, length
);
862 in_loc
= roundup(in_loc
, length
);
863 ret
= iio_buffer_add_demux(buffer
, &p
, in_loc
, out_loc
, length
);
865 goto error_clear_mux_table
;
869 /* Relies on scan_timestamp being last */
870 if (buffer
->scan_timestamp
) {
871 length
= iio_storage_bytes_for_timestamp(indio_dev
);
872 out_loc
= roundup(out_loc
, length
);
873 in_loc
= roundup(in_loc
, length
);
874 ret
= iio_buffer_add_demux(buffer
, &p
, in_loc
, out_loc
, length
);
876 goto error_clear_mux_table
;
880 buffer
->demux_bounce
= kzalloc(out_loc
, GFP_KERNEL
);
881 if (buffer
->demux_bounce
== NULL
) {
883 goto error_clear_mux_table
;
887 error_clear_mux_table
:
888 iio_buffer_demux_free(buffer
);
893 static int iio_update_demux(struct iio_dev
*indio_dev
)
895 struct iio_buffer
*buffer
;
898 list_for_each_entry(buffer
, &indio_dev
->buffer_list
, buffer_list
) {
899 ret
= iio_buffer_update_demux(indio_dev
, buffer
);
901 goto error_clear_mux_table
;
905 error_clear_mux_table
:
906 list_for_each_entry(buffer
, &indio_dev
->buffer_list
, buffer_list
)
907 iio_buffer_demux_free(buffer
);
912 static int iio_enable_buffers(struct iio_dev
*indio_dev
,
913 struct iio_device_config
*config
)
915 struct iio_buffer
*buffer
;
918 indio_dev
->active_scan_mask
= config
->scan_mask
;
919 indio_dev
->scan_timestamp
= config
->scan_timestamp
;
920 indio_dev
->scan_bytes
= config
->scan_bytes
;
922 iio_update_demux(indio_dev
);
925 if (indio_dev
->setup_ops
->preenable
) {
926 ret
= indio_dev
->setup_ops
->preenable(indio_dev
);
928 dev_dbg(&indio_dev
->dev
,
929 "Buffer not started: buffer preenable failed (%d)\n", ret
);
930 goto err_undo_config
;
934 if (indio_dev
->info
->update_scan_mode
) {
935 ret
= indio_dev
->info
936 ->update_scan_mode(indio_dev
,
937 indio_dev
->active_scan_mask
);
939 dev_dbg(&indio_dev
->dev
,
940 "Buffer not started: update scan mode failed (%d)\n",
942 goto err_run_postdisable
;
946 if (indio_dev
->info
->hwfifo_set_watermark
)
947 indio_dev
->info
->hwfifo_set_watermark(indio_dev
,
950 list_for_each_entry(buffer
, &indio_dev
->buffer_list
, buffer_list
) {
951 ret
= iio_buffer_enable(buffer
, indio_dev
);
953 goto err_disable_buffers
;
956 indio_dev
->currentmode
= config
->mode
;
958 if (indio_dev
->setup_ops
->postenable
) {
959 ret
= indio_dev
->setup_ops
->postenable(indio_dev
);
961 dev_dbg(&indio_dev
->dev
,
962 "Buffer not started: postenable failed (%d)\n", ret
);
963 goto err_disable_buffers
;
970 list_for_each_entry_continue_reverse(buffer
, &indio_dev
->buffer_list
,
972 iio_buffer_disable(buffer
, indio_dev
);
974 indio_dev
->currentmode
= INDIO_DIRECT_MODE
;
975 if (indio_dev
->setup_ops
->postdisable
)
976 indio_dev
->setup_ops
->postdisable(indio_dev
);
978 indio_dev
->active_scan_mask
= NULL
;
983 static int iio_disable_buffers(struct iio_dev
*indio_dev
)
985 struct iio_buffer
*buffer
;
989 /* Wind down existing buffers - iff there are any */
990 if (list_empty(&indio_dev
->buffer_list
))
994 * If things go wrong at some step in disable we still need to continue
995 * to perform the other steps, otherwise we leave the device in a
996 * inconsistent state. We return the error code for the first error we
1000 if (indio_dev
->setup_ops
->predisable
) {
1001 ret2
= indio_dev
->setup_ops
->predisable(indio_dev
);
1006 list_for_each_entry(buffer
, &indio_dev
->buffer_list
, buffer_list
) {
1007 ret2
= iio_buffer_disable(buffer
, indio_dev
);
1012 indio_dev
->currentmode
= INDIO_DIRECT_MODE
;
1014 if (indio_dev
->setup_ops
->postdisable
) {
1015 ret2
= indio_dev
->setup_ops
->postdisable(indio_dev
);
1020 iio_free_scan_mask(indio_dev
, indio_dev
->active_scan_mask
);
1021 indio_dev
->active_scan_mask
= NULL
;
1026 static int __iio_update_buffers(struct iio_dev
*indio_dev
,
1027 struct iio_buffer
*insert_buffer
,
1028 struct iio_buffer
*remove_buffer
)
1030 struct iio_device_config new_config
;
1033 ret
= iio_verify_update(indio_dev
, insert_buffer
, remove_buffer
,
1038 if (insert_buffer
) {
1039 ret
= iio_buffer_request_update(indio_dev
, insert_buffer
);
1041 goto err_free_config
;
1044 ret
= iio_disable_buffers(indio_dev
);
1046 goto err_deactivate_all
;
1049 iio_buffer_deactivate(remove_buffer
);
1051 iio_buffer_activate(indio_dev
, insert_buffer
);
1053 /* If no buffers in list, we are done */
1054 if (list_empty(&indio_dev
->buffer_list
))
1057 ret
= iio_enable_buffers(indio_dev
, &new_config
);
1059 goto err_deactivate_all
;
1065 * We've already verified that the config is valid earlier. If things go
1066 * wrong in either enable or disable the most likely reason is an IO
1067 * error from the device. In this case there is no good recovery
1068 * strategy. Just make sure to disable everything and leave the device
1069 * in a sane state. With a bit of luck the device might come back to
1070 * life again later and userspace can try again.
1072 iio_buffer_deactivate_all(indio_dev
);
1075 iio_free_scan_mask(indio_dev
, new_config
.scan_mask
);
1079 int iio_update_buffers(struct iio_dev
*indio_dev
,
1080 struct iio_buffer
*insert_buffer
,
1081 struct iio_buffer
*remove_buffer
)
1085 if (insert_buffer
== remove_buffer
)
1088 mutex_lock(&indio_dev
->info_exist_lock
);
1089 mutex_lock(&indio_dev
->mlock
);
1091 if (insert_buffer
&& iio_buffer_is_active(insert_buffer
))
1092 insert_buffer
= NULL
;
1094 if (remove_buffer
&& !iio_buffer_is_active(remove_buffer
))
1095 remove_buffer
= NULL
;
1097 if (!insert_buffer
&& !remove_buffer
) {
1102 if (indio_dev
->info
== NULL
) {
1107 ret
= __iio_update_buffers(indio_dev
, insert_buffer
, remove_buffer
);
1110 mutex_unlock(&indio_dev
->mlock
);
1111 mutex_unlock(&indio_dev
->info_exist_lock
);
1115 EXPORT_SYMBOL_GPL(iio_update_buffers
);
1117 void iio_disable_all_buffers(struct iio_dev
*indio_dev
)
1119 iio_disable_buffers(indio_dev
);
1120 iio_buffer_deactivate_all(indio_dev
);
1123 static ssize_t
iio_buffer_store_enable(struct device
*dev
,
1124 struct device_attribute
*attr
,
1129 bool requested_state
;
1130 struct iio_dev
*indio_dev
= dev_to_iio_dev(dev
);
1133 ret
= strtobool(buf
, &requested_state
);
1137 mutex_lock(&indio_dev
->mlock
);
1139 /* Find out if it is in the list */
1140 inlist
= iio_buffer_is_active(indio_dev
->buffer
);
1141 /* Already in desired state */
1142 if (inlist
== requested_state
)
1145 if (requested_state
)
1146 ret
= __iio_update_buffers(indio_dev
,
1147 indio_dev
->buffer
, NULL
);
1149 ret
= __iio_update_buffers(indio_dev
,
1150 NULL
, indio_dev
->buffer
);
1153 mutex_unlock(&indio_dev
->mlock
);
1154 return (ret
< 0) ? ret
: len
;
1157 static const char * const iio_scan_elements_group_name
= "scan_elements";
1159 static ssize_t
iio_buffer_show_watermark(struct device
*dev
,
1160 struct device_attribute
*attr
,
1163 struct iio_dev
*indio_dev
= dev_to_iio_dev(dev
);
1164 struct iio_buffer
*buffer
= indio_dev
->buffer
;
1166 return sprintf(buf
, "%u\n", buffer
->watermark
);
1169 static ssize_t
iio_buffer_store_watermark(struct device
*dev
,
1170 struct device_attribute
*attr
,
1174 struct iio_dev
*indio_dev
= dev_to_iio_dev(dev
);
1175 struct iio_buffer
*buffer
= indio_dev
->buffer
;
1179 ret
= kstrtouint(buf
, 10, &val
);
1185 mutex_lock(&indio_dev
->mlock
);
1187 if (val
> buffer
->length
) {
1192 if (iio_buffer_is_active(indio_dev
->buffer
)) {
1197 buffer
->watermark
= val
;
1199 mutex_unlock(&indio_dev
->mlock
);
1201 return ret
? ret
: len
;
1204 static ssize_t
iio_dma_show_data_available(struct device
*dev
,
1205 struct device_attribute
*attr
,
1208 struct iio_dev
*indio_dev
= dev_to_iio_dev(dev
);
1211 bytes
= iio_buffer_data_available(indio_dev
->buffer
);
1213 return sprintf(buf
, "%zu\n", bytes
);
1216 static DEVICE_ATTR(length
, S_IRUGO
| S_IWUSR
, iio_buffer_read_length
,
1217 iio_buffer_write_length
);
1218 static struct device_attribute dev_attr_length_ro
= __ATTR(length
,
1219 S_IRUGO
, iio_buffer_read_length
, NULL
);
1220 static DEVICE_ATTR(enable
, S_IRUGO
| S_IWUSR
,
1221 iio_buffer_show_enable
, iio_buffer_store_enable
);
1222 static DEVICE_ATTR(watermark
, S_IRUGO
| S_IWUSR
,
1223 iio_buffer_show_watermark
, iio_buffer_store_watermark
);
1224 static struct device_attribute dev_attr_watermark_ro
= __ATTR(watermark
,
1225 S_IRUGO
, iio_buffer_show_watermark
, NULL
);
1226 static DEVICE_ATTR(data_available
, S_IRUGO
,
1227 iio_dma_show_data_available
, NULL
);
1229 static struct attribute
*iio_buffer_attrs
[] = {
1230 &dev_attr_length
.attr
,
1231 &dev_attr_enable
.attr
,
1232 &dev_attr_watermark
.attr
,
1233 &dev_attr_data_available
.attr
,
1236 int iio_buffer_alloc_sysfs_and_mask(struct iio_dev
*indio_dev
)
1238 struct iio_dev_attr
*p
;
1239 struct attribute
**attr
;
1240 struct iio_buffer
*buffer
= indio_dev
->buffer
;
1241 int ret
, i
, attrn
, attrcount
, attrcount_orig
= 0;
1242 const struct iio_chan_spec
*channels
;
1244 channels
= indio_dev
->channels
;
1246 int ml
= indio_dev
->masklength
;
1248 for (i
= 0; i
< indio_dev
->num_channels
; i
++)
1249 ml
= max(ml
, channels
[i
].scan_index
+ 1);
1250 indio_dev
->masklength
= ml
;
1257 if (buffer
->attrs
) {
1258 while (buffer
->attrs
[attrcount
] != NULL
)
1262 attr
= kcalloc(attrcount
+ ARRAY_SIZE(iio_buffer_attrs
) + 1,
1263 sizeof(struct attribute
*), GFP_KERNEL
);
1267 memcpy(attr
, iio_buffer_attrs
, sizeof(iio_buffer_attrs
));
1268 if (!buffer
->access
->set_length
)
1269 attr
[0] = &dev_attr_length_ro
.attr
;
1271 if (buffer
->access
->flags
& INDIO_BUFFER_FLAG_FIXED_WATERMARK
)
1272 attr
[2] = &dev_attr_watermark_ro
.attr
;
1275 memcpy(&attr
[ARRAY_SIZE(iio_buffer_attrs
)], buffer
->attrs
,
1276 sizeof(struct attribute
*) * attrcount
);
1278 attr
[attrcount
+ ARRAY_SIZE(iio_buffer_attrs
)] = NULL
;
1280 buffer
->buffer_group
.name
= "buffer";
1281 buffer
->buffer_group
.attrs
= attr
;
1283 indio_dev
->groups
[indio_dev
->groupcounter
++] = &buffer
->buffer_group
;
1285 if (buffer
->scan_el_attrs
!= NULL
) {
1286 attr
= buffer
->scan_el_attrs
->attrs
;
1287 while (*attr
++ != NULL
)
1290 attrcount
= attrcount_orig
;
1291 INIT_LIST_HEAD(&buffer
->scan_el_dev_attr_list
);
1292 channels
= indio_dev
->channels
;
1295 for (i
= 0; i
< indio_dev
->num_channels
; i
++) {
1296 if (channels
[i
].scan_index
< 0)
1299 ret
= iio_buffer_add_channel_sysfs(indio_dev
,
1302 goto error_cleanup_dynamic
;
1304 if (channels
[i
].type
== IIO_TIMESTAMP
)
1305 indio_dev
->scan_index_timestamp
=
1306 channels
[i
].scan_index
;
1308 if (indio_dev
->masklength
&& buffer
->scan_mask
== NULL
) {
1309 buffer
->scan_mask
= kcalloc(BITS_TO_LONGS(indio_dev
->masklength
),
1310 sizeof(*buffer
->scan_mask
),
1312 if (buffer
->scan_mask
== NULL
) {
1314 goto error_cleanup_dynamic
;
1319 buffer
->scan_el_group
.name
= iio_scan_elements_group_name
;
1321 buffer
->scan_el_group
.attrs
= kcalloc(attrcount
+ 1,
1322 sizeof(buffer
->scan_el_group
.attrs
[0]),
1324 if (buffer
->scan_el_group
.attrs
== NULL
) {
1326 goto error_free_scan_mask
;
1328 if (buffer
->scan_el_attrs
)
1329 memcpy(buffer
->scan_el_group
.attrs
, buffer
->scan_el_attrs
,
1330 sizeof(buffer
->scan_el_group
.attrs
[0])*attrcount_orig
);
1331 attrn
= attrcount_orig
;
1333 list_for_each_entry(p
, &buffer
->scan_el_dev_attr_list
, l
)
1334 buffer
->scan_el_group
.attrs
[attrn
++] = &p
->dev_attr
.attr
;
1335 indio_dev
->groups
[indio_dev
->groupcounter
++] = &buffer
->scan_el_group
;
1339 error_free_scan_mask
:
1340 kfree(buffer
->scan_mask
);
1341 error_cleanup_dynamic
:
1342 iio_free_chan_devattr_list(&buffer
->scan_el_dev_attr_list
);
1343 kfree(indio_dev
->buffer
->buffer_group
.attrs
);
1348 void iio_buffer_free_sysfs_and_mask(struct iio_dev
*indio_dev
)
1350 if (!indio_dev
->buffer
)
1353 kfree(indio_dev
->buffer
->scan_mask
);
1354 kfree(indio_dev
->buffer
->buffer_group
.attrs
);
1355 kfree(indio_dev
->buffer
->scan_el_group
.attrs
);
1356 iio_free_chan_devattr_list(&indio_dev
->buffer
->scan_el_dev_attr_list
);
1360 * iio_validate_scan_mask_onehot() - Validates that exactly one channel is selected
1361 * @indio_dev: the iio device
1362 * @mask: scan mask to be checked
1364 * Return true if exactly one bit is set in the scan mask, false otherwise. It
1365 * can be used for devices where only one channel can be active for sampling at
1368 bool iio_validate_scan_mask_onehot(struct iio_dev
*indio_dev
,
1369 const unsigned long *mask
)
1371 return bitmap_weight(mask
, indio_dev
->masklength
) == 1;
1373 EXPORT_SYMBOL_GPL(iio_validate_scan_mask_onehot
);
1375 static const void *iio_demux(struct iio_buffer
*buffer
,
1378 struct iio_demux_table
*t
;
1380 if (list_empty(&buffer
->demux_list
))
1382 list_for_each_entry(t
, &buffer
->demux_list
, l
)
1383 memcpy(buffer
->demux_bounce
+ t
->to
,
1384 datain
+ t
->from
, t
->length
);
1386 return buffer
->demux_bounce
;
1389 static int iio_push_to_buffer(struct iio_buffer
*buffer
, const void *data
)
1391 const void *dataout
= iio_demux(buffer
, data
);
1394 ret
= buffer
->access
->store_to(buffer
, dataout
);
1399 * We can't just test for watermark to decide if we wake the poll queue
1400 * because read may request less samples than the watermark.
1402 wake_up_interruptible_poll(&buffer
->pollq
, EPOLLIN
| EPOLLRDNORM
);
1407 * iio_push_to_buffers() - push to a registered buffer.
1408 * @indio_dev: iio_dev structure for device.
1411 int iio_push_to_buffers(struct iio_dev
*indio_dev
, const void *data
)
1414 struct iio_buffer
*buf
;
1416 list_for_each_entry(buf
, &indio_dev
->buffer_list
, buffer_list
) {
1417 ret
= iio_push_to_buffer(buf
, data
);
1424 EXPORT_SYMBOL_GPL(iio_push_to_buffers
);
1427 * iio_buffer_release() - Free a buffer's resources
1428 * @ref: Pointer to the kref embedded in the iio_buffer struct
1430 * This function is called when the last reference to the buffer has been
1431 * dropped. It will typically free all resources allocated by the buffer. Do not
1432 * call this function manually, always use iio_buffer_put() when done using a
1435 static void iio_buffer_release(struct kref
*ref
)
1437 struct iio_buffer
*buffer
= container_of(ref
, struct iio_buffer
, ref
);
1439 buffer
->access
->release(buffer
);
1443 * iio_buffer_get() - Grab a reference to the buffer
1444 * @buffer: The buffer to grab a reference for, may be NULL
1446 * Returns the pointer to the buffer that was passed into the function.
1448 struct iio_buffer
*iio_buffer_get(struct iio_buffer
*buffer
)
1451 kref_get(&buffer
->ref
);
1455 EXPORT_SYMBOL_GPL(iio_buffer_get
);
1458 * iio_buffer_put() - Release the reference to the buffer
1459 * @buffer: The buffer to release the reference for, may be NULL
1461 void iio_buffer_put(struct iio_buffer
*buffer
)
1464 kref_put(&buffer
->ref
, iio_buffer_release
);
1466 EXPORT_SYMBOL_GPL(iio_buffer_put
);
1469 * iio_device_attach_buffer - Attach a buffer to a IIO device
1470 * @indio_dev: The device the buffer should be attached to
1471 * @buffer: The buffer to attach to the device
1473 * This function attaches a buffer to a IIO device. The buffer stays attached to
1474 * the device until the device is freed. The function should only be called at
1475 * most once per device.
1477 void iio_device_attach_buffer(struct iio_dev
*indio_dev
,
1478 struct iio_buffer
*buffer
)
1480 indio_dev
->buffer
= iio_buffer_get(buffer
);
1482 EXPORT_SYMBOL_GPL(iio_device_attach_buffer
);