2 * Copyright (C) 2012 Google, Inc.
4 * This software is licensed under the terms of the GNU General Public
5 * License version 2, as published by the Free Software Foundation, and
6 * may be copied, distributed, and modified under those terms.
8 * This program is distributed in the hope that it will be useful,
9 * but WITHOUT ANY WARRANTY; without even the implied warranty of
10 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
11 * GNU General Public License for more details.
15 #define pr_fmt(fmt) "persistent_ram: " fmt
17 #include <linux/device.h>
18 #include <linux/err.h>
19 #include <linux/errno.h>
20 #include <linux/init.h>
22 #include <linux/kernel.h>
23 #include <linux/list.h>
24 #include <linux/memblock.h>
25 #include <linux/pstore_ram.h>
26 #include <linux/rslib.h>
27 #include <linux/slab.h>
28 #include <linux/uaccess.h>
29 #include <linux/vmalloc.h>
32 struct persistent_ram_buffer
{
39 #define PERSISTENT_RAM_SIG (0x43474244) /* DBGC */
41 static inline size_t buffer_size(struct persistent_ram_zone
*prz
)
43 return atomic_read(&prz
->buffer
->size
);
46 static inline size_t buffer_start(struct persistent_ram_zone
*prz
)
48 return atomic_read(&prz
->buffer
->start
);
51 /* increase and wrap the start pointer, returning the old value */
52 static size_t buffer_start_add(struct persistent_ram_zone
*prz
, size_t a
)
56 unsigned long flags
= 0;
58 if (!(prz
->flags
& PRZ_FLAG_NO_LOCK
))
59 raw_spin_lock_irqsave(&prz
->buffer_lock
, flags
);
61 old
= atomic_read(&prz
->buffer
->start
);
63 while (unlikely(new >= prz
->buffer_size
))
64 new -= prz
->buffer_size
;
65 atomic_set(&prz
->buffer
->start
, new);
67 if (!(prz
->flags
& PRZ_FLAG_NO_LOCK
))
68 raw_spin_unlock_irqrestore(&prz
->buffer_lock
, flags
);
73 /* increase the size counter until it hits the max size */
74 static void buffer_size_add(struct persistent_ram_zone
*prz
, size_t a
)
78 unsigned long flags
= 0;
80 if (!(prz
->flags
& PRZ_FLAG_NO_LOCK
))
81 raw_spin_lock_irqsave(&prz
->buffer_lock
, flags
);
83 old
= atomic_read(&prz
->buffer
->size
);
84 if (old
== prz
->buffer_size
)
88 if (new > prz
->buffer_size
)
89 new = prz
->buffer_size
;
90 atomic_set(&prz
->buffer
->size
, new);
93 if (!(prz
->flags
& PRZ_FLAG_NO_LOCK
))
94 raw_spin_unlock_irqrestore(&prz
->buffer_lock
, flags
);
97 static void notrace
persistent_ram_encode_rs8(struct persistent_ram_zone
*prz
,
98 uint8_t *data
, size_t len
, uint8_t *ecc
)
101 uint16_t par
[prz
->ecc_info
.ecc_size
];
103 /* Initialize the parity buffer */
104 memset(par
, 0, sizeof(par
));
105 encode_rs8(prz
->rs_decoder
, data
, len
, par
, 0);
106 for (i
= 0; i
< prz
->ecc_info
.ecc_size
; i
++)
110 static int persistent_ram_decode_rs8(struct persistent_ram_zone
*prz
,
111 void *data
, size_t len
, uint8_t *ecc
)
114 uint16_t par
[prz
->ecc_info
.ecc_size
];
116 for (i
= 0; i
< prz
->ecc_info
.ecc_size
; i
++)
118 return decode_rs8(prz
->rs_decoder
, data
, par
, len
,
119 NULL
, 0, NULL
, 0, NULL
);
122 static void notrace
persistent_ram_update_ecc(struct persistent_ram_zone
*prz
,
123 unsigned int start
, unsigned int count
)
125 struct persistent_ram_buffer
*buffer
= prz
->buffer
;
126 uint8_t *buffer_end
= buffer
->data
+ prz
->buffer_size
;
129 int ecc_block_size
= prz
->ecc_info
.block_size
;
130 int ecc_size
= prz
->ecc_info
.ecc_size
;
131 int size
= ecc_block_size
;
136 block
= buffer
->data
+ (start
& ~(ecc_block_size
- 1));
137 par
= prz
->par_buffer
+ (start
/ ecc_block_size
) * ecc_size
;
140 if (block
+ ecc_block_size
> buffer_end
)
141 size
= buffer_end
- block
;
142 persistent_ram_encode_rs8(prz
, block
, size
, par
);
143 block
+= ecc_block_size
;
145 } while (block
< buffer
->data
+ start
+ count
);
148 static void persistent_ram_update_header_ecc(struct persistent_ram_zone
*prz
)
150 struct persistent_ram_buffer
*buffer
= prz
->buffer
;
152 if (!prz
->ecc_info
.ecc_size
)
155 persistent_ram_encode_rs8(prz
, (uint8_t *)buffer
, sizeof(*buffer
),
159 static void persistent_ram_ecc_old(struct persistent_ram_zone
*prz
)
161 struct persistent_ram_buffer
*buffer
= prz
->buffer
;
165 if (!prz
->ecc_info
.ecc_size
)
168 block
= buffer
->data
;
169 par
= prz
->par_buffer
;
170 while (block
< buffer
->data
+ buffer_size(prz
)) {
172 int size
= prz
->ecc_info
.block_size
;
173 if (block
+ size
> buffer
->data
+ prz
->buffer_size
)
174 size
= buffer
->data
+ prz
->buffer_size
- block
;
175 numerr
= persistent_ram_decode_rs8(prz
, block
, size
, par
);
177 pr_devel("error in block %p, %d\n", block
, numerr
);
178 prz
->corrected_bytes
+= numerr
;
179 } else if (numerr
< 0) {
180 pr_devel("uncorrectable error in block %p\n", block
);
183 block
+= prz
->ecc_info
.block_size
;
184 par
+= prz
->ecc_info
.ecc_size
;
188 static int persistent_ram_init_ecc(struct persistent_ram_zone
*prz
,
189 struct persistent_ram_ecc_info
*ecc_info
)
192 struct persistent_ram_buffer
*buffer
= prz
->buffer
;
196 if (!ecc_info
|| !ecc_info
->ecc_size
)
199 prz
->ecc_info
.block_size
= ecc_info
->block_size
?: 128;
200 prz
->ecc_info
.ecc_size
= ecc_info
->ecc_size
?: 16;
201 prz
->ecc_info
.symsize
= ecc_info
->symsize
?: 8;
202 prz
->ecc_info
.poly
= ecc_info
->poly
?: 0x11d;
204 ecc_blocks
= DIV_ROUND_UP(prz
->buffer_size
- prz
->ecc_info
.ecc_size
,
205 prz
->ecc_info
.block_size
+
206 prz
->ecc_info
.ecc_size
);
207 ecc_total
= (ecc_blocks
+ 1) * prz
->ecc_info
.ecc_size
;
208 if (ecc_total
>= prz
->buffer_size
) {
209 pr_err("%s: invalid ecc_size %u (total %zu, buffer size %zu)\n",
210 __func__
, prz
->ecc_info
.ecc_size
,
211 ecc_total
, prz
->buffer_size
);
215 prz
->buffer_size
-= ecc_total
;
216 prz
->par_buffer
= buffer
->data
+ prz
->buffer_size
;
217 prz
->par_header
= prz
->par_buffer
+
218 ecc_blocks
* prz
->ecc_info
.ecc_size
;
221 * first consecutive root is 0
222 * primitive element to generate roots = 1
224 prz
->rs_decoder
= init_rs(prz
->ecc_info
.symsize
, prz
->ecc_info
.poly
,
225 0, 1, prz
->ecc_info
.ecc_size
);
226 if (prz
->rs_decoder
== NULL
) {
227 pr_info("init_rs failed\n");
231 prz
->corrected_bytes
= 0;
234 numerr
= persistent_ram_decode_rs8(prz
, buffer
, sizeof(*buffer
),
237 pr_info("error in header, %d\n", numerr
);
238 prz
->corrected_bytes
+= numerr
;
239 } else if (numerr
< 0) {
240 pr_info("uncorrectable error in header\n");
247 ssize_t
persistent_ram_ecc_string(struct persistent_ram_zone
*prz
,
248 char *str
, size_t len
)
252 if (!prz
->ecc_info
.ecc_size
)
255 if (prz
->corrected_bytes
|| prz
->bad_blocks
)
256 ret
= snprintf(str
, len
, ""
257 "\n%d Corrected bytes, %d unrecoverable blocks\n",
258 prz
->corrected_bytes
, prz
->bad_blocks
);
260 ret
= snprintf(str
, len
, "\nNo errors detected\n");
265 static void notrace
persistent_ram_update(struct persistent_ram_zone
*prz
,
266 const void *s
, unsigned int start
, unsigned int count
)
268 struct persistent_ram_buffer
*buffer
= prz
->buffer
;
269 memcpy_toio(buffer
->data
+ start
, s
, count
);
270 persistent_ram_update_ecc(prz
, start
, count
);
273 static int notrace
persistent_ram_update_user(struct persistent_ram_zone
*prz
,
274 const void __user
*s
, unsigned int start
, unsigned int count
)
276 struct persistent_ram_buffer
*buffer
= prz
->buffer
;
277 int ret
= unlikely(__copy_from_user(buffer
->data
+ start
, s
, count
)) ?
279 persistent_ram_update_ecc(prz
, start
, count
);
283 void persistent_ram_save_old(struct persistent_ram_zone
*prz
)
285 struct persistent_ram_buffer
*buffer
= prz
->buffer
;
286 size_t size
= buffer_size(prz
);
287 size_t start
= buffer_start(prz
);
293 persistent_ram_ecc_old(prz
);
294 prz
->old_log
= kmalloc(size
, GFP_KERNEL
);
297 pr_err("failed to allocate buffer\n");
301 prz
->old_log_size
= size
;
302 memcpy_fromio(prz
->old_log
, &buffer
->data
[start
], size
- start
);
303 memcpy_fromio(prz
->old_log
+ size
- start
, &buffer
->data
[0], start
);
306 int notrace
persistent_ram_write(struct persistent_ram_zone
*prz
,
307 const void *s
, unsigned int count
)
313 if (unlikely(c
> prz
->buffer_size
)) {
314 s
+= c
- prz
->buffer_size
;
315 c
= prz
->buffer_size
;
318 buffer_size_add(prz
, c
);
320 start
= buffer_start_add(prz
, c
);
322 rem
= prz
->buffer_size
- start
;
323 if (unlikely(rem
< c
)) {
324 persistent_ram_update(prz
, s
, start
, rem
);
329 persistent_ram_update(prz
, s
, start
, c
);
331 persistent_ram_update_header_ecc(prz
);
336 int notrace
persistent_ram_write_user(struct persistent_ram_zone
*prz
,
337 const void __user
*s
, unsigned int count
)
339 int rem
, ret
= 0, c
= count
;
342 if (unlikely(!access_ok(VERIFY_READ
, s
, count
)))
344 if (unlikely(c
> prz
->buffer_size
)) {
345 s
+= c
- prz
->buffer_size
;
346 c
= prz
->buffer_size
;
349 buffer_size_add(prz
, c
);
351 start
= buffer_start_add(prz
, c
);
353 rem
= prz
->buffer_size
- start
;
354 if (unlikely(rem
< c
)) {
355 ret
= persistent_ram_update_user(prz
, s
, start
, rem
);
361 ret
= persistent_ram_update_user(prz
, s
, start
, c
);
363 persistent_ram_update_header_ecc(prz
);
365 return unlikely(ret
) ? ret
: count
;
368 size_t persistent_ram_old_size(struct persistent_ram_zone
*prz
)
370 return prz
->old_log_size
;
373 void *persistent_ram_old(struct persistent_ram_zone
*prz
)
378 void persistent_ram_free_old(struct persistent_ram_zone
*prz
)
382 prz
->old_log_size
= 0;
385 void persistent_ram_zap(struct persistent_ram_zone
*prz
)
387 atomic_set(&prz
->buffer
->start
, 0);
388 atomic_set(&prz
->buffer
->size
, 0);
389 persistent_ram_update_header_ecc(prz
);
392 static void *persistent_ram_vmap(phys_addr_t start
, size_t size
,
393 unsigned int memtype
)
396 phys_addr_t page_start
;
397 unsigned int page_count
;
402 page_start
= start
- offset_in_page(start
);
403 page_count
= DIV_ROUND_UP(size
+ offset_in_page(start
), PAGE_SIZE
);
406 prot
= pgprot_noncached(PAGE_KERNEL
);
408 prot
= pgprot_writecombine(PAGE_KERNEL
);
410 pages
= kmalloc_array(page_count
, sizeof(struct page
*), GFP_KERNEL
);
412 pr_err("%s: Failed to allocate array for %u pages\n",
413 __func__
, page_count
);
417 for (i
= 0; i
< page_count
; i
++) {
418 phys_addr_t addr
= page_start
+ i
* PAGE_SIZE
;
419 pages
[i
] = pfn_to_page(addr
>> PAGE_SHIFT
);
421 vaddr
= vmap(pages
, page_count
, VM_MAP
, prot
);
427 static void *persistent_ram_iomap(phys_addr_t start
, size_t size
,
428 unsigned int memtype
)
432 if (!request_mem_region(start
, size
, "persistent_ram")) {
433 pr_err("request mem region (0x%llx@0x%llx) failed\n",
434 (unsigned long long)size
, (unsigned long long)start
);
439 va
= ioremap(start
, size
);
441 va
= ioremap_wc(start
, size
);
446 static int persistent_ram_buffer_map(phys_addr_t start
, phys_addr_t size
,
447 struct persistent_ram_zone
*prz
, int memtype
)
452 if (pfn_valid(start
>> PAGE_SHIFT
))
453 prz
->vaddr
= persistent_ram_vmap(start
, size
, memtype
);
455 prz
->vaddr
= persistent_ram_iomap(start
, size
, memtype
);
458 pr_err("%s: Failed to map 0x%llx pages at 0x%llx\n", __func__
,
459 (unsigned long long)size
, (unsigned long long)start
);
463 prz
->buffer
= prz
->vaddr
+ offset_in_page(start
);
464 prz
->buffer_size
= size
- sizeof(struct persistent_ram_buffer
);
469 static int persistent_ram_post_init(struct persistent_ram_zone
*prz
, u32 sig
,
470 struct persistent_ram_ecc_info
*ecc_info
)
474 ret
= persistent_ram_init_ecc(prz
, ecc_info
);
478 sig
^= PERSISTENT_RAM_SIG
;
480 if (prz
->buffer
->sig
== sig
) {
481 if (buffer_size(prz
) > prz
->buffer_size
||
482 buffer_start(prz
) > buffer_size(prz
))
483 pr_info("found existing invalid buffer, size %zu, start %zu\n",
484 buffer_size(prz
), buffer_start(prz
));
486 pr_debug("found existing buffer, size %zu, start %zu\n",
487 buffer_size(prz
), buffer_start(prz
));
488 persistent_ram_save_old(prz
);
492 pr_debug("no valid data in buffer (sig = 0x%08x)\n",
496 /* Rewind missing or invalid memory area. */
497 prz
->buffer
->sig
= sig
;
498 persistent_ram_zap(prz
);
503 void persistent_ram_free(struct persistent_ram_zone
*prz
)
509 if (pfn_valid(prz
->paddr
>> PAGE_SHIFT
)) {
513 release_mem_region(prz
->paddr
, prz
->size
);
517 persistent_ram_free_old(prz
);
521 struct persistent_ram_zone
*persistent_ram_new(phys_addr_t start
, size_t size
,
522 u32 sig
, struct persistent_ram_ecc_info
*ecc_info
,
523 unsigned int memtype
, u32 flags
)
525 struct persistent_ram_zone
*prz
;
528 prz
= kzalloc(sizeof(struct persistent_ram_zone
), GFP_KERNEL
);
530 pr_err("failed to allocate persistent ram zone\n");
534 /* Initialize general buffer state. */
535 raw_spin_lock_init(&prz
->buffer_lock
);
538 ret
= persistent_ram_buffer_map(start
, size
, prz
, memtype
);
542 ret
= persistent_ram_post_init(prz
, sig
, ecc_info
);
548 persistent_ram_free(prz
);