updated on Sun Jan 15 16:02:00 UTC 2012

[aur-mirror.git] / kernel26-uksm / uksm-2.6.38-20120110.patch

diff -urN linux-2.6.38/arch/x86/kernel/entry_32.S uksm-2.6.38-zhang/arch/x86/kernel/entry_32.S
--- linux-2.6.38/arch/x86/kernel/entry_32.S     2011-03-15 09:20:32.000000000 +0800
+++ uksm-2.6.38-zhang/arch/x86/kernel/entry_32.S        2012-01-09 10:05:23.642269166 +0800
@@ -1413,7 +1413,7 @@
        CFI_ADJUST_CFA_OFFSET 4
        jmp error_code
        CFI_ENDPROC
-END(apf_page_fault)
+END(async_page_fault)
 #endif
 
 /*
diff -urN linux-2.6.38/arch/x86/kernel/entry_64.S uksm-2.6.38-zhang/arch/x86/kernel/entry_64.S
--- linux-2.6.38/arch/x86/kernel/entry_64.S     2011-03-15 09:20:32.000000000 +0800
+++ uksm-2.6.38-zhang/arch/x86/kernel/entry_64.S        2012-01-09 10:05:23.642269166 +0800
@@ -1248,7 +1248,7 @@
        decl PER_CPU_VAR(irq_count)
        jmp  error_exit
        CFI_ENDPROC
-END(do_hypervisor_callback)
+END(xen_do_hypervisor_callback)
 
 /*
  * Hypervisor uses this for application faults while it executes.
diff -urN linux-2.6.38/fs/exec.c uksm-2.6.38-zhang/fs/exec.c
--- linux-2.6.38/fs/exec.c      2011-03-15 09:20:32.000000000 +0800
+++ uksm-2.6.38-zhang/fs/exec.c 2012-01-09 10:05:55.168936883 +0800
@@ -19,7 +19,7 @@
  * current->executable is only used by the procfs.  This allows a dispatch
  * table to check for several different types  of binary formats.  We keep
  * trying until we recognize the file or we run out of supported binary
- * formats. 
+ * formats.
  */
 
 #include <linux/slab.h>
@@ -55,6 +55,7 @@
 #include <linux/fs_struct.h>
 #include <linux/pipe_fs_i.h>
 #include <linux/oom.h>
+#include <linux/ksm.h>
 
 #include <asm/uaccess.h>
 #include <asm/mmu_context.h>
@@ -85,7 +86,7 @@
        insert ? list_add(&fmt->lh, &formats) :
                 list_add_tail(&fmt->lh, &formats);
        write_unlock(&binfmt_lock);
-       return 0;       
+       return 0;
 }
 
 EXPORT_SYMBOL(__register_binfmt);
@@ -1106,7 +1107,7 @@
           group */
 
        current->self_exec_id++;
-                       
+
        flush_signal_handlers(current, 0);
        flush_old_files(current->files);
 }
@@ -1196,8 +1197,8 @@
        return res;
 }
 
-/* 
- * Fill the binprm structure from the inode. 
+/*
+ * Fill the binprm structure from the inode.
  * Check permissions, then read the first 128 (BINPRM_BUF_SIZE) bytes
  *
  * This may be called multiple times for binary chains (scripts for example).
diff -urN linux-2.6.38/fs/proc/meminfo.c uksm-2.6.38-zhang/fs/proc/meminfo.c
--- linux-2.6.38/fs/proc/meminfo.c      2011-03-15 09:20:32.000000000 +0800
+++ uksm-2.6.38-zhang/fs/proc/meminfo.c 2012-01-09 10:05:56.362270256 +0800
@@ -87,6 +87,10 @@
                "SUnreclaim:     %8lu kB\n"
                "KernelStack:    %8lu kB\n"
                "PageTables:     %8lu kB\n"
+#ifdef CONFIG_KSM
+               "KsmSharing:     %8lu kB\n"
+               "KsmZeroPages:   %8lu kB\n"
+#endif
 #ifdef CONFIG_QUICKLIST
                "Quicklists:     %8lu kB\n"
 #endif
@@ -145,6 +149,10 @@
                K(global_page_state(NR_SLAB_UNRECLAIMABLE)),
                global_page_state(NR_KERNEL_STACK) * THREAD_SIZE / 1024,
                K(global_page_state(NR_PAGETABLE)),
+#ifdef CONFIG_KSM
+               K(global_page_state(NR_KSM_PAGES_SHARING)),
+               K(global_page_state(NR_KSM_ZERO_PAGES)),
+#endif
 #ifdef CONFIG_QUICKLIST
                K(quicklist_total_size()),
 #endif
Binary files linux-2.6.38/.gitignore.swp and uksm-2.6.38-zhang/.gitignore.swp differ
diff -urN linux-2.6.38/include/linux/ksm.h uksm-2.6.38-zhang/include/linux/ksm.h
--- linux-2.6.38/include/linux/ksm.h    2011-03-15 09:20:32.000000000 +0800
+++ uksm-2.6.38-zhang/include/linux/ksm.h       2012-01-09 10:11:24.218947858 +0800
@@ -20,23 +20,10 @@
                        struct vm_area_struct *vma, unsigned long address);
 
 #ifdef CONFIG_KSM
-int ksm_madvise(struct vm_area_struct *vma, unsigned long start,
-               unsigned long end, int advice, unsigned long *vm_flags);
-int __ksm_enter(struct mm_struct *mm);
-void __ksm_exit(struct mm_struct *mm);
 
-static inline int ksm_fork(struct mm_struct *mm, struct mm_struct *oldmm)
-{
-       if (test_bit(MMF_VM_MERGEABLE, &oldmm->flags))
-               return __ksm_enter(mm);
-       return 0;
-}
-
-static inline void ksm_exit(struct mm_struct *mm)
-{
-       if (test_bit(MMF_VM_MERGEABLE, &mm->flags))
-               __ksm_exit(mm);
-}
+extern unsigned long zero_pfn __read_mostly;
+extern unsigned long ksm_zero_pfn __read_mostly;
+extern struct page *empty_ksm_zero_page;
 
 /*
  * A KSM page is one of those write-protected "shared pages" or "merged pages"
@@ -62,6 +49,13 @@
                                (PAGE_MAPPING_ANON | PAGE_MAPPING_KSM);
 }
 
+/* must be done before linked to mm */
+extern void ksm_vma_add_new(struct vm_area_struct *vma);
+
+extern void ksm_remove_vma(struct vm_area_struct *vma);
+extern int unmerge_ksm_pages(struct vm_area_struct *vma,
+                                   unsigned long start, unsigned long end);
+
 /*
  * When do_swap_page() first faults in from swap what used to be a KSM page,
  * no problem, it will be assigned to this vma's anon_vma; but thereafter,
@@ -90,16 +84,184 @@
                  struct vm_area_struct *, unsigned long, void *), void *arg);
 void ksm_migrate_page(struct page *newpage, struct page *oldpage);
 
-#else  /* !CONFIG_KSM */
+/* Each rung of this ladder is a list of VMAs having a same scan ratio */
+struct scan_rung {
+       struct list_head vma_list;
+       //spinlock_t vma_list_lock;
+       //struct semaphore sem;
+       struct list_head *current_scan;
+       unsigned int pages_to_scan;
+       unsigned char round_finished; /* rung is ready for the next round */
+       unsigned char busy_searched;
+       unsigned long fully_scanned_slots;
+       unsigned long scan_ratio;
+       unsigned long vma_num;
+       //unsigned long vma_finished;
+       unsigned long scan_turn;
+};
+
+struct vma_slot {
+       struct list_head ksm_list;
+       struct list_head slot_list;
+       unsigned long dedup_ratio;
+       unsigned long dedup_num;
+       int ksm_index; /* -1 if vma is not in inter-table,
+                               positive otherwise */
+       unsigned long pages_scanned;
+       unsigned long last_scanned;
+       unsigned long pages_to_scan;
+       struct scan_rung *rung;
+       struct page **rmap_list_pool;
+       unsigned long *pool_counts;
+       unsigned long pool_size;
+       struct vm_area_struct *vma;
+       struct mm_struct *mm;
+       unsigned long ctime_j;
+       unsigned long pages;
+       unsigned char need_sort;
+       unsigned char need_rerand;
+       unsigned long slot_scanned; /* It's scanned in this round */
+       unsigned long fully_scanned; /* the above four to be merged to status bits */
+       unsigned long pages_cowed; /* pages cowed this round */
+       unsigned long pages_merged; /* pages merged this round */
+
+       /* used for dup vma pair */
+       struct radix_tree_root dup_tree;
+};
 
-static inline int ksm_fork(struct mm_struct *mm, struct mm_struct *oldmm)
-{
-       return 0;
-}
+/*
+ * A few notes about the KSM scanning process,
+ * to make it easier to understand the data structures below:
+ *
+ * In order to reduce excessive scanning, KSM sorts the memory pages by their
+ * contents into a data structure that holds pointers to the pages' locations.
+ *
+ * Since the contents of the pages may change at any moment, KSM cannot just
+ * insert the pages into a normal sorted tree and expect it to find anything.
+ * Therefore KSM uses two data structures - the stable and the unstable tree.
+ *
+ * The stable tree holds pointers to all the merged pages (ksm pages), sorted
+ * by their contents.  Because each such page is write-protected, searching on
+ * this tree is fully assured to be working (except when pages are unmapped),
+ * and therefore this tree is called the stable tree.
+ *
+ * In addition to the stable tree, KSM uses a second data structure called the
+ * unstable tree: this tree holds pointers to pages which have been found to
+ * be "unchanged for a period of time".  The unstable tree sorts these pages
+ * by their contents, but since they are not write-protected, KSM cannot rely
+ * upon the unstable tree to work correctly - the unstable tree is liable to
+ * be corrupted as its contents are modified, and so it is called unstable.
+ *
+ * KSM solves this problem by several techniques:
+ *
+ * 1) The unstable tree is flushed every time KSM completes scanning all
+ *    memory areas, and then the tree is rebuilt again from the beginning.
+ * 2) KSM will only insert into the unstable tree, pages whose hash value
+ *    has not changed since the previous scan of all memory areas.
+ * 3) The unstable tree is a RedBlack Tree - so its balancing is based on the
+ *    colors of the nodes and not on their contents, assuring that even when
+ *    the tree gets "corrupted" it won't get out of balance, so scanning time
+ *    remains the same (also, searching and inserting nodes in an rbtree uses
+ *    the same algorithm, so we have no overhead when we flush and rebuild).
+ * 4) KSM never flushes the stable tree, which means that even if it were to
+ *    take 10 attempts to find a page in the unstable tree, once it is found,
+ *    it is secured in the stable tree.  (When we scan a new page, we first
+ *    compare it against the stable tree, and then against the unstable tree.)
+ */
 
-static inline void ksm_exit(struct mm_struct *mm)
-{
-}
+
+/**
+ * node of either the stable or unstale rbtree
+ *
+ */
+struct tree_node {
+       struct rb_node node; /* link in the main (un)stable rbtree */
+       struct rb_root sub_root; /* rb_root for sublevel collision rbtree */
+       u32 hash;
+       unsigned long count; /* how many sublevel tree nodes */
+       struct list_head all_list; /* all tree nodes in stable/unstable tree */
+};
+
+
+/**
+ * struct stable_node - node of the stable rbtree
+ * @node: rb node of this ksm page in the stable tree
+ * @hlist: hlist head of rmap_items using this ksm page
+ * @kpfn: page frame number of this ksm page
+ */
+struct stable_node {
+       struct rb_node node; /* link in sub-rbtree */
+       struct tree_node *tree_node; /* it's tree node root in stable tree, NULL if it's in hell list */
+       struct hlist_head hlist;
+       unsigned long kpfn;
+       u32 hash_max; /* if ==0 then it's not been calculated yet */
+       //struct vm_area_struct *old_vma;
+       struct list_head all_list; /* in a list for all stable nodes */
+};
+
+
+
+
+/**
+ * struct node_vma - group rmap_items linked in a same stable
+ * node together.
+ */
+struct node_vma {
+       union {
+               struct vma_slot *slot;
+               unsigned long key;  /* slot is used as key sorted on hlist */
+       };
+       struct hlist_node hlist;
+       struct hlist_head rmap_hlist;
+       struct stable_node *head;
+       unsigned long last_update;
+};
+
+/**
+ * struct rmap_item - reverse mapping item for virtual addresses
+ * @rmap_list: next rmap_item in mm_slot's singly-linked rmap_list
+ * @anon_vma: pointer to anon_vma for this mm,address, when in stable tree
+ * @mm: the memory structure this rmap_item is pointing into
+ * @address: the virtual address this rmap_item tracks (+ flags in low bits)
+ * @node: rb node of this rmap_item in the unstable tree
+ * @head: pointer to stable_node heading this list in the stable tree
+ * @hlist: link into hlist of rmap_items hanging off that stable_node
+ */
+struct rmap_item {
+       struct vma_slot *slot;
+       struct page *page;
+       unsigned long address;  /* + low bits used for flags below */
+       /* Appendded to (un)stable tree on which scan round */
+       unsigned long append_round;
+
+       /* Which rung scan turn it was last scanned */
+       //unsigned long last_scan;
+       unsigned long entry_index;
+       union {
+               struct {/* when in unstable tree */
+                       struct rb_node node;
+                       struct tree_node *tree_node;
+                       u32 hash_max;
+               };
+               struct { /* when in stable tree */
+                       struct node_vma *head;
+                       struct hlist_node hlist;
+                       struct anon_vma *anon_vma;
+               };
+       };
+} __attribute__((aligned(4)));
+
+struct rmap_list_entry {
+       union {
+               struct rmap_item *item;
+               unsigned long addr;
+       };
+       // lowest bit is used for is_addr tag
+       //unsigned char is_addr;
+} __attribute__((aligned(4))); // 4 aligned to fit in to pages
+
+//extern struct semaphore ksm_scan_sem;
+#else  /* !CONFIG_KSM */
 
 static inline int PageKsm(struct page *page)
 {
@@ -107,8 +269,9 @@
 }
 
 #ifdef CONFIG_MMU
-static inline int ksm_madvise(struct vm_area_struct *vma, unsigned long start,
-               unsigned long end, int advice, unsigned long *vm_flags)
+
+extern inline int unmerge_ksm_pages(struct vm_area_struct *vma,
+                                   unsigned long start, unsigned long end)
 {
        return 0;
 }
diff -urN linux-2.6.38/include/linux/mm_types.h uksm-2.6.38-zhang/include/linux/mm_types.h
--- linux-2.6.38/include/linux/mm_types.h       2011-03-15 09:20:32.000000000 +0800
+++ uksm-2.6.38-zhang/include/linux/mm_types.h  2012-01-09 10:05:57.562270296 +0800
@@ -183,6 +183,9 @@
 #ifdef CONFIG_NUMA
        struct mempolicy *vm_policy;    /* NUMA policy for the VMA */
 #endif
+#ifdef CONFIG_KSM
+       struct vma_slot *ksm_vma_slot;
+#endif
 };
 
 struct core_thread {
diff -urN linux-2.6.38/include/linux/mmzone.h uksm-2.6.38-zhang/include/linux/mmzone.h
--- linux-2.6.38/include/linux/mmzone.h 2011-03-15 09:20:32.000000000 +0800
+++ uksm-2.6.38-zhang/include/linux/mmzone.h    2012-01-09 10:05:57.562270296 +0800
@@ -115,6 +115,10 @@
        NUMA_OTHER,             /* allocation from other node */
 #endif
        NR_ANON_TRANSPARENT_HUGEPAGES,
+#ifdef CONFIG_KSM
+       NR_KSM_PAGES_SHARING,
+       NR_KSM_ZERO_PAGES,
+#endif
        NR_VM_ZONE_STAT_ITEMS };
 
 /*
@@ -344,7 +348,7 @@
        ZONE_PADDING(_pad1_)
 
        /* Fields commonly accessed by the page reclaim scanner */
-       spinlock_t              lru_lock;       
+       spinlock_t              lru_lock;
        struct zone_lru {
                struct list_head list;
        } lru[NR_LRU_LISTS];
@@ -722,7 +726,7 @@
 }
 
 /**
- * is_highmem - helper function to quickly check if a struct zone is a 
+ * is_highmem - helper function to quickly check if a struct zone is a
  *              highmem zone or not.  This is an attempt to keep references
  *              to ZONE_{DMA/NORMAL/HIGHMEM/etc} in general code to a minimum.
  * @zone - pointer to struct zone variable
diff -urN linux-2.6.38/include/linux/sched.h uksm-2.6.38-zhang/include/linux/sched.h
--- linux-2.6.38/include/linux/sched.h  2011-03-15 09:20:32.000000000 +0800
+++ uksm-2.6.38-zhang/include/linux/sched.h     2012-01-09 10:05:57.815603639 +0800
@@ -433,7 +433,6 @@
 # define MMF_DUMP_MASK_DEFAULT_ELF     0
 #endif
                                        /* leave room for more dump flags */
-#define MMF_VM_MERGEABLE       16      /* KSM may merge identical pages */
 #define MMF_VM_HUGEPAGE                17      /* set when VM_HUGEPAGE is set on vma */
 
 #define MMF_INIT_MASK          (MMF_DUMPABLE_MASK | MMF_DUMP_FILTER_MASK)
@@ -1280,9 +1279,9 @@
        unsigned long stack_canary;
 #endif
 
-       /* 
+       /*
         * pointers to (original) parent process, youngest child, younger sibling,
-        * older sibling, respectively.  (p->father can be replaced with 
+        * older sibling, respectively.  (p->father can be replaced with
         * p->real_parent->pid)
         */
        struct task_struct *real_parent; /* real parent process */
@@ -2080,7 +2079,7 @@
        spin_unlock_irqrestore(&tsk->sighand->siglock, flags);
 
        return ret;
-}      
+}
 
 extern void block_all_signals(int (*notifier)(void *priv), void *priv,
                              sigset_t *mask);
diff -urN linux-2.6.38/kernel/fork.c uksm-2.6.38-zhang/kernel/fork.c
--- linux-2.6.38/kernel/fork.c  2011-03-15 09:20:32.000000000 +0800
+++ uksm-2.6.38-zhang/kernel/fork.c     2012-01-09 10:05:59.635603699 +0800
@@ -328,9 +328,6 @@
        rb_link = &mm->mm_rb.rb_node;
        rb_parent = NULL;
        pprev = &mm->mmap;
-       retval = ksm_fork(mm, oldmm);
-       if (retval)
-               goto out;
        retval = khugepaged_fork(mm, oldmm);
        if (retval)
                goto out;
@@ -353,7 +350,7 @@
                                goto fail_nomem;
                        charge = len;
                }
-               tmp = kmem_cache_alloc(vm_area_cachep, GFP_KERNEL);
+               tmp = kmem_cache_zalloc(vm_area_cachep, GFP_KERNEL);
                if (!tmp)
                        goto fail_nomem;
                *tmp = *mpnt;
@@ -406,7 +403,9 @@
                __vma_link_rb(mm, tmp, rb_link, rb_parent);
                rb_link = &tmp->vm_rb.rb_right;
                rb_parent = &tmp->vm_rb;
-
+#ifdef CONFIG_KSM
+               ksm_vma_add_new(tmp);
+#endif
                mm->map_count++;
                retval = copy_page_range(mm, oldmm, mpnt);
 
@@ -549,7 +548,6 @@
 
        if (atomic_dec_and_test(&mm->mm_users)) {
                exit_aio(mm);
-               ksm_exit(mm);
                khugepaged_exit(mm); /* must run before exit_mmap */
                exit_mmap(mm);
                set_mm_exe_file(mm, NULL);
diff -urN linux-2.6.38/mm/ksm.c uksm-2.6.38-zhang/mm/ksm.c
--- linux-2.6.38/mm/ksm.c       2011-03-15 09:20:32.000000000 +0800
+++ uksm-2.6.38-zhang/mm/ksm.c  2012-01-09 10:05:59.862270375 +0800
@@ -12,6 +12,47 @@
  *     Hugh Dickins
  *
  * This work is licensed under the terms of the GNU GPL, version 2.
+ *
+ *
+ *
+ * Ultra KSM. Copyright (C) 2011 Nai Xia
+ *
+ * This is an improvement upon KSM. Its features:
+ * 1. Full system scan:
+ *      It automatically scans all user processes' anonymous VMAs. Kernel-user
+ *      interaction to submit a memory area to KSM is no longer needed.
+ *
+ * 2. Rich area detection based on random sampling:
+ *      It automatically detects rich areas containing abundant duplicated
+ *      pages based on their randomly-sampled history. Rich areas are given
+ *      a full scan speed. Poor areas are sampled at a reasonable speed with
+ *      very low CPU consumption.
+ *
+ * 3. Per-page scan speed improvement:
+ *      A new hash algorithm(random_sample_hash) is proposed. Quite usually,
+ *      it's enough to distinguish pages by hashing their partial content
+ *      instead of full pages. This algorithm can automatically adapt to this
+ *      situation. For the best case, only one 32-bit-word/page is needed to
+ *      get the hash value for distinguishing pages. For the worst case, it's as
+ *      fast as SuperFastHash.
+ *
+ * 4. Thrashing area avoidance:
+ *      Thrashing area(an VMA that has frequent Ksm page break-out) can be
+ *      filtered out. My benchmark shows it's more efficient than KSM's per-page
+ *      hash value based volatile page detection.
+ *
+ * 5. Hash-value-based identical page detection:
+ *      It no longer uses "memcmp" based page detection any more.
+ *
+ * 6. Misc changes upon KSM:
+ *      * It has a fully x86-opitmized memcmp dedicated for 4-byte-aligned page
+ *        comparison. It's much faster than default C version on x86.
+ *      * rmap_item now has an struct *page member to loosely cache a
+ *        address-->page mapping, which reduces too much time-costly
+ *        follow_page().
+ *      * The VMA creation/exit procedures are hooked to let the Ultra KSM know.
+ *      * try_to_merge_two_pages() now can revert a pte if it fails. No break_
+ *        ksm is needed for this case.
  */
 
 #include <linux/errno.h>
@@ -33,142 +74,168 @@
 #include <linux/mmu_notifier.h>
 #include <linux/swap.h>
 #include <linux/ksm.h>
-#include <linux/hash.h>
+#include <linux/crypto.h>
+#include <linux/scatterlist.h>
+#include <crypto/hash.h>
+#include <linux/random.h>
+#include <linux/math64.h>
+#include <linux/gcd.h>
 #include <linux/freezer.h>
 
 #include <asm/tlbflush.h>
 #include "internal.h"
 
+#ifdef CONFIG_X86
+#undef memcmp
+
+#ifdef CONFIG_X86_32
+#define memcmp memcmpx86_32
 /*
- * A few notes about the KSM scanning process,
- * to make it easier to understand the data structures below:
- *
- * In order to reduce excessive scanning, KSM sorts the memory pages by their
- * contents into a data structure that holds pointers to the pages' locations.
- *
- * Since the contents of the pages may change at any moment, KSM cannot just
- * insert the pages into a normal sorted tree and expect it to find anything.
- * Therefore KSM uses two data structures - the stable and the unstable tree.
- *
- * The stable tree holds pointers to all the merged pages (ksm pages), sorted
- * by their contents.  Because each such page is write-protected, searching on
- * this tree is fully assured to be working (except when pages are unmapped),
- * and therefore this tree is called the stable tree.
- *
- * In addition to the stable tree, KSM uses a second data structure called the
- * unstable tree: this tree holds pointers to pages which have been found to
- * be "unchanged for a period of time".  The unstable tree sorts these pages
- * by their contents, but since they are not write-protected, KSM cannot rely
- * upon the unstable tree to work correctly - the unstable tree is liable to
- * be corrupted as its contents are modified, and so it is called unstable.
- *
- * KSM solves this problem by several techniques:
- *
- * 1) The unstable tree is flushed every time KSM completes scanning all
- *    memory areas, and then the tree is rebuilt again from the beginning.
- * 2) KSM will only insert into the unstable tree, pages whose hash value
- *    has not changed since the previous scan of all memory areas.
- * 3) The unstable tree is a RedBlack Tree - so its balancing is based on the
- *    colors of the nodes and not on their contents, assuring that even when
- *    the tree gets "corrupted" it won't get out of balance, so scanning time
- *    remains the same (also, searching and inserting nodes in an rbtree uses
- *    the same algorithm, so we have no overhead when we flush and rebuild).
- * 4) KSM never flushes the stable tree, which means that even if it were to
- *    take 10 attempts to find a page in the unstable tree, once it is found,
- *    it is secured in the stable tree.  (When we scan a new page, we first
- *    compare it against the stable tree, and then against the unstable tree.)
+ * Compare 4-byte-aligned address s1 and s2, with length n
  */
+int memcmpx86_32(void *s1, void *s2, size_t n)
+{
+       size_t num = n / 4;
+       register int res;
 
-/**
- * struct mm_slot - ksm information per mm that is being scanned
- * @link: link to the mm_slots hash list
- * @mm_list: link into the mm_slots list, rooted in ksm_mm_head
- * @rmap_list: head for this mm_slot's singly-linked list of rmap_items
- * @mm: the mm that this information is valid for
- */
-struct mm_slot {
-       struct hlist_node link;
-       struct list_head mm_list;
-       struct rmap_item *rmap_list;
-       struct mm_struct *mm;
-};
+       __asm__ __volatile__
+       (
+        "testl %3,%3\n\t"
+        "repe; cmpsd\n\t"
+        "je        1f\n\t"
+        "sbbl      %0,%0\n\t"
+        "orl       $1,%0\n"
+        "1:"
+        : "=&a" (res), "+&S" (s1), "+&D" (s2), "+&c" (num)
+        : "0" (0)
+        : "cc");
 
-/**
- * struct ksm_scan - cursor for scanning
- * @mm_slot: the current mm_slot we are scanning
- * @address: the next address inside that to be scanned
- * @rmap_list: link to the next rmap to be scanned in the rmap_list
- * @seqnr: count of completed full scans (needed when removing unstable node)
- *
- * There is only the one ksm_scan instance of this cursor structure.
+       return res;
+}
+
+/*
+ * Check the page is all zero ?
  */
-struct ksm_scan {
-       struct mm_slot *mm_slot;
-       unsigned long address;
-       struct rmap_item **rmap_list;
-       unsigned long seqnr;
-};
+static int is_full_zero(const void *s1, size_t len)
+{
+       unsigned char same;
 
-/**
- * struct stable_node - node of the stable rbtree
- * @node: rb node of this ksm page in the stable tree
- * @hlist: hlist head of rmap_items using this ksm page
- * @kpfn: page frame number of this ksm page
- */
-struct stable_node {
-       struct rb_node node;
-       struct hlist_head hlist;
-       unsigned long kpfn;
-};
+       len /= 4;
 
-/**
- * struct rmap_item - reverse mapping item for virtual addresses
- * @rmap_list: next rmap_item in mm_slot's singly-linked rmap_list
- * @anon_vma: pointer to anon_vma for this mm,address, when in stable tree
- * @mm: the memory structure this rmap_item is pointing into
- * @address: the virtual address this rmap_item tracks (+ flags in low bits)
- * @oldchecksum: previous checksum of the page at that virtual address
- * @node: rb node of this rmap_item in the unstable tree
- * @head: pointer to stable_node heading this list in the stable tree
- * @hlist: link into hlist of rmap_items hanging off that stable_node
- */
-struct rmap_item {
-       struct rmap_item *rmap_list;
-       struct anon_vma *anon_vma;      /* when stable */
-       struct mm_struct *mm;
-       unsigned long address;          /* + low bits used for flags below */
-       unsigned int oldchecksum;       /* when unstable */
-       union {
-               struct rb_node node;    /* when node of unstable tree */
-               struct {                /* when listed from stable tree */
-                       struct stable_node *head;
-                       struct hlist_node hlist;
-               };
-       };
-};
+       __asm__ __volatile__
+       ("repe; scasl;"
+        "sete %0"
+        : "=qm" (same), "+D" (s1), "+c" (len)
+        : "a" (0)
+        : "cc");
+
+       return same;
+}
 
-#define SEQNR_MASK     0x0ff   /* low bits of unstable tree seqnr */
-#define UNSTABLE_FLAG  0x100   /* is a node of the unstable tree */
-#define STABLE_FLAG    0x200   /* is listed from the stable tree */
 
-/* The stable and unstable tree heads */
-static struct rb_root root_stable_tree = RB_ROOT;
-static struct rb_root root_unstable_tree = RB_ROOT;
+#elif defined(CONFIG_X86_64)
+#define memcmp memcmpx86_64
+/*
+ * Compare 8-byte-aligned address s1 and s2, with length n
+ */
+int memcmpx86_64(void *s1, void *s2, size_t n)
+{
+       size_t num = n / 8;
+       register int res;
 
-#define MM_SLOTS_HASH_SHIFT 10
-#define MM_SLOTS_HASH_HEADS (1 << MM_SLOTS_HASH_SHIFT)
-static struct hlist_head mm_slots_hash[MM_SLOTS_HASH_HEADS];
+       __asm__ __volatile__
+       (
+        "testq %q3,%q3\n\t"
+        "repe; cmpsq\n\t"
+        "je        1f\n\t"
+        "sbbq      %q0,%q0\n\t"
+        "orq       $1,%q0\n"
+        "1:"
+        : "=&a" (res), "+&S" (s1), "+&D" (s2), "+&c" (num)
+        : "0" (0)
+        : "cc");
 
-static struct mm_slot ksm_mm_head = {
-       .mm_list = LIST_HEAD_INIT(ksm_mm_head.mm_list),
-};
-static struct ksm_scan ksm_scan = {
-       .mm_slot = &ksm_mm_head,
-};
+       return res;
+}
+
+static int is_full_zero(const void *s1, size_t len)
+{
+       unsigned char same;
+
+       len /= 8;
+
+       __asm__ __volatile__
+       ("repe; scasq;"
+        "sete %0"
+        : "=qm" (same), "+D" (s1), "+c" (len)
+        : "a" (0)
+        : "cc");
+
+       return same;
+}
+
+#endif
+#else
+static int is_full_zero(const void *s1, size_t len)
+{
+       unsigned long *src = s1;
+       int i;
 
+       len /= sizeof(*src);
+
+       for (i = 0; i < len; i++) {
+               if (src[i])
+                       return 0;
+       }
+
+       return 1;
+}
+#endif
+
+#define U64_MAX                (~((u64)0))
+
+
+/*
+ * Flags for rmap_item to judge if it's listed in the stable/unstable tree.
+ * The flags use the low bits of rmap_item.address
+ */
+#define UNSTABLE_FLAG  0x1
+#define STABLE_FLAG    0x2
+#define get_rmap_addr(x)       ((x)->address & PAGE_MASK)
+
+/*
+ * rmap_list_entry helpers
+ */
+#define IS_ADDR_FLAG   1
+#define is_addr(ptr)           ((unsigned long)(ptr) & IS_ADDR_FLAG)
+#define set_is_addr(ptr)       ((ptr) |= IS_ADDR_FLAG)
+#define get_clean_addr(ptr)    (((ptr) & ~(__typeof__(ptr))IS_ADDR_FLAG))
+
+
+/*
+ * High speed caches for frequently allocated and freed structs
+ */
 static struct kmem_cache *rmap_item_cache;
 static struct kmem_cache *stable_node_cache;
-static struct kmem_cache *mm_slot_cache;
+static struct kmem_cache *node_vma_cache;
+static struct kmem_cache *vma_slot_cache;
+static struct kmem_cache *tree_node_cache;
+#define KSM_KMEM_CACHE(__struct, __flags) kmem_cache_create("ksm_"#__struct,\
+               sizeof(struct __struct), __alignof__(struct __struct),\
+               (__flags), NULL)
+
+/* The scan rounds ksmd is currently in */
+static unsigned long long ksm_scan_round = 1;
+
+/* The number of pages has been scanned since the start up */
+static u64 ksm_pages_scanned;
+
+/* The number of pages has been scanned when last scan round finished */
+static u64 ksm_pages_scanned_last;
+
+/* If the scanned number is tooo large, we encode it here */
+static u64 pages_scanned_stored;
+static unsigned long pages_scanned_base;
 
 /* The number of nodes in the stable tree */
 static unsigned long ksm_pages_shared;
@@ -179,345 +246,403 @@
 /* The number of nodes in the unstable tree */
 static unsigned long ksm_pages_unshared;
 
-/* The number of rmap_items in use: to calculate pages_volatile */
-static unsigned long ksm_rmap_items;
-
-/* Number of pages ksmd should scan in one batch */
-static unsigned int ksm_thread_pages_to_scan = 100;
+/*
+ * Number of pages ksmd should scan in one batch. This is the top speed for
+ * richly duplicated areas.
+ */
+static unsigned long ksm_scan_batch_pages = 60000;
 
 /* Milliseconds ksmd should sleep between batches */
-static unsigned int ksm_thread_sleep_millisecs = 20;
+static unsigned int ksm_sleep_jiffies = 2;
+
+/*
+ * The threshold used to filter out thrashing areas,
+ * If it == 0, filtering is disabled, otherwise it's the percentage up-bound
+ * of the thrashing ratio of all areas. Any area with a bigger thrashing ratio
+ * will be considered as having a zero duplication ratio.
+ */
+static unsigned int ksm_thrash_threshold = 50;
+
+/* To avoid the float point arithmetic, this is the scale of a
+ * deduplication ratio number.
+ */
+#define KSM_DEDUP_RATIO_SCALE  100
+
+
+#define KSM_SCAN_RATIO_MAX     125
+
+/* minimum scan ratio for a vma, in unit of 1/KSM_SCAN_RATIO_MAX */
+static unsigned int ksm_min_scan_ratio = 1;
+
+/*
+ * After each scan round, the scan ratio of an area with a big deduplication
+ * ratio is upgraded by *=ksm_scan_ratio_delta
+ */
+static unsigned int ksm_scan_ratio_delta = 5;
+
+/*
+ * Inter-vma duplication number table page pointer array, initialized at
+ * startup. Whenever ksmd finds that two areas have an identical page,
+ * their corresponding table entry is increased. After each scan round
+ * is finished, this table is scanned to calculate the estimated
+ * duplication ratio for VMAs. Limited number(2048) of VMAs are
+ * supported by now. We will migrate it to more scalable data structures
+ * in the future.
+ */
+#define KSM_DUP_VMA_MAX                2048
+
+#define INDIRECT_OFFSET                1
+
+/*
+ * For mapping of vma_slot and its index in inter-vma duplication number
+ * table
+ */
+static struct radix_tree_root ksm_vma_tree;
+static unsigned long ksm_vma_tree_num;
+static unsigned long ksm_vma_tree_index_end;
+
+/* Array of all scan_rung, ksm_scan_ladder[0] having the minimum scan ratio */
+static struct scan_rung *ksm_scan_ladder;
+static unsigned int ksm_scan_ladder_size;
+
+/* The number of VMAs we are keeping track of */
+static unsigned long ksm_vma_slot_num;
+
+/* How many times the ksmd has slept since startup */
+static u64 ksm_sleep_times;
 
 #define KSM_RUN_STOP   0
 #define KSM_RUN_MERGE  1
-#define KSM_RUN_UNMERGE        2
-static unsigned int ksm_run = KSM_RUN_STOP;
+static unsigned int ksm_run = KSM_RUN_MERGE;
 
 static DECLARE_WAIT_QUEUE_HEAD(ksm_thread_wait);
 static DEFINE_MUTEX(ksm_thread_mutex);
-static DEFINE_SPINLOCK(ksm_mmlist_lock);
 
-#define KSM_KMEM_CACHE(__struct, __flags) kmem_cache_create("ksm_"#__struct,\
-               sizeof(struct __struct), __alignof__(struct __struct),\
-               (__flags), NULL)
+/*
+ * List vma_slot_new is for newly created vma_slot waiting to be added by
+ * ksmd. If one cannot be added(e.g. due to it's too small), it's moved to
+ * vma_slot_noadd. vma_slot_del is the list for vma_slot whose corresponding
+ * VMA has been removed/freed.
+ */
+struct list_head vma_slot_new = LIST_HEAD_INIT(vma_slot_new);
+struct list_head vma_slot_noadd = LIST_HEAD_INIT(vma_slot_noadd);
+struct list_head vma_slot_del = LIST_HEAD_INIT(vma_slot_del);
+static DEFINE_SPINLOCK(vma_slot_list_lock);
 
-static int __init ksm_slab_init(void)
+/* The unstable tree heads */
+static struct rb_root root_unstable_tree = RB_ROOT;
+
+/*
+ * All tree_nodes are in a list to be freed at once when unstable tree is
+ * freed after each scan round.
+ */
+static struct list_head unstable_tree_node_list =
+                               LIST_HEAD_INIT(unstable_tree_node_list);
+
+/* List contains all stable nodes */
+static struct list_head stable_node_list = LIST_HEAD_INIT(stable_node_list);
+
+/*
+ * When the hash strength is changed, the stable tree must be delta_hashed and
+ * re-structured. We use two set of below structs to speed up the
+ * re-structuring of stable tree.
+ */
+static struct list_head
+stable_tree_node_list[2] = {LIST_HEAD_INIT(stable_tree_node_list[0]),
+                           LIST_HEAD_INIT(stable_tree_node_list[1])};
+
+static struct list_head *stable_tree_node_listp = &stable_tree_node_list[0];
+static struct rb_root root_stable_tree[2] = {RB_ROOT, RB_ROOT};
+static struct rb_root *root_stable_treep = &root_stable_tree[0];
+static unsigned long stable_tree_index;
+
+/* The hash strength needed to hash a full page */
+#define HASH_STRENGTH_FULL             (PAGE_SIZE / sizeof(u32))
+
+/* The hash strength needed for loop-back hashing */
+#define HASH_STRENGTH_MAX              (HASH_STRENGTH_FULL + 10)
+
+/* The random offsets in a page */
+static u32 *random_nums;
+
+/* The hash strength */
+static unsigned long hash_strength = HASH_STRENGTH_FULL >> 4;
+
+/* The delta value each time the hash strength increases or decreases */
+static unsigned long hash_strength_delta;
+#define HASH_STRENGTH_DELTA_MAX        5
+
+/* The time we have saved due to random_sample_hash */
+static u64 rshash_pos;
+
+/* The time we have wasted due to hash collision */
+static u64 rshash_neg;
+
+struct ksm_benefit {
+       u64 pos;
+       u64 neg;
+       u64 scanned;
+       unsigned long base;
+} benefit;
+
+/*
+ * The relative cost of memcmp, compared to 1 time unit of random sample
+ * hash, this value is tested when ksm module is initialized
+ */
+static unsigned long memcmp_cost;
+
+static unsigned long  rshash_neg_cont_zero;
+static unsigned long  rshash_cont_obscure;
+
+/* The possible states of hash strength adjustment heuristic */
+enum rshash_states {
+               RSHASH_STILL,
+               RSHASH_TRYUP,
+               RSHASH_TRYDOWN,
+               RSHASH_NEW,
+               RSHASH_PRE_STILL,
+};
+
+/* The possible direction we are about to adjust hash strength */
+enum rshash_direct {
+       GO_UP,
+       GO_DOWN,
+       OBSCURE,
+       STILL,
+};
+
+/* random sampling hash state machine */
+static struct {
+       enum rshash_states state;
+       enum rshash_direct pre_direct;
+       u8 below_count;
+       /* Keep a lookup window of size 5, iff above_count/below_count > 3
+        * in this window we stop trying.
+        */
+       u8 lookup_window_index;
+       u64 stable_benefit;
+       unsigned long turn_point_down;
+       unsigned long turn_benefit_down;
+       unsigned long turn_point_up;
+       unsigned long turn_benefit_up;
+       unsigned long stable_point;
+} rshash_state;
+
+/*zero page hash table, hash_strength [0 ~ HASH_STRENGTH_MAX]*/
+static u32 *zero_hash_table;
+
+static inline struct node_vma *alloc_node_vma(void)
 {
-       rmap_item_cache = KSM_KMEM_CACHE(rmap_item, 0);
-       if (!rmap_item_cache)
-               goto out;
+       struct node_vma *node_vma;
+       node_vma = kmem_cache_zalloc(node_vma_cache, GFP_KERNEL);
+       if (node_vma) {
+               INIT_HLIST_HEAD(&node_vma->rmap_hlist);
+               INIT_HLIST_NODE(&node_vma->hlist);
+               node_vma->last_update = 0;
+       }
+       return node_vma;
+}
 
-       stable_node_cache = KSM_KMEM_CACHE(stable_node, 0);
-       if (!stable_node_cache)
-               goto out_free1;
+static inline void free_node_vma(struct node_vma *node_vma)
+{
+       kmem_cache_free(node_vma_cache, node_vma);
+}
 
-       mm_slot_cache = KSM_KMEM_CACHE(mm_slot, 0);
-       if (!mm_slot_cache)
-               goto out_free2;
 
-       return 0;
+static inline struct vma_slot *alloc_vma_slot(void)
+{
+       struct vma_slot *slot;
 
-out_free2:
-       kmem_cache_destroy(stable_node_cache);
-out_free1:
-       kmem_cache_destroy(rmap_item_cache);
-out:
-       return -ENOMEM;
+       /*
+        * In case ksm is not initialized by now.
+        * Oops, we need to consider the call site of ksm_init() in the future.
+        */
+       if (!vma_slot_cache)
+               return NULL;
+
+       slot = kmem_cache_zalloc(vma_slot_cache, GFP_KERNEL);
+       if (slot) {
+               INIT_LIST_HEAD(&slot->ksm_list);
+               INIT_LIST_HEAD(&slot->slot_list);
+               INIT_RADIX_TREE(&slot->dup_tree, GFP_KERNEL);
+               slot->ksm_index = -1;
+               slot->need_rerand = 1;
+       }
+       return slot;
 }
 
-static void __init ksm_slab_free(void)
+static inline void free_vma_slot(struct vma_slot *vma_slot)
 {
-       kmem_cache_destroy(mm_slot_cache);
-       kmem_cache_destroy(stable_node_cache);
-       kmem_cache_destroy(rmap_item_cache);
-       mm_slot_cache = NULL;
+       kmem_cache_free(vma_slot_cache, vma_slot);
 }
 
+
+
 static inline struct rmap_item *alloc_rmap_item(void)
 {
        struct rmap_item *rmap_item;
 
        rmap_item = kmem_cache_zalloc(rmap_item_cache, GFP_KERNEL);
-       if (rmap_item)
-               ksm_rmap_items++;
+       if (rmap_item) {
+               /* bug on lowest bit is not clear for flag use */
+               BUG_ON(is_addr(rmap_item));
+       }
        return rmap_item;
 }
 
 static inline void free_rmap_item(struct rmap_item *rmap_item)
 {
-       ksm_rmap_items--;
-       rmap_item->mm = NULL;   /* debug safety */
+       rmap_item->slot = NULL; /* debug safety */
        kmem_cache_free(rmap_item_cache, rmap_item);
 }
 
 static inline struct stable_node *alloc_stable_node(void)
 {
-       return kmem_cache_alloc(stable_node_cache, GFP_KERNEL);
+       struct stable_node *node;
+       node = kmem_cache_alloc(stable_node_cache, GFP_KERNEL | GFP_ATOMIC);
+       if (!node)
+               return NULL;
+
+       INIT_HLIST_HEAD(&node->hlist);
+       list_add(&node->all_list, &stable_node_list);
+       return node;
 }
 
 static inline void free_stable_node(struct stable_node *stable_node)
 {
+       list_del(&stable_node->all_list);
        kmem_cache_free(stable_node_cache, stable_node);
 }
 
-static inline struct mm_slot *alloc_mm_slot(void)
+static inline struct tree_node *alloc_tree_node(struct list_head *list)
 {
-       if (!mm_slot_cache)     /* initialization failed */
+       struct tree_node *node;
+       node = kmem_cache_zalloc(tree_node_cache, GFP_KERNEL | GFP_ATOMIC);
+       if (!node)
                return NULL;
-       return kmem_cache_zalloc(mm_slot_cache, GFP_KERNEL);
+
+       list_add(&node->all_list, list);
+       return node;
 }
 
-static inline void free_mm_slot(struct mm_slot *mm_slot)
+static inline void free_tree_node(struct tree_node *node)
 {
-       kmem_cache_free(mm_slot_cache, mm_slot);
+       list_del(&node->all_list);
+       kmem_cache_free(tree_node_cache, node);
 }
 
-static struct mm_slot *get_mm_slot(struct mm_struct *mm)
+static void ksm_drop_anon_vma(struct rmap_item *rmap_item)
 {
-       struct mm_slot *mm_slot;
-       struct hlist_head *bucket;
-       struct hlist_node *node;
+       struct anon_vma *anon_vma = rmap_item->anon_vma;
 
-       bucket = &mm_slots_hash[hash_ptr(mm, MM_SLOTS_HASH_SHIFT)];
-       hlist_for_each_entry(mm_slot, node, bucket, link) {
-               if (mm == mm_slot->mm)
-                       return mm_slot;
-       }
-       return NULL;
+       drop_anon_vma(anon_vma);
 }
 
-static void insert_to_mm_slots_hash(struct mm_struct *mm,
-                                   struct mm_slot *mm_slot)
+
+/**
+ * Remove a stable node from stable_tree, may unlink from its tree_node and
+ * may remove its parent tree_node if no other stable node is pending.
+ *
+ * @stable_node        The node need to be removed
+ * @unlink_rb          Will this node be unlinked from the rbtree?
+ * @remove_tree_       node Will its tree_node be removed if empty?
+ */
+static void remove_node_from_stable_tree(struct stable_node *stable_node,
+                                        int unlink_rb,  int remove_tree_node)
 {
-       struct hlist_head *bucket;
+       struct node_vma *node_vma;
+       struct rmap_item *rmap_item;
+       struct hlist_node *hlist, *rmap_hlist, *n;
 
-       bucket = &mm_slots_hash[hash_ptr(mm, MM_SLOTS_HASH_SHIFT)];
-       mm_slot->mm = mm;
-       hlist_add_head(&mm_slot->link, bucket);
-}
+       if (!hlist_empty(&stable_node->hlist)) {
+               hlist_for_each_entry_safe(node_vma, hlist, n,
+                                         &stable_node->hlist, hlist) {
+                       hlist_for_each_entry(rmap_item, rmap_hlist,
+                                            &node_vma->rmap_hlist, hlist) {
+                               ksm_pages_sharing--;
 
-static inline int in_stable_tree(struct rmap_item *rmap_item)
-{
-       return rmap_item->address & STABLE_FLAG;
-}
+                               ksm_drop_anon_vma(rmap_item);
+                               rmap_item->address &= PAGE_MASK;
+                       }
+                       free_node_vma(node_vma);
+                       cond_resched();
+               }
 
-static void hold_anon_vma(struct rmap_item *rmap_item,
-                         struct anon_vma *anon_vma)
-{
-       rmap_item->anon_vma = anon_vma;
-       get_anon_vma(anon_vma);
-}
+               /* the last one is counted as shared */
+               ksm_pages_shared--;
+               ksm_pages_sharing++;
+       }
 
-static void ksm_drop_anon_vma(struct rmap_item *rmap_item)
-{
-       struct anon_vma *anon_vma = rmap_item->anon_vma;
+       if (stable_node->tree_node && unlink_rb) {
+               rb_erase(&stable_node->node,
+                        &stable_node->tree_node->sub_root);
+
+               if (RB_EMPTY_ROOT(&stable_node->tree_node->sub_root) &&
+                   remove_tree_node) {
+                       rb_erase(&stable_node->tree_node->node,
+                                root_stable_treep);
+                       free_tree_node(stable_node->tree_node);
+               } else {
+                       stable_node->tree_node->count--;
+               }
+       }
 
-       drop_anon_vma(anon_vma);
+       free_stable_node(stable_node);
 }
 
-/*
- * ksmd, and unmerge_and_remove_all_rmap_items(), must not touch an mm's
- * page tables after it has passed through ksm_exit() - which, if necessary,
- * takes mmap_sem briefly to serialize against them.  ksm_exit() does not set
- * a special flag: they can just back out as soon as mm_users goes to zero.
- * ksm_test_exit() is used throughout to make this test for exit: in some
- * places for correctness, in some places just to avoid unnecessary work.
- */
-static inline bool ksm_test_exit(struct mm_struct *mm)
-{
-       return atomic_read(&mm->mm_users) == 0;
-}
 
 /*
- * We use break_ksm to break COW on a ksm page: it's a stripped down
+ * get_ksm_page: checks if the page indicated by the stable node
+ * is still its ksm page, despite having held no reference to it.
+ * In which case we can trust the content of the page, and it
+ * returns the gotten page; but if the page has now been zapped,
+ * remove the stale node from the stable tree and return NULL.
  *
- *     if (get_user_pages(current, mm, addr, 1, 1, 1, &page, NULL) == 1)
- *             put_page(page);
+ * You would expect the stable_node to hold a reference to the ksm page.
+ * But if it increments the page's count, swapping out has to wait for
+ * ksmd to come around again before it can free the page, which may take
+ * seconds or even minutes: much too unresponsive.  So instead we use a
+ * "keyhole reference": access to the ksm page from the stable node peeps
+ * out through its keyhole to see if that page still holds the right key,
+ * pointing back to this stable node.  This relies on freeing a PageAnon
+ * page to reset its page->mapping to NULL, and relies on no other use of
+ * a page to put something that might look like our key in page->mapping.
  *
- * but taking great care only to touch a ksm page, in a VM_MERGEABLE vma,
- * in case the application has unmapped and remapped mm,addr meanwhile.
- * Could a ksm page appear anywhere else?  Actually yes, in a VM_PFNMAP
- * mmap of /dev/mem or /dev/kmem, where we would not want to touch it.
+ * include/linux/pagemap.h page_cache_get_speculative() is a good reference,
+ * but this is different - made simpler by ksm_thread_mutex being held, but
+ * interesting for assuming that no other use of the struct page could ever
+ * put our expected_mapping into page->mapping (or a field of the union which
+ * coincides with page->mapping).  The RCU calls are not for KSM at all, but
+ * to keep the page_count protocol described with page_cache_get_speculative.
+ *
+ * Note: it is possible that get_ksm_page() will return NULL one moment,
+ * then page the next, if the page is in between page_freeze_refs() and
+ * page_unfreeze_refs(): this shouldn't be a problem anywhere, the page
+ * is on its way to being freed; but it is an anomaly to bear in mind.
+ *
+ * @unlink_rb:                 if the removal of this node will firstly unlink from
+ * its rbtree. stable_node_reinsert will prevent this when restructuring the
+ * node from its old tree.
+ *
+ * @remove_tree_node:  if this is the last one of its tree_node, will the
+ * tree_node be freed ? If we are inserting stable node, this tree_node may
+ * be reused, so don't free it.
  */
-static int break_ksm(struct vm_area_struct *vma, unsigned long addr)
+static struct page *get_ksm_page(struct stable_node *stable_node,
+                                int unlink_rb, int remove_tree_node)
 {
        struct page *page;
-       int ret = 0;
+       void *expected_mapping;
 
-       do {
-               cond_resched();
-               page = follow_page(vma, addr, FOLL_GET);
-               if (IS_ERR_OR_NULL(page))
-                       break;
-               if (PageKsm(page))
-                       ret = handle_mm_fault(vma->vm_mm, vma, addr,
-                                                       FAULT_FLAG_WRITE);
-               else
-                       ret = VM_FAULT_WRITE;
-               put_page(page);
-       } while (!(ret & (VM_FAULT_WRITE | VM_FAULT_SIGBUS | VM_FAULT_OOM)));
-       /*
-        * We must loop because handle_mm_fault() may back out if there's
-        * any difficulty e.g. if pte accessed bit gets updated concurrently.
-        *
-        * VM_FAULT_WRITE is what we have been hoping for: it indicates that
-        * COW has been broken, even if the vma does not permit VM_WRITE;
-        * but note that a concurrent fault might break PageKsm for us.
-        *
-        * VM_FAULT_SIGBUS could occur if we race with truncation of the
-        * backing file, which also invalidates anonymous pages: that's
-        * okay, that truncation will have unmapped the PageKsm for us.
-        *
-        * VM_FAULT_OOM: at the time of writing (late July 2009), setting
-        * aside mem_cgroup limits, VM_FAULT_OOM would only be set if the
-        * current task has TIF_MEMDIE set, and will be OOM killed on return
-        * to user; and ksmd, having no mm, would never be chosen for that.
-        *
-        * But if the mm is in a limited mem_cgroup, then the fault may fail
-        * with VM_FAULT_OOM even if the current task is not TIF_MEMDIE; and
-        * even ksmd can fail in this way - though it's usually breaking ksm
-        * just to undo a merge it made a moment before, so unlikely to oom.
-        *
-        * That's a pity: we might therefore have more kernel pages allocated
-        * than we're counting as nodes in the stable tree; but ksm_do_scan
-        * will retry to break_cow on each pass, so should recover the page
-        * in due course.  The important thing is to not let VM_MERGEABLE
-        * be cleared while any such pages might remain in the area.
-        */
-       return (ret & VM_FAULT_OOM) ? -ENOMEM : 0;
-}
-
-static void break_cow(struct rmap_item *rmap_item)
-{
-       struct mm_struct *mm = rmap_item->mm;
-       unsigned long addr = rmap_item->address;
-       struct vm_area_struct *vma;
-
-       /*
-        * It is not an accident that whenever we want to break COW
-        * to undo, we also need to drop a reference to the anon_vma.
-        */
-       ksm_drop_anon_vma(rmap_item);
-
-       down_read(&mm->mmap_sem);
-       if (ksm_test_exit(mm))
-               goto out;
-       vma = find_vma(mm, addr);
-       if (!vma || vma->vm_start > addr)
-               goto out;
-       if (!(vma->vm_flags & VM_MERGEABLE) || !vma->anon_vma)
-               goto out;
-       break_ksm(vma, addr);
-out:
-       up_read(&mm->mmap_sem);
-}
-
-static struct page *page_trans_compound_anon(struct page *page)
-{
-       if (PageTransCompound(page)) {
-               struct page *head = compound_trans_head(page);
-               /*
-                * head may actually be splitted and freed from under
-                * us but it's ok here.
-                */
-               if (PageAnon(head))
-                       return head;
-       }
-       return NULL;
-}
-
-static struct page *get_mergeable_page(struct rmap_item *rmap_item)
-{
-       struct mm_struct *mm = rmap_item->mm;
-       unsigned long addr = rmap_item->address;
-       struct vm_area_struct *vma;
-       struct page *page;
-
-       down_read(&mm->mmap_sem);
-       if (ksm_test_exit(mm))
-               goto out;
-       vma = find_vma(mm, addr);
-       if (!vma || vma->vm_start > addr)
-               goto out;
-       if (!(vma->vm_flags & VM_MERGEABLE) || !vma->anon_vma)
-               goto out;
-
-       page = follow_page(vma, addr, FOLL_GET);
-       if (IS_ERR_OR_NULL(page))
-               goto out;
-       if (PageAnon(page) || page_trans_compound_anon(page)) {
-               flush_anon_page(vma, page, addr);
-               flush_dcache_page(page);
-       } else {
-               put_page(page);
-out:           page = NULL;
-       }
-       up_read(&mm->mmap_sem);
-       return page;
-}
-
-static void remove_node_from_stable_tree(struct stable_node *stable_node)
-{
-       struct rmap_item *rmap_item;
-       struct hlist_node *hlist;
-
-       hlist_for_each_entry(rmap_item, hlist, &stable_node->hlist, hlist) {
-               if (rmap_item->hlist.next)
-                       ksm_pages_sharing--;
-               else
-                       ksm_pages_shared--;
-               ksm_drop_anon_vma(rmap_item);
-               rmap_item->address &= PAGE_MASK;
-               cond_resched();
-       }
-
-       rb_erase(&stable_node->node, &root_stable_tree);
-       free_stable_node(stable_node);
-}
-
-/*
- * get_ksm_page: checks if the page indicated by the stable node
- * is still its ksm page, despite having held no reference to it.
- * In which case we can trust the content of the page, and it
- * returns the gotten page; but if the page has now been zapped,
- * remove the stale node from the stable tree and return NULL.
- *
- * You would expect the stable_node to hold a reference to the ksm page.
- * But if it increments the page's count, swapping out has to wait for
- * ksmd to come around again before it can free the page, which may take
- * seconds or even minutes: much too unresponsive.  So instead we use a
- * "keyhole reference": access to the ksm page from the stable node peeps
- * out through its keyhole to see if that page still holds the right key,
- * pointing back to this stable node.  This relies on freeing a PageAnon
- * page to reset its page->mapping to NULL, and relies on no other use of
- * a page to put something that might look like our key in page->mapping.
- *
- * include/linux/pagemap.h page_cache_get_speculative() is a good reference,
- * but this is different - made simpler by ksm_thread_mutex being held, but
- * interesting for assuming that no other use of the struct page could ever
- * put our expected_mapping into page->mapping (or a field of the union which
- * coincides with page->mapping).  The RCU calls are not for KSM at all, but
- * to keep the page_count protocol described with page_cache_get_speculative.
- *
- * Note: it is possible that get_ksm_page() will return NULL one moment,
- * then page the next, if the page is in between page_freeze_refs() and
- * page_unfreeze_refs(): this shouldn't be a problem anywhere, the page
- * is on its way to being freed; but it is an anomaly to bear in mind.
- */
-static struct page *get_ksm_page(struct stable_node *stable_node)
-{
-       struct page *page;
-       void *expected_mapping;
-
-       page = pfn_to_page(stable_node->kpfn);
-       expected_mapping = (void *)stable_node +
-                               (PAGE_MAPPING_ANON | PAGE_MAPPING_KSM);
-       rcu_read_lock();
-       if (page->mapping != expected_mapping)
-               goto stale;
-       if (!get_page_unless_zero(page))
-               goto stale;
-       if (page->mapping != expected_mapping) {
+       page = pfn_to_page(stable_node->kpfn);
+       expected_mapping = (void *)stable_node +
+                               (PAGE_MAPPING_ANON | PAGE_MAPPING_KSM);
+       rcu_read_lock();
+       if (page->mapping != expected_mapping)
+               goto stale;
+       if (!get_page_unless_zero(page))
+               goto stale;
+       if (page->mapping != expected_mapping) {
                put_page(page);
                goto stale;
        }
@@ -525,7 +650,8 @@
        return page;
 stale:
        rcu_read_unlock();
-       remove_node_from_stable_tree(stable_node);
+       remove_node_from_stable_tree(stable_node, unlink_rb, remove_tree_node);
+
        return NULL;
 }
 
@@ -533,32 +659,46 @@
  * Removing rmap_item from stable or unstable tree.
  * This function will clean the information from the stable/unstable tree.
  */
-static void remove_rmap_item_from_tree(struct rmap_item *rmap_item)
+static inline void remove_rmap_item_from_tree(struct rmap_item *rmap_item)
 {
        if (rmap_item->address & STABLE_FLAG) {
                struct stable_node *stable_node;
+               struct node_vma *node_vma;
                struct page *page;
 
-               stable_node = rmap_item->head;
-               page = get_ksm_page(stable_node);
+               node_vma = rmap_item->head;
+               stable_node = node_vma->head;
+               page = get_ksm_page(stable_node, 1, 1);
                if (!page)
                        goto out;
 
+               /*
+                * page lock is needed because it's racing with
+                * try_to_unmap_ksm(), etc.
+                */
                lock_page(page);
                hlist_del(&rmap_item->hlist);
+
+               if (hlist_empty(&node_vma->rmap_hlist)) {
+                       hlist_del(&node_vma->hlist);
+                       free_node_vma(node_vma);
+               }
                unlock_page(page);
-               put_page(page);
 
-               if (stable_node->hlist.first)
-                       ksm_pages_sharing--;
-               else
+               put_page(page);
+               if (hlist_empty(&stable_node->hlist)) {
+                       /* do NOT call remove_node_from_stable_tree() here,
+                        * it's possible for a forked rmap_item not in
+                        * stable tree while the in-tree rmap_items were
+                        * deleted.
+                        */
                        ksm_pages_shared--;
+               } else
+                       ksm_pages_sharing--;
 
-               ksm_drop_anon_vma(rmap_item);
-               rmap_item->address &= PAGE_MASK;
 
+               ksm_drop_anon_vma(rmap_item);
        } else if (rmap_item->address & UNSTABLE_FLAG) {
-               unsigned char age;
                /*
                 * Usually ksmd can and must skip the rb_erase, because
                 * root_unstable_tree was already reset to RB_ROOT.
@@ -566,169 +706,454 @@
                 * if this rmap_item was inserted by this scan, rather
                 * than left over from before.
                 */
-               age = (unsigned char)(ksm_scan.seqnr - rmap_item->address);
-               BUG_ON(age > 1);
-               if (!age)
-                       rb_erase(&rmap_item->node, &root_unstable_tree);
-
+               if (rmap_item->append_round == ksm_scan_round) {
+                       rb_erase(&rmap_item->node,
+                                &rmap_item->tree_node->sub_root);
+                       if (RB_EMPTY_ROOT(&rmap_item->tree_node->sub_root)) {
+                               rb_erase(&rmap_item->tree_node->node,
+                                        &root_unstable_tree);
+
+                               free_tree_node(rmap_item->tree_node);
+                       } else
+                               rmap_item->tree_node->count--;
+               }
                ksm_pages_unshared--;
-               rmap_item->address &= PAGE_MASK;
        }
+
+       rmap_item->address &= PAGE_MASK;
+       rmap_item->hash_max = 0;
+
 out:
        cond_resched();         /* we're called from many long loops */
 }
 
-static void remove_trailing_rmap_items(struct mm_slot *mm_slot,
-                                      struct rmap_item **rmap_list)
+/**
+ * Need to do two things:
+ * 1. check if slot was moved to del list
+ * 2. make sure the mmap_sem is manipulated under valid vma.
+ *
+ * My concern here is that in some cases, this may make
+ * vma_slot_list_lock() waiters to serialized further by some
+ * sem->wait_lock, can this really be expensive?
+ *
+ *
+ * @return
+ * 0: if successfully locked mmap_sem
+ * -ENOENT: this slot was moved to del list
+ * -EBUSY: vma lock failed
+ */
+static int try_down_read_slot_mmap_sem(struct vma_slot *slot)
 {
-       while (*rmap_list) {
-               struct rmap_item *rmap_item = *rmap_list;
-               *rmap_list = rmap_item->rmap_list;
-               remove_rmap_item_from_tree(rmap_item);
-               free_rmap_item(rmap_item);
+       struct vm_area_struct *vma;
+       struct mm_struct *mm;
+       struct rw_semaphore *sem;
+
+       spin_lock(&vma_slot_list_lock);
+
+       /* the slot_list was removed and inited from new list, when it enters
+        * ksm_list. If now it's not empty, then it must be moved to del list
+        */
+       if (!list_empty(&slot->slot_list)) {
+               spin_unlock(&vma_slot_list_lock);
+               return -ENOENT;
+       }
+
+       BUG_ON(slot->pages != vma_pages(slot->vma));
+       /* Ok, vma still valid */
+       vma = slot->vma;
+       mm = vma->vm_mm;
+       sem = &mm->mmap_sem;
+       if (down_read_trylock(sem)) {
+               spin_unlock(&vma_slot_list_lock);
+               return 0;
        }
+
+       spin_unlock(&vma_slot_list_lock);
+       return -EBUSY;
 }
 
-/*
- * Though it's very tempting to unmerge in_stable_tree(rmap_item)s rather
- * than check every pte of a given vma, the locking doesn't quite work for
- * that - an rmap_item is assigned to the stable tree after inserting ksm
- * page and upping mmap_sem.  Nor does it fit with the way we skip dup'ing
- * rmap_items from parent to child at fork time (so as not to waste time
- * if exit comes before the next scan reaches it).
- *
- * Similarly, although we'd like to remove rmap_items (so updating counts
- * and freeing memory) when unmerging an area, it's easier to leave that
- * to the next pass of ksmd - consider, for example, how ksmd might be
- * in cmp_and_merge_page on one of the rmap_items we would be removing.
- */
-static int unmerge_ksm_pages(struct vm_area_struct *vma,
-                            unsigned long start, unsigned long end)
+static inline unsigned long
+vma_page_address(struct page *page, struct vm_area_struct *vma)
 {
-       unsigned long addr;
-       int err = 0;
+       pgoff_t pgoff = page->index << (PAGE_CACHE_SHIFT - PAGE_SHIFT);
+       unsigned long address;
 
-       for (addr = start; addr < end && !err; addr += PAGE_SIZE) {
-               if (ksm_test_exit(vma->vm_mm))
-                       break;
-               if (signal_pending(current))
-                       err = -ERESTARTSYS;
-               else
-                       err = break_ksm(vma, addr);
+       address = vma->vm_start + ((pgoff - vma->vm_pgoff) << PAGE_SHIFT);
+       if (unlikely(address < vma->vm_start || address >= vma->vm_end)) {
+               /* page should be within @vma mapping range */
+               return -EFAULT;
        }
-       return err;
+       return address;
 }
 
-#ifdef CONFIG_SYSFS
 /*
- * Only called through the sysfs control interface:
+ * Test if the mm is exiting
  */
-static int unmerge_and_remove_all_rmap_items(void)
+static inline bool ksm_test_exit(struct mm_struct *mm)
+{
+       return atomic_read(&mm->mm_users) == 0;
+}
+
+/* return 0 on success with the item's mmap_sem locked */
+static inline int get_mergeable_page_lock_mmap(struct rmap_item *item)
 {
-       struct mm_slot *mm_slot;
        struct mm_struct *mm;
        struct vm_area_struct *vma;
-       int err = 0;
+       struct vma_slot *slot = item->slot;
+       int err = -EINVAL;
 
-       spin_lock(&ksm_mmlist_lock);
-       ksm_scan.mm_slot = list_entry(ksm_mm_head.mm_list.next,
-                                               struct mm_slot, mm_list);
-       spin_unlock(&ksm_mmlist_lock);
-
-       for (mm_slot = ksm_scan.mm_slot;
-                       mm_slot != &ksm_mm_head; mm_slot = ksm_scan.mm_slot) {
-               mm = mm_slot->mm;
-               down_read(&mm->mmap_sem);
-               for (vma = mm->mmap; vma; vma = vma->vm_next) {
-                       if (ksm_test_exit(mm))
-                               break;
-                       if (!(vma->vm_flags & VM_MERGEABLE) || !vma->anon_vma)
-                               continue;
-                       err = unmerge_ksm_pages(vma,
-                                               vma->vm_start, vma->vm_end);
-                       if (err)
-                               goto error;
-               }
-
-               remove_trailing_rmap_items(mm_slot, &mm_slot->rmap_list);
-
-               spin_lock(&ksm_mmlist_lock);
-               ksm_scan.mm_slot = list_entry(mm_slot->mm_list.next,
-                                               struct mm_slot, mm_list);
-               if (ksm_test_exit(mm)) {
-                       hlist_del(&mm_slot->link);
-                       list_del(&mm_slot->mm_list);
-                       spin_unlock(&ksm_mmlist_lock);
-
-                       free_mm_slot(mm_slot);
-                       clear_bit(MMF_VM_MERGEABLE, &mm->flags);
-                       up_read(&mm->mmap_sem);
-                       mmdrop(mm);
-               } else {
-                       spin_unlock(&ksm_mmlist_lock);
-                       up_read(&mm->mmap_sem);
-               }
+       struct page *page;
+
+       BUG_ON(!item->slot);
+       /*
+        * try_down_read_slot_mmap_sem() returns non-zero if the slot
+        * has been removed by ksm_remove_vma().
+        */
+       if (try_down_read_slot_mmap_sem(slot))
+               return -EBUSY;
+
+       mm = slot->vma->vm_mm;
+       vma = slot->vma;
+
+       if (ksm_test_exit(mm))
+               goto failout_up;
+
+       page = item->page;
+       rcu_read_lock();
+       if (!get_page_unless_zero(page)) {
+               rcu_read_unlock();
+               goto failout_up;
        }
 
-       ksm_scan.seqnr = 0;
+       /* No need to consider huge page here. */
+       if (item->slot->vma->anon_vma != page_anon_vma(page) ||
+           vma_page_address(page, item->slot->vma) != get_rmap_addr(item)) {
+               /*
+                * TODO:
+                * should we release this item becase of its stale page
+                * mapping?
+                */
+               put_page(page);
+               rcu_read_unlock();
+               goto failout_up;
+       }
+       rcu_read_unlock();
        return 0;
 
-error:
+failout_up:
        up_read(&mm->mmap_sem);
-       spin_lock(&ksm_mmlist_lock);
-       ksm_scan.mm_slot = &ksm_mm_head;
-       spin_unlock(&ksm_mmlist_lock);
        return err;
 }
-#endif /* CONFIG_SYSFS */
 
-static u32 calc_checksum(struct page *page)
+/*
+ * What kind of VMA is considered ?
+ */
+static inline int vma_can_enter(struct vm_area_struct *vma)
 {
-       u32 checksum;
-       void *addr = kmap_atomic(page, KM_USER0);
-       checksum = jhash2(addr, PAGE_SIZE / 4, 17);
-       kunmap_atomic(addr, KM_USER0);
-       return checksum;
+       return !(vma->vm_flags & (VM_PFNMAP | VM_IO  | VM_DONTEXPAND |
+                                 VM_RESERVED  | VM_HUGETLB | VM_INSERTPAGE |
+                                 VM_NONLINEAR | VM_MIXEDMAP | VM_SAO |
+                                 VM_SHARED  | VM_MAYSHARE | VM_GROWSUP
+                                 | VM_GROWSDOWN));
 }
 
-static int memcmp_pages(struct page *page1, struct page *page2)
+/*
+ * Called whenever a fresh new vma is created A new vma_slot.
+ * is created and inserted into a global list Must be called.
+ * after vma is inserted to its mm                         .
+ */
+inline void ksm_vma_add_new(struct vm_area_struct *vma)
 {
-       char *addr1, *addr2;
-       int ret;
+       struct vma_slot *slot;
 
-       addr1 = kmap_atomic(page1, KM_USER0);
-       addr2 = kmap_atomic(page2, KM_USER1);
-       ret = memcmp(addr1, addr2, PAGE_SIZE);
-       kunmap_atomic(addr2, KM_USER1);
-       kunmap_atomic(addr1, KM_USER0);
-       return ret;
-}
+       if (!vma_can_enter(vma)) {
+               vma->ksm_vma_slot = NULL;
+               return;
+       }
 
-static inline int pages_identical(struct page *page1, struct page *page2)
-{
-       return !memcmp_pages(page1, page2);
+       slot = alloc_vma_slot();
+       if (!slot) {
+               vma->ksm_vma_slot = NULL;
+               return;
+       }
+
+       vma->ksm_vma_slot = slot;
+       slot->vma = vma;
+       slot->mm = vma->vm_mm;
+       slot->ctime_j = jiffies;
+       slot->pages = vma_pages(vma);
+       spin_lock(&vma_slot_list_lock);
+       list_add_tail(&slot->slot_list, &vma_slot_new);
+       spin_unlock(&vma_slot_list_lock);
 }
 
-static int write_protect_page(struct vm_area_struct *vma, struct page *page,
-                             pte_t *orig_pte)
+/*
+ * Called after vma is unlinked from its mm
+ */
+void ksm_remove_vma(struct vm_area_struct *vma)
 {
-       struct mm_struct *mm = vma->vm_mm;
-       unsigned long addr;
-       pte_t *ptep;
-       spinlock_t *ptl;
-       int swapped;
-       int err = -EFAULT;
+       struct vma_slot *slot;
 
-       addr = page_address_in_vma(page, vma);
-       if (addr == -EFAULT)
-               goto out;
+       if (!vma->ksm_vma_slot)
+               return;
 
-       BUG_ON(PageTransCompound(page));
-       ptep = page_check_address(page, mm, addr, &ptl, 0);
-       if (!ptep)
-               goto out;
+       slot = vma->ksm_vma_slot;
+       spin_lock(&vma_slot_list_lock);
+       if (list_empty(&slot->slot_list)) {
+               /**
+                * This slot has been added by ksmd, so move to the del list
+                * waiting ksmd to free it.
+                */
+               list_add_tail(&slot->slot_list, &vma_slot_del);
+       } else {
+               /**
+                * It's still on new list. It's ok to free slot directly.
+                */
+               list_del(&slot->slot_list);
+               free_vma_slot(slot);
+       }
+       spin_unlock(&vma_slot_list_lock);
+       vma->ksm_vma_slot = NULL;
+}
 
-       if (pte_write(*ptep) || pte_dirty(*ptep)) {
+/*   32/3 < they < 32/2 */
+#define shiftl 8
+#define shiftr 12
+
+#define HASH_FROM_TO(from, to)                                 \
+for (index = from; index < to; index++) {              \
+       pos = random_nums[index];                       \
+       hash += key[pos];                               \
+       hash += (hash << shiftl);                       \
+       hash ^= (hash >> shiftr);                       \
+}
+
+
+#define HASH_FROM_DOWN_TO(from, to)                    \
+for (index = from - 1; index >= to; index--) {         \
+       hash ^= (hash >> shiftr);                       \
+       hash ^= (hash >> (shiftr*2));                   \
+       hash -= (hash << shiftl);                       \
+       hash += (hash << (shiftl*2));                   \
+       pos = random_nums[index];                       \
+       hash -= key[pos];                               \
+}
+
+/*
+ * The main random sample hash function.
+ */
+static u32 random_sample_hash(void *addr, u32 hash_strength)
+{
+       u32 hash = 0xdeadbeef;
+       int index, pos, loop = hash_strength;
+       u32 *key = (u32 *)addr;
+
+       if (loop > HASH_STRENGTH_FULL)
+               loop = HASH_STRENGTH_FULL;
+
+       HASH_FROM_TO(0, loop);
+
+       if (hash_strength > HASH_STRENGTH_FULL) {
+               loop = hash_strength - HASH_STRENGTH_FULL;
+               HASH_FROM_TO(0, loop);
+       }
+
+       return hash;
+}
+
+
+/**
+ * It's used when hash strength is adjusted
+ *
+ * @addr The page's virtual address
+ * @from The original hash strength
+ * @to   The hash strength changed to
+ * @hash The hash value generated with "from" hash value
+ *
+ * return the hash value
+ */
+static u32 delta_hash(void *addr, int from, int to, u32 hash)
+{
+       u32 *key = (u32 *)addr;
+       int index, pos; /* make sure they are int type */
+
+       if (to > from) {
+               if (from >= HASH_STRENGTH_FULL) {
+                       from -= HASH_STRENGTH_FULL;
+                       to -= HASH_STRENGTH_FULL;
+                       HASH_FROM_TO(from, to);
+               } else if (to <= HASH_STRENGTH_FULL) {
+                       HASH_FROM_TO(from, to);
+               } else {
+                       HASH_FROM_TO(from, HASH_STRENGTH_FULL);
+                       HASH_FROM_TO(0, to - HASH_STRENGTH_FULL);
+               }
+       } else {
+               if (from <= HASH_STRENGTH_FULL) {
+                       HASH_FROM_DOWN_TO(from, to);
+               } else if (to >= HASH_STRENGTH_FULL) {
+                       from -= HASH_STRENGTH_FULL;
+                       to -= HASH_STRENGTH_FULL;
+                       HASH_FROM_DOWN_TO(from, to);
+               } else {
+                       HASH_FROM_DOWN_TO(from - HASH_STRENGTH_FULL, 0);
+                       HASH_FROM_DOWN_TO(HASH_STRENGTH_FULL, to);
+               }
+       }
+
+       return hash;
+}
+
+
+
+
+#define CAN_OVERFLOW_U64(x, delta) (U64_MAX - (x) < (delta))
+
+/**
+ *
+ * Called when: rshash_pos or rshash_neg is about to overflow or a scan round
+ * has finished.
+ *
+ */
+static inline void encode_benefit(void)
+{
+       u64 scanned_delta, pos_delta, neg_delta;
+       unsigned long base = benefit.base;
+
+       scanned_delta = (ksm_pages_scanned - ksm_pages_scanned_last) >> base;
+       pos_delta = rshash_pos >> base;
+       neg_delta = rshash_neg >> base;
+
+       if (CAN_OVERFLOW_U64(benefit.pos, pos_delta) ||
+           CAN_OVERFLOW_U64(benefit.neg, neg_delta) ||
+           CAN_OVERFLOW_U64(benefit.scanned, scanned_delta)) {
+               benefit.scanned >>= 1;
+               benefit.neg >>= 1;
+               benefit.pos >>= 1;
+               benefit.base++;
+               scanned_delta >>= 1;
+               pos_delta >>= 1;
+               neg_delta >>= 1;
+       }
+
+       benefit.pos += pos_delta;
+       benefit.neg += neg_delta;
+       benefit.scanned += scanned_delta;
+
+       BUG_ON(!benefit.scanned);
+
+       rshash_pos = rshash_neg = 0;
+
+       /* -1 to make rshash_adjust() work */
+       ksm_pages_scanned_last = ksm_pages_scanned - 1;
+}
+
+static inline void reset_benefit(void)
+{
+       benefit.pos = 0;
+       benefit.neg = 0;
+       benefit.base = 0;
+       benefit.scanned = 0;
+}
+
+static inline void inc_rshash_pos(unsigned long delta)
+{
+       if (CAN_OVERFLOW_U64(rshash_pos, delta))
+               encode_benefit();
+
+       rshash_pos += delta;
+}
+
+static inline void inc_rshash_neg(unsigned long delta)
+{
+       if (CAN_OVERFLOW_U64(rshash_neg, delta))
+               encode_benefit();
+
+       rshash_neg += delta;
+}
+
+
+static inline u32 page_hash(struct page *page, unsigned long hash_strength,
+                           int cost_accounting)
+{
+       u32 val;
+       unsigned long delta;
+
+       void *addr = kmap_atomic(page, KM_USER0);
+
+       val = random_sample_hash(addr, hash_strength);
+       kunmap_atomic(addr, KM_USER0);
+
+       if (cost_accounting) {
+               if (HASH_STRENGTH_FULL > hash_strength)
+                       delta = HASH_STRENGTH_FULL - hash_strength;
+               else
+                       delta = 0;
+
+               inc_rshash_pos(delta);
+       }
+
+       return val;
+}
+
+static int memcmp_pages(struct page *page1, struct page *page2,
+                       int cost_accounting)
+{
+       char *addr1, *addr2;
+       int ret;
+
+       addr1 = kmap_atomic(page1, KM_USER0);
+       addr2 = kmap_atomic(page2, KM_USER1);
+       ret = memcmp(addr1, addr2, PAGE_SIZE);
+       kunmap_atomic(addr2, KM_USER1);
+       kunmap_atomic(addr1, KM_USER0);
+
+       if (cost_accounting)
+               inc_rshash_neg(memcmp_cost);
+
+       return ret;
+}
+
+static inline int pages_identical(struct page *page1, struct page *page2)
+{
+       return !memcmp_pages(page1, page2, 0);
+}
+
+static inline int is_page_full_zero(struct page *page)
+{
+       char *addr;
+       int ret;
+
+       addr = kmap_atomic(page, KM_USER0);
+       ret = is_full_zero(addr, PAGE_SIZE);
+       kunmap_atomic(addr, KM_USER0);
+
+       return ret;
+}
+
+static int write_protect_page(struct vm_area_struct *vma, struct page *page,
+                             pte_t *orig_pte, pte_t *old_pte)
+{
+       struct mm_struct *mm = vma->vm_mm;
+       unsigned long addr;
+       pte_t *ptep;
+       spinlock_t *ptl;
+       int swapped;
+       int err = -EFAULT;
+
+       addr = page_address_in_vma(page, vma);
+       if (addr == -EFAULT)
+               goto out;
+
+       BUG_ON(PageTransCompound(page));
+       ptep = page_check_address(page, mm, addr, &ptl, 0);
+       if (!ptep)
+               goto out;
+
+       if (old_pte)
+               *old_pte = *ptep;
+
+       if (pte_write(*ptep) || pte_dirty(*ptep)) {
                pte_t entry;
 
                swapped = PageSwapCache(page);
@@ -765,6 +1190,11 @@
        return err;
 }
 
+#define MERGE_ERR_PGERR                1 /* the page is invalid cannot continue */
+#define MERGE_ERR_COLLI                2 /* there is a collision */
+#define MERGE_ERR_CHANGED      3 /* the page has changed since last hash */
+
+
 /**
  * replace_page - replace page in vma by new ksm page
  * @vma:      vma that holds the pte pointing to page
@@ -772,7 +1202,7 @@
  * @kpage:    the ksm page we replace page by
  * @orig_pte: the original value of the pte
  *
- * Returns 0 on success, -EFAULT on failure.
+ * Returns 0 on success, MERGE_ERR_PGERR on failure.
  */
 static int replace_page(struct vm_area_struct *vma, struct page *page,
                        struct page *kpage, pte_t orig_pte)
@@ -783,8 +1213,10 @@
        pmd_t *pmd;
        pte_t *ptep;
        spinlock_t *ptl;
+       pte_t entry;
+
        unsigned long addr;
-       int err = -EFAULT;
+       int err = MERGE_ERR_PGERR;
 
        addr = page_address_in_vma(page, vma);
        if (addr == -EFAULT)
@@ -809,12 +1241,20 @@
                goto out;
        }
 
-       get_page(kpage);
-       page_add_anon_rmap(kpage, vma, addr);
-
        flush_cache_page(vma, addr, pte_pfn(*ptep));
        ptep_clear_flush(vma, addr, ptep);
-       set_pte_at_notify(mm, addr, ptep, mk_pte(kpage, vma->vm_page_prot));
+       entry = mk_pte(kpage, vma->vm_page_prot);
+
+       /* special treatment is needed for zero_page */
+       if ((page_to_pfn(kpage) == ksm_zero_pfn) ||
+                               (page_to_pfn(kpage) == zero_pfn))
+               entry = pte_mkspecial(entry);
+       else {
+               get_page(kpage);
+               page_add_anon_rmap(kpage, vma, addr);
+       }
+
+       set_pte_at_notify(mm, addr, ptep, entry);
 
        page_remove_rmap(page);
        if (!page_mapped(page))
@@ -827,6 +1267,85 @@
        return err;
 }
 
+
+/**
+ *  Fully hash a page with HASH_STRENGTH_MAX return a non-zero hash value. The
+ *  zero hash value at HASH_STRENGTH_MAX is used to indicated that its
+ *  hash_max member has not been calculated.
+ *
+ * @page The page needs to be hashed
+ * @hash_old The hash value calculated with current hash strength
+ *
+ * return the new hash value calculated at HASH_STRENGTH_MAX
+ */
+static inline u32 page_hash_max(struct page *page, u32 hash_old)
+{
+       u32 hash_max = 0;
+       void *addr;
+
+       addr = kmap_atomic(page, KM_USER0);
+       hash_max = delta_hash(addr, hash_strength,
+                             HASH_STRENGTH_MAX, hash_old);
+
+       kunmap_atomic(addr, KM_USER0);
+
+       if (!hash_max)
+               hash_max = 1;
+
+       inc_rshash_neg(HASH_STRENGTH_MAX - hash_strength);
+       return hash_max;
+}
+
+/*
+ * We compare the hash again, to ensure that it is really a hash collision
+ * instead of being caused by page write.
+ */
+static inline int check_collision(struct rmap_item *rmap_item,
+                                 u32 hash)
+{
+       int err;
+       struct page *page = rmap_item->page;
+
+       /* if this rmap_item has already been hash_maxed, then the collision
+        * must appears in the second-level rbtree search. In this case we check
+        * if its hash_max value has been changed. Otherwise, the collision
+        * happens in the first-level rbtree search, so we check against it's
+        * current hash value.
+        */
+       if (rmap_item->hash_max) {
+               inc_rshash_neg(memcmp_cost);
+               inc_rshash_neg(HASH_STRENGTH_MAX - hash_strength);
+
+               if (rmap_item->hash_max == page_hash_max(page, hash))
+                       err = MERGE_ERR_COLLI;
+               else
+                       err = MERGE_ERR_CHANGED;
+       } else {
+               inc_rshash_neg(memcmp_cost + hash_strength);
+
+               if (page_hash(page, hash_strength, 0) == hash)
+                       err = MERGE_ERR_COLLI;
+               else
+                       err = MERGE_ERR_CHANGED;
+       }
+
+       return err;
+}
+
+static struct page *page_trans_compound_anon(struct page *page)
+{
+       if (PageTransCompound(page)) {
+               struct page *head = compound_trans_head(page);
+               /*
+                * head may actually be splitted and freed from under
+                * us but it's ok here.
+                */
+               if (PageAnon(head))
+                       return head;
+       }
+       return NULL;
+}
+
 static int page_trans_compound_anon_split(struct page *page)
 {
        int ret = 0;
@@ -854,30 +1373,36 @@
        return ret;
 }
 
-/*
- * try_to_merge_one_page - take two pages and merge them into one
- * @vma: the vma that holds the pte pointing to page
- * @page: the PageAnon page that we want to replace with kpage
- * @kpage: the PageKsm page that we want to map instead of page,
- *         or NULL the first time when we want to use page as kpage.
+/**
+ * Try to merge a rmap_item.page with a kpage in stable node. kpage must
+ * already be a ksm page.
  *
- * This function returns 0 if the pages were merged, -EFAULT otherwise.
+ * @return 0 if the pages were merged, -EFAULT otherwise.
  */
-static int try_to_merge_one_page(struct vm_area_struct *vma,
-                                struct page *page, struct page *kpage)
+static int try_to_merge_with_ksm_page(struct rmap_item *rmap_item,
+                                     struct page *kpage, u32 hash)
 {
+       struct vm_area_struct *vma = rmap_item->slot->vma;
+       struct mm_struct *mm = vma->vm_mm;
        pte_t orig_pte = __pte(0);
-       int err = -EFAULT;
+       int err = MERGE_ERR_PGERR;
+       struct page *page;
 
-       if (page == kpage)                      /* ksm page forked */
-               return 0;
+       if (ksm_test_exit(mm))
+               goto out;
+
+       page = rmap_item->page;
 
-       if (!(vma->vm_flags & VM_MERGEABLE))
+       if (page == kpage) { /* ksm page forked */
+               err = 0;
                goto out;
+       }
+
        if (PageTransCompound(page) && page_trans_compound_anon_split(page))
                goto out;
        BUG_ON(PageTransCompound(page));
-       if (!PageAnon(page))
+
+       if (!PageAnon(page) || !PageKsm(kpage))
                goto out;
 
        /*
@@ -895,18 +1420,27 @@
         * ptes are necessarily already write-protected.  But in either
         * case, we need to lock and check page_count is not raised.
         */
-       if (write_protect_page(vma, page, &orig_pte) == 0) {
+       if (write_protect_page(vma, page, &orig_pte, NULL) == 0) {
                if (!kpage) {
+                       long map_sharing = atomic_read(&page->_mapcount);
                        /*
                         * While we hold page lock, upgrade page from
                         * PageAnon+anon_vma to PageKsm+NULL stable_node:
                         * stable_tree_insert() will update stable_node.
                         */
                        set_page_stable_node(page, NULL);
+                       if (map_sharing)
+                               add_zone_page_state(page_zone(page),
+                                                   NR_KSM_PAGES_SHARING,
+                                                   map_sharing);
                        mark_page_accessed(page);
                        err = 0;
-               } else if (pages_identical(page, kpage))
-                       err = replace_page(vma, page, kpage, orig_pte);
+               } else {
+                       if (pages_identical(page, kpage))
+                               err = replace_page(vma, page, kpage, orig_pte);
+                       else
+                               err = check_collision(rmap_item, hash);
+               }
        }
 
        if ((vma->vm_flags & VM_LOCKED) && kpage && !err) {
@@ -924,378 +1458,2697 @@
        return err;
 }
 
-/*
- * try_to_merge_with_ksm_page - like try_to_merge_two_pages,
- * but no new kernel page is allocated: kpage must already be a ksm page.
+
+
+/**
+ * If two pages fail to merge in try_to_merge_two_pages, then we have a chance
+ * to restore a page mapping that has been changed in try_to_merge_two_pages.
  *
- * This function returns 0 if the pages were merged, -EFAULT otherwise.
+ * @return 0 on success.
  */
-static int try_to_merge_with_ksm_page(struct rmap_item *rmap_item,
-                                     struct page *page, struct page *kpage)
+static int restore_ksm_page_pte(struct vm_area_struct *vma, unsigned long addr,
+                            pte_t orig_pte, pte_t wprt_pte)
 {
-       struct mm_struct *mm = rmap_item->mm;
-       struct vm_area_struct *vma;
+       struct mm_struct *mm = vma->vm_mm;
+       pgd_t *pgd;
+       pud_t *pud;
+       pmd_t *pmd;
+       pte_t *ptep;
+       spinlock_t *ptl;
+
        int err = -EFAULT;
 
-       down_read(&mm->mmap_sem);
-       if (ksm_test_exit(mm))
-               goto out;
-       vma = find_vma(mm, rmap_item->address);
-       if (!vma || vma->vm_start > rmap_item->address)
+       pgd = pgd_offset(mm, addr);
+       if (!pgd_present(*pgd))
+               goto out;
+
+       pud = pud_offset(pgd, addr);
+       if (!pud_present(*pud))
+               goto out;
+
+       pmd = pmd_offset(pud, addr);
+       if (!pmd_present(*pmd))
+               goto out;
+
+       ptep = pte_offset_map_lock(mm, pmd, addr, &ptl);
+       if (!pte_same(*ptep, wprt_pte)) {
+               /* already copied, let it be */
+               pte_unmap_unlock(ptep, ptl);
+               goto out;
+       }
+
+       /*
+        * Good boy, still here. When we still get the ksm page, it does not
+        * return to the free page pool, there is no way that a pte was changed
+        * to other page and gets back to this page. And remind that ksm page
+        * do not reuse in do_wp_page(). So it's safe to restore the original
+        * pte.
+        */
+       flush_cache_page(vma, addr, pte_pfn(*ptep));
+       ptep_clear_flush(vma, addr, ptep);
+       set_pte_at_notify(mm, addr, ptep, orig_pte);
+
+       pte_unmap_unlock(ptep, ptl);
+       err = 0;
+out:
+       return err;
+}
+
+/**
+ * try_to_merge_two_pages() - take two identical pages and prepare
+ * them to be merged into one page(rmap_item->page)
+ *
+ * @return 0 if we successfully merged two identical pages into
+ *         one ksm page. MERGE_ERR_COLLI if it's only a hash collision
+ *         search in rbtree. MERGE_ERR_CHANGED if rmap_item has been
+ *         changed since it's hashed. MERGE_ERR_PGERR otherwise.
+ *
+ */
+static int try_to_merge_two_pages(struct rmap_item *rmap_item,
+                                 struct rmap_item *tree_rmap_item,
+                                 u32 hash)
+{
+       pte_t orig_pte1 = __pte(0), orig_pte2 = __pte(0);
+       pte_t wprt_pte1 = __pte(0), wprt_pte2 = __pte(0);
+       struct vm_area_struct *vma1 = rmap_item->slot->vma;
+       struct vm_area_struct *vma2 = tree_rmap_item->slot->vma;
+       struct page *page = rmap_item->page;
+       struct page *tree_page = tree_rmap_item->page;
+       int err = MERGE_ERR_PGERR;
+
+       long map_sharing;
+       struct address_space *saved_mapping;
+
+
+       if (rmap_item->page == tree_rmap_item->page)
+               goto out;
+
+       if (PageTransCompound(page) && page_trans_compound_anon_split(page))
+               goto out;
+       BUG_ON(PageTransCompound(page));
+
+       if (PageTransCompound(tree_page) && page_trans_compound_anon_split(tree_page))
+               goto out;
+       BUG_ON(PageTransCompound(tree_page));
+
+       if (!PageAnon(page) || !PageAnon(tree_page))
+               goto out;
+
+       if (!trylock_page(page))
+               goto out;
+
+
+       if (write_protect_page(vma1, page, &wprt_pte1, &orig_pte1) != 0) {
+               unlock_page(page);
+               goto out;
+       }
+
+       /*
+        * While we hold page lock, upgrade page from
+        * PageAnon+anon_vma to PageKsm+NULL stable_node:
+        * stable_tree_insert() will update stable_node.
+        */
+       saved_mapping = page->mapping;
+       map_sharing = atomic_read(&page->_mapcount);
+       set_page_stable_node(page, NULL);
+       if (map_sharing)
+               add_zone_page_state(page_zone(page),
+                                   NR_KSM_PAGES_SHARING,
+                                   map_sharing);
+       mark_page_accessed(page);
+       unlock_page(page);
+
+       if (!trylock_page(tree_page))
+               goto restore_out;
+
+       if (write_protect_page(vma2, tree_page, &wprt_pte2, &orig_pte2) != 0) {
+               unlock_page(tree_page);
+               goto restore_out;
+       }
+
+       if (pages_identical(page, tree_page)) {
+               err = replace_page(vma2, tree_page, page, wprt_pte2);
+               if (err)
+                       goto restore_out;
+
+               if ((vma2->vm_flags & VM_LOCKED)) {
+                       munlock_vma_page(tree_page);
+                       if (!PageMlocked(page)) {
+                               unlock_page(tree_page);
+                               lock_page(page);
+                               mlock_vma_page(page);
+                               tree_page = page; /* for final unlock */
+                       }
+               }
+
+               unlock_page(tree_page);
+
+               goto out; /* success */
+
+       } else {
+               if (page_hash(page, hash_strength, 0) ==
+                   page_hash(tree_page, hash_strength, 0)) {
+                       inc_rshash_neg(memcmp_cost + hash_strength * 2);
+                       err = MERGE_ERR_COLLI;
+               } else
+                       err = MERGE_ERR_CHANGED;
+
+               unlock_page(tree_page);
+       }
+
+restore_out:
+       lock_page(page);
+       if (!restore_ksm_page_pte(vma1, get_rmap_addr(rmap_item),
+                                 orig_pte1, wprt_pte1))
+               page->mapping = saved_mapping;
+
+       unlock_page(page);
+out:
+       return err;
+}
+
+static inline int hash_cmp(u32 new_val, u32 node_val)
+{
+       if (new_val > node_val)
+               return 1;
+       else if (new_val < node_val)
+               return -1;
+       else
+               return 0;
+}
+
+static inline u32 rmap_item_hash_max(struct rmap_item *item, u32 hash)
+{
+       u32 hash_max = item->hash_max;
+
+       if (!hash_max) {
+               hash_max = page_hash_max(item->page, hash);
+
+               item->hash_max = hash_max;
+       }
+
+       return hash_max;
+}
+
+
+
+/**
+ * stable_tree_search() - search the stable tree for a page
+ *
+ * @item:      the rmap_item we are comparing with
+ * @hash:      the hash value of this item->page already calculated
+ *
+ * @return     the page we have found, NULL otherwise. The page returned has
+ *             been gotten.
+ */
+static struct page *stable_tree_search(struct rmap_item *item, u32 hash)
+{
+       struct rb_node *node = root_stable_treep->rb_node;
+       struct tree_node *tree_node;
+       unsigned long hash_max;
+       struct page *page = item->page;
+       struct stable_node *stable_node;
+
+       stable_node = page_stable_node(page);
+       if (stable_node) {
+               /* ksm page forked, that is
+                * if (PageKsm(page) && !in_stable_tree(rmap_item))
+                * it's actually gotten once outside.
+                */
+               get_page(page);
+               return page;
+       }
+
+       while (node) {
+               int cmp;
+
+               tree_node = rb_entry(node, struct tree_node, node);
+
+               cmp = hash_cmp(hash, tree_node->hash);
+
+               if (cmp < 0)
+                       node = node->rb_left;
+               else if (cmp > 0)
+                       node = node->rb_right;
+               else
+                       break;
+       }
+
+       if (!node)
+               return NULL;
+
+       if (tree_node->count == 1) {
+               stable_node = rb_entry(tree_node->sub_root.rb_node,
+                                      struct stable_node, node);
+               BUG_ON(!stable_node);
+
+               goto get_page_out;
+       }
+
+       /*
+        * ok, we have to search the second
+        * level subtree, hash the page to a
+        * full strength.
+        */
+       node = tree_node->sub_root.rb_node;
+       BUG_ON(!node);
+       hash_max = rmap_item_hash_max(item, hash);
+
+       while (node) {
+               int cmp;
+
+               stable_node = rb_entry(node, struct stable_node, node);
+
+               cmp = hash_cmp(hash_max, stable_node->hash_max);
+
+               if (cmp < 0)
+                       node = node->rb_left;
+               else if (cmp > 0)
+                       node = node->rb_right;
+               else
+                       goto get_page_out;
+       }
+
+       return NULL;
+
+get_page_out:
+       page = get_ksm_page(stable_node, 1, 1);
+       return page;
+}
+
+
+/**
+ * try_to_merge_with_stable_page() - when two rmap_items need to be inserted
+ * into stable tree, the page was found to be identical to a stable ksm page,
+ * this is the last chance we can merge them into one.
+ *
+ * @item1:     the rmap_item holding the page which we wanted to insert
+ *             into stable tree.
+ * @item2:     the other rmap_item we found when unstable tree search
+ * @oldpage:   the page currently mapped by the two rmap_items
+ * @tree_page:         the page we found identical in stable tree node
+ * @success1:  return if item1 is successfully merged
+ * @success2:  return if item2 is successfully merged
+ */
+static void try_merge_with_stable(struct rmap_item *item1,
+                                 struct rmap_item *item2,
+                                 struct page **kpage,
+                                 struct page *tree_page,
+                                 int *success1, int *success2)
+{
+       spinlock_t *ptl1, *ptl2;
+       pte_t *ptep1, *ptep2;
+       unsigned long addr1, addr2;
+       struct vm_area_struct *vma1 = item1->slot->vma;
+       struct vm_area_struct *vma2 = item2->slot->vma;
+
+       *success1 = 0;
+       *success2 = 0;
+
+       if (unlikely(*kpage == tree_page)) {
+               /* I don't think this can really happen */
+               goto success_both;
+       }
+
+       if (!PageAnon(*kpage) || !PageKsm(*kpage))
+               goto failed;
+
+       if (!trylock_page(tree_page))
+               goto failed;
+
+       /* If the oldpage is still ksm and still pointed
+        * to in the right place, and still write protected,
+        * we are confident it's not changed, no need to
+        * memcmp anymore.
+        * be ware, we cannot take nested pte locks,
+        * deadlock risk.
+        */
+       addr1 = get_rmap_addr(item1);
+
+       ptep1 = page_check_address(*kpage, vma1->vm_mm, addr1, &ptl1, 0);
+       if (!ptep1)
+               goto failed;
+
+       if (pte_write(*ptep1)) {
+               /* has changed, abort! */
+               pte_unmap_unlock(ptep1, ptl1);
+               goto failed;
+       }
+
+       get_page(tree_page);
+       page_add_anon_rmap(tree_page, vma1, addr1);
+
+       flush_cache_page(vma1, addr1, pte_pfn(*ptep1));
+       ptep_clear_flush(vma1, addr1, ptep1);
+       set_pte_at_notify(vma1->vm_mm, addr1, ptep1,
+                         mk_pte(tree_page, vma1->vm_page_prot));
+
+       page_remove_rmap(*kpage);
+       put_page(*kpage);
+
+       pte_unmap_unlock(ptep1, ptl1);
+
+
+       /* ok, then vma2, remind that pte1 already set */
+       addr2 = get_rmap_addr(item2);
+
+       ptep2 = page_check_address(*kpage, vma2->vm_mm, addr2, &ptl2, 0);
+       if (!ptep2)
+               goto success1;
+
+       if (pte_write(*ptep2)) {
+               /* has changed, abort! */
+               pte_unmap_unlock(ptep2, ptl2);
+               goto success1;
+       }
+
+       get_page(tree_page);
+       page_add_anon_rmap(tree_page, vma2, addr2);
+
+       flush_cache_page(vma2, addr2, pte_pfn(*ptep2));
+       ptep_clear_flush(vma2, addr2, ptep2);
+       set_pte_at_notify(vma2->vm_mm, addr2, ptep2,
+                         mk_pte(tree_page, vma2->vm_page_prot));
+
+       page_remove_rmap(*kpage);
+       put_page(*kpage);
+
+       pte_unmap_unlock(ptep2, ptl2);
+
+
+success_both:
+       *success2 = 1;
+success1:
+       *success1 = 1;
+
+
+       if ((*success1 && vma1->vm_flags & VM_LOCKED) ||
+           (*success2 && vma2->vm_flags & VM_LOCKED)) {
+               munlock_vma_page(*kpage);
+               if (!PageMlocked(tree_page))
+                       mlock_vma_page(tree_page);
+       }
+
+       /*
+        * We do not need oldpage any more in the caller, so can break the lock
+        * now.
+        */
+       unlock_page(*kpage);
+       *kpage = tree_page; /* Get unlocked outside. */
+failed:
+       return;
+}
+
+static inline void stable_node_hash_max(struct stable_node *node,
+                                        struct page *page, u32 hash)
+{
+       u32 hash_max = node->hash_max;
+
+       if (!hash_max) {
+               hash_max = page_hash_max(page, hash);
+               node->hash_max = hash_max;
+       }
+}
+
+static inline
+struct stable_node *new_stable_node(struct tree_node *tree_node,
+                                   struct page *kpage, u32 hash_max)
+{
+       struct stable_node *new_stable_node;
+
+       new_stable_node = alloc_stable_node();
+       if (!new_stable_node)
+               return NULL;
+
+       new_stable_node->kpfn = page_to_pfn(kpage);
+       new_stable_node->hash_max = hash_max;
+       new_stable_node->tree_node = tree_node;
+       set_page_stable_node(kpage, new_stable_node);
+
+       return new_stable_node;
+}
+
+static inline
+struct stable_node *first_level_insert(struct tree_node *tree_node,
+                                      struct rmap_item *rmap_item,
+                                      struct rmap_item *tree_rmap_item,
+                                      struct page **kpage, u32 hash,
+                                      int *success1, int *success2)
+{
+       int cmp;
+       struct page *tree_page;
+       u32 hash_max = 0;
+       struct stable_node *stable_node, *new_snode;
+       struct rb_node *parent = NULL, **new;
+
+       /* this tree node contains no sub-tree yet */
+       stable_node = rb_entry(tree_node->sub_root.rb_node,
+                              struct stable_node, node);
+
+       tree_page = get_ksm_page(stable_node, 1, 0);
+       if (tree_page) {
+               cmp = memcmp_pages(*kpage, tree_page, 1);
+               if (!cmp) {
+                       try_merge_with_stable(rmap_item, tree_rmap_item, kpage,
+                                             tree_page, success1, success2);
+                       put_page(tree_page);
+                       if (!*success1 && !*success2)
+                               goto failed;
+
+                       return stable_node;
+
+               } else {
+                       /*
+                        * collision in first level try to create a subtree.
+                        * A new node need to be created.
+                        */
+                       put_page(tree_page);
+
+                       stable_node_hash_max(stable_node, tree_page,
+                                            tree_node->hash);
+                       hash_max = rmap_item_hash_max(rmap_item, hash);
+                       cmp = hash_cmp(hash_max, stable_node->hash_max);
+
+                       parent = &stable_node->node;
+                       if (cmp < 0) {
+                               new = &parent->rb_left;
+                       } else if (cmp > 0) {
+                               new = &parent->rb_right;
+                       } else {
+                               goto failed;
+                       }
+               }
+
+       } else {
+               /* the only stable_node deleted, we reuse its tree_node.
+                */
+               parent = NULL;
+               new = &tree_node->sub_root.rb_node;
+       }
+
+       new_snode = new_stable_node(tree_node, *kpage, hash_max);
+       if (!new_snode)
+               goto failed;
+
+       rb_link_node(&new_snode->node, parent, new);
+       rb_insert_color(&new_snode->node, &tree_node->sub_root);
+       tree_node->count++;
+       *success1 = *success2 = 1;
+
+       return new_snode;
+
+failed:
+       return NULL;
+}
+
+static inline
+struct stable_node *stable_subtree_insert(struct tree_node *tree_node,
+                                         struct rmap_item *rmap_item,
+                                         struct rmap_item *tree_rmap_item,
+                                         struct page **kpage, u32 hash,
+                                         int *success1, int *success2)
+{
+       struct page *tree_page;
+       u32 hash_max;
+       struct stable_node *stable_node, *new_snode;
+       struct rb_node *parent, **new;
+
+research:
+       parent = NULL;
+       new = &tree_node->sub_root.rb_node;
+       BUG_ON(!*new);
+       hash_max = rmap_item_hash_max(rmap_item, hash);
+       while (*new) {
+               int cmp;
+
+               stable_node = rb_entry(*new, struct stable_node, node);
+
+               cmp = hash_cmp(hash_max, stable_node->hash_max);
+
+               if (cmp < 0) {
+                       parent = *new;
+                       new = &parent->rb_left;
+               } else if (cmp > 0) {
+                       parent = *new;
+                       new = &parent->rb_right;
+               } else {
+                       tree_page = get_ksm_page(stable_node, 1, 0);
+                       if (tree_page) {
+                               cmp = memcmp_pages(*kpage, tree_page, 1);
+                               if (!cmp) {
+                                       try_merge_with_stable(rmap_item,
+                                               tree_rmap_item, kpage,
+                                               tree_page, success1, success2);
+
+                                       put_page(tree_page);
+                                       if (!*success1 && !*success2)
+                                               goto failed;
+                                       /*
+                                        * successfully merged with a stable
+                                        * node
+                                        */
+                                       return stable_node;
+                               } else {
+                                       put_page(tree_page);
+                                       goto failed;
+                               }
+                       } else {
+                               /*
+                                * stable node may be deleted,
+                                * and subtree maybe
+                                * restructed, cannot
+                                * continue, research it.
+                                */
+                               if (tree_node->count) {
+                                       goto research;
+                               } else {
+                                       /* reuse the tree node*/
+                                       parent = NULL;
+                                       new = &tree_node->sub_root.rb_node;
+                               }
+                       }
+               }
+       }
+
+       new_snode = new_stable_node(tree_node, *kpage, hash_max);
+       if (!new_snode)
+               goto failed;
+
+       rb_link_node(&new_snode->node, parent, new);
+       rb_insert_color(&new_snode->node, &tree_node->sub_root);
+       tree_node->count++;
+       *success1 = *success2 = 1;
+
+       return new_snode;
+
+failed:
+       return NULL;
+}
+
+
+/**
+ * stable_tree_insert() - try to insert a merged page in unstable tree to
+ * the stable tree
+ *
+ * @kpage:             the page need to be inserted
+ * @hash:              the current hash of this page
+ * @rmap_item:         the rmap_item being scanned
+ * @tree_rmap_item:    the rmap_item found on unstable tree
+ * @success1:          return if rmap_item is merged
+ * @success2:          return if tree_rmap_item is merged
+ *
+ * @return             the stable_node on stable tree if at least one
+ *                     rmap_item is inserted into stable tree, NULL
+ *                     otherwise.
+ */
+static struct stable_node *
+stable_tree_insert(struct page **kpage, u32 hash,
+                  struct rmap_item *rmap_item,
+                  struct rmap_item *tree_rmap_item,
+                  int *success1, int *success2)
+{
+       struct rb_node **new = &root_stable_treep->rb_node;
+       struct rb_node *parent = NULL;
+       struct stable_node *stable_node;
+       struct tree_node *tree_node;
+       u32 hash_max = 0;
+
+       *success1 = *success2 = 0;
+
+       while (*new) {
+               int cmp;
+
+               tree_node = rb_entry(*new, struct tree_node, node);
+
+               cmp = hash_cmp(hash, tree_node->hash);
+
+               if (cmp < 0) {
+                       parent = *new;
+                       new = &parent->rb_left;
+               } else if (cmp > 0) {
+                       parent = *new;
+                       new = &parent->rb_right;
+               } else
+                       break;
+       }
+
+       if (*new) {
+               if (tree_node->count == 1) {
+                       stable_node = first_level_insert(tree_node, rmap_item,
+                                               tree_rmap_item, kpage,
+                                               hash, success1, success2);
+               } else {
+                       stable_node = stable_subtree_insert(tree_node,
+                                       rmap_item, tree_rmap_item, kpage,
+                                       hash, success1, success2);
+               }
+       } else {
+
+               /* no tree node found */
+               tree_node = alloc_tree_node(stable_tree_node_listp);
+               if (!tree_node) {
+                       stable_node = NULL;
+                       goto out;
+               }
+
+               stable_node = new_stable_node(tree_node, *kpage, hash_max);
+               if (!stable_node) {
+                       free_tree_node(tree_node);
+                       goto out;
+               }
+
+               tree_node->hash = hash;
+               rb_link_node(&tree_node->node, parent, new);
+               rb_insert_color(&tree_node->node, root_stable_treep);
+               parent = NULL;
+               new = &tree_node->sub_root.rb_node;
+
+               rb_link_node(&stable_node->node, parent, new);
+               rb_insert_color(&stable_node->node, &tree_node->sub_root);
+               tree_node->count++;
+               *success1 = *success2 = 1;
+       }
+
+out:
+       return stable_node;
+}
+
+
+/**
+ * get_tree_rmap_item_page() - try to get the page and lock the mmap_sem
+ *
+ * @return     0 on success, -EBUSY if unable to lock the mmap_sem,
+ *             -EINVAL if the page mapping has been changed.
+ */
+static inline int get_tree_rmap_item_page(struct rmap_item *tree_rmap_item)
+{
+       int err;
+
+       err = get_mergeable_page_lock_mmap(tree_rmap_item);
+
+       if (err == -EINVAL) {
+               /* its page map has been changed, remove it */
+               remove_rmap_item_from_tree(tree_rmap_item);
+       }
+
+       /* The page is gotten and mmap_sem is locked now. */
+       return err;
+}
+
+
+/**
+ * unstable_tree_search_insert() - search an unstable tree rmap_item with the
+ * same hash value. Get its page and trylock the mmap_sem
+ */
+static inline
+struct rmap_item *unstable_tree_search_insert(struct rmap_item *rmap_item,
+                                             u32 hash)
+
+{
+       struct rb_node **new = &root_unstable_tree.rb_node;
+       struct rb_node *parent = NULL;
+       struct tree_node *tree_node;
+       u32 hash_max;
+       struct rmap_item *tree_rmap_item;
+
+       while (*new) {
+               int cmp;
+
+               tree_node = rb_entry(*new, struct tree_node, node);
+
+               cmp = hash_cmp(hash, tree_node->hash);
+
+               if (cmp < 0) {
+                       parent = *new;
+                       new = &parent->rb_left;
+               } else if (cmp > 0) {
+                       parent = *new;
+                       new = &parent->rb_right;
+               } else
+                       break;
+       }
+
+       if (*new) {
+               /* got the tree_node */
+               if (tree_node->count == 1) {
+                       tree_rmap_item = rb_entry(tree_node->sub_root.rb_node,
+                                                 struct rmap_item, node);
+                       BUG_ON(!tree_rmap_item);
+
+                       goto get_page_out;
+               }
+
+               /* well, search the collision subtree */
+               new = &tree_node->sub_root.rb_node;
+               BUG_ON(!*new);
+               hash_max = rmap_item_hash_max(rmap_item, hash);
+
+               while (*new) {
+                       int cmp;
+
+                       tree_rmap_item = rb_entry(*new, struct rmap_item,
+                                                 node);
+
+                       cmp = hash_cmp(hash_max, tree_rmap_item->hash_max);
+                       parent = *new;
+                       if (cmp < 0)
+                               new = &parent->rb_left;
+                       else if (cmp > 0)
+                               new = &parent->rb_right;
+                       else
+                               goto get_page_out;
+               }
+       } else {
+               /* alloc a new tree_node */
+               tree_node = alloc_tree_node(&unstable_tree_node_list);
+               if (!tree_node)
+                       return NULL;
+
+               tree_node->hash = hash;
+               rb_link_node(&tree_node->node, parent, new);
+               rb_insert_color(&tree_node->node, &root_unstable_tree);
+               parent = NULL;
+               new = &tree_node->sub_root.rb_node;
+       }
+
+       /* did not found even in sub-tree */
+       rmap_item->tree_node = tree_node;
+       rmap_item->address |= UNSTABLE_FLAG;
+       rmap_item->append_round = ksm_scan_round;
+       rb_link_node(&rmap_item->node, parent, new);
+       rb_insert_color(&rmap_item->node, &tree_node->sub_root);
+
+       ksm_pages_unshared++;
+       return NULL;
+
+get_page_out:
+       if (tree_rmap_item->page == rmap_item->page)
+               return NULL;
+
+       if (get_tree_rmap_item_page(tree_rmap_item))
+               return NULL;
+
+       return tree_rmap_item;
+}
+
+static void enter_vma_tree(struct vma_slot *slot)
+{
+       unsigned long i;
+       int ret;
+
+       i = ksm_vma_tree_index_end;
+
+       ret = radix_tree_insert(&ksm_vma_tree, i, slot);
+       BUG_ON(ret);
+
+       slot->ksm_index = i;
+       ksm_vma_tree_num++;
+       ksm_vma_tree_index_end++;
+}
+
+static inline void get_sub_dup_vma(struct vma_slot **slot,
+                                  struct vma_slot **sub_slot)
+{
+       struct vma_slot *tmp;
+
+       if ((*slot)->ksm_index > (*sub_slot)->ksm_index) {
+               tmp = *slot;
+               *slot = *sub_slot;
+               *sub_slot = tmp;
+       }
+}
+
+/*
+ * Inc or dec the dup pages stored in a slot, return the dup page num after
+ * the operation.
+ */
+static inline unsigned long dup_pages_mod(void **slot, int inc)
+{
+       unsigned long item, ret;
+
+       item = (unsigned long)(*slot) >> INDIRECT_OFFSET;
+       if (inc) {
+               item++;
+               BUG_ON(!item);
+       } else {
+               BUG_ON(!item);
+               item--;
+       }
+       ret = item;
+       item <<= INDIRECT_OFFSET;
+       *slot = (void *)item;
+
+       return ret;
+}
+
+static void inc_dup_vma(struct vma_slot *slot, struct vma_slot *sub_slot)
+{
+       void **dup_slot;
+       unsigned long dup_pages;
+       int ret;
+
+       if (slot->ksm_index == -1)
+               enter_vma_tree(slot);
+
+       if (sub_slot->ksm_index == -1)
+               enter_vma_tree(sub_slot);
+
+       get_sub_dup_vma(&slot, &sub_slot);
+
+       dup_slot = radix_tree_lookup_slot(&slot->dup_tree, sub_slot->ksm_index);
+       if (dup_slot)
+               goto found;
+
+       /*
+        * In order to store dup_pages in radix tree, we must make
+        * radix_tree_is_indirect_ptr() happy.
+        */
+       dup_pages = 1 << INDIRECT_OFFSET;
+
+       /* no such entry yet, insert one */
+       ret = radix_tree_insert(&slot->dup_tree, sub_slot->ksm_index,
+                               (void *)dup_pages);
+       BUG_ON(ret);
+
+       return;
+
+found:
+       dup_pages_mod(dup_slot, 1);
+}
+
+static void dec_dup_vma(struct vma_slot *slot, struct vma_slot *sub_slot)
+{
+       void **dup_slot;
+       unsigned long dup_pages;
+
+       BUG_ON(slot->ksm_index == -1 || sub_slot->ksm_index == -1);
+
+       get_sub_dup_vma(&slot, &sub_slot);
+
+       dup_slot = radix_tree_lookup_slot(&slot->dup_tree, sub_slot->ksm_index);
+       BUG_ON(!dup_slot);
+
+       dup_pages = dup_pages_mod(dup_slot, 0);
+
+       /* dup_pages == 0, we need to kick it out */
+       if (!dup_pages)
+               radix_tree_delete(&slot->dup_tree, sub_slot->ksm_index);
+}
+
+static void hold_anon_vma(struct rmap_item *rmap_item,
+                         struct anon_vma *anon_vma)
+{
+       rmap_item->anon_vma = anon_vma;
+       get_anon_vma(anon_vma);
+}
+
+
+/**
+ * stable_tree_append() - append a rmap_item to a stable node. Deduplication
+ * ratio statistics is done in this function.
+ *
+ */
+static void stable_tree_append(struct rmap_item *rmap_item,
+                              struct stable_node *stable_node)
+{
+       struct node_vma *node_vma = NULL, *new_node_vma, *node_vma_iter = NULL;
+       struct hlist_node *hlist, *cont_p = NULL;
+       unsigned long key = (unsigned long)rmap_item->slot;
+
+       BUG_ON(!stable_node);
+       rmap_item->address |= STABLE_FLAG;
+       rmap_item->append_round = ksm_scan_round;
+
+       if (hlist_empty(&stable_node->hlist)) {
+               ksm_pages_shared++;
+               goto node_vma_new;
+       } else {
+               ksm_pages_sharing++;
+       }
+
+       hlist_for_each_entry(node_vma, hlist, &stable_node->hlist, hlist) {
+               if (node_vma->last_update == ksm_scan_round)
+                       inc_dup_vma(rmap_item->slot, node_vma->slot);
+
+               if (node_vma->key >= key)
+                       break;
+       }
+
+       cont_p = hlist;
+
+       if (node_vma && node_vma->key == key) {
+               if (node_vma->last_update == ksm_scan_round) {
+                       /**
+                        * we consider this page a inner duplicate, cancel
+                        * other updates
+                        */
+                       hlist_for_each_entry(node_vma_iter, hlist,
+                                            &stable_node->hlist, hlist) {
+                               if (node_vma_iter->key == key)
+                                       break;
+
+                               /* only need to increase the same vma */
+                               if (node_vma_iter->last_update ==
+                                   ksm_scan_round) {
+                                       dec_dup_vma(rmap_item->slot,
+                                                   node_vma_iter->slot);
+                               }
+                       }
+               } else {
+                       /**
+                        * Although it's same vma, it contains no duplicate for this
+                        * round. Continue scan other vma.
+                        */
+                       hlist_for_each_entry_continue(node_vma_iter,
+                                                     hlist, hlist) {
+                               if (node_vma_iter->last_update ==
+                                   ksm_scan_round) {
+                                       inc_dup_vma(rmap_item->slot,
+                                                   node_vma_iter->slot);
+                               }
+                       }
+
+               }
+
+               goto node_vma_ok;
+       }
+
+node_vma_new:
+       /* no same vma already in node, alloc a new node_vma */
+       new_node_vma = alloc_node_vma();
+       BUG_ON(!new_node_vma);
+       new_node_vma->head = stable_node;
+       new_node_vma->slot = rmap_item->slot;
+
+       if (!node_vma) {
+               hlist_add_head(&new_node_vma->hlist, &stable_node->hlist);
+       } else if (node_vma->key != key) {
+               if (node_vma->key < key)
+                       hlist_add_after(&node_vma->hlist, &new_node_vma->hlist);
+               else {
+                       hlist_for_each_entry_continue(node_vma_iter, cont_p,
+                                                     hlist) {
+                               if (node_vma_iter->last_update ==
+                                   ksm_scan_round) {
+                                       inc_dup_vma(rmap_item->slot,
+                                                   node_vma_iter->slot);
+                               }
+                       }
+                       hlist_add_before(&new_node_vma->hlist,
+                                        &node_vma->hlist);
+               }
+
+       }
+       node_vma = new_node_vma;
+
+node_vma_ok: /* ok, ready to add to the list */
+       rmap_item->head = node_vma;
+       hlist_add_head(&rmap_item->hlist, &node_vma->rmap_hlist);
+       node_vma->last_update = ksm_scan_round;
+       hold_anon_vma(rmap_item, rmap_item->slot->vma->anon_vma);
+       rmap_item->slot->pages_merged++;
+}
+
+/*
+ * We use break_ksm to break COW on a ksm page: it's a stripped down
+ *
+ *     if (get_user_pages(current, mm, addr, 1, 1, 1, &page, NULL) == 1)
+ *             put_page(page);
+ *
+ * but taking great care only to touch a ksm page, in a VM_MERGEABLE vma,
+ * in case the application has unmapped and remapped mm,addr meanwhile.
+ * Could a ksm page appear anywhere else?  Actually yes, in a VM_PFNMAP
+ * mmap of /dev/mem or /dev/kmem, where we would not want to touch it.
+ */
+static int break_ksm(struct vm_area_struct *vma, unsigned long addr)
+{
+       struct page *page;
+       int ret = 0;
+
+       do {
+               cond_resched();
+               page = follow_page(vma, addr, FOLL_GET);
+               if (IS_ERR_OR_NULL(page))
+                       break;
+               if (PageKsm(page)) {
+                       ret = handle_mm_fault(vma->vm_mm, vma, addr,
+                                             FAULT_FLAG_WRITE);
+               } else
+                       ret = VM_FAULT_WRITE;
+               put_page(page);
+       } while (!(ret & (VM_FAULT_WRITE | VM_FAULT_SIGBUS | VM_FAULT_OOM)));
+       /*
+        * We must loop because handle_mm_fault() may back out if there's
+        * any difficulty e.g. if pte accessed bit gets updated concurrently.
+        *
+        * VM_FAULT_WRITE is what we have been hoping for: it indicates that
+        * COW has been broken, even if the vma does not permit VM_WRITE;
+        * but note that a concurrent fault might break PageKsm for us.
+        *
+        * VM_FAULT_SIGBUS could occur if we race with truncation of the
+        * backing file, which also invalidates anonymous pages: that's
+        * okay, that truncation will have unmapped the PageKsm for us.
+        *
+        * VM_FAULT_OOM: at the time of writing (late July 2009), setting
+        * aside mem_cgroup limits, VM_FAULT_OOM would only be set if the
+        * current task has TIF_MEMDIE set, and will be OOM killed on return
+        * to user; and ksmd, having no mm, would never be chosen for that.
+        *
+        * But if the mm is in a limited mem_cgroup, then the fault may fail
+        * with VM_FAULT_OOM even if the current task is not TIF_MEMDIE; and
+        * even ksmd can fail in this way - though it's usually breaking ksm
+        * just to undo a merge it made a moment before, so unlikely to oom.
+        *
+        * That's a pity: we might therefore have more kernel pages allocated
+        * than we're counting as nodes in the stable tree; but ksm_do_scan
+        * will retry to break_cow on each pass, so should recover the page
+        * in due course.  The important thing is to not let VM_MERGEABLE
+        * be cleared while any such pages might remain in the area.
+        */
+       return (ret & VM_FAULT_OOM) ? -ENOMEM : 0;
+}
+
+static void break_cow(struct rmap_item *rmap_item)
+{
+       struct vm_area_struct *vma = rmap_item->slot->vma;
+       struct mm_struct *mm = vma->vm_mm;
+       unsigned long addr = get_rmap_addr(rmap_item);
+
+       if (ksm_test_exit(mm))
+               goto out;
+
+       break_ksm(vma, addr);
+out:
+       return;
+}
+
+/*
+ * Though it's very tempting to unmerge in_stable_tree(rmap_item)s rather
+ * than check every pte of a given vma, the locking doesn't quite work for
+ * that - an rmap_item is assigned to the stable tree after inserting ksm
+ * page and upping mmap_sem.  Nor does it fit with the way we skip dup'ing
+ * rmap_items from parent to child at fork time (so as not to waste time
+ * if exit comes before the next scan reaches it).
+ *
+ * Similarly, although we'd like to remove rmap_items (so updating counts
+ * and freeing memory) when unmerging an area, it's easier to leave that
+ * to the next pass of ksmd - consider, for example, how ksmd might be
+ * in cmp_and_merge_page on one of the rmap_items we would be removing.
+ */
+inline int unmerge_ksm_pages(struct vm_area_struct *vma,
+                     unsigned long start, unsigned long end)
+{
+       unsigned long addr;
+       int err = 0;
+
+       for (addr = start; addr < end && !err; addr += PAGE_SIZE) {
+               if (ksm_test_exit(vma->vm_mm))
+                       break;
+               if (signal_pending(current))
+                       err = -ERESTARTSYS;
+               else
+                       err = break_ksm(vma, addr);
+       }
+       return err;
+}
+
+static inline void inc_ksm_pages_scanned(void)
+{
+       u64 delta;
+
+
+       if (ksm_pages_scanned == U64_MAX) {
+               encode_benefit();
+
+               delta = ksm_pages_scanned >> pages_scanned_base;
+
+               if (CAN_OVERFLOW_U64(pages_scanned_stored, delta)) {
+                       pages_scanned_stored >>= 1;
+                       delta >>= 1;
+                       pages_scanned_base++;
+               }
+
+               pages_scanned_stored += delta;
+
+               ksm_pages_scanned = ksm_pages_scanned_last = 0;
+       }
+
+       ksm_pages_scanned++;
+}
+
+static inline int find_zero_page_hash(int strength, u32 hash)
+{
+       return (zero_hash_table[strength] == hash);
+}
+
+static
+int cmp_and_merge_zero_page(struct vm_area_struct *vma, struct page *page)
+{
+       struct page *zero_page = empty_ksm_zero_page;
+       struct mm_struct *mm = vma->vm_mm;
+       pte_t orig_pte = __pte(0);
+       int err = -EFAULT;
+
+       if (ksm_test_exit(mm))
+               goto out;
+
+       if (PageTransCompound(page) && page_trans_compound_anon_split(page))
+               goto out;
+       BUG_ON(PageTransCompound(page));
+
+       if (!PageAnon(page))
+               goto out;
+
+       if (!trylock_page(page))
+               goto out;
+
+       if (write_protect_page(vma, page, &orig_pte, 0) == 0) {
+               if (is_page_full_zero(page))
+                       err = replace_page(vma, page, zero_page, orig_pte);
+       }
+
+       unlock_page(page);
+out:
+       return err;
+}
+
+/*
+ * cmp_and_merge_page() - first see if page can be merged into the stable
+ * tree; if not, compare hash to previous and if it's the same, see if page
+ * can be inserted into the unstable tree, or merged with a page already there
+ * and both transferred to the stable tree.
+ *
+ * @page: the page that we are searching identical page to.
+ * @rmap_item: the reverse mapping into the virtual address of this page
+ */
+static void cmp_and_merge_page(struct rmap_item *rmap_item)
+{
+       struct rmap_item *tree_rmap_item;
+       struct page *page;
+       struct page *kpage = NULL;
+       u32 hash, hash_max;
+       int err;
+       unsigned int success1, success2;
+       struct stable_node *snode;
+       int cmp;
+       struct rb_node *parent = NULL, **new;
+
+       remove_rmap_item_from_tree(rmap_item);
+       inc_ksm_pages_scanned();
+
+       page = rmap_item->page;
+
+       hash = page_hash(page, hash_strength, 1);
+
+       /*if the page content all zero, re-map to zero-page*/
+       if (find_zero_page_hash(hash_strength, hash)) {
+               if (!cmp_and_merge_zero_page(rmap_item->slot->vma, page)) {
+                       __inc_zone_page_state(page, NR_KSM_ZERO_PAGES);
+                       return ;
+               } else {
+                       inc_rshash_neg(memcmp_cost / 2);
+               }
+       }
+       //ksm_pages_scanned++;
+
+       /* We first start with searching the page inside the stable tree */
+       kpage = stable_tree_search(rmap_item, hash);
+       if (kpage) {
+               err = try_to_merge_with_ksm_page(rmap_item, kpage,
+                                                hash);
+               if (!err) {
+                       /*
+                        * The page was successfully merged, add
+                        * its rmap_item to the stable tree.
+                        * page lock is needed because it's
+                        * racing with try_to_unmap_ksm(), etc.
+                        */
+                       lock_page(kpage);
+                       stable_tree_append(rmap_item, page_stable_node(kpage));
+                       unlock_page(kpage);
+                       put_page(kpage);
+                       return; /* success */
+               }
+               put_page(kpage);
+
+               /*
+                * if it's a collision and it has been search in sub-rbtree
+                * (hash_max != 0), we want to abort, because if it is
+                * successfully merged in unstable tree, the collision trends to
+                * happen again.
+                */
+               if (err == MERGE_ERR_COLLI && rmap_item->hash_max)
+                       return;
+       }
+
+       tree_rmap_item =
+               unstable_tree_search_insert(rmap_item, hash);
+       if (tree_rmap_item) {
+               err = try_to_merge_two_pages(rmap_item, tree_rmap_item, hash);
+               /*
+                * As soon as we merge this page, we want to remove the
+                * rmap_item of the page we have merged with from the unstable
+                * tree, and insert it instead as new node in the stable tree.
+                */
+               if (!err) {
+                       kpage = page;
+                       remove_rmap_item_from_tree(tree_rmap_item);
+                       lock_page(kpage);
+                       snode = stable_tree_insert(&kpage, hash,
+                                                  rmap_item, tree_rmap_item,
+                                                  &success1, &success2);
+
+                       if (success1)
+                               stable_tree_append(rmap_item, snode);
+                       else
+                               break_cow(rmap_item);
+
+                       if (success2)
+                               stable_tree_append(tree_rmap_item, snode);
+                       else
+                               break_cow(tree_rmap_item);
+
+                       /*
+                        * The original kpage may be unlocked inside
+                        * stable_tree_insert() already.
+                        */
+                       unlock_page(kpage);
+
+               } else if (err == MERGE_ERR_COLLI) {
+                       if (tree_rmap_item->tree_node->count == 1) {
+                               rmap_item_hash_max(tree_rmap_item,
+                               tree_rmap_item->tree_node->hash);
+                       } else
+                               BUG_ON(!(tree_rmap_item->hash_max));
+
+                       hash_max = rmap_item_hash_max(rmap_item, hash);
+                       cmp = hash_cmp(hash_max, tree_rmap_item->hash_max);
+                       parent = &tree_rmap_item->node;
+                       if (cmp < 0)
+                               new = &parent->rb_left;
+                       else if (cmp > 0)
+                               new = &parent->rb_right;
+                       else
+                               goto put_up_out;
+
+                       rmap_item->tree_node = tree_rmap_item->tree_node;
+                       rmap_item->address |= UNSTABLE_FLAG;
+                       rmap_item->append_round = ksm_scan_round;
+                       rb_link_node(&rmap_item->node, parent, new);
+                       rb_insert_color(&rmap_item->node,
+                                       &tree_rmap_item->tree_node->sub_root);
+                       rmap_item->tree_node->count++;
+               }
+put_up_out:
+               put_page(tree_rmap_item->page);
+               up_read(&tree_rmap_item->slot->vma->vm_mm->mmap_sem);
+       }
+}
+
+
+
+
+static inline unsigned long get_pool_index(struct vma_slot *slot,
+                                          unsigned long index)
+{
+       unsigned long pool_index;
+
+       pool_index = (sizeof(struct rmap_list_entry *) * index) >> PAGE_SHIFT;
+       if (pool_index >= slot->pool_size)
+               BUG();
+       return pool_index;
+}
+
+static inline unsigned long index_page_offset(unsigned long index)
+{
+       return offset_in_page(sizeof(struct rmap_list_entry *) * index);
+}
+
+static inline
+struct rmap_list_entry *get_rmap_list_entry(struct vma_slot *slot,
+                                           unsigned long index, int need_alloc)
+{
+       unsigned long pool_index;
+       void *addr;
+
+
+       pool_index = get_pool_index(slot, index);
+       if (!slot->rmap_list_pool[pool_index]) {
+               if (!need_alloc)
+                       return NULL;
+
+               slot->rmap_list_pool[pool_index] =
+                       alloc_page(GFP_KERNEL | __GFP_ZERO);
+               BUG_ON(!slot->rmap_list_pool[pool_index]);
+       }
+
+       addr = kmap(slot->rmap_list_pool[pool_index]);
+       addr += index_page_offset(index);
+
+       return addr;
+}
+
+static inline void put_rmap_list_entry(struct vma_slot *slot,
+                                      unsigned long index)
+{
+       unsigned long pool_index;
+
+       pool_index = get_pool_index(slot, index);
+       BUG_ON(!slot->rmap_list_pool[pool_index]);
+       kunmap(slot->rmap_list_pool[pool_index]);
+}
+
+static inline int entry_is_new(struct rmap_list_entry *entry)
+{
+       return !entry->item;
+}
+
+static inline unsigned long get_index_orig_addr(struct vma_slot *slot,
+                                               unsigned long index)
+{
+       return slot->vma->vm_start + (index << PAGE_SHIFT);
+}
+
+static inline unsigned long get_entry_address(struct rmap_list_entry *entry)
+{
+       unsigned long addr;
+
+       if (is_addr(entry->addr))
+               addr = get_clean_addr(entry->addr);
+       else if (entry->item)
+               addr = get_rmap_addr(entry->item);
+       else
+               BUG();
+
+       return addr;
+}
+
+static inline struct rmap_item *get_entry_item(struct rmap_list_entry *entry)
+{
+       if (is_addr(entry->addr))
+               return NULL;
+
+       return entry->item;
+}
+
+static inline void inc_rmap_list_pool_count(struct vma_slot *slot,
+                                           unsigned long index)
+{
+       unsigned long pool_index;
+
+       pool_index = get_pool_index(slot, index);
+       BUG_ON(!slot->rmap_list_pool[pool_index]);
+       slot->pool_counts[pool_index]++;
+}
+
+static inline void dec_rmap_list_pool_count(struct vma_slot *slot,
+                                           unsigned long index)
+{
+       unsigned long pool_index;
+
+       pool_index = get_pool_index(slot, index);
+       BUG_ON(!slot->rmap_list_pool[pool_index]);
+       BUG_ON(!slot->pool_counts[pool_index]);
+       slot->pool_counts[pool_index]--;
+}
+
+static inline int entry_has_rmap(struct rmap_list_entry *entry)
+{
+       return !is_addr(entry->addr) && entry->item;
+}
+
+static inline void swap_entries(struct rmap_list_entry *entry1,
+                               unsigned long index1,
+                               struct rmap_list_entry *entry2,
+                               unsigned long index2)
+{
+       struct rmap_list_entry tmp;
+
+       /* swapping two new entries is meaningless */
+       BUG_ON(entry_is_new(entry1) && entry_is_new(entry2));
+
+       tmp = *entry1;
+       *entry1 = *entry2;
+       *entry2 = tmp;
+
+       if (entry_has_rmap(entry1))
+               entry1->item->entry_index = index1;
+
+       if (entry_has_rmap(entry2))
+               entry2->item->entry_index = index2;
+
+       if (entry_has_rmap(entry1) && !entry_has_rmap(entry2)) {
+               inc_rmap_list_pool_count(entry1->item->slot, index1);
+               dec_rmap_list_pool_count(entry1->item->slot, index2);
+       } else if (!entry_has_rmap(entry1) && entry_has_rmap(entry2)) {
+               inc_rmap_list_pool_count(entry2->item->slot, index2);
+               dec_rmap_list_pool_count(entry2->item->slot, index1);
+       }
+}
+
+static inline void free_entry_item(struct rmap_list_entry *entry)
+{
+       unsigned long index;
+       struct rmap_item *item;
+
+       if (!is_addr(entry->addr)) {
+               BUG_ON(!entry->item);
+               item = entry->item;
+               entry->addr = get_rmap_addr(item);
+               set_is_addr(entry->addr);
+               index = item->entry_index;
+               remove_rmap_item_from_tree(item);
+               dec_rmap_list_pool_count(item->slot, index);
+               free_rmap_item(item);
+       }
+}
+
+static inline int pool_entry_boundary(unsigned long index)
+{
+       unsigned long linear_addr;
+
+       linear_addr = sizeof(struct rmap_list_entry *) * index;
+       return index && !offset_in_page(linear_addr);
+}
+
+static inline void try_free_last_pool(struct vma_slot *slot,
+                                     unsigned long index)
+{
+       unsigned long pool_index;
+
+       pool_index = get_pool_index(slot, index);
+       if (slot->rmap_list_pool[pool_index] &&
+           !slot->pool_counts[pool_index]) {
+               __free_page(slot->rmap_list_pool[pool_index]);
+               slot->rmap_list_pool[pool_index] = NULL;
+               slot->need_sort = 1;
+       }
+
+}
+
+static inline unsigned long vma_item_index(struct vm_area_struct *vma,
+                                          struct rmap_item *item)
+{
+       return (get_rmap_addr(item) - vma->vm_start) >> PAGE_SHIFT;
+}
+
+static int within_same_pool(struct vma_slot *slot,
+                           unsigned long i, unsigned long j)
+{
+       unsigned long pool_i, pool_j;
+
+       pool_i = get_pool_index(slot, i);
+       pool_j = get_pool_index(slot, j);
+
+       return (pool_i == pool_j);
+}
+
+static void sort_rmap_entry_list(struct vma_slot *slot)
+{
+       unsigned long i, j;
+       struct rmap_list_entry *entry, *swap_entry;
+
+       entry = get_rmap_list_entry(slot, 0, 0);
+       for (i = 0; i < slot->pages; ) {
+
+               if (!entry)
+                       goto skip_whole_pool;
+
+               if (entry_is_new(entry))
+                       goto next_entry;
+
+               if (is_addr(entry->addr)) {
+                       entry->addr = 0;
+                       goto next_entry;
+               }
+
+               j = vma_item_index(slot->vma, entry->item);
+               if (j == i)
+                       goto next_entry;
+
+               if (within_same_pool(slot, i, j))
+                       swap_entry = entry + j - i;
+               else
+                       swap_entry = get_rmap_list_entry(slot, j, 1);
+
+               swap_entries(entry, i, swap_entry, j);
+               if (!within_same_pool(slot, i, j))
+                       put_rmap_list_entry(slot, j);
+               continue;
+
+skip_whole_pool:
+               i += PAGE_SIZE / sizeof(*entry);
+               if (i < slot->pages)
+                       entry = get_rmap_list_entry(slot, i, 0);
+               continue;
+
+next_entry:
+               if (i >= slot->pages - 1 ||
+                   !within_same_pool(slot, i, i + 1)) {
+                       put_rmap_list_entry(slot, i);
+                       if (i + 1 < slot->pages)
+                               entry = get_rmap_list_entry(slot, i + 1, 0);
+               } else
+                       entry++;
+               i++;
+               continue;
+       }
+
+       /* free empty pool entries which contain no rmap_item */
+       /* CAN be simplied to based on only pool_counts when bug freed !!!!! */
+       for (i = 0; i < slot->pool_size; i++) {
+               unsigned char has_rmap;
+               void *addr;
+
+               if (!slot->rmap_list_pool[i])
+                       continue;
+
+               has_rmap = 0;
+               addr = kmap(slot->rmap_list_pool[i]);
+               BUG_ON(!addr);
+               for (j = 0; j < PAGE_SIZE / sizeof(*entry); j++) {
+                       entry = (struct rmap_list_entry *)addr + j;
+                       if (is_addr(entry->addr))
+                               continue;
+                       if (!entry->item)
+                               continue;
+                       has_rmap = 1;
+               }
+               kunmap(slot->rmap_list_pool[i]);
+               if (!has_rmap) {
+                       BUG_ON(slot->pool_counts[i]);
+                       __free_page(slot->rmap_list_pool[i]);
+                       slot->rmap_list_pool[i] = NULL;
+               }
+       }
+
+       slot->need_sort = 0;
+}
+
+/*
+ * vma_fully_scanned() - if all the pages in this slot have been scanned.
+ */
+static inline int vma_fully_scanned(struct vma_slot *slot)
+{
+       return slot->pages_scanned && !(slot->pages_scanned % slot->pages);
+}
+
+/**
+ * get_next_rmap_item() - Get the next rmap_item in a vma_slot according to
+ * its random permutation. This function is embedded with the random
+ * permutation index management code.
+ */
+static struct rmap_item *get_next_rmap_item(struct vma_slot *slot)
+{
+       unsigned long rand_range, addr, swap_index, scan_index;
+       struct rmap_item *item = NULL;
+       struct rmap_list_entry *scan_entry, *swap_entry = NULL;
+       struct page *page;
+
+       scan_index = swap_index = slot->pages_scanned % slot->pages;
+
+       if (pool_entry_boundary(scan_index))
+               try_free_last_pool(slot, scan_index - 1);
+
+       if (vma_fully_scanned(slot)) {
+               slot->need_rerand = slot->need_sort;
+               if (slot->need_sort)
+                       sort_rmap_entry_list(slot);
+       }
+
+       scan_entry = get_rmap_list_entry(slot, scan_index, 1);
+       if (entry_is_new(scan_entry)) {
+               scan_entry->addr = get_index_orig_addr(slot, scan_index);
+               set_is_addr(scan_entry->addr);
+       }
+
+       if (slot->need_rerand) {
+               rand_range = slot->pages - scan_index;
+               BUG_ON(!rand_range);
+               swap_index = scan_index + (random32() % rand_range);
+       }
+
+       if (swap_index != scan_index) {
+               swap_entry = get_rmap_list_entry(slot, swap_index, 1);
+               if (entry_is_new(swap_entry)) {
+                       swap_entry->addr = get_index_orig_addr(slot,
+                                                              swap_index);
+                       set_is_addr(swap_entry->addr);
+               }
+               swap_entries(scan_entry, scan_index, swap_entry, swap_index);
+       }
+
+       addr = get_entry_address(scan_entry);
+       item = get_entry_item(scan_entry);
+       BUG_ON(addr > slot->vma->vm_end || addr < slot->vma->vm_start);
+
+       page = follow_page(slot->vma, addr, FOLL_GET);
+       if (IS_ERR_OR_NULL(page))
+               goto nopage;
+
+       if (!PageAnon(page) && !page_trans_compound_anon(page))
+               goto putpage;
+
+       /*check is zero_page pfn or ksm_zero_page*/
+       if ((page_to_pfn(page) == zero_pfn)
+                       || (page_to_pfn(page) == ksm_zero_pfn))
+               goto putpage;
+
+       flush_anon_page(slot->vma, page, addr);
+       flush_dcache_page(page);
+
+       if (!item) {
+               item = alloc_rmap_item();
+               if (item) {
+                       /* It has already been zeroed */
+                       item->slot = slot;
+                       item->address = addr;
+                       item->entry_index = scan_index;
+                       scan_entry->item = item;
+                       inc_rmap_list_pool_count(slot, scan_index);
+               } else
+                       goto putpage;
+       }
+
+       BUG_ON(item->slot != slot);
+       /* the page may have changed */
+       item->page = page;
+       put_rmap_list_entry(slot, scan_index);
+       if (swap_entry)
+               put_rmap_list_entry(slot, swap_index);
+       return item;
+
+putpage:
+       put_page(page);
+       page = NULL;
+nopage:
+       /* no page, store addr back and free rmap_item if possible */
+       free_entry_item(scan_entry);
+       put_rmap_list_entry(slot, scan_index);
+       if (swap_entry)
+               put_rmap_list_entry(slot, swap_index);
+       return NULL;
+}
+
+static inline int in_stable_tree(struct rmap_item *rmap_item)
+{
+       return rmap_item->address & STABLE_FLAG;
+}
+
+/**
+ * scan_vma_one_page() - scan the next page in a vma_slot. Called with
+ * mmap_sem locked.
+ */
+static void scan_vma_one_page(struct vma_slot *slot)
+{
+       struct mm_struct *mm;
+       struct rmap_item *rmap_item = NULL;
+       struct vm_area_struct *vma = slot->vma;
+
+       mm = vma->vm_mm;
+       BUG_ON(!mm);
+       BUG_ON(!slot);
+
+       rmap_item = get_next_rmap_item(slot);
+       if (!rmap_item)
+               goto out1;
+
+       if (PageKsm(rmap_item->page) && in_stable_tree(rmap_item))
+               goto out2;
+
+       cmp_and_merge_page(rmap_item);
+out2:
+       put_page(rmap_item->page);
+out1:
+       slot->pages_scanned++;
+       slot->slot_scanned = 1;
+       if (vma_fully_scanned(slot)) {
+               slot->fully_scanned = 1;
+               slot->rung->fully_scanned_slots++;
+               BUG_ON(!slot->rung->fully_scanned_slots);
+       }
+}
+
+static unsigned long get_vma_random_scan_num(struct vma_slot *slot,
+                                            unsigned long scan_ratio)
+{
+       return slot->pages * scan_ratio / KSM_SCAN_RATIO_MAX;
+}
+
+static inline void vma_rung_enter(struct vma_slot *slot,
+                                 struct scan_rung *rung)
+{
+       unsigned long pages_to_scan;
+       struct scan_rung *old_rung = slot->rung;
+
+       /* leave the old rung it was in */
+       BUG_ON(list_empty(&slot->ksm_list));
+
+       if (old_rung->current_scan == &slot->ksm_list)
+               old_rung->current_scan = slot->ksm_list.next;
+       list_del_init(&slot->ksm_list);
+       old_rung->vma_num--;
+       if (slot->fully_scanned)
+               old_rung->fully_scanned_slots--;
+
+       if (old_rung->current_scan == &old_rung->vma_list) {
+               /* This rung finishes a round */
+               old_rung->round_finished = 1;
+               old_rung->current_scan = old_rung->vma_list.next;
+               BUG_ON(old_rung->current_scan == &old_rung->vma_list &&
+                      !list_empty(&old_rung->vma_list));
+       }
+
+       /* enter the new rung */
+       while (!(pages_to_scan =
+               get_vma_random_scan_num(slot, rung->scan_ratio))) {
+               rung++;
+               BUG_ON(rung > &ksm_scan_ladder[ksm_scan_ladder_size - 1]);
+       }
+       if (list_empty(&rung->vma_list))
+               rung->current_scan = &slot->ksm_list;
+       list_add(&slot->ksm_list, &rung->vma_list);
+       slot->rung = rung;
+       slot->pages_to_scan = pages_to_scan;
+       slot->rung->vma_num++;
+       if (slot->fully_scanned)
+               rung->fully_scanned_slots++;
+
+       BUG_ON(rung->current_scan == &rung->vma_list &&
+              !list_empty(&rung->vma_list));
+}
+
+static inline void vma_rung_up(struct vma_slot *slot)
+{
+       if (slot->rung == &ksm_scan_ladder[ksm_scan_ladder_size-1])
+               return;
+
+       vma_rung_enter(slot, slot->rung + 1);
+}
+
+static inline void vma_rung_down(struct vma_slot *slot)
+{
+       if (slot->rung == &ksm_scan_ladder[0])
+               return;
+
+       vma_rung_enter(slot, slot->rung - 1);
+}
+
+/**
+ * cal_dedup_ratio() - Calculate the deduplication ratio for this slot.
+ */
+static unsigned long cal_dedup_ratio(struct vma_slot *slot)
+{
+       struct vma_slot *slot2;
+       void **dup_slot;
+       unsigned long dup_pages;
+       unsigned long dedup_num, pages1, scanned1;
+       unsigned long ret;
+       int i;
+
+       if (!slot->pages_scanned)
+               return 0;
+
+       pages1 = slot->pages;
+       scanned1 = slot->pages_scanned - slot->last_scanned;
+       BUG_ON(scanned1 > slot->pages_scanned);
+
+       for (i = slot->ksm_index; i < ksm_vma_tree_index_end; i++) {
+               unsigned long pages2, scanned2;
+
+               dup_slot = radix_tree_lookup_slot(&slot->dup_tree, i);
+               if (!dup_slot)
+                       continue;
+
+               dup_pages = (unsigned long)(*dup_slot) >> INDIRECT_OFFSET;
+
+               slot2 = radix_tree_lookup(&ksm_vma_tree, i);
+               BUG_ON(!slot2 || !slot2->pages_scanned);
+
+               pages2 = slot2->pages;
+               scanned2 = slot2->pages_scanned - slot2->last_scanned;
+               BUG_ON(scanned2 > slot2->pages_scanned);
+
+               BUG_ON(!scanned1 || !scanned2);
+
+               dedup_num = dup_pages * pages1 / scanned1 * pages2 / scanned2;
+               slot->dedup_num += dedup_num;
+               slot2->dedup_num += dedup_num;
+       }
+
+       ret = (slot->dedup_num * KSM_DEDUP_RATIO_SCALE / pages1);
+
+       /* Thrashing area filtering */
+       if (ksm_thrash_threshold) {
+               if (slot->pages_cowed * 100 / slot->pages_merged
+                   > ksm_thrash_threshold) {
+                       ret = 0;
+               } else {
+                       ret = ret * (slot->pages_merged - slot->pages_cowed)
+                             / slot->pages_merged;
+               }
+       }
+
+       return ret;
+}
+
+
+/**
+ * stable_node_reinsert() - When the hash_strength has been adjusted, the
+ * stable tree need to be restructured, this is the function re-inserting the
+ * stable node.
+ */
+static inline void stable_node_reinsert(struct stable_node *new_node,
+                                       struct page *page,
+                                       struct rb_root *root_treep,
+                                       struct list_head *tree_node_listp,
+                                       u32 hash)
+{
+       struct rb_node **new = &root_treep->rb_node;
+       struct rb_node *parent = NULL;
+       struct stable_node *stable_node;
+       struct tree_node *tree_node;
+       struct page *tree_page;
+       int cmp;
+
+       while (*new) {
+               int cmp;
+
+               tree_node = rb_entry(*new, struct tree_node, node);
+
+               cmp = hash_cmp(hash, tree_node->hash);
+
+               if (cmp < 0) {
+                       parent = *new;
+                       new = &parent->rb_left;
+               } else if (cmp > 0) {
+                       parent = *new;
+                       new = &parent->rb_right;
+               } else
+                       break;
+       }
+
+       if (*new) {
+               /* find a stable tree node with same first level hash value */
+               stable_node_hash_max(new_node, page, hash);
+               if (tree_node->count == 1) {
+                       stable_node = rb_entry(tree_node->sub_root.rb_node,
+                                              struct stable_node, node);
+                       tree_page = get_ksm_page(stable_node, 1, 0);
+                       if (tree_page) {
+                               stable_node_hash_max(stable_node,
+                                                     tree_page, hash);
+                               put_page(tree_page);
+
+                               /* prepare for stable node insertion */
+
+                               cmp = hash_cmp(new_node->hash_max,
+                                                  stable_node->hash_max);
+                               parent = &stable_node->node;
+                               if (cmp < 0)
+                                       new = &parent->rb_left;
+                               else if (cmp > 0)
+                                       new = &parent->rb_right;
+                               else
+                                       goto failed;
+
+                               goto add_node;
+                       } else {
+                               /* the only stable_node deleted, the tree node
+                                * was not deleted.
+                                */
+                               goto tree_node_reuse;
+                       }
+               }
+
+               /* well, search the collision subtree */
+               new = &tree_node->sub_root.rb_node;
+               parent = NULL;
+               BUG_ON(!*new);
+               while (*new) {
+                       int cmp;
+
+                       stable_node = rb_entry(*new, struct stable_node, node);
+
+                       cmp = hash_cmp(new_node->hash_max,
+                                          stable_node->hash_max);
+
+                       if (cmp < 0) {
+                               parent = *new;
+                               new = &parent->rb_left;
+                       } else if (cmp > 0) {
+                               parent = *new;
+                               new = &parent->rb_right;
+                       } else {
+                               /* oh, no, still a collision */
+                               goto failed;
+                       }
+               }
+
+               goto add_node;
+       }
+
+       /* no tree node found */
+       tree_node = alloc_tree_node(tree_node_listp);
+       if (!tree_node) {
+               printk(KERN_ERR "UKSM: memory allocation error!\n");
+               goto failed;
+       } else {
+               tree_node->hash = hash;
+               rb_link_node(&tree_node->node, parent, new);
+               rb_insert_color(&tree_node->node, root_treep);
+
+tree_node_reuse:
+               /* prepare for stable node insertion */
+               parent = NULL;
+               new = &tree_node->sub_root.rb_node;
+       }
+
+add_node:
+       rb_link_node(&new_node->node, parent, new);
+       rb_insert_color(&new_node->node, &tree_node->sub_root);
+       new_node->tree_node = tree_node;
+       tree_node->count++;
+       return;
+
+failed:
+       /* This can only happen when two nodes have collided
+        * in two levels.
+        */
+       new_node->tree_node = NULL;
+       return;
+}
+
+static inline void free_all_tree_nodes(struct list_head *list)
+{
+       struct tree_node *node, *tmp;
+
+       list_for_each_entry_safe(node, tmp, list, all_list) {
+               free_tree_node(node);
+       }
+}
+
+/**
+ * stable_tree_delta_hash() - Delta hash the stable tree from previous hash
+ * strength to the current hash_strength. It re-structures the hole tree.
+ */
+static inline void stable_tree_delta_hash(u32 prev_hash_strength)
+{
+       struct stable_node *node, *tmp;
+       struct rb_root *root_new_treep;
+       struct list_head *new_tree_node_listp;
+
+       stable_tree_index = (stable_tree_index + 1) % 2;
+       root_new_treep = &root_stable_tree[stable_tree_index];
+       new_tree_node_listp = &stable_tree_node_list[stable_tree_index];
+       *root_new_treep = RB_ROOT;
+       BUG_ON(!list_empty(new_tree_node_listp));
+
+       /*
+        * we need to be safe, the node could be removed by get_ksm_page()
+        */
+       list_for_each_entry_safe(node, tmp, &stable_node_list, all_list) {
+               void *addr;
+               struct page *node_page;
+               u32 hash;
+
+               /*
+                * We are completely re-structuring the stable nodes to a new
+                * stable tree. We don't want to touch the old tree unlinks and
+                * old tree_nodes. The old tree_nodes will be freed at once.
+                */
+               node_page = get_ksm_page(node, 0, 0);
+               if (!node_page)
+                       continue;
+
+               if (node->tree_node) {
+                       hash = node->tree_node->hash;
+
+                       addr = kmap_atomic(node_page, KM_USER0);
+
+                       hash = delta_hash(addr, prev_hash_strength,
+                                         hash_strength, hash);
+                       kunmap_atomic(addr, KM_USER0);
+               } else {
+                       /*
+                        *it was not inserted to rbtree due to collision in last
+                        *round scan.
+                        */
+                       hash = page_hash(node_page, hash_strength, 0);
+               }
+
+               stable_node_reinsert(node, node_page, root_new_treep,
+                                    new_tree_node_listp, hash);
+               put_page(node_page);
+       }
+
+       root_stable_treep = root_new_treep;
+       free_all_tree_nodes(stable_tree_node_listp);
+       BUG_ON(!list_empty(stable_tree_node_listp));
+       stable_tree_node_listp = new_tree_node_listp;
+}
+
+static inline void inc_hash_strength(unsigned long delta)
+{
+       hash_strength += 1 << delta;
+       if (hash_strength > HASH_STRENGTH_MAX)
+               hash_strength = HASH_STRENGTH_MAX;
+}
+
+static inline void dec_hash_strength(unsigned long delta)
+{
+       unsigned long change = 1 << delta;
+
+       if (hash_strength <= change + 1)
+               hash_strength = 1;
+       else
+               hash_strength -= change;
+}
+
+static inline void inc_hash_strength_delta(void)
+{
+       hash_strength_delta++;
+       if (hash_strength_delta > HASH_STRENGTH_DELTA_MAX)
+               hash_strength_delta = HASH_STRENGTH_DELTA_MAX;
+}
+
+/*
+static inline unsigned long get_current_neg_ratio(void)
+{
+       if (!rshash_pos || rshash_neg > rshash_pos)
+               return 100;
+
+       return div64_u64(100 * rshash_neg , rshash_pos);
+}
+*/
+
+static inline unsigned long get_current_neg_ratio(void)
+{
+       u64 pos = benefit.pos;
+       u64 neg = benefit.neg;
+
+       if (!neg)
+               return 0;
+
+       if (!pos || neg > pos)
+               return 100;
+
+       if (neg > div64_u64(U64_MAX, 100))
+               pos = div64_u64(pos, 100);
+       else
+               neg *= 100;
+
+       return div64_u64(neg, pos);
+}
+
+static inline unsigned long get_current_benefit(void)
+{
+       u64 pos = benefit.pos;
+       u64 neg = benefit.neg;
+       u64 scanned = benefit.scanned;
+
+       if (neg > pos)
+               return 0;
+
+       return div64_u64((pos - neg), scanned);
+}
+
+static inline int judge_rshash_direction(void)
+{
+       u64 current_neg_ratio, stable_benefit;
+       u64 current_benefit, delta = 0;
+       int ret = STILL;
+
+       /* In case the system are still for a long time. */
+       if (ksm_scan_round % 1024 == 3) {
+               ret = OBSCURE;
+               goto out;
+       }
+
+       current_neg_ratio = get_current_neg_ratio();
+
+       if (current_neg_ratio == 0) {
+               rshash_neg_cont_zero++;
+               if (rshash_neg_cont_zero > 2)
+                       return GO_DOWN;
+               else
+                       return STILL;
+       }
+       rshash_neg_cont_zero = 0;
+
+       if (current_neg_ratio > 90) {
+               ret = GO_UP;
                goto out;
+       }
 
-       err = try_to_merge_one_page(vma, page, kpage);
-       if (err)
+       current_benefit = get_current_benefit();
+       stable_benefit = rshash_state.stable_benefit;
+
+       if (!stable_benefit) {
+               ret = OBSCURE;
                goto out;
+       }
+
+       if (current_benefit > stable_benefit)
+               delta = current_benefit - stable_benefit;
+       else if (current_benefit < stable_benefit)
+               delta = stable_benefit - current_benefit;
+
+       delta = div64_u64(100 * delta , stable_benefit);
+
+       if (delta > 50) {
+               rshash_cont_obscure++;
+               if (rshash_cont_obscure > 2)
+                       return OBSCURE;
+               else
+                       return STILL;
+       }
 
-       /* Must get reference to anon_vma while still holding mmap_sem */
-       hold_anon_vma(rmap_item, vma->anon_vma);
 out:
-       up_read(&mm->mmap_sem);
-       return err;
+       rshash_cont_obscure = 0;
+       return ret;
 }
 
-/*
- * try_to_merge_two_pages - take two identical pages and prepare them
- * to be merged into one page.
- *
- * This function returns the kpage if we successfully merged two identical
- * pages into one ksm page, NULL otherwise.
- *
- * Note that this function upgrades page to ksm page: if one of the pages
- * is already a ksm page, try_to_merge_with_ksm_page should be used.
+/**
+ * rshash_adjust() - The main function to control the random sampling state
+ * machine for hash strength adapting.
  */
-static struct page *try_to_merge_two_pages(struct rmap_item *rmap_item,
-                                          struct page *page,
-                                          struct rmap_item *tree_rmap_item,
-                                          struct page *tree_page)
+static void rshash_adjust(void)
 {
-       int err;
+       unsigned long prev_hash_strength = hash_strength;
 
-       err = try_to_merge_with_ksm_page(rmap_item, page, NULL);
-       if (!err) {
-               err = try_to_merge_with_ksm_page(tree_rmap_item,
-                                                       tree_page, page);
-               /*
-                * If that fails, we have a ksm page with only one pte
-                * pointing to it: so break it.
-                */
-               if (err)
-                       break_cow(rmap_item);
-       }
-       return err ? NULL : page;
-}
+       if (ksm_pages_scanned == ksm_pages_scanned_last)
+               return;
 
-/*
- * stable_tree_search - search for page inside the stable tree
- *
- * This function checks if there is a page inside the stable tree
- * with identical content to the page that we are scanning right now.
- *
- * This function returns the stable tree node of identical content if found,
- * NULL otherwise.
- */
-static struct page *stable_tree_search(struct page *page)
-{
-       struct rb_node *node = root_stable_tree.rb_node;
-       struct stable_node *stable_node;
+       encode_benefit();
 
-       stable_node = page_stable_node(page);
-       if (stable_node) {                      /* ksm page forked */
-               get_page(page);
-               return page;
+       switch (rshash_state.state) {
+       case RSHASH_STILL:
+               switch (judge_rshash_direction()) {
+               case GO_UP:
+                       if (rshash_state.pre_direct == GO_DOWN)
+                               hash_strength_delta = 0;
+
+                       inc_hash_strength(hash_strength_delta);
+                       inc_hash_strength_delta();
+                       rshash_state.stable_benefit = get_current_benefit();
+                       rshash_state.pre_direct = GO_UP;
+                       break;
+
+               case GO_DOWN:
+                       if (rshash_state.pre_direct == GO_UP)
+                               hash_strength_delta = 0;
+
+                       dec_hash_strength(hash_strength_delta);
+                       inc_hash_strength_delta();
+                       rshash_state.stable_benefit = get_current_benefit();
+                       rshash_state.pre_direct = GO_DOWN;
+                       break;
+
+               case OBSCURE:
+                       rshash_state.stable_point = hash_strength;
+                       rshash_state.turn_point_down = hash_strength;
+                       rshash_state.turn_point_up = hash_strength;
+                       rshash_state.turn_benefit_down = get_current_benefit();
+                       rshash_state.turn_benefit_up = get_current_benefit();
+                       rshash_state.lookup_window_index = 0;
+                       rshash_state.state = RSHASH_TRYDOWN;
+                       dec_hash_strength(hash_strength_delta);
+                       inc_hash_strength_delta();
+                       break;
+
+               case STILL:
+                       break;
+               default:
+                       BUG();
+               }
+               break;
+
+       case RSHASH_TRYDOWN:
+               if (rshash_state.lookup_window_index++ % 5 == 0)
+                       rshash_state.below_count = 0;
+
+               if (get_current_benefit() < rshash_state.stable_benefit)
+                       rshash_state.below_count++;
+               else if (get_current_benefit() >
+                        rshash_state.turn_benefit_down) {
+                       rshash_state.turn_point_down = hash_strength;
+                       rshash_state.turn_benefit_down = get_current_benefit();
+               }
+
+               if (rshash_state.below_count >= 3 ||
+                   judge_rshash_direction() == GO_UP ||
+                   hash_strength == 1) {
+                       hash_strength = rshash_state.stable_point;
+                       hash_strength_delta = 0;
+                       inc_hash_strength(hash_strength_delta);
+                       inc_hash_strength_delta();
+                       rshash_state.lookup_window_index = 0;
+                       rshash_state.state = RSHASH_TRYUP;
+                       hash_strength_delta = 0;
+               } else {
+                       dec_hash_strength(hash_strength_delta);
+                       inc_hash_strength_delta();
+               }
+               break;
+
+       case RSHASH_TRYUP:
+               if (rshash_state.lookup_window_index++ % 5 == 0)
+                       rshash_state.below_count = 0;
+
+               if (get_current_benefit() < rshash_state.turn_benefit_down)
+                       rshash_state.below_count++;
+               else if (get_current_benefit() > rshash_state.turn_benefit_up) {
+                       rshash_state.turn_point_up = hash_strength;
+                       rshash_state.turn_benefit_up = get_current_benefit();
+               }
+
+               if (rshash_state.below_count >= 3 ||
+                   judge_rshash_direction() == GO_DOWN ||
+                   hash_strength == HASH_STRENGTH_MAX) {
+                       hash_strength = rshash_state.turn_benefit_up >
+                               rshash_state.turn_benefit_down ?
+                               rshash_state.turn_point_up :
+                               rshash_state.turn_point_down;
+
+                       rshash_state.state = RSHASH_PRE_STILL;
+               } else {
+                       inc_hash_strength(hash_strength_delta);
+                       inc_hash_strength_delta();
+               }
+
+               break;
+
+       case RSHASH_NEW:
+       case RSHASH_PRE_STILL:
+               rshash_state.stable_benefit = get_current_benefit();
+               rshash_state.state = RSHASH_STILL;
+               hash_strength_delta = 0;
+               break;
+       default:
+               BUG();
        }
 
-       while (node) {
-               struct page *tree_page;
-               int ret;
+       /* rshash_neg = rshash_pos = 0; */
+       reset_benefit();
 
-               cond_resched();
-               stable_node = rb_entry(node, struct stable_node, node);
-               tree_page = get_ksm_page(stable_node);
-               if (!tree_page)
-                       return NULL;
+       if (prev_hash_strength != hash_strength)
+               stable_tree_delta_hash(prev_hash_strength);
+}
 
-               ret = memcmp_pages(page, tree_page);
+static void free_vma_dup_tree(struct vma_slot *slot)
+{
+       struct vma_slot *tmp_slot;
+       int i;
 
-               if (ret < 0) {
-                       put_page(tree_page);
-                       node = node->rb_left;
-               } else if (ret > 0) {
-                       put_page(tree_page);
-                       node = node->rb_right;
-               } else
-                       return tree_page;
+       /* step 1: free entries in smaller vmas' dup tree */
+       for (i = 0; i < slot->ksm_index; i++) {
+               tmp_slot = radix_tree_lookup(&ksm_vma_tree, i);
+               if (tmp_slot)
+                       radix_tree_delete(&tmp_slot->dup_tree, slot->ksm_index);
        }
 
-       return NULL;
+       /* step 2: free my own dup tree */
+       for (i = slot->ksm_index; i < ksm_vma_tree_index_end; i++)
+               radix_tree_delete(&slot->dup_tree, i);
+
+       BUG_ON(slot->dup_tree.rnode);
 }
 
-/*
- * stable_tree_insert - insert rmap_item pointing to new ksm page
- * into the stable tree.
- *
- * This function returns the stable tree node just allocated on success,
- * NULL otherwise.
+/**
+ * round_update_ladder() - The main function to do update of all the
+ * adjustments whenever a scan round is finished.
  */
-static struct stable_node *stable_tree_insert(struct page *kpage)
+static void round_update_ladder(void)
 {
-       struct rb_node **new = &root_stable_tree.rb_node;
-       struct rb_node *parent = NULL;
-       struct stable_node *stable_node;
+       int i;
+       struct vma_slot *slot, *tmp_slot;
+       unsigned long dedup_ratio_max = 0, dedup_ratio_mean = 0;
+       unsigned long threshold;
+
+       for (i = 0; i < ksm_vma_tree_index_end; i++) {
+               slot = radix_tree_lookup(&ksm_vma_tree, i);
+
+               if (slot) {
+                       slot->dedup_ratio = cal_dedup_ratio(slot);
+                       if (dedup_ratio_max < slot->dedup_ratio)
+                               dedup_ratio_max = slot->dedup_ratio;
+                       dedup_ratio_mean += slot->dedup_ratio;
+               }
+       }
 
-       while (*new) {
-               struct page *tree_page;
-               int ret;
+       dedup_ratio_mean /= ksm_vma_slot_num;
+       threshold = dedup_ratio_mean;
 
-               cond_resched();
-               stable_node = rb_entry(*new, struct stable_node, node);
-               tree_page = get_ksm_page(stable_node);
-               if (!tree_page)
-                       return NULL;
+       for (i = 0; i < ksm_vma_tree_index_end; i++) {
+               slot = radix_tree_lookup(&ksm_vma_tree, i);
 
-               ret = memcmp_pages(kpage, tree_page);
-               put_page(tree_page);
+               if (slot) {
+                       if (slot->dedup_ratio &&
+                           slot->dedup_ratio >= threshold) {
+                               vma_rung_up(slot);
+                       } else {
+                               vma_rung_down(slot);
+                       }
 
-               parent = *new;
-               if (ret < 0)
-                       new = &parent->rb_left;
-               else if (ret > 0)
-                       new = &parent->rb_right;
-               else {
+                       free_vma_dup_tree(slot);
+                       radix_tree_delete(&ksm_vma_tree, i);
+                       ksm_vma_tree_num--;
+                       slot->ksm_index = -1;
+                       slot->slot_scanned = 0;
+                       slot->dedup_ratio = 0;
+                       slot->dedup_num = 0;
+               }
+       }
+
+       for (i = 0; i < ksm_scan_ladder_size; i++) {
+               list_for_each_entry_safe(slot, tmp_slot,
+                                        &ksm_scan_ladder[i].vma_list,
+                                        ksm_list) {
                        /*
-                        * It is not a bug that stable_tree_search() didn't
-                        * find this node: because at that time our page was
-                        * not yet write-protected, so may have changed since.
+                        * The slots were scanned but not in inter_tab, their
+                        * dedup must be 0.
                         */
-                       return NULL;
+                       if (slot->slot_scanned) {
+                               BUG_ON(slot->dedup_ratio != 0);
+                               vma_rung_down(slot);
+                       }
+
+                       slot->dedup_ratio = 0;
                }
        }
 
-       stable_node = alloc_stable_node();
-       if (!stable_node)
-               return NULL;
+       BUG_ON(ksm_vma_tree_num != 0);
+       ksm_vma_tree_index_end = 0;
 
-       rb_link_node(&stable_node->node, parent, new);
-       rb_insert_color(&stable_node->node, &root_stable_tree);
+       for (i = 0; i < ksm_scan_ladder_size; i++) {
+               ksm_scan_ladder[i].round_finished = 0;
+               ksm_scan_ladder[i].busy_searched = 0;
+
+               list_for_each_entry(slot, &ksm_scan_ladder[i].vma_list,
+                                   ksm_list) {
+                       slot->last_scanned = slot->pages_scanned;
+                       slot->slot_scanned = 0;
+                       slot->pages_cowed = 0;
+                       slot->pages_merged = 0;
+                       if (slot->fully_scanned) {
+                               slot->fully_scanned = 0;
+                               ksm_scan_ladder[i].fully_scanned_slots--;
+                       }
+                       BUG_ON(slot->ksm_index != -1);
+               }
 
-       INIT_HLIST_HEAD(&stable_node->hlist);
+               BUG_ON(ksm_scan_ladder[i].fully_scanned_slots);
+       }
 
-       stable_node->kpfn = page_to_pfn(kpage);
-       set_page_stable_node(kpage, stable_node);
+       rshash_adjust();
 
-       return stable_node;
+       //ksm_pages_scanned_last = ksm_pages_scanned;
 }
 
-/*
- * unstable_tree_search_insert - search for identical page,
- * else insert rmap_item into the unstable tree.
- *
- * This function searches for a page in the unstable tree identical to the
- * page currently being scanned; and if no identical page is found in the
- * tree, we insert rmap_item as a new object into the unstable tree.
- *
- * This function returns pointer to rmap_item found to be identical
- * to the currently scanned page, NULL otherwise.
- *
- * This function does both searching and inserting, because they share
- * the same walking algorithm in an rbtree.
- */
-static
-struct rmap_item *unstable_tree_search_insert(struct rmap_item *rmap_item,
-                                             struct page *page,
-                                             struct page **tree_pagep)
+static inline unsigned int ksm_pages_to_scan(unsigned int batch_pages)
+{
+       return totalram_pages * batch_pages / 1000000;
+}
 
+static inline void cal_ladder_pages_to_scan(unsigned int num)
 {
-       struct rb_node **new = &root_unstable_tree.rb_node;
-       struct rb_node *parent = NULL;
+       int i;
 
-       while (*new) {
-               struct rmap_item *tree_rmap_item;
-               struct page *tree_page;
-               int ret;
+       for (i = 0; i < ksm_scan_ladder_size; i++) {
+               ksm_scan_ladder[i].pages_to_scan = num
+                       * ksm_scan_ladder[i].scan_ratio / KSM_SCAN_RATIO_MAX;
+       }
+       ksm_scan_ladder[0].pages_to_scan /= 16;
+       ksm_scan_ladder[1].pages_to_scan /= 4;
+}
 
-               cond_resched();
-               tree_rmap_item = rb_entry(*new, struct rmap_item, node);
-               tree_page = get_mergeable_page(tree_rmap_item);
-               if (IS_ERR_OR_NULL(tree_page))
-                       return NULL;
+static inline void ksm_del_vma_slot(struct vma_slot *slot)
+{
+       int i, j;
+       struct rmap_list_entry *entry;
+       struct vma_slot *tmp;
 
-               /*
-                * Don't substitute a ksm page for a forked page.
-                */
-               if (page == tree_page) {
-                       put_page(tree_page);
-                       return NULL;
-               }
+       /* mutex lock contention maybe intensive, other idea ? */
+       BUG_ON(list_empty(&slot->ksm_list) || !slot->rung);
 
-               ret = memcmp_pages(page, tree_page);
+       if (slot->rung->current_scan == &slot->ksm_list)
+               slot->rung->current_scan = slot->rung->current_scan->next;
 
-               parent = *new;
-               if (ret < 0) {
-                       put_page(tree_page);
-                       new = &parent->rb_left;
-               } else if (ret > 0) {
-                       put_page(tree_page);
-                       new = &parent->rb_right;
-               } else {
-                       *tree_pagep = tree_page;
-                       return tree_rmap_item;
-               }
+       list_del_init(&slot->ksm_list);
+       slot->rung->vma_num--;
+       if (slot->fully_scanned)
+               slot->rung->fully_scanned_slots--;
+
+       if (slot->rung->current_scan == &slot->rung->vma_list) {
+               /* This rung finishes a round */
+               slot->rung->round_finished = 1;
+               slot->rung->current_scan = slot->rung->vma_list.next;
+               BUG_ON(slot->rung->current_scan == &slot->rung->vma_list
+                      && !list_empty(&slot->rung->vma_list));
        }
 
-       rmap_item->address |= UNSTABLE_FLAG;
-       rmap_item->address |= (ksm_scan.seqnr & SEQNR_MASK);
-       rb_link_node(&rmap_item->node, parent, new);
-       rb_insert_color(&rmap_item->node, &root_unstable_tree);
+       if (slot->ksm_index == -1)
+               goto skip;
 
-       ksm_pages_unshared++;
-       return NULL;
+       tmp = radix_tree_delete(&ksm_vma_tree, slot->ksm_index);
+       BUG_ON(!tmp || tmp != slot);
+       free_vma_dup_tree(slot);
+       ksm_vma_tree_num--;
+       if (slot->ksm_index == ksm_vma_tree_index_end - 1)
+               ksm_vma_tree_index_end--;
+
+skip:
+       if (!slot->rmap_list_pool)
+               goto out;
+
+       for (i = 0; i < slot->pool_size; i++) {
+               void *addr;
+
+               if (!slot->rmap_list_pool[i])
+                       continue;
+
+               addr = kmap(slot->rmap_list_pool[i]);
+               BUG_ON(!addr);
+               for (j = 0; j < PAGE_SIZE / sizeof(*entry); j++) {
+                       entry = (struct rmap_list_entry *)addr + j;
+                       if (is_addr(entry->addr))
+                               continue;
+                       if (!entry->item)
+                               continue;
+
+                       remove_rmap_item_from_tree(entry->item);
+                       free_rmap_item(entry->item);
+                       slot->pool_counts[i]--;
+               }
+               BUG_ON(slot->pool_counts[i]);
+               kunmap(slot->rmap_list_pool[i]);
+               __free_page(slot->rmap_list_pool[i]);
+       }
+       kfree(slot->rmap_list_pool);
+       kfree(slot->pool_counts);
+
+out:
+       slot->rung = NULL;
+       free_vma_slot(slot);
+       BUG_ON(!ksm_vma_slot_num);
+       ksm_vma_slot_num--;
 }
 
-/*
- * stable_tree_append - add another rmap_item to the linked list of
- * rmap_items hanging off a given node of the stable tree, all sharing
- * the same ksm page.
- */
-static void stable_tree_append(struct rmap_item *rmap_item,
-                              struct stable_node *stable_node)
+
+static inline void cleanup_vma_slots(void)
 {
-       rmap_item->head = stable_node;
-       rmap_item->address |= STABLE_FLAG;
-       hlist_add_head(&rmap_item->hlist, &stable_node->hlist);
+       struct vma_slot *slot;
 
-       if (rmap_item->hlist.next)
-               ksm_pages_sharing++;
-       else
-               ksm_pages_shared++;
+       spin_lock(&vma_slot_list_lock);
+       while (!list_empty(&vma_slot_del)) {
+               slot = list_entry(vma_slot_del.next,
+                                 struct vma_slot, slot_list);
+               list_del(&slot->slot_list);
+               spin_unlock(&vma_slot_list_lock);
+               ksm_del_vma_slot(slot);
+               spin_lock(&vma_slot_list_lock);
+       }
+       spin_unlock(&vma_slot_list_lock);
 }
 
-/*
- * cmp_and_merge_page - first see if page can be merged into the stable tree;
- * if not, compare checksum to previous and if it's the same, see if page can
- * be inserted into the unstable tree, or merged with a page already there and
- * both transferred to the stable tree.
- *
- * @page: the page that we are searching identical page to.
- * @rmap_item: the reverse mapping into the virtual address of this page
+static inline int rung_fully_scanned(struct scan_rung *rung)
+{
+       return (rung->fully_scanned_slots == rung->vma_num &&
+               rung->fully_scanned_slots);
+}
+
+/**
+ * ksm_do_scan()  - the main worker function.
  */
-static void cmp_and_merge_page(struct page *page, struct rmap_item *rmap_item)
+static void ksm_do_scan(void)
 {
-       struct rmap_item *tree_rmap_item;
-       struct page *tree_page = NULL;
-       struct stable_node *stable_node;
-       struct page *kpage;
-       unsigned int checksum;
-       int err;
+       struct vma_slot *slot, *iter;
+       struct list_head *next_scan, *iter_head;
+       struct mm_struct *busy_mm;
+       unsigned char round_finished, all_rungs_emtpy;
+       int i, err;
+       unsigned long rest_pages;
+
+       might_sleep();
+
+       rest_pages = 0;
+repeat_all:
+       for (i = ksm_scan_ladder_size - 1; i >= 0; i--) {
+               struct scan_rung *rung = &ksm_scan_ladder[i];
 
-       remove_rmap_item_from_tree(rmap_item);
+               if (!rung->pages_to_scan)
+                       continue;
 
-       /* We first start with searching the page inside the stable tree */
-       kpage = stable_tree_search(page);
-       if (kpage) {
-               err = try_to_merge_with_ksm_page(rmap_item, page, kpage);
-               if (!err) {
-                       /*
-                        * The page was successfully merged:
-                        * add its rmap_item to the stable tree.
-                        */
-                       lock_page(kpage);
-                       stable_tree_append(rmap_item, page_stable_node(kpage));
-                       unlock_page(kpage);
+               if (list_empty(&rung->vma_list)) {
+                       rung->pages_to_scan = 0;
+                       continue;
                }
-               put_page(kpage);
-               return;
-       }
-
-       /*
-        * If the hash value of the page has changed from the last time
-        * we calculated it, this page is changing frequently: therefore we
-        * don't want to insert it in the unstable tree, and we don't want
-        * to waste our time searching for something identical to it there.
-        */
-       checksum = calc_checksum(page);
-       if (rmap_item->oldchecksum != checksum) {
-               rmap_item->oldchecksum = checksum;
-               return;
-       }
 
-       tree_rmap_item =
-               unstable_tree_search_insert(rmap_item, page, &tree_page);
-       if (tree_rmap_item) {
-               kpage = try_to_merge_two_pages(rmap_item, page,
-                                               tree_rmap_item, tree_page);
-               put_page(tree_page);
                /*
-                * As soon as we merge this page, we want to remove the
-                * rmap_item of the page we have merged with from the unstable
-                * tree, and insert it instead as new node in the stable tree.
+                * if a higher rung is fully scanned, its rest pages should be
+                * propagated to the lower rungs. This can prevent the higher
+                * rung from waiting a long time while it still has its
+                * pages_to_scan quota.
+                *
                 */
-               if (kpage) {
-                       remove_rmap_item_from_tree(tree_rmap_item);
+               if (rung_fully_scanned(rung)) {
+                       rest_pages += rung->pages_to_scan;
+                       rung->pages_to_scan = 0;
+                       continue;
+               }
 
-                       lock_page(kpage);
-                       stable_node = stable_tree_insert(kpage);
-                       if (stable_node) {
-                               stable_tree_append(tree_rmap_item, stable_node);
-                               stable_tree_append(rmap_item, stable_node);
+               rung->pages_to_scan += rest_pages;
+               rest_pages = 0;
+               while (rung->pages_to_scan && likely(!freezing(current))) {
+cleanup:
+                       cleanup_vma_slots();
+
+                       if (list_empty(&rung->vma_list))
+                               break;
+
+rescan:
+                       BUG_ON(rung->current_scan == &rung->vma_list &&
+                              !list_empty(&rung->vma_list));
+
+                       slot = list_entry(rung->current_scan,
+                                        struct vma_slot, ksm_list);
+
+
+                       if (slot->fully_scanned)
+                               goto next_scan;
+
+                       err = try_down_read_slot_mmap_sem(slot);
+                       if (err == -ENOENT)
+                               goto cleanup;
+
+                       busy_mm = slot->mm;
+
+busy:
+                       if (err == -EBUSY) {
+                               /* skip other vmas on the same mm */
+                               rung->busy_searched = 1;
+                               iter = slot;
+                               iter_head = slot->ksm_list.next;
+
+                               while (iter_head != &rung->vma_list) {
+                                       iter = list_entry(iter_head,
+                                                         struct vma_slot,
+                                                         ksm_list);
+                                       if (iter->vma->vm_mm != busy_mm)
+                                               break;
+                                       iter_head = iter_head->next;
+                               }
+
+                               if (iter->vma->vm_mm != busy_mm) {
+                                       rung->current_scan = &iter->ksm_list;
+                                       goto rescan;
+                               } else {
+                                       /* at the end, but still busy */
+                                       rung->current_scan = iter->ksm_list.next;
+                                       goto next_scan;
+                                       break;
+                               }
                        }
-                       unlock_page(kpage);
 
-                       /*
-                        * If we fail to insert the page into the stable tree,
-                        * we will have 2 virtual addresses that are pointing
-                        * to a ksm page left outside the stable tree,
-                        * in which case we need to break_cow on both.
-                        */
-                       if (!stable_node) {
-                               break_cow(tree_rmap_item);
-                               break_cow(rmap_item);
+                       BUG_ON(!vma_can_enter(slot->vma));
+                       if (ksm_test_exit(slot->vma->vm_mm)) {
+                               busy_mm = slot->vma->vm_mm;
+                               up_read(&slot->vma->vm_mm->mmap_sem);
+                               err = -EBUSY;
+                               goto busy;
+                       }
+
+                       if (rung->busy_searched)
+                               rung->busy_searched = 0;
+                       /* Ok, we have take the mmap_sem, ready to scan */
+                       scan_vma_one_page(slot);
+                       up_read(&slot->vma->vm_mm->mmap_sem);
+                       rung->pages_to_scan--;
+
+                       if ((slot->pages_scanned &&
+                            slot->pages_scanned % slot->pages_to_scan == 0)
+                           || slot->fully_scanned) {
+next_scan:
+                               next_scan = rung->current_scan->next;
+                               if (next_scan == &rung->vma_list) {
+                                       /*
+                                        * All the slots in this rung
+                                        * have been traveled in this
+                                        * round.
+                                        */
+                                       rung->round_finished = 1;
+                                       rung->current_scan =
+                                               rung->vma_list.next;
+                                       if (rung_fully_scanned(rung) ||
+                                           rung->busy_searched) {
+                                               /*
+                                                * All the pages in all slots
+                                                * have been scanned. Or we
+                                                * did not make any progress
+                                                * because of busy mm.
+                                                */
+                                               rest_pages +=
+                                                       rung->pages_to_scan;
+                                               rung->pages_to_scan = 0;
+                                               break;
+                                       }
+                               } else {
+                                       rung->current_scan = next_scan;
+                               }
                        }
+
+                       cond_resched();
                }
+
+               if (freezing(current))
+                       break;
        }
-}
 
-static struct rmap_item *get_next_rmap_item(struct mm_slot *mm_slot,
-                                           struct rmap_item **rmap_list,
-                                           unsigned long addr)
-{
-       struct rmap_item *rmap_item;
+       if (freezing(current))
+               return;
 
-       while (*rmap_list) {
-               rmap_item = *rmap_list;
-               if ((rmap_item->address & PAGE_MASK) == addr)
-                       return rmap_item;
-               if (rmap_item->address > addr)
+       round_finished = 1;
+       all_rungs_emtpy = 1;
+       for (i = 0; i < ksm_scan_ladder_size; i++) {
+               struct scan_rung *rung = &ksm_scan_ladder[i];
+
+               if (!list_empty(&rung->vma_list)) {
+                       all_rungs_emtpy = 0;
+                       if (!rung->round_finished)
+                               round_finished = 0;
                        break;
-               *rmap_list = rmap_item->rmap_list;
-               remove_rmap_item_from_tree(rmap_item);
-               free_rmap_item(rmap_item);
+               }
        }
 
-       rmap_item = alloc_rmap_item();
-       if (rmap_item) {
-               /* It has already been zeroed */
-               rmap_item->mm = mm_slot->mm;
-               rmap_item->address = addr;
-               rmap_item->rmap_list = *rmap_list;
-               *rmap_list = rmap_item;
-       }
-       return rmap_item;
-}
+       if (all_rungs_emtpy)
+               round_finished = 0;
 
-static struct rmap_item *scan_get_next_rmap_item(struct page **page)
-{
-       struct mm_struct *mm;
-       struct mm_slot *slot;
-       struct vm_area_struct *vma;
-       struct rmap_item *rmap_item;
+       cleanup_vma_slots();
 
-       if (list_empty(&ksm_mm_head.mm_list))
-               return NULL;
+       if (round_finished) {
+               round_update_ladder();
 
-       slot = ksm_scan.mm_slot;
-       if (slot == &ksm_mm_head) {
                /*
                 * A number of pages can hang around indefinitely on per-cpu
                 * pagevecs, raised page count preventing write_protect_page
@@ -1308,266 +4161,160 @@
                 */
                lru_add_drain_all();
 
+               /* sync with ksm_remove_vma for rb_erase */
+               ksm_scan_round++;
                root_unstable_tree = RB_ROOT;
-
-               spin_lock(&ksm_mmlist_lock);
-               slot = list_entry(slot->mm_list.next, struct mm_slot, mm_list);
-               ksm_scan.mm_slot = slot;
-               spin_unlock(&ksm_mmlist_lock);
-next_mm:
-               ksm_scan.address = 0;
-               ksm_scan.rmap_list = &slot->rmap_list;
-       }
-
-       mm = slot->mm;
-       down_read(&mm->mmap_sem);
-       if (ksm_test_exit(mm))
-               vma = NULL;
-       else
-               vma = find_vma(mm, ksm_scan.address);
-
-       for (; vma; vma = vma->vm_next) {
-               if (!(vma->vm_flags & VM_MERGEABLE))
-                       continue;
-               if (ksm_scan.address < vma->vm_start)
-                       ksm_scan.address = vma->vm_start;
-               if (!vma->anon_vma)
-                       ksm_scan.address = vma->vm_end;
-
-               while (ksm_scan.address < vma->vm_end) {
-                       if (ksm_test_exit(mm))
-                               break;
-                       *page = follow_page(vma, ksm_scan.address, FOLL_GET);
-                       if (IS_ERR_OR_NULL(*page)) {
-                               ksm_scan.address += PAGE_SIZE;
-                               cond_resched();
-                               continue;
-                       }
-                       if (PageAnon(*page) ||
-                           page_trans_compound_anon(*page)) {
-                               flush_anon_page(vma, *page, ksm_scan.address);
-                               flush_dcache_page(*page);
-                               rmap_item = get_next_rmap_item(slot,
-                                       ksm_scan.rmap_list, ksm_scan.address);
-                               if (rmap_item) {
-                                       ksm_scan.rmap_list =
-                                                       &rmap_item->rmap_list;
-                                       ksm_scan.address += PAGE_SIZE;
-                               } else
-                                       put_page(*page);
-                               up_read(&mm->mmap_sem);
-                               return rmap_item;
-                       }
-                       put_page(*page);
-                       ksm_scan.address += PAGE_SIZE;
-                       cond_resched();
-               }
-       }
-
-       if (ksm_test_exit(mm)) {
-               ksm_scan.address = 0;
-               ksm_scan.rmap_list = &slot->rmap_list;
-       }
-       /*
-        * Nuke all the rmap_items that are above this current rmap:
-        * because there were no VM_MERGEABLE vmas with such addresses.
-        */
-       remove_trailing_rmap_items(slot, ksm_scan.rmap_list);
-
-       spin_lock(&ksm_mmlist_lock);
-       ksm_scan.mm_slot = list_entry(slot->mm_list.next,
-                                               struct mm_slot, mm_list);
-       if (ksm_scan.address == 0) {
-               /*
-                * We've completed a full scan of all vmas, holding mmap_sem
-                * throughout, and found no VM_MERGEABLE: so do the same as
-                * __ksm_exit does to remove this mm from all our lists now.
-                * This applies either when cleaning up after __ksm_exit
-                * (but beware: we can reach here even before __ksm_exit),
-                * or when all VM_MERGEABLE areas have been unmapped (and
-                * mmap_sem then protects against race with MADV_MERGEABLE).
-                */
-               hlist_del(&slot->link);
-               list_del(&slot->mm_list);
-               spin_unlock(&ksm_mmlist_lock);
-
-               free_mm_slot(slot);
-               clear_bit(MMF_VM_MERGEABLE, &mm->flags);
-               up_read(&mm->mmap_sem);
-               mmdrop(mm);
-       } else {
-               spin_unlock(&ksm_mmlist_lock);
-               up_read(&mm->mmap_sem);
+               free_all_tree_nodes(&unstable_tree_node_list);
        }
 
-       /* Repeat until we've completed scanning the whole list */
-       slot = ksm_scan.mm_slot;
-       if (slot != &ksm_mm_head)
-               goto next_mm;
-
-       ksm_scan.seqnr++;
-       return NULL;
-}
-
-/**
- * ksm_do_scan  - the ksm scanner main worker function.
- * @scan_npages - number of pages we want to scan before we return.
- */
-static void ksm_do_scan(unsigned int scan_npages)
-{
-       struct rmap_item *rmap_item;
-       struct page *uninitialized_var(page);
+       for (i = 0; i < ksm_scan_ladder_size; i++) {
+               struct scan_rung *rung = &ksm_scan_ladder[i];
 
-       while (scan_npages-- && likely(!freezing(current))) {
-               cond_resched();
-               rmap_item = scan_get_next_rmap_item(&page);
-               if (!rmap_item)
-                       return;
-               if (!PageKsm(page) || !in_stable_tree(rmap_item))
-                       cmp_and_merge_page(page, rmap_item);
-               put_page(page);
+               /*
+                * Before we can go sleep, we should make sure that all the
+                * pages_to_scan quota for this scan has been finished
+                */
+               if (!list_empty(&rung->vma_list) && rung->pages_to_scan)
+                       goto repeat_all;
        }
+
+       cal_ladder_pages_to_scan(ksm_scan_batch_pages);
 }
 
 static int ksmd_should_run(void)
 {
-       return (ksm_run & KSM_RUN_MERGE) && !list_empty(&ksm_mm_head.mm_list);
+       return ksm_run & KSM_RUN_MERGE;
 }
 
-static int ksm_scan_thread(void *nothing)
-{
-       set_freezable();
-       set_user_nice(current, 5);
-
-       while (!kthread_should_stop()) {
-               mutex_lock(&ksm_thread_mutex);
-               if (ksmd_should_run())
-                       ksm_do_scan(ksm_thread_pages_to_scan);
-               mutex_unlock(&ksm_thread_mutex);
-
-               try_to_freeze();
+#define __round_mask(x, y) ((__typeof__(x))((y)-1))
+#define round_up(x, y) ((((x)-1) | __round_mask(x, y))+1)
 
-               if (ksmd_should_run()) {
-                       schedule_timeout_interruptible(
-                               msecs_to_jiffies(ksm_thread_sleep_millisecs));
-               } else {
-                       wait_event_freezable(ksm_thread_wait,
-                               ksmd_should_run() || kthread_should_stop());
-               }
-       }
-       return 0;
+static inline unsigned long vma_pool_size(struct vm_area_struct *vma)
+{
+       return round_up(sizeof(struct rmap_list_entry) * vma_pages(vma),
+                       PAGE_SIZE) >> PAGE_SHIFT;
 }
 
-int ksm_madvise(struct vm_area_struct *vma, unsigned long start,
-               unsigned long end, int advice, unsigned long *vm_flags)
+/**
+ *
+ *
+ *
+ * @param slot
+ *
+ * @return int , 1 on success, 0 on failure
+ */
+static int ksm_vma_enter(struct vma_slot *slot)
 {
-       struct mm_struct *mm = vma->vm_mm;
-       int err;
-
-       switch (advice) {
-       case MADV_MERGEABLE:
-               /*
-                * Be somewhat over-protective for now!
-                */
-               if (*vm_flags & (VM_MERGEABLE | VM_SHARED  | VM_MAYSHARE   |
-                                VM_PFNMAP    | VM_IO      | VM_DONTEXPAND |
-                                VM_RESERVED  | VM_HUGETLB | VM_INSERTPAGE |
-                                VM_NONLINEAR | VM_MIXEDMAP | VM_SAO))
-                       return 0;               /* just ignore the advice */
-
-               if (!test_bit(MMF_VM_MERGEABLE, &mm->flags)) {
-                       err = __ksm_enter(mm);
-                       if (err)
-                               return err;
-               }
-
-               *vm_flags |= VM_MERGEABLE;
-               break;
+       struct scan_rung *rung;
+       unsigned long pages_to_scan, pool_size;
 
-       case MADV_UNMERGEABLE:
-               if (!(*vm_flags & VM_MERGEABLE))
-                       return 0;               /* just ignore the advice */
+       BUG_ON(slot->pages != vma_pages(slot->vma));
+       rung = &ksm_scan_ladder[0];
 
-               if (vma->anon_vma) {
-                       err = unmerge_ksm_pages(vma, start, end);
-                       if (err)
-                               return err;
+       pages_to_scan = get_vma_random_scan_num(slot, rung->scan_ratio);
+       if (pages_to_scan) {
+               if (list_empty(&rung->vma_list))
+                       rung->current_scan = &slot->ksm_list;
+               BUG_ON(!list_empty(&slot->ksm_list));
+
+               list_add(&slot->ksm_list, &rung->vma_list);
+               slot->rung = rung;
+               slot->pages_to_scan = pages_to_scan;
+               slot->rung->vma_num++;
+               BUG_ON(PAGE_SIZE % sizeof(struct rmap_list_entry) != 0);
+
+               pool_size = vma_pool_size(slot->vma);
+
+               slot->rmap_list_pool = kzalloc(sizeof(struct page *) *
+                                              pool_size, GFP_NOWAIT);
+               slot->pool_counts = kzalloc(sizeof(unsigned long) * pool_size,
+                                           GFP_NOWAIT);
+               slot->pool_size = pool_size;
+               if (!slot->rmap_list_pool)
+                       goto failed;
+
+               if (!slot->pool_counts) {
+                       kfree(slot->rmap_list_pool);
+                       goto failed;
                }
 
-               *vm_flags &= ~VM_MERGEABLE;
-               break;
+               BUG_ON(rung->current_scan == &rung->vma_list &&
+                      !list_empty(&rung->vma_list));
+
+               ksm_vma_slot_num++;
+               BUG_ON(!ksm_vma_slot_num);
+               return 1;
        }
 
+failed:
        return 0;
 }
 
-int __ksm_enter(struct mm_struct *mm)
-{
-       struct mm_slot *mm_slot;
-       int needs_wakeup;
-
-       mm_slot = alloc_mm_slot();
-       if (!mm_slot)
-               return -ENOMEM;
-
-       /* Check ksm_run too?  Would need tighter locking */
-       needs_wakeup = list_empty(&ksm_mm_head.mm_list);
 
-       spin_lock(&ksm_mmlist_lock);
-       insert_to_mm_slots_hash(mm, mm_slot);
-       /*
-        * Insert just behind the scanning cursor, to let the area settle
-        * down a little; when fork is followed by immediate exec, we don't
-        * want ksmd to waste time setting up and tearing down an rmap_list.
-        */
-       list_add_tail(&mm_slot->mm_list, &ksm_scan.mm_slot->mm_list);
-       spin_unlock(&ksm_mmlist_lock);
+static void ksm_enter_all_slots(void)
+{
+       struct vma_slot *slot;
+       int added;
 
-       set_bit(MMF_VM_MERGEABLE, &mm->flags);
-       atomic_inc(&mm->mm_count);
+       spin_lock(&vma_slot_list_lock);
+       while (!list_empty(&vma_slot_new)) {
+               slot = list_entry(vma_slot_new.next,
+                                 struct vma_slot, slot_list);
+               /**
+                * slots are sorted by ctime_j, if one found to be too
+                * young, just stop scanning the rest ones.
+                */
+               /*
 
-       if (needs_wakeup)
-               wake_up_interruptible(&ksm_thread_wait);
+                       if (time_before(jiffies, slot->ctime_j +
+                                       msecs_to_jiffies(1000))) {
+                               spin_unlock(&vma_slot_list_lock);
+                               return;
+                       }
+               */
 
-       return 0;
+               list_del_init(&slot->slot_list);
+               added = 0;
+               if (vma_can_enter(slot->vma))
+                       added = ksm_vma_enter(slot);
+
+               if (!added) {
+                       /* Put back to new list to be del by its creator */
+                       slot->ctime_j = jiffies;
+                       list_del(&slot->slot_list);
+                       list_add_tail(&slot->slot_list, &vma_slot_noadd);
+               }
+               spin_unlock(&vma_slot_list_lock);
+               cond_resched();
+               spin_lock(&vma_slot_list_lock);
+       }
+       spin_unlock(&vma_slot_list_lock);
 }
 
-void __ksm_exit(struct mm_struct *mm)
+static int ksm_scan_thread(void *nothing)
 {
-       struct mm_slot *mm_slot;
-       int easy_to_free = 0;
+       set_freezable();
+       set_user_nice(current, 5);
 
-       /*
-        * This process is exiting: if it's straightforward (as is the
-        * case when ksmd was never running), free mm_slot immediately.
-        * But if it's at the cursor or has rmap_items linked to it, use
-        * mmap_sem to synchronize with any break_cows before pagetables
-        * are freed, and leave the mm_slot on the list for ksmd to free.
-        * Beware: ksm may already have noticed it exiting and freed the slot.
-        */
+       while (!kthread_should_stop()) {
+               mutex_lock(&ksm_thread_mutex);
+               if (ksmd_should_run()) {
+                       ksm_enter_all_slots();
+                       ksm_do_scan();
+               }
+               mutex_unlock(&ksm_thread_mutex);
+
+               try_to_freeze();
 
-       spin_lock(&ksm_mmlist_lock);
-       mm_slot = get_mm_slot(mm);
-       if (mm_slot && ksm_scan.mm_slot != mm_slot) {
-               if (!mm_slot->rmap_list) {
-                       hlist_del(&mm_slot->link);
-                       list_del(&mm_slot->mm_list);
-                       easy_to_free = 1;
+               if (ksmd_should_run()) {
+                       schedule_timeout_interruptible(ksm_sleep_jiffies);
+                       ksm_sleep_times++;
                } else {
-                       list_move(&mm_slot->mm_list,
-                                 &ksm_scan.mm_slot->mm_list);
+                       wait_event_freezable(ksm_thread_wait,
+                               ksmd_should_run() || kthread_should_stop());
                }
        }
-       spin_unlock(&ksm_mmlist_lock);
-
-       if (easy_to_free) {
-               free_mm_slot(mm_slot);
-               clear_bit(MMF_VM_MERGEABLE, &mm->flags);
-               mmdrop(mm);
-       } else if (mm_slot) {
-               down_write(&mm->mmap_sem);
-               up_write(&mm->mmap_sem);
-       }
+       return 0;
 }
 
 struct page *ksm_does_need_to_copy(struct page *page,
@@ -1597,11 +4344,13 @@
                        unsigned long *vm_flags)
 {
        struct stable_node *stable_node;
+       struct node_vma *node_vma;
        struct rmap_item *rmap_item;
-       struct hlist_node *hlist;
+       struct hlist_node *hlist, *rmap_hlist;
        unsigned int mapcount = page_mapcount(page);
        int referenced = 0;
        int search_new_forks = 0;
+       unsigned long address;
 
        VM_BUG_ON(!PageKsm(page));
        VM_BUG_ON(!PageLocked(page));
@@ -1609,38 +4358,51 @@
        stable_node = page_stable_node(page);
        if (!stable_node)
                return 0;
-again:
-       hlist_for_each_entry(rmap_item, hlist, &stable_node->hlist, hlist) {
-               struct anon_vma *anon_vma = rmap_item->anon_vma;
-               struct anon_vma_chain *vmac;
-               struct vm_area_struct *vma;
-
-               anon_vma_lock(anon_vma);
-               list_for_each_entry(vmac, &anon_vma->head, same_anon_vma) {
-                       vma = vmac->vma;
-                       if (rmap_item->address < vma->vm_start ||
-                           rmap_item->address >= vma->vm_end)
-                               continue;
-                       /*
-                        * Initially we examine only the vma which covers this
-                        * rmap_item; but later, if there is still work to do,
-                        * we examine covering vmas in other mms: in case they
-                        * were forked from the original since ksmd passed.
-                        */
-                       if ((rmap_item->mm == vma->vm_mm) == search_new_forks)
-                               continue;
 
-                       if (memcg && !mm_match_cgroup(vma->vm_mm, memcg))
-                               continue;
 
-                       referenced += page_referenced_one(page, vma,
-                               rmap_item->address, &mapcount, vm_flags);
-                       if (!search_new_forks || !mapcount)
-                               break;
+again:
+       hlist_for_each_entry(node_vma, hlist, &stable_node->hlist, hlist) {
+               hlist_for_each_entry(rmap_item, rmap_hlist,
+                                    &node_vma->rmap_hlist, hlist) {
+                       struct anon_vma *anon_vma = rmap_item->anon_vma;
+                       struct anon_vma_chain *vmac;
+                       struct vm_area_struct *vma;
+
+                       anon_vma_lock(anon_vma);
+                       list_for_each_entry(vmac, &anon_vma->head,
+                                           same_anon_vma) {
+                               vma = vmac->vma;
+                               address = get_rmap_addr(rmap_item);
+
+                               if (address < vma->vm_start ||
+                                   address >= vma->vm_end)
+                                       continue;
+                               /*
+                                * Initially we examine only the vma which
+                                * covers this rmap_item; but later, if there
+                                * is still work to do, we examine covering
+                                * vmas in other mms: in case they were forked
+                                * from the original since ksmd passed.
+                                */
+                               if ((rmap_item->slot->vma == vma) ==
+                                   search_new_forks)
+                                       continue;
+
+                               if (memcg &&
+                                   !mm_match_cgroup(vma->vm_mm, memcg))
+                                       continue;
+
+                               referenced +=
+                                       page_referenced_one(page, vma,
+                                               address, &mapcount, vm_flags);
+                               if (!search_new_forks || !mapcount)
+                                       break;
+                       }
+
+                       anon_vma_unlock(anon_vma);
+                       if (!mapcount)
+                               goto out;
                }
-               anon_vma_unlock(anon_vma);
-               if (!mapcount)
-                       goto out;
        }
        if (!search_new_forks++)
                goto again;
@@ -1651,10 +4413,12 @@
 int try_to_unmap_ksm(struct page *page, enum ttu_flags flags)
 {
        struct stable_node *stable_node;
-       struct hlist_node *hlist;
+       struct node_vma *node_vma;
+       struct hlist_node *hlist, *rmap_hlist;
        struct rmap_item *rmap_item;
        int ret = SWAP_AGAIN;
        int search_new_forks = 0;
+       unsigned long address;
 
        VM_BUG_ON(!PageKsm(page));
        VM_BUG_ON(!PageLocked(page));
@@ -1663,34 +4427,42 @@
        if (!stable_node)
                return SWAP_FAIL;
 again:
-       hlist_for_each_entry(rmap_item, hlist, &stable_node->hlist, hlist) {
-               struct anon_vma *anon_vma = rmap_item->anon_vma;
-               struct anon_vma_chain *vmac;
-               struct vm_area_struct *vma;
-
-               anon_vma_lock(anon_vma);
-               list_for_each_entry(vmac, &anon_vma->head, same_anon_vma) {
-                       vma = vmac->vma;
-                       if (rmap_item->address < vma->vm_start ||
-                           rmap_item->address >= vma->vm_end)
-                               continue;
-                       /*
-                        * Initially we examine only the vma which covers this
-                        * rmap_item; but later, if there is still work to do,
-                        * we examine covering vmas in other mms: in case they
-                        * were forked from the original since ksmd passed.
-                        */
-                       if ((rmap_item->mm == vma->vm_mm) == search_new_forks)
-                               continue;
-
-                       ret = try_to_unmap_one(page, vma,
-                                       rmap_item->address, flags);
-                       if (ret != SWAP_AGAIN || !page_mapped(page)) {
-                               anon_vma_unlock(anon_vma);
-                               goto out;
+       hlist_for_each_entry(node_vma, hlist, &stable_node->hlist, hlist) {
+               hlist_for_each_entry(rmap_item, rmap_hlist,
+                                    &node_vma->rmap_hlist, hlist) {
+                       struct anon_vma *anon_vma = rmap_item->anon_vma;
+                       struct anon_vma_chain *vmac;
+                       struct vm_area_struct *vma;
+
+                       anon_vma_lock(anon_vma);
+                       list_for_each_entry(vmac, &anon_vma->head,
+                                           same_anon_vma) {
+                               vma = vmac->vma;
+                               address = get_rmap_addr(rmap_item);
+
+                               if (address < vma->vm_start ||
+                                   address >= vma->vm_end)
+                                       continue;
+                               /*
+                                * Initially we examine only the vma which
+                                * covers this rmap_item; but later, if there
+                                * is still work to do, we examine covering
+                                * vmas in other mms: in case they were forked
+                                * from the original since ksmd passed.
+                                */
+                               if ((rmap_item->slot->vma == vma) ==
+                                   search_new_forks)
+                                       continue;
+
+                               ret = try_to_unmap_one(page, vma,
+                                                      address, flags);
+                               if (ret != SWAP_AGAIN || !page_mapped(page)) {
+                                       anon_vma_unlock(anon_vma);
+                                       goto out;
+                               }
                        }
+                       anon_vma_unlock(anon_vma);
                }
-               anon_vma_unlock(anon_vma);
        }
        if (!search_new_forks++)
                goto again;
@@ -1703,10 +4475,12 @@
                  struct vm_area_struct *, unsigned long, void *), void *arg)
 {
        struct stable_node *stable_node;
-       struct hlist_node *hlist;
+       struct node_vma *node_vma;
+       struct hlist_node *hlist, *rmap_hlist;
        struct rmap_item *rmap_item;
        int ret = SWAP_AGAIN;
        int search_new_forks = 0;
+       unsigned long address;
 
        VM_BUG_ON(!PageKsm(page));
        VM_BUG_ON(!PageLocked(page));
@@ -1715,33 +4489,35 @@
        if (!stable_node)
                return ret;
 again:
-       hlist_for_each_entry(rmap_item, hlist, &stable_node->hlist, hlist) {
-               struct anon_vma *anon_vma = rmap_item->anon_vma;
-               struct anon_vma_chain *vmac;
-               struct vm_area_struct *vma;
-
-               anon_vma_lock(anon_vma);
-               list_for_each_entry(vmac, &anon_vma->head, same_anon_vma) {
-                       vma = vmac->vma;
-                       if (rmap_item->address < vma->vm_start ||
-                           rmap_item->address >= vma->vm_end)
-                               continue;
-                       /*
-                        * Initially we examine only the vma which covers this
-                        * rmap_item; but later, if there is still work to do,
-                        * we examine covering vmas in other mms: in case they
-                        * were forked from the original since ksmd passed.
-                        */
-                       if ((rmap_item->mm == vma->vm_mm) == search_new_forks)
-                               continue;
-
-                       ret = rmap_one(page, vma, rmap_item->address, arg);
-                       if (ret != SWAP_AGAIN) {
-                               anon_vma_unlock(anon_vma);
-                               goto out;
+       hlist_for_each_entry(node_vma, hlist, &stable_node->hlist, hlist) {
+               hlist_for_each_entry(rmap_item, rmap_hlist,
+                                    &node_vma->rmap_hlist, hlist) {
+                       struct anon_vma *anon_vma = rmap_item->anon_vma;
+                       struct anon_vma_chain *vmac;
+                       struct vm_area_struct *vma;
+
+                       anon_vma_lock(anon_vma);
+                       list_for_each_entry(vmac, &anon_vma->head,
+                                           same_anon_vma) {
+                               vma = vmac->vma;
+                               address = get_rmap_addr(rmap_item);
+
+                               if (address < vma->vm_start ||
+                                   address >= vma->vm_end)
+                                       continue;
+
+                               if ((rmap_item->slot->vma == vma) ==
+                                   search_new_forks)
+                                       continue;
+
+                               ret = rmap_one(page, vma, address, arg);
+                               if (ret != SWAP_AGAIN) {
+                                       anon_vma_unlock(anon_vma);
+                                       goto out;
+                               }
                        }
+                       anon_vma_unlock(anon_vma);
                }
-               anon_vma_unlock(anon_vma);
        }
        if (!search_new_forks++)
                goto again;
@@ -1771,7 +4547,7 @@
 {
        struct rb_node *node;
 
-       for (node = rb_first(&root_stable_tree); node; node = rb_next(node)) {
+       for (node = rb_first(root_stable_treep); node; node = rb_next(node)) {
                struct stable_node *stable_node;
 
                stable_node = rb_entry(node, struct stable_node, node);
@@ -1810,7 +4586,7 @@
                 */
                while ((stable_node = ksm_check_stable_tree(mn->start_pfn,
                                        mn->start_pfn + mn->nr_pages)) != NULL)
-                       remove_node_from_stable_tree(stable_node);
+                       remove_node_from_stable_tree(stable_node, 1, 1);
                /* fallthrough */
 
        case MEM_CANCEL_OFFLINE:
@@ -1835,7 +4611,7 @@
 static ssize_t sleep_millisecs_show(struct kobject *kobj,
                                    struct kobj_attribute *attr, char *buf)
 {
-       return sprintf(buf, "%u\n", ksm_thread_sleep_millisecs);
+       return sprintf(buf, "%u\n", jiffies_to_msecs(ksm_sleep_jiffies));
 }
 
 static ssize_t sleep_millisecs_store(struct kobject *kobj,
@@ -1849,34 +4625,58 @@
        if (err || msecs > UINT_MAX)
                return -EINVAL;
 
-       ksm_thread_sleep_millisecs = msecs;
+       ksm_sleep_jiffies = msecs_to_jiffies(msecs);
 
        return count;
 }
 KSM_ATTR(sleep_millisecs);
 
-static ssize_t pages_to_scan_show(struct kobject *kobj,
+static ssize_t min_scan_ratio_show(struct kobject *kobj,
+                                   struct kobj_attribute *attr, char *buf)
+{
+       return sprintf(buf, "%u\n", ksm_min_scan_ratio);
+}
+
+static ssize_t min_scan_ratio_store(struct kobject *kobj,
+                                    struct kobj_attribute *attr,
+                                    const char *buf, size_t count)
+{
+       unsigned long msr;
+       int err;
+
+       err = strict_strtoul(buf, 10, &msr);
+       if (err || msr > UINT_MAX)
+               return -EINVAL;
+
+       ksm_min_scan_ratio = msr;
+
+       return count;
+}
+KSM_ATTR(min_scan_ratio);
+
+static ssize_t scan_batch_pages_show(struct kobject *kobj,
                                  struct kobj_attribute *attr, char *buf)
 {
-       return sprintf(buf, "%u\n", ksm_thread_pages_to_scan);
+       return sprintf(buf, "%lu\n", ksm_scan_batch_pages);
 }
 
-static ssize_t pages_to_scan_store(struct kobject *kobj,
+static ssize_t scan_batch_pages_store(struct kobject *kobj,
                                   struct kobj_attribute *attr,
                                   const char *buf, size_t count)
 {
        int err;
-       unsigned long nr_pages;
+       unsigned long batch_pages;
 
-       err = strict_strtoul(buf, 10, &nr_pages);
-       if (err || nr_pages > UINT_MAX)
+       err = strict_strtoul(buf, 10, &batch_pages);
+       if (err || batch_pages > UINT_MAX)
                return -EINVAL;
 
-       ksm_thread_pages_to_scan = nr_pages;
+       ksm_scan_batch_pages = batch_pages;
+       cal_ladder_pages_to_scan(ksm_scan_batch_pages);
 
        return count;
 }
-KSM_ATTR(pages_to_scan);
+KSM_ATTR(scan_batch_pages);
 
 static ssize_t run_show(struct kobject *kobj, struct kobj_attribute *attr,
                        char *buf)
@@ -1893,28 +4693,12 @@
        err = strict_strtoul(buf, 10, &flags);
        if (err || flags > UINT_MAX)
                return -EINVAL;
-       if (flags > KSM_RUN_UNMERGE)
+       if (flags > KSM_RUN_MERGE)
                return -EINVAL;
 
-       /*
-        * KSM_RUN_MERGE sets ksmd running, and 0 stops it running.
-        * KSM_RUN_UNMERGE stops it running and unmerges all rmap_items,
-        * breaking COW to free the pages_shared (but leaves mm_slots
-        * on the list for when ksmd may be set running again).
-        */
-
        mutex_lock(&ksm_thread_mutex);
        if (ksm_run != flags) {
                ksm_run = flags;
-               if (flags & KSM_RUN_UNMERGE) {
-                       current->flags |= PF_OOM_ORIGIN;
-                       err = unmerge_and_remove_all_rmap_items();
-                       current->flags &= ~PF_OOM_ORIGIN;
-                       if (err) {
-                               ksm_run = KSM_RUN_STOP;
-                               count = err;
-                       }
-               }
        }
        mutex_unlock(&ksm_thread_mutex);
 
@@ -1925,6 +4709,30 @@
 }
 KSM_ATTR(run);
 
+
+static ssize_t thrash_threshold_show(struct kobject *kobj,
+                                    struct kobj_attribute *attr, char *buf)
+{
+       return sprintf(buf, "%u\n", ksm_thrash_threshold);
+}
+
+static ssize_t thrash_threshold_store(struct kobject *kobj,
+                                     struct kobj_attribute *attr,
+                                     const char *buf, size_t count)
+{
+       int err;
+       unsigned long flags;
+
+       err = strict_strtoul(buf, 10, &flags);
+       if (err || flags > 99)
+               return -EINVAL;
+
+       ksm_thrash_threshold = flags;
+
+       return count;
+}
+KSM_ATTR(thrash_threshold);
+
 static ssize_t pages_shared_show(struct kobject *kobj,
                                 struct kobj_attribute *attr, char *buf)
 {
@@ -1946,60 +4754,291 @@
 }
 KSM_ATTR_RO(pages_unshared);
 
-static ssize_t pages_volatile_show(struct kobject *kobj,
-                                  struct kobj_attribute *attr, char *buf)
+static ssize_t full_scans_show(struct kobject *kobj,
+                              struct kobj_attribute *attr, char *buf)
 {
-       long ksm_pages_volatile;
+       return sprintf(buf, "%llu\n", ksm_scan_round);
+}
+KSM_ATTR_RO(full_scans);
 
-       ksm_pages_volatile = ksm_rmap_items - ksm_pages_shared
-                               - ksm_pages_sharing - ksm_pages_unshared;
-       /*
-        * It was not worth any locking to calculate that statistic,
-        * but it might therefore sometimes be negative: conceal that.
-        */
-       if (ksm_pages_volatile < 0)
-               ksm_pages_volatile = 0;
-       return sprintf(buf, "%ld\n", ksm_pages_volatile);
+static ssize_t pages_scanned_show(struct kobject *kobj,
+                                 struct kobj_attribute *attr, char *buf)
+{
+       unsigned long base = 0;
+       u64 delta, ret;
+
+       if (pages_scanned_stored) {
+               base = pages_scanned_base;
+               ret = pages_scanned_stored;
+               delta = ksm_pages_scanned >> base;
+               if (CAN_OVERFLOW_U64(ret, delta)) {
+                       ret >>= 1;
+                       delta >>= 1;
+                       base++;
+                       ret += delta;
+               }
+       } else {
+               ret = ksm_pages_scanned;
+       }
+
+       while (ret > ULONG_MAX) {
+               ret >>= 1;
+               base++;
+       }
+
+       if (base)
+               return sprintf(buf, "%lu * 2^%lu\n", (unsigned long)ret, base);
+       else
+               return sprintf(buf, "%lu\n", (unsigned long)ret);
 }
-KSM_ATTR_RO(pages_volatile);
+KSM_ATTR_RO(pages_scanned);
 
-static ssize_t full_scans_show(struct kobject *kobj,
-                              struct kobj_attribute *attr, char *buf)
+static ssize_t hash_strength_show(struct kobject *kobj,
+                                 struct kobj_attribute *attr, char *buf)
 {
-       return sprintf(buf, "%lu\n", ksm_scan.seqnr);
+       return sprintf(buf, "%lu\n", hash_strength);
 }
-KSM_ATTR_RO(full_scans);
+KSM_ATTR_RO(hash_strength);
+
+static ssize_t sleep_times_show(struct kobject *kobj,
+                                 struct kobj_attribute *attr, char *buf)
+{
+       return sprintf(buf, "%llu\n", ksm_sleep_times);
+}
+KSM_ATTR_RO(sleep_times);
+
 
 static struct attribute *ksm_attrs[] = {
        &sleep_millisecs_attr.attr,
-       &pages_to_scan_attr.attr,
+       &scan_batch_pages_attr.attr,
        &run_attr.attr,
        &pages_shared_attr.attr,
        &pages_sharing_attr.attr,
        &pages_unshared_attr.attr,
-       &pages_volatile_attr.attr,
        &full_scans_attr.attr,
+       &min_scan_ratio_attr.attr,
+       &pages_scanned_attr.attr,
+       &hash_strength_attr.attr,
+       &sleep_times_attr.attr,
+       &thrash_threshold_attr.attr,
        NULL,
 };
 
 static struct attribute_group ksm_attr_group = {
        .attrs = ksm_attrs,
-       .name = "ksm",
+       .name = "uksm",
 };
 #endif /* CONFIG_SYSFS */
 
+static inline void init_scan_ladder(void)
+{
+       int i;
+       unsigned long mul = 1;
+
+       unsigned long pages_to_scan;
+
+       pages_to_scan = ksm_scan_batch_pages;
+
+       for (i = 0; i < ksm_scan_ladder_size; i++,
+             mul *= ksm_scan_ratio_delta) {
+
+               ksm_scan_ladder[i].scan_ratio = ksm_min_scan_ratio * mul;
+               INIT_LIST_HEAD(&ksm_scan_ladder[i].vma_list);
+               ksm_scan_ladder[i].vma_num = 0;
+               ksm_scan_ladder[i].round_finished = 0;
+               ksm_scan_ladder[i].fully_scanned_slots = 0;
+               ksm_scan_ladder[i].busy_searched = 0;
+       }
+
+       cal_ladder_pages_to_scan(ksm_scan_batch_pages);
+}
+
+static inline int cal_positive_negative_costs(void)
+{
+       struct page *p1, *p2;
+       unsigned char *addr1, *addr2;
+       unsigned long i, time_start, hash_cost;
+       unsigned long loopnum = 0;
+
+       /*IMPORTANT: volatile is needed to prevent over-optimization by gcc. */
+       volatile u32 hash;
+       volatile int ret;
+
+       p1 = alloc_page(GFP_KERNEL);
+       if (!p1)
+               return -ENOMEM;
+
+       p2 = alloc_page(GFP_KERNEL);
+       if (!p2)
+               return -ENOMEM;
+
+       addr1 = kmap_atomic(p1, KM_USER0);
+       addr2 = kmap_atomic(p2, KM_USER1);
+       memset(addr1, random32(), PAGE_SIZE);
+       memcpy(addr2, addr1, PAGE_SIZE);
+
+       /* make sure that the two pages differ in last byte */
+       addr2[PAGE_SIZE-1] = ~addr2[PAGE_SIZE-1];
+       kunmap_atomic(addr2, KM_USER1);
+       kunmap_atomic(addr1, KM_USER0);
+
+       time_start = jiffies;
+       while (jiffies - time_start < 100) {
+               for (i = 0; i < 100; i++)
+                       hash = page_hash(p1, HASH_STRENGTH_FULL, 0);
+               loopnum += 100;
+       }
+       hash_cost = (jiffies - time_start);
+
+       time_start = jiffies;
+       for (i = 0; i < loopnum; i++)
+               ret = pages_identical(p1, p2);
+       memcmp_cost = HASH_STRENGTH_FULL * (jiffies - time_start);
+       memcmp_cost /= hash_cost;
+       printk(KERN_INFO "UKSM: relative memcmp_cost = %lu.\n", memcmp_cost);
+
+       __free_page(p1);
+       __free_page(p2);
+       return 0;
+}
+
+static int init_zeropage_hash_table(void)
+{
+       struct page *page;
+       char *addr;
+       int i;
+
+       page = alloc_page(GFP_KERNEL);
+       if (!page)
+               return -ENOMEM;
+
+       addr = kmap_atomic(page, KM_USER0);
+       memset(addr, 0, PAGE_SIZE);
+       kunmap_atomic(addr, KM_USER0);
+
+       zero_hash_table = kmalloc(HASH_STRENGTH_MAX * sizeof(u32),
+               GFP_KERNEL);
+       if (!zero_hash_table)
+               return -ENOMEM;
+
+       for (i = 0; i < HASH_STRENGTH_MAX; i++)
+               zero_hash_table[i] = page_hash(page, i, 0);
+
+       __free_page(page);
+
+       return 0;
+}
+
+static inline int init_random_sampling(void)
+{
+       unsigned long i;
+       random_nums = kmalloc(PAGE_SIZE, GFP_KERNEL);
+       if (!random_nums)
+               return -ENOMEM;
+
+       for (i = 0; i < HASH_STRENGTH_FULL; i++)
+               random_nums[i] = i;
+
+       for (i = 0; i < HASH_STRENGTH_FULL; i++) {
+               unsigned long rand_range, swap_index, tmp;
+
+               rand_range = HASH_STRENGTH_FULL - i;
+               swap_index = i + random32() % rand_range;
+               tmp = random_nums[i];
+               random_nums[i] =  random_nums[swap_index];
+               random_nums[swap_index] = tmp;
+       }
+
+       rshash_state.state = RSHASH_NEW;
+       rshash_state.below_count = 0;
+       rshash_state.lookup_window_index = 0;
+
+       return cal_positive_negative_costs();
+}
+
+static int __init ksm_slab_init(void)
+{
+       rmap_item_cache = KSM_KMEM_CACHE(rmap_item, 0);
+       if (!rmap_item_cache)
+               goto out;
+
+       stable_node_cache = KSM_KMEM_CACHE(stable_node, 0);
+       if (!stable_node_cache)
+               goto out_free1;
+
+       node_vma_cache = KSM_KMEM_CACHE(node_vma, 0);
+       if (!node_vma_cache)
+               goto out_free2;
+
+       vma_slot_cache = KSM_KMEM_CACHE(vma_slot, 0);
+       if (!vma_slot_cache)
+               goto out_free3;
+
+       tree_node_cache = KSM_KMEM_CACHE(tree_node, 0);
+       if (!tree_node_cache)
+               goto out_free4;
+
+       return 0;
+
+out_free4:
+       kmem_cache_destroy(vma_slot_cache);
+out_free3:
+       kmem_cache_destroy(node_vma_cache);
+out_free2:
+       kmem_cache_destroy(stable_node_cache);
+out_free1:
+       kmem_cache_destroy(rmap_item_cache);
+out:
+       return -ENOMEM;
+}
+
+static void __init ksm_slab_free(void)
+{
+       kmem_cache_destroy(stable_node_cache);
+       kmem_cache_destroy(rmap_item_cache);
+       kmem_cache_destroy(node_vma_cache);
+       kmem_cache_destroy(vma_slot_cache);
+       kmem_cache_destroy(tree_node_cache);
+}
+
 static int __init ksm_init(void)
 {
        struct task_struct *ksm_thread;
        int err;
+       unsigned int sr = ksm_min_scan_ratio;
+
+       ksm_scan_ladder_size = 1;
+       while (sr < KSM_SCAN_RATIO_MAX) {
+               sr *= ksm_scan_ratio_delta;
+               ksm_scan_ladder_size++;
+       }
+       ksm_scan_ladder = kzalloc(sizeof(struct scan_rung) *
+                                 ksm_scan_ladder_size, GFP_KERNEL);
+       if (!ksm_scan_ladder) {
+               printk(KERN_ERR "uksm scan ladder allocation failed, size=%d\n",
+                      ksm_scan_ladder_size);
+               err = ENOMEM;
+               goto out;
+       }
+       init_scan_ladder();
+
+       INIT_RADIX_TREE(&ksm_vma_tree, GFP_KERNEL);
+
+       err = init_random_sampling();
+       if (err)
+               goto out_free2;
 
        err = ksm_slab_init();
        if (err)
-               goto out;
+               goto out_free1;
 
-       ksm_thread = kthread_run(ksm_scan_thread, NULL, "ksmd");
+       err = init_zeropage_hash_table();
+       if (err)
+               goto out_free0;
+
+       ksm_thread = kthread_run(ksm_scan_thread, NULL, "uksmd");
        if (IS_ERR(ksm_thread)) {
-               printk(KERN_ERR "ksm: creating kthread failed\n");
+               printk(KERN_ERR "uksm: creating kthread failed\n");
                err = PTR_ERR(ksm_thread);
                goto out_free;
        }
@@ -2007,7 +5046,7 @@
 #ifdef CONFIG_SYSFS
        err = sysfs_create_group(mm_kobj, &ksm_attr_group);
        if (err) {
-               printk(KERN_ERR "ksm: register sysfs failed\n");
+               printk(KERN_ERR "uksm: register sysfs failed\n");
                kthread_stop(ksm_thread);
                goto out_free;
        }
@@ -2026,8 +5065,20 @@
        return 0;
 
 out_free:
+       kfree(zero_hash_table);
+out_free0:
        ksm_slab_free();
+out_free1:
+       kfree(random_nums);
+out_free2:
+       kfree(ksm_scan_ladder);
 out:
        return err;
 }
+
+#ifdef MODULE
 module_init(ksm_init)
+#else
+late_initcall(ksm_init);
+#endif
+
diff -urN linux-2.6.38/mm/madvise.c uksm-2.6.38-zhang/mm/madvise.c
--- linux-2.6.38/mm/madvise.c   2011-03-15 09:20:32.000000000 +0800
+++ uksm-2.6.38-zhang/mm/madvise.c      2012-01-09 10:05:59.862270375 +0800
@@ -65,12 +65,6 @@
                }
                new_flags &= ~VM_DONTCOPY;
                break;
-       case MADV_MERGEABLE:
-       case MADV_UNMERGEABLE:
-               error = ksm_madvise(vma, start, end, behavior, &new_flags);
-               if (error)
-                       goto out;
-               break;
        case MADV_HUGEPAGE:
        case MADV_NOHUGEPAGE:
                error = hugepage_madvise(vma, &new_flags, behavior);
@@ -285,10 +279,6 @@
        case MADV_REMOVE:
        case MADV_WILLNEED:
        case MADV_DONTNEED:
-#ifdef CONFIG_KSM
-       case MADV_MERGEABLE:
-       case MADV_UNMERGEABLE:
-#endif
 #ifdef CONFIG_TRANSPARENT_HUGEPAGE
        case MADV_HUGEPAGE:
        case MADV_NOHUGEPAGE:
diff -urN linux-2.6.38/mm/memory.c uksm-2.6.38-zhang/mm/memory.c
--- linux-2.6.38/mm/memory.c    2011-03-15 09:20:32.000000000 +0800
+++ uksm-2.6.38-zhang/mm/memory.c       2012-01-09 10:09:28.235610655 +0800
@@ -112,6 +112,37 @@
 unsigned long zero_pfn __read_mostly;
 unsigned long highest_memmap_pfn __read_mostly;
 
+#ifdef CONFIG_KSM
+unsigned long ksm_zero_pfn __read_mostly;
+struct page *empty_ksm_zero_page;
+
+static int __init setup_ksm_zero_page(void)
+{
+       unsigned long addr;
+       addr = __get_free_pages(GFP_KERNEL | __GFP_ZERO, 0);
+       if (!addr)
+               panic("Oh boy, that early out of memory?");
+
+       empty_ksm_zero_page = virt_to_page((void *) addr);
+       SetPageReserved(empty_ksm_zero_page);
+
+       ksm_zero_pfn = page_to_pfn(empty_ksm_zero_page);
+
+       return 0;
+}
+core_initcall(setup_ksm_zero_page);
+
+static inline int is_ksm_zero_pfn(unsigned long pfn)
+{
+       return pfn == ksm_zero_pfn;
+}
+#else
+static inline int is_ksm_zero_pfn(unsigned long pfn)
+{
+       return 0;
+}
+#endif
+
 /*
  * CONFIG_MMU architectures set up ZERO_PAGE in their paging_init()
  */
@@ -123,6 +154,7 @@
 core_initcall(init_zero_pfn);
 
 
+
 #if defined(SPLIT_RSS_COUNTING)
 
 static void __sync_task_rss_stat(struct task_struct *task, struct mm_struct *mm)
@@ -609,7 +641,7 @@
                        goto check_pfn;
                if (vma->vm_flags & (VM_PFNMAP | VM_MIXEDMAP))
                        return NULL;
-               if (!is_zero_pfn(pfn))
+               if (!is_zero_pfn(pfn) && !is_ksm_zero_pfn(pfn))
                        print_bad_pte(vma, addr, pte, NULL);
                return NULL;
        }
@@ -631,7 +663,7 @@
                }
        }
 
-       if (is_zero_pfn(pfn))
+       if (is_zero_pfn(pfn) || is_ksm_zero_pfn(pfn))
                return NULL;
 check_pfn:
        if (unlikely(pfn > highest_memmap_pfn)) {
@@ -719,6 +751,10 @@
                        rss[MM_ANONPAGES]++;
                else
                        rss[MM_FILEPAGES]++;
+#ifdef CONFIG_KSM
+               if (PageKsm(page)) /* follows page_dup_rmap() */
+                       inc_zone_page_state(page, NR_KSM_PAGES_SHARING);
+#endif
        }
 
 out_set_pte:
@@ -1341,7 +1377,8 @@
        page = vm_normal_page(vma, address, pte);
        if (unlikely(!page)) {
                if ((flags & FOLL_DUMP) ||
-                   !is_zero_pfn(pte_pfn(pte)))
+                   !is_zero_pfn(pte_pfn(pte)) ||
+                   !is_ksm_zero_pfn(pte_pfn(pte)))
                        goto bad_page;
                page = pte_page(pte);
        }
@@ -1423,7 +1460,7 @@
 
        VM_BUG_ON(!!pages != !!(gup_flags & FOLL_GET));
 
-       /* 
+       /*
         * Require read or write permissions.
         * If FOLL_FORCE is set, we only require the "MAY" flags.
         */
@@ -1470,7 +1507,8 @@
                                page = vm_normal_page(gate_vma, start, *pte);
                                if (!page) {
                                        if (!(gup_flags & FOLL_DUMP) &&
-                                            is_zero_pfn(pte_pfn(*pte)))
+                                           (is_zero_pfn(pte_pfn(*pte)) ||
+                                            is_ksm_zero_pfn(pte_pfn(*pte))))
                                                page = pte_page(*pte);
                                        else {
                                                pte_unmap(pte);
@@ -2158,8 +2196,13 @@
                        clear_page(kaddr);
                kunmap_atomic(kaddr, KM_USER0);
                flush_dcache_page(dst);
-       } else
+       } else {
                copy_user_highpage(dst, src, va, vma);
+#ifdef CONFIG_KSM
+               if (vma->ksm_vma_slot && PageKsm(src))
+                       vma->ksm_vma_slot->pages_cowed++;
+#endif
+       }
 }
 
 /*
@@ -2353,10 +2396,15 @@
        if (unlikely(anon_vma_prepare(vma)))
                goto oom;
 
-       if (is_zero_pfn(pte_pfn(orig_pte))) {
+       if (is_zero_pfn(pte_pfn(orig_pte))
+                       || is_ksm_zero_pfn(pte_pfn(orig_pte))) {
                new_page = alloc_zeroed_user_highpage_movable(vma, address);
                if (!new_page)
                        goto oom;
+#ifdef CONFIG_KSM
+               if (vma->ksm_vma_slot && is_ksm_zero_pfn(pte_pfn(orig_pte)))
+                       vma->ksm_vma_slot->pages_cowed++;
+#endif
        } else {
                new_page = alloc_page_vma(GFP_HIGHUSER_MOVABLE, vma, address);
                if (!new_page)
@@ -2378,6 +2426,11 @@
                                dec_mm_counter_fast(mm, MM_FILEPAGES);
                                inc_mm_counter_fast(mm, MM_ANONPAGES);
                        }
+#ifdef CONFIG_KSM
+                       if (is_ksm_zero_pfn(pte_pfn(orig_pte)))
+                               __dec_zone_page_state(old_page,
+                                               NR_KSM_ZERO_PAGES);
+#endif
                } else
                        inc_mm_counter_fast(mm, MM_ANONPAGES);
                flush_cache_page(vma, address, pte_pfn(orig_pte));
Binary files linux-2.6.38/mm/.memory.c.swp and uksm-2.6.38-zhang/mm/.memory.c.swp differ
diff -urN linux-2.6.38/mm/mmap.c uksm-2.6.38-zhang/mm/mmap.c
--- linux-2.6.38/mm/mmap.c      2011-03-15 09:20:32.000000000 +0800
+++ uksm-2.6.38-zhang/mm/mmap.c 2012-01-09 10:05:59.872270374 +0800
@@ -30,6 +30,7 @@
 #include <linux/perf_event.h>
 #include <linux/audit.h>
 #include <linux/khugepaged.h>
+#include <linux/ksm.h>
 
 #include <asm/uaccess.h>
 #include <asm/cacheflush.h>
@@ -65,7 +66,7 @@
  * MAP_SHARED  r: (no) no      r: (yes) yes    r: (no) yes     r: (no) yes
  *             w: (no) no      w: (no) no      w: (yes) yes    w: (no) no
  *             x: (no) no      x: (no) yes     x: (no) yes     x: (yes) yes
- *             
+ *
  * MAP_PRIVATE r: (no) no      r: (yes) yes    r: (no) yes     r: (no) yes
  *             w: (no) no      w: (no) no      w: (copy) copy  w: (no) no
  *             x: (no) no      x: (no) yes     x: (no) yes     x: (yes) yes
@@ -240,6 +241,9 @@
                        removed_exe_file_vma(vma->vm_mm);
        }
        mpol_put(vma_policy(vma));
+#ifdef CONFIG_KSM
+       ksm_remove_vma(vma);
+#endif
        kmem_cache_free(vm_area_cachep, vma);
        return next;
 }
@@ -529,9 +533,20 @@
        long adjust_next = 0;
        int remove_next = 0;
 
+/*
+ * to avoid deadlock, ksm_remove_vma must be done before any spin_lock is
+ * acquired
+ */
+#ifdef CONFIG_KSM
+       ksm_remove_vma(vma);
+#endif
+
        if (next && !insert) {
                struct vm_area_struct *exporter = NULL;
 
+#ifdef CONFIG_KSM
+               ksm_remove_vma(next);
+#endif
                if (end >= next->vm_end) {
                        /*
                         * vma expands, overlapping all the next, and
@@ -616,10 +631,10 @@
                if (adjust_next)
                        vma_prio_tree_remove(next, root);
        }
-
        vma->vm_start = start;
        vma->vm_end = end;
        vma->vm_pgoff = pgoff;
+
        if (adjust_next) {
                next->vm_start += adjust_next << PAGE_SHIFT;
                next->vm_pgoff += adjust_next;
@@ -672,10 +687,22 @@
                 */
                if (remove_next == 2) {
                        next = vma->vm_next;
+#ifdef CONFIG_KSM
+                       ksm_remove_vma(next);
+#endif
                        goto again;
                }
+       } else {
+#ifdef CONFIG_KSM
+               if (next && !insert)
+                       ksm_vma_add_new(next);
+#endif
        }
 
+#ifdef CONFIG_KSM
+       ksm_vma_add_new(vma);
+#endif
+
        validate_mm(mm);
 
        return 0;
@@ -1352,6 +1379,9 @@
 
        vma_link(mm, vma, prev, rb_link, rb_parent);
        file = vma->vm_file;
+#ifdef CONFIG_KSM
+       ksm_vma_add_new(vma);
+#endif
 
        /* Once vma denies write, undo our temporary denial count */
        if (correct_wcount)
@@ -1378,6 +1408,9 @@
        unmap_region(mm, vma, prev, vma->vm_start, vma->vm_end);
        charged = 0;
 free_vma:
+#ifdef CONFIG_KSM
+       ksm_remove_vma(vma);
+#endif
        kmem_cache_free(vm_area_cachep, vma);
 unacct_error:
        if (charged)
@@ -1453,7 +1486,7 @@
                addr = vma->vm_end;
        }
 }
-#endif 
+#endif
 
 void arch_unmap_area(struct mm_struct *mm, unsigned long addr)
 {
@@ -2014,6 +2047,10 @@
        else
                err = vma_adjust(vma, vma->vm_start, addr, vma->vm_pgoff, new);
 
+#ifdef CONFIG_KSM
+       ksm_vma_add_new(new);
+#endif
+
        /* Success. */
        if (!err)
                return 0;
@@ -2250,6 +2287,9 @@
        vma->vm_flags = flags;
        vma->vm_page_prot = vm_get_page_prot(flags);
        vma_link(mm, vma, prev, rb_link, rb_parent);
+#ifdef CONFIG_KSM
+       ksm_vma_add_new(vma);
+#endif
 out:
        perf_event_mmap(vma);
        mm->total_vm += len >> PAGE_SHIFT;
@@ -2273,6 +2313,12 @@
        /* mm's last user has gone, and its about to be pulled down */
        mmu_notifier_release(mm);
 
+       /*
+        * Taking write lock on mmap_sem does not harm others,
+        * but it's crucial for uksm to avoid races.
+        */
+       down_write(&mm->mmap_sem);
+
        if (mm->locked_vm) {
                vma = mm->mmap;
                while (vma) {
@@ -2306,6 +2352,11 @@
        while (vma)
                vma = remove_vma(vma);
 
+       mm->mmap = NULL;
+       mm->mm_rb = RB_ROOT;
+       mm->mmap_cache = NULL;
+       up_write(&mm->mmap_sem);
+
        BUG_ON(mm->nr_ptes > (FIRST_USER_ADDRESS+PMD_SIZE-1)>>PMD_SHIFT);
 }
 
@@ -2397,6 +2448,9 @@
                        if (new_vma->vm_ops && new_vma->vm_ops->open)
                                new_vma->vm_ops->open(new_vma);
                        vma_link(mm, new_vma, prev, rb_link, rb_parent);
+#ifdef CONFIG_KSM
+                       ksm_vma_add_new(new_vma);
+#endif
                }
        }
        return new_vma;
@@ -2502,11 +2556,14 @@
        ret = insert_vm_struct(mm, vma);
        if (ret)
                goto out;
-
        mm->total_vm += len >> PAGE_SHIFT;
 
        perf_event_mmap(vma);
 
+#ifdef CONFIG_KSM
+       ksm_vma_add_new(vma);
+#endif
+
        return 0;
 
 out:
diff -urN linux-2.6.38/mm/mremap.c uksm-2.6.38-zhang/mm/mremap.c
--- linux-2.6.38/mm/mremap.c    2011-03-15 09:20:32.000000000 +0800
+++ uksm-2.6.38-zhang/mm/mremap.c       2012-01-09 10:05:59.872270374 +0800
@@ -191,8 +191,7 @@
         * pages recently unmapped.  But leave vma->vm_flags as it was,
         * so KSM can come around to merge on vma and new_vma afterwards.
         */
-       err = ksm_madvise(vma, old_addr, old_addr + old_len,
-                                               MADV_UNMERGEABLE, &vm_flags);
+       err = unmerge_ksm_pages(vma, old_addr, old_addr + old_len);
        if (err)
                return err;
 
diff -urN linux-2.6.38/mm/rmap.c uksm-2.6.38-zhang/mm/rmap.c
--- linux-2.6.38/mm/rmap.c      2011-03-15 09:20:32.000000000 +0800
+++ uksm-2.6.38-zhang/mm/rmap.c 2012-01-09 10:05:59.875603707 +0800
@@ -817,9 +817,9 @@
 
 /**
  * __page_set_anon_rmap - set up new anonymous rmap
- * @page:      Page to add to rmap     
+ * @page:      Page to add to rmap
  * @vma:       VM area to add page to.
- * @address:   User virtual address of the mapping     
+ * @address:   User virtual address of the mapping
  * @exclusive: the page is exclusively owned by the current process
  */
 static void __page_set_anon_rmap(struct page *page,
@@ -905,9 +905,12 @@
                        __inc_zone_page_state(page,
                                              NR_ANON_TRANSPARENT_HUGEPAGES);
        }
-       if (unlikely(PageKsm(page)))
+#ifdef CONFIG_KSM
+       if (unlikely(PageKsm(page))) {
+               __inc_zone_page_state(page, NR_KSM_PAGES_SHARING);
                return;
-
+       }
+#endif
        VM_BUG_ON(!PageLocked(page));
        VM_BUG_ON(address < vma->vm_start || address >= vma->vm_end);
        if (first)
@@ -965,6 +968,10 @@
  */
 void page_remove_rmap(struct page *page)
 {
+#ifdef CONFIG_KSM
+       if (PageKsm(page))
+               __dec_zone_page_state(page, NR_KSM_PAGES_SHARING);
+#endif
        /* page still mapped by someone else? */
        if (!atomic_add_negative(-1, &page->_mapcount))
                return;
diff -urN linux-2.6.38/security/apparmor/capability_names.h uksm-2.6.38-zhang/security/apparmor/capability_names.h
--- linux-2.6.38/security/apparmor/capability_names.h   1970-01-01 08:00:00.000000000 +0800
+++ uksm-2.6.38-zhang/security/apparmor/capability_names.h      2012-01-10 09:30:37.569678996 +0800
@@ -0,0 +1,37 @@
+static const char *capability_names[] = {
+[0]  = "chown",
+[1]  = "dac_override",
+[2]  = "dac_read_search",
+[3]  = "fowner",
+[4]  = "fsetid",
+[5]  = "kill",
+[6]  = "setgid",
+[7]  = "setuid",
+[8]  = "setpcap",
+[9]  = "linux_immutable",
+[10]  = "net_bind_service",
+[11]  = "net_broadcast",
+[12]  = "net_admin",
+[13]  = "net_raw",
+[14]  = "ipc_lock",
+[15]  = "ipc_owner",
+[16]  = "sys_module",
+[17]  = "sys_rawio",
+[18]  = "sys_chroot",
+[19]  = "sys_ptrace",
+[20]  = "sys_pacct",
+[21]  = "sys_admin",
+[22]  = "sys_boot",
+[23]  = "sys_nice",
+[24]  = "sys_resource",
+[25]  = "sys_time",
+[26]  = "sys_tty_config",
+[27]  = "mknod",
+[28]  = "lease",
+[29]  = "audit_write",
+[30]  = "audit_control",
+[31]  = "setfcap",
+[32]  = "mac_override",
+[33]  = "mac_admin",
+[34]  = "syslog",
+};
diff -urN linux-2.6.38/security/apparmor/rlim_names.h uksm-2.6.38-zhang/security/apparmor/rlim_names.h
--- linux-2.6.38/security/apparmor/rlim_names.h 1970-01-01 08:00:00.000000000 +0800
+++ uksm-2.6.38-zhang/security/apparmor/rlim_names.h    2012-01-10 09:30:41.073012457 +0800
@@ -0,0 +1,36 @@
+static const char *rlim_names[] = {
+[0]  = "cpu",
+[1]  = "fsize",
+[2]  = "data",
+[3]  = "stack",
+[4]  = "core",
+[5]  = "rss",
+[6]  = "nproc",
+[7]  = "nofile",
+[8]  = "memlock",
+[9]  = "as",
+[10]  = "locks",
+[11]  = "sigpending",
+[12]  = "msgqueue",
+[13]  = "nice",
+[14]  = "rtprio",
+[15]  = "rttime",
+};
+static const int rlim_map[] = {
+RLIMIT_CPU,
+RLIMIT_FSIZE,
+RLIMIT_DATA,
+RLIMIT_STACK,
+RLIMIT_CORE,
+RLIMIT_RSS,
+RLIMIT_NPROC,
+RLIMIT_NOFILE,
+RLIMIT_MEMLOCK,
+RLIMIT_AS,
+RLIMIT_LOCKS,
+RLIMIT_SIGPENDING,
+RLIMIT_MSGQUEUE,
+RLIMIT_NICE,
+RLIMIT_RTPRIO,
+RLIMIT_RTTIME,
+};