autotiering.patch

diff --git a/Makefile b/Makefile
index 6886f2290..2294e8a39 100644
--- a/Makefile
+++ b/Makefile
@@ -2,7 +2,7 @@
 VERSION = 5
 PATCHLEVEL = 3
 SUBLEVEL = 0
-EXTRAVERSION =
+EXTRAVERSION = -autotiering
 NAME = Bobtail Squid
 
 # *DOCUMENTATION*
diff --git a/drivers/acpi/hmat/hmat.c b/drivers/acpi/hmat/hmat.c
index 96b7d39a9..f86fe7130 100644
--- a/drivers/acpi/hmat/hmat.c
+++ b/drivers/acpi/hmat/hmat.c
@@ -14,14 +14,18 @@
 #include <linux/init.h>
 #include <linux/list.h>
 #include <linux/list_sort.h>
+#include <linux/memory.h>
+#include <linux/mutex.h>
 #include <linux/node.h>
 #include <linux/sysfs.h>
 
-static __initdata u8 hmat_revision;
+static u8 hmat_revision;
 
-static __initdata LIST_HEAD(targets);
-static __initdata LIST_HEAD(initiators);
-static __initdata LIST_HEAD(localities);
+static LIST_HEAD(targets);
+static LIST_HEAD(initiators);
+static LIST_HEAD(localities);
+
+static DEFINE_MUTEX(target_lock);
 
 /*
  * The defined enum order is used to prioritize attributes to break ties when
@@ -36,11 +40,19 @@ enum locality_types {
 
 static struct memory_locality *localities_types[4];
 
+struct target_cache {
+	struct list_head node;
+	struct node_cache_attrs cache_attrs;
+};
+
 struct memory_target {
 	struct list_head node;
 	unsigned int memory_pxm;
 	unsigned int processor_pxm;
 	struct node_hmem_attrs hmem_attrs;
+	struct list_head caches;
+	struct node_cache_attrs cache_attrs;
+	bool registered;
 };
 
 struct memory_initiator {
@@ -53,7 +65,7 @@ struct memory_locality {
 	struct acpi_hmat_locality *hmat_loc;
 };
 
-static __init struct memory_initiator *find_mem_initiator(unsigned int cpu_pxm)
+static struct memory_initiator *find_mem_initiator(unsigned int cpu_pxm)
 {
 	struct memory_initiator *initiator;
 
@@ -63,7 +75,7 @@ static __init struct memory_initiator *find_mem_initiator(unsigned int cpu_pxm)
 	return NULL;
 }
 
-static __init struct memory_target *find_mem_target(unsigned int mem_pxm)
+static struct memory_target *find_mem_target(unsigned int mem_pxm)
 {
 	struct memory_target *target;
 
@@ -110,6 +122,7 @@ static __init void alloc_memory_target(unsigned int mem_pxm)
 	target->memory_pxm = mem_pxm;
 	target->processor_pxm = PXM_INVAL;
 	list_add_tail(&target->node, &targets);
+	INIT_LIST_HEAD(&target->caches);
 }
 
 static __init const char *hmat_data_type(u8 type)
@@ -148,7 +161,7 @@ static __init const char *hmat_data_type_suffix(u8 type)
 	}
 }
 
-static __init u32 hmat_normalize(u16 entry, u64 base, u8 type)
+static u32 hmat_normalize(u16 entry, u64 base, u8 type)
 {
 	u32 value;
 
@@ -183,7 +196,7 @@ static __init u32 hmat_normalize(u16 entry, u64 base, u8 type)
 	return value;
 }
 
-static __init void hmat_update_target_access(struct memory_target *target,
+static void hmat_update_target_access(struct memory_target *target,
 					     u8 type, u32 value)
 {
 	switch (type) {
@@ -314,7 +327,8 @@ static __init int hmat_parse_cache(union acpi_subtable_headers *header,
 				   const unsigned long end)
 {
 	struct acpi_hmat_cache *cache = (void *)header;
-	struct node_cache_attrs cache_attrs;
+	struct memory_target *target;
+	struct target_cache *tcache;
 	u32 attrs;
 
 	if (cache->header.length < sizeof(*cache)) {
@@ -328,37 +342,47 @@ static __init int hmat_parse_cache(union acpi_subtable_headers *header,
 		cache->memory_PD, cache->cache_size, attrs,
 		cache->number_of_SMBIOShandles);
 
-	cache_attrs.size = cache->cache_size;
-	cache_attrs.level = (attrs & ACPI_HMAT_CACHE_LEVEL) >> 4;
-	cache_attrs.line_size = (attrs & ACPI_HMAT_CACHE_LINE_SIZE) >> 16;
+	target = find_mem_target(cache->memory_PD);
+	if (!target)
+		return 0;
+
+	tcache = kzalloc(sizeof(*tcache), GFP_KERNEL);
+	if (!tcache) {
+		pr_notice_once("Failed to allocate HMAT cache info\n");
+		return 0;
+	}
+
+	tcache->cache_attrs.size = cache->cache_size;
+	tcache->cache_attrs.level = (attrs & ACPI_HMAT_CACHE_LEVEL) >> 4;
+	tcache->cache_attrs.line_size = (attrs & ACPI_HMAT_CACHE_LINE_SIZE) >> 16;
 
 	switch ((attrs & ACPI_HMAT_CACHE_ASSOCIATIVITY) >> 8) {
 	case ACPI_HMAT_CA_DIRECT_MAPPED:
-		cache_attrs.indexing = NODE_CACHE_DIRECT_MAP;
+		tcache->cache_attrs.indexing = NODE_CACHE_DIRECT_MAP;
 		break;
 	case ACPI_HMAT_CA_COMPLEX_CACHE_INDEXING:
-		cache_attrs.indexing = NODE_CACHE_INDEXED;
+		tcache->cache_attrs.indexing = NODE_CACHE_INDEXED;
 		break;
 	case ACPI_HMAT_CA_NONE:
 	default:
-		cache_attrs.indexing = NODE_CACHE_OTHER;
+		tcache->cache_attrs.indexing = NODE_CACHE_OTHER;
 		break;
 	}
 
 	switch ((attrs & ACPI_HMAT_WRITE_POLICY) >> 12) {
 	case ACPI_HMAT_CP_WB:
-		cache_attrs.write_policy = NODE_CACHE_WRITE_BACK;
+		tcache->cache_attrs.write_policy = NODE_CACHE_WRITE_BACK;
 		break;
 	case ACPI_HMAT_CP_WT:
-		cache_attrs.write_policy = NODE_CACHE_WRITE_THROUGH;
+		tcache->cache_attrs.write_policy = NODE_CACHE_WRITE_THROUGH;
 		break;
 	case ACPI_HMAT_CP_NONE:
 	default:
-		cache_attrs.write_policy = NODE_CACHE_WRITE_OTHER;
+		tcache->cache_attrs.write_policy = NODE_CACHE_WRITE_OTHER;
 		break;
 	}
+	list_add_tail(&tcache->node, &target->caches);
 
-	node_add_cache(pxm_to_node(cache->memory_PD), &cache_attrs);
 	return 0;
 }
 
@@ -435,7 +459,7 @@ static __init int srat_parse_mem_affinity(union acpi_subtable_headers *header,
 	return 0;
 }
 
-static __init u32 hmat_initiator_perf(struct memory_target *target,
+static u32 hmat_initiator_perf(struct memory_target *target,
 			       struct memory_initiator *initiator,
 			       struct acpi_hmat_locality *hmat_loc)
 {
@@ -473,7 +497,7 @@ static __init u32 hmat_initiator_perf(struct memory_target *target,
 			      hmat_loc->data_type);
 }
 
-static __init bool hmat_update_best(u8 type, u32 value, u32 *best)
+static bool hmat_update_best(u8 type, u32 value, u32 *best)
 {
 	bool updated = false;
 
@@ -517,7 +541,7 @@ static int initiator_cmp(void *priv, struct list_head *a, struct list_head *b)
 	return ia->processor_pxm - ib->processor_pxm;
 }
 
-static __init void hmat_register_target_initiators(struct memory_target *target)
+static void hmat_register_target_initiators(struct memory_target *target)
 {
 	static DECLARE_BITMAP(p_nodes, MAX_NUMNODES);
 	struct memory_initiator *initiator;
@@ -577,29 +601,80 @@ static __init void hmat_register_target_initiators(struct memory_target *target)
 	}
 }
 
-static __init void hmat_register_target_perf(struct memory_target *target)
+static void hmat_register_target_cache(struct memory_target *target)
+{
+	unsigned mem_nid = pxm_to_node(target->memory_pxm);
+	struct target_cache *tcache;
+
+	list_for_each_entry(tcache, &target->caches, node)
+		node_add_cache(mem_nid, &tcache->cache_attrs);
+}
+
+static void hmat_register_target_perf(struct memory_target *target)
 {
 	unsigned mem_nid = pxm_to_node(target->memory_pxm);
 	node_set_perf_attrs(mem_nid, &target->hmem_attrs, 0);
 }
 
-static __init void hmat_register_targets(void)
+static void hmat_register_target(struct memory_target *target)
 {
-	struct memory_target *target;
+	if (!node_online(pxm_to_node(target->memory_pxm)))
+		return;
 
-	list_for_each_entry(target, &targets, node) {
+	mutex_lock(&target_lock);
+	if (!target->registered) {
 		hmat_register_target_initiators(target);
+		hmat_register_target_cache(target);
 		hmat_register_target_perf(target);
+		target->registered = true;
 	}
+	mutex_unlock(&target_lock);
+}
+
+static void hmat_register_targets(void)
+{
+	struct memory_target *target;
+
+	list_for_each_entry(target, &targets, node)
+		hmat_register_target(target);
+}
+
+static int hmat_callback(struct notifier_block *self,
+			 unsigned long action, void *arg)
+{
+	struct memory_target *target;
+	struct memory_notify *mnb = arg;
+	int pxm, nid = mnb->status_change_nid;
+
+	if (nid == NUMA_NO_NODE || action != MEM_ONLINE)
+		return NOTIFY_OK;
+
+	pxm = node_to_pxm(nid);
+	target = find_mem_target(pxm);
+	if (!target)
+		return NOTIFY_OK;
+
+	hmat_register_target(target);
+	return NOTIFY_OK;
 }
 
+static struct notifier_block hmat_callback_nb = {
+	.notifier_call = hmat_callback,
+	.priority = 2,
+};
+
 static __init void hmat_free_structures(void)
 {
 	struct memory_target *target, *tnext;
 	struct memory_locality *loc, *lnext;
 	struct memory_initiator *initiator, *inext;
+	struct target_cache *tcache, *cnext;
 
 	list_for_each_entry_safe(target, tnext, &targets, node) {
+		list_for_each_entry_safe(tcache, cnext, &target->caches, node) {
+			list_del(&tcache->node);
+			kfree(tcache);
+		}
 		list_del(&target->node);
 		kfree(target);
 	}
@@ -658,6 +733,10 @@ static __init int hmat_init(void)
 		}
 	}
 	hmat_register_targets();
+
+	/* Keep the table and structures if the notifier may use them */
+	if (!register_hotmemory_notifier(&hmat_callback_nb))
+		return 0;
 out_put:
 	hmat_free_structures();
 	acpi_put_table(tbl);
diff --git a/drivers/base/node.c b/drivers/base/node.c
index 75b7e6f65..9d5f0abf1 100644
--- a/drivers/base/node.c
+++ b/drivers/base/node.c
@@ -17,6 +17,7 @@
 #include <linux/nodemask.h>
 #include <linux/cpu.h>
 #include <linux/device.h>
+#include <linux/list_sort.h>
 #include <linux/pm_runtime.h>
 #include <linux/swap.h>
 #include <linux/slab.h>
@@ -74,6 +75,9 @@ struct node_access_nodes {
 	unsigned		access;
 #ifdef CONFIG_HMEM_REPORTING
 	struct node_hmem_attrs	hmem_attrs;
+	struct list_head	target_list;
+	struct list_head	target_siblings;
+	int			initiator_node;
 #endif
 };
 #define to_access_nodes(dev) container_of(dev, struct node_access_nodes, dev)
@@ -102,6 +106,13 @@ static const struct attribute_group *node_access_node_groups[] = {
 	NULL,
 };
 
+#define TERMINAL_NODE -1
+static int node_migration[MAX_NUMNODES] = {[0 ...  MAX_NUMNODES - 1] = TERMINAL_NODE};
+static int node_demotion[MAX_NUMNODES] = {[0 ...  MAX_NUMNODES - 1] = TERMINAL_NODE};
+static int node_promotion[MAX_NUMNODES] = {[0 ...  MAX_NUMNODES - 1] = TERMINAL_NODE};
+static DEFINE_SPINLOCK(node_migration_lock);
+static DEFINE_SPINLOCK(node_kswapd_failure_lock);
+
 static void node_remove_accesses(struct node *node)
 {
 	struct node_access_nodes *c, *cnext;
@@ -117,6 +128,17 @@ static void node_access_release(struct device *dev)
 	kfree(to_access_nodes(dev));
 }
 
+static struct node_access_nodes *node_find_access_node(struct node *node,
+						       unsigned access)
+{
+	struct node_access_nodes *access_node;
+
+	list_for_each_entry(access_node, &node->access_list, list_node)
+		if (access_node->access == access)
+			return access_node;
+	return NULL;
+}
+
 static struct node_access_nodes *node_init_node_access(struct node *node,
 						       unsigned access)
 {
@@ -139,6 +161,12 @@ static struct node_access_nodes *node_init_node_access(struct node *node,
 	if (dev_set_name(dev, "access%u", access))
 		goto free;
 
+#ifdef CONFIG_HMEM_REPORTING
+	INIT_LIST_HEAD(&access_node->target_list);
+	INIT_LIST_HEAD(&access_node->target_siblings);
+	access_node->initiator_node = -1;
+#endif
+
 	if (device_register(dev))
 		goto free_name;
 
@@ -175,6 +203,58 @@ static struct attribute *access_attrs[] = {
 	NULL,
 };
 
+/*
+ * Sort priority: read latency, write latency, read bandwidth, write bandwidth,
+ * and finally capacity. We'll prefer to migrate to nodes with more plentiful
+ * memory.
+ */
+static int access_cmp(void *priv, struct list_head *a, struct list_head *b)
+{
+	struct node_access_nodes *access_a = container_of(a,
+						struct node_access_nodes,
+						target_siblings);
+	struct node_access_nodes *access_b = container_of(b,
+						struct node_access_nodes,
+						target_siblings);
+	struct node *node_a, *node_b;
+	struct sysinfo ia, ib;
+
+	if (access_a->hmem_attrs.read_latency != 0 &&
+	    access_b->hmem_attrs.read_latency != 0 &&
+	    access_a->hmem_attrs.read_latency !=
+			access_b->hmem_attrs.read_latency)
+		return access_a->hmem_attrs.read_latency -
+				access_b->hmem_attrs.read_latency;
+
+	if (access_a->hmem_attrs.write_latency != 0 &&
+	    access_b->hmem_attrs.write_latency != 0 &&
+	    access_a->hmem_attrs.write_latency !=
+			access_b->hmem_attrs.write_latency)
+		return access_a->hmem_attrs.write_latency -
+				access_b->hmem_attrs.write_latency;
+
+	if (access_a->hmem_attrs.read_bandwidth != 0 &&
+	    access_b->hmem_attrs.read_bandwidth != 0 &&
+	    access_a->hmem_attrs.read_bandwidth !=
+			access_b->hmem_attrs.read_bandwidth)
+		return access_b->hmem_attrs.read_bandwidth -
+				access_a->hmem_attrs.read_bandwidth;
+
+	if (access_a->hmem_attrs.write_bandwidth != 0 &&
+	    access_b->hmem_attrs.write_bandwidth != 0 &&
+	    access_a->hmem_attrs.write_bandwidth !=
+			access_b->hmem_attrs.write_bandwidth)
+		return access_b->hmem_attrs.write_bandwidth -
+				access_a->hmem_attrs.write_bandwidth;
+
+	node_a = to_node(access_a->dev.parent);
+	node_b = to_node(access_b->dev.parent);
+	si_meminfo_node(&ia, node_a->dev.id);
+	si_meminfo_node(&ib, node_b->dev.id);
+
+	return ib.totalram - ia.totalram;
+}
+
 /**
  * node_set_perf_attrs - Set the performance values for given access class
  * @nid: Node identifier to be set
@@ -184,27 +264,57 @@ static struct attribute *access_attrs[] = {
 void node_set_perf_attrs(unsigned int nid, struct node_hmem_attrs *hmem_attrs,
 			 unsigned access)
 {
-	struct node_access_nodes *c;
-	struct node *node;
-	int i;
+	struct node_access_nodes *m, *c;
+	struct node *node, *initiator;
+	int i, cpu_nid;
 
 	if (WARN_ON_ONCE(!node_online(nid)))
 		return;
 
 	node = node_devices[nid];
-	c = node_init_node_access(node, access);
-	if (!c)
+	m = node_init_node_access(node, access);
+	if (!m)
 		return;
 
-	c->hmem_attrs = *hmem_attrs;
+	m->hmem_attrs = *hmem_attrs;
 	for (i = 0; access_attrs[i] != NULL; i++) {
-		if (sysfs_add_file_to_group(&c->dev.kobj, access_attrs[i],
+		if (sysfs_add_file_to_group(&m->dev.kobj, access_attrs[i],
 					    "initiators")) {
 			pr_info("failed to add performance attribute to node %d\n",
 				nid);
 			break;
 		}
 	}
+
+	if (access != 0)
+		return;
+
+	cpu_nid = m->initiator_node;
+	if (cpu_nid < 0)
+		return;
+
+	initiator = node_devices[cpu_nid];
+	if (!initiator)
+		return;
+
+	c = node_find_access_node(initiator, access);
+	if (!c)
+		return;
+
+	list_add_tail(&m->target_siblings, &c->target_list);
+	list_sort(NULL, &c->target_list, access_cmp);
+
+	i = -1;
+	list_for_each_entry(m, &c->target_list, target_siblings) {
+		node = to_node(m->dev.parent);
+		if (i >= 0) {
+			node_demotion[i] = node->dev.id;
+			node_promotion[node->dev.id] = i;
+                        node_migration[i] = node->dev.id;
+                        node_migration[node->dev.id] = i;
+		}
+		i = node->dev.id;
+	}
 }
 
 /**
@@ -531,6 +641,249 @@ static ssize_t node_read_distance(struct device *dev,
 }
 static DEVICE_ATTR(distance, S_IRUGO, node_read_distance, NULL);
 
+static ssize_t kswapd_failures_show(struct device *dev,
+				   struct device_attribute *attr,
+				   char *buf)
+{
+	struct pglist_data *pgdat = NODE_DATA(dev->id);
+	return sprintf(buf, "%d\n", pgdat->kswapd_failures);
+}
+
+static ssize_t migration_path_show(struct device *dev,
+				   struct device_attribute *attr,
+				   char *buf)
+{
+	return sprintf(buf, "%d\n", node_migration[dev->id]);
+}
+
+static ssize_t promotion_path_show(struct device *dev,
+				   struct device_attribute *attr,
+				   char *buf)
+{
+	return sprintf(buf, "%d\n", node_promotion[dev->id]);
+}
+
+static ssize_t demotion_path_show(struct device *dev,
+				   struct device_attribute *attr,
+				   char *buf)
+{
+	return sprintf(buf, "%d\n", node_demotion[dev->id]);
+}
+
+static ssize_t kswapd_failures_store(struct device *dev,
+				    struct device_attribute *attr,
+				    const char *buf, size_t count)
+{
+	int err, nid = dev->id;
+	struct pglist_data *pgdat = NODE_DATA(nid);
+	long nr_failures;
+
+	err = kstrtol(buf, 0, &nr_failures);
+	if (err)
+		return -EINVAL;
+
+	if (nr_failures < 0)
+		return -EINVAL;
+
+	spin_lock(&node_kswapd_failure_lock);
+	WRITE_ONCE(pgdat->kswapd_failures, nr_failures);
+	spin_unlock(&node_kswapd_failure_lock);
+
+	return count;
+}
+
+static ssize_t migration_path_store(struct device *dev,
+				    struct device_attribute *attr,
+				    const char *buf, size_t count)
+{
+	int i, err, nid = dev->id;
+	nodemask_t visited = NODE_MASK_NONE;
+	long next;
+
+	err = kstrtol(buf, 0, &next);
+	if (err)
+		return -EINVAL;
+
+	if (next < 0) {
+		spin_lock(&node_migration_lock);
+		WRITE_ONCE(node_migration[nid], TERMINAL_NODE);
+		spin_unlock(&node_migration_lock);
+		return count;
+	}
+	if (next >= MAX_NUMNODES || !node_online(next))
+		return -EINVAL;
+
+	/*
+	 * Follow the entire migration path from 'nid' through the point where
+	 * we hit a TERMINAL_NODE.
+	 *
+	 * Don't allow loops migration cycles in the path.
+	 */
+	node_set(nid, visited);
+	spin_lock(&node_migration_lock);
+	for (i = next; node_migration[i] != TERMINAL_NODE;
+	     i = node_migration[i]) {
+		/* Fail if we have visited this node already */
+		if (node_test_and_set(i, visited)) {
+			spin_unlock(&node_migration_lock);
+			return -EINVAL;
+		}
+	}
+        WRITE_ONCE(node_migration[nid], next);
+
+        spin_unlock(&node_migration_lock);
+
+	return count;
+}
+
+static ssize_t demotion_path_store(struct device *dev,
+				    struct device_attribute *attr,
+				    const char *buf, size_t count)
+{
+	int i, err, nid = dev->id;
+	nodemask_t visited = NODE_MASK_NONE;
+	long next;
+
+	err = kstrtol(buf, 0, &next);
+	if (err)
+		return -EINVAL;
+
+	if (next < 0) {
+		spin_lock(&node_migration_lock);
+		WRITE_ONCE(node_demotion[nid], TERMINAL_NODE);
+		spin_unlock(&node_migration_lock);
+		return count;
+	}
+	if (next >= MAX_NUMNODES || !node_online(next))
+		return -EINVAL;
+
+	/*
+	 * Follow the entire migration path from 'nid' through the point where
+	 * we hit a TERMINAL_NODE.
+	 *
+	 * Don't allow loops migration cycles in the path.
+	 */
+	node_set(nid, visited);
+	spin_lock(&node_migration_lock);
+	for (i = next; node_demotion[i] != TERMINAL_NODE;
+	     i = node_demotion[i]) {
+		/* Fail if we have visited this node already */
+		if (node_test_and_set(i, visited)) {
+			spin_unlock(&node_migration_lock);
+			return -EINVAL;
+		}
+	}
+
+        WRITE_ONCE(node_demotion[nid], next);
+
+	spin_unlock(&node_migration_lock);
+
+	return count;
+}
+
+static ssize_t promotion_path_store(struct device *dev,
+				    struct device_attribute *attr,
+				    const char *buf, size_t count)
+{
+	int i, err, nid = dev->id;
+	nodemask_t visited = NODE_MASK_NONE;
+	long next;
+
+	err = kstrtol(buf, 0, &next);
+	if (err)
+		return -EINVAL;
+
+	if (next < 0) {
+		spin_lock(&node_migration_lock);
+		WRITE_ONCE(node_promotion[nid], TERMINAL_NODE);
+		spin_unlock(&node_migration_lock);
+		return count;
+	}
+	if (next >= MAX_NUMNODES || !node_online(next))
+		return -EINVAL;
+
+	/*
+	 * Follow the entire migration path from 'nid' through the point where
+	 * we hit a TERMINAL_NODE.
+	 *
+	 * Don't allow loops migration cycles in the path.
+	 */
+	node_set(nid, visited);
+	spin_lock(&node_migration_lock);
+	for (i = next; node_promotion[i] != TERMINAL_NODE;
+	     i = node_promotion[i]) {
+		/* Fail if we have visited this node already */
+		if (node_test_and_set(i, visited)) {
+			spin_unlock(&node_migration_lock);
+			return -EINVAL;
+		}
+	}
+        WRITE_ONCE(node_promotion[nid], next);
+
+        spin_unlock(&node_migration_lock);
+
+	return count;
+}
+static DEVICE_ATTR_RW(migration_path);
+static DEVICE_ATTR_RW(promotion_path);
+static DEVICE_ATTR_RW(demotion_path);
+static DEVICE_ATTR_RW(kswapd_failures);
+
+/**
+ * next_migration_node() - Get the next node in the migration path
+ * @current_node: The starting node to lookup the next node
+ *
+ * @returns: node id for next memory node in the migration path hierarchy from
+ * 	     @current_node; -1 if @current_node is terminal or its migration
+ * 	     node is not online.
+ */
+int next_migration_node(int current_node)
+{
+	int nid = READ_ONCE(node_migration[current_node]);
+
+	if (nid >= 0 && node_online(nid))
+		return nid;
+	return TERMINAL_NODE;
+}
+
+int next_promotion_node(int current_node)
+{
+	int nid = READ_ONCE(node_promotion[current_node]);
+
+	if (nid >= 0 && node_online(nid))
+		return nid;
+	return TERMINAL_NODE;
+}
+
+int next_demotion_node(int current_node)
+{
+	int nid = READ_ONCE(node_demotion[current_node]);
+
+	if (nid >= 0 && node_online(nid))
+		return nid;
+	return TERMINAL_NODE;
+}
+
+int is_top_node(int current_node) {
+	if (current_node == TERMINAL_NODE)
+		return false;
+
+	if (next_promotion_node(current_node) == TERMINAL_NODE)
+		return true;
+	else
+		return false;
+}
+
+int is_bottom_node(int current_node) {
+	if (current_node == TERMINAL_NODE)
+		return false;
+
+	if (next_demotion_node(current_node) == TERMINAL_NODE)
+		return true;
+	else
+		return false;
+}
+
 static struct attribute *node_dev_attrs[] = {
 	&dev_attr_cpumap.attr,
 	&dev_attr_cpulist.attr,
@@ -538,6 +891,10 @@ static struct attribute *node_dev_attrs[] = {
 	&dev_attr_numastat.attr,
 	&dev_attr_distance.attr,
 	&dev_attr_vmstat.attr,
+	&dev_attr_migration_path.attr,
+	&dev_attr_promotion_path.attr,
+	&dev_attr_demotion_path.attr,
+	&dev_attr_kswapd_failures.attr,
 	NULL
 };
 ATTRIBUTE_GROUPS(node_dev);
@@ -714,6 +1071,9 @@ int register_memory_node_under_compute_node(unsigned int mem_nid,
 	if (ret)
 		goto err;
 
+#ifdef CONFIG_HMEM_REPORTING
+	target->initiator_node = cpu_nid;
+#endif
 	return 0;
  err:
 	sysfs_remove_link_from_group(&initiator->dev.kobj, "targets",
diff --git a/include/linux/exchange.h b/include/linux/exchange.h
new file mode 100644
index 000000000..7a70ac29f
--- /dev/null
+++ b/include/linux/exchange.h
@@ -0,0 +1,33 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+#ifndef _LINUX_EXCHANGE_H
+#define _LINUX_EXCHANGE_H
+
+#include <linux/migrate.h>
+#include <linux/list.h>
+
+struct exchange_page_info {
+	struct page *from_page;
+	struct page *to_page;
+
+	struct anon_vma *from_anon_vma;
+	struct anon_vma *to_anon_vma;
+
+	int from_page_was_mapped;
+	int to_page_was_mapped;
+
+	pgoff_t from_index, to_index;
+
+	struct list_head list;
+};
+
+int exchange_two_pages(struct page *, struct page *, enum migrate_mode mode);
+int try_exchange_page(struct page *page, int dst_nid);
+int exchange_pages(struct list_head *, enum migrate_mode mode);
+int exchange_pages_concur(struct list_head *exchange_list,
+		enum migrate_mode mode);
+int exchange_pages_between_nodes_batch(const int from_nid, const int to_nid);
+int exchange_page_lists_mthread(struct page **to, struct page **from, int nr_pages);
+int exchange_page_mthread(struct page *to, struct page *from, int nr_pages);
+void wakeup_kexchanged(int node);
+
+#endif /* _LINUX_EXCHANGE_H */
diff --git a/include/linux/hugetlb.h b/include/linux/hugetlb.h
index edfca4278..50126c47c 100644
--- a/include/linux/hugetlb.h
+++ b/include/linux/hugetlb.h
@@ -101,7 +101,9 @@ long hugetlb_unreserve_pages(struct inode *inode, long start, long end,
 						long freed);
 bool isolate_huge_page(struct page *page, struct list_head *list);
 void putback_active_hugepage(struct page *page);
-void move_hugetlb_state(struct page *oldpage, struct page *newpage, int reason);
+struct migrate_detail; /* hack for crazy include ordering */
+void move_hugetlb_state(struct page *oldpage, struct page *newpage,
+			struct migrate_detail *m_detail);
 void free_huge_page(struct page *page);
 void hugetlb_fix_reserve_counts(struct inode *inode);
 extern struct mutex *hugetlb_fault_mutex_table;
@@ -192,7 +194,7 @@ static inline bool isolate_huge_page(struct page *page, struct list_head *list)
 	return false;
 }
 #define putback_active_hugepage(p)	do {} while (0)
-#define move_hugetlb_state(old, new, reason)	do {} while (0)
+#define move_hugetlb_state(old, new, m_detail)	do {} while (0)
 
 static inline unsigned long hugetlb_change_protection(struct vm_area_struct *vma,
 		unsigned long address, unsigned long end, pgprot_t newprot)
diff --git a/include/linux/ksm.h b/include/linux/ksm.h
index e48b1e453..461793e87 100644
--- a/include/linux/ksm.h
+++ b/include/linux/ksm.h
@@ -53,6 +53,7 @@ struct page *ksm_might_need_to_copy(struct page *page,
 
 void rmap_walk_ksm(struct page *page, struct rmap_walk_control *rwc);
 void ksm_migrate_page(struct page *newpage, struct page *oldpage);
+void ksm_exchange_page(struct page *to_page, struct page *from_page);
 bool reuse_ksm_page(struct page *page,
 			struct vm_area_struct *vma, unsigned long address);
 
@@ -88,6 +89,10 @@ static inline void rmap_walk_ksm(struct page *page,
 static inline void ksm_migrate_page(struct page *newpage, struct page *oldpage)
 {
 }
+static inline void ksm_exchange_page(struct page *to_page,
+		struct page *from_page)
+{
+}
 static inline bool reuse_ksm_page(struct page *page,
 			struct vm_area_struct *vma, unsigned long address)
 {
diff --git a/include/linux/migrate.h b/include/linux/migrate.h
index 7f04754c7..9ad306396 100644
--- a/include/linux/migrate.h
+++ b/include/linux/migrate.h
@@ -25,9 +25,45 @@ enum migrate_reason {
 	MR_MEMPOLICY_MBIND,
 	MR_NUMA_MISPLACED,
 	MR_CONTIG_RANGE,
+	MR_DEMOTION,
+	MR_PROMOTION,
 	MR_TYPES
 };
 
+enum migrate_fail_reason {
+	MFR_UNKNOWN,
+	MFR_DST_NODE_FULL,
+	MFR_NUMA_ISOLATE,
+	MFR_NOMEM_FAIL,
+	MFR_REFCOUNT_FAIL,
+	MFR_NR_REASONS
+};
+
+enum migrate_hmem_reason {
+	MR_HMEM_UNKNOWN,
+	MR_HMEM_DEMOTE,
+	MR_HMEM_LOCAL_PROMOTE,
+	MR_HMEM_REMOTE_PROMOTE,
+	MR_HMEM_MIGRATE,
+	MR_HMEM_NR_REASONS
+};
+
+enum migrate_hmem_fail_reason {
+	MR_HMEM_UNKNOWN_FAIL,
+	MR_HMEM_LOCAL_PROMOTE_FAIL,
+	MR_HMEM_REMOTE_PROMOTE_FAIL,
+	MR_HMEM_MIGRATE_FAIL,
+	MR_HMEM_NR_FAIL_REASONS
+};
+
+struct migrate_detail {
+	enum migrate_reason reason;
+	enum migrate_fail_reason fail_reason;
+	enum migrate_hmem_reason h_reason;
+	enum migrate_hmem_reason h_reason_orig;
+	enum migrate_hmem_fail_reason h_fail_reason;
+};
+
 /* In mm/debug.c; also keep sync with include/trace/events/migrate.h */
 extern const char *migrate_reason_names[MR_TYPES];
 
@@ -66,7 +102,8 @@ extern int migrate_page(struct address_space *mapping,
 			struct page *newpage, struct page *page,
 			enum migrate_mode mode);
 extern int migrate_pages(struct list_head *l, new_page_t new, free_page_t free,
-		unsigned long private, enum migrate_mode mode, int reason);
+		unsigned long private, enum migrate_mode mode,
+		struct migrate_detail *m_detail);
 extern int isolate_movable_page(struct page *page, isolate_mode_t mode);
 extern void putback_movable_page(struct page *page);
 
@@ -78,6 +115,8 @@ extern int migrate_huge_page_move_mapping(struct address_space *mapping,
 				  struct page *newpage, struct page *page);
 extern int migrate_page_move_mapping(struct address_space *mapping,
 		struct page *newpage, struct page *page, int extra_count);
+
+extern int copy_page_multithread(struct page *to, struct page *from, int nr_pages);
 #else
 
 static inline void putback_movable_pages(struct list_head *l) {}
@@ -104,6 +143,7 @@ static inline int migrate_huge_page_move_mapping(struct address_space *mapping,
 	return -ENOSYS;
 }
 
+
 #endif /* CONFIG_MIGRATION */
 
 #ifdef CONFIG_COMPACTION
@@ -125,6 +165,17 @@ static inline void __ClearPageMovable(struct page *page)
 extern bool pmd_trans_migrating(pmd_t pmd);
 extern int migrate_misplaced_page(struct page *page,
 				  struct vm_area_struct *vma, int node);
+extern int migrate_demote_mapping(struct page *page);
+extern int migrate_promote_mapping(struct page *page);
+
+extern int wakeup_kdemoted(int dst_cpu, struct page *fault_page);
+extern int try_demote_from_busy_node(struct page *fault_page, int busy_nid,
+		unsigned int mode);
+extern int try_demote_page_cache(struct pglist_data *pgdat, struct lruvec *lruvec);
+extern bool migrate_balanced_pgdat(struct pglist_data *pgdat, unsigned int order);
+extern void numamigrate_fail_reason(struct migrate_detail *m_detail,
+				enum migrate_hmem_reason h_reason);
+extern void numamigrate_reason(struct migrate_detail *m_detail, int src_nid, int dst_nid);
 #else
 static inline bool pmd_trans_migrating(pmd_t pmd)
 {
@@ -135,6 +186,41 @@ static inline int migrate_misplaced_page(struct page *page,
 {
 	return -EAGAIN; /* can't migrate now */
 }
+static inline int migrate_demote_mapping(struct page *page)
+{
+	return -ENOSYS;
+}
+static inline int migrate_promote_mapping(struct page *page)
+{
+	return -ENOSYS;
+}
+static inline int wakeup_kdemoted(int dst_cpu, struct page *fault_page)
+{
+	return false;
+}
+static inline int try_demote_from_busy_node(struct page *fault_page, int busy_nid,
+		unsigned int mode)
+{
+	return false;
+}
+static inline int try_demote_page_cache(struct pglist_data *pgdat,
+				struct lruvec *lruvec)
+{
+	return false;
+}
+static inline bool migrate_balanced_pgdat(struct pglist_data *pgdat,
+				unsigned int order)
+{
+	return false;
+}
+static inline void numamigrate_fail_reason(struct migrate_detail *m_detail,
+				enum migrate_hmem_reason h_reason)
+{
+}
+static inline void numamigrate_reason(struct migrate_detail *m_detail,
+				int src_nid, int dst_nid)
+{
+}
 #endif /* CONFIG_NUMA_BALANCING */
 
 #if defined(CONFIG_NUMA_BALANCING) && defined(CONFIG_TRANSPARENT_HUGEPAGE)
@@ -155,6 +241,13 @@ static inline int migrate_misplaced_transhuge_page(struct mm_struct *mm,
 #endif /* CONFIG_NUMA_BALANCING && CONFIG_TRANSPARENT_HUGEPAGE*/
 
 
+#ifdef CONFIG_BLOCK
+bool buffer_migrate_lock_buffers(struct buffer_head *head,
+			enum migrate_mode mode);
+#endif
+
+int writeout(struct address_space *mapping, struct page *page);
+
 #ifdef CONFIG_MIGRATION
 
 /*
diff --git a/include/linux/migrate_mode.h b/include/linux/migrate_mode.h
index 883c99249..0b78c3461 100644
--- a/include/linux/migrate_mode.h
+++ b/include/linux/migrate_mode.h
@@ -16,7 +16,12 @@ enum migrate_mode {
 	MIGRATE_ASYNC,
 	MIGRATE_SYNC_LIGHT,
 	MIGRATE_SYNC,
-	MIGRATE_SYNC_NO_COPY,
+
+	MIGRATE_MODE_MASK    = 7,
+	MIGRATE_SYNC_NO_COPY = 1<<3,
+	MIGRATE_SINGLETHREAD = 0,
+	MIGRATE_MT           = 1<<4,
+	MIGRATE_CONCUR       = 1<<5,
 };
 
 #endif		/* MIGRATE_MODE_H_INCLUDED */
diff --git a/include/linux/mm.h b/include/linux/mm.h
index 0334ca97c..b56ae309b 100644
--- a/include/linux/mm.h
+++ b/include/linux/mm.h
@@ -219,6 +219,7 @@ extern int overcommit_kbytes_handler(struct ctl_table *, int, void __user *,
 #define PAGE_ALIGNED(addr)	IS_ALIGNED((unsigned long)(addr), PAGE_SIZE)
 
 #define lru_to_page(head) (list_entry((head)->prev, struct page, lru))
+#define lru_to_page_head(head) (list_entry((head)->next, struct page, lru))
 
 /*
  * Linux kernel virtual memory manager primitives.
@@ -2249,6 +2250,7 @@ extern void zone_pcp_reset(struct zone *zone);
 extern int min_free_kbytes;
 extern int watermark_boost_factor;
 extern int watermark_scale_factor;
+extern void promote_init(void);
 
 /* nommu.c */
 extern atomic_long_t mmap_pages_allocated;
diff --git a/include/linux/mm_inline.h b/include/linux/mm_inline.h
index 6f2fef7b0..3963d2a9c 100644
--- a/include/linux/mm_inline.h
+++ b/include/linux/mm_inline.h
@@ -4,6 +4,9 @@
 
 #include <linux/huge_mm.h>
 #include <linux/swap.h>
+#ifdef CONFIG_PAGE_BALANCING
+#include <linux/page_balancing.h>
+#endif
 
 /**
  * page_is_file_cache - should the page be on a file LRU or anon LRU?
@@ -44,6 +47,27 @@ static __always_inline void update_lru_size(struct lruvec *lruvec,
 #endif
 }
 
+/*
+ * Update LRU sizes after isolating pages. The LRU size updates must
+ * be complete before mem_cgroup_update_lru_size due to a santity check.
+ */
+static __always_inline void update_lru_sizes(struct lruvec *lruvec,
+			enum lru_list lru, unsigned long *nr_zone_taken)
+{
+	int zid;
+
+	for (zid = 0; zid < MAX_NR_ZONES; zid++) {
+		if (!nr_zone_taken[zid])
+			continue;
+
+		__update_lru_size(lruvec, lru, zid, -nr_zone_taken[zid]);
+#ifdef CONFIG_MEMCG
+		mem_cgroup_update_lru_size(lruvec, lru, zid, -nr_zone_taken[zid]);
+#endif
+	}
+
+}
+
 static __always_inline void add_page_to_lru_list(struct page *page,
 				struct lruvec *lruvec, enum lru_list lru)
 {
@@ -63,6 +87,7 @@ static __always_inline void del_page_from_lru_list(struct page *page,
 {
 	list_del(&page->lru);
 	update_lru_size(lruvec, lru, page_zonenum(page), -hpage_nr_pages(page));
+	del_page_from_deferred_list(page);
 }
 
 /**
diff --git a/include/linux/mmzone.h b/include/linux/mmzone.h
index 3f38c30d2..3361d51be 100644
--- a/include/linux/mmzone.h
+++ b/include/linux/mmzone.h
@@ -100,6 +100,12 @@ struct free_area {
 	unsigned long		nr_free;
 };
 
+struct lap_area {
+	struct list_head	lap_list; // Least-Accessed Page list
+	long long		nr_free;
+	unsigned long		demotion_count; // demotion count per LAP level
+};
+
 /* Used for pages not on another list */
 static inline void add_to_free_area(struct page *page, struct free_area *area,
 			     int migratetype)
@@ -206,7 +212,21 @@ enum zone_stat_item {
 	NR_ZSPAGES,		/* allocated in zsmalloc */
 #endif
 	NR_FREE_CMA_PAGES,
-	NR_VM_ZONE_STAT_ITEMS };
+#define HMEM_MIGRATE(__hmem_name)      __hmem_name ## _SRC, __hmem_name ## _DEST
+	HMEM_MIGRATE_UNKNOWN,
+	HMEM_MIGRATE_FIRST_ENTRY = HMEM_MIGRATE_UNKNOWN,
+	HMEM_MIGRATE(MR_HMEM_DEMOTE),
+	HMEM_MIGRATE(MR_HMEM_LOCAL_PROMOTE),
+	HMEM_MIGRATE(MR_HMEM_REMOTE_PROMOTE),
+	HMEM_MIGRATE(MR_HMEM_MIGRATE),
+#define HMEM_MIGRATE_FAIL(__hmem_name)      __hmem_name ## FAIL_SRC, __hmem_name ## FAIL_DEST
+	HMEM_MIGRATE_UNKNOWN_FAIL,
+	HMEM_MIGRATE_FAIL_FIRST_ENTRY = HMEM_MIGRATE_UNKNOWN_FAIL,
+	HMEM_MIGRATE_FAIL(MR_HMEM_LOCAL_PROMOTE),
+	HMEM_MIGRATE_FAIL(MR_HMEM_REMOTE_PROMOTE),
+	HMEM_MIGRATE_FAIL(MR_HMEM_MIGRATE),
+	NR_VM_ZONE_STAT_ITEMS
+};
 
 enum node_stat_item {
 	NR_LRU_BASE,
@@ -242,6 +262,9 @@ enum node_stat_item {
 	NR_DIRTIED,		/* page dirtyings since bootup */
 	NR_WRITTEN,		/* page writings since bootup */
 	NR_KERNEL_MISC_RECLAIMABLE,	/* reclaimable non-slab kernel pages */
+	NR_DEFERRED,		/* migration fail pages */
+	NR_TRACKED,		/* tracked pages by APM */
+	NR_FREE_PROMOTE,	/* page at free promote area */
 	NR_VM_NODE_STAT_ITEMS
 };
 
@@ -761,6 +784,11 @@ typedef struct pglist_data {
 	unsigned long split_queue_len;
 #endif
 
+#ifdef CONFIG_PAGE_BALANCING
+	struct list_head deferred_list;
+	// FIXME: Need to use macro
+	struct lap_area lap_area[9]; // MAX_ACCESS_LEVEL + 1
+#endif
 	/* Fields commonly accessed by the page reclaim scanner */
 	struct lruvec		lruvec;
 
@@ -821,6 +849,7 @@ extern void init_currently_empty_zone(struct zone *zone, unsigned long start_pfn
 				     unsigned long size);
 
 extern void lruvec_init(struct lruvec *lruvec);
+extern void deferred_list_init(struct pglist_data *pgdat);
 
 static inline struct pglist_data *lruvec_pgdat(struct lruvec *lruvec)
 {
diff --git a/include/linux/node.h b/include/linux/node.h
index 4866f32a0..b45d082b2 100644
--- a/include/linux/node.h
+++ b/include/linux/node.h
@@ -134,6 +134,11 @@ static inline int register_one_node(int nid)
 	return error;
 }
 
+extern int next_migration_node(int current_node);
+extern int next_promotion_node(int current_node);
+extern int next_demotion_node(int current_node);
+extern int is_top_node(int current_node);
+extern int is_bottom_node(int current_node);
 extern void unregister_one_node(int nid);
 extern int register_cpu_under_node(unsigned int cpu, unsigned int nid);
 extern int unregister_cpu_under_node(unsigned int cpu, unsigned int nid);
@@ -176,6 +181,29 @@ static inline void register_hugetlbfs_with_node(node_registration_func_t reg,
 						node_registration_func_t unreg)
 {
 }
+
+static inline int next_migration_node(int current_node)
+{
+	return -1;
+}
+
+static inline int next_promotion_node(int current_node)
+{
+	return -1;
+}
+
+static inline int next_demotion_node(int current_node)
+{
+	return -1;
+}
+static inline int is_top_node(int current_node)
+{
+	return -1;
+}
+static inline int is_bottom_node(int current_node)
+{
+	return -1;
+}
 #endif
 
 #define to_node(device) container_of(device, struct node, dev)
diff --git a/include/linux/page_balancing.h b/include/linux/page_balancing.h
new file mode 100644
index 000000000..16e799a29
--- /dev/null
+++ b/include/linux/page_balancing.h
@@ -0,0 +1,132 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+#ifndef __LINUX_PAGE_BALANCING_H
+#define __LINUX_PAGE_BALANCING_H
+#include <linux/sched/sysctl.h>
+
+#define ACCESS_HISTORY_SIZE 8
+#define MAX_ACCESS_LEVEL ACCESS_HISTORY_SIZE
+#define MEDIAN_ACCESS_LEVEL (ACCESS_HISTORY_SIZE >> 1)
+
+#ifdef CONFIG_PAGE_BALANCING
+struct page_info {
+	struct list_head list;
+	unsigned long pfn;
+	int8_t last_cpu; // for free_promote area
+	u8 access_bitmap;
+};
+#endif
+
+extern struct page_ext_operations page_info_ops;
+
+#ifdef CONFIG_PAGE_BALANCING
+extern struct page *get_page_from_page_info(struct page_info *page_info);
+extern struct page_info *get_page_info_from_page(struct page *page);
+extern struct page_ext *get_page_ext(struct page_info *page_info);
+
+extern void set_page_to_page_info(struct page *page, struct page_info *page_info);
+extern void clear_page_info(struct page *page);
+extern void del_page_from_deferred_list(struct page *page);
+extern void del_page_from_lap_list(struct page *page);
+extern void copy_page_info(struct page *oldpage, struct page *newpage);
+extern void exchange_page_info(struct page *from_page, struct page *to_page);
+extern void SetPageDeferred(struct page *page);
+extern void ClearPageDeferred(struct page *page);
+extern unsigned int PageDeferred(struct page *page);
+extern void SetPageDemoted(struct page *page);
+extern void ClearPageDemoted(struct page *page);
+extern unsigned int PageDemoted(struct page *page);
+extern void lock_busy(struct page *page);
+extern unsigned int trylock_busy(struct page *page);
+extern void unlock_busy(struct page *page);
+extern void add_page_for_exchange(struct page *page, int node);
+extern void add_page_for_tracking(struct page *page, unsigned int prev_lv);
+extern unsigned int mod_page_access_lv(struct page *page, unsigned int accessed);
+extern unsigned int get_page_access_lv(struct page *page);
+extern void reset_page_access_lv(struct page *page);
+
+extern int get_page_last_cpu(struct page *page);
+extern void set_page_last_cpu(struct page *page, int cpu);
+
+/* Finding best nodes */
+extern int find_best_demotion_node(struct page *page);
+extern int find_best_migration_node(struct page *page, int target_nid);
+
+/* User-space parameters */
+extern unsigned int background_demotion;
+extern unsigned int batch_demotion;
+extern unsigned int thp_mt_copy;
+
+#ifdef CONFIG_PAGE_BALANCING_DEBUG
+extern void trace_dump_page(struct page *page, const char *msg);
+#else /* CONFIG_PAGE_BALANCING_DEBUG */
+static inline void trace_dump_page(struct page *page, const char *msg) {
+}
+#endif /* CONFIG_PAGE_BALANCING_DEBUG */
+
+#else /* CONFIG_PAGE_BALANCING */
+static inline void *get_page_from_page_info(struct page_info *page_info)
+{
+	return NULL;
+}
+static inline void *get_page_ext(struct page_info *page_info)
+{
+	return NULL;
+}
+static inline void set_page_to_page_info(struct page *page,
+				struct page_info *page_info) {
+}
+static inline void clear_page_info(struct page *page) {
+}
+static inline void del_page_from_deferred_list(struct page *page) {
+}
+static inline void del_page_from_lap_list(struct page *page)
+}
+static inline void copy_page_info(struct page *oldpage, struct page *newpage) {
+}
+static inline void exchange_page_info(struct page *from_page, struct page *to_page) {
+}
+static inline void SetPageDeferred(struct page *page) {
+}
+static inline void ClearPageDeferred(struct page *page) {
+}
+static inline unsigned int PageDeferred(struct page *page) {
+	return 0;
+}
+static inline void SetPageDemoted(struct page *page) {
+}
+static inline void ClearPageDemoted(struct page *page) {
+}
+static inline unsigned int PageDemoted(struct page *page) {
+	return 0;
+}
+static inline void lock_busy(struct page *page) {
+}
+static inline unsigned int trylock_busy(struct page *page) {
+	return 0;
+}
+static inline void unlock_busy(struct page *page) {
+}
+static inline void add_page_for_exchange(struct page *page, int node) {
+}
+static inline void add_page_for_tracking(struct page *page, unsigned int prev_lv) {
+}
+static inline unsigned int mod_page_access_lv(struct page *page, unsigned int accessed) {
+	/* always hot */
+	return MAX_ACCESS_LEVEL;
+}
+static inline unsigned int get_page_access_lv(struct page *page) {
+	/* always hot */
+	return MAX_ACCESS_LEVEL;
+}
+static inline void reset_page_access_lv(struct page *page) {
+}
+
+static inline int find_best_demotion_node(struct page *page) {
+	return NUMA_NO_NODE;
+}
+static inline int  find_best_migration_node(struct page *page, int target_nid) {
+	return NUMA_NO_NODE;
+}
+
+#endif /* CONFIG_PAGE_BALANCING */
+#endif /* __LINUX_PAGE_BALANCING_H */
diff --git a/include/linux/page_ext.h b/include/linux/page_ext.h
index 095929517..70f9e557c 100644
--- a/include/linux/page_ext.h
+++ b/include/linux/page_ext.h
@@ -22,6 +22,13 @@ enum page_ext_flags {
 	PAGE_EXT_YOUNG,
 	PAGE_EXT_IDLE,
 #endif
+#ifdef CONFIG_PAGE_BALANCING
+	PAGE_EXT_BALALCING,
+	PAGE_EXT_BUSY_LOCK,
+	PAGE_EXT_TRACKED,
+	PAGE_EXT_DEFERRED,
+	PAGE_EXT_DEMOTED,
+#endif
 };
 
 /*
diff --git a/include/linux/sched/numa_balancing.h b/include/linux/sched/numa_balancing.h
index 3988762ef..6f6c7f586 100644
--- a/include/linux/sched/numa_balancing.h
+++ b/include/linux/sched/numa_balancing.h
@@ -14,6 +14,8 @@
 #define TNF_SHARED	0x04
 #define TNF_FAULT_LOCAL	0x08
 #define TNF_MIGRATE_FAIL 0x10
+#define TNF_YOUNG	0x20
+#define TNF_WRITE	0x40
 
 #ifdef CONFIG_NUMA_BALANCING
 extern void task_numa_fault(int last_node, int node, int pages, int flags);
@@ -21,7 +23,7 @@ extern pid_t task_numa_group_id(struct task_struct *p);
 extern void set_numabalancing_state(bool enabled);
 extern void task_numa_free(struct task_struct *p, bool final);
 extern bool should_numa_migrate_memory(struct task_struct *p, struct page *page,
-					int src_nid, int dst_cpu);
+				        int src_nid, int dst_cpu);
 #else
 static inline void task_numa_fault(int last_node, int node, int pages,
 				   int flags)
@@ -38,7 +40,7 @@ static inline void task_numa_free(struct task_struct *p, bool final)
 {
 }
 static inline bool should_numa_migrate_memory(struct task_struct *p,
-				struct page *page, int src_nid, int dst_cpu)
+			        struct page *page, int src_nid, int dst_cpu)
 {
 	return true;
 }
diff --git a/include/linux/sched/sysctl.h b/include/linux/sched/sysctl.h
index d4f6215ee..0c9498dce 100644
--- a/include/linux/sched/sysctl.h
+++ b/include/linux/sched/sysctl.h
@@ -33,11 +33,20 @@ enum sched_tunable_scaling {
 };
 extern enum sched_tunable_scaling sysctl_sched_tunable_scaling;
 
+/* NUMA balancing extended modes */
+#define NUMA_BALANCING_CPM		0x1
+#define NUMA_BALANCING_OPM		0x2
+#define NUMA_BALANCING_EXCHANGE		0x4
+
+extern unsigned int sysctl_numa_balancing_mode;
 extern unsigned int sysctl_numa_balancing_scan_delay;
 extern unsigned int sysctl_numa_balancing_scan_period_min;
 extern unsigned int sysctl_numa_balancing_scan_period_max;
 extern unsigned int sysctl_numa_balancing_scan_size;
 
+/* NUMA balancing extended mode options */
+extern unsigned int sysctl_numa_balancing_extended_mode;
+
 #ifdef CONFIG_SCHED_DEBUG
 extern __read_mostly unsigned int sysctl_sched_migration_cost;
 extern __read_mostly unsigned int sysctl_sched_nr_migrate;
@@ -90,6 +99,10 @@ extern int sysctl_numa_balancing(struct ctl_table *table, int write,
 				 void __user *buffer, size_t *lenp,
 				 loff_t *ppos);
 
+extern int sysctl_numa_balancing_extended(struct ctl_table *table, int write,
+				 void __user *buffer, size_t *lenp,
+				 loff_t *ppos);
+
 extern int sysctl_schedstats(struct ctl_table *table, int write,
 				 void __user *buffer, size_t *lenp,
 				 loff_t *ppos);
diff --git a/include/linux/swap.h b/include/linux/swap.h
index de2c67a33..2d34f048f 100644
--- a/include/linux/swap.h
+++ b/include/linux/swap.h
@@ -686,5 +686,25 @@ static inline bool mem_cgroup_swap_full(struct page *page)
 }
 #endif
 
+static inline bool reclaim_anon_pages(struct mem_cgroup *memcg,
+				      int node_id)
+{
+	/* Always age anon pages when we have swap */
+	if (memcg == NULL) {
+		if (get_nr_swap_pages() > 0)
+			return true;
+	} else {
+		if (mem_cgroup_get_nr_swap_pages(memcg) > 0)
+			return true;
+	}
+
+	/* Also age anon pages if we can auto-migrate them */
+	if (next_demotion_node(node_id) >= 0)
+		return true;
+
+	/* No way to reclaim anon pages */
+	return false;
+}
+
 #endif /* __KERNEL__*/
 #endif /* _LINUX_SWAP_H */
diff --git a/include/linux/vm_event_item.h b/include/linux/vm_event_item.h
index 47a3441cf..c5125998e 100644
--- a/include/linux/vm_event_item.h
+++ b/include/linux/vm_event_item.h
@@ -51,7 +51,33 @@ enum vm_event_item { PGPGIN, PGPGOUT, PSWPIN, PSWPOUT,
 		NUMA_PAGE_MIGRATE,
 #endif
 #ifdef CONFIG_MIGRATION
-		PGMIGRATE_SUCCESS, PGMIGRATE_FAIL,
+		PGMIGRATE_SUCCESS,
+		PGMIGRATE_FAIL,
+		PGMIGRATE_DST_NODE_FULL_FAIL,
+		PGMIGRATE_NUMA_ISOLATE_FAIL,
+		PGMIGRATE_NOMEM_FAIL,
+		PGMIGRATE_REFCOUNT_FAIL,
+		PGEXCHANGE_SUCCESS,
+		PGEXCHANGE_FAIL,
+		PGEXCHANGE_NO_PAGE_FAIL,
+		PGEXCHANGE_NODE_UNMATCH_FAIL,
+		PGEXCHANGE_LIST_EMPTY_FAIL,
+		PGEXCHANGE_SCAN_FAIL,
+		PGEXCHANGE_BUSY_FAIL,
+		PGACTIVATE_DEFERRED,
+		PGACTIVATE_DEFERRED_LOCAL,
+		PGDEMOTE_NO_PAGE_FAIL,
+		PGDEMOTE_NO_LRU_FAIL,
+		PGDEMOTE_BUSY_FAIL,
+		PGDEMOTE_BACKGROUND,
+		PGDEMOTE_FILE,
+		PGPROMOTE_EMPTY_POOL_FAIL,
+		PGPROMOTE_NO_PAGE_FAIL,
+		PGPROMOTE_LOW_FREQ_FAIL,
+		PGPROMOTE_FREE_AREA,
+		PGREPROMOTE,
+		PGFREE_DEMOTED,
+		NR_PAGE_SKIPPED,
 #endif
 #ifdef CONFIG_COMPACTION
 		COMPACTMIGRATE_SCANNED, COMPACTFREE_SCANNED,
diff --git a/include/linux/vmstat.h b/include/linux/vmstat.h
index bdeda4b07..3ac620e7c 100644
--- a/include/linux/vmstat.h
+++ b/include/linux/vmstat.h
@@ -263,6 +263,16 @@ extern void dec_zone_state(struct zone *, enum zone_stat_item);
 extern void __dec_zone_state(struct zone *, enum zone_stat_item);
 extern void __dec_node_state(struct pglist_data *, enum node_stat_item);
 
+enum migrate_hmem_reason;
+enum migrate_hmem_fail_reason;
+void inc_hmem_state(enum migrate_hmem_reason, struct page *, struct page *);
+void inc_hmem_src_state(enum migrate_hmem_reason, struct page *);
+void inc_hmem_dst_state(enum migrate_hmem_reason, struct page *);
+void inc_hmem_fail_state(enum migrate_hmem_fail_reason, int src_nid, int dst_nid,
+		int is_huge);
+void inc_hmem_fail_src_state(enum migrate_hmem_fail_reason, int nid, int is_huge);
+void inc_hmem_fail_dst_state(enum migrate_hmem_fail_reason, int nid, int is_huge);
+
 void quiet_vmstat(void);
 void cpu_vm_stats_fold(int cpu);
 void refresh_zone_stat_thresholds(void);
@@ -369,6 +379,16 @@ static inline void quiet_vmstat(void) { }
 
 static inline void drain_zonestat(struct zone *zone,
 			struct per_cpu_pageset *pset) { }
+
+static inline void inc_hmem_state(unsigned int reason, struct page *src,
+				  struct page *dst) { }
+static inline void inc_hmem_src_state(enum migrate_hmem_reason, struct page *) { }
+static inline void inc_hmem_dst_state(enum migrate_hmem_reason, struct page *) { }
+static inline void inc_hmem_fail_src_state(enum migrate_hmem_fail_reason,
+				struct page *) { }
+static inline void inc_hmem_fail_dst_state(enum migrate_hmem_fail_reason,
+				struct page *) { }
+
 #endif		/* CONFIG_SMP */
 
 static inline void __mod_zone_freepage_state(struct zone *zone, int nr_pages,
diff --git a/include/trace/events/migrate.h b/include/trace/events/migrate.h
index 705b33d1e..414b1b52e 100644
--- a/include/trace/events/migrate.h
+++ b/include/trace/events/migrate.h
@@ -6,6 +6,7 @@
 #define _TRACE_MIGRATE_H
 
 #include <linux/tracepoint.h>
+#include <trace/events/mmflags.h>
 
 #define MIGRATE_MODE						\
 	EM( MIGRATE_ASYNC,	"MIGRATE_ASYNC")		\
@@ -20,7 +21,23 @@
 	EM( MR_SYSCALL,		"syscall_or_cpuset")		\
 	EM( MR_MEMPOLICY_MBIND,	"mempolicy_mbind")		\
 	EM( MR_NUMA_MISPLACED,	"numa_misplaced")		\
-	EMe(MR_CONTIG_RANGE,	"contig_range")
+	EM( MR_CONTIG_RANGE,	"contig_range")			\
+	EM( MR_DEMOTION,	"demotion")			\
+	EMe(MR_PROMOTION,	"promotion")
+
+
+#define HMEM_MIGRATE_REASON					\
+	EM(MR_HMEM_LOCAL_PROMOTE,	"lp")	\
+	EM(MR_HMEM_REMOTE_PROMOTE,	"rp")	\
+	EM(MR_HMEM_MIGRATE,		"migrate")		\
+	EMe( MR_DEMOTION,		"demotion")
+
+
+#define HMEM_MIGRATE_FAIL_REASON				\
+	EM(MR_HMEM_LOCAL_PROMOTE_FAIL,	"lpf")			\
+	EM(MR_HMEM_REMOTE_PROMOTE_FAIL, "rpf")			\
+	EMe(MR_HMEM_MIGRATE_FAIL,	"mf")
+
 
 /*
  * First define the enums in the above macros to be exported to userspace
@@ -70,6 +87,34 @@ TRACE_EVENT(mm_migrate_pages,
 		__print_symbolic(__entry->mode, MIGRATE_MODE),
 		__print_symbolic(__entry->reason, MIGRATE_REASON))
 );
+
+TRACE_EVENT(mm_migrate_move_page,
+
+	TP_PROTO(struct page *from, struct page *to, int status, int reason),
+
+	TP_ARGS(from, to, status, reason),
+
+	TP_STRUCT__entry(
+		__field(struct page *, from)
+		__field(struct page *, to)
+		__field(int, status)
+		__field(int, reason)
+	),
+
+	TP_fast_assign(
+		__entry->from = from;
+		__entry->to = to;
+		__entry->status = status;
+		__entry->reason = reason;
+	),
+
+	TP_printk("node from=%d to=%d status=%d flags=%s reason=%s refs=%d",
+		page_to_nid(__entry->from), page_to_nid(__entry->to),
+		__entry->status,
+		show_page_flags(__entry->from->flags & ((1UL << NR_PAGEFLAGS) - 1)),
+                __print_symbolic(__entry->reason, MIGRATE_REASON),
+		page_ref_count(__entry->from))
+);
 #endif /* _TRACE_MIGRATE_H */
 
 /* This part must be outside protection */
diff --git a/init/main.c b/init/main.c
index 96f8d5af5..1e4eac751 100644
--- a/init/main.c
+++ b/init/main.c
@@ -1188,6 +1188,7 @@ static noinline void __init kernel_init_freeable(void)
 	page_alloc_init_late();
 	/* Initialize page ext after all struct pages are initialized. */
 	page_ext_init();
+	promote_init();
 
 	do_basic_setup();
 
diff --git a/kernel/sched/core.c b/kernel/sched/core.c
index df9f1fe56..8fe11c705 100644
--- a/kernel/sched/core.c
+++ b/kernel/sched/core.c
@@ -2603,6 +2603,8 @@ static void __sched_fork(unsigned long clone_flags, struct task_struct *p)
 }
 
 DEFINE_STATIC_KEY_FALSE(sched_numa_balancing);
+unsigned int sysctl_numa_balancing_mode;
+unsigned int sysctl_numa_balancing_extended_mode;
 
 #ifdef CONFIG_NUMA_BALANCING
 
@@ -2618,20 +2620,32 @@ void set_numabalancing_state(bool enabled)
 int sysctl_numa_balancing(struct ctl_table *table, int write,
 			 void __user *buffer, size_t *lenp, loff_t *ppos)
 {
-	struct ctl_table t;
 	int err;
-	int state = static_branch_likely(&sched_numa_balancing);
 
 	if (write && !capable(CAP_SYS_ADMIN))
 		return -EPERM;
 
-	t = *table;
-	t.data = &state;
-	err = proc_dointvec_minmax(&t, write, buffer, lenp, ppos);
+	err = proc_dointvec_minmax(table, write, buffer, lenp, ppos);
 	if (err < 0)
 		return err;
 	if (write)
-		set_numabalancing_state(state);
+		set_numabalancing_state(*(int *)table->data);
+	return err;
+}
+
+int sysctl_numa_balancing_extended(struct ctl_table *table, int write,
+			 void __user *buffer, size_t *lenp, loff_t *ppos)
+{
+	int err;
+
+	if (write && !capable(CAP_SYS_ADMIN))
+		return -EPERM;
+
+	if (!sysctl_numa_balancing_mode)
+		return -EPERM;
+
+	err = proc_dointvec_minmax(table, write, buffer, lenp, ppos);
+
 	return err;
 }
 #endif
diff --git a/kernel/sched/fair.c b/kernel/sched/fair.c
index 500f5db0d..64c16a912 100644
--- a/kernel/sched/fair.c
+++ b/kernel/sched/fair.c
@@ -2516,7 +2516,7 @@ static void reset_ptenuma_scan(struct task_struct *p)
 	 * expensive, to avoid any form of compiler optimizations:
 	 */
 	WRITE_ONCE(p->mm->numa_scan_seq, READ_ONCE(p->mm->numa_scan_seq) + 1);
-	p->mm->numa_scan_offset = 0;
+	WRITE_ONCE(p->mm->numa_scan_offset, 0);
 }
 
 /*
@@ -2618,6 +2618,7 @@ void task_numa_work(struct callback_head *work)
 			start = max(start, vma->vm_start);
 			end = ALIGN(start + (pages << PAGE_SHIFT), HPAGE_SIZE);
 			end = min(end, vma->vm_end);
+			WRITE_ONCE(mm->numa_scan_offset, end);
 			nr_pte_updates = change_prot_numa(vma, start, end);
 
 			/*
@@ -2647,9 +2648,7 @@ void task_numa_work(struct callback_head *work)
 	 * would find the !migratable VMA on the next scan but not reset the
 	 * scanner to the start so check it now.
 	 */
-	if (vma)
-		mm->numa_scan_offset = start;
-	else
+	if (!vma)
 		reset_ptenuma_scan(p);
 	up_read(&mm->mmap_sem);
 
diff --git a/kernel/sched/sched.h b/kernel/sched/sched.h
index 802b1f340..4979d1fc8 100644
--- a/kernel/sched/sched.h
+++ b/kernel/sched/sched.h
@@ -66,6 +66,10 @@
 #include <linux/task_work.h>
 #include <linux/tsacct_kern.h>
 
+#ifdef CONFIG_PAGE_BALANCING
+#include <linux/page_balancing.h>
+#endif
+
 #include <asm/tlb.h>
 
 #ifdef CONFIG_PARAVIRT
diff --git a/kernel/sysctl.c b/kernel/sysctl.c
index 078950d96..1c7bdf9f6 100644
--- a/kernel/sysctl.c
+++ b/kernel/sysctl.c
@@ -126,7 +126,9 @@ static int sixty = 60;
 
 static int __maybe_unused neg_one = -1;
 static int __maybe_unused two = 2;
+static int __maybe_unused three = 3;
 static int __maybe_unused four = 4;
+static int __maybe_unused seven = 7;
 static unsigned long zero_ul;
 static unsigned long one_ul = 1;
 static unsigned long long_max = LONG_MAX;
@@ -417,9 +419,18 @@ static struct ctl_table kern_table[] = {
 		.proc_handler	= proc_dointvec_minmax,
 		.extra1		= SYSCTL_ONE,
 	},
+	{
+		.procname	= "numa_balancing_extended",
+		.data		= &sysctl_numa_balancing_extended_mode,
+		.maxlen		= sizeof(unsigned int),
+		.mode		= 0644,
+		.proc_handler	= sysctl_numa_balancing_extended,
+		.extra1		= &zero_ul,
+		.extra2		= &seven,
+	},
 	{
 		.procname	= "numa_balancing",
-		.data		= NULL, /* filled in by handler */
+		.data		= &sysctl_numa_balancing_mode,
 		.maxlen		= sizeof(unsigned int),
 		.mode		= 0644,
 		.proc_handler	= sysctl_numa_balancing,
diff --git a/mm/Kconfig b/mm/Kconfig
index 56cec636a..0f5d6bb0c 100644
--- a/mm/Kconfig
+++ b/mm/Kconfig
@@ -726,6 +726,12 @@ config GUP_GET_PTE_LOW_HIGH
 config ARCH_HAS_PTE_SPECIAL
 	bool
 
+config PAGE_FAULT_PROFILE
+	bool "Page Promotion/Demotion Latency Profile"
+	def_bool n
+	help
+	  Profile promotion and demotion latency on tiered memory systems
+
 #
 # Some architectures require a special hugepage directory format that is
 # required to support multiple hugepage sizes. For example a4fe3ce76
diff --git a/mm/Kconfig.debug b/mm/Kconfig.debug
index 82b6a2089..3f493ec57 100644
--- a/mm/Kconfig.debug
+++ b/mm/Kconfig.debug
@@ -98,6 +98,28 @@ config PAGE_POISONING_ZERO
 
 	   If unsure, say N
 
+config PAGE_BALANCING
+	bool "Add extended numa balancing modes"
+	depends on DEBUG_KERNEL
+	depends on MIGRATION
+	select PAGE_EXTENSION
+	---help---
+	   Enable extended numa balancing modes including conservative promotion
+	   and migration(CPM), opportunistic promotion and migration(OPM), and
+	   page exchange. Also, this support THP migration.
+
+	   If unsure, say N
+
+config PAGE_BALANCING_DEBUG
+	bool "Track extended numa balancing stats"
+	depends on MIGRATION
+	depends on PAGE_BALANCING
+	select PAGE_EXTENSION
+	---help---
+	   Enable tracking page promotion, demotion, migration and exchange stats.
+	   It also track stats about Least Accessed Page list statistics.
+
+
 config DEBUG_PAGE_REF
 	bool "Enable tracepoint to track down page reference manipulation"
 	depends on DEBUG_KERNEL
diff --git a/mm/Makefile b/mm/Makefile
index d0b295c3b..d53485f04 100644
--- a/mm/Makefile
+++ b/mm/Makefile
@@ -45,6 +45,10 @@ obj-y			:= filemap.o mempool.o oom_kill.o fadvise.o \
 page-alloc-y := page_alloc.o
 page-alloc-$(CONFIG_SHUFFLE_PAGE_ALLOCATOR) += shuffle.o
 
+obj-y += copy_page.o
+obj-y += exchange_page.o
+obj-y += exchange.o
+
 obj-y += page-alloc.o
 obj-y += init-mm.o
 obj-y += memblock.o
@@ -105,3 +109,7 @@ obj-$(CONFIG_PERCPU_STATS) += percpu-stats.o
 obj-$(CONFIG_ZONE_DEVICE) += memremap.o
 obj-$(CONFIG_HMM_MIRROR) += hmm.o
 obj-$(CONFIG_MEMFD_CREATE) += memfd.o
+obj-$(CONFIG_PAGE_BALANCING) += page_balancing.o
+
+#for debugging
+CFLAGS_page_balancing.o = -Og
diff --git a/mm/compaction.c b/mm/compaction.c
index 952dc2fb2..7732ca3ac 100644
--- a/mm/compaction.c
+++ b/mm/compaction.c
@@ -2071,6 +2071,7 @@ bool compaction_zonelist_suitable(struct alloc_context *ac, int order,
 static enum compact_result
 compact_zone(struct compact_control *cc, struct capture_control *capc)
 {
+	struct migrate_detail m_detail = {};
 	enum compact_result ret;
 	unsigned long start_pfn = cc->zone->zone_start_pfn;
 	unsigned long end_pfn = zone_end_pfn(cc->zone);
@@ -2183,9 +2184,10 @@ compact_zone(struct compact_control *cc, struct capture_control *capc)
 			;
 		}
 
+		m_detail.reason = MR_COMPACTION;
 		err = migrate_pages(&cc->migratepages, compaction_alloc,
 				compaction_free, (unsigned long)cc, cc->mode,
-				MR_COMPACTION);
+				&m_detail);
 
 		trace_mm_compaction_migratepages(cc->nr_migratepages, err,
 							&cc->migratepages);
diff --git a/mm/copy_page.c b/mm/copy_page.c
new file mode 100644
index 000000000..71da8fee2
--- /dev/null
+++ b/mm/copy_page.c
@@ -0,0 +1,249 @@
+/*
+ * Parallel page copy routine.
+ * Use DMA engine to copy page data
+ *
+ * Zi Yan <zi.yan@cs.rutgers.edu>
+ *
+ */
+
+#include <linux/sysctl.h>
+#include <linux/highmem.h>
+#include <linux/workqueue.h>
+#include <linux/slab.h>
+#include <linux/freezer.h>
+
+
+unsigned int limit_mt_num = 4;
+
+/* ======================== multi-threaded copy page ======================== */
+
+struct copy_item {
+	char *to;
+	char *from;
+	unsigned long chunk_size;
+};
+
+struct copy_page_info {
+	struct work_struct copy_page_work;
+	unsigned long num_items;
+	struct copy_item item_list[0];
+};
+
+static void copy_page_routine(char *vto, char *vfrom,
+	unsigned long chunk_size)
+{
+	memcpy(vto, vfrom, chunk_size);
+}
+
+static void copy_page_work_queue_thread(struct work_struct *work)
+{
+	struct copy_page_info *my_work = (struct copy_page_info *)work;
+	int i;
+
+	for (i = 0; i < my_work->num_items; ++i)
+		copy_page_routine(my_work->item_list[i].to,
+						  my_work->item_list[i].from,
+						  my_work->item_list[i].chunk_size);
+}
+
+int copy_page_multithread(struct page *to, struct page *from, int nr_pages)
+{
+	unsigned int total_mt_num = limit_mt_num;
+#ifdef CONFIG_PAGE_MIGRATION_PROFILE
+	int to_node = page_to_nid(to);
+#else
+	int to_node = numa_node_id();
+#endif
+	int i;
+	struct copy_page_info *work_items[32] = {0};
+	char *vto, *vfrom;
+	unsigned long chunk_size;
+	const struct cpumask *per_node_cpumask = cpumask_of_node(to_node);
+	int cpu_id_list[32] = {0};
+	int cpu;
+	int err = 0;
+
+	total_mt_num = min_t(unsigned int, total_mt_num,
+						 cpumask_weight(per_node_cpumask));
+	if (total_mt_num > 1)
+		total_mt_num = (total_mt_num / 2) * 2;
+
+	if (total_mt_num > 32 || total_mt_num < 1)
+		return -ENODEV;
+
+	for (cpu = 0; cpu < total_mt_num; ++cpu) {
+		work_items[cpu] = kzalloc(sizeof(struct copy_page_info)
+						+ sizeof(struct copy_item), GFP_KERNEL);
+		if (!work_items[cpu]) {
+			err = -ENOMEM;
+			goto free_work_items;
+		}
+	}
+
+	i = 0;
+	for_each_cpu(cpu, per_node_cpumask) {
+		if (i >= total_mt_num)
+			break;
+		cpu_id_list[i] = cpu;
+		++i;
+	}
+
+	vfrom = kmap(from);
+	vto = kmap(to);
+	chunk_size = PAGE_SIZE*nr_pages / total_mt_num;
+
+	for (i = 0; i < total_mt_num; ++i) {
+		INIT_WORK((struct work_struct *)work_items[i],
+				  copy_page_work_queue_thread);
+
+		work_items[i]->num_items = 1;
+		work_items[i]->item_list[0].to = vto + i * chunk_size;
+		work_items[i]->item_list[0].from = vfrom + i * chunk_size;
+		work_items[i]->item_list[0].chunk_size = chunk_size;
+
+		queue_work_on(cpu_id_list[i],
+					  system_highpri_wq,
+					  (struct work_struct *)work_items[i]);
+	}
+
+	/* Wait until it finishes  */
+	for (i = 0; i < total_mt_num; ++i)
+		flush_work((struct work_struct *)work_items[i]);
+
+	kunmap(to);
+	kunmap(from);
+
+free_work_items:
+	for (cpu = 0; cpu < total_mt_num; ++cpu)
+		if (work_items[cpu])
+			kfree(work_items[cpu]);
+
+	return err;
+}
+
+int copy_page_lists_mt(struct page **to, struct page **from, int nr_items)
+{
+	int err = 0;
+	unsigned int total_mt_num = limit_mt_num;
+#ifdef CONFIG_PAGE_MIGRATION_PROFILE
+	int to_node = page_to_nid(*to);
+#else
+	int to_node = numa_node_id();
+#endif
+	int i;
+	struct copy_page_info *work_items[32] = {0};
+	const struct cpumask *per_node_cpumask = cpumask_of_node(to_node);
+	int cpu_id_list[32] = {0};
+	int cpu;
+	int max_items_per_thread;
+	int item_idx;
+
+	total_mt_num = min_t(unsigned int, total_mt_num,
+						 cpumask_weight(per_node_cpumask));
+
+
+	if (total_mt_num > 32)
+		return -ENODEV;
+
+	/* Each threads get part of each page, if nr_items < totla_mt_num */
+	if (nr_items < total_mt_num)
+		max_items_per_thread = nr_items;
+	else
+		max_items_per_thread = (nr_items / total_mt_num) +
+				((nr_items % total_mt_num)?1:0);
+
+
+	for (cpu = 0; cpu < total_mt_num; ++cpu) {
+		work_items[cpu] = kzalloc(sizeof(struct copy_page_info) +
+					sizeof(struct copy_item)*max_items_per_thread, GFP_KERNEL);
+		if (!work_items[cpu]) {
+			err = -ENOMEM;
+			goto free_work_items;
+		}
+	}
+
+	i = 0;
+	for_each_cpu(cpu, per_node_cpumask) {
+		if (i >= total_mt_num)
+			break;
+		cpu_id_list[i] = cpu;
+		++i;
+	}
+
+	if (nr_items < total_mt_num) {
+		for (cpu = 0; cpu < total_mt_num; ++cpu) {
+			INIT_WORK((struct work_struct *)work_items[cpu],
+					  copy_page_work_queue_thread);
+			work_items[cpu]->num_items = max_items_per_thread;
+		}
+
+		for (item_idx = 0; item_idx < nr_items; ++item_idx) {
+			unsigned long chunk_size = PAGE_SIZE * hpage_nr_pages(from[item_idx]) / total_mt_num;
+			char *vfrom = kmap(from[item_idx]);
+			char *vto = kmap(to[item_idx]);
+			VM_BUG_ON(PAGE_SIZE * hpage_nr_pages(from[item_idx]) % total_mt_num);
+			BUG_ON(hpage_nr_pages(to[item_idx]) !=
+				   hpage_nr_pages(from[item_idx]));
+
+			for (cpu = 0; cpu < total_mt_num; ++cpu) {
+				work_items[cpu]->item_list[item_idx].to = vto + chunk_size * cpu;
+				work_items[cpu]->item_list[item_idx].from = vfrom + chunk_size * cpu;
+				work_items[cpu]->item_list[item_idx].chunk_size =
+					chunk_size;
+			}
+		}
+
+		for (cpu = 0; cpu < total_mt_num; ++cpu)
+			queue_work_on(cpu_id_list[cpu],
+						  system_highpri_wq,
+						  (struct work_struct *)work_items[cpu]);
+	} else {
+		item_idx = 0;
+		for (cpu = 0; cpu < total_mt_num; ++cpu) {
+			int num_xfer_per_thread = nr_items / total_mt_num;
+			int per_cpu_item_idx;
+
+			if (cpu < (nr_items % total_mt_num))
+				num_xfer_per_thread += 1;
+
+			INIT_WORK((struct work_struct *)work_items[cpu],
+					  copy_page_work_queue_thread);
+
+			work_items[cpu]->num_items = num_xfer_per_thread;
+			for (per_cpu_item_idx = 0; per_cpu_item_idx < work_items[cpu]->num_items;
+				 ++per_cpu_item_idx, ++item_idx) {
+				work_items[cpu]->item_list[per_cpu_item_idx].to = kmap(to[item_idx]);
+				work_items[cpu]->item_list[per_cpu_item_idx].from =
+					kmap(from[item_idx]);
+				work_items[cpu]->item_list[per_cpu_item_idx].chunk_size =
+					PAGE_SIZE * hpage_nr_pages(from[item_idx]);
+
+				BUG_ON(hpage_nr_pages(to[item_idx]) !=
+					   hpage_nr_pages(from[item_idx]));
+			}
+
+			queue_work_on(cpu_id_list[cpu],
+						  system_highpri_wq,
+						  (struct work_struct *)work_items[cpu]);
+		}
+		if (item_idx != nr_items)
+			pr_err("%s: only %d out of %d pages are transferred\n", __func__,
+				item_idx - 1, nr_items);
+	}
+
+	/* Wait until it finishes  */
+	for (i = 0; i < total_mt_num; ++i)
+		flush_work((struct work_struct *)work_items[i]);
+
+	for (i = 0; i < nr_items; ++i) {
+			kunmap(to[i]);
+			kunmap(from[i]);
+	}
+
+free_work_items:
+	for (cpu = 0; cpu < total_mt_num; ++cpu)
+		if (work_items[cpu])
+			kfree(work_items[cpu]);
+
+	return err;
+}
diff --git a/mm/debug.c b/mm/debug.c
index 8345bb6e4..b9a1809f4 100644
--- a/mm/debug.c
+++ b/mm/debug.c
@@ -25,6 +25,8 @@ const char *migrate_reason_names[MR_TYPES] = {
 	"mempolicy_mbind",
 	"numa_misplaced",
 	"cma",
+	"demotion",
+	"promotion",
 };
 
 const struct trace_print_flags pageflag_names[] = {
diff --git a/mm/exchange.c b/mm/exchange.c
new file mode 100644
index 000000000..8e7372203
--- /dev/null
+++ b/mm/exchange.c
@@ -0,0 +1,1821 @@
+/*
+ * Exchange two in-use pages. Page flags and page->mapping are exchanged
+ * as well. Only anonymous pages are supported.
+ *
+ * Copyright (C) 2016 NVIDIA, Zi Yan <ziy@nvidia.com>
+ *
+ * This work is licensed under the terms of the GNU GPL, version 2.
+ */
+
+#include <linux/syscalls.h>
+#include <linux/migrate.h>
+#include <linux/exchange.h>
+#include <linux/security.h>
+#include <linux/cpuset.h>
+#include <linux/mm_inline.h>
+#include <linux/page_idle.h>
+#include <linux/page-flags.h>
+#include <linux/ksm.h>
+#include <linux/memcontrol.h>
+#include <linux/balloon_compaction.h>
+#include <linux/buffer_head.h>
+#include <linux/fs.h> /* buffer_migrate_page  */
+#include <linux/backing-dev.h>
+#include <linux/sched/mm.h>
+#include <linux/sched/sysctl.h>
+#include <linux/page_balancing.h>
+
+
+#include "internal.h"
+
+// #define DEBUG
+
+#ifdef DEBUG
+#define START_TIME(ts) ts = rdtsc()
+#define END_TIME(msg, ts) trace_printk("%s: %llu\n", msg, rdtsc() - ts)
+#else
+#define START_TIME(ts) {}
+#define END_TIME(msg, ts) {}
+#endif
+
+unsigned int exchange_concur = 1;
+unsigned int exchange_mt = 1;
+
+
+/*
+ * Move a list of individual pages
+ */
+struct pages_to_node {
+	unsigned long from_addr;
+	int from_status;
+
+	unsigned long to_addr;
+	int to_status;
+};
+
+struct page_flags {
+	unsigned int page_error :1;
+	unsigned int page_referenced:1;
+	unsigned int page_uptodate:1;
+	unsigned int page_active:1;
+	unsigned int page_unevictable:1;
+	unsigned int page_checked:1;
+	unsigned int page_mappedtodisk:1;
+	unsigned int page_dirty:1;
+	unsigned int page_is_young:1;
+	unsigned int page_is_idle:1;
+	unsigned int page_swapcache:1;
+	unsigned int page_writeback:1;
+	unsigned int page_private:1;
+	unsigned int page_doublemap:1;
+	unsigned int _pad:2;
+};
+
+static void pr_dump_page(struct page *page, const char *msg)
+{
+	pr_debug("dump:%s page(%p):0x%lx,"
+		"count:%d,mapcount:%d,mapping:%p,index:%#lx,flags:%#lx(%pGp),%s,order:%d"
+		",%s"
+		",%s,page_nid:%d"
+		"\n",
+		msg,
+		page,
+		page_to_pfn(page),
+		page_ref_count(page),
+		PageSlab(page)?0:page_mapcount(page),
+		page->mapping, page_to_pgoff(page),
+		page->flags, &page->flags,
+		PageCompound(page)?"compound_page":"single_page",
+		compound_order(page),
+		PageDirty(page)?"dirty":"clean",
+		PageDeferred(page)?"deferred":"nondeferred",
+		page_to_nid(page)
+		);
+}
+
+#define UNROLL2(x)   x x
+#define UNROLL4(x)   UNROLL2(x)   UNROLL2(x)
+#define UNROLL8(x)   UNROLL4(x)   UNROLL4(x)
+#define UNROLL16(x)  UNROLL8(x)   UNROLL8(x)
+#define UNROLL32(x)  UNROLL16(x)  UNROLL16(x)
+#define UNROLL64(x)  UNROLL32(x)  UNROLL32(x)
+#define UNROLL128(x) UNROLL64(x)  UNROLL64(x)
+#define UNROLL256(x) UNROLL128(x) UNROLL128(x)
+#define UNROLL512(x) UNROLL256(x) UNROLL256(x)
+
+#define SWAP_PAGE(from, to, tmp, index, chunk) \
+		tmp = *((u64*)(from + index)); \
+		*((u64*)(from + index)) = *((u64*)(to + index)); \
+		*((u64*)(to + index)) = tmp; \
+		index = index + chunk;
+
+static void exchange_page(char *to, char *from)
+{
+	u64 tmp;
+	int chunk_size = sizeof(tmp);
+	int i = 0;
+
+	// for (i = 0; i < PAGE_SIZE; i += chunck_size * 4) {
+	// 	tmp = *((u64*)(from + i));
+	// 	*((u64*)(from + i)) = *((u64*)(to + i));
+	// 	*((u64*)(to + i)) = tmp;
+	// }
+	while (i < PAGE_SIZE) {
+		UNROLL512(SWAP_PAGE(from, to, tmp, i, chunk_size))
+	}
+}
+
+static inline void exchange_highpage(struct page *to, struct page *from)
+{
+	char *vfrom, *vto;
+
+	vfrom = kmap_atomic(from);
+	vto = kmap_atomic(to);
+	exchange_page(vto, vfrom);
+	kunmap_atomic(vto);
+	kunmap_atomic(vfrom);
+}
+
+static void exchange_huge_page(struct page *dst, struct page *src)
+{
+	int i;
+	int nr_pages;
+
+	if (PageHuge(src)) {
+		/* hugetlbfs page */
+		struct hstate *h = page_hstate(src);
+		nr_pages = pages_per_huge_page(h);
+
+		if (unlikely(nr_pages > MAX_ORDER_NR_PAGES)) {
+			//__exchange_gigantic_page(dst, src, nr_pages);
+			return;
+		}
+	} else {
+		/* thp page */
+		BUG_ON(!PageTransHuge(src));
+		nr_pages = hpage_nr_pages(src);
+	}
+
+	for (i = 0; i < nr_pages; i++) {
+		exchange_highpage(dst + i, src + i);
+	}
+}
+
+/*
+ * Copy the page to its new location without polluting cache
+ */
+static void exchange_page_flags(struct page *to_page, struct page *from_page)
+{
+	int from_cpupid, to_cpupid;
+	struct page_flags from_page_flags = {0}, to_page_flags = {0};
+	struct mem_cgroup *to_memcg = page_memcg(to_page),
+					  *from_memcg = page_memcg(from_page);
+
+	from_cpupid = page_cpupid_xchg_last(from_page, -1);
+
+	from_page_flags.page_error = PageError(from_page);
+	if (from_page_flags.page_error)
+		ClearPageError(from_page);
+	from_page_flags.page_referenced = TestClearPageReferenced(from_page);
+	from_page_flags.page_uptodate = PageUptodate(from_page);
+	ClearPageUptodate(from_page);
+	from_page_flags.page_active = TestClearPageActive(from_page);
+	from_page_flags.page_unevictable = TestClearPageUnevictable(from_page);
+	from_page_flags.page_checked = PageChecked(from_page);
+	if (from_page_flags.page_checked)
+		ClearPageChecked(from_page);
+	from_page_flags.page_mappedtodisk = PageMappedToDisk(from_page);
+	ClearPageMappedToDisk(from_page);
+	from_page_flags.page_dirty = PageDirty(from_page);
+	ClearPageDirty(from_page);
+	from_page_flags.page_is_young = test_and_clear_page_young(from_page);
+	from_page_flags.page_is_idle = page_is_idle(from_page);
+	clear_page_idle(from_page);
+	from_page_flags.page_swapcache = PageSwapCache(from_page);
+	/*from_page_flags.page_private = PagePrivate(from_page);*/
+	/*ClearPagePrivate(from_page);*/
+	from_page_flags.page_writeback = test_clear_page_writeback(from_page);
+	from_page_flags.page_doublemap = PageDoubleMap(from_page);
+
+
+	to_cpupid = page_cpupid_xchg_last(to_page, -1);
+
+	to_page_flags.page_error = PageError(to_page);
+	if (to_page_flags.page_error)
+		ClearPageError(to_page);
+	to_page_flags.page_referenced = TestClearPageReferenced(to_page);
+	to_page_flags.page_uptodate = PageUptodate(to_page);
+	ClearPageUptodate(to_page);
+	to_page_flags.page_active = TestClearPageActive(to_page);
+	to_page_flags.page_unevictable = TestClearPageUnevictable(to_page);
+	to_page_flags.page_checked = PageChecked(to_page);
+	if (to_page_flags.page_checked)
+		ClearPageChecked(to_page);
+	to_page_flags.page_mappedtodisk = PageMappedToDisk(to_page);
+	ClearPageMappedToDisk(to_page);
+	to_page_flags.page_dirty = PageDirty(to_page);
+	ClearPageDirty(to_page);
+	to_page_flags.page_is_young = test_and_clear_page_young(to_page);
+	to_page_flags.page_is_idle = page_is_idle(to_page);
+	clear_page_idle(to_page);
+	to_page_flags.page_swapcache = PageSwapCache(to_page);
+	/*to_page_flags.page_private = PagePrivate(to_page);*/
+	/*ClearPagePrivate(to_page);*/
+	to_page_flags.page_writeback = test_clear_page_writeback(to_page);
+	to_page_flags.page_doublemap = PageDoubleMap(to_page);
+
+	/* set to_page */
+	if (from_page_flags.page_error)
+		SetPageError(to_page);
+	if (from_page_flags.page_referenced)
+		SetPageReferenced(to_page);
+	if (from_page_flags.page_uptodate)
+		SetPageUptodate(to_page);
+	if (from_page_flags.page_active) {
+		VM_BUG_ON_PAGE(from_page_flags.page_unevictable, from_page);
+		SetPageActive(to_page);
+	} else if (from_page_flags.page_unevictable)
+		SetPageUnevictable(to_page);
+	if (from_page_flags.page_checked)
+		SetPageChecked(to_page);
+	if (from_page_flags.page_mappedtodisk)
+		SetPageMappedToDisk(to_page);
+
+	/* Move dirty on pages not done by migrate_page_move_mapping() */
+	if (from_page_flags.page_dirty)
+		SetPageDirty(to_page);
+
+	if (from_page_flags.page_is_young)
+		set_page_young(to_page);
+	if (from_page_flags.page_is_idle)
+		set_page_idle(to_page);
+	if (from_page_flags.page_doublemap)
+		SetPageDoubleMap(to_page);
+
+	/* set from_page */
+	if (to_page_flags.page_error)
+		SetPageError(from_page);
+	if (to_page_flags.page_referenced)
+		SetPageReferenced(from_page);
+	if (to_page_flags.page_uptodate)
+		SetPageUptodate(from_page);
+	if (to_page_flags.page_active) {
+		VM_BUG_ON_PAGE(to_page_flags.page_unevictable, from_page);
+		SetPageActive(from_page);
+	} else if (to_page_flags.page_unevictable)
+		SetPageUnevictable(from_page);
+	if (to_page_flags.page_checked)
+		SetPageChecked(from_page);
+	if (to_page_flags.page_mappedtodisk)
+		SetPageMappedToDisk(from_page);
+
+	/* Move dirty on pages not done by migrate_page_move_mapping() */
+	if (to_page_flags.page_dirty)
+		SetPageDirty(from_page);
+
+	if (to_page_flags.page_is_young)
+		set_page_young(from_page);
+	if (to_page_flags.page_is_idle)
+		set_page_idle(from_page);
+	if (to_page_flags.page_doublemap)
+		SetPageDoubleMap(from_page);
+
+	/*
+	 * Copy NUMA information to the new page, to prevent over-eager
+	 * future migrations of this same page.
+	 */
+	page_cpupid_xchg_last(to_page, from_cpupid);
+	page_cpupid_xchg_last(from_page, to_cpupid);
+
+	ksm_exchange_page(to_page, from_page);
+	/*
+	 * Please do not reorder this without considering how mm/ksm.c's
+	 * get_ksm_page() depends upon ksm_migrate_page() and PageSwapCache().
+	 */
+	ClearPageSwapCache(to_page);
+	ClearPageSwapCache(from_page);
+	if (from_page_flags.page_swapcache)
+		SetPageSwapCache(to_page);
+	if (to_page_flags.page_swapcache)
+		SetPageSwapCache(from_page);
+
+
+#ifdef CONFIG_PAGE_OWNER
+	/* exchange page owner  */
+	BUG();
+#endif
+	/* exchange mem cgroup  */
+	to_page->mem_cgroup = from_memcg;
+	from_page->mem_cgroup = to_memcg;
+
+	/* exchange page info */
+	exchange_page_info(from_page, to_page);
+
+}
+
+/*
+ * Replace the page in the mapping.
+ *
+ * The number of remaining references must be:
+ * 1 for anonymous pages without a mapping
+ * 2 for pages with a mapping
+ * 3 for pages with a mapping and PagePrivate/PagePrivate2 set.
+ */
+
+static int exchange_page_move_mapping(struct address_space *to_mapping,
+			struct address_space *from_mapping,
+			struct page *to_page, struct page *from_page,
+			struct buffer_head *to_head, struct buffer_head *from_head,
+			enum migrate_mode mode,
+			int to_extra_count, int from_extra_count)
+{
+	int to_expected_count = 1 + to_extra_count,
+		from_expected_count = 1 + from_extra_count;
+	unsigned long from_page_index = from_page->index;
+	unsigned long to_page_index = to_page->index;
+	int to_swapbacked = PageSwapBacked(to_page),
+		from_swapbacked = PageSwapBacked(from_page);
+	struct address_space *to_mapping_value = to_page->mapping,
+						 *from_mapping_value = from_page->mapping;
+
+	VM_BUG_ON_PAGE(to_mapping != page_mapping(to_page), to_page);
+	VM_BUG_ON_PAGE(from_mapping != page_mapping(from_page), from_page);
+	VM_BUG_ON(PageCompound(from_page) != PageCompound(to_page));
+
+	if (!to_mapping) {
+		/* Anonymous page without mapping */
+		if (page_count(to_page) != to_expected_count)
+			return -EAGAIN;
+	}
+
+	if (!from_mapping) {
+		/* Anonymous page without mapping */
+		if (page_count(from_page) != from_expected_count)
+			return -EAGAIN;
+	}
+
+	/* both are anonymous pages  */
+	if (!from_mapping && !to_mapping) {
+		/* from_page  */
+		from_page->index = to_page_index;
+		from_page->mapping = to_mapping_value;
+
+		ClearPageSwapBacked(from_page);
+		if (to_swapbacked)
+			SetPageSwapBacked(from_page);
+
+
+		/* to_page  */
+		to_page->index = from_page_index;
+		to_page->mapping = from_mapping_value;
+
+		ClearPageSwapBacked(to_page);
+		if (from_swapbacked)
+			SetPageSwapBacked(to_page);
+	} else if (!from_mapping && to_mapping) { /* from is anonymous, to is file-backed  */
+		struct zone *from_zone, *to_zone;
+		void **to_pslot;
+		int dirty;
+
+		from_zone = page_zone(from_page);
+		to_zone = page_zone(to_page);
+
+		spin_lock_irq(&to_mapping->i_pages.xa_lock);
+
+		to_pslot = radix_tree_lookup_slot(&to_mapping->i_pages, page_index(to_page));
+
+		to_expected_count += 1 + page_has_private(to_page);
+		if (page_count(to_page) != to_expected_count ||
+			radix_tree_deref_slot_protected(to_pslot, &to_mapping->i_pages.xa_lock)
+			!= to_page) {
+			spin_unlock_irq(&to_mapping->i_pages.xa_lock);
+			return -EAGAIN;
+		}
+
+		if (!page_ref_freeze(to_page, to_expected_count)) {
+			spin_unlock_irq(&to_mapping->i_pages.xa_lock);
+			pr_debug("cannot freeze page count\n");
+			return -EAGAIN;
+		}
+
+		if (((mode & MIGRATETYPE_MASK) == MIGRATE_ASYNC) && to_head &&
+				!buffer_migrate_lock_buffers(to_head, mode)) {
+			page_ref_unfreeze(to_page, to_expected_count);
+			spin_unlock_irq(&to_mapping->i_pages.xa_lock);
+
+			pr_debug("cannot lock buffer head\n");
+			return -EAGAIN;
+		}
+
+		/*
+		 * Now we know that no one else is looking at the page:
+		 * no turning back from here.
+		 */
+		ClearPageSwapBacked(from_page);
+		ClearPageSwapBacked(to_page);
+
+		/* from_page  */
+		from_page->index = to_page_index;
+		from_page->mapping = to_mapping_value;
+		/* to_page  */
+		to_page->index = from_page_index;
+		to_page->mapping = from_mapping_value;
+
+		get_page(from_page); /* add cache reference  */
+		if (to_swapbacked)
+			__SetPageSwapBacked(from_page);
+		else
+			VM_BUG_ON_PAGE(PageSwapCache(to_page), to_page);
+
+		if (from_swapbacked)
+			__SetPageSwapBacked(to_page);
+		else
+			VM_BUG_ON_PAGE(PageSwapCache(from_page), from_page);
+
+		dirty = PageDirty(to_page);
+
+		radix_tree_replace_slot(&to_mapping->i_pages, to_pslot, from_page);
+
+		/* drop cache reference */
+		page_ref_unfreeze(to_page, to_expected_count - 1);
+
+		spin_unlock(&to_mapping->i_pages.xa_lock);
+
+		/*
+		 * If moved to a different zone then also account
+		 * the page for that zone. Other VM counters will be
+		 * taken care of when we establish references to the
+		 * new page and drop references to the old page.
+		 *
+		 * Note that anonymous pages are accounted for
+		 * via NR_FILE_PAGES and NR_ANON_MAPPED if they
+		 * are mapped to swap space.
+		 */
+		if (to_zone != from_zone) {
+			__dec_node_state(to_zone->zone_pgdat, NR_FILE_PAGES);
+			__inc_node_state(from_zone->zone_pgdat, NR_FILE_PAGES);
+			if (PageSwapBacked(to_page) && !PageSwapCache(to_page)) {
+				__dec_node_state(to_zone->zone_pgdat, NR_SHMEM);
+				__inc_node_state(from_zone->zone_pgdat, NR_SHMEM);
+			}
+			if (dirty && mapping_cap_account_dirty(to_mapping)) {
+				__dec_node_state(to_zone->zone_pgdat, NR_FILE_DIRTY);
+				__dec_zone_state(to_zone, NR_ZONE_WRITE_PENDING);
+				__inc_node_state(from_zone->zone_pgdat, NR_FILE_DIRTY);
+				__inc_zone_state(from_zone, NR_ZONE_WRITE_PENDING);
+			}
+		}
+		local_irq_enable();
+
+	} else {
+		/* from is file-backed to is anonymous: fold this to the case above */
+		/* both are file-backed  */
+		BUG();
+	}
+	return MIGRATEPAGE_SUCCESS;
+}
+
+static int exchange_from_to_pages(struct page *to_page, struct page *from_page,
+				enum migrate_mode mode)
+{
+	int rc = -EBUSY;
+	struct address_space *to_page_mapping, *from_page_mapping;
+	struct buffer_head *to_head = NULL, *to_bh = NULL;
+#ifdef DEBUG
+	u64 timestamp;
+#endif
+
+	VM_BUG_ON_PAGE(!PageLocked(from_page), from_page);
+	VM_BUG_ON_PAGE(!PageLocked(to_page), to_page);
+
+	/* copy page->mapping not use page_mapping()  */
+	to_page_mapping = page_mapping(to_page);
+	from_page_mapping = page_mapping(from_page);
+
+	/* from_page has to be anonymous page  */
+	BUG_ON(from_page_mapping);
+	BUG_ON(PageWriteback(from_page));
+	/* writeback has to finish */
+	BUG_ON(PageWriteback(to_page));
+
+	pr_dump_page(from_page, "exchange anonymous page: from ");
+
+	/* to_page is anonymous  */
+	if (!to_page_mapping) {
+		pr_dump_page(to_page, "exchange anonymous page: to ");
+exchange_mappings:
+		START_TIME(timestamp);
+		/* actual page mapping exchange */
+		rc = exchange_page_move_mapping(to_page_mapping, from_page_mapping,
+							to_page, from_page, NULL, NULL, mode, 0, 0);
+		END_TIME("exchange_page_move_mapping", timestamp);
+	} else {
+		if (to_page_mapping->a_ops->migratepage == buffer_migrate_page) {
+
+			pr_dump_page(to_page, "exchange has migratepage: to ");
+
+			if (!page_has_buffers(to_page))
+				goto exchange_mappings;
+
+			to_head = page_buffers(to_page);
+
+			rc = exchange_page_move_mapping(to_page_mapping,
+					from_page_mapping, to_page, from_page,
+					to_head, NULL, mode, 0, 0);
+
+			if (rc != MIGRATEPAGE_SUCCESS)
+				return rc;
+
+			/*
+			 * In the async case, migrate_page_move_mapping locked the buffers
+			 * with an IRQ-safe spinlock held. In the sync case, the buffers
+			 * need to be locked now
+			 */
+			if ((mode & MIGRATETYPE_MASK) != MIGRATE_ASYNC)
+				BUG_ON(!buffer_migrate_lock_buffers(to_head, mode));
+
+			ClearPagePrivate(to_page);
+			set_page_private(from_page, page_private(to_page));
+			set_page_private(to_page, 0);
+			/* transfer private page count  */
+			put_page(to_page);
+			get_page(from_page);
+
+			to_bh = to_head;
+			do {
+				set_bh_page(to_bh, from_page, bh_offset(to_bh));
+				to_bh = to_bh->b_this_page;
+
+			} while (to_bh != to_head);
+
+			SetPagePrivate(from_page);
+
+			to_bh = to_head;
+		} else if (!to_page_mapping->a_ops->migratepage) {
+			/* fallback_migrate_page  */
+			pr_dump_page(to_page, "exchange no migratepage: to ");
+
+			if (PageDirty(to_page)) {
+				if ((mode & MIGRATETYPE_MASK) != MIGRATE_SYNC)
+					return -EBUSY;
+				return writeout(to_page_mapping, to_page);
+			}
+			if (page_has_private(to_page) &&
+				!try_to_release_page(to_page, GFP_KERNEL))
+				return -EAGAIN;
+
+			goto exchange_mappings;
+		}
+	}
+
+
+	/* actual page data exchange  */
+	if (rc != MIGRATEPAGE_SUCCESS)
+		return rc;
+
+	rc = -EFAULT;
+
+	START_TIME(timestamp);
+	if (mode & MIGRATE_MT)
+		rc = exchange_page_mthread(to_page, from_page,
+				hpage_nr_pages(from_page));
+	if (rc) {
+		if (PageHuge(from_page) || PageTransHuge(from_page))
+			exchange_huge_page(to_page, from_page);
+		else
+			exchange_highpage(to_page, from_page);
+		rc = 0;
+	}
+	END_TIME("exchange_highpage", timestamp);
+
+	/*
+	 * 1. buffer_migrate_page:
+	 *   private flag should be transferred from to_page to from_page
+	 *
+	 * 2. anon<->anon, fallback_migrate_page:
+	 *   both have none private flags or to_page's is cleared.
+	 * */
+	VM_BUG_ON(!((page_has_private(from_page) && !page_has_private(to_page)) ||
+				(!page_has_private(from_page) && !page_has_private(to_page))));
+
+	// START_TIME(timestamp);
+	exchange_page_flags(to_page, from_page);
+	// END_TIME("exchange_page_flags", timestamp);
+
+
+	pr_dump_page(from_page, "after exchange: from ");
+	pr_dump_page(to_page, "after exchange: to ");
+
+	if (to_bh) {
+		VM_BUG_ON(to_bh != to_head);
+		do {
+			unlock_buffer(to_bh);
+			put_bh(to_bh);
+			to_bh = to_bh->b_this_page;
+
+		} while (to_bh != to_head);
+	}
+
+	return rc;
+}
+
+static int unmap_and_exchange(struct page *from_page,
+		struct page *to_page, enum migrate_mode mode)
+{
+	int rc = -EAGAIN;
+	struct anon_vma *from_anon_vma = NULL;
+	struct anon_vma *to_anon_vma = NULL;
+	/*bool is_from_lru = !__PageMovable(from_page);*/
+	/*bool is_to_lru = !__PageMovable(to_page);*/
+	int from_page_was_mapped = 0;
+	int to_page_was_mapped = 0;
+	int from_page_count = 0, to_page_count = 0;
+	int from_map_count = 0, to_map_count = 0;
+	unsigned long from_flags, to_flags;
+	pgoff_t from_index, to_index;
+	struct address_space *from_mapping, *to_mapping;
+#ifdef DEBUG
+	u64 timestamp;
+#endif
+
+	//START_TIME(timestamp);
+	if (!trylock_page(from_page)) {
+		if ((mode & MIGRATETYPE_MASK) == MIGRATE_ASYNC)
+			goto out;
+		lock_page(from_page);
+	}
+
+	if (!trylock_page(to_page)) {
+		if ((mode & MIGRATETYPE_MASK) == MIGRATE_ASYNC)
+			goto out;
+		lock_page(to_page);
+	}
+	//END_TIME("trylock_page*2", timestamp);
+
+
+	/* from_page is supposed to be an anonymous page */
+	VM_BUG_ON_PAGE(PageWriteback(from_page), from_page);
+
+	if (PageWriteback(to_page)) {
+		/*
+		 * Only in the case of a full synchronous migration is it
+		 * necessary to wait for PageWriteback. In the async case,
+		 * the retry loop is too short and in the sync-light case,
+		 * the overhead of stalling is too much
+		 */
+		if ((mode & MIGRATETYPE_MASK) != MIGRATE_SYNC) {
+			rc = -EBUSY;
+			goto out_unlock;
+		}
+		wait_on_page_writeback(to_page);
+	}
+
+	/*
+	 * By try_to_unmap(), page->mapcount goes down to 0 here. In this case,
+	 * we cannot notice that anon_vma is freed while we migrates a page.
+	 * This get_anon_vma() delays freeing anon_vma pointer until the end
+	 * of migration. File cache pages are no problem because of page_lock()
+	 * File Caches may use write_page() or lock_page() in migration, then,
+	 * just care Anon page here.
+	 *
+	 * Only page_get_anon_vma() understands the subtleties of
+	 * getting a hold on an anon_vma from outside one of its mms.
+	 * But if we cannot get anon_vma, then we won't need it anyway,
+	 * because that implies that the anon page is no longer mapped
+	 * (and cannot be remapped so long as we hold the page lock).
+	 */
+	//START_TIME(timestamp);
+	if (PageAnon(from_page) && !PageKsm(from_page))
+		from_anon_vma = page_get_anon_vma(from_page);
+
+	if (PageAnon(to_page) && !PageKsm(to_page))
+		to_anon_vma = page_get_anon_vma(to_page);
+	//END_TIME("page_get_anon_vma*2", timestamp);
+
+	/*if (unlikely(!is_from_lru)) {*/
+		/*VM_BUG_ON_PAGE(1, from_page);*/
+		/*goto out_unlock_both;*/
+	/*}*/
+
+	/*if (unlikely(!is_to_lru)) {*/
+		/*pr_debug("exchange non-lru to_page\n");*/
+		/*goto out_unlock_both;*/
+	/*}*/
+
+	from_page_count = page_count(from_page);
+	from_map_count = page_mapcount(from_page);
+	to_page_count = page_count(to_page);
+	to_map_count = page_mapcount(to_page);
+	from_flags = from_page->flags;
+	to_flags = to_page->flags;
+	from_mapping = from_page->mapping;
+	to_mapping = to_page->mapping;
+	from_index = from_page->index;
+	to_index = to_page->index;
+
+	START_TIME(timestamp);
+	/*
+	 * Corner case handling:
+	 * 1. When a new swap-cache page is read into, it is added to the LRU
+	 * and treated as swapcache but it has no rmap yet.
+	 * Calling try_to_unmap() against a page->mapping==NULL page will
+	 * trigger a BUG.  So handle it here.
+	 * 2. An orphaned page (see truncate_complete_page) might have
+	 * fs-private metadata. The page can be picked up due to memory
+	 * offlining.  Everywhere else except page reclaim, the page is
+	 * invisible to the vm, so the page can not be migrated.  So try to
+	 * free the metadata, so the page can be freed.
+	 */
+	if (!from_page->mapping) {
+		VM_BUG_ON_PAGE(PageAnon(from_page), from_page);
+		if (page_has_private(from_page)) {
+			try_to_free_buffers(from_page);
+			goto out_unlock_both;
+		}
+	} else if (page_mapped(from_page)) {
+		/* Establish migration ptes */
+		VM_BUG_ON_PAGE(PageAnon(from_page) && !PageKsm(from_page) &&
+					   !from_anon_vma, from_page);
+		try_to_unmap(from_page,
+			TTU_MIGRATION|TTU_IGNORE_MLOCK|TTU_IGNORE_ACCESS);
+		from_page_was_mapped = 1;
+	}
+	END_TIME("try_to_unmap from_page", timestamp);
+
+	START_TIME(timestamp);
+	if (!to_page->mapping) {
+		VM_BUG_ON_PAGE(PageAnon(to_page), to_page);
+		if (page_has_private(to_page)) {
+			try_to_free_buffers(to_page);
+			goto out_unlock_both_remove_from_migration_pte;
+		}
+	} else if (page_mapped(to_page)) {
+		/* Establish migration ptes */
+		VM_BUG_ON_PAGE(PageAnon(to_page) && !PageKsm(to_page) &&
+						!to_anon_vma, to_page);
+		try_to_unmap(to_page,
+			TTU_MIGRATION|TTU_IGNORE_MLOCK|TTU_IGNORE_ACCESS);
+		to_page_was_mapped = 1;
+	}
+	END_TIME("try_to_unmap to_page", timestamp);
+
+
+
+	//START_TIME(timestamp);
+	if (!page_mapped(from_page) && !page_mapped(to_page)) {
+		rc = exchange_from_to_pages(to_page, from_page, mode);
+		pr_debug("exchange_from_to_pages from: %lx, to %lx: %d\n", page_to_pfn(from_page), page_to_pfn(to_page), rc);
+	}
+	//END_TIME("exchange_from_to_pages",timestamp);
+
+
+	START_TIME(timestamp);
+	/* In remove_migration_ptes(), page_walk_vma() assumes
+	 * from_page and to_page have the same index.
+	 * Thus, we restore old_page->index here.
+	 * Here to_page is the old_page.
+	 */
+	if (to_page_was_mapped) {
+		if (rc == MIGRATEPAGE_SUCCESS)
+			swap(to_page->index, to_index);
+
+		remove_migration_ptes(to_page,
+			rc == MIGRATEPAGE_SUCCESS ? from_page : to_page, false);
+
+		if (rc == MIGRATEPAGE_SUCCESS)
+			swap(to_page->index, to_index);
+	}
+
+out_unlock_both_remove_from_migration_pte:
+	if (from_page_was_mapped) {
+		if (rc == MIGRATEPAGE_SUCCESS)
+			swap(from_page->index, from_index);
+
+		remove_migration_ptes(from_page,
+			rc == MIGRATEPAGE_SUCCESS ? to_page : from_page, false);
+
+		if (rc == MIGRATEPAGE_SUCCESS)
+			swap(from_page->index, from_index);
+	}
+	END_TIME("remove_migration_ptes*2", timestamp);
+
+
+
+	if (rc == MIGRATEPAGE_SUCCESS) {
+		if (from_page_count != page_count(to_page) ||
+			to_page_count != page_count(from_page) ||
+			from_map_count != page_mapcount(to_page) ||
+			to_map_count != page_mapcount(from_page)) {
+
+			if (page_mapping(from_page) &&
+				!page_mapping(from_page)->a_ops->migratepage &&
+				to_page_count == page_count(from_page) + 1 &&
+				to_map_count == page_mapcount(from_page) &&
+				from_page_count == page_count(to_page) &&
+				from_map_count == page_mapcount(to_page)) {
+
+			} else if ((PageWaiters(from_page)?
+				to_page_count < page_count(from_page):
+				to_page_count == page_count(from_page))&&
+				to_map_count == page_mapcount(from_page) &&
+
+				(PageWaiters(to_page)?
+				from_page_count < page_count(to_page):
+				from_page_count == page_count(to_page) )&&
+				from_map_count == page_mapcount(to_page)) {
+			} else {
+
+
+			pr_debug("anon<->file: from_page_was_mapped: %d, to_page_was_mapped: %d\n",
+				from_page_was_mapped, to_page_was_mapped);
+			pr_debug("before: from_page_count: %d, from_map_count: %d, from_flags: %#lx(%pGp), from_mapping: %p, "
+				"to_page_count: %d, to_map_count: %d, to_flags: %#lx(%pGp), to_mapping: %p\n",
+				from_page_count, from_map_count, from_flags, &from_flags, from_mapping,
+				to_page_count, to_map_count, to_flags, &to_flags, to_mapping);
+
+
+			pr_dump_page(from_page, "after exchange: from");
+			pr_dump_page(to_page, "after exchange: to");
+			}
+		}
+	} else {
+		if (from_page_count != page_count(from_page) ||
+			to_page_count != page_count(to_page) ||
+			from_map_count != page_mapcount(from_page) ||
+			to_map_count != page_mapcount(to_page)) {
+
+			if (page_mapping(to_page) &&
+				!page_mapping(to_page)->a_ops->migratepage &&
+				to_page_count == page_count(to_page) + 1 &&
+				to_map_count == page_mapcount(to_page) &&
+				from_page_count == page_count(from_page) &&
+				from_map_count == page_mapcount(from_page)) {
+
+			} else if ((PageWaiters(to_page)?
+				to_page_count < page_count(to_page):
+				to_page_count == page_count(to_page) )&&
+				to_map_count == page_mapcount(to_page) &&
+
+				(PageWaiters(from_page)?
+				from_page_count < page_count(from_page):
+				from_page_count == page_count(from_page) )&&
+				from_map_count == page_mapcount(from_page)) {
+			} else {
+			pr_debug("anon<->file: from_page_was_mapped: %d, to_page_was_mapped: %d, rc: %d\n",
+				from_page_was_mapped, to_page_was_mapped, rc);
+			pr_debug("before: from_page_count: %d, from_map_count: %d, from_flags: %#lx(%pGp), from_mapping: %p, "
+				"to_page_count: %d, to_map_count: %d, to_flags: %#lx(%pGp), to_mapping: %p\n",
+				from_page_count, from_map_count, from_flags, &from_flags, from_mapping,
+				to_page_count, to_map_count, to_flags, &to_flags, to_mapping);
+
+
+			pr_dump_page(from_page, "exchange failed: from");
+			pr_dump_page(to_page, "exchange failed: to");
+			}
+		}
+	}
+out_unlock_both:
+	//START_TIME(timestamp);
+	if (to_anon_vma)
+		put_anon_vma(to_anon_vma);
+	unlock_page(to_page);
+	//END_TIME("put_anon_vma(to_anon_vma)", timestamp);
+out_unlock:
+	//START_TIME(timestamp);
+	/* Drop an anon_vma reference if we took one */
+	if (from_anon_vma)
+		put_anon_vma(from_anon_vma);
+	unlock_page(from_page);
+	//END_TIME("put_anon_vma(from_anon_vma)", timestamp);
+out:
+	return rc;
+}
+
+static bool can_be_exchanged(struct page *from, struct page *to)
+{
+	if (PageCompound(from) != PageCompound(to))
+		return false;
+
+	if (PageHuge(from) != PageHuge(to))
+		return false;
+
+	if (PageHuge(from) || PageHuge(to))
+		return false;
+
+	if (compound_order(from) != compound_order(to))
+		return false;
+
+	return true;
+}
+
+/*
+ * Exchange pages in the exchange_list
+ *
+ * Caller should release the exchange_list resource.
+ *
+ * */
+int exchange_pages(struct list_head *exchange_list,
+			enum migrate_mode mode)
+{
+	struct exchange_page_info *one_pair, *one_pair2;
+	int failed = 0;
+
+	list_for_each_entry_safe(one_pair, one_pair2, exchange_list, list) {
+		struct page *from_page = one_pair->from_page;
+		struct page *to_page = one_pair->to_page;
+		int rc = -EBUSY;
+		int retry = 0;
+#ifdef DEBUG
+		u64 timestamp;
+#endif
+
+again:
+		if (page_count(from_page) == 1) {
+			/* page was freed from under us. So we are done  */
+			ClearPageActive(from_page);
+			ClearPageUnevictable(from_page);
+
+			put_page(from_page);
+			dec_node_page_state(from_page, NR_ISOLATED_ANON +
+					page_is_file_cache(from_page));
+
+			if (page_count(to_page) == 1) {
+				ClearPageActive(to_page);
+				ClearPageUnevictable(to_page);
+				put_page(to_page);
+			} else
+				goto putback_to_page;
+
+			continue;
+		}
+
+		if (page_count(to_page) == 1) {
+			/* page was freed from under us. So we are done  */
+			ClearPageActive(to_page);
+			ClearPageUnevictable(to_page);
+
+			put_page(to_page);
+			dec_node_page_state(to_page, NR_ISOLATED_ANON +
+					page_is_file_cache(to_page));
+
+			continue;
+		}
+
+		/* TODO: compound page not supported */
+		if (!can_be_exchanged(from_page, to_page) ||
+			page_mapping(from_page)
+			/* allow to_page to be file-backed page  */
+			/*|| page_mapping(to_page)*/
+			) {
+			++failed;
+			goto putback;
+		}
+
+		//START_TIME(timestamp);
+		rc = unmap_and_exchange(from_page, to_page, mode);
+		//END_TIME("unmap_and_exchange", timestamp);
+
+		if (rc == -EAGAIN && retry < 3) {
+			++retry;
+			goto again;
+		}
+
+		if (rc != MIGRATEPAGE_SUCCESS) {
+			++failed;
+			trace_dump_page(from_page, "from_page");
+			trace_dump_page(to_page, "to_page");
+			count_vm_event(PGEXCHANGE_FAIL);
+		}
+
+		if (rc == MIGRATEPAGE_SUCCESS) {
+			count_vm_events(PGEXCHANGE_SUCCESS, hpage_nr_pages(from_page));
+			count_vm_events(PGEXCHANGE_SUCCESS, hpage_nr_pages(to_page));
+		}
+
+		START_TIME(timestamp);
+putback:
+		dec_node_page_state(from_page, NR_ISOLATED_ANON +
+				page_is_file_cache(from_page));
+
+		putback_lru_page(from_page);
+
+putback_to_page:
+		/*if (!__PageMovable(to_page)) {*/
+		dec_node_page_state(to_page, NR_ISOLATED_ANON +
+				page_is_file_cache(to_page));
+
+		putback_lru_page(to_page);
+		/*} else {*/
+			/*putback_movable_page(to_page);*/
+		/*}*/
+		END_TIME("putback_lru_page*2", timestamp);
+	}
+	return failed;
+}
+
+
+/* page1's ref_cout: 2, page2's ref_cout: 1 */
+int exchange_two_pages(struct page *page1, struct page *page2,
+				enum migrate_mode mode)
+{
+	struct exchange_page_info page_info;
+	LIST_HEAD(exchange_list);
+	int err = -EFAULT;
+	int pagevec_flushed = 0;
+	int ret;
+#ifdef DEBUG
+	u64 last_timestamp;
+#endif
+	int migrate_concur =  mode & MIGRATE_CONCUR;
+
+	if (!(PageLRU(page1) && PageLRU(page2)))
+		return -EBUSY;
+
+	if (page_count(page1) != 2 || page_count(page2) != 1)
+		return -EBUSY;
+
+retry_isolate1:
+	if (!get_page_unless_zero(page1))
+		return -EBUSY;
+	err = isolate_lru_page(page1);
+	if (err) {
+		if (!pagevec_flushed) {
+			migrate_prep();
+			pagevec_flushed = 1;
+			goto retry_isolate1;
+		}
+		put_page(page1); // get_page_unless_zero()
+		return err;
+	} else {
+		put_page(page1); // get_page_unless_zero()
+		put_page(page1); // isolate_lru_page()
+	}
+	inc_node_page_state(page1,
+			NR_ISOLATED_ANON + page_is_file_cache(page1));
+
+retry_isolate2:
+	if (!get_page_unless_zero(page2)) {
+		dec_node_page_state(page1,
+				NR_ISOLATED_ANON + page_is_file_cache(page1));
+		get_page(page1);
+		putback_lru_page(page1);
+		return -EBUSY;
+	}
+	err = isolate_lru_page(page2);
+	if (err) {
+		if (!pagevec_flushed) {
+			migrate_prep();
+			pagevec_flushed = 1;
+			goto retry_isolate2;
+		}
+		dec_node_page_state(page1,
+				NR_ISOLATED_ANON + page_is_file_cache(page1));
+		get_page(page1);
+		putback_lru_page(page1);
+
+		put_page(page2);
+		return err;
+	} else {
+		put_page(page2);
+	}
+	inc_node_page_state(page2,
+			NR_ISOLATED_ANON + page_is_file_cache(page2));
+
+	VM_BUG_ON_PAGE(PageTail(page1), page1);
+	VM_BUG_ON_PAGE(PageTail(page2), page2);
+
+	page_info.from_page = page1;
+	page_info.to_page = page2;
+	INIT_LIST_HEAD(&page_info.list);
+	list_add(&page_info.list, &exchange_list);
+
+	//START_TIME(last_timestamp);
+	if (migrate_concur)
+		ret = exchange_pages_concur(&exchange_list, mode);
+	else
+		ret = exchange_pages(&exchange_list, mode);
+
+	//END_TIME("exchange_pages", last_timestamp);
+	return ret;
+}
+
+static int unmap_pair_pages_concur(struct exchange_page_info *one_pair,
+				int force, enum migrate_mode mode)
+{
+	int rc = -EAGAIN;
+	struct anon_vma *anon_vma_from_page = NULL, *anon_vma_to_page = NULL;
+	struct page *from_page = one_pair->from_page;
+	struct page *to_page = one_pair->to_page;
+#ifdef CONFIG_PAGE_MIGRATION_PROFILE
+	u64 timestamp;
+#endif
+
+	one_pair->from_index = from_page->index;
+	one_pair->to_index = to_page->index;
+	/* from_page lock down  */
+	if (!trylock_page(from_page)) {
+		if (!force || ((mode & MIGRATE_MODE_MASK) == MIGRATE_ASYNC))
+			goto out;
+
+		lock_page(from_page);
+	}
+
+	BUG_ON(PageWriteback(from_page));
+
+	/*
+	 * By try_to_unmap(), page->mapcount goes down to 0 here. In this case,
+	 * we cannot notice that anon_vma is freed while we migrates a page.
+	 * This get_anon_vma() delays freeing anon_vma pointer until the end
+	 * of migration. File cache pages are no problem because of page_lock()
+	 * File Caches may use write_page() or lock_page() in migration, then,
+	 * just care Anon page here.
+	 *
+	 * Only page_get_anon_vma() understands the subtleties of
+	 * getting a hold on an anon_vma from outside one of its mms.
+	 * But if we cannot get anon_vma, then we won't need it anyway,
+	 * because that implies that the anon page is no longer mapped
+	 * (and cannot be remapped so long as we hold the page lock).
+	 */
+	if (PageAnon(from_page) && !PageKsm(from_page))
+		one_pair->from_anon_vma = anon_vma_from_page
+					= page_get_anon_vma(from_page);
+
+	/* to_page lock down  */
+	if (!trylock_page(to_page)) {
+		if (!force || ((mode & MIGRATE_MODE_MASK) == MIGRATE_ASYNC))
+			goto out_unlock;
+
+		lock_page(to_page);
+	}
+
+#ifdef CONFIG_PAGE_MIGRATION_PROFILE
+	timestamp = rdtsc();
+	current->move_pages_breakdown.lock_page_cycles += timestamp -
+		current->move_pages_breakdown.last_timestamp;
+	current->move_pages_breakdown.last_timestamp = timestamp;
+#endif
+
+	BUG_ON(PageWriteback(to_page));
+
+	/*
+	 * By try_to_unmap(), page->mapcount goes down to 0 here. In this case,
+	 * we cannot notice that anon_vma is freed while we migrates a page.
+	 * This get_anon_vma() delays freeing anon_vma pointer until the end
+	 * of migration. File cache pages are no problem because of page_lock()
+	 * File Caches may use write_page() or lock_page() in migration, then,
+	 * just care Anon page here.
+	 *
+	 * Only page_get_anon_vma() understands the subtleties of
+	 * getting a hold on an anon_vma from outside one of its mms.
+	 * But if we cannot get anon_vma, then we won't need it anyway,
+	 * because that implies that the anon page is no longer mapped
+	 * (and cannot be remapped so long as we hold the page lock).
+	 */
+	if (PageAnon(to_page) && !PageKsm(to_page))
+		one_pair->to_anon_vma = anon_vma_to_page = page_get_anon_vma(to_page);
+
+	/*
+	 * Corner case handling:
+	 * 1. When a new swap-cache page is read into, it is added to the LRU
+	 * and treated as swapcache but it has no rmap yet.
+	 * Calling try_to_unmap() against a page->mapping==NULL page will
+	 * trigger a BUG.  So handle it here.
+	 * 2. An orphaned page (see truncate_complete_page) might have
+	 * fs-private metadata. The page can be picked up due to memory
+	 * offlining.  Everywhere else except page reclaim, the page is
+	 * invisible to the vm, so the page can not be migrated.  So try to
+	 * free the metadata, so the page can be freed.
+	 */
+	if (!from_page->mapping) {
+		VM_BUG_ON_PAGE(PageAnon(from_page), from_page);
+		if (page_has_private(from_page)) {
+			try_to_free_buffers(from_page);
+			goto out_unlock_both;
+		}
+	} else if (page_mapped(from_page)) {
+		/* Establish migration ptes */
+		VM_BUG_ON_PAGE(PageAnon(from_page) && !PageKsm(from_page) &&
+					   !anon_vma_from_page, from_page);
+		try_to_unmap(from_page,
+			TTU_MIGRATION|TTU_IGNORE_MLOCK|TTU_IGNORE_ACCESS);
+
+		one_pair->from_page_was_mapped = 1;
+	}
+
+	if (!to_page->mapping) {
+		VM_BUG_ON_PAGE(PageAnon(to_page), to_page);
+		if (page_has_private(to_page)) {
+			try_to_free_buffers(to_page);
+			goto out_unlock_both;
+		}
+	} else if (page_mapped(to_page)) {
+		/* Establish migration ptes */
+		VM_BUG_ON_PAGE(PageAnon(to_page) && !PageKsm(to_page) &&
+					   !anon_vma_to_page, to_page);
+		try_to_unmap(to_page,
+			TTU_MIGRATION|TTU_IGNORE_MLOCK|TTU_IGNORE_ACCESS);
+
+		one_pair->to_page_was_mapped = 1;
+	}
+
+	return MIGRATEPAGE_SUCCESS;
+
+out_unlock_both:
+	if (anon_vma_to_page)
+		put_anon_vma(anon_vma_to_page);
+	unlock_page(to_page);
+out_unlock:
+	/* Drop an anon_vma reference if we took one */
+	if (anon_vma_from_page)
+		put_anon_vma(anon_vma_from_page);
+	unlock_page(from_page);
+out:
+
+	return rc;
+}
+
+static int exchange_page_mapping_concur(struct list_head *unmapped_list_ptr,
+					   struct list_head *exchange_list_ptr,
+						enum migrate_mode mode)
+{
+	int nr_failed = 0;
+	struct address_space *to_page_mapping, *from_page_mapping;
+	struct exchange_page_info *one_pair, *one_pair2;
+
+	list_for_each_entry_safe(one_pair, one_pair2, unmapped_list_ptr, list) {
+		struct page *from_page = one_pair->from_page;
+		struct page *to_page = one_pair->to_page;
+		int rc = -EBUSY;
+
+		VM_BUG_ON_PAGE(!PageLocked(from_page), from_page);
+		VM_BUG_ON_PAGE(!PageLocked(to_page), to_page);
+
+		/* copy page->mapping not use page_mapping()  */
+		to_page_mapping = page_mapping(to_page);
+		from_page_mapping = page_mapping(from_page);
+
+		BUG_ON(from_page_mapping);
+		BUG_ON(to_page_mapping);
+
+		BUG_ON(PageWriteback(from_page));
+		BUG_ON(PageWriteback(to_page));
+
+		/* actual page mapping exchange */
+		if (!page_mapped(from_page) && !page_mapped(to_page))
+			rc = exchange_page_move_mapping(to_page_mapping, from_page_mapping,
+								to_page, from_page, NULL, NULL, mode, 0, 0);
+
+		if (rc) {
+			if (one_pair->from_page_was_mapped)
+				remove_migration_ptes(from_page, from_page, false);
+			if (one_pair->to_page_was_mapped)
+				remove_migration_ptes(to_page, to_page, false);
+
+			if (one_pair->from_anon_vma)
+				put_anon_vma(one_pair->from_anon_vma);
+			unlock_page(from_page);
+
+			if (one_pair->to_anon_vma)
+				put_anon_vma(one_pair->to_anon_vma);
+			unlock_page(to_page);
+
+			mod_node_page_state(page_pgdat(from_page), NR_ISOLATED_ANON +
+					page_is_file_cache(from_page), -hpage_nr_pages(from_page));
+			putback_lru_page(from_page);
+
+			mod_node_page_state(page_pgdat(to_page), NR_ISOLATED_ANON +
+					page_is_file_cache(to_page), -hpage_nr_pages(to_page));
+			putback_lru_page(to_page);
+
+			one_pair->from_page = NULL;
+			one_pair->to_page = NULL;
+
+			list_del(&one_pair->list);
+			++nr_failed;
+		}
+	}
+
+	return nr_failed;
+}
+
+static int exchange_page_data_concur(struct list_head *unmapped_list_ptr,
+									enum migrate_mode mode)
+{
+	struct exchange_page_info *one_pair;
+	int num_pages = 0, idx = 0;
+	struct page **src_page_list = NULL, **dst_page_list = NULL;
+	unsigned long size = 0;
+	int rc = -EFAULT;
+#ifdef CONFIG_PAGE_MIGRATION_PROFILE
+	u64 timestamp;
+#endif
+
+	if (list_empty(unmapped_list_ptr))
+		return 0;
+
+	/* form page list  */
+	list_for_each_entry(one_pair, unmapped_list_ptr, list) {
+		++num_pages;
+		size += PAGE_SIZE * hpage_nr_pages(one_pair->from_page);
+	}
+
+	src_page_list = kzalloc(sizeof(struct page *)*num_pages, GFP_KERNEL);
+	if (!src_page_list)
+		return -ENOMEM;
+	dst_page_list = kzalloc(sizeof(struct page *)*num_pages, GFP_KERNEL);
+	if (!dst_page_list)
+		return -ENOMEM;
+
+	list_for_each_entry(one_pair, unmapped_list_ptr, list) {
+		src_page_list[idx] = one_pair->from_page;
+		dst_page_list[idx] = one_pair->to_page;
+		++idx;
+	}
+
+	BUG_ON(idx != num_pages);
+
+#ifdef CONFIG_PAGE_MIGRATION_PROFILE
+	timestamp = rdtsc();
+	current->move_pages_breakdown.change_page_mapping_cycles += timestamp -
+		current->move_pages_breakdown.last_timestamp;
+	current->move_pages_breakdown.last_timestamp = timestamp;
+#endif
+
+	if (mode & MIGRATE_MT)
+		rc = exchange_page_lists_mthread(dst_page_list, src_page_list,
+				num_pages);
+
+	if (rc) {
+		list_for_each_entry(one_pair, unmapped_list_ptr, list) {
+			if (PageHuge(one_pair->from_page) ||
+				PageTransHuge(one_pair->from_page)) {
+				exchange_huge_page(one_pair->to_page, one_pair->from_page);
+			} else {
+				exchange_highpage(one_pair->to_page, one_pair->from_page);
+			}
+		}
+	}
+
+	list_for_each_entry(one_pair, unmapped_list_ptr, list) {
+		exchange_page_flags(one_pair->to_page, one_pair->from_page);
+	}
+
+#ifdef CONFIG_PAGE_MIGRATION_PROFILE
+	timestamp = rdtsc();
+	current->move_pages_breakdown.copy_page_cycles += timestamp -
+		current->move_pages_breakdown.last_timestamp;
+	current->move_pages_breakdown.last_timestamp = timestamp;
+#endif
+
+	kfree(src_page_list);
+	kfree(dst_page_list);
+
+
+	return rc;
+}
+
+static int remove_migration_ptes_concur(struct list_head *unmapped_list_ptr)
+{
+	struct exchange_page_info *iterator;
+	LIST_HEAD(putback_list);
+#ifdef CONFIG_PAGE_MIGRATION_PROFILE
+	u64 timestamp;
+#endif
+#ifdef DEBUG
+	u64 timestamp, last_timestamp, list_timestamp;
+	list_timestamp = last_timestamp = timestamp = rdtsc();
+#endif
+
+	list_for_each_entry(iterator, unmapped_list_ptr, list) {
+		struct page *from_page = iterator->from_page;
+		struct page *to_page = iterator->to_page;
+
+		swap(from_page->index, iterator->from_index);
+		if (iterator->from_page_was_mapped)
+			remove_migration_ptes(iterator->from_page, iterator->to_page, false);
+		swap(from_page->index, iterator->from_index);
+
+		swap(to_page->index, iterator->to_index);
+		if (iterator->to_page_was_mapped)
+			remove_migration_ptes(iterator->to_page, iterator->from_page, false);
+		swap(to_page->index, iterator->to_index);
+#ifdef DEBUG
+		timestamp = rdtsc();
+		trace_printk("remove_migration_ptes timestamp: %llu\n", timestamp - last_timestamp);
+		last_timestamp = timestamp;
+#endif
+
+#ifdef CONFIG_PAGE_MIGRATION_PROFILE
+		timestamp = rdtsc();
+		current->move_pages_breakdown.remove_migration_ptes_cycles += timestamp -
+			current->move_pages_breakdown.last_timestamp;
+		current->move_pages_breakdown.last_timestamp = timestamp;
+#endif
+		dec_node_page_state(iterator->from_page, NR_ISOLATED_ANON +
+				page_is_file_cache(iterator->from_page));
+		count_vm_events(PGEXCHANGE_SUCCESS, hpage_nr_pages(iterator->from_page));
+
+
+		dec_node_page_state(iterator->to_page, NR_ISOLATED_ANON +
+				page_is_file_cache(iterator->to_page));
+		count_vm_events(PGEXCHANGE_SUCCESS, hpage_nr_pages(iterator->to_page));
+
+
+		if (iterator->from_anon_vma)
+			put_anon_vma(iterator->from_anon_vma);
+		unlock_page(iterator->from_page);
+		if (iterator->to_anon_vma)
+			put_anon_vma(iterator->to_anon_vma);
+		unlock_page(iterator->to_page);
+
+		putback_lru_page(iterator->from_page);
+		iterator->from_page = NULL;
+
+
+
+#ifdef CONFIG_PAGE_MIGRATION_PROFILE
+		timestamp = rdtsc();
+		current->move_pages_breakdown.putback_old_page_cycles += timestamp -
+			current->move_pages_breakdown.last_timestamp;
+		current->move_pages_breakdown.last_timestamp = timestamp;
+#endif
+		putback_lru_page(iterator->to_page);
+		iterator->to_page = NULL;
+
+#ifdef DEBUG
+		timestamp = rdtsc();
+		trace_printk("swap page timestamp: %llu\n", timestamp - last_timestamp);
+		last_timestamp = timestamp;
+#endif
+#ifdef CONFIG_PAGE_MIGRATION_PROFILE
+		timestamp = rdtsc();
+		current->move_pages_breakdown.putback_new_page_cycles += timestamp -
+			current->move_pages_breakdown.last_timestamp;
+		current->move_pages_breakdown.last_timestamp = timestamp;
+#endif
+	}
+
+	return 0;
+}
+
+int exchange_pages_concur(struct list_head *exchange_list,
+		enum migrate_mode mode)
+{
+	struct exchange_page_info *one_pair, *one_pair2;
+	int pass = 0;
+	int retry = 1;
+	int nr_failed = 0;
+	int nr_succeeded = 0;
+	int rc = 0;
+	LIST_HEAD(serialized_list);
+	LIST_HEAD(unmapped_list);
+#ifdef CONFIG_PAGE_MIGRATION_PROFILE
+	u64 timestamp;
+
+	timestamp = rdtsc();
+	current->move_pages_breakdown.enter_unmap_and_move_cycles += timestamp -
+		current->move_pages_breakdown.last_timestamp;
+	current->move_pages_breakdown.last_timestamp = timestamp;
+#endif
+
+	for(pass = 0; pass < 1 && retry; pass++) {
+		retry = 0;
+
+		/* unmap and get new page for page_mapping(page) == NULL */
+		list_for_each_entry_safe(one_pair, one_pair2, exchange_list, list) {
+			struct page *from_page = one_pair->from_page;
+			struct page *to_page = one_pair->to_page;
+			cond_resched();
+
+			if (page_count(from_page) == 1) {
+				/* page was freed from under us. So we are done  */
+				ClearPageActive(from_page);
+				ClearPageUnevictable(from_page);
+
+				put_page(from_page);
+				dec_node_page_state(from_page, NR_ISOLATED_ANON +
+						page_is_file_cache(from_page));
+
+				if (page_count(to_page) == 1) {
+					ClearPageActive(to_page);
+					ClearPageUnevictable(to_page);
+					put_page(to_page);
+				} else {
+					mod_node_page_state(page_pgdat(to_page), NR_ISOLATED_ANON +
+							page_is_file_cache(to_page), -hpage_nr_pages(to_page));
+					putback_lru_page(to_page);
+				}
+				list_del(&one_pair->list);
+
+				continue;
+			}
+
+			if (page_count(to_page) == 1) {
+				/* page was freed from under us. So we are done  */
+				ClearPageActive(to_page);
+				ClearPageUnevictable(to_page);
+
+				put_page(to_page);
+
+				dec_node_page_state(to_page, NR_ISOLATED_ANON +
+						page_is_file_cache(to_page));
+
+				mod_node_page_state(page_pgdat(from_page), NR_ISOLATED_ANON +
+						page_is_file_cache(from_page), -hpage_nr_pages(from_page));
+				putback_lru_page(from_page);
+
+				list_del(&one_pair->list);
+				continue;
+			}
+		/* We do not exchange huge pages and file-backed pages concurrently */
+			if (PageHuge(one_pair->from_page) || PageHuge(one_pair->to_page)) {
+				rc = -ENODEV;
+			}
+			else if ((page_mapping(one_pair->from_page) != NULL) ||
+					 (page_mapping(one_pair->from_page) != NULL)) {
+				rc = -ENODEV;
+			}
+			else
+				rc = unmap_pair_pages_concur(one_pair, 1, mode);
+
+			switch(rc) {
+			case -ENODEV:
+				list_move(&one_pair->list, &serialized_list);
+				break;
+			case -ENOMEM:
+				goto out;
+			case -EAGAIN:
+				retry++;
+				break;
+			case MIGRATEPAGE_SUCCESS:
+				list_move(&one_pair->list, &unmapped_list);
+				nr_succeeded++;
+				break;
+			default:
+				/*
+				 * Permanent failure (-EBUSY, -ENOSYS, etc.):
+				 * unlike -EAGAIN case, the failed page is
+				 * removed from migration page list and not
+				 * retried in the next outer loop.
+				 */
+				list_move(&one_pair->list, &serialized_list);
+				nr_failed++;
+				break;
+			}
+		}
+
+#ifdef CONFIG_PAGE_MIGRATION_PROFILE
+		timestamp = rdtsc();
+		current->move_pages_breakdown.unmap_page_cycles += timestamp -
+			current->move_pages_breakdown.last_timestamp;
+		current->move_pages_breakdown.last_timestamp = timestamp;
+#endif
+
+		/* move page->mapping to new page, only -EAGAIN could happen  */
+		exchange_page_mapping_concur(&unmapped_list, exchange_list, mode);
+
+		/* copy pages in unmapped_list */
+		exchange_page_data_concur(&unmapped_list, mode);
+
+		/* remove migration pte, if old_page is NULL?, unlock old and new
+		 * pages, put anon_vma, put old and new pages */
+		remove_migration_ptes_concur(&unmapped_list);
+	}
+
+	nr_failed += retry;
+	rc = nr_failed;
+
+	exchange_pages(&serialized_list, mode);
+out:
+	list_splice(&unmapped_list, exchange_list);
+	list_splice(&serialized_list, exchange_list);
+
+	return nr_failed?-EFAULT:0;
+}
+
+int try_exchange_page(struct page *page, int dst_nid)
+{
+	struct pglist_data *pgdat = NODE_DATA(dst_nid);
+	struct page *dst_page = NULL;
+	struct page_info *pi, *pi2;
+	struct page_ext *page_ext;
+	int src_nid;
+	int last_nid = NUMA_NO_NODE;
+	int scan, nr_to_scan = 16;
+#ifdef DEBUG
+	u64 timestamp;
+#endif
+	enum migrate_mode mode = MIGRATE_SYNC;
+
+	src_nid = page_to_nid(page);
+	scan = 0;
+
+	if (!(sysctl_numa_balancing_extended_mode & NUMA_BALANCING_EXCHANGE))
+		return false;
+
+	spin_lock_irq(&pgdat->lru_lock);
+
+	/* Check that whether page is now exchanging or not*/
+	if (list_empty(&pgdat->deferred_list)) {
+		count_vm_event(PGEXCHANGE_LIST_EMPTY_FAIL);
+		spin_unlock_irq(&pgdat->lru_lock);
+		return false;
+	}
+
+	if (!trylock_busy(page)) {
+		spin_unlock_irq(&pgdat->lru_lock);
+		return false;
+	}
+
+	list_for_each_entry_safe(pi, pi2, &pgdat->deferred_list, list) {
+
+		if (unlikely(scan > nr_to_scan))
+			break;
+
+		dst_page = get_page_from_page_info(pi);
+		if (!dst_page) {
+			count_vm_event(PGEXCHANGE_NO_PAGE_FAIL);
+			trace_printk("exch: pfn:%lu,last_cpu:%d\n", pi->pfn, pi->last_cpu);
+			page_ext = get_page_ext(pi);
+			clear_bit(PAGE_EXT_DEFERRED, &page_ext->flags);
+			clear_bit(PAGE_EXT_BUSY_LOCK, &page_ext->flags);
+			__mod_node_page_state(pgdat, NR_DEFERRED, -1);
+			list_del(&pi->list);
+			goto skip_page;
+		}
+
+		last_nid = cpu_to_node(pi->last_cpu);
+
+		if (last_nid == src_nid) {
+			if ((PageTransHuge(page) ^ PageTransHuge(dst_page)))
+				goto skip_page;
+
+			if (unlikely(!PageLRU(dst_page)))
+				goto skip_page;
+
+			if (unlikely(!trylock_busy(dst_page))) {
+				count_vm_event(PGEXCHANGE_BUSY_FAIL);
+				goto skip_page;
+			}
+
+			del_page_from_deferred_list(dst_page);
+			break;
+		}
+
+		if (last_nid != src_nid) {
+			list_move_tail(&pi->list, &pgdat->deferred_list);
+			count_vm_event(PGEXCHANGE_NODE_UNMATCH_FAIL);
+		}
+skip_page:
+		count_vm_event(NR_PAGE_SKIPPED);
+		scan++;
+	}
+
+	spin_unlock_irq(&pgdat->lru_lock);
+
+	if (unlikely(scan > nr_to_scan)) {
+		count_vm_event(PGEXCHANGE_SCAN_FAIL);
+		goto unlock_src_fail;
+	}
+
+	if (PageTransHuge(page) && PageTransHuge(dst_page)) {
+		// mode |= MIGRATE_CONCUR;
+		mode |= MIGRATE_MT;
+		unlock_page(page);
+	}
+
+	//put_page(page);
+	if (exchange_two_pages(page, dst_page, mode)) {
+
+		if (PageTransHuge(page))
+			lock_page(page);
+
+		//get_page(page);
+		goto unlock_dst_fail;
+	}
+
+	unlock_busy(page);
+	unlock_busy(dst_page);
+	return true;
+
+unlock_dst_fail:
+	unlock_busy(dst_page);
+
+unlock_src_fail:
+	unlock_busy(page);
+	return false;
+}
+
+#ifdef CONFIG_SYSFS
+static ssize_t exchange_mt_show(struct kobject *kobj,
+		struct kobj_attribute *attr, char *buf)
+{
+	return sprintf(buf, "%u\n", exchange_mt);
+}
+static ssize_t exchange_mt_store(struct kobject *kobj,
+		struct kobj_attribute *attr,
+		const char *buf, size_t count)
+{
+	unsigned long mt;
+	int err;
+
+	err = kstrtoul(buf, 10, &mt);
+	if (err || mt > 1 || mt < 0)
+		return -EINVAL;
+
+	exchange_mt = mt;
+
+	return count;
+}
+static struct kobj_attribute exchange_mt_attr =
+__ATTR(exchange_mt, 0644, exchange_mt_show,
+		exchange_mt_store);
+
+static ssize_t exchange_concur_show(struct kobject *kobj,
+		struct kobj_attribute *attr, char *buf)
+{
+	return sprintf(buf, "%u\n", exchange_concur);
+}
+static ssize_t exchange_concur_store(struct kobject *kobj,
+		struct kobj_attribute *attr,
+		const char *buf, size_t count)
+{
+	unsigned long concur;
+	int err;
+
+	err = kstrtoul(buf, 10, &concur);
+	if (err || concur > 1 || concur < 0)
+		return -EINVAL;
+
+	exchange_concur = concur;
+
+	return count;
+}
+static struct kobj_attribute exchange_concur_attr =
+__ATTR(exchange_concur, 0644, exchange_concur_show,
+		exchange_concur_store);
+
+static struct attribute *numa_exchange_attr[] = {
+	&exchange_concur_attr.attr,
+	&exchange_mt_attr.attr,
+	NULL,
+};
+
+static struct attribute_group numa_exchange_attr_group = {
+	.attrs = numa_exchange_attr,
+};
+static int __init numa_exchange_init_sysfs(struct kobject **exchange_kobj) {
+	int err;
+
+	*exchange_kobj = kobject_create_and_add("exchange", mm_kobj);
+	if (unlikely(!*exchange_kobj)) {
+		pr_err("failed to create exchange kobject\n");
+		return -ENOMEM;
+	}
+
+	err = sysfs_create_group(*exchange_kobj, &numa_exchange_attr_group);
+	if (err) {
+		pr_err("failed to register exchange group\n");
+		goto delete_obj;
+	}
+
+	return 0;
+
+	sysfs_remove_group(*exchange_kobj, &numa_exchange_attr_group);
+delete_obj:
+	kobject_put(*exchange_kobj);
+	return err;
+}
+
+static void __init numa_exchange_exit_sysfs(struct kobject *exchange_kobj)
+{
+	sysfs_remove_group(exchange_kobj, &numa_exchange_attr_group);
+	kobject_put(exchange_kobj);
+}
+#else
+static inline int numa_exchange_init_sysfs(struct kobject **exchange_kobj)
+{
+	return 0;
+}
+static inline void numa_exchange_exit_sysfs(struct kobject *exchange_kobj)
+{
+}
+#endif
+
+static int __init numa_exchange_init(void)
+{
+	int err;
+	struct kobject *sysfs_exchange_kobj;
+
+	err = numa_exchange_init_sysfs(&sysfs_exchange_kobj);
+	if (err) {
+		pr_err("failed start numa_exchange_init becasue sysfs\n");
+		goto err_sysfs;
+	}
+	return 0;
+
+	numa_exchange_exit_sysfs(sysfs_exchange_kobj);
+err_sysfs:
+	return err;
+}
+subsys_initcall(numa_exchange_init);
diff --git a/mm/exchange_page.c b/mm/exchange_page.c
new file mode 100644
index 000000000..07c1e3389
--- /dev/null
+++ b/mm/exchange_page.c
@@ -0,0 +1,241 @@
+/*
+ * This implements parallel page copy function through multi threaded
+ * work queues.
+ *
+ * Zi Yan <ziy@nvidia.com>
+ *
+ * This work is licensed under the terms of the GNU GPL, version 2.
+ */
+#include <linux/highmem.h>
+#include <linux/workqueue.h>
+#include <linux/slab.h>
+#include <linux/freezer.h>
+
+#define UNROLL2(x)   x x
+#define UNROLL4(x)   UNROLL2(x)   UNROLL2(x)
+#define UNROLL8(x)   UNROLL4(x)   UNROLL4(x)
+#define UNROLL16(x)  UNROLL8(x)   UNROLL8(x)
+#define UNROLL32(x)  UNROLL16(x)  UNROLL16(x)
+#define UNROLL64(x)  UNROLL32(x)  UNROLL32(x)
+#define UNROLL128(x) UNROLL64(x)  UNROLL64(x)
+#define UNROLL256(x) UNROLL128(x) UNROLL128(x)
+#define UNROLL512(x) UNROLL256(x) UNROLL256(x)
+
+#define SWAP_PAGE(from, to, tmp, index, chunk) \
+		tmp = *((u64*)(from + index)); \
+		*((u64*)(from + index)) = *((u64*)(to + index)); \
+		*((u64*)(to + index)) = tmp; \
+		index = index + chunk;
+
+/*
+ * nr_copythreads can be the highest number of threads for given node
+ * on any architecture. The actual number of copy threads will be
+ * limited by the cpumask weight of the target node.
+ */
+extern unsigned int limit_mt_num;
+
+struct copy_page_info {
+	struct work_struct copy_page_work;
+	char *to;
+	char *from;
+	unsigned long chunk_size;
+};
+
+static void exchange_page_routine(char *to, char *from, unsigned long chunk_size)
+{
+	u64 tmp;
+	int i = 0;
+	int pieces = sizeof(tmp);
+
+	while (i < chunk_size) {
+		UNROLL512(SWAP_PAGE(from, to, tmp, i, pieces))
+	}
+}
+
+static void exchange_page_work_queue_thread(struct work_struct *work)
+{
+	struct copy_page_info *my_work = (struct copy_page_info*)work;
+
+	exchange_page_routine(my_work->to,
+							  my_work->from,
+							  my_work->chunk_size);
+}
+
+int exchange_page_mthread(struct page *to, struct page *from, int nr_pages)
+{
+	int total_mt_num = limit_mt_num;
+#ifdef CONFIG_PAGE_MIGRATION_PROFILE
+	int to_node = page_to_nid(to);
+#else
+	int to_node = numa_node_id();
+#endif
+	int i;
+	struct copy_page_info *work_items;
+	char *vto, *vfrom;
+	unsigned long chunk_size;
+	const struct cpumask *per_node_cpumask = cpumask_of_node(to_node);
+	int cpu_id_list[32] = {0};
+	int cpu;
+
+	total_mt_num = min_t(unsigned int, total_mt_num,
+						 cpumask_weight(per_node_cpumask));
+
+	if (total_mt_num > 1)
+		total_mt_num = (total_mt_num / 2) * 2;
+
+	if (total_mt_num > 32 || total_mt_num < 1)
+		return -ENODEV;
+
+	work_items = kvzalloc(sizeof(struct copy_page_info)*total_mt_num,
+						 GFP_KERNEL);
+	if (!work_items)
+		return -ENOMEM;
+
+	i = 0;
+	for_each_cpu(cpu, per_node_cpumask) {
+		if (i >= total_mt_num)
+			break;
+		cpu_id_list[i] = cpu;
+		++i;
+	}
+
+	/* XXX: assume no highmem  */
+	vfrom = kmap(from);
+	vto = kmap(to);
+	chunk_size = PAGE_SIZE*nr_pages / total_mt_num;
+
+	for (i = 0; i < total_mt_num; ++i) {
+		INIT_WORK((struct work_struct *)&work_items[i],
+				exchange_page_work_queue_thread);
+
+		work_items[i].to = vto + i * chunk_size;
+		work_items[i].from = vfrom + i * chunk_size;
+		work_items[i].chunk_size = chunk_size;
+
+		queue_work_on(cpu_id_list[i],
+					  system_highpri_wq,
+					  (struct work_struct *)&work_items[i]);
+	}
+
+	/* Wait until it finishes  */
+	flush_workqueue(system_highpri_wq);
+
+	kunmap(to);
+	kunmap(from);
+
+	kvfree(work_items);
+
+	return 0;
+}
+
+int exchange_page_lists_mthread(struct page **to, struct page **from, int nr_pages) 
+{
+	int err = 0;
+	int total_mt_num = limit_mt_num;
+#ifdef CONFIG_PAGE_MIGRATION_PROFILE
+	int to_node = page_to_nid(*to);
+#else
+	int to_node = numa_node_id();
+#endif
+	int i;
+	struct copy_page_info *work_items;
+	const struct cpumask *per_node_cpumask = cpumask_of_node(to_node);
+	int cpu_id_list[32] = {0};
+	int cpu;
+	int item_idx;
+
+
+	total_mt_num = min_t(unsigned int, total_mt_num,
+						 cpumask_weight(per_node_cpumask));
+
+	if (total_mt_num > 32 || total_mt_num < 1)
+		return -ENODEV;
+
+	if (nr_pages < total_mt_num) {
+		int residual_nr_pages = nr_pages - rounddown_pow_of_two(nr_pages);
+
+		if (residual_nr_pages) {
+			for (i = 0; i < residual_nr_pages; ++i) {
+				BUG_ON(hpage_nr_pages(to[i]) != hpage_nr_pages(from[i]));
+				err = exchange_page_mthread(to[i], from[i], hpage_nr_pages(to[i]));
+				VM_BUG_ON(err);
+			}
+			nr_pages = rounddown_pow_of_two(nr_pages);
+			to = &to[residual_nr_pages];
+			from = &from[residual_nr_pages];
+		}
+
+		work_items = kvzalloc(sizeof(struct copy_page_info)*total_mt_num,
+							 GFP_KERNEL);
+	} else
+		work_items = kvzalloc(sizeof(struct copy_page_info)*nr_pages,
+							 GFP_KERNEL);
+	if (!work_items)
+		return -ENOMEM;
+
+	i = 0;
+	for_each_cpu(cpu, per_node_cpumask) {
+		if (i >= total_mt_num)
+			break;
+		cpu_id_list[i] = cpu;
+		++i;
+	}
+
+	if (nr_pages < total_mt_num) {
+		for (cpu = 0; cpu < total_mt_num; ++cpu)
+			INIT_WORK((struct work_struct *)&work_items[cpu],
+					  exchange_page_work_queue_thread);
+		cpu = 0;
+		for (item_idx = 0; item_idx < nr_pages; ++item_idx) {
+			unsigned long chunk_size = nr_pages * PAGE_SIZE * hpage_nr_pages(from[item_idx]) / total_mt_num;
+			char *vfrom = kmap(from[item_idx]);
+			char *vto = kmap(to[item_idx]);
+			VM_BUG_ON(PAGE_SIZE * hpage_nr_pages(from[item_idx]) % total_mt_num);
+			VM_BUG_ON(total_mt_num % nr_pages);
+			BUG_ON(hpage_nr_pages(to[item_idx]) !=
+				   hpage_nr_pages(from[item_idx]));
+
+			for (i = 0; i < (total_mt_num/nr_pages); ++cpu, ++i) {
+				work_items[cpu].to = vto + chunk_size * i;
+				work_items[cpu].from = vfrom + chunk_size * i;
+				work_items[cpu].chunk_size = chunk_size;
+			}
+		}
+		if (cpu != total_mt_num)
+			pr_err("%s: only %d out of %d pages are transferred\n", __func__,
+				cpu - 1, total_mt_num);
+
+		for (cpu = 0; cpu < total_mt_num; ++cpu)
+			queue_work_on(cpu_id_list[cpu],
+						  system_highpri_wq,
+						  (struct work_struct *)&work_items[cpu]);
+	} else {
+		for (i = 0; i < nr_pages; ++i) {
+			int thread_idx = i % total_mt_num;
+
+			INIT_WORK((struct work_struct *)&work_items[i], exchange_page_work_queue_thread);
+
+			/* XXX: assume no highmem  */
+			work_items[i].to = kmap(to[i]);
+			work_items[i].from = kmap(from[i]);
+			work_items[i].chunk_size = PAGE_SIZE * hpage_nr_pages(from[i]);
+
+			BUG_ON(hpage_nr_pages(to[i]) != hpage_nr_pages(from[i]));
+
+			queue_work_on(cpu_id_list[thread_idx], system_highpri_wq, (struct work_struct *)&work_items[i]);
+		}
+	}
+
+	/* Wait until it finishes  */
+	flush_workqueue(system_highpri_wq);
+
+	for (i = 0; i < nr_pages; ++i) {
+			kunmap(to[i]);
+			kunmap(from[i]);
+	}
+
+	kvfree(work_items);
+
+	return err;
+}
+
diff --git a/mm/gup.c b/mm/gup.c
index 98f13ab37..5d8547c98 100644
--- a/mm/gup.c
+++ b/mm/gup.c
@@ -1489,14 +1489,16 @@ static long check_and_migrate_cma_pages(struct task_struct *tsk,
 	}
 
 	if (!list_empty(&cma_page_list)) {
+		struct migrate_detail m_detail = {};
 		/*
 		 * drop the above get_user_pages reference.
 		 */
 		for (i = 0; i < nr_pages; i++)
 			put_page(pages[i]);
 
+		m_detail.reason = MR_CONTIG_RANGE;
 		if (migrate_pages(&cma_page_list, new_non_cma_page,
-				  NULL, 0, MIGRATE_SYNC, MR_CONTIG_RANGE)) {
+				  NULL, 0, MIGRATE_SYNC, &m_detail)) {
 			/*
 			 * some of the pages failed migration. Do get_user_pages
 			 * without migration.
diff --git a/mm/huge_memory.c b/mm/huge_memory.c
index de1f15969..4ee72ae46 100644
--- a/mm/huge_memory.c
+++ b/mm/huge_memory.c
@@ -33,6 +33,7 @@
 #include <linux/oom.h>
 #include <linux/numa.h>
 #include <linux/page_owner.h>
+#include <linux/sched/sysctl.h>
 
 #include <asm/tlb.h>
 #include <asm/pgalloc.h>
@@ -1539,11 +1540,16 @@ vm_fault_t do_huge_pmd_numa_page(struct vm_fault *vmf, pmd_t pmd)
 	bool migrated = false;
 	bool was_writable;
 	int flags = 0;
+	int mode = sysctl_numa_balancing_extended_mode;
 
 	vmf->ptl = pmd_lock(vma->vm_mm, vmf->pmd);
 	if (unlikely(!pmd_same(pmd, *vmf->pmd)))
 		goto out_unlock;
 
+	/* Only migrate if accessed twice */
+	if (pmd_young(*vmf->pmd))
+		flags |= TNF_YOUNG;
+
 	/*
 	 * If there are potential migrations, wait for completion and retry
 	 * without disrupting NUMA hinting information. Do not relock and
@@ -1563,9 +1569,21 @@ vm_fault_t do_huge_pmd_numa_page(struct vm_fault *vmf, pmd_t pmd)
 	page_nid = page_to_nid(page);
 	last_cpupid = page_cpupid_last(page);
 	count_vm_numa_event(NUMA_HINT_FAULTS);
+
 	if (page_nid == this_nid) {
 		count_vm_numa_event(NUMA_HINT_FAULTS_LOCAL);
 		flags |= TNF_FAULT_LOCAL;
+
+		if (mode & NUMA_BALANCING_EXCHANGE) {
+			struct pglist_data *pgdat = page_pgdat(page);
+
+			spin_lock_irq(&pgdat->lru_lock);
+			if (PageDeferred(page)) {
+				del_page_from_deferred_list(page);
+				count_vm_events(PGACTIVATE_DEFERRED_LOCAL, hpage_nr_pages(page));
+			}
+			spin_unlock_irq(&pgdat->lru_lock);
+		}
 	}
 
 	/* See similar comment in do_numa_page for explanation */
@@ -1954,17 +1972,36 @@ int change_huge_pmd(struct vm_area_struct *vma, pmd_t *pmd,
 	}
 #endif
 
-	/*
-	 * Avoid trapping faults against the zero page. The read-only
-	 * data is likely to be read-cached on the local CPU and
-	 * local/remote hits to the zero page are not interesting.
-	 */
-	if (prot_numa && is_huge_zero_pmd(*pmd))
-		goto unlock;
+	if (prot_numa) {
+		struct page *page;
+		int prev_lv;
+		int mode = sysctl_numa_balancing_extended_mode;
+		/*
+		 * Avoid trapping faults against the zero page. The read-only
+		 * data is likely to be read-cached on the local CPU and
+		 * local/remote hits to the zero page are not interesting.
+		 */
+		if (is_huge_zero_pmd(*pmd))
+			goto unlock;
 
-	if (prot_numa && pmd_protnone(*pmd))
-		goto unlock;
+		page = pmd_page(*pmd);
 
+		/* Avoid TLB flush if possible */
+		if (pmd_protnone(*pmd)) {
+			if (mode & NUMA_BALANCING_OPM) {
+				/* The page is not accessed in last scan period */
+				prev_lv = mod_page_access_lv(page, 0);
+				add_page_for_tracking(page, prev_lv);
+			}
+			goto unlock;
+		}
+
+		/* The page is accessed in last scan period */
+		if (mode & NUMA_BALANCING_OPM) {
+			prev_lv = mod_page_access_lv(page, 1);
+			add_page_for_tracking(page, prev_lv);
+		}
+	}
 	/*
 	 * In case prot_numa, we are under down_read(mmap_sem). It's critical
 	 * to not clear pmd intermittently to avoid race with MADV_DONTNEED
diff --git a/mm/hugetlb.c b/mm/hugetlb.c
index 6d7296dd1..9b25e875c 100644
--- a/mm/hugetlb.c
+++ b/mm/hugetlb.c
@@ -27,6 +27,7 @@
 #include <linux/swapops.h>
 #include <linux/jhash.h>
 #include <linux/numa.h>
+#include <linux/migrate.h>
 
 #include <asm/page.h>
 #include <asm/pgtable.h>
@@ -5030,12 +5031,13 @@ void putback_active_hugepage(struct page *page)
 	put_page(page);
 }
 
-void move_hugetlb_state(struct page *oldpage, struct page *newpage, int reason)
+void move_hugetlb_state(struct page *oldpage, struct page *newpage,
+			struct migrate_detail *m_detail)
 {
 	struct hstate *h = page_hstate(oldpage);
 
 	hugetlb_cgroup_migrate(oldpage, newpage);
-	set_page_owner_migrate_reason(newpage, reason);
+	set_page_owner_migrate_reason(newpage, m_detail->reason);
 
 	/*
 	 * transfer temporary state of the new huge page. This is
diff --git a/mm/internal.h b/mm/internal.h
index e32390802..299365e34 100644
--- a/mm/internal.h
+++ b/mm/internal.h
@@ -165,6 +165,23 @@ extern void post_alloc_hook(struct page *page, unsigned int order,
 					gfp_t gfp_flags);
 extern int user_min_free_kbytes;
 
+#ifdef CONFIG_PAGE_BALANCING
+struct free_promote_area {
+	struct list_head free_list;
+	uint32_t nr_free;
+};
+
+enum page_type {
+	BASEPAGE,
+	HUGEPAGE,
+	NR_PAGE_TYPE
+};
+
+extern struct free_promote_area *promote_area[NR_PAGE_TYPE];
+extern struct page* alloc_promote_page(struct page *page, unsigned long data);
+extern struct page* alloc_demote_page(struct page *page, unsigned long data);
+#endif
+
 #if defined CONFIG_COMPACTION || defined CONFIG_CMA
 
 /*
diff --git a/mm/ksm.c b/mm/ksm.c
index 3dc434641..8418d85fd 100644
--- a/mm/ksm.c
+++ b/mm/ksm.c
@@ -2717,6 +2717,41 @@ void ksm_migrate_page(struct page *newpage, struct page *oldpage)
 		set_page_stable_node(oldpage, NULL);
 	}
 }
+
+void ksm_exchange_page(struct page *to_page, struct page *from_page)
+{
+	struct stable_node *to_stable_node, *from_stable_node;
+
+	VM_BUG_ON_PAGE(!PageLocked(to_page), to_page);
+	VM_BUG_ON_PAGE(!PageLocked(from_page), from_page);
+
+	to_stable_node = page_stable_node(to_page);
+	from_stable_node = page_stable_node(from_page);
+	if (to_stable_node) {
+		VM_BUG_ON_PAGE(to_stable_node->kpfn != page_to_pfn(from_page),
+				from_page);
+		to_stable_node->kpfn = page_to_pfn(to_page);
+		/*
+		 * newpage->mapping was set in advance; now we need smp_wmb()
+		 * to make sure that the new stable_node->kpfn is visible
+		 * to get_ksm_page() before it can see that oldpage->mapping
+		 * has gone stale (or that PageSwapCache has been cleared).
+		 */
+		smp_wmb();
+	}
+	if (from_stable_node) {
+		VM_BUG_ON_PAGE(from_stable_node->kpfn != page_to_pfn(to_page),
+				to_page);
+		from_stable_node->kpfn = page_to_pfn(from_page);
+		/*
+		 * newpage->mapping was set in advance; now we need smp_wmb()
+		 * to make sure that the new stable_node->kpfn is visible
+		 * to get_ksm_page() before it can see that oldpage->mapping
+		 * has gone stale (or that PageSwapCache has been cleared).
+		 */
+		smp_wmb();
+	}
+}
 #endif /* CONFIG_MIGRATION */
 
 #ifdef CONFIG_MEMORY_HOTREMOVE
diff --git a/mm/memory-failure.c b/mm/memory-failure.c
index 7ef849da8..bbc93480d 100644
--- a/mm/memory-failure.c
+++ b/mm/memory-failure.c
@@ -1677,6 +1677,7 @@ static int get_any_page(struct page *page, unsigned long pfn, int flags)
 static int soft_offline_huge_page(struct page *page, int flags)
 {
 	int ret;
+	struct migrate_detail m_detail = {};
 	unsigned long pfn = page_to_pfn(page);
 	struct page *hpage = compound_head(page);
 	LIST_HEAD(pagelist);
@@ -1705,8 +1706,9 @@ static int soft_offline_huge_page(struct page *page, int flags)
 		return -EBUSY;
 	}
 
+	m_detail.reason = MR_MEMORY_FAILURE;
 	ret = migrate_pages(&pagelist, new_page, NULL, MPOL_MF_MOVE_ALL,
-				MIGRATE_SYNC, MR_MEMORY_FAILURE);
+				MIGRATE_SYNC, &m_detail);
 	if (ret) {
 		pr_info("soft offline: %#lx: hugepage migration failed %d, type %lx (%pGp)\n",
 			pfn, ret, page->flags, &page->flags);
@@ -1736,6 +1738,7 @@ static int soft_offline_huge_page(struct page *page, int flags)
 static int __soft_offline_page(struct page *page, int flags)
 {
 	int ret;
+	struct migrate_detail m_detail = {};
 	unsigned long pfn = page_to_pfn(page);
 
 	/*
@@ -1795,8 +1798,9 @@ static int __soft_offline_page(struct page *page, int flags)
 			inc_node_page_state(page, NR_ISOLATED_ANON +
 						page_is_file_cache(page));
 		list_add(&page->lru, &pagelist);
+		m_detail.reason = MR_MEMORY_FAILURE;
 		ret = migrate_pages(&pagelist, new_page, NULL, MPOL_MF_MOVE_ALL,
-					MIGRATE_SYNC, MR_MEMORY_FAILURE);
+					MIGRATE_SYNC, &m_detail);
 		if (ret) {
 			if (!list_empty(&pagelist))
 				putback_movable_pages(&pagelist);
diff --git a/mm/memory.c b/mm/memory.c
index e2bb51b62..241dfd9fe 100644
--- a/mm/memory.c
+++ b/mm/memory.c
@@ -72,6 +72,11 @@
 #include <linux/oom.h>
 #include <linux/numa.h>
 
+#ifdef CONFIG_PAGE_BALANCING
+#include <linux/page_balancing.h>
+#include <linux/sched/sysctl.h>
+#endif
+
 #include <asm/io.h>
 #include <asm/mmu_context.h>
 #include <asm/pgalloc.h>
@@ -3624,14 +3629,33 @@ static int numa_migrate_prep(struct page *page, struct vm_area_struct *vma,
 				unsigned long addr, int page_nid,
 				int *flags)
 {
+	int dst_nid = numa_node_id();
+	int mode = sysctl_numa_balancing_extended_mode;
+	struct pglist_data *pgdat = page_pgdat(page);
+
 	get_page(page);
 
 	count_vm_numa_event(NUMA_HINT_FAULTS);
-	if (page_nid == numa_node_id()) {
+
+	if (page_nid == dst_nid) {
 		count_vm_numa_event(NUMA_HINT_FAULTS_LOCAL);
 		*flags |= TNF_FAULT_LOCAL;
 	}
 
+	/* Check local or remote NUMA page fault */
+	if (mode & NUMA_BALANCING_EXCHANGE)
+	{
+		/* Local NUMA_HINT_FAULT */
+		if (page_nid == dst_nid) {
+			spin_lock_irq(&pgdat->lru_lock);
+			if (PageDeferred(page)) {
+				del_page_from_deferred_list(page);
+				count_vm_event(PGACTIVATE_DEFERRED_LOCAL);
+			}
+			spin_unlock_irq(&pgdat->lru_lock);
+		}
+	}
+
 	return mpol_misplaced(page, vma, addr);
 }
 
@@ -3642,7 +3666,7 @@ static vm_fault_t do_numa_page(struct vm_fault *vmf)
 	int page_nid = NUMA_NO_NODE;
 	int last_cpupid;
 	int target_nid;
-	bool migrated = false;
+	int migrated_nid;
 	pte_t pte, old_pte;
 	bool was_writable = pte_savedwrite(vmf->orig_pte);
 	int flags = 0;
@@ -3671,6 +3695,12 @@ static vm_fault_t do_numa_page(struct vm_fault *vmf)
 	ptep_modify_prot_commit(vma, vmf->address, vmf->pte, old_pte, pte);
 	update_mmu_cache(vma, vmf->address, vmf->pte);
 
+	if (pte_young(old_pte))
+		flags |= TNF_YOUNG;
+
+	if (vmf->flags & FAULT_FLAG_WRITE)
+		flags |= TNF_WRITE;
+
 	page = vm_normal_page(vma, vmf->address, pte);
 	if (!page) {
 		pte_unmap_unlock(vmf->pte, vmf->ptl);
@@ -3712,9 +3742,9 @@ static vm_fault_t do_numa_page(struct vm_fault *vmf)
 	}
 
 	/* Migrate to the requested node */
-	migrated = migrate_misplaced_page(page, vma, target_nid);
-	if (migrated) {
-		page_nid = target_nid;
+	migrated_nid = migrate_misplaced_page(page, vma, target_nid);
+	if (migrated_nid != NUMA_NO_NODE) {
+		page_nid = migrated_nid;
 		flags |= TNF_MIGRATED;
 	} else
 		flags |= TNF_MIGRATE_FAIL;
diff --git a/mm/memory_hotplug.c b/mm/memory_hotplug.c
index c73f09913..02a81b1a1 100644
--- a/mm/memory_hotplug.c
+++ b/mm/memory_hotplug.c
@@ -1392,9 +1392,11 @@ do_migrate_range(unsigned long start_pfn, unsigned long end_pfn)
 		put_page(page);
 	}
 	if (!list_empty(&source)) {
+		struct migrate_detail m_detail = {};
 		/* Allocate a new page from the nearest neighbor node */
+		m_detail.reason = MR_MEMORY_HOTPLUG;
 		ret = migrate_pages(&source, new_node_page, NULL, 0,
-					MIGRATE_SYNC, MR_MEMORY_HOTPLUG);
+					MIGRATE_SYNC, &m_detail);
 		if (ret) {
 			list_for_each_entry(page, &source, lru) {
 				pr_warn("migrating pfn %lx failed ret:%d ",
diff --git a/mm/mempolicy.c b/mm/mempolicy.c
index 65e0874fc..f0d521d47 100644
--- a/mm/mempolicy.c
+++ b/mm/mempolicy.c
@@ -105,6 +105,10 @@
 
 #include "internal.h"
 
+#ifdef CONFIG_PAGE_BALANCING
+#include <linux/page_balancing.h>
+#endif
+
 /* Internal flags */
 #define MPOL_MF_DISCONTIG_OK (MPOL_MF_INTERNAL << 0)	/* Skip checks for continuous vmas */
 #define MPOL_MF_INVERT (MPOL_MF_INTERNAL << 1)		/* Invert check for nodemask */
@@ -1040,8 +1044,9 @@ static int migrate_to_node(struct mm_struct *mm, int source, int dest,
 			flags | MPOL_MF_DISCONTIG_OK, &pagelist);
 
 	if (!list_empty(&pagelist)) {
+		struct migrate_detail m_detail = { .reason = MR_SYSCALL };
 		err = migrate_pages(&pagelist, alloc_new_node_page, NULL, dest,
-					MIGRATE_SYNC, MR_SYSCALL);
+					MIGRATE_SYNC, &m_detail);
 		if (err)
 			putback_movable_pages(&pagelist);
 	}
@@ -1297,9 +1302,12 @@ static long do_mbind(unsigned long start, unsigned long len,
 		int nr_failed = 0;
 
 		if (!list_empty(&pagelist)) {
+			struct migrate_detail m_detail = {};
+
+			m_detail.reason = MR_MEMPOLICY_MBIND;
 			WARN_ON_ONCE(flags & MPOL_MF_LAZY);
 			nr_failed = migrate_pages(&pagelist, new_page, NULL,
-				start, MIGRATE_SYNC, MR_MEMPOLICY_MBIND);
+				start, MIGRATE_SYNC, &m_detail);
 			if (nr_failed)
 				putback_movable_pages(&pagelist);
 		}
@@ -2343,7 +2351,8 @@ static void sp_free(struct sp_node *n)
  * Policy determination "mimics" alloc_page_vma().
  * Called from fault path where we know the vma and faulting address.
  */
-int mpol_misplaced(struct page *page, struct vm_area_struct *vma, unsigned long addr)
+int mpol_misplaced(struct page *page, struct vm_area_struct *vma,
+		   unsigned long addr)
 {
 	struct mempolicy *pol;
 	struct zoneref *z;
@@ -2353,6 +2362,11 @@ int mpol_misplaced(struct page *page, struct vm_area_struct *vma, unsigned long
 	int thisnid = cpu_to_node(thiscpu);
 	int polnid = NUMA_NO_NODE;
 	int ret = -1;
+	int mode = sysctl_numa_balancing_extended_mode;
+
+	/* Set page least fault cpu */
+	if (mode & NUMA_BALANCING_OPM || mode & NUMA_BALANCING_EXCHANGE)
+		set_page_last_cpu(page, thiscpu);
 
 	pol = get_vma_policy(vma, addr);
 	if (!(pol->flags & MPOL_F_MOF))
@@ -2400,7 +2414,6 @@ int mpol_misplaced(struct page *page, struct vm_area_struct *vma, unsigned long
 		if (!should_numa_migrate_memory(current, page, curnid, thiscpu))
 			goto out;
 	}
-
 	if (curnid != polnid)
 		ret = polnid;
 out:
diff --git a/mm/migrate.c b/mm/migrate.c
index a42858d8e..e1e31e37c 100644
--- a/mm/migrate.c
+++ b/mm/migrate.c
@@ -47,6 +47,9 @@
 #include <linux/page_owner.h>
 #include <linux/sched/mm.h>
 #include <linux/ptrace.h>
+#include <linux/exchange.h>
+#include <linux/sched/sysctl.h>
+#include <linux/memcontrol.h>
 
 #include <asm/tlbflush.h>
 
@@ -403,8 +406,10 @@ int migrate_page_move_mapping(struct address_space *mapping,
 
 	if (!mapping) {
 		/* Anonymous page without mapping */
-		if (page_count(page) != expected_count)
+		if (page_count(page) != expected_count) {
+			count_vm_events(PGMIGRATE_REFCOUNT_FAIL, hpage_nr_pages(page));
 			return -EAGAIN;
+		}
 
 		/* No turning back from here */
 		newpage->index = page->index;
@@ -581,9 +586,13 @@ static void copy_huge_page(struct page *dst, struct page *src)
 		nr_pages = hpage_nr_pages(src);
 	}
 
-	for (i = 0; i < nr_pages; i++) {
-		cond_resched();
-		copy_highpage(dst + i, src + i);
+	if (PageTransHuge(src) && thp_mt_copy) {
+		copy_page_multithread(dst, src, nr_pages);
+	} else {
+		for (i = 0; i < nr_pages; i++) {
+			cond_resched();
+			copy_highpage(dst + i, src + i);
+		}
 	}
 }
 
@@ -646,6 +655,7 @@ void migrate_page_states(struct page *newpage, struct page *page)
 		end_page_writeback(newpage);
 
 	copy_page_owner(page, newpage);
+	copy_page_info(page, newpage);
 
 	mem_cgroup_migrate(page, newpage);
 }
@@ -695,7 +705,7 @@ EXPORT_SYMBOL(migrate_page);
 
 #ifdef CONFIG_BLOCK
 /* Returns true if all buffers are successfully locked */
-static bool buffer_migrate_lock_buffers(struct buffer_head *head,
+bool buffer_migrate_lock_buffers(struct buffer_head *head,
 							enum migrate_mode mode)
 {
 	struct buffer_head *bh = head;
@@ -845,7 +855,7 @@ int buffer_migrate_page_norefs(struct address_space *mapping,
 /*
  * Writeback a page to clean the dirty state
  */
-static int writeout(struct address_space *mapping, struct page *page)
+int writeout(struct address_space *mapping, struct page *page)
 {
 	struct writeback_control wbc = {
 		.sync_mode = WB_SYNC_NONE,
@@ -924,7 +934,8 @@ static int fallback_migrate_page(struct address_space *mapping,
  *  MIGRATEPAGE_SUCCESS - success
  */
 static int move_to_new_page(struct page *newpage, struct page *page,
-				enum migrate_mode mode)
+				enum migrate_mode mode,
+				struct migrate_detail *m_detail)
 {
 	struct address_space *mapping;
 	int rc = -EAGAIN;
@@ -992,65 +1003,30 @@ static int move_to_new_page(struct page *newpage, struct page *page,
 		if (!PageMappingFlags(page))
 			page->mapping = NULL;
 
+		inc_hmem_src_state(m_detail->h_reason, page);
+		inc_hmem_dst_state(m_detail->h_reason, newpage);
+
 		if (likely(!is_zone_device_page(newpage)))
 			flush_dcache_page(newpage);
 
+                trace_mm_migrate_move_page(page, newpage, rc, m_detail->reason);
 	}
 out:
 	return rc;
 }
 
-static int __unmap_and_move(struct page *page, struct page *newpage,
-				int force, enum migrate_mode mode)
+static int __unmap_and_move(new_page_t get_new_page,
+			    free_page_t put_new_page,
+			    unsigned long private, struct page *page,
+			    enum migrate_mode mode,
+			    struct migrate_detail *m_detail)
 {
 	int rc = -EAGAIN;
 	int page_was_mapped = 0;
 	struct anon_vma *anon_vma = NULL;
 	bool is_lru = !__PageMovable(page);
 
-	if (!trylock_page(page)) {
-		if (!force || mode == MIGRATE_ASYNC)
-			goto out;
-
-		/*
-		 * It's not safe for direct compaction to call lock_page.
-		 * For example, during page readahead pages are added locked
-		 * to the LRU. Later, when the IO completes the pages are
-		 * marked uptodate and unlocked. However, the queueing
-		 * could be merging multiple pages for one bio (e.g.
-		 * mpage_readpages). If an allocation happens for the
-		 * second or third page, the process can end up locking
-		 * the same page twice and deadlocking. Rather than
-		 * trying to be clever about what pages can be locked,
-		 * avoid the use of lock_page for direct compaction
-		 * altogether.
-		 */
-		if (current->flags & PF_MEMALLOC)
-			goto out;
-
-		lock_page(page);
-	}
-
-	if (PageWriteback(page)) {
-		/*
-		 * Only in the case of a full synchronous migration is it
-		 * necessary to wait for PageWriteback. In the async case,
-		 * the retry loop is too short and in the sync-light case,
-		 * the overhead of stalling is too much
-		 */
-		switch (mode) {
-		case MIGRATE_SYNC:
-		case MIGRATE_SYNC_NO_COPY:
-			break;
-		default:
-			rc = -EBUSY;
-			goto out_unlock;
-		}
-		if (!force)
-			goto out_unlock;
-		wait_on_page_writeback(page);
-	}
-
+	struct page *newpage;
 	/*
 	 * By try_to_unmap(), page->mapcount goes down to 0 here. In this case,
 	 * we cannot notice that anon_vma is freed while we migrates a page.
@@ -1068,6 +1044,12 @@ static int __unmap_and_move(struct page *page, struct page *newpage,
 	if (PageAnon(page) && !PageKsm(page))
 		anon_vma = page_get_anon_vma(page);
 
+	newpage = get_new_page(page, private);
+	if (!newpage) {
+		rc = -ENOMEM;
+		goto out;
+	}
+
 	/*
 	 * Block others from accessing the new page when we get around to
 	 * establishing additional references. We are usually the only one
@@ -1077,11 +1059,11 @@ static int __unmap_and_move(struct page *page, struct page *newpage,
 	 * This is much like races on refcount of oldpage: just don't BUG().
 	 */
 	if (unlikely(!trylock_page(newpage)))
-		goto out_unlock;
+		goto out_put;
 
 	if (unlikely(!is_lru)) {
-		rc = move_to_new_page(newpage, page, mode);
-		goto out_unlock_both;
+		rc = move_to_new_page(newpage, page, mode, m_detail);
+		goto out_unlock;
 	}
 
 	/*
@@ -1100,7 +1082,7 @@ static int __unmap_and_move(struct page *page, struct page *newpage,
 		VM_BUG_ON_PAGE(PageAnon(page), page);
 		if (page_has_private(page)) {
 			try_to_free_buffers(page);
-			goto out_unlock_both;
+			goto out_unlock;
 		}
 	} else if (page_mapped(page)) {
 		/* Establish migration ptes */
@@ -1112,20 +1094,14 @@ static int __unmap_and_move(struct page *page, struct page *newpage,
 	}
 
 	if (!page_mapped(page))
-		rc = move_to_new_page(newpage, page, mode);
+		rc = move_to_new_page(newpage, page, mode, m_detail);
 
 	if (page_was_mapped)
 		remove_migration_ptes(page,
 			rc == MIGRATEPAGE_SUCCESS ? newpage : page, false);
-
-out_unlock_both:
-	unlock_page(newpage);
 out_unlock:
-	/* Drop an anon_vma reference if we took one */
-	if (anon_vma)
-		put_anon_vma(anon_vma);
-	unlock_page(page);
-out:
+	unlock_page(newpage);
+out_put:
 	/*
 	 * If migration is successful, decrease refcount of the newpage
 	 * which will not free the page because new page owner increased
@@ -1136,15 +1112,87 @@ static int __unmap_and_move(struct page *page, struct page *newpage,
 	 * state.
 	 */
 	if (rc == MIGRATEPAGE_SUCCESS) {
+		set_page_owner_migrate_reason(newpage, m_detail->reason);
 		if (unlikely(!is_lru))
 			put_page(newpage);
 		else
 			putback_lru_page(newpage);
+	} else if (put_new_page) {
+		put_new_page(newpage, private);
+	} else {
+		put_page(newpage);
 	}
-
+out:
+	/* Drop an anon_vma reference if we took one */
+	if (anon_vma)
+		put_anon_vma(anon_vma);
 	return rc;
 }
 
+static struct page *alloc_node_page(struct page *page, unsigned long node)
+{
+	/*
+	 * The flags are set to allocate only on the desired node in the
+	 * migration path, and to fail fast if not immediately available. We
+	 * are already doing memory reclaim, we don't want heroic efforts to
+	 * get a page.
+	 */
+	gfp_t mask = GFP_NOWAIT | __GFP_NOWARN | __GFP_NORETRY |
+			__GFP_NOMEMALLOC | __GFP_THISNODE | __GFP_HIGHMEM |
+			__GFP_MOVABLE;
+	struct page *newpage;
+
+	if (PageTransHuge(page)) {
+		mask |= __GFP_COMP;
+		newpage = alloc_pages_node(node, mask, HPAGE_PMD_ORDER);
+		if (newpage)
+			prep_transhuge_page(newpage);
+	} else
+		newpage = alloc_pages_node(node, mask, 0);
+
+	return newpage;
+}
+
+static int migrate_mapping(struct page *page, int next_nid,
+			   struct migrate_detail *m_detail)
+{
+	VM_BUG_ON_PAGE(!PageLocked(page), page);
+	VM_BUG_ON_PAGE(PageHuge(page), page);
+	VM_BUG_ON_PAGE(PageLRU(page), page);
+
+	if (next_nid < 0)
+		return -ENOSYS;
+	if (PageTransHuge(page) && !thp_migration_supported())
+		return -ENOMEM;
+
+	if (m_detail->reason == MR_PROMOTION)
+		pr_info_once("promote page from %d to %d\n",
+			page_to_nid(page), next_nid);
+
+	/* MIGRATE_ASYNC is the most light weight and never blocks.*/
+	return __unmap_and_move(alloc_node_page, NULL, next_nid,
+				page, MIGRATE_ASYNC, m_detail);
+}
+
+/**
+ * migrate_demote_mapping() - Migrate this page and its mappings to its
+ *                            demotion node.
+ * @page: A locked, isolated, non-huge page that should migrate to its current
+ *        node's demotion target, if available.
+ *
+ * @returns: MIGRATEPAGE_SUCCESS if successful, -errno otherwise.
+ */
+int migrate_demote_mapping(struct page *page)
+{
+	struct migrate_detail m_detail = {
+		.reason = MR_DEMOTION,
+		.h_reason = MR_HMEM_DEMOTE,
+	};
+
+	return migrate_mapping(page, next_demotion_node(page_to_nid(page)),
+			       &m_detail);
+}
+
 /*
  * gcc 4.7 and 4.8 on arm get an ICEs when inlining unmap_and_move().  Work
  * around it.
@@ -1164,18 +1212,13 @@ static ICE_noinline int unmap_and_move(new_page_t get_new_page,
 				   free_page_t put_new_page,
 				   unsigned long private, struct page *page,
 				   int force, enum migrate_mode mode,
-				   enum migrate_reason reason)
+				   struct migrate_detail *m_detail)
 {
-	int rc = MIGRATEPAGE_SUCCESS;
-	struct page *newpage;
+	int rc = -EAGAIN;
 
 	if (!thp_migration_supported() && PageTransHuge(page))
 		return -ENOMEM;
 
-	newpage = get_new_page(page, private);
-	if (!newpage)
-		return -ENOMEM;
-
 	if (page_count(page) == 1) {
 		/* page was freed from under us. So we are done. */
 		ClearPageActive(page);
@@ -1186,17 +1229,57 @@ static ICE_noinline int unmap_and_move(new_page_t get_new_page,
 				__ClearPageIsolated(page);
 			unlock_page(page);
 		}
-		if (put_new_page)
-			put_new_page(newpage, private);
-		else
-			put_page(newpage);
+		rc = MIGRATEPAGE_SUCCESS;
 		goto out;
 	}
 
-	rc = __unmap_and_move(page, newpage, force, mode);
-	if (rc == MIGRATEPAGE_SUCCESS)
-		set_page_owner_migrate_reason(newpage, reason);
+	if (!trylock_page(page)) {
+		if (!force || mode == MIGRATE_ASYNC)
+			return rc;
 
+		/*
+		 * It's not safe for direct compaction to call lock_page.
+		 * For example, during page readahead pages are added locked
+		 * to the LRU. Later, when the IO completes the pages are
+		 * marked uptodate and unlocked. However, the queueing
+		 * could be merging multiple pages for one bio (e.g.
+		 * mpage_readpages). If an allocation happens for the
+		 * second or third page, the process can end up locking
+		 * the same page twice and deadlocking. Rather than
+		 * trying to be clever about what pages can be locked,
+		 * avoid the use of lock_page for direct compaction
+		 * altogether.
+		 */
+		if (current->flags & PF_MEMALLOC)
+			return rc;
+
+		lock_page(page);
+	}
+
+	if (PageWriteback(page)) {
+		/*
+		 * Only in the case of a full synchronous migration is it
+		 * necessary to wait for PageWriteback. In the async case,
+		 * the retry loop is too short and in the sync-light case,
+		 * the overhead of stalling is too much
+		 */
+		switch (mode) {
+		case MIGRATE_SYNC:
+		case MIGRATE_SYNC_NO_COPY:
+			break;
+		default:
+			rc = -EBUSY;
+			goto out_unlock;
+		}
+		if (!force)
+			goto out_unlock;
+		wait_on_page_writeback(page);
+	}
+	rc = __unmap_and_move(get_new_page, put_new_page, private,
+			      page, mode, m_detail);
+
+out_unlock:
+	unlock_page(page);
 out:
 	if (rc != -EAGAIN) {
 		/*
@@ -1224,7 +1307,7 @@ static ICE_noinline int unmap_and_move(new_page_t get_new_page,
 	 */
 	if (rc == MIGRATEPAGE_SUCCESS) {
 		put_page(page);
-		if (reason == MR_MEMORY_FAILURE) {
+		if (m_detail->reason == MR_MEMORY_FAILURE) {
 			/*
 			 * Set PG_HWPoison on just freed page
 			 * intentionally. Although it's rather weird,
@@ -1237,9 +1320,8 @@ static ICE_noinline int unmap_and_move(new_page_t get_new_page,
 		if (rc != -EAGAIN) {
 			if (likely(!__PageMovable(page))) {
 				putback_lru_page(page);
-				goto put_new;
+				goto done;
 			}
-
 			lock_page(page);
 			if (PageMovable(page))
 				putback_movable_page(page);
@@ -1248,13 +1330,8 @@ static ICE_noinline int unmap_and_move(new_page_t get_new_page,
 			unlock_page(page);
 			put_page(page);
 		}
-put_new:
-		if (put_new_page)
-			put_new_page(newpage, private);
-		else
-			put_page(newpage);
 	}
-
+done:
 	return rc;
 }
 
@@ -1279,7 +1356,8 @@ static ICE_noinline int unmap_and_move(new_page_t get_new_page,
 static int unmap_and_move_huge_page(new_page_t get_new_page,
 				free_page_t put_new_page, unsigned long private,
 				struct page *hpage, int force,
-				enum migrate_mode mode, int reason)
+				enum migrate_mode mode,
+				struct migrate_detail *m_detail)
 {
 	int rc = -EAGAIN;
 	int page_was_mapped = 0;
@@ -1338,7 +1416,7 @@ static int unmap_and_move_huge_page(new_page_t get_new_page,
 	}
 
 	if (!page_mapped(hpage))
-		rc = move_to_new_page(new_hpage, hpage, mode);
+		rc = move_to_new_page(new_hpage, hpage, mode, m_detail);
 
 	if (page_was_mapped)
 		remove_migration_ptes(hpage,
@@ -1351,7 +1429,7 @@ static int unmap_and_move_huge_page(new_page_t get_new_page,
 		put_anon_vma(anon_vma);
 
 	if (rc == MIGRATEPAGE_SUCCESS) {
-		move_hugetlb_state(hpage, new_hpage, reason);
+		move_hugetlb_state(hpage, new_hpage, m_detail);
 		put_new_page = NULL;
 	}
 
@@ -1386,7 +1464,7 @@ static int unmap_and_move_huge_page(new_page_t get_new_page,
  * @private:		Private data to be passed on to get_new_page()
  * @mode:		The migration mode that specifies the constraints for
  *			page migration, if any.
- * @reason:		The reason for page migration.
+ * @m_detail:		The reason for page migration in detail.
  *
  * The function returns after 10 attempts or if no pages are movable any more
  * because the list has become empty or no retryable pages exist any more.
@@ -1397,7 +1475,7 @@ static int unmap_and_move_huge_page(new_page_t get_new_page,
  */
 int migrate_pages(struct list_head *from, new_page_t get_new_page,
 		free_page_t put_new_page, unsigned long private,
-		enum migrate_mode mode, int reason)
+		enum migrate_mode mode, struct migrate_detail *m_detail)
 {
 	int retry = 1;
 	int nr_failed = 0;
@@ -1421,11 +1499,11 @@ int migrate_pages(struct list_head *from, new_page_t get_new_page,
 			if (PageHuge(page))
 				rc = unmap_and_move_huge_page(get_new_page,
 						put_new_page, private, page,
-						pass > 2, mode, reason);
+						pass > 2, mode, m_detail);
 			else
 				rc = unmap_and_move(get_new_page, put_new_page,
 						private, page, pass > 2, mode,
-						reason);
+						m_detail);
 
 			switch(rc) {
 			case -ENOMEM:
@@ -1450,6 +1528,9 @@ int migrate_pages(struct list_head *from, new_page_t get_new_page,
 					}
 				}
 				nr_failed++;
+
+                                count_vm_events(PGMIGRATE_NOMEM_FAIL,
+                                                hpage_nr_pages(page));
 				goto out;
 			case -EAGAIN:
 				retry++;
@@ -1472,11 +1553,13 @@ int migrate_pages(struct list_head *from, new_page_t get_new_page,
 	nr_failed += retry;
 	rc = nr_failed;
 out:
-	if (nr_succeeded)
+	if (nr_succeeded) {
 		count_vm_events(PGMIGRATE_SUCCESS, nr_succeeded);
-	if (nr_failed)
+	}
+	if (nr_failed) {
 		count_vm_events(PGMIGRATE_FAIL, nr_failed);
-	trace_mm_migrate_pages(nr_succeeded, nr_failed, mode, reason);
+	}
+	trace_mm_migrate_pages(nr_succeeded, nr_failed, mode, m_detail->reason);
 
 	if (!swapwrite)
 		current->flags &= ~PF_SWAPWRITE;
@@ -1500,13 +1583,15 @@ static int store_status(int __user *status, int start, int value, int nr)
 static int do_move_pages_to_node(struct mm_struct *mm,
 		struct list_head *pagelist, int node)
 {
+	struct migrate_detail m_detail = {};
 	int err;
 
 	if (list_empty(pagelist))
 		return 0;
 
+	m_detail.reason = MR_SYSCALL;
 	err = migrate_pages(pagelist, alloc_new_node_page, NULL, node,
-			MIGRATE_SYNC, MR_SYSCALL);
+			MIGRATE_SYNC, &m_detail);
 	if (err)
 		putback_movable_pages(pagelist);
 	return err;
@@ -1847,10 +1932,11 @@ COMPAT_SYSCALL_DEFINE6(move_pages, pid_t, pid, compat_ulong_t, nr_pages,
  * Returns true if this is a safe migration target node for misplaced NUMA
  * pages. Currently it only checks the watermarks which crude
  */
-static bool migrate_balanced_pgdat(struct pglist_data *pgdat,
-				   unsigned long nr_migrate_pages)
+bool migrate_balanced_pgdat(struct pglist_data *pgdat,
+				   unsigned int order)
 {
 	int z;
+	unsigned long nr_migratepages = 1UL << order;
 
 	for (z = pgdat->nr_zones - 1; z >= 0; z--) {
 		struct zone *zone = pgdat->node_zones + z;
@@ -1859,10 +1945,10 @@ static bool migrate_balanced_pgdat(struct pglist_data *pgdat,
 			continue;
 
 		/* Avoid waking kswapd by allocating pages_to_migrate pages. */
-		if (!zone_watermark_ok(zone, 0,
-				       high_wmark_pages(zone) +
-				       nr_migrate_pages,
-				       0, 0))
+		if (!zone_watermark_ok(zone, order,
+				      high_wmark_pages(zone) +
+				      nr_migratepages,
+				      ZONE_MOVABLE, 0))
 			continue;
 		return true;
 	}
@@ -1884,18 +1970,80 @@ static struct page *alloc_misplaced_dst_page(struct page *page,
 	return newpage;
 }
 
-static int numamigrate_isolate_page(pg_data_t *pgdat, struct page *page)
+/* Seperate Local/Remote promotion and AutoNUMA migration */
+void numamigrate_reason(struct migrate_detail *m_detail, int src_nid, int dst_nid)
 {
-	int page_lru;
+	int promote_nid = next_promotion_node(src_nid);
+	int migrate_nid = next_migration_node(src_nid);
+
+	/* promote & migrate path were not defined*/
+	if (promote_nid == NUMA_NO_NODE && migrate_nid == NUMA_NO_NODE) {
+		m_detail->reason = MR_NUMA_MISPLACED;
+		m_detail->h_reason = MR_HMEM_MIGRATE;
+		m_detail->h_reason_orig = MR_HMEM_MIGRATE;
+		return;
+	}
+
+	/* Local promotion */
+	if (dst_nid == promote_nid) {
+		m_detail->reason = MR_PROMOTION;
+		m_detail->h_reason = MR_HMEM_LOCAL_PROMOTE;
+		m_detail->h_reason_orig = MR_HMEM_LOCAL_PROMOTE;
+	/* Remote promotion */
+	} else if ((dst_nid != promote_nid) && (promote_nid != NUMA_NO_NODE)
+			&& (dst_nid != migrate_nid)) {
+		m_detail->reason = MR_PROMOTION;
+		m_detail->h_reason = MR_HMEM_REMOTE_PROMOTE;
+		m_detail->h_reason_orig = MR_HMEM_REMOTE_PROMOTE;
+	/* Migrate bewteen same tier memory nodes */
+	} else if (dst_nid == migrate_nid) {
+		m_detail->reason = MR_NUMA_MISPLACED;
+		m_detail->h_reason = MR_HMEM_MIGRATE;
+		m_detail->h_reason_orig = MR_HMEM_MIGRATE;
+	}
+}
+
+void numamigrate_fail_reason(struct migrate_detail *m_detail,
+		enum migrate_hmem_reason h_reason)
+{
+	switch(h_reason) {
+		case MR_HMEM_LOCAL_PROMOTE:
+			m_detail->h_fail_reason = MR_HMEM_LOCAL_PROMOTE_FAIL;
+			break;
+		case MR_HMEM_REMOTE_PROMOTE:
+			m_detail->h_fail_reason = MR_HMEM_REMOTE_PROMOTE_FAIL;
+			break;
+		case MR_HMEM_MIGRATE:
+			m_detail->h_fail_reason = MR_HMEM_MIGRATE_FAIL;
+			break;
+		default:
+			/* Other cases are not considered, yet. */
+			m_detail->h_fail_reason = MR_HMEM_UNKNOWN_FAIL;
+			break;
+	}
+}
 
+int numamigrate_isolate_page(pg_data_t *pgdat, struct page *page,
+                struct migrate_detail *m_detail)
+{
+	int page_lru;
 	VM_BUG_ON_PAGE(compound_order(page) && !PageTransHuge(page), page);
 
 	/* Avoid migrating to a node that is nearly full */
-	if (!migrate_balanced_pgdat(pgdat, 1UL << compound_order(page)))
+	if (!migrate_balanced_pgdat(pgdat, compound_order(page))) {
+		/* Get current migrate fail reason */
+		m_detail->fail_reason = MFR_DST_NODE_FULL;
+		numamigrate_fail_reason(m_detail, m_detail->h_reason);
 		return 0;
+	}
 
-	if (isolate_lru_page(page))
+	if (isolate_lru_page(page)) {
+                count_vm_events(PGMIGRATE_NUMA_ISOLATE_FAIL,
+                                hpage_nr_pages(page));
+		m_detail->fail_reason = MFR_NUMA_ISOLATE;
+		m_detail->h_fail_reason = MR_HMEM_UNKNOWN_FAIL;
 		return 0;
+        }
 
 	/*
 	 * migrate_misplaced_transhuge_page() skips page migration's usual
@@ -1906,6 +2054,8 @@ static int numamigrate_isolate_page(pg_data_t *pgdat, struct page *page)
 	 */
 	if (PageTransHuge(page) && page_count(page) != 3) {
 		putback_lru_page(page);
+		m_detail->fail_reason = MFR_REFCOUNT_FAIL;
+		m_detail->h_fail_reason = MR_HMEM_UNKNOWN_FAIL;
 		return 0;
 	}
 
@@ -1928,6 +2078,298 @@ bool pmd_trans_migrating(pmd_t pmd)
 	return PageLocked(page);
 }
 
+int promote_prep(struct page *page, struct migrate_detail *m_detail)
+{
+	int page_lru;
+	VM_BUG_ON_PAGE(compound_order(page) && !PageTransHuge(page), page);
+
+	if (isolate_lru_page(page)) {
+		count_vm_events(PGMIGRATE_NUMA_ISOLATE_FAIL,
+				hpage_nr_pages(page));
+		m_detail->fail_reason = MFR_NUMA_ISOLATE;
+		m_detail->h_fail_reason = MR_HMEM_UNKNOWN_FAIL;
+		return 0;
+	}
+
+	/*
+	 * migrate_misplaced_transhuge_page() skips page migration's usual
+	 * check on page_count(), so we must do it here, now that the page
+	 * has been isolated: a GUP pin, or any other pin, prevents migration.
+	 * The expected page count is 3: 1 for page's mapcount and 1 for the
+	 * caller's pin and 1 for the reference taken by isolate_lru_page().
+	 */
+	if (PageTransHuge(page) && page_count(page) != 3) {
+		putback_lru_page(page);
+		m_detail->fail_reason = MFR_REFCOUNT_FAIL;
+		m_detail->h_fail_reason = MR_HMEM_UNKNOWN_FAIL;
+		return 0;
+	}
+
+	page_lru = page_is_file_cache(page);
+	mod_node_page_state(page_pgdat(page), NR_ISOLATED_ANON + page_lru,
+				hpage_nr_pages(page));
+
+	/*
+	 * Isolating the page has taken another reference, so the
+	 * caller's reference can be safely dropped without the page
+	 * disappearing underneath us during migration.
+	 */
+	put_page(page);
+	return 1;
+}
+
+int try_promote_page(struct page *page, int dst_nid, struct migrate_detail *m_detail)
+{
+	int nr_remaining;
+	int page_lru;
+	int cpu = smp_processor_id();
+	LIST_HEAD(migratepages);
+
+	if (isolate_lru_page(page)) {
+		count_vm_events(PGMIGRATE_NUMA_ISOLATE_FAIL,
+				hpage_nr_pages(page));
+		m_detail->fail_reason = MFR_NUMA_ISOLATE;
+		m_detail->h_fail_reason = MR_HMEM_UNKNOWN_FAIL;
+		return 0;
+	}
+
+	page_lru = page_is_file_cache(page);
+	mod_node_page_state(page_pgdat(page), NR_ISOLATED_ANON + page_lru,
+				hpage_nr_pages(page));
+	put_page(page);
+
+	list_add(&page->lru, &migratepages);
+
+	nr_remaining = migrate_pages(&migratepages, alloc_promote_page,
+			NULL, cpu, MIGRATE_ASYNC, m_detail);
+
+	if (nr_remaining) {
+		get_page(page);
+		return 0;
+	}
+
+	return 1;
+}
+
+int try_demote_page_cache(struct pglist_data *pgdat, struct lruvec *lruvec)
+{
+	struct list_head *evict_list;
+	struct page *page = NULL;
+	enum lru_list lru;
+
+	spin_lock_irq(&pgdat->lru_lock);
+	for_each_evictable_lru(lru) {
+		/* anon page skip */
+		if (!is_file_lru(lru))
+			continue;
+
+		evict_list = &lruvec->lists[lru];
+		if (!list_empty(evict_list)) {
+			page = lru_to_page(evict_list);
+
+			if (PageWriteback(page))
+				goto unlock_lru_fail;
+
+			if (trylock_busy(page)) {
+				if (PageLRU(page)) {
+					int lru = page_lru(page);
+					get_page(page);
+					ClearPageLRU(page);
+					del_page_from_lru_list(page, lruvec, lru);
+					spin_unlock_irq(&pgdat->lru_lock);
+				} else
+					goto unlock_lru_fail;
+
+				if (trylock_page(page)) {
+					if (!PageHuge(page)) {
+						int rc = migrate_demote_mapping(page);
+
+						if (rc == MIGRATEPAGE_SUCCESS) {
+							unlock_page(page);
+							unlock_busy(page);
+							put_page(page);
+							count_vm_event(PGDEMOTE_FILE);
+							return true;
+						}
+					}
+					unlock_page(page);
+				}
+				unlock_busy(page);
+				return false;
+			} else
+				goto unlock_lru_fail;
+		}
+	}
+
+unlock_lru_fail:
+	spin_unlock_irq(&pgdat->lru_lock);
+	return false;
+}
+
+int try_demote_from_busy_node(struct page *fault_page, int busy_nid, unsigned int mode)
+{
+	struct pglist_data *pgdat = NODE_DATA(busy_nid);
+	struct mem_cgroup *memcg = get_mem_cgroup_from_page(fault_page);
+	struct lruvec *lruvec = mem_cgroup_lruvec(pgdat, memcg);
+	struct page_info *pi, *pi2;
+	struct page *page = NULL;
+	struct page *cold_page = NULL;
+	struct page_ext *page_ext;
+	struct migrate_detail m_detail = {
+		.reason = MR_DEMOTION,
+		.h_reason = MR_HMEM_DEMOTE,
+	};
+	int nr_remaining;
+	int i, fault_lv, lv;
+	int selected = 0;
+	int bt_lv = 0;
+	int ret = -EBUSY;
+	LIST_HEAD(coldpages);
+
+	fault_lv = get_page_access_lv(fault_page);
+	if (fault_lv > MAX_ACCESS_LEVEL || fault_lv < 0)
+		return false;
+
+	/* 1. Try to demote page cache (active & inactive file page) */
+	if (try_demote_page_cache(pgdat, lruvec))
+		return false;
+
+	spin_lock_irq(&pgdat->lru_lock);
+
+	for (i = 0; list_empty(&pgdat->lap_area[i].lap_list) && (i <= MAX_ACCESS_LEVEL); i++)
+		bt_lv++;
+
+	if (fault_lv <= bt_lv) {
+		count_vm_event(PGPROMOTE_LOW_FREQ_FAIL);
+		spin_unlock_irq(&pgdat->lru_lock);
+		return false;
+	}
+
+	/* 2. Get anon cold page */
+	for (lv = bt_lv; lv < fault_lv; lv++) {
+		if (list_empty(&pgdat->lap_area[lv].lap_list))
+			continue;
+
+		list_for_each_entry_safe(pi, pi2, &pgdat->lap_area[lv].lap_list, list) {
+			page = get_page_from_page_info(pi);
+			if (!page) {
+				count_vm_event(PGDEMOTE_NO_PAGE_FAIL);
+				trace_printk("demote: pfn:%lu,last_cpu:%d\n",
+						pi->pfn, pi->last_cpu);
+				page_ext = get_page_ext(pi);
+				clear_bit(PAGE_EXT_TRACKED, &page_ext->flags);
+				clear_bit(PAGE_EXT_BUSY_LOCK, &page_ext->flags);
+				__mod_node_page_state(pgdat, NR_TRACKED, -1);
+				list_del(&pi->list);
+				continue;
+			}
+
+			if (unlikely(!PageLRU(page))) {
+				count_vm_event(PGDEMOTE_NO_LRU_FAIL);
+				del_page_from_lap_list(page);
+				continue;
+			}
+
+			if (unlikely(page_count(page) > 1)) {
+				del_page_from_lap_list(page);
+				continue;
+			}
+
+			if (unlikely(!trylock_busy(page))) {
+				count_vm_event(PGDEMOTE_BUSY_FAIL);
+				continue;
+			}
+
+			/* page is locked by busy_lock */
+
+			// page exchange
+			if (mode & NUMA_BALANCING_EXCHANGE) {
+				del_page_from_lap_list(page);
+				selected = 1;
+				(pgdat->lap_area[lv].demotion_count)++;
+				break;
+				/* page is still locked by busy_lock */
+
+			} else {  // page demotion
+				switch (__isolate_lru_page(page, 0)) {
+					case 0:
+						del_page_from_lap_list(page);
+						(pgdat->lap_area[lv].demotion_count)++;
+						selected = 1;
+
+						list_move(&page->lru, &coldpages);
+
+						lruvec = mem_cgroup_page_lruvec(page, pgdat);
+						update_lru_size(lruvec, page_lru(page), page_zonenum(page),
+								-hpage_nr_pages(page));
+						unlock_busy(page);
+						cold_page = page;
+						/* page is unlocked for busy_lock */
+						break;
+					case -EINVAL:
+					case -EBUSY:
+						del_page_from_lap_list(page);
+						continue;
+					default:
+						BUG();
+				}
+
+				/* Break loop: list_for_each_entry_safe */
+				if (selected)
+					break;
+			}
+		}
+
+		/* Break loop: for (lv = 0; lv < fault_lv; lv++) */
+		if (selected)
+			break;
+	}
+
+	spin_unlock_irq(&pgdat->lru_lock);
+
+	if (!selected)
+		return false;
+
+	if (mode & NUMA_BALANCING_EXCHANGE) {
+
+		/*
+		 * Now, this try to exchange between fault page cold page. If it
+		 * return true, page exchange wil be successed.
+		 */
+
+		cold_page = page;
+
+		/* Prepare page exchange between fault page and cold page */
+		if (trylock_busy(fault_page)) {
+			/* Try to exchange between fault page and cold page */
+			ret = exchange_two_pages(fault_page, cold_page, MIGRATE_SYNC);
+			unlock_busy(fault_page);
+			unlock_busy(cold_page);
+			/* The ref_count of both pages are modified 1 */
+
+			if (ret)  {// exchange fail
+				get_page(fault_page); // page's ref is 2
+				return false;
+			} else	 // exchagne success
+				return true;
+		} else {
+			unlock_busy(cold_page);
+			return false;
+		}
+
+	} else {
+
+		nr_remaining = migrate_pages(&coldpages, alloc_demote_page,
+				NULL, -1, MIGRATE_SYNC, &m_detail);
+
+		if (nr_remaining) {
+			putback_movable_pages(&coldpages);
+			return false;
+		}
+	}
+	return true;
+}
+
 /*
  * Attempt to migrate a misplaced page to the specified destination
  * node. Caller is expected to have an elevated reference count on
@@ -1936,10 +2378,18 @@ bool pmd_trans_migrating(pmd_t pmd)
 int migrate_misplaced_page(struct page *page, struct vm_area_struct *vma,
 			   int node)
 {
-	pg_data_t *pgdat = NODE_DATA(node);
+	struct migrate_detail m_detail = {};
 	int isolated;
+	int page_nid;
 	int nr_remaining;
 	LIST_HEAD(migratepages);
+	int data = node;
+	unsigned int mode = sysctl_numa_balancing_extended_mode;
+	void *alloc_page_func = alloc_misplaced_dst_page;
+#ifdef CONFIG_PAGE_FAULT_PROFILE
+	u64 start_ts, end_ts;
+	int reserved = 0;
+#endif
 
 	/*
 	 * Don't migrate file pages that are mapped in multiple processes
@@ -1956,30 +2406,131 @@ int migrate_misplaced_page(struct page *page, struct vm_area_struct *vma,
 	if (page_is_file_cache(page) && PageDirty(page))
 		goto out;
 
-	isolated = numamigrate_isolate_page(pgdat, page);
-	if (!isolated)
+	page_nid = page_to_nid(page);
+
+#ifdef CONFIG_PAGE_FAULT_PROFILE
+	start_ts = rdtsc();
+#endif
+	/* Define migrate reason by using promotion node info */
+	numamigrate_reason(&m_detail, page_nid, node);
+	isolated = numamigrate_isolate_page(NODE_DATA(node), page, &m_detail);
+	if (!isolated) {
+		if(m_detail.fail_reason == MFR_NUMA_ISOLATE
+				|| m_detail.fail_reason == MFR_REFCOUNT_FAIL)
+			goto migrate_fail;
+
+		if (mode & NUMA_BALANCING_EXCHANGE) {
+			if (m_detail.h_fail_reason == MR_HMEM_MIGRATE_FAIL &&
+					node == next_migration_node(page_nid)) {
+				if (try_exchange_page(page, node)) {
+					return NUMA_NO_NODE;
+				}
+
+				add_page_for_exchange(page, node);
+				goto migrate_fail;
+			}
+		}
+
+		if (mode & NUMA_BALANCING_OPM) {
+			if (background_demotion) {
+				data = smp_processor_id();
+				if (wakeup_kdemoted(data, page)) {
+					if (promote_prep(page, &m_detail)) {
+						alloc_page_func = alloc_promote_page;
+#ifdef CONFIG_PAGE_FAULT_PROFILE
+						reserved = 1;
+#endif
+						goto isolated;
+					}
+				}
+			} else { /* Promote page with On-demand demotion */
+
+#ifdef CONFIG_PAGE_FAULT_PROFILE
+				start_ts = rdtsc();
+#endif
+				if (try_demote_from_busy_node(page, node, mode)) {
+#ifdef CONFIG_PAGE_FAULT_PROFILE
+					end_ts = rdtsc();
+					trace_printk("ondemand_exchange %d %d %llu -1 -1 %d\n",
+						     page_nid, node,
+						     end_ts - start_ts, reserved);
+#endif
+					if (mode & NUMA_BALANCING_EXCHANGE) {
+						return node;
+					} else {
+						if (numamigrate_isolate_page(NODE_DATA(node),
+									page, &m_detail))
+							goto isolated;
+					}
+				}
+			}
+		}
+
+		/* CPM or OPM(Demotion Fail or Demotion Success with Promotin Fail) */
+		if (mode & NUMA_BALANCING_CPM || mode & NUMA_BALANCING_OPM) {
+			data = find_best_migration_node(page, node);
+			if (data != NUMA_NO_NODE && data != node) {
+				if (numamigrate_isolate_page(NODE_DATA(data), page, &m_detail)) {
+					numamigrate_reason(&m_detail, page_nid, data);
+					node = data;
+					goto isolated;
+				}
+			}
+		}
+
+		numamigrate_fail_reason(&m_detail, m_detail.h_reason_orig);
+migrate_fail:
+		/*
+		 * Increment zoneinfo stat about migration fail only if
+		 * this migration failed finally.
+		 */
+		if (m_detail.h_fail_reason != MR_HMEM_UNKNOWN_FAIL) {
+			/* Get original migrate fail reason */
+			inc_hmem_fail_state(m_detail.h_fail_reason, page_nid, node, 0);
+			count_vm_events(PGMIGRATE_DST_NODE_FULL_FAIL, hpage_nr_pages(page));
+		}
+
 		goto out;
+	}
 
+isolated:
 	list_add(&page->lru, &migratepages);
-	nr_remaining = migrate_pages(&migratepages, alloc_misplaced_dst_page,
-				     NULL, node, MIGRATE_ASYNC,
-				     MR_NUMA_MISPLACED);
+
+	nr_remaining = migrate_pages(&migratepages, alloc_page_func,
+				     NULL, data, MIGRATE_ASYNC, &m_detail);
+	// migrate success: page's ref is 1
+	// migrate fail: page's ref is 2
 	if (nr_remaining) {
 		if (!list_empty(&migratepages)) {
 			list_del(&page->lru);
 			dec_node_page_state(page, NR_ISOLATED_ANON +
 					page_is_file_cache(page));
-			putback_lru_page(page);
+			putback_lru_page(page); // page's ref is 1
+		} else {
+			printk("nr_remaining:%d,page_count:%d\n", nr_remaining, page_count(page));
+			goto out;
 		}
-		isolated = 0;
-	} else
+		// isolated = 0;
+		node = NUMA_NO_NODE;
+	} else {
 		count_vm_numa_event(NUMA_PAGE_MIGRATE);
+#ifdef CONFIG_PAGE_FAULT_PROFILE
+		end_ts = rdtsc();
+		if (m_detail.reason == MR_PROMOTION) {
+			trace_printk("promotion %d %d %llu -1 %d %d\n", page_nid, node,
+				     end_ts - start_ts, m_detail.h_reason, reserved);
+		}
+		else if (m_detail.reason == MR_NUMA_MISPLACED)
+			trace_printk("migration %d %d %llu -1 %d %d\n", page_nid, node,
+				     end_ts - start_ts, m_detail.h_reason, reserved);
+#endif
+	}
 	BUG_ON(!list_empty(&migratepages));
-	return isolated;
+	return node;
 
 out:
 	put_page(page);
-	return 0;
+	return NUMA_NO_NODE;
 }
 #endif /* CONFIG_NUMA_BALANCING */
 
@@ -1995,25 +2546,106 @@ int migrate_misplaced_transhuge_page(struct mm_struct *mm,
 				struct page *page, int node)
 {
 	spinlock_t *ptl;
-	pg_data_t *pgdat = NODE_DATA(node);
 	int isolated = 0;
+	int page_nid;
 	struct page *new_page = NULL;
 	int page_lru = page_is_file_cache(page);
 	unsigned long start = address & HPAGE_PMD_MASK;
+	struct migrate_detail m_detail = {};
+	LIST_HEAD(batch_page_list);
+	int mode = sysctl_numa_balancing_extended_mode;
+	int cpu = -1;
+	int dst_nid;
 
-	new_page = alloc_pages_node(node,
-		(GFP_TRANSHUGE_LIGHT | __GFP_THISNODE),
-		HPAGE_PMD_ORDER);
-	if (!new_page)
-		goto out_fail;
-	prep_transhuge_page(new_page);
+	page_nid = page_to_nid(page);
+
+	/* Define migrate reason by using promotion node info */
+	numamigrate_reason(&m_detail, page_nid, node);
 
-	isolated = numamigrate_isolate_page(pgdat, page);
+	isolated = numamigrate_isolate_page(NODE_DATA(node), page, &m_detail);
 	if (!isolated) {
-		put_page(new_page);
+		if(m_detail.fail_reason == MFR_NUMA_ISOLATE
+				|| m_detail.fail_reason == MFR_REFCOUNT_FAIL)
+			goto isolate_thp_fail;
+
+		numamigrate_fail_reason(&m_detail, m_detail.h_reason_orig);
+
+		if (mode & NUMA_BALANCING_EXCHANGE) {
+			if (m_detail.h_fail_reason == MR_HMEM_MIGRATE_FAIL &&
+					node == next_migration_node(page_nid)) {
+				if (try_exchange_page(page, node))
+					return isolated;
+				add_page_for_exchange(page, node);
+				goto isolate_thp_fail;
+			}
+		}
+
+		if (mode & NUMA_BALANCING_OPM) {
+			if (background_demotion) {
+				cpu = smp_processor_id();
+				if (wakeup_kdemoted(cpu, page)) {
+					if (promote_prep(page, &m_detail))
+						goto isolated_thp;
+					else
+						cpu = -1;
+				}
+			} else { /* Promote page with On-demand demotion */
+				if (try_demote_from_busy_node(page, node, mode)) {
+					if (mode & NUMA_BALANCING_EXCHANGE) {
+						return node;
+					} else {
+						if (numamigrate_isolate_page(NODE_DATA(node),
+									page, &m_detail))
+							goto isolated_thp;
+					}
+				}
+			}
+		}
+
+		/* CPM or OPM(Demotion Fail or Demotion Success with Promotin Fail) */
+		if (mode & NUMA_BALANCING_CPM || mode & NUMA_BALANCING_OPM) {
+			dst_nid = find_best_migration_node(page, node);
+			if (dst_nid != NUMA_NO_NODE && dst_nid != node) {
+				if (numamigrate_isolate_page(NODE_DATA(dst_nid), page, &m_detail)) {
+					numamigrate_reason(&m_detail, page_nid, dst_nid);
+					node = dst_nid;
+					goto isolated_thp;
+				}
+			}
+		}
+
+isolate_thp_fail:
+		/*
+		 * Increment zoneinfo stat about migration fail only if
+		 * this migration failed finally.
+		 */
+		if (m_detail.h_fail_reason != MR_HMEM_UNKNOWN_FAIL) {
+			/* Get original migrate fail reason */
+			inc_hmem_fail_state(m_detail.h_fail_reason, page_nid, node, 1);
+			count_vm_events(PGMIGRATE_DST_NODE_FULL_FAIL, hpage_nr_pages(page));
+		}
+
+		/* Existing path */
 		goto out_fail;
 	}
 
+isolated_thp:
+	if (cpu >= 0) /* page will be promoted to reserved page area */
+		new_page = alloc_promote_page(page, cpu);
+	else /* Existing path */
+		new_page = alloc_pages_node(node,
+			(GFP_TRANSHUGE_LIGHT | __GFP_THISNODE),
+			HPAGE_PMD_ORDER);
+
+	if (!new_page) {
+		get_page(page);
+		putback_lru_page(page);
+		mod_node_page_state(page_pgdat(page),
+			 NR_ISOLATED_ANON + page_lru, -HPAGE_PMD_NR);
+		goto out_fail;
+	}
+	prep_transhuge_page(new_page);
+
 	/* Prepare a page as a migration target */
 	__SetPageLocked(new_page);
 	if (PageSwapBacked(page))
@@ -2050,6 +2682,7 @@ int migrate_misplaced_transhuge_page(struct mm_struct *mm,
 		goto out_unlock;
 	}
 
+
 	entry = mk_huge_pmd(new_page, vma->vm_page_prot);
 	entry = maybe_pmd_mkwrite(pmd_mkdirty(entry), vma);
 
@@ -2083,6 +2716,9 @@ int migrate_misplaced_transhuge_page(struct mm_struct *mm,
 
 	spin_unlock(ptl);
 
+	inc_hmem_src_state(m_detail.h_reason, page);
+	inc_hmem_dst_state(m_detail.h_reason, new_page);
+
 	/* Take an "isolate" reference and put new page on the LRU. */
 	get_page(new_page);
 	putback_lru_page(new_page);
@@ -2096,8 +2732,7 @@ int migrate_misplaced_transhuge_page(struct mm_struct *mm,
 	count_vm_numa_events(NUMA_PAGE_MIGRATE, HPAGE_PMD_NR);
 
 	mod_node_page_state(page_pgdat(page),
-			NR_ISOLATED_ANON + page_lru,
-			-HPAGE_PMD_NR);
+			NR_ISOLATED_ANON + page_lru, -HPAGE_PMD_NR);
 	return isolated;
 
 out_fail:
@@ -2113,6 +2748,8 @@ int migrate_misplaced_transhuge_page(struct mm_struct *mm,
 out_unlock:
 	unlock_page(page);
 	put_page(page);
+	mod_node_page_state(page_pgdat(page),
+			NR_ISOLATED_ANON + page_lru, -HPAGE_PMD_NR);
 	return 0;
 }
 #endif /* CONFIG_NUMA_BALANCING */
diff --git a/mm/mmzone.c b/mm/mmzone.c
index 4686fdc23..8bd2bdb51 100644
--- a/mm/mmzone.c
+++ b/mm/mmzone.c
@@ -10,6 +10,10 @@
 #include <linux/mm.h>
 #include <linux/mmzone.h>
 
+#ifdef CONFIG_PAGE_BALANCING
+#include <linux/page_balancing.h>
+#endif
+
 struct pglist_data *first_online_pgdat(void)
 {
 	return NODE_DATA(first_online_node);
diff --git a/mm/mprotect.c b/mm/mprotect.c
index bf38dfbbb..4b0e7a787 100644
--- a/mm/mprotect.c
+++ b/mm/mprotect.c
@@ -28,6 +28,9 @@
 #include <linux/ksm.h>
 #include <linux/uaccess.h>
 #include <linux/mm_inline.h>
+#include <linux/page_balancing.h>
+#include <linux/sched/sysctl.h>
+
 #include <asm/pgtable.h>
 #include <asm/cacheflush.h>
 #include <asm/mmu_context.h>
@@ -43,6 +46,8 @@ static unsigned long change_pte_range(struct vm_area_struct *vma, pmd_t *pmd,
 	spinlock_t *ptl;
 	unsigned long pages = 0;
 	int target_node = NUMA_NO_NODE;
+	LIST_HEAD(coldpages);
+	int mode = sysctl_numa_balancing_extended_mode;
 
 	/*
 	 * Can be called with only the mmap_sem for reading by
@@ -72,6 +77,7 @@ static unsigned long change_pte_range(struct vm_area_struct *vma, pmd_t *pmd,
 		if (pte_present(oldpte)) {
 			pte_t ptent;
 			bool preserve_write = prot_numa && pte_write(oldpte);
+			int prev_lv;
 
 			/*
 			 * Avoid trapping faults against the zero or KSM
@@ -79,6 +85,7 @@ static unsigned long change_pte_range(struct vm_area_struct *vma, pmd_t *pmd,
 			 */
 			if (prot_numa) {
 				struct page *page;
+				int nid;
 
 				page = vm_normal_page(vma, addr, oldpte);
 				if (!page || PageKsm(page))
@@ -97,15 +104,29 @@ static unsigned long change_pte_range(struct vm_area_struct *vma, pmd_t *pmd,
 				if (page_is_file_cache(page) && PageDirty(page))
 					continue;
 
+				nid = page_to_nid(page);
+
 				/* Avoid TLB flush if possible */
-				if (pte_protnone(oldpte))
+				if (pte_protnone(oldpte)) {
+					if (mode & NUMA_BALANCING_OPM) {
+						/* The page is not accessed in last scan period */
+						prev_lv = mod_page_access_lv(page, 0);
+						add_page_for_tracking(page, prev_lv);
+					}
 					continue;
+				}
+
+				/* The page is accessed in last scan period */
+				if (mode & NUMA_BALANCING_OPM) {
+					prev_lv = mod_page_access_lv(page, 1);
+					add_page_for_tracking(page, prev_lv);
+				}
 
 				/*
 				 * Don't mess with PTEs if page is already on the node
 				 * a single-threaded process is running on.
 				 */
-				if (target_node == page_to_nid(page))
+				if (target_node == nid)
 					continue;
 			}
 
@@ -122,6 +143,8 @@ static unsigned long change_pte_range(struct vm_area_struct *vma, pmd_t *pmd,
 			}
 			ptep_modify_prot_commit(vma, addr, pte, oldpte, ptent);
 			pages++;
+
+
 		} else if (IS_ENABLED(CONFIG_MIGRATION)) {
 			swp_entry_t entry = pte_to_swp_entry(oldpte);
 
diff --git a/mm/page_alloc.c b/mm/page_alloc.c
index 9c9194959..4ac511a1f 100644
--- a/mm/page_alloc.c
+++ b/mm/page_alloc.c
@@ -68,6 +68,8 @@
 #include <linux/lockdep.h>
 #include <linux/nmi.h>
 #include <linux/psi.h>
+#include <linux/page_balancing.h>
+#include <linux/sched/sysctl.h>
 
 #include <asm/sections.h>
 #include <asm/tlbflush.h>
@@ -75,6 +77,27 @@
 #include "internal.h"
 #include "shuffle.h"
 
+//#define DEBUG
+
+#ifdef CONFIG_PAGE_BALANCING
+#define MAX_NR_FREE_PAGES 16
+#define MIN_NR_FREE_PAGES (MAX_NR_FREE_PAGES >> 2) // 25%
+
+#define MAX_NR_FREE_HPAGES 4
+#define MIN_NR_FREE_HPAGES 3
+//#define MIN_NR_FREE_HPAGES (MAX_NR_FREE_HPAGES >> 2) // 25%
+struct free_promote_area *promote_area[2];
+
+struct task_struct **page_demoted;
+struct mem_cgroup **kdemoted_memcg;
+wait_queue_head_t *kdemoted_wait;
+int *kdemoted_wakeup;
+spinlock_t *promote_lock;
+
+void add_page_for_promotion(struct page *page, struct free_promote_area *area);
+void free_promote_page(struct page* page);
+#endif
+
 /* prevent >1 _updater_ of zone percpu pageset ->high and ->batch fields */
 static DEFINE_MUTEX(pcp_batch_high_lock);
 #define MIN_PERCPU_PAGELIST_FRACTION	(8)
@@ -670,7 +693,34 @@ static void bad_page(struct page *page, const char *reason,
 
 void free_compound_page(struct page *page)
 {
-	__free_pages_ok(page, compound_order(page));
+	if (PageDemoted(page) && PageTransHuge(page)) {
+		struct page *newpage;
+		int cpu = get_page_last_cpu(page);
+		enum page_type type = HUGEPAGE;
+
+		count_vm_event(PGFREE_DEMOTED);
+		ClearPageDemoted(page);
+
+		__free_pages_ok(page, compound_order(page));
+
+		/*
+		 * FIXME free page must be added to free promote page pool
+		 * witout new page allocation.
+		 */
+		newpage = __alloc_pages_nodemask(GFP_TRANSHUGE_LIGHT | __GFP_THISNODE,
+				HPAGE_PMD_ORDER, cpu_to_node(cpu), NULL);
+		if (newpage) {
+			prep_transhuge_page(newpage);
+			spin_lock_irq(&promote_lock[cpu]);
+			add_page_for_promotion(newpage, &promote_area[type][cpu]);
+			spin_unlock_irq(&promote_lock[cpu]);
+		} else {
+			printk(KERN_ERR "failed to alloc page\n");
+			return;
+		}
+	} else
+		__free_pages_ok(page, compound_order(page));
+
 }
 
 void prep_compound_page(struct page *page, unsigned int order)
@@ -1168,6 +1218,8 @@ static __always_inline bool free_pages_prepare(struct page *page,
 	page->flags &= ~PAGE_FLAGS_CHECK_AT_PREP;
 	reset_page_owner(page, order);
 
+	clear_page_info(page);
+
 	if (!PageHighMem(page)) {
 		debug_check_no_locks_freed(page_address(page),
 					   PAGE_SIZE << order);
@@ -3056,6 +3108,13 @@ void free_unref_page(struct page *page)
 	if (!free_unref_page_prepare(page, pfn))
 		return;
 
+	if (PageDemoted(page)) {
+		free_promote_page(page);
+		count_vm_event(PGFREE_DEMOTED);
+		ClearPageDemoted(page);
+		return;
+	}
+
 	local_irq_save(flags);
 	free_unref_page_commit(page, pfn);
 	local_irq_restore(flags);
@@ -4740,6 +4799,9 @@ __alloc_pages_nodemask(gfp_t gfp_mask, unsigned int order, int preferred_nid,
 		page = NULL;
 	}
 
+	if (page != NULL)
+		reset_page_access_lv(page);
+
 	trace_mm_page_alloc(page, order, alloc_mask, ac.migratetype);
 
 	return page;
@@ -6668,6 +6730,7 @@ static void __meminit pgdat_init_internals(struct pglist_data *pgdat)
 	pgdat_page_ext_init(pgdat);
 	spin_lock_init(&pgdat->lru_lock);
 	lruvec_init(node_lruvec(pgdat));
+	deferred_list_init(pgdat);
 }
 
 static void __meminit zone_init_internals(struct zone *zone, enum zone_type idx, int nid,
@@ -8275,6 +8338,8 @@ static int __alloc_contig_migrate_range(struct compact_control *cc,
 	migrate_prep();
 
 	while (pfn < end || !list_empty(&cc->migratepages)) {
+		struct migrate_detail m_detail = {};
+
 		if (fatal_signal_pending(current)) {
 			ret = -EINTR;
 			break;
@@ -8297,8 +8362,9 @@ static int __alloc_contig_migrate_range(struct compact_control *cc,
 							&cc->migratepages);
 		cc->nr_migratepages -= nr_reclaimed;
 
+		m_detail.reason = MR_CONTIG_RANGE;
 		ret = migrate_pages(&cc->migratepages, alloc_migrate_target,
-				    NULL, 0, cc->mode, MR_CONTIG_RANGE);
+				    NULL, 0, cc->mode, &m_detail);
 	}
 	if (ret < 0) {
 		putback_movable_pages(&cc->migratepages);
@@ -8618,3 +8684,552 @@ bool set_hwpoison_free_buddy_page(struct page *page)
 	return hwpoisoned;
 }
 #endif
+
+#ifdef CONFIG_PAGE_BALANCING
+
+void deferred_list_init(struct pglist_data *pgdat)
+{
+	int lv;
+
+	INIT_LIST_HEAD(&pgdat->deferred_list);
+	for (lv = 0; lv <= MAX_ACCESS_LEVEL; lv++) {
+		INIT_LIST_HEAD(&pgdat->lap_area[lv].lap_list);
+		pgdat->lap_area[lv].nr_free = 0;
+		pgdat->lap_area[lv].demotion_count = 0;
+	}
+}
+
+struct page* alloc_promote_page(struct page *page, unsigned long data) {
+	struct page_info *pi;
+	struct page *newpage;
+	int cpu = (int) data;
+	int nid = cpu_to_node(cpu);
+	struct pglist_data *pgdat = NODE_DATA(nid);
+	enum page_type type;
+
+	/*
+	 * 1) Find the free list in node and cpu number
+	 * 2) Delete a page from the free list if it is not empty
+	 * 3) Return the page
+	 */
+
+	type = PageTransHuge(page);
+
+	spin_lock_irq(&promote_lock[cpu]);
+	if (list_empty(&promote_area[type][cpu].free_list)) {
+#ifdef DEBUG
+		printk(KERN_ERR "Failed to alloc page(node[%d], cpu[%d])\n",
+				nid, cpu);
+#endif
+		count_vm_event(PGPROMOTE_EMPTY_POOL_FAIL);
+		goto out_unlock;
+	}
+
+	pi = list_first_entry_or_null(&promote_area[type][cpu].free_list, struct page_info, list);
+	if (pi == NULL)
+		goto out_unlock;
+
+	newpage = get_page_from_page_info(pi);
+	if (newpage == NULL) {
+		printk("alloc_promote_page: cannot get a free page! \n");
+		count_vm_event(PGPROMOTE_NO_PAGE_FAIL);
+		goto out_unlock;
+	}
+
+	list_del(&pi->list);
+	(promote_area[type][cpu].nr_free)--;
+
+	spin_unlock_irq(&promote_lock[cpu]);
+
+	mod_node_page_state(pgdat, NR_FREE_PROMOTE, -hpage_nr_pages(newpage));
+
+#ifdef DEBUG
+	printk(KERN_INFO "page is allocated from [cpu%d],[nr_free:%d] pool\n",
+			cpu, promote_area[type][cpu].nr_free);
+#endif
+	count_vm_event(PGPROMOTE_FREE_AREA);
+
+	return newpage;
+
+out_unlock:
+	spin_unlock_irq(&promote_lock[cpu]);
+	return NULL;
+}
+EXPORT_SYMBOL(alloc_promote_page);
+
+struct page *alloc_demote_page(struct page *page, unsigned long data)
+{
+	int dst_nid;
+	struct page *newpage;
+	gfp_t mask = (GFP_HIGHUSER_MOVABLE |
+			__GFP_THISNODE | __GFP_NOMEMALLOC |
+			__GFP_NORETRY | __GFP_NOWARN) &
+			~__GFP_RECLAIM;
+	int order = 0;
+
+	dst_nid = find_best_demotion_node(page);
+
+	/* No memory */
+	if (dst_nid == NUMA_NO_NODE)
+		return NULL;
+
+	if (PageTransHuge(page)) {
+		mask |= GFP_TRANSHUGE_LIGHT;
+		order = HPAGE_PMD_ORDER;
+	}
+
+	newpage = __alloc_pages_node(dst_nid, mask, order);
+
+	if (!newpage)
+		return NULL;
+
+	if (PageTransHuge(page))
+		prep_transhuge_page(newpage);
+
+	if (background_demotion) {
+		int cur_cpu = data;
+		/* Set page pool(per cpu) number */
+		set_page_last_cpu(page, cur_cpu);
+		SetPageDemoted(page);
+	}
+
+	return newpage;
+}
+EXPORT_SYMBOL(alloc_demote_page);
+
+void add_page_for_promotion(struct page *page, struct free_promote_area *area)
+{
+	struct page_info *pi;
+	struct pglist_data *pgdat = page_pgdat(page);
+
+	pi = get_page_info_from_page(page);
+	if (!pi)
+		return;
+	list_add(&pi->list, &area->free_list);
+	pi->pfn = page_to_pfn(page);
+	(area->nr_free)++;
+	mod_node_page_state(pgdat, NR_FREE_PROMOTE, hpage_nr_pages(page));
+}
+EXPORT_SYMBOL(add_page_for_promotion);
+
+void free_promote_page(struct page* page)
+{
+	int cpu = get_page_last_cpu(page);
+	enum page_type type = PageTransHuge(page);
+
+	spin_lock_irq(&promote_lock[cpu]);
+	page_ref_inc(page);
+	add_page_for_promotion(page, &promote_area[type][cpu]);
+	spin_unlock_irq(&promote_lock[cpu]);
+
+#ifdef DEBUG
+	printk("page is released to the promote pool[%d]\n", cpu);
+#endif
+}
+EXPORT_SYMBOL(free_promote_page);
+
+
+int __kdemoted(int nid, int cpu, int nr_to_demote, enum page_type type)
+{
+	struct page_info *pi, *pi2;
+	struct page *page = NULL;
+	struct page_ext *page_ext;
+	struct migrate_detail m_detail = {
+		.reason = MR_DEMOTION,
+		.h_reason = MR_HMEM_DEMOTE,
+	};
+	struct pglist_data *pgdat = NODE_DATA(nid);
+	struct lruvec *lruvec = NULL;
+	int nr_remaining;
+	int lv;
+	int nr_taken = 0;
+	int nr_pages;
+#ifdef CONFIG_PAGE_FAULT_PROFILE
+	u64 start_ts, end_ts;
+	int dst_nid = NUMA_NO_NODE;
+#endif
+
+	LIST_HEAD(coldpages);
+
+	if (nr_to_demote <= 0)
+		return false;
+
+#ifdef CONFIG_PAGE_FAULT_PROFILE
+	start_ts = rdtsc();
+#endif
+	/* 1. Try to demote page cache (active & inactive file page) */
+	if (kdemoted_memcg[cpu] != NULL) {
+		lruvec = mem_cgroup_lruvec(pgdat, kdemoted_memcg[cpu]);
+
+		if (type == BASEPAGE) {
+			if(try_demote_page_cache(pgdat, lruvec))
+				return true;
+		} else { /* until THP size */
+			int nr_pages = nr_to_demote << HPAGE_PMD_ORDER;
+			int p;
+			for (p = 0; p < nr_pages; p ++) {
+				if(!try_demote_page_cache(pgdat, lruvec))
+					break;
+			}
+
+			if (p == nr_pages)
+				return true;
+		}
+	}
+
+	spin_lock_irq(&pgdat->lru_lock);
+
+	/* 2. Get anon cold page */
+	for (lv = 0; lv < MAX_ACCESS_LEVEL; lv++) {
+
+		if (nr_taken >= nr_to_demote)
+			break;
+
+		if (list_empty(&pgdat->lap_area[lv].lap_list))
+			continue;
+
+		list_for_each_entry_safe(pi, pi2, &pgdat->lap_area[lv].lap_list, list) {
+			page = get_page_from_page_info(pi);
+			if (!page) {
+				count_vm_event(PGDEMOTE_NO_PAGE_FAIL);
+				trace_printk("kdemoted: pfn:%lu,last_cpu:%d\n",
+						pi->pfn, pi->last_cpu);
+				page_ext = get_page_ext(pi);
+				clear_bit(PAGE_EXT_TRACKED, &page_ext->flags);
+				clear_bit(PAGE_EXT_BUSY_LOCK, &page_ext->flags);
+				__mod_node_page_state(pgdat, NR_TRACKED, -1);
+				list_del(&pi->list);
+				continue;
+			}
+
+			if (PageTransHuge(page) != type)
+				continue;
+
+			nr_pages = 1;
+
+			if (unlikely(!PageLRU(page))) {
+				count_vm_event(PGDEMOTE_NO_LRU_FAIL);
+				del_page_from_lap_list(page);
+				continue;
+			}
+
+			if (type == BASEPAGE) {
+				if (unlikely(page_count(page) > 1)) {
+					del_page_from_lap_list(page);
+					continue;
+				}
+			}
+
+			if (unlikely(!trylock_busy(page))) {
+				count_vm_event(PGDEMOTE_BUSY_FAIL);
+				continue;
+			}
+			/* page is locked by busy_lock */
+
+			switch (__isolate_lru_page(page, 0)) { //get_page()
+				case 0:
+
+#ifdef CONFIG_PAGE_FAULT_PROFILE
+					dst_nid = next_demotion_node(page_to_nid(page));
+#endif
+					del_page_from_lap_list(page);
+					(pgdat->lap_area[lv].demotion_count)++;
+
+					nr_taken += nr_pages;
+
+					list_move(&page->lru, &coldpages);
+
+					lruvec = mem_cgroup_page_lruvec(page, pgdat);
+					update_lru_size(lruvec, page_lru(page), page_zonenum(page),
+							-hpage_nr_pages(page));
+					unlock_busy(page);
+					/* page is unlocked for busy_lock */
+					break;
+				case -EINVAL:
+				case -EBUSY:
+					del_page_from_lap_list(page);
+					continue;
+				default:
+					BUG();
+			}
+
+			if (nr_taken >= nr_to_demote)
+				break;
+		}
+	}
+	spin_unlock_irq(&pgdat->lru_lock);
+
+	if (!nr_taken)
+		return false;
+
+	nr_remaining = migrate_pages(&coldpages, alloc_demote_page,
+			NULL, cpu, MIGRATE_SYNC, &m_detail);
+
+	if (nr_remaining) {
+		putback_movable_pages(&coldpages);
+		return false;
+	}
+
+	count_vm_events(PGDEMOTE_BACKGROUND, nr_taken - nr_remaining);
+#ifdef CONFIG_PAGE_FAULT_PROFILE
+	end_ts = rdtsc();
+	trace_printk("demotion %d %d %llu %d\n", nid, dst_nid, end_ts - start_ts,
+		     nr_to_demote);
+#endif
+
+	return true;
+}
+
+int wakeup_kdemoted(int cpu, struct page *page)
+{
+	int nr_free, i;
+	int bt_lv = 0;
+	int nid = cpu_to_node(cpu);
+	int lv = get_page_access_lv(page);
+	struct pglist_data *pgdat = NODE_DATA(nid);
+	enum page_type type = PageTransHuge(page);
+	int thp_enabled = transparent_hugepage_flags & (1 << TRANSPARENT_HUGEPAGE_FLAG);
+
+	/* If THP is enabled, only allow for THP pages to proactively promote */
+	if (thp_enabled && (type == BASEPAGE))
+		return 0;
+
+	for (i = 0; list_empty(&pgdat->lap_area[i].lap_list) && (i <= MAX_ACCESS_LEVEL); i++)
+		bt_lv++;
+
+	if (lv <= bt_lv) {
+		count_vm_event(PGPROMOTE_LOW_FREQ_FAIL);
+		return 0;
+	}
+
+	nr_free = promote_area[type][cpu].nr_free;
+
+	if (type == BASEPAGE) {
+		if (nr_free > MIN_NR_FREE_PAGES)
+			return 1;
+	} else if (type == HUGEPAGE) {
+		if (nr_free > MIN_NR_FREE_HPAGES)
+			return 1;
+	}
+
+	/* kdemoted is busy */
+	if (!waitqueue_active(&kdemoted_wait[cpu])) {
+		return 0;
+	}
+
+	kdemoted_wakeup[cpu] = 1;
+	kdemoted_memcg[cpu] = get_mem_cgroup_from_page(page);
+
+	wake_up_interruptible(&kdemoted_wait[cpu]);
+	return 0;
+}
+EXPORT_SYMBOL(wakeup_kdemoted);
+
+static int kdemoted_wait_event(int cpu)
+{
+	int mode = sysctl_numa_balancing_extended_mode;
+
+	return (mode & NUMA_BALANCING_OPM) && kdemoted_wakeup[cpu];
+}
+
+int kdemoted(void *p)
+{
+	int *ptr_cpu = (int *)p;
+	int cpu = *ptr_cpu;
+	int nid = cpu_to_node(cpu);
+	int ret = true;
+	int nr_to_demote = 0;
+	int max_nr_free_pages = MAX_NR_FREE_PAGES;
+	enum page_type type = BASEPAGE;
+
+	printk("kdemoted init: cpu[%d],nid[%d]\n", cpu, nid);
+	set_freezable();
+
+	while(!kthread_should_stop()) {
+		int thp_enabled =
+			transparent_hugepage_flags & (1 << TRANSPARENT_HUGEPAGE_FLAG);
+		wait_event_interruptible(kdemoted_wait[cpu],
+						kdemoted_wait_event(cpu));
+
+		for (type = BASEPAGE; type < NR_PAGE_TYPE; type++) {
+			if (type == BASEPAGE)
+				max_nr_free_pages = MAX_NR_FREE_PAGES;
+			else if (type == HUGEPAGE)
+				max_nr_free_pages = MAX_NR_FREE_HPAGES;
+
+			/* Skip THP */
+			if (!thp_enabled && type == HUGEPAGE)
+				continue;
+			else if (thp_enabled && type == BASEPAGE)
+				continue;
+
+			if (batch_demotion)
+				nr_to_demote =	max_nr_free_pages - promote_area[type][cpu].nr_free;
+			else
+				nr_to_demote = 1;
+
+#ifdef DEBUG
+			printk("kdemoted%d,type:%s before nr_free:%d,"
+					"nr_to_demote:%d\n",
+					cpu, type?"hugepage":"basepage",
+					promote_area[type][cpu].nr_free,
+					nr_to_demote);
+#endif
+			ret = __kdemoted(nid, cpu, nr_to_demote, type);
+#ifdef DEBUG
+			printk("kdemoted%d,type:%s after nr_free:%d,"
+					"nr_to_demote:%d\n",
+					cpu, type?"hugepage":"basepage",
+					promote_area[type][cpu].nr_free,
+					nr_to_demote);
+#endif
+			spin_lock_irq(&promote_lock[cpu]);
+			if (promote_area[type][cpu].nr_free >= max_nr_free_pages
+					|| next_demotion_node(nid) == NUMA_NO_NODE
+					|| ret == false)
+				kdemoted_wakeup[cpu] = 0;
+			spin_unlock_irq(&promote_lock[cpu]);
+		}
+	}
+
+	return 0;
+}
+EXPORT_SYMBOL(kdemoted);
+
+int page_demoted_run(int *cpu)
+{
+	int nid = cpu_to_node(*cpu);
+	int ret = 0;
+
+	page_demoted[*cpu] = kthread_run(kdemoted, cpu, "kdemoted%d", *cpu);
+	if (IS_ERR(page_demoted[*cpu])) {
+		pr_err("Failed to start kdemoted on node%d/cpu%d\n", nid, *cpu);
+		ret = PTR_ERR(page_demoted[*cpu]);
+		page_demoted[*cpu] = NULL;
+	}
+	return ret;
+}
+
+void kdemoted_stop(int cpu)
+{
+	if (page_demoted[cpu]) {
+		kthread_stop(page_demoted[cpu]);
+		page_demoted = NULL;
+	}
+}
+
+static int __init kdemoted_init(void)
+{
+	const struct cpumask *cpumasks = cpu_online_mask;
+	int nr_cpus = cpumask_weight(cpumasks);
+	int *cpu = kmalloc(sizeof(int) * nr_cpus, GFP_KERNEL);
+	int i;
+	int node;
+
+	page_demoted = kmalloc(sizeof(struct task_struct *) * nr_cpus, GFP_KERNEL);
+	kdemoted_wait = kmalloc(sizeof(wait_queue_head_t) * nr_cpus, GFP_KERNEL);
+	kdemoted_wakeup = kmalloc(sizeof(int) * nr_cpus, GFP_KERNEL);
+	kdemoted_memcg = kmalloc(sizeof(struct mem_cgroup *) * nr_cpus, GFP_KERNEL);
+
+	for_each_cpu(i, cpumasks) {
+		cpu[i] = i;
+		node = cpu_to_node(cpu[i]);
+		page_demoted[i] = kmalloc(sizeof(struct task_struct), GFP_KERNEL);
+		init_waitqueue_head(&kdemoted_wait[i]);
+		kdemoted_wakeup[i] = 0;
+		kdemoted_memcg[i] = NULL;
+
+		page_demoted_run(&cpu[i]);
+		do_set_cpus_allowed(page_demoted[i], cpumask_of_node(node));
+	}
+	return 0;
+}
+module_init(kdemoted_init)
+
+void __init promote_init(void)
+{
+	const struct cpumask *cpumasks = cpu_online_mask;
+	enum page_type type = BASEPAGE;
+	int nr_cpus = cpumask_weight(cpumasks);
+	int cpu;
+#if 0
+	struct page *page;
+	int nr_page;
+#endif
+	int order = 0;
+	int max_nr_free_pages = MAX_NR_FREE_PAGES;
+
+	struct page_info pi;
+
+	gfp_t bmask = (GFP_HIGHUSER_MOVABLE |
+			__GFP_THISNODE | __GFP_NOMEMALLOC |
+			__GFP_NORETRY | __GFP_NOWARN) &
+			~__GFP_RECLAIM; // basepage mask;
+	gfp_t hmask = bmask | GFP_TRANSHUGE_LIGHT; //hugepage mask;
+	gfp_t mask;
+	pi.pfn = 0;
+	pi.last_cpu = 0;
+
+	printk(KERN_INFO "promote_init called\n");
+
+	promote_lock = kmalloc(sizeof(spinlock_t) * nr_cpus, GFP_KERNEL);
+	if (!promote_lock) {
+		printk(KERN_ERR "kmalloc failed in promote_lock\n");
+		return;
+	}
+
+	for (type = BASEPAGE; type < NR_PAGE_TYPE; type++) {
+		promote_area[type] = kmalloc(sizeof(struct free_promote_area) * nr_cpus, GFP_KERNEL);
+		if (!promote_area[type]) {
+			printk(KERN_ERR "kmalloc failed in promote_area\n");
+			return;
+		}
+
+		if (type == HUGEPAGE) {
+			mask = hmask;
+			max_nr_free_pages = MAX_NR_FREE_HPAGES;
+			order = HPAGE_PMD_ORDER;
+		} else {
+			mask = bmask;
+			max_nr_free_pages = MAX_NR_FREE_PAGES;
+			order = 0;
+		}
+
+		/* Initialized values */
+		for_each_cpu(cpu, cpumasks) {
+			INIT_LIST_HEAD(&promote_area[type][cpu].free_list);
+			promote_area[type][cpu].nr_free = 0;
+			spin_lock_init(&promote_lock[cpu]);
+
+
+#if 0 /* maybe unnecessary..? */
+			for(nr_page = 0; nr_page < max_nr_free_pages; nr_page++) {
+				page = __alloc_pages_nodemask(mask, order,
+						cpu_to_node(cpu), NULL);
+
+				if (!page) {
+					printk(KERN_ERR "failed to alloc page\n");
+					return;
+				}
+
+				if (PageTransHuge(page))
+					prep_transhuge_page(page);
+				add_page_for_promotion(page, &promote_area[type][cpu]);
+			}
+#endif
+		}
+	}
+
+	printk(KERN_INFO "free_promote_init success!\n");
+	printk(KERN_INFO "Free pages per cpu\n");
+	printk(KERN_INFO "NR_CPUS:%d\n", nr_cpus);
+	printk(KERN_INFO "free_promote_area[%d-%d]: (base, %d), (huge, %d)\n",
+			0, nr_cpus - 1,
+			nr_cpus * MAX_NR_FREE_PAGES, nr_cpus * MAX_NR_FREE_HPAGES);
+	printk(KERN_INFO "added page extension size:%ld B\n", sizeof(pi));
+	printk(KERN_INFO "struct list_head list size: %ld B\n", sizeof(pi.list));
+	printk(KERN_INFO "unsigned long pfn size: %ld B\n", sizeof(pi.pfn));
+	printk(KERN_INFO "int8_t last_cpu size: %ld B\n", sizeof(pi.last_cpu));
+	printk(KERN_INFO "access_bitmap size: %ld B\n", sizeof(pi.access_bitmap));
+}
+#endif
diff --git a/mm/page_balancing.c b/mm/page_balancing.c
new file mode 100644
index 000000000..5f922807f
--- /dev/null
+++ b/mm/page_balancing.c
@@ -0,0 +1,1119 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Balance pages in tiered memory system. This scheme includes page promotion,
+ * demotion, and exchange across NUMA nodes.
+ *
+ * Author: Jonghyeon Kim <tome01@ajou.ac.kr>
+ */
+
+#include <linux/debugfs.h>
+#include <linux/mm.h>
+#include <linux/huge_mm.h>
+#include <linux/uaccess.h>
+#include <linux/jump_label.h>
+#include <linux/exchange.h>
+#include <linux/memcontrol.h>
+#include <linux/node.h>
+
+#include <linux/page_balancing.h>
+
+#include "internal.h"
+
+unsigned int background_demotion = 0;
+unsigned int batch_demotion = 0;
+unsigned int thp_mt_copy = 0;
+unsigned int skip_lower_tier = 1;
+
+static bool need_page_balancing(void)
+{
+	return true;
+}
+
+static inline struct page_info *get_page_info(struct page_ext *page_ext)
+{
+	return (void *)page_ext + page_info_ops.offset;
+}
+
+struct page_ext *get_page_ext(struct page_info *page_info)
+{
+	return (void *)page_info - page_info_ops.offset;
+}
+
+struct page *get_page_from_page_info(struct page_info *page_info)
+{
+	if (page_info->pfn)
+		return pfn_to_page(page_info->pfn);
+	else
+		return NULL;
+}
+
+struct page_info *get_page_info_from_page(struct page *page)
+{
+	struct page_ext *page_ext = lookup_page_ext(page);
+	if (!page_ext)
+		return NULL;
+	return get_page_info(page_ext);
+}
+
+void set_page_to_page_info(struct page *page,
+				struct page_info *page_info)
+{
+	page_info->pfn = page_to_pfn(page);
+}
+
+unsigned int __get_page_access_lv(struct page_info *pi)
+{
+	int i;
+	unsigned int lv = 0;
+	u8 bitmap = pi->access_bitmap;
+
+	for (i = 0; i < ACCESS_HISTORY_SIZE; i++) {
+		lv = lv + (bitmap & 1);
+		bitmap = bitmap >> 1;
+	}
+	return lv;
+}
+
+static inline unsigned int __PageTracked(struct page_ext *page_ext)
+{
+	return test_bit(PAGE_EXT_TRACKED, &page_ext->flags);
+}
+
+static inline void __SetPageTracked(struct page_ext *page_ext)
+{
+	set_bit(PAGE_EXT_TRACKED, &page_ext->flags);
+}
+
+static inline void __ClearPageTracked(struct page_ext *page_ext)
+{
+	clear_bit(PAGE_EXT_TRACKED, &page_ext->flags);
+}
+
+static inline unsigned int __PageDeferred(struct page_ext *page_ext)
+{
+	return test_bit(PAGE_EXT_DEFERRED, &page_ext->flags);
+}
+
+static inline void __SetPageDeferred(struct page_ext *page_ext)
+{
+	set_bit(PAGE_EXT_DEFERRED, &page_ext->flags);
+}
+
+static inline void __ClearPageDeferred(struct page_ext *page_ext)
+{
+	clear_bit(PAGE_EXT_DEFERRED, &page_ext->flags);
+}
+
+
+static inline unsigned int __PageDemoted(struct page_ext *page_ext)
+{
+	return test_bit(PAGE_EXT_DEMOTED, &page_ext->flags);
+}
+
+static inline void __SetPageDemoted(struct page_ext *page_ext)
+{
+	set_bit(PAGE_EXT_DEMOTED, &page_ext->flags);
+}
+
+static inline void __ClearPageDemoted(struct page_ext *page_ext)
+{
+	clear_bit(PAGE_EXT_DEMOTED, &page_ext->flags);
+}
+
+static inline void __del_page_from_deferred_list(struct page_ext *page_ext,
+				struct page *page)
+{
+	struct page_info *pi = get_page_info(page_ext);
+	struct pglist_data *pgdat = page_pgdat(page);
+
+	if (__PageTracked(page_ext)) {
+		unsigned int lv = __get_page_access_lv(pi);
+		if (--(pgdat->lap_area[lv].nr_free) < 0)
+			pgdat->lap_area[lv].nr_free = 0;
+
+		__ClearPageTracked(page_ext);
+		__mod_lruvec_page_state(page, NR_TRACKED, -hpage_nr_pages(page));
+		list_del(&pi->list);
+	} else if (__PageDeferred(page_ext)) {
+		__ClearPageDeferred(page_ext);
+		__mod_lruvec_page_state(page, NR_DEFERRED, -hpage_nr_pages(page));
+		list_del(&pi->list);
+	}
+}
+
+static inline unsigned int __PageBusyLock(struct page_ext *page_ext)
+{
+	return test_bit(PAGE_EXT_BUSY_LOCK, &page_ext->flags);
+}
+
+static inline void __lock_busy(struct page_ext *page_ext)
+{
+	set_bit(PAGE_EXT_BUSY_LOCK, &page_ext->flags);
+}
+
+static inline unsigned int __trylock_busy(struct page_ext *page_ext)
+{
+	if (test_bit(PAGE_EXT_BUSY_LOCK, &page_ext->flags))
+		return 0;
+	else {
+		__lock_busy(page_ext);
+		return 1;
+	}
+}
+
+static inline void __unlock_busy(struct page_ext *page_ext)
+{
+	clear_bit(PAGE_EXT_BUSY_LOCK, &page_ext->flags);
+}
+
+static inline void __clear_page_info(struct page_ext *page_ext)
+{
+	struct page_info *pi = get_page_info(page_ext);
+	pi->pfn = 0;
+	pi->access_bitmap = 0;
+}
+
+unsigned int PageTracked(struct page *page)
+{
+	struct page_ext *page_ext;
+
+	if (!(sysctl_numa_balancing_extended_mode & NUMA_BALANCING_OPM))
+		return 0;
+
+	page_ext = lookup_page_ext(page);
+	if (unlikely(!page_ext))
+		return 0;
+
+	return __PageTracked(page_ext);
+}
+
+void ClearPageTracked(struct page *page)
+{
+	struct page_ext *page_ext;
+
+	if (!(sysctl_numa_balancing_extended_mode & NUMA_BALANCING_OPM))
+		return;
+
+	page_ext = lookup_page_ext(page);
+	if (unlikely(!page_ext))
+		return;
+
+	if (!__PageTracked(page_ext))
+		return;
+
+	VM_BUG_ON(!__PageTracked(page_ext));
+
+	__ClearPageTracked(page_ext);
+	__mod_lruvec_page_state(page, NR_TRACKED, -hpage_nr_pages(page));
+}
+
+unsigned int PageDeferred(struct page *page)
+{
+	struct page_ext *page_ext;
+
+	if (!(sysctl_numa_balancing_extended_mode & NUMA_BALANCING_EXCHANGE))
+		return 0;
+
+	page_ext = lookup_page_ext(page);
+	if (unlikely(!page_ext))
+		return 0;
+
+	return __PageDeferred(page_ext);
+}
+
+
+void ClearPageDeferred(struct page *page)
+{
+	struct page_ext *page_ext;
+
+	if (!(sysctl_numa_balancing_extended_mode & NUMA_BALANCING_EXCHANGE))
+		return;
+
+	page_ext = lookup_page_ext(page);
+	if (unlikely(!page_ext))
+		return;
+
+	if (!__PageDeferred(page_ext))
+		return;
+
+	VM_BUG_ON(!__PageDeferred(page_ext));
+
+	__ClearPageDeferred(page_ext);
+	__mod_lruvec_page_state(page, NR_DEFERRED, -hpage_nr_pages(page));
+}
+
+unsigned int PageDemoted(struct page *page)
+{
+	struct page_ext *page_ext = lookup_page_ext(page);
+	if (unlikely(!page_ext))
+		return 0;
+	return __PageDemoted(page_ext);
+}
+
+
+void ClearPageDemoted(struct page *page)
+{
+	struct page_ext *page_ext;
+
+	page_ext = lookup_page_ext(page);
+	if (unlikely(!page_ext))
+		return;
+
+	if (!__PageDemoted(page_ext))
+		return;
+
+	__ClearPageDemoted(page_ext);
+}
+
+#ifdef CONFIG_PAGE_BALANCING_DEBUG
+void trace_dump_page(struct page *page, const char *msg)
+{
+	trace_printk("dump:%s page(%p):0x%lx,"
+		"refcount:%d,mapcount:%d,mapping:%p,index:%#lx,flags:%#lx(%pGp),"
+		"%s,%s,%s,%s,page_nid:%d\n",
+		msg,
+		page,
+		page_to_pfn(page),
+		page_ref_count(page),
+		PageSlab(page)?0:page_mapcount(page),
+		page->mapping, page_to_pgoff(page),
+		page->flags, &page->flags,
+		PageCompound(page)?"compound_page":"single_page",
+		PageDirty(page)?"dirty":"clean",
+		PageDeferred(page)?"deferred":"nondeferred",
+		PageTracked(page)?"tracked":"nontracked",
+		page_to_nid(page)
+		);
+}
+
+static void print_access_history(const char *msg, struct page *page,
+		struct page_info *pi)
+{
+	char buf[10];
+	unsigned int i, node_id = page_to_nid(page);
+	unsigned long pfn = page_to_pfn(page);
+	char hi, lo;
+	u8 bitmap = pi->access_bitmap;
+
+	for (i = 0; i < ACCESS_HISTORY_SIZE; i++) {
+		if (bitmap & 1)
+			buf[i] = '1';
+		else
+			buf[i] = '0';
+
+		bitmap = bitmap >> 1;
+	}
+
+	buf[ACCESS_HISTORY_SIZE] = '\0';
+
+	trace_printk("%s pfn:[%6lx],access:[%8s],lv:[%u],node:[%u],last_cpu[%d]\n",
+			msg, pfn, buf, __get_page_access_lv(pi), node_id, pi->last_cpu);
+}
+#else
+static inline void print_access_history(const char *msg, struct page *page,
+		struct page_info *pi)
+{
+}
+#endif /* CONFIG_PAGE_BALANCING_DEBUG */
+
+void SetPageTracked(struct page *page)
+{
+	struct page_ext *page_ext;
+
+	if (!(sysctl_numa_balancing_extended_mode & NUMA_BALANCING_OPM))
+		return;
+
+	page_ext = lookup_page_ext(page);
+	if (unlikely(!page_ext))
+		return;
+
+	__SetPageTracked(page_ext);
+	__mod_lruvec_page_state(page, NR_TRACKED, hpage_nr_pages(page));
+}
+
+void SetPageDeferred(struct page *page)
+{
+	struct page_ext *page_ext;
+
+	if (!(sysctl_numa_balancing_extended_mode & NUMA_BALANCING_EXCHANGE))
+		return;
+
+	page_ext = lookup_page_ext(page);
+	if (unlikely(!page_ext))
+		return;
+
+	__SetPageDeferred(page_ext);
+	__mod_lruvec_page_state(page, NR_DEFERRED, hpage_nr_pages(page));
+}
+
+void SetPageDemoted(struct page *page)
+{
+	struct page_ext *page_ext;
+
+	if (!(sysctl_numa_balancing_extended_mode & NUMA_BALANCING_OPM))
+		return;
+	page_ext = lookup_page_ext(page);
+	if (unlikely(!page_ext))
+		return;
+
+	__SetPageDemoted(page_ext);
+}
+
+void clear_page_info(struct page *page)
+{
+	struct page_ext *page_ext = lookup_page_ext(page);
+	if (unlikely(!page_ext))
+		return;
+	__clear_page_info(page_ext);
+}
+
+/* page should be locked from pgdat->lru_lock */
+void del_page_from_deferred_list(struct page *page)
+{
+	struct page_ext *page_ext;
+	int mode = sysctl_numa_balancing_extended_mode;
+	mode = mode & (NUMA_BALANCING_EXCHANGE | NUMA_BALANCING_OPM);
+
+	if (!mode)
+		return;
+
+	page_ext = lookup_page_ext(page);
+	__del_page_from_deferred_list(page_ext, page);
+}
+
+/* page, lap_list should be locked from pgdat->lru_lock */
+void del_page_from_lap_list(struct page *page)
+{
+	del_page_from_deferred_list(page);
+}
+
+void copy_page_info(struct page *oldpage, struct page *newpage)
+{
+	struct page_ext *old_ext, *new_ext;
+	struct page_info *old_pi, *new_pi;
+	int mode = sysctl_numa_balancing_extended_mode;
+	mode = mode & (NUMA_BALANCING_CPM | NUMA_BALANCING_OPM);
+
+	if (!mode)
+		return;
+
+	old_ext = lookup_page_ext(oldpage);
+	new_ext = lookup_page_ext(newpage);
+	if (unlikely(!old_ext || !new_ext))
+		return;
+
+	old_pi = get_page_info(old_ext);
+	new_pi = get_page_info(new_ext);
+
+	if (mode & NUMA_BALANCING_OPM) {
+		new_pi->access_bitmap = old_pi->access_bitmap;
+
+		print_access_history("migrate-old", oldpage, old_pi);
+		print_access_history("migrate-new", newpage, new_pi);
+	}
+
+}
+
+void exchange_page_info(struct page *from_page, struct page *to_page)
+{
+	struct page_ext *from_ext = lookup_page_ext(from_page);
+	struct page_ext *to_ext = lookup_page_ext(to_page);
+	struct page_info *from_pi, *to_pi;
+	struct page_info tmp_pi;
+
+	if (unlikely(!from_ext || !to_ext))
+		return;
+	from_pi = get_page_info(from_ext);
+	to_pi = get_page_info(to_ext);
+
+	tmp_pi.pfn = 0;
+	tmp_pi.access_bitmap = 0;
+
+	if (sysctl_numa_balancing_extended_mode & NUMA_BALANCING_OPM) {
+		tmp_pi.access_bitmap = to_pi->access_bitmap;
+		to_pi->access_bitmap = from_pi->access_bitmap;
+		from_pi->access_bitmap = tmp_pi.access_bitmap;
+
+		print_access_history("exchange-from", from_page, from_pi);
+		print_access_history("exchange-  to", to_page, to_pi);
+	}
+
+}
+
+int get_page_last_cpu(struct page *page)
+{
+	struct page_ext *page_ext = lookup_page_ext(page);
+	struct page_info *pi;
+	if (unlikely(!page_ext))
+		return NUMA_NO_NODE;
+	pi = get_page_info(page_ext);
+	return pi->last_cpu;
+}
+
+void set_page_last_cpu(struct page *page, int cpu)
+{
+	struct page_ext *page_ext = lookup_page_ext(page);
+	struct page_info *pi;
+	if (unlikely(!page_ext))
+		return;
+	pi = get_page_info(page_ext);
+	pi->last_cpu = cpu;
+}
+
+unsigned int PageBusyLock(struct page *page)
+{
+	struct page_ext *page_ext = lookup_page_ext(page);
+	if (unlikely(!page_ext))
+		return 0;
+	return __PageBusyLock(page_ext);
+}
+
+void lock_busy(struct page *page)
+{
+	struct page_ext *page_ext = lookup_page_ext(page);
+	if (unlikely(!page_ext))
+		return;
+	__lock_busy(page_ext);
+}
+
+unsigned int trylock_busy(struct page *page)
+{
+	struct page_ext *page_ext = lookup_page_ext(page);
+	if (unlikely(!page_ext))
+		return 0;
+	return __trylock_busy(page_ext);
+}
+
+void unlock_busy(struct page *page)
+{
+	struct page_ext *page_ext = lookup_page_ext(page);
+	if (unlikely(!page_ext))
+		return;
+	__unlock_busy(page_ext);
+}
+
+void add_page_for_tracking(struct page *page, unsigned int prev_lv)
+{
+	struct page_ext *page_ext;
+	struct page_info *pi;
+	struct pglist_data *pgdat;
+	unsigned int lv;
+	int recent;
+	int thp_enabled = transparent_hugepage_flags & (1 << TRANSPARENT_HUGEPAGE_FLAG);
+	int mode = sysctl_numa_balancing_extended_mode;
+	mode = mode & NUMA_BALANCING_OPM;
+
+	if (!mode)
+		return;
+
+	if (skip_lower_tier) {
+		/*
+		 * The lowest tier memory node does not need to mark cold page.
+		 * So, we skip followed process that add page to cold page list.
+		 */
+		if (is_bottom_node(page_to_nid(page)))
+			return;
+	}
+
+	/* Skip tail pages */
+	if (PageTail(page))
+		return;
+
+	/* If THP is enabled, only allow tracking THP pages */
+	if (thp_enabled && !PageTransHuge(page))
+		return;
+
+	if (page_count(page) > 1)
+		return;
+
+	page_ext = lookup_page_ext(page);
+	if (unlikely(!page_ext))
+		return;
+
+	pgdat = page_pgdat(page);
+
+	spin_lock_irq(&pgdat->lru_lock);
+
+	pi = get_page_info(page_ext);
+	lv = __get_page_access_lv(pi);
+
+	if(__PageBusyLock(page_ext)
+		|| __PageDeferred(page_ext)
+		|| !PageLRU(page))
+	{
+		spin_unlock_irq(&pgdat->lru_lock);
+		return;
+	}
+
+	VM_BUG_ON_PAGE(!PageLRU(page), page);
+	VM_BUG_ON_PAGE(__PageBusyLock(page_ext), page);
+	VM_BUG_ON_PAGE(__PageDeferred(page_ext), page);
+
+	set_page_to_page_info(page, pi);
+
+	/* Other lv page move to lap_list with changed lv */
+	if (__PageTracked(page_ext)) {
+		if (lv != prev_lv) {
+			if (--(pgdat->lap_area[prev_lv].nr_free) < 0)
+				pgdat->lap_area[prev_lv].nr_free = 0;
+			(pgdat->lap_area[lv].nr_free)++;
+		}
+
+		recent = pi->access_bitmap & 0x1;
+		// Recently accessd
+		if (recent == 1) {
+			list_move_tail(&pi->list, &pgdat->lap_area[lv].lap_list);
+			print_access_history("    accessed", page, pi);
+		} else { // Recently not accessed
+			list_move(&pi->list, &pgdat->lap_area[lv].lap_list);
+			print_access_history("not_accessed", page, pi);
+		}
+
+	/* Add page to lap_list */
+	} else {
+		__SetPageTracked(page_ext);
+
+		recent = pi->access_bitmap & 0x1;
+		// Recently accessd
+		if (recent == 1) {
+			list_add_tail(&pi->list, &pgdat->lap_area[lv].lap_list);
+			print_access_history("    accessed", page, pi);
+		} else { // Recently not accessed
+			list_add(&pi->list, &pgdat->lap_area[lv].lap_list);
+			print_access_history("not_accessed", page, pi);
+		}
+
+		__mod_lruvec_page_state(page, NR_TRACKED, hpage_nr_pages(page));
+		(pgdat->lap_area[lv].nr_free)++;
+	}
+
+	spin_unlock_irq(&pgdat->lru_lock);
+}
+
+void add_page_for_exchange(struct page *page, int node)
+{
+	struct page_ext *page_ext;
+	struct page_info *pi;
+	struct pglist_data *pgdat;
+
+	if (!(sysctl_numa_balancing_extended_mode & NUMA_BALANCING_EXCHANGE))
+		return;
+
+	page_ext = lookup_page_ext(page);
+	if (unlikely(!page_ext))
+		return;
+
+	pgdat = page_pgdat(page);
+
+	spin_lock_irq(&pgdat->lru_lock);
+
+	//FIXME: defererd page move to head of list
+	if (__PageDeferred(page_ext)
+		|| __PageBusyLock(page_ext)
+		|| !PageLRU(page)) {
+		spin_unlock_irq(&pgdat->lru_lock);
+		return;
+	}
+
+	VM_BUG_ON_PAGE(!PageLRU(page), page);
+	VM_BUG_ON_PAGE(__PageDeferred(page_ext), page);
+	VM_BUG_ON_PAGE(__PageBusyLock(page_ext), page);
+
+	pi = get_page_info(page_ext);
+	set_page_to_page_info(page, pi);
+
+	if (__PageTracked(page_ext)) {
+		__ClearPageTracked(page_ext);
+		__mod_lruvec_page_state(page, NR_TRACKED, -hpage_nr_pages(page));
+		__SetPageDeferred(page_ext);
+		list_move(&pi->list, &pgdat->deferred_list);
+	} else {
+		__SetPageDeferred(page_ext);
+		list_add(&pi->list, &pgdat->deferred_list);
+	}
+
+	spin_unlock_irq(&pgdat->lru_lock);
+
+	VM_BUG_ON_PAGE(__PageTracked(page_ext), page);
+
+	mod_lruvec_page_state(page, NR_DEFERRED, hpage_nr_pages(page));
+}
+
+unsigned int mod_page_access_lv(struct page *page, unsigned int accessed)
+{
+	struct page_ext *page_ext = lookup_page_ext(page);
+	struct page_info *pi = get_page_info(page_ext);
+	unsigned int prev_lv = __get_page_access_lv(pi);
+
+	// Shfit Left, Recently accessed bit is LSB
+	pi->access_bitmap = ((pi->access_bitmap) << 1);
+	if (accessed)
+		pi->access_bitmap |= 0x1;
+	else
+		pi->access_bitmap &= 0xfe;
+	return prev_lv;
+}
+
+unsigned int get_page_access_lv(struct page *page)
+{
+	struct page_ext *page_ext;
+	struct page_info *pi;
+
+	if (!(sysctl_numa_balancing_extended_mode & NUMA_BALANCING_OPM))
+		return -1;
+	page_ext = lookup_page_ext(page);
+	pi = get_page_info(page_ext);
+
+	return __get_page_access_lv(pi);
+}
+
+void reset_page_access_lv(struct page *page)
+{
+	struct page_ext *page_ext;
+	struct page_info *pi;
+
+	if (!(sysctl_numa_balancing_extended_mode & NUMA_BALANCING_OPM))
+		return;
+	page_ext = lookup_page_ext(page);
+	pi = get_page_info(page_ext);
+
+	pi->access_bitmap = (u8) ~(1 << ACCESS_HISTORY_SIZE);
+}
+
+/* Traverse migratable nodes from start_nid to all same-tier memory nodes */
+static int traverse_migratable_nodes(int start_nid, int order, int hold)
+{
+	int temp_nid = next_migration_node(start_nid);
+	int dst_nid = NUMA_NO_NODE;
+
+	/* Hold start_nid on temp_nid */
+	if (hold)
+		temp_nid = start_nid;
+
+	if (start_nid == NUMA_NO_NODE || temp_nid == NUMA_NO_NODE)
+		return dst_nid;
+
+	do {
+		if (migrate_balanced_pgdat(NODE_DATA(temp_nid), order)) {
+			dst_nid = temp_nid;
+			break;
+		}
+		temp_nid = next_migration_node(temp_nid);
+
+		/* migration path fail */
+		if (temp_nid == NUMA_NO_NODE) {
+			dst_nid = NUMA_NO_NODE;
+			break;
+		}
+	} while (temp_nid != start_nid);
+
+	return dst_nid;
+}
+
+int find_best_demotion_node(struct page *page)
+{
+	int order = compound_order(page);
+	int page_nid = page_to_nid(page);
+
+	int last_cpu = get_page_last_cpu(page);
+	int last_nid;
+
+	int dst_nid = NUMA_NO_NODE;
+	int sub_nid;
+
+	if (last_cpu < 0)
+		last_nid = page_nid;
+	else
+		last_nid = cpu_to_node(last_cpu);
+
+	if (!is_top_node(page_nid) || !is_top_node(last_nid))
+		return NUMA_NO_NODE;
+
+	sub_nid = next_demotion_node(last_nid);
+	dst_nid = traverse_migratable_nodes(sub_nid, order, 1);
+
+	return dst_nid;
+}
+EXPORT_SYMBOL(find_best_demotion_node);
+
+/* Find best node for migration */
+int find_best_migration_node(struct page *page, int target_nid)
+{
+	int order = compound_order(page);
+	int page_nid = page_to_nid(page);
+	int first_nid = next_promotion_node(page_nid);
+
+	int dst_nid;
+
+	dst_nid = traverse_migratable_nodes(target_nid, order, 1);
+#if 0 /* Verbose version */
+	/* Migration between same-tier memory nodes */
+	if (is_top_node(page_nid)) {
+	} else { /* Promotion */
+		if (first_nid == target_nid) {
+		} /* Remote Promotion */
+		else {
+			/* Find migratable lower-tier node*/
+			if (dst_nid == NUMA_NO_NODE)
+				dst_nid = traverse_migratable_nodes(page_nid, order, 0);
+		}
+	}
+#else /* Simple version */
+	/* Find migratable lower-tier node*/
+	if (dst_nid == NUMA_NO_NODE // Fail first try
+			&& !is_top_node(page_nid) // DCPMM page
+			&& first_nid != target_nid) // Remote Promotion
+		dst_nid = traverse_migratable_nodes(page_nid, order, 0);
+#endif
+
+	return dst_nid;
+}
+EXPORT_SYMBOL(find_best_migration_node);
+
+static void init_pages_in_zone(pg_data_t *pgdat, struct zone *zone)
+{
+	unsigned long pfn = zone->zone_start_pfn;
+	unsigned long end_pfn = zone_end_pfn(zone);
+	unsigned long count = 0;
+
+	/*
+	 * Walk the zone in pageblock_nr_pages steps. If a page block spans
+	 * a zone boundary, it will be double counted between zones. This does
+	 * not matter as the mixed block count will still be correct
+	 */
+	for (; pfn < end_pfn; ) {
+		unsigned long block_end_pfn;
+
+		if (!pfn_valid(pfn)) {
+			pfn = ALIGN(pfn + 1, MAX_ORDER_NR_PAGES);
+			continue;
+		}
+
+		block_end_pfn = ALIGN(pfn + 1, pageblock_nr_pages);
+		block_end_pfn = min(block_end_pfn, end_pfn);
+
+		for (; pfn < block_end_pfn; pfn++) {
+			struct page *page;
+			struct page_ext *page_ext;
+			struct page_info *pi;
+
+			if (!pfn_valid_within(pfn))
+				continue;
+
+			page = pfn_to_page(pfn);
+
+			if (page_zone(page) != zone)
+				continue;
+
+			/*
+			 * To avoid having to grab zone->lock, be a little
+			 * careful when reading buddy page order. The only
+			 * danger is that we skip too much and potentially miss
+			 * some early allocated pages, which is better than
+			 * heavy lock contention.
+			 */
+			if (PageBuddy(page)) {
+				unsigned long order = page_order_unsafe(page);
+
+				if (order > 0 && order < MAX_ORDER)
+					pfn += (1UL << order) - 1;
+				continue;
+			}
+
+			if (PageReserved(page))
+				continue;
+
+			page_ext = lookup_page_ext(page);
+			if (unlikely(!page_ext))
+				continue;
+
+			/* Maybe overlapping zone */
+			if (test_bit(PAGE_EXT_BALALCING, &page_ext->flags))
+				continue;
+
+			pi = get_page_info(page_ext);
+
+			/* Found early allocated page */
+			__set_bit(PAGE_EXT_BALALCING, &page_ext->flags);
+			__ClearPageTracked(page_ext);
+			__ClearPageDeferred(page_ext);
+			__ClearPageDemoted(page_ext);
+			pi->pfn = 0;
+			pi->last_cpu = -1;
+			pi->access_bitmap = (u8) ~(1 << ACCESS_HISTORY_SIZE);
+
+			count++;
+		}
+		cond_resched();
+	}
+
+	printk("Node %d, zone %8s: page info found early allocated %lu pages\n",
+		pgdat->node_id, zone->name, count);
+}
+
+static void init_zones_in_node(pg_data_t *pgdat)
+{
+	struct zone *zone;
+	struct zone *node_zones = pgdat->node_zones;
+
+	for (zone = node_zones; zone - node_zones < MAX_NR_ZONES; ++zone) {
+		if (!populated_zone(zone))
+			continue;
+
+		init_pages_in_zone(pgdat, zone);
+	}
+}
+
+static void init_early_allocated_pages(void)
+{
+	pg_data_t *pgdat;
+
+	for_each_online_pgdat(pgdat)
+		init_zones_in_node(pgdat);
+}
+
+static void init_page_balancing(void)
+{
+	init_early_allocated_pages();
+}
+
+struct page_ext_operations page_info_ops = {
+	.size = sizeof(struct page_info),
+	.need = need_page_balancing,
+	.init = init_page_balancing,
+};
+
+#ifdef CONFIG_SYSFS
+static ssize_t background_demotion_show(struct kobject *kobj,
+		struct kobj_attribute *attr, char *buf)
+{
+	switch (background_demotion) {
+	case 0:
+		return sprintf(buf, "%u - Disabled.\n",
+				background_demotion);
+	case 1:
+		return sprintf(buf, "%u - Enabled background page demotion\n",
+				background_demotion);
+	default:
+		return sprintf(buf, "error\n");
+	}
+}
+
+static ssize_t background_demotion_store(struct kobject *kobj,
+		struct kobj_attribute *attr,
+		const char *buf, size_t count)
+{
+	unsigned long enable;
+	int err;
+
+	err = kstrtoul(buf, 10, &enable);
+	if (err || enable < 0 || enable > 1)
+		return -EINVAL;
+
+	background_demotion = enable;
+
+	return count;
+}
+
+static struct kobj_attribute background_demotion_attr =
+__ATTR(background_demotion, 0644, background_demotion_show,
+		background_demotion_store);
+
+static ssize_t nr_reserved_pages_show(struct kobject *kobj,
+		struct kobj_attribute *attr, char *buf)
+{
+	int cpu;
+	int size = 0;
+	char buf_nr_free[10];
+	const struct cpumask *cpumasks = cpu_online_mask;
+	enum page_type type = BASEPAGE;
+
+	/* maybe not accurate */
+	for (type = BASEPAGE; type < NR_PAGE_TYPE; type++) {
+		for_each_cpu(cpu, cpumasks) {
+			size += sprintf(buf_nr_free, "%u ",
+					promote_area[type][cpu].nr_free);
+			strcat(buf, buf_nr_free);
+		}
+		strcat(buf, "\n");
+		size++;
+	}
+
+	return size;
+}
+
+static struct kobj_attribute nr_reserved_pages_attr =
+__ATTR(nr_reserved_pages, 0644, nr_reserved_pages_show, NULL);
+
+static ssize_t batch_demotion_show(struct kobject *kobj,
+		struct kobj_attribute *attr, char *buf)
+{
+	switch (batch_demotion) {
+	case 0:
+		return sprintf(buf, "%u - Disabled. batch size is 1\n",
+				batch_demotion);
+	case 1:
+		return sprintf(buf, "%u - Enabled. batch size is defined by current free reserved pages\n",
+				batch_demotion);
+	default:
+		return sprintf(buf, "error\n");
+	}
+}
+
+static ssize_t batch_demotion_store(struct kobject *kobj,
+		struct kobj_attribute *attr,
+		const char *buf, size_t count)
+{
+	unsigned long enable;
+	int err;
+
+	err = kstrtoul(buf, 10, &enable);
+	if (err || enable < 0 || enable > 1)
+		return -EINVAL;
+
+	batch_demotion = enable;
+
+	return count;
+}
+
+static struct kobj_attribute batch_demotion_attr =
+__ATTR(batch_demotion, 0644, batch_demotion_show,
+		batch_demotion_store);
+
+static ssize_t thp_mt_copy_show(struct kobject *kobj,
+		struct kobj_attribute *attr, char *buf)
+{
+	switch (thp_mt_copy) {
+	case 0:
+		return sprintf(buf, "%u - Disabled. single-thread copy\n",
+				thp_mt_copy);
+	case 1:
+		return sprintf(buf, "%u - Enabled. multi-thread(4) copys\n",
+				thp_mt_copy);
+	default:
+		return sprintf(buf, "error\n");
+	}
+}
+
+static ssize_t thp_mt_copy_store(struct kobject *kobj,
+		struct kobj_attribute *attr,
+		const char *buf, size_t count)
+{
+	unsigned long enable;
+	int err;
+
+	err = kstrtoul(buf, 10, &enable);
+	if (err || enable < 0 || enable > 1)
+		return -EINVAL;
+
+	thp_mt_copy = enable;
+
+	return count;
+}
+
+static struct kobj_attribute thp_mt_copy_attr =
+__ATTR(thp_mt_copy, 0644, thp_mt_copy_show,
+		thp_mt_copy_store);
+
+static ssize_t skip_lower_tier_show(struct kobject *kobj,
+		struct kobj_attribute *attr, char *buf)
+{
+	switch (skip_lower_tier) {
+	case 0:
+		return sprintf(buf, "%u - Disabled. tracking all pages\n",
+				skip_lower_tier);
+	case 1:
+		return sprintf(buf, "%u - Enabled. skip tracking lower-tier pages\n",
+				skip_lower_tier);
+	default:
+		return sprintf(buf, "error\n");
+	}
+}
+
+static ssize_t skip_lower_tier_store(struct kobject *kobj,
+		struct kobj_attribute *attr,
+		const char *buf, size_t count)
+{
+	unsigned long enable;
+	int err;
+
+	err = kstrtoul(buf, 10, &enable);
+	if (err || enable < 0 || enable > 1)
+		return -EINVAL;
+
+	skip_lower_tier = enable;
+
+	return count;
+}
+
+static struct kobj_attribute skip_lower_tier_attr =
+__ATTR(skip_lower_tier, 0644, skip_lower_tier_show,
+		skip_lower_tier_store);
+
+static struct attribute *page_balancing_attr[] = {
+	&background_demotion_attr.attr,
+	&batch_demotion_attr.attr,
+	&thp_mt_copy_attr.attr,
+	&skip_lower_tier_attr.attr,
+	&nr_reserved_pages_attr.attr,
+	NULL,
+};
+
+static struct attribute_group page_balancing_attr_group = {
+	.attrs = page_balancing_attr,
+};
+
+static void __init page_balancing_exit_sysfs(struct kobject *page_balancing_kobj)
+{
+	sysfs_remove_group(page_balancing_kobj, &page_balancing_attr_group);
+	kobject_put(page_balancing_kobj);
+}
+
+static int __init page_balancing_init_sysfs(struct kobject **page_balancing_kobj) {
+	int err;
+
+	*page_balancing_kobj = kobject_create_and_add("page_balancing", mm_kobj);
+	if (unlikely(!*page_balancing_kobj)) {
+		pr_err("failed to create page_balancing kobject\n");
+		return -ENOMEM;
+	}
+
+	err = sysfs_create_group(*page_balancing_kobj, &page_balancing_attr_group);
+	if (err) {
+		pr_err("failed to register page_balancing group\n");
+		goto delete_obj;
+	}
+
+	return 0;
+
+delete_obj:
+	page_balancing_exit_sysfs(*page_balancing_kobj);
+	return err;
+}
+
+#else
+static inline int page_balancing_init_sysfs(struct kobject **page_balancing_kobj)
+{
+	return 0;
+}
+static inline void page_balancing_exit_sysfs(struct kobject *page_balancing_kobj)
+{
+}
+#endif
+
+static int __init page_balancing_init(void)
+{
+	int err;
+	struct kobject *sysfs_page_balancing_kobj;
+
+	err = page_balancing_init_sysfs(&sysfs_page_balancing_kobj);
+	if (err) {
+		pr_err("failed start page_balancing_init becasue sysfs\n");
+		goto err_sysfs;
+	}
+	return 0;
+
+err_sysfs:
+	return err;
+}
+subsys_initcall(page_balancing_init);
diff --git a/mm/page_ext.c b/mm/page_ext.c
index 5f5769c7d..c4a55265d 100644
--- a/mm/page_ext.c
+++ b/mm/page_ext.c
@@ -8,6 +8,9 @@
 #include <linux/kmemleak.h>
 #include <linux/page_owner.h>
 #include <linux/page_idle.h>
+#ifdef CONFIG_PAGE_BALANCING
+#include <linux/page_balancing.h>
+#endif
 
 /*
  * struct page extension
@@ -65,6 +68,9 @@ static struct page_ext_operations *page_ext_ops[] = {
 #if defined(CONFIG_IDLE_PAGE_TRACKING) && !defined(CONFIG_64BIT)
 	&page_idle_ops,
 #endif
+#ifdef CONFIG_PAGE_BALANCING
+	&page_info_ops,
+#endif
 };
 
 static unsigned long total_usage;
diff --git a/mm/rmap.c b/mm/rmap.c
index 003377e24..e65805e43 100644
--- a/mm/rmap.c
+++ b/mm/rmap.c
@@ -1392,6 +1392,9 @@ static bool try_to_unmap_one(struct page *page, struct vm_area_struct *vma,
 		if (!pvmw.pte && (flags & TTU_MIGRATION)) {
 			VM_BUG_ON_PAGE(PageHuge(page) || !PageTransCompound(page), page);
 
+			if (!PageAnon(page))
+				continue;
+
 			set_pmd_migration_entry(&pvmw, page);
 			continue;
 		}
diff --git a/mm/swap.c b/mm/swap.c
index ae300397d..24d123075 100644
--- a/mm/swap.c
+++ b/mm/swap.c
@@ -35,6 +35,7 @@
 #include <linux/uio.h>
 #include <linux/hugetlb.h>
 #include <linux/page_idle.h>
+#include <linux/sched/sysctl.h>
 
 #include "internal.h"
 
@@ -281,6 +282,7 @@ static void __activate_page(struct page *page, struct lruvec *lruvec,
 		del_page_from_lru_list(page, lruvec, lru);
 		SetPageActive(page);
 		lru += LRU_ACTIVE;
+
 		add_page_to_lru_list(page, lruvec, lru);
 		trace_mm_lru_activate(page);
 
@@ -790,7 +792,6 @@ void release_pages(struct page **pages, int nr)
 		/* Clear Active bit in case of parallel mark_page_accessed */
 		__ClearPageActive(page);
 		__ClearPageWaiters(page);
-
 		list_add(&page->lru, &pages_to_free);
 	}
 	if (locked_pgdat)
diff --git a/mm/vmscan.c b/mm/vmscan.c
index a6c5d0b28..ca0aa3e59 100644
--- a/mm/vmscan.c
+++ b/mm/vmscan.c
@@ -57,6 +57,7 @@
 
 #include <linux/swapops.h>
 #include <linux/balloon_compaction.h>
+#include <linux/sched/sysctl.h>
 
 #include "internal.h"
 
@@ -339,7 +340,7 @@ unsigned long zone_reclaimable_pages(struct zone *zone)
 
 	nr = zone_page_state_snapshot(zone, NR_ZONE_INACTIVE_FILE) +
 		zone_page_state_snapshot(zone, NR_ZONE_ACTIVE_FILE);
-	if (get_nr_swap_pages() > 0)
+	if (reclaim_anon_pages(NULL, zone_to_nid(zone)))
 		nr += zone_page_state_snapshot(zone, NR_ZONE_INACTIVE_ANON) +
 			zone_page_state_snapshot(zone, NR_ZONE_ACTIVE_ANON);
 
@@ -1639,28 +1640,6 @@ int __isolate_lru_page(struct page *page, isolate_mode_t mode)
 	return ret;
 }
 
-
-/*
- * Update LRU sizes after isolating pages. The LRU size updates must
- * be complete before mem_cgroup_update_lru_size due to a santity check.
- */
-static __always_inline void update_lru_sizes(struct lruvec *lruvec,
-			enum lru_list lru, unsigned long *nr_zone_taken)
-{
-	int zid;
-
-	for (zid = 0; zid < MAX_NR_ZONES; zid++) {
-		if (!nr_zone_taken[zid])
-			continue;
-
-		__update_lru_size(lruvec, lru, zid, -nr_zone_taken[zid]);
-#ifdef CONFIG_MEMCG
-		mem_cgroup_update_lru_size(lruvec, lru, zid, -nr_zone_taken[zid]);
-#endif
-	}
-
-}
-
 /**
  * pgdat->lru_lock is heavily contended.  Some of the functions that
  * shrink the lists perform better by taking out a batch of pages
@@ -2188,7 +2167,7 @@ static bool inactive_list_is_low(struct lruvec *lruvec, bool file,
 	 * If we don't have swap space, anonymous page deactivation
 	 * is pointless.
 	 */
-	if (!file && !total_swap_pages)
+	if (!file && !reclaim_anon_pages(NULL, pgdat->node_id))
 		return false;
 
 	inactive = lruvec_lru_size(lruvec, inactive_lru, sc->reclaim_idx);
@@ -2263,7 +2242,7 @@ static void get_scan_count(struct lruvec *lruvec, struct mem_cgroup *memcg,
 	enum lru_list lru;
 
 	/* If we have no swap space, do not bother scanning anon pages. */
-	if (!sc->may_swap || mem_cgroup_get_nr_swap_pages(memcg) <= 0) {
+	if (!sc->may_swap || !reclaim_anon_pages(memcg, pgdat->node_id)) {
 		scan_balance = SCAN_FILE;
 		goto out;
 	}
@@ -2626,7 +2605,7 @@ static inline bool should_continue_reclaim(struct pglist_data *pgdat,
 	 */
 	pages_for_compaction = compact_gap(sc->order);
 	inactive_lru_pages = node_page_state(pgdat, NR_INACTIVE_FILE);
-	if (get_nr_swap_pages() > 0)
+	if (!reclaim_anon_pages(NULL, pgdat->node_id))
 		inactive_lru_pages += node_page_state(pgdat, NR_INACTIVE_ANON);
 	if (sc->nr_reclaimed < pages_for_compaction &&
 			inactive_lru_pages > pages_for_compaction)
@@ -3314,7 +3293,7 @@ static void age_active_anon(struct pglist_data *pgdat,
 {
 	struct mem_cgroup *memcg;
 
-	if (!total_swap_pages)
+	if (!reclaim_anon_pages(NULL, pgdat->node_id))
 		return;
 
 	memcg = mem_cgroup_iter(NULL, NULL, NULL);
diff --git a/mm/vmstat.c b/mm/vmstat.c
index fd7e16ca6..859adb44c 100644
--- a/mm/vmstat.c
+++ b/mm/vmstat.c
@@ -28,6 +28,7 @@
 #include <linux/mm_inline.h>
 #include <linux/page_ext.h>
 #include <linux/page_owner.h>
+#include <linux/migrate.h>
 
 #include "internal.h"
 
@@ -528,6 +529,140 @@ void inc_zone_page_state(struct page *page, enum zone_stat_item item)
 }
 EXPORT_SYMBOL(inc_zone_page_state);
 
+void inc_zone_state(struct zone *zone, enum zone_stat_item item)
+{
+	mod_zone_state(zone, item, 1, 1);
+}
+EXPORT_SYMBOL(inc_zone_state);
+
+void inc_hmem_state(enum migrate_hmem_reason hr, struct page *src, struct page *dst)
+{
+	/*
+	 * There are two zone stats for each 'migrate_hmem_reason',
+	 * a source and a destination.  Given the hmem_reason,
+	 * calculate the two corresponding zone stats:
+	 */
+	int zone_stat_src = HMEM_MIGRATE_FIRST_ENTRY + 2 * hr - 1;
+	int zone_stat_dst = HMEM_MIGRATE_FIRST_ENTRY + 2 * hr;
+
+	/*
+	 * HMEM_MIGRATE_FIRST_ENTRY is also the "unknown" which will
+	 * be tolerated for now since all code paths have not had
+	 * hmem migrations reasons added.
+	 *
+	 * An invalid value here probably comes from an uninitialized
+	 * stack instance of 'struct migrate_detail'.
+	 */
+	if (WARN_ON_ONCE(hr < MR_HMEM_UNKNOWN || hr >= MR_HMEM_NR_REASONS))
+		return;
+	if (hr == MR_HMEM_UNKNOWN)
+		return;
+	mod_zone_page_state(page_zone(src), zone_stat_src, hpage_nr_pages(src));
+	mod_zone_page_state(page_zone(dst), zone_stat_dst, hpage_nr_pages(dst));
+}
+EXPORT_SYMBOL(inc_hmem_state);
+
+void inc_hmem_src_state(enum migrate_hmem_reason hr, struct page *src)
+{
+	/*
+	 * There are one zone stats for each 'migrate_hmem_reason',
+	 * a source page.  Given the hmem_reason, calculate the 
+         * corresponding zone stats:
+	 */
+	int zone_stat_src = HMEM_MIGRATE_FIRST_ENTRY + 2 * hr - 1;
+
+	/*
+	 * HMEM_MIGRATE_FIRST_ENTRY is also the "unknown" which will
+	 * be tolerated for now since all code paths have not had
+	 * hmem migrations reasons added.
+	 *
+	 * An invalid value here probably comes from an uninitialized
+	 * stack instance of 'struct migrate_detail'.
+	 */
+	if (WARN_ON_ONCE(hr < MR_HMEM_UNKNOWN || hr >= MR_HMEM_NR_REASONS))
+		return;
+	if (hr == MR_HMEM_UNKNOWN)
+		return;
+	mod_zone_page_state(page_zone(src), zone_stat_src, hpage_nr_pages(src));
+}
+EXPORT_SYMBOL(inc_hmem_src_state);
+
+void inc_hmem_dst_state(enum migrate_hmem_reason hr, struct page *dst)
+{
+	/*
+	 * There are one zone stats for each 'migrate_hmem_reason',
+	 * a source page.  Given the hmem_reason, calculate the 
+         * corresponding zone stats:
+	 */
+	int zone_stat_dst = HMEM_MIGRATE_FIRST_ENTRY + 2 * hr;
+
+	/*
+	 * HMEM_MIGRATE_FIRST_ENTRY is also the "unknown" which will
+	 * be tolerated for now since all code paths have not had
+	 * hmem migrations reasons added.
+	 *
+	 * An invalid value here probably comes from an uninitialized
+	 * stack instance of 'struct migrate_detail'.
+	 */
+	if (WARN_ON_ONCE(hr < MR_HMEM_UNKNOWN || hr >= MR_HMEM_NR_REASONS))
+		return;
+	if (hr == MR_HMEM_UNKNOWN)
+		return;
+	mod_zone_page_state(page_zone(dst), zone_stat_dst, hpage_nr_pages(dst));
+}
+EXPORT_SYMBOL(inc_hmem_dst_state);
+
+void inc_hmem_fail_state(enum migrate_hmem_fail_reason hr, int src_nid,
+				int dst_nid, int is_huge)
+{
+	struct pglist_data *pgdat_src = NODE_DATA(src_nid);
+	struct pglist_data *pgdat_dst = NODE_DATA(dst_nid);
+	struct zone *zone_src = &pgdat_src->node_zones[ZONE_NORMAL];
+	struct zone *zone_dst = &pgdat_dst->node_zones[ZONE_NORMAL];
+	int zone_stat_src = HMEM_MIGRATE_FAIL_FIRST_ENTRY + 2 * hr - 1;
+	int zone_stat_dst = HMEM_MIGRATE_FAIL_FIRST_ENTRY + 2 * hr;
+	int nr_pages = is_huge ? 512 : 1;
+
+	if (WARN_ON_ONCE(hr < MR_HMEM_UNKNOWN_FAIL || hr >= MR_HMEM_NR_FAIL_REASONS))
+		return;
+	if (hr == MR_HMEM_UNKNOWN_FAIL)
+		return;
+
+	mod_zone_state(zone_src, zone_stat_src, nr_pages, 0);
+	mod_zone_state(zone_dst, zone_stat_dst, nr_pages, 0);
+}
+EXPORT_SYMBOL(inc_hmem_fail_state);
+
+void inc_hmem_fail_src_state(enum migrate_hmem_fail_reason hr, int src_nid, int is_huge)
+{
+	struct pglist_data *pgdat = NODE_DATA(src_nid);
+	struct zone *zone = &pgdat->node_zones[ZONE_NORMAL];
+	int zone_stat_src = HMEM_MIGRATE_FAIL_FIRST_ENTRY + 2 * hr - 1;
+	int nr_pages = is_huge ? 512 : 1;
+
+	if (WARN_ON_ONCE(hr < MR_HMEM_UNKNOWN_FAIL || hr >= MR_HMEM_NR_FAIL_REASONS))
+		return;
+	if (hr == MR_HMEM_UNKNOWN_FAIL)
+		return;
+	mod_zone_state(zone, zone_stat_src, nr_pages, 0);
+}
+EXPORT_SYMBOL(inc_hmem_fail_src_state);
+
+void inc_hmem_fail_dst_state(enum migrate_hmem_fail_reason hr, int dst_nid, int is_huge)
+{
+	struct pglist_data *pgdat = NODE_DATA(dst_nid);
+	struct zone *zone = &pgdat->node_zones[ZONE_NORMAL];
+	int zone_stat_dst = HMEM_MIGRATE_FAIL_FIRST_ENTRY + 2 * hr;
+	int nr_pages = is_huge ? 512 : 1;
+
+	if (WARN_ON_ONCE(hr < MR_HMEM_UNKNOWN_FAIL || hr >= MR_HMEM_NR_FAIL_REASONS))
+		return;
+	if (hr == MR_HMEM_UNKNOWN_FAIL)
+		return;
+	mod_zone_state(zone, zone_stat_dst, nr_pages, 0);
+}
+EXPORT_SYMBOL(inc_hmem_fail_dst_state);
+
 void dec_zone_page_state(struct page *page, enum zone_stat_item item)
 {
 	mod_zone_state(page_zone(page), item, -1, -1);
@@ -1123,6 +1258,22 @@ const char * const vmstat_text[] = {
 	"nr_zspages",
 #endif
 	"nr_free_cma",
+	"hmem_unknown",
+	"hmem_demote_src",
+	"hmem_demote_dst",
+	"hmem_promote_local_src",
+	"hmem_promote_local_dst",
+	"hmem_promote_remote_src",
+	"hmem_promote_remote_dst",
+	"hmem_migrate_src",
+	"hmem_migrate_dst",
+	"hmem_unknown_fail",
+	"hmem_promote_local_fail_src",
+	"hmem_promote_local_fail_dst",
+	"hmem_promote_remote_fail_src",
+	"hmem_promote_remote_fail_dst",
+	"hmem_migrate_fail_src",
+	"hmem_migrate_fail_dst",
 
 	/* enum numa_stat_item counters */
 #ifdef CONFIG_NUMA
@@ -1165,6 +1316,9 @@ const char * const vmstat_text[] = {
 	"nr_dirtied",
 	"nr_written",
 	"nr_kernel_misc_reclaimable",
+	"nr_deferred",
+	"nr_tracked",
+	"nr_reserved_pages",
 
 	/* enum writeback_stat_item counters */
 	"nr_dirty_threshold",
@@ -1223,6 +1377,31 @@ const char * const vmstat_text[] = {
 #ifdef CONFIG_MIGRATION
 	"pgmigrate_success",
 	"pgmigrate_fail",
+	"pgmigrate_dst_node_full_fail",
+	"pgmigrate_numa_isolate_fail",
+	"pgmigrate_nomem_fail",
+	"pgmigrate_refcount_fail",
+	"pgexchange_success",
+	"pgexchange_fail",
+	"pgexchange_no_page_fail",
+	"pgexchange_node_unmatch_fail",
+	"pgexchange_list_empty_fail",
+	"pgexchange_scan_fail",
+	"pgexchange_busy_fail",
+	"pgactivate_deferred",
+	"pgactivate_deferred_local",
+	"pgdemote_no_page_fail",
+	"pgdemote_no_lru_fail",
+	"pgdemote_busy_fail",
+	"pgdemote_background",
+	"pgdemote_file",
+	"pgpromote_empty_pool_fail",
+	"pgpromote_no_page_fail",
+	"pgpromote_low_freq_fail",
+	"pgpromote_reserved",
+	"pgrepromote",
+	"pgfree_demoted",
+	"nr_page_skipped",
 #endif
 #ifdef CONFIG_COMPACTION
 	"compact_migrate_scanned",
@@ -1509,6 +1688,43 @@ static int pagetypeinfo_show(struct seq_file *m, void *arg)
 	return 0;
 }
 
+static void lap_show_print(struct seq_file *m, pg_data_t *pgdat)
+{
+	int order;
+
+	seq_printf(m, "Node %d ", pgdat->node_id);
+	for (order = 0; order <= MAX_ACCESS_LEVEL; order++)
+		seq_printf(m, "%7llu ", pgdat->lap_area[order].nr_free);
+	seq_putc(m, '\n');
+}
+
+static void lap_count_show_print(struct seq_file *m, pg_data_t *pgdat)
+{
+	int order;
+
+	seq_printf(m, "Node %d ", pgdat->node_id);
+	for (order = 0; order <= MAX_ACCESS_LEVEL; order++)
+		seq_printf(m, "%7lu ", pgdat->lap_area[order].demotion_count);
+	seq_putc(m, '\n');
+}
+
+static int lap_show(struct seq_file *m, void *arg)
+{
+	pg_data_t *pgdat = (pg_data_t *)arg;
+
+	/* check memoryless node */
+	if (!node_state(pgdat->node_id, N_MEMORY))
+		return 0;
+
+	seq_printf(m, "LAP pages at order\n");
+	lap_show_print(m, pgdat);
+
+	seq_printf(m, "LAP page demotion counts\n");
+	lap_count_show_print(m, pgdat);
+
+	return 0;
+}
+
 static const struct seq_operations fragmentation_op = {
 	.start	= frag_start,
 	.next	= frag_next,
@@ -1523,6 +1739,14 @@ static const struct seq_operations pagetypeinfo_op = {
 	.show	= pagetypeinfo_show,
 };
 
+static const struct seq_operations leastaccessedpage_op = {
+	.start	= frag_start,
+	.next	= frag_next,
+	.stop	= frag_stop,
+	.show	= lap_show,
+};
+
+
 static bool is_zone_first_populated(pg_data_t *pgdat, struct zone *zone)
 {
 	int zid;
@@ -1973,6 +2197,7 @@ void __init init_mm_internals(void)
 	proc_create_seq("pagetypeinfo", 0444, NULL, &pagetypeinfo_op);
 	proc_create_seq("vmstat", 0444, NULL, &vmstat_op);
 	proc_create_seq("zoneinfo", 0444, NULL, &zoneinfo_op);
+	proc_create_seq("lapinfo", 0444, NULL, &leastaccessedpage_op);
 #endif
 }