mm/hugetlb_sysfs.c - linux/kernel/git/gregkh/usb - Git at Google

 // SPDX-License-Identifier: GPL-2.0-only
 /*
  * HugeTLB sysfs interfaces.
  * (C) Nadia Yvette Chambers, April 2004
  */

 #include <linux/swap.h>
 #include <linux/page_owner.h>
 #include <linux/page-isolation.h>

 #include "hugetlb_vmemmap.h"
 #include "hugetlb_internal.h"

 #define HSTATE_ATTR_RO(_name) \
 	static struct kobj_attribute _name##_attr = __ATTR_RO(_name)

 #define HSTATE_ATTR_WO(_name) \
 	static struct kobj_attribute _name##_attr = __ATTR_WO(_name)

 #define HSTATE_ATTR(_name) \
 	static struct kobj_attribute _name##_attr = __ATTR_RW(_name)

 static struct kobject *hugepages_kobj;
 static struct kobject *hstate_kobjs[HUGE_MAX_HSTATE];

 static struct hstate *kobj_to_node_hstate(struct kobject *kobj, int *nidp);

 static struct hstate *kobj_to_hstate(struct kobject *kobj, int *nidp)
 {
 	int i;

 	for (i = 0; i < HUGE_MAX_HSTATE; i++)
 		if (hstate_kobjs[i] == kobj) {
 			if (nidp)
 				*nidp = NUMA_NO_NODE;
 			return &hstates[i];
 		}

 	return kobj_to_node_hstate(kobj, nidp);
 }

 static ssize_t nr_hugepages_show_common(struct kobject *kobj,
 					struct kobj_attribute *attr, char *buf)
 {
 	struct hstate *h;
 	unsigned long nr_huge_pages;
 	int nid;

 	h = kobj_to_hstate(kobj, &nid);
 	if (nid == NUMA_NO_NODE)
 		nr_huge_pages = h->nr_huge_pages;
 	else
 		nr_huge_pages = h->nr_huge_pages_node[nid];

 	return sysfs_emit(buf, "%lu\n", nr_huge_pages);
 }

 static ssize_t nr_hugepages_store_common(bool obey_mempolicy,
 					 struct kobject *kobj, const char *buf,
 					 size_t len)
 {
 	struct hstate *h;
 	unsigned long count;
 	int nid;
 	int err;

 	err = kstrtoul(buf, 10, &count);
 	if (err)
 		return err;

 	h = kobj_to_hstate(kobj, &nid);
 	return __nr_hugepages_store_common(obey_mempolicy, h, nid, count, len);
 }

 static ssize_t nr_hugepages_show(struct kobject *kobj,
 				       struct kobj_attribute *attr, char *buf)
 {
 	return nr_hugepages_show_common(kobj, attr, buf);
 }

 static ssize_t nr_hugepages_store(struct kobject *kobj,
 	       struct kobj_attribute *attr, const char *buf, size_t len)
 {
 	return nr_hugepages_store_common(false, kobj, buf, len);
 }
 HSTATE_ATTR(nr_hugepages);

 #ifdef CONFIG_NUMA

 /*
  * hstate attribute for optionally mempolicy-based constraint on persistent
  * huge page alloc/free.
  */
 static ssize_t nr_hugepages_mempolicy_show(struct kobject *kobj,
 					   struct kobj_attribute *attr,
 					   char *buf)
 {
 	return nr_hugepages_show_common(kobj, attr, buf);
 }

 static ssize_t nr_hugepages_mempolicy_store(struct kobject *kobj,
 	       struct kobj_attribute *attr, const char *buf, size_t len)
 {
 	return nr_hugepages_store_common(true, kobj, buf, len);
 }
 HSTATE_ATTR(nr_hugepages_mempolicy);
 #endif


 static ssize_t nr_overcommit_hugepages_show(struct kobject *kobj,
 					struct kobj_attribute *attr, char *buf)
 {
 	struct hstate *h = kobj_to_hstate(kobj, NULL);
 	return sysfs_emit(buf, "%lu\n", h->nr_overcommit_huge_pages);
 }

 static ssize_t nr_overcommit_hugepages_store(struct kobject *kobj,
 		struct kobj_attribute *attr, const char *buf, size_t count)
 {
 	int err;
 	unsigned long input;
 	struct hstate *h = kobj_to_hstate(kobj, NULL);

 	if (hstate_is_gigantic_no_runtime(h))
 		return -EINVAL;

 	err = kstrtoul(buf, 10, &input);
 	if (err)
 		return err;

 	spin_lock_irq(&hugetlb_lock);
 	h->nr_overcommit_huge_pages = input;
 	spin_unlock_irq(&hugetlb_lock);

 	return count;
 }
 HSTATE_ATTR(nr_overcommit_hugepages);

 static ssize_t free_hugepages_show(struct kobject *kobj,
 					struct kobj_attribute *attr, char *buf)
 {
 	struct hstate *h;
 	unsigned long free_huge_pages;
 	int nid;

 	h = kobj_to_hstate(kobj, &nid);
 	if (nid == NUMA_NO_NODE)
 		free_huge_pages = h->free_huge_pages;
 	else
 		free_huge_pages = h->free_huge_pages_node[nid];

 	return sysfs_emit(buf, "%lu\n", free_huge_pages);
 }
 HSTATE_ATTR_RO(free_hugepages);

 static ssize_t resv_hugepages_show(struct kobject *kobj,
 					struct kobj_attribute *attr, char *buf)
 {
 	struct hstate *h = kobj_to_hstate(kobj, NULL);
 	return sysfs_emit(buf, "%lu\n", h->resv_huge_pages);
 }
 HSTATE_ATTR_RO(resv_hugepages);

 static ssize_t surplus_hugepages_show(struct kobject *kobj,
 					struct kobj_attribute *attr, char *buf)
 {
 	struct hstate *h;
 	unsigned long surplus_huge_pages;
 	int nid;

 	h = kobj_to_hstate(kobj, &nid);
 	if (nid == NUMA_NO_NODE)
 		surplus_huge_pages = h->surplus_huge_pages;
 	else
 		surplus_huge_pages = h->surplus_huge_pages_node[nid];

 	return sysfs_emit(buf, "%lu\n", surplus_huge_pages);
 }
 HSTATE_ATTR_RO(surplus_hugepages);

 static ssize_t demote_store(struct kobject *kobj,
 	       struct kobj_attribute *attr, const char *buf, size_t len)
 {
 	unsigned long nr_demote;
 	unsigned long nr_available;
 	nodemask_t nodes_allowed, *n_mask;
 	struct hstate *h;
 	int err;
 	int nid;

 	err = kstrtoul(buf, 10, &nr_demote);
 	if (err)
 		return err;
 	h = kobj_to_hstate(kobj, &nid);

 	if (nid != NUMA_NO_NODE) {
 		init_nodemask_of_node(&nodes_allowed, nid);
 		n_mask = &nodes_allowed;
 	} else {
 		n_mask = &node_states[N_MEMORY];
 	}

 	/* Synchronize with other sysfs operations modifying huge pages */
 	mutex_lock(&h->resize_lock);
 	spin_lock_irq(&hugetlb_lock);

 	while (nr_demote) {
 		long rc;

 		/*
 		 * Check for available pages to demote each time thorough the
 		 * loop as demote_pool_huge_page will drop hugetlb_lock.
 		 */
 		if (nid != NUMA_NO_NODE)
 			nr_available = h->free_huge_pages_node[nid];
 		else
 			nr_available = h->free_huge_pages;
 		nr_available -= h->resv_huge_pages;
 		if (!nr_available)
 			break;

 		rc = demote_pool_huge_page(h, n_mask, nr_demote);
 		if (rc < 0) {
 			err = rc;
 			break;
 		}

 		nr_demote -= rc;
 	}

 	spin_unlock_irq(&hugetlb_lock);
 	mutex_unlock(&h->resize_lock);

 	if (err)
 		return err;
 	return len;
 }
 HSTATE_ATTR_WO(demote);

 static ssize_t demote_size_show(struct kobject *kobj,
 					struct kobj_attribute *attr, char *buf)
 {
 	struct hstate *h = kobj_to_hstate(kobj, NULL);
 	unsigned long demote_size = (PAGE_SIZE << h->demote_order) / SZ_1K;

 	return sysfs_emit(buf, "%lukB\n", demote_size);
 }

 static ssize_t demote_size_store(struct kobject *kobj,
 					struct kobj_attribute *attr,
 					const char *buf, size_t count)
 {
 	struct hstate *h, *demote_hstate;
 	unsigned long demote_size;
 	unsigned int demote_order;

 	demote_size = (unsigned long)memparse(buf, NULL);

 	demote_hstate = size_to_hstate(demote_size);
 	if (!demote_hstate)
 		return -EINVAL;
 	demote_order = demote_hstate->order;
 	if (demote_order < HUGETLB_PAGE_ORDER)
 		return -EINVAL;

 	/* demote order must be smaller than hstate order */
 	h = kobj_to_hstate(kobj, NULL);
 	if (demote_order >= h->order)
 		return -EINVAL;

 	/* resize_lock synchronizes access to demote size and writes */
 	mutex_lock(&h->resize_lock);
 	h->demote_order = demote_order;
 	mutex_unlock(&h->resize_lock);

 	return count;
 }
 HSTATE_ATTR(demote_size);

 static struct attribute *hstate_attrs[] = {
 	&nr_hugepages_attr.attr,
 	&nr_overcommit_hugepages_attr.attr,
 	&free_hugepages_attr.attr,
 	&resv_hugepages_attr.attr,
 	&surplus_hugepages_attr.attr,
 #ifdef CONFIG_NUMA
 	&nr_hugepages_mempolicy_attr.attr,
 #endif
 	NULL,
 };

 static const struct attribute_group hstate_attr_group = {
 	.attrs = hstate_attrs,
 };

 static struct attribute *hstate_demote_attrs[] = {
 	&demote_size_attr.attr,
 	&demote_attr.attr,
 	NULL,
 };

 static const struct attribute_group hstate_demote_attr_group = {
 	.attrs = hstate_demote_attrs,
 };

 static int hugetlb_sysfs_add_hstate(struct hstate *h, struct kobject *parent,
 				    struct kobject **hstate_kobjs,
 				    const struct attribute_group *hstate_attr_group)
 {
 	int retval;
 	int hi = hstate_index(h);

 	hstate_kobjs[hi] = kobject_create_and_add(h->name, parent);
 	if (!hstate_kobjs[hi])
 		return -ENOMEM;

 	retval = sysfs_create_group(hstate_kobjs[hi], hstate_attr_group);
 	if (retval) {
 		kobject_put(hstate_kobjs[hi]);
 		hstate_kobjs[hi] = NULL;
 		return retval;
 	}

 	if (h->demote_order) {
 		retval = sysfs_create_group(hstate_kobjs[hi],
 					    &hstate_demote_attr_group);
 		if (retval) {
 			pr_warn("HugeTLB unable to create demote interfaces for %s\n", h->name);
 			sysfs_remove_group(hstate_kobjs[hi], hstate_attr_group);
 			kobject_put(hstate_kobjs[hi]);
 			hstate_kobjs[hi] = NULL;
 			return retval;
 		}
 	}

 	return 0;
 }

 #ifdef CONFIG_NUMA
 static bool hugetlb_sysfs_initialized __ro_after_init;

 /*
  * node_hstate/s - associate per node hstate attributes, via their kobjects,
  * with node devices in node_devices[] using a parallel array.  The array
  * index of a node device or _hstate == node id.
  * This is here to avoid any static dependency of the node device driver, in
  * the base kernel, on the hugetlb module.
  */
 struct node_hstate {
 	struct kobject		*hugepages_kobj;
 	struct kobject		*hstate_kobjs[HUGE_MAX_HSTATE];
 };
 static struct node_hstate node_hstates[MAX_NUMNODES];

 /*
  * A subset of global hstate attributes for node devices
  */
 static struct attribute *per_node_hstate_attrs[] = {
 	&nr_hugepages_attr.attr,
 	&free_hugepages_attr.attr,
 	&surplus_hugepages_attr.attr,
 	NULL,
 };

 static const struct attribute_group per_node_hstate_attr_group = {
 	.attrs = per_node_hstate_attrs,
 };

 /*
  * kobj_to_node_hstate - lookup global hstate for node device hstate attr kobj.
  * Returns node id via non-NULL nidp.
  */
 static struct hstate *kobj_to_node_hstate(struct kobject *kobj, int *nidp)
 {
 	int nid;

 	for (nid = 0; nid < nr_node_ids; nid++) {
 		struct node_hstate *nhs = &node_hstates[nid];
 		int i;
 		for (i = 0; i < HUGE_MAX_HSTATE; i++)
 			if (nhs->hstate_kobjs[i] == kobj) {
 				if (nidp)
 					*nidp = nid;
 				return &hstates[i];
 			}
 	}

 	BUG();
 	return NULL;
 }

 /*
  * Unregister hstate attributes from a single node device.
  * No-op if no hstate attributes attached.
  */
 void hugetlb_unregister_node(struct node *node)
 {
 	struct hstate *h;
 	struct node_hstate *nhs = &node_hstates[node->dev.id];

 	if (!nhs->hugepages_kobj)
 		return;		/* no hstate attributes */

 	for_each_hstate(h) {
 		int idx = hstate_index(h);
 		struct kobject *hstate_kobj = nhs->hstate_kobjs[idx];

 		if (!hstate_kobj)
 			continue;
 		if (h->demote_order)
 			sysfs_remove_group(hstate_kobj, &hstate_demote_attr_group);
 		sysfs_remove_group(hstate_kobj, &per_node_hstate_attr_group);
 		kobject_put(hstate_kobj);
 		nhs->hstate_kobjs[idx] = NULL;
 	}

 	kobject_put(nhs->hugepages_kobj);
 	nhs->hugepages_kobj = NULL;
 }


 /*
  * Register hstate attributes for a single node device.
  * No-op if attributes already registered.
  */
 void hugetlb_register_node(struct node *node)
 {
 	struct hstate *h;
 	struct node_hstate *nhs = &node_hstates[node->dev.id];
 	int err;

 	if (!hugetlb_sysfs_initialized)
 		return;

 	if (nhs->hugepages_kobj)
 		return;		/* already allocated */

 	nhs->hugepages_kobj = kobject_create_and_add("hugepages",
 							&node->dev.kobj);
 	if (!nhs->hugepages_kobj)
 		return;

 	for_each_hstate(h) {
 		err = hugetlb_sysfs_add_hstate(h, nhs->hugepages_kobj,
 						nhs->hstate_kobjs,
 						&per_node_hstate_attr_group);
 		if (err) {
 			pr_err("HugeTLB: Unable to add hstate %s for node %d\n",
 				h->name, node->dev.id);
 			hugetlb_unregister_node(node);
 			break;
 		}
 	}
 }

 /*
  * hugetlb init time:  register hstate attributes for all registered node
  * devices of nodes that have memory.  All on-line nodes should have
  * registered their associated device by this time.
  */
 static void __init hugetlb_register_all_nodes(void)
 {
 	int nid;

 	for_each_online_node(nid)
 		hugetlb_register_node(node_devices[nid]);
 }
 #else	/* !CONFIG_NUMA */

 static struct hstate *kobj_to_node_hstate(struct kobject *kobj, int *nidp)
 {
 	BUG();
 	if (nidp)
 		*nidp = -1;
 	return NULL;
 }

 static void hugetlb_register_all_nodes(void) { }

 #endif

 void __init hugetlb_sysfs_init(void)
 {
 	struct hstate *h;
 	int err;

 	hugepages_kobj = kobject_create_and_add("hugepages", mm_kobj);
 	if (!hugepages_kobj)
 		return;

 	for_each_hstate(h) {
 		err = hugetlb_sysfs_add_hstate(h, hugepages_kobj,
 					 hstate_kobjs, &hstate_attr_group);
 		if (err)
 			pr_err("HugeTLB: Unable to add hstate %s\n", h->name);
 	}

 #ifdef CONFIG_NUMA
 	hugetlb_sysfs_initialized = true;
 #endif
 	hugetlb_register_all_nodes();
 }
	// SPDX-License-Identifier: GPL-2.0-only
	/*
	* HugeTLB sysfs interfaces.
	* (C) Nadia Yvette Chambers, April 2004
	*/

	#include <linux/swap.h>
	#include <linux/page_owner.h>
	#include <linux/page-isolation.h>

	#include "hugetlb_vmemmap.h"
	#include "hugetlb_internal.h"

	#define HSTATE_ATTR_RO(_name) \
	static struct kobj_attribute _name##_attr = __ATTR_RO(_name)

	#define HSTATE_ATTR_WO(_name) \
	static struct kobj_attribute _name##_attr = __ATTR_WO(_name)

	#define HSTATE_ATTR(_name) \
	static struct kobj_attribute _name##_attr = __ATTR_RW(_name)

	static struct kobject *hugepages_kobj;
	static struct kobject *hstate_kobjs[HUGE_MAX_HSTATE];

	static struct hstate kobj_to_node_hstate(struct kobject kobj, int *nidp);

	static struct hstate kobj_to_hstate(struct kobject kobj, int *nidp)
	{
	int i;

	for (i = 0; i < HUGE_MAX_HSTATE; i++)
	if (hstate_kobjs[i] == kobj) {
	if (nidp)
	*nidp = NUMA_NO_NODE;
	return &hstates[i];
	}

	return kobj_to_node_hstate(kobj, nidp);
	}

	static ssize_t nr_hugepages_show_common(struct kobject *kobj,
	struct kobj_attribute attr, char buf)
	{
	struct hstate *h;
	unsigned long nr_huge_pages;
	int nid;

	h = kobj_to_hstate(kobj, &nid);
	if (nid == NUMA_NO_NODE)
	nr_huge_pages = h->nr_huge_pages;
	else
	nr_huge_pages = h->nr_huge_pages_node[nid];

	return sysfs_emit(buf, "%lu\n", nr_huge_pages);
	}

	static ssize_t nr_hugepages_store_common(bool obey_mempolicy,
	struct kobject kobj, const char buf,
	size_t len)
	{
	struct hstate *h;
	unsigned long count;
	int nid;
	int err;

	err = kstrtoul(buf, 10, &count);
	if (err)
	return err;

	h = kobj_to_hstate(kobj, &nid);
	return __nr_hugepages_store_common(obey_mempolicy, h, nid, count, len);
	}

	static ssize_t nr_hugepages_show(struct kobject *kobj,
	struct kobj_attribute attr, char buf)
	{
	return nr_hugepages_show_common(kobj, attr, buf);
	}

	static ssize_t nr_hugepages_store(struct kobject *kobj,
	struct kobj_attribute attr, const char buf, size_t len)
	{
	return nr_hugepages_store_common(false, kobj, buf, len);
	}
	HSTATE_ATTR(nr_hugepages);

	#ifdef CONFIG_NUMA

	/*
	* hstate attribute for optionally mempolicy-based constraint on persistent
	* huge page alloc/free.
	*/
	static ssize_t nr_hugepages_mempolicy_show(struct kobject *kobj,
	struct kobj_attribute *attr,
	char *buf)
	{
	return nr_hugepages_show_common(kobj, attr, buf);
	}

	static ssize_t nr_hugepages_mempolicy_store(struct kobject *kobj,
	struct kobj_attribute attr, const char buf, size_t len)
	{
	return nr_hugepages_store_common(true, kobj, buf, len);
	}
	HSTATE_ATTR(nr_hugepages_mempolicy);
	#endif


	static ssize_t nr_overcommit_hugepages_show(struct kobject *kobj,
	struct kobj_attribute attr, char buf)
	{
	struct hstate *h = kobj_to_hstate(kobj, NULL);
	return sysfs_emit(buf, "%lu\n", h->nr_overcommit_huge_pages);
	}

	static ssize_t nr_overcommit_hugepages_store(struct kobject *kobj,
	struct kobj_attribute attr, const char buf, size_t count)
	{
	int err;
	unsigned long input;
	struct hstate *h = kobj_to_hstate(kobj, NULL);

	if (hstate_is_gigantic_no_runtime(h))
	return -EINVAL;

	err = kstrtoul(buf, 10, &input);
	if (err)
	return err;

	spin_lock_irq(&hugetlb_lock);
	h->nr_overcommit_huge_pages = input;
	spin_unlock_irq(&hugetlb_lock);

	return count;
	}
	HSTATE_ATTR(nr_overcommit_hugepages);

	static ssize_t free_hugepages_show(struct kobject *kobj,
	struct kobj_attribute attr, char buf)
	{
	struct hstate *h;
	unsigned long free_huge_pages;
	int nid;

	h = kobj_to_hstate(kobj, &nid);
	if (nid == NUMA_NO_NODE)
	free_huge_pages = h->free_huge_pages;
	else
	free_huge_pages = h->free_huge_pages_node[nid];

	return sysfs_emit(buf, "%lu\n", free_huge_pages);
	}
	HSTATE_ATTR_RO(free_hugepages);

	static ssize_t resv_hugepages_show(struct kobject *kobj,
	struct kobj_attribute attr, char buf)
	{
	struct hstate *h = kobj_to_hstate(kobj, NULL);
	return sysfs_emit(buf, "%lu\n", h->resv_huge_pages);
	}
	HSTATE_ATTR_RO(resv_hugepages);

	static ssize_t surplus_hugepages_show(struct kobject *kobj,
	struct kobj_attribute attr, char buf)
	{
	struct hstate *h;
	unsigned long surplus_huge_pages;
	int nid;

	h = kobj_to_hstate(kobj, &nid);
	if (nid == NUMA_NO_NODE)
	surplus_huge_pages = h->surplus_huge_pages;
	else
	surplus_huge_pages = h->surplus_huge_pages_node[nid];

	return sysfs_emit(buf, "%lu\n", surplus_huge_pages);
	}
	HSTATE_ATTR_RO(surplus_hugepages);

	static ssize_t demote_store(struct kobject *kobj,
	struct kobj_attribute attr, const char buf, size_t len)
	{
	unsigned long nr_demote;
	unsigned long nr_available;
	nodemask_t nodes_allowed, *n_mask;
	struct hstate *h;
	int err;
	int nid;

	err = kstrtoul(buf, 10, &nr_demote);
	if (err)
	return err;
	h = kobj_to_hstate(kobj, &nid);

	if (nid != NUMA_NO_NODE) {
	init_nodemask_of_node(&nodes_allowed, nid);
	n_mask = &nodes_allowed;
	} else {
	n_mask = &node_states[N_MEMORY];
	}

	/* Synchronize with other sysfs operations modifying huge pages */
	mutex_lock(&h->resize_lock);
	spin_lock_irq(&hugetlb_lock);

	while (nr_demote) {
	long rc;

	/*
	* Check for available pages to demote each time thorough the
	* loop as demote_pool_huge_page will drop hugetlb_lock.
	*/
	if (nid != NUMA_NO_NODE)
	nr_available = h->free_huge_pages_node[nid];
	else
	nr_available = h->free_huge_pages;
	nr_available -= h->resv_huge_pages;
	if (!nr_available)
	break;

	rc = demote_pool_huge_page(h, n_mask, nr_demote);
	if (rc < 0) {
	err = rc;
	break;
	}

	nr_demote -= rc;
	}

	spin_unlock_irq(&hugetlb_lock);
	mutex_unlock(&h->resize_lock);

	if (err)
	return err;
	return len;
	}
	HSTATE_ATTR_WO(demote);

	static ssize_t demote_size_show(struct kobject *kobj,
	struct kobj_attribute attr, char buf)
	{
	struct hstate *h = kobj_to_hstate(kobj, NULL);
	unsigned long demote_size = (PAGE_SIZE << h->demote_order) / SZ_1K;

	return sysfs_emit(buf, "%lukB\n", demote_size);
	}

	static ssize_t demote_size_store(struct kobject *kobj,
	struct kobj_attribute *attr,
	const char *buf, size_t count)
	{
	struct hstate h, demote_hstate;
	unsigned long demote_size;
	unsigned int demote_order;

	demote_size = (unsigned long)memparse(buf, NULL);

	demote_hstate = size_to_hstate(demote_size);
	if (!demote_hstate)
	return -EINVAL;
	demote_order = demote_hstate->order;
	if (demote_order < HUGETLB_PAGE_ORDER)
	return -EINVAL;

	/* demote order must be smaller than hstate order */
	h = kobj_to_hstate(kobj, NULL);
	if (demote_order >= h->order)
	return -EINVAL;

	/* resize_lock synchronizes access to demote size and writes */
	mutex_lock(&h->resize_lock);
	h->demote_order = demote_order;
	mutex_unlock(&h->resize_lock);

	return count;
	}
	HSTATE_ATTR(demote_size);

	static struct attribute *hstate_attrs[] = {
	&nr_hugepages_attr.attr,
	&nr_overcommit_hugepages_attr.attr,
	&free_hugepages_attr.attr,
	&resv_hugepages_attr.attr,
	&surplus_hugepages_attr.attr,
	#ifdef CONFIG_NUMA
	&nr_hugepages_mempolicy_attr.attr,
	#endif
	NULL,
	};

	static const struct attribute_group hstate_attr_group = {
	.attrs = hstate_attrs,
	};

	static struct attribute *hstate_demote_attrs[] = {
	&demote_size_attr.attr,
	&demote_attr.attr,
	NULL,
	};

	static const struct attribute_group hstate_demote_attr_group = {
	.attrs = hstate_demote_attrs,
	};

	static int hugetlb_sysfs_add_hstate(struct hstate h, struct kobject parent,
	struct kobject **hstate_kobjs,
	const struct attribute_group *hstate_attr_group)
	{
	int retval;
	int hi = hstate_index(h);

	hstate_kobjs[hi] = kobject_create_and_add(h->name, parent);
	if (!hstate_kobjs[hi])
	return -ENOMEM;

	retval = sysfs_create_group(hstate_kobjs[hi], hstate_attr_group);
	if (retval) {
	kobject_put(hstate_kobjs[hi]);
	hstate_kobjs[hi] = NULL;
	return retval;
	}

	if (h->demote_order) {
	retval = sysfs_create_group(hstate_kobjs[hi],
	&hstate_demote_attr_group);
	if (retval) {
	pr_warn("HugeTLB unable to create demote interfaces for %s\n", h->name);
	sysfs_remove_group(hstate_kobjs[hi], hstate_attr_group);
	kobject_put(hstate_kobjs[hi]);
	hstate_kobjs[hi] = NULL;
	return retval;
	}
	}

	return 0;
	}

	#ifdef CONFIG_NUMA
	static bool hugetlb_sysfs_initialized __ro_after_init;

	/*
	* node_hstate/s - associate per node hstate attributes, via their kobjects,
	* with node devices in node_devices[] using a parallel array. The array
	* index of a node device or _hstate == node id.
	* This is here to avoid any static dependency of the node device driver, in
	* the base kernel, on the hugetlb module.
	*/
	struct node_hstate {
	struct kobject *hugepages_kobj;
	struct kobject *hstate_kobjs[HUGE_MAX_HSTATE];
	};
	static struct node_hstate node_hstates[MAX_NUMNODES];

	/*
	* A subset of global hstate attributes for node devices
	*/
	static struct attribute *per_node_hstate_attrs[] = {
	&nr_hugepages_attr.attr,
	&free_hugepages_attr.attr,
	&surplus_hugepages_attr.attr,
	NULL,
	};

	static const struct attribute_group per_node_hstate_attr_group = {
	.attrs = per_node_hstate_attrs,
	};

	/*
	* kobj_to_node_hstate - lookup global hstate for node device hstate attr kobj.
	* Returns node id via non-NULL nidp.
	*/
	static struct hstate kobj_to_node_hstate(struct kobject kobj, int *nidp)
	{
	int nid;

	for (nid = 0; nid < nr_node_ids; nid++) {
	struct node_hstate *nhs = &node_hstates[nid];
	int i;
	for (i = 0; i < HUGE_MAX_HSTATE; i++)
	if (nhs->hstate_kobjs[i] == kobj) {
	if (nidp)
	*nidp = nid;
	return &hstates[i];
	}
	}

	BUG();
	return NULL;
	}

	/*
	* Unregister hstate attributes from a single node device.
	* No-op if no hstate attributes attached.
	*/
	void hugetlb_unregister_node(struct node *node)
	{
	struct hstate *h;
	struct node_hstate *nhs = &node_hstates[node->dev.id];

	if (!nhs->hugepages_kobj)
	return; /* no hstate attributes */

	for_each_hstate(h) {
	int idx = hstate_index(h);
	struct kobject *hstate_kobj = nhs->hstate_kobjs[idx];

	if (!hstate_kobj)
	continue;
	if (h->demote_order)
	sysfs_remove_group(hstate_kobj, &hstate_demote_attr_group);
	sysfs_remove_group(hstate_kobj, &per_node_hstate_attr_group);
	kobject_put(hstate_kobj);
	nhs->hstate_kobjs[idx] = NULL;
	}

	kobject_put(nhs->hugepages_kobj);
	nhs->hugepages_kobj = NULL;
	}


	/*
	* Register hstate attributes for a single node device.
	* No-op if attributes already registered.
	*/
	void hugetlb_register_node(struct node *node)
	{
	struct hstate *h;
	struct node_hstate *nhs = &node_hstates[node->dev.id];
	int err;

	if (!hugetlb_sysfs_initialized)
	return;

	if (nhs->hugepages_kobj)
	return; /* already allocated */

	nhs->hugepages_kobj = kobject_create_and_add("hugepages",
	&node->dev.kobj);
	if (!nhs->hugepages_kobj)
	return;

	for_each_hstate(h) {
	err = hugetlb_sysfs_add_hstate(h, nhs->hugepages_kobj,
	nhs->hstate_kobjs,
	&per_node_hstate_attr_group);
	if (err) {
	pr_err("HugeTLB: Unable to add hstate %s for node %d\n",
	h->name, node->dev.id);
	hugetlb_unregister_node(node);
	break;
	}
	}
	}

	/*
	* hugetlb init time: register hstate attributes for all registered node
	* devices of nodes that have memory. All on-line nodes should have
	* registered their associated device by this time.
	*/
	static void __init hugetlb_register_all_nodes(void)
	{
	int nid;

	for_each_online_node(nid)
	hugetlb_register_node(node_devices[nid]);
	}
	#else /* !CONFIG_NUMA */

	static struct hstate kobj_to_node_hstate(struct kobject kobj, int *nidp)
	{
	BUG();
	if (nidp)
	*nidp = -1;
	return NULL;
	}

	static void hugetlb_register_all_nodes(void) { }

	#endif

	void __init hugetlb_sysfs_init(void)
	{
	struct hstate *h;
	int err;

	hugepages_kobj = kobject_create_and_add("hugepages", mm_kobj);
	if (!hugepages_kobj)
	return;

	for_each_hstate(h) {
	err = hugetlb_sysfs_add_hstate(h, hugepages_kobj,
	hstate_kobjs, &hstate_attr_group);
	if (err)
	pr_err("HugeTLB: Unable to add hstate %s\n", h->name);
	}

	#ifdef CONFIG_NUMA
	hugetlb_sysfs_initialized = true;
	#endif
	hugetlb_register_all_nodes();
	}