Merge branch 'akpm' (patches from Andrew)
Merge misc updates from Andrew Morton: - a few misc things - ocfs2 updates - most of MM * emailed patches from Andrew Morton <akpm@linux-foundation.org>: (159 commits) tools/testing/selftests/proc/proc-self-syscall.c: remove duplicate include proc: more robust bulk read test proc: test /proc/*/maps, smaps, smaps_rollup, statm proc: use seq_puts() everywhere proc: read kernel cpu stat pointer once proc: remove unused argument in proc_pid_lookup() fs/proc/thread_self.c: code cleanup for proc_setup_thread_self() fs/proc/self.c: code cleanup for proc_setup_self() proc: return exit code 4 for skipped tests mm,mremap: bail out earlier in mremap_to under map pressure mm/sparse: fix a bad comparison mm/memory.c: do_fault: avoid usage of stale vm_area_struct writeback: fix inode cgroup switching comment mm/huge_memory.c: fix "orig_pud" set but not used mm/hotplug: fix an imbalance with DEBUG_PAGEALLOC mm/memcontrol.c: fix bad line in comment mm/cma.c: cma_declare_contiguous: correct err handling mm/page_ext.c: fix an imbalance with kmemleak mm/compaction: pass pgdat to too_many_isolated() instead of zone mm: remove zone_lru_lock() function, access ->lru_lock directly ...
This commit is contained in:
Linus Torvalds
commit
8dcd175bc3
213 changed files with 4924 additions and 2321 deletions
|
|
@ -365,7 +365,7 @@ unlocked_inode_to_wb_begin(struct inode *inode, struct wb_lock_cookie *cookie)
|
|||
rcu_read_lock();
|
||||
|
||||
/*
|
||||
* Paired with store_release in inode_switch_wb_work_fn() and
|
||||
* Paired with store_release in inode_switch_wbs_work_fn() and
|
||||
* ensures that we see the new wb if we see cleared I_WB_SWITCH.
|
||||
*/
|
||||
cookie->locked = smp_load_acquire(&inode->i_state) & I_WB_SWITCH;
|
||||
|
|
|
|||
|
|
@ -4,15 +4,18 @@
|
|||
*
|
||||
* Common interface definitions for making balloon pages movable by compaction.
|
||||
*
|
||||
* Despite being perfectly possible to perform ballooned pages migration, they
|
||||
* make a special corner case to compaction scans because balloon pages are not
|
||||
* enlisted at any LRU list like the other pages we do compact / migrate.
|
||||
* Balloon page migration makes use of the general non-lru movable page
|
||||
* feature.
|
||||
*
|
||||
* page->private is used to reference the responsible balloon device.
|
||||
* page->mapping is used in context of non-lru page migration to reference
|
||||
* the address space operations for page isolation/migration/compaction.
|
||||
*
|
||||
* As the page isolation scanning step a compaction thread does is a lockless
|
||||
* procedure (from a page standpoint), it might bring some racy situations while
|
||||
* performing balloon page compaction. In order to sort out these racy scenarios
|
||||
* and safely perform balloon's page compaction and migration we must, always,
|
||||
* ensure following these three simple rules:
|
||||
* ensure following these simple rules:
|
||||
*
|
||||
* i. when updating a balloon's page ->mapping element, strictly do it under
|
||||
* the following lock order, independently of the far superior
|
||||
|
|
@ -21,19 +24,8 @@
|
|||
* +--spin_lock_irq(&b_dev_info->pages_lock);
|
||||
* ... page->mapping updates here ...
|
||||
*
|
||||
* ii. before isolating or dequeueing a balloon page from the balloon device
|
||||
* pages list, the page reference counter must be raised by one and the
|
||||
* extra refcount must be dropped when the page is enqueued back into
|
||||
* the balloon device page list, thus a balloon page keeps its reference
|
||||
* counter raised only while it is under our special handling;
|
||||
*
|
||||
* iii. after the lockless scan step have selected a potential balloon page for
|
||||
* isolation, re-test the PageBalloon mark and the PagePrivate flag
|
||||
* under the proper page lock, to ensure isolating a valid balloon page
|
||||
* (not yet isolated, nor under release procedure)
|
||||
*
|
||||
* iv. isolation or dequeueing procedure must clear PagePrivate flag under
|
||||
* page lock together with removing page from balloon device page list.
|
||||
* ii. isolation or dequeueing procedure must remove the page from balloon
|
||||
* device page list under b_dev_info->pages_lock.
|
||||
*
|
||||
* The functions provided by this interface are placed to help on coping with
|
||||
* the aforementioned balloon page corner case, as well as to ensure the simple
|
||||
|
|
@ -103,7 +95,7 @@ extern int balloon_page_migrate(struct address_space *mapping,
|
|||
static inline void balloon_page_insert(struct balloon_dev_info *balloon,
|
||||
struct page *page)
|
||||
{
|
||||
__SetPageBalloon(page);
|
||||
__SetPageOffline(page);
|
||||
__SetPageMovable(page, balloon->inode->i_mapping);
|
||||
set_page_private(page, (unsigned long)balloon);
|
||||
list_add(&page->lru, &balloon->pages);
|
||||
|
|
@ -119,7 +111,7 @@ static inline void balloon_page_insert(struct balloon_dev_info *balloon,
|
|||
*/
|
||||
static inline void balloon_page_delete(struct page *page)
|
||||
{
|
||||
__ClearPageBalloon(page);
|
||||
__ClearPageOffline(page);
|
||||
__ClearPageMovable(page);
|
||||
set_page_private(page, 0);
|
||||
/*
|
||||
|
|
@ -149,13 +141,13 @@ static inline gfp_t balloon_mapping_gfp_mask(void)
|
|||
static inline void balloon_page_insert(struct balloon_dev_info *balloon,
|
||||
struct page *page)
|
||||
{
|
||||
__SetPageBalloon(page);
|
||||
__SetPageOffline(page);
|
||||
list_add(&page->lru, &balloon->pages);
|
||||
}
|
||||
|
||||
static inline void balloon_page_delete(struct page *page)
|
||||
{
|
||||
__ClearPageBalloon(page);
|
||||
__ClearPageOffline(page);
|
||||
list_del(&page->lru);
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -32,6 +32,7 @@ struct kernfs_node;
|
|||
struct kernfs_ops;
|
||||
struct kernfs_open_file;
|
||||
struct seq_file;
|
||||
struct poll_table_struct;
|
||||
|
||||
#define MAX_CGROUP_TYPE_NAMELEN 32
|
||||
#define MAX_CGROUP_ROOT_NAMELEN 64
|
||||
|
|
@ -574,6 +575,9 @@ struct cftype {
|
|||
ssize_t (*write)(struct kernfs_open_file *of,
|
||||
char *buf, size_t nbytes, loff_t off);
|
||||
|
||||
__poll_t (*poll)(struct kernfs_open_file *of,
|
||||
struct poll_table_struct *pt);
|
||||
|
||||
#ifdef CONFIG_DEBUG_LOCK_ALLOC
|
||||
struct lock_class_key lockdep_key;
|
||||
#endif
|
||||
|
|
|
|||
|
|
@ -88,14 +88,13 @@ extern int sysctl_compact_memory;
|
|||
extern int sysctl_compaction_handler(struct ctl_table *table, int write,
|
||||
void __user *buffer, size_t *length, loff_t *ppos);
|
||||
extern int sysctl_extfrag_threshold;
|
||||
extern int sysctl_extfrag_handler(struct ctl_table *table, int write,
|
||||
void __user *buffer, size_t *length, loff_t *ppos);
|
||||
extern int sysctl_compact_unevictable_allowed;
|
||||
|
||||
extern int fragmentation_index(struct zone *zone, unsigned int order);
|
||||
extern enum compact_result try_to_compact_pages(gfp_t gfp_mask,
|
||||
unsigned int order, unsigned int alloc_flags,
|
||||
const struct alloc_context *ac, enum compact_priority prio);
|
||||
const struct alloc_context *ac, enum compact_priority prio,
|
||||
struct page **page);
|
||||
extern void reset_isolation_suitable(pg_data_t *pgdat);
|
||||
extern enum compact_result compaction_suitable(struct zone *zone, int order,
|
||||
unsigned int alloc_flags, int classzone_idx);
|
||||
|
|
@ -227,8 +226,8 @@ static inline void wakeup_kcompactd(pg_data_t *pgdat, int order, int classzone_i
|
|||
|
||||
#endif /* CONFIG_COMPACTION */
|
||||
|
||||
#if defined(CONFIG_COMPACTION) && defined(CONFIG_SYSFS) && defined(CONFIG_NUMA)
|
||||
struct node;
|
||||
#if defined(CONFIG_COMPACTION) && defined(CONFIG_SYSFS) && defined(CONFIG_NUMA)
|
||||
extern int compaction_register_node(struct node *node);
|
||||
extern void compaction_unregister_node(struct node *node);
|
||||
|
||||
|
|
|
|||
|
|
@ -1095,7 +1095,7 @@ static inline void set_dev_node(struct device *dev, int node)
|
|||
#else
|
||||
static inline int dev_to_node(struct device *dev)
|
||||
{
|
||||
return -1;
|
||||
return NUMA_NO_NODE;
|
||||
}
|
||||
static inline void set_dev_node(struct device *dev, int node)
|
||||
{
|
||||
|
|
|
|||
|
|
@ -7,6 +7,13 @@
|
|||
#include <linux/bitops.h>
|
||||
#include <linux/jump_label.h>
|
||||
|
||||
/*
|
||||
* Return code to denote that requested number of
|
||||
* frontswap pages are unused(moved to page cache).
|
||||
* Used in in shmem_unuse and try_to_unuse.
|
||||
*/
|
||||
#define FRONTSWAP_PAGES_UNUSED 2
|
||||
|
||||
struct frontswap_ops {
|
||||
void (*init)(unsigned); /* this swap type was just swapon'ed */
|
||||
int (*store)(unsigned, pgoff_t, struct page *); /* store a page */
|
||||
|
|
|
|||
|
|
@ -2091,7 +2091,7 @@ static inline void init_sync_kiocb(struct kiocb *kiocb, struct file *filp)
|
|||
* I_WB_SWITCH Cgroup bdi_writeback switching in progress. Used to
|
||||
* synchronize competing switching instances and to tell
|
||||
* wb stat updates to grab the i_pages lock. See
|
||||
* inode_switch_wb_work_fn() for details.
|
||||
* inode_switch_wbs_work_fn() for details.
|
||||
*
|
||||
* I_OVL_INUSE Used by overlayfs to get exclusive ownership on upper
|
||||
* and work dirs among overlayfs mounts.
|
||||
|
|
|
|||
|
|
@ -24,21 +24,21 @@ struct vm_area_struct;
|
|||
#define ___GFP_HIGH 0x20u
|
||||
#define ___GFP_IO 0x40u
|
||||
#define ___GFP_FS 0x80u
|
||||
#define ___GFP_WRITE 0x100u
|
||||
#define ___GFP_NOWARN 0x200u
|
||||
#define ___GFP_RETRY_MAYFAIL 0x400u
|
||||
#define ___GFP_NOFAIL 0x800u
|
||||
#define ___GFP_NORETRY 0x1000u
|
||||
#define ___GFP_MEMALLOC 0x2000u
|
||||
#define ___GFP_COMP 0x4000u
|
||||
#define ___GFP_ZERO 0x8000u
|
||||
#define ___GFP_NOMEMALLOC 0x10000u
|
||||
#define ___GFP_HARDWALL 0x20000u
|
||||
#define ___GFP_THISNODE 0x40000u
|
||||
#define ___GFP_ATOMIC 0x80000u
|
||||
#define ___GFP_ACCOUNT 0x100000u
|
||||
#define ___GFP_DIRECT_RECLAIM 0x200000u
|
||||
#define ___GFP_KSWAPD_RECLAIM 0x400000u
|
||||
#define ___GFP_ZERO 0x100u
|
||||
#define ___GFP_ATOMIC 0x200u
|
||||
#define ___GFP_DIRECT_RECLAIM 0x400u
|
||||
#define ___GFP_KSWAPD_RECLAIM 0x800u
|
||||
#define ___GFP_WRITE 0x1000u
|
||||
#define ___GFP_NOWARN 0x2000u
|
||||
#define ___GFP_RETRY_MAYFAIL 0x4000u
|
||||
#define ___GFP_NOFAIL 0x8000u
|
||||
#define ___GFP_NORETRY 0x10000u
|
||||
#define ___GFP_MEMALLOC 0x20000u
|
||||
#define ___GFP_COMP 0x40000u
|
||||
#define ___GFP_NOMEMALLOC 0x80000u
|
||||
#define ___GFP_HARDWALL 0x100000u
|
||||
#define ___GFP_THISNODE 0x200000u
|
||||
#define ___GFP_ACCOUNT 0x400000u
|
||||
#ifdef CONFIG_LOCKDEP
|
||||
#define ___GFP_NOLOCKDEP 0x800000u
|
||||
#else
|
||||
|
|
|
|||
|
|
@ -371,6 +371,8 @@ struct page *alloc_huge_page_nodemask(struct hstate *h, int preferred_nid,
|
|||
nodemask_t *nmask);
|
||||
struct page *alloc_huge_page_vma(struct hstate *h, struct vm_area_struct *vma,
|
||||
unsigned long address);
|
||||
struct page *alloc_migrate_huge_page(struct hstate *h, gfp_t gfp_mask,
|
||||
int nid, nodemask_t *nmask);
|
||||
int huge_add_to_page_cache(struct page *page, struct address_space *mapping,
|
||||
pgoff_t idx);
|
||||
|
||||
|
|
@ -493,17 +495,54 @@ static inline pgoff_t basepage_index(struct page *page)
|
|||
extern int dissolve_free_huge_page(struct page *page);
|
||||
extern int dissolve_free_huge_pages(unsigned long start_pfn,
|
||||
unsigned long end_pfn);
|
||||
static inline bool hugepage_migration_supported(struct hstate *h)
|
||||
{
|
||||
|
||||
#ifdef CONFIG_ARCH_ENABLE_HUGEPAGE_MIGRATION
|
||||
#ifndef arch_hugetlb_migration_supported
|
||||
static inline bool arch_hugetlb_migration_supported(struct hstate *h)
|
||||
{
|
||||
if ((huge_page_shift(h) == PMD_SHIFT) ||
|
||||
(huge_page_shift(h) == PGDIR_SHIFT))
|
||||
(huge_page_shift(h) == PUD_SHIFT) ||
|
||||
(huge_page_shift(h) == PGDIR_SHIFT))
|
||||
return true;
|
||||
else
|
||||
return false;
|
||||
#else
|
||||
return false;
|
||||
}
|
||||
#endif
|
||||
#else
|
||||
static inline bool arch_hugetlb_migration_supported(struct hstate *h)
|
||||
{
|
||||
return false;
|
||||
}
|
||||
#endif
|
||||
|
||||
static inline bool hugepage_migration_supported(struct hstate *h)
|
||||
{
|
||||
return arch_hugetlb_migration_supported(h);
|
||||
}
|
||||
|
||||
/*
|
||||
* Movability check is different as compared to migration check.
|
||||
* It determines whether or not a huge page should be placed on
|
||||
* movable zone or not. Movability of any huge page should be
|
||||
* required only if huge page size is supported for migration.
|
||||
* There wont be any reason for the huge page to be movable if
|
||||
* it is not migratable to start with. Also the size of the huge
|
||||
* page should be large enough to be placed under a movable zone
|
||||
* and still feasible enough to be migratable. Just the presence
|
||||
* in movable zone does not make the migration feasible.
|
||||
*
|
||||
* So even though large huge page sizes like the gigantic ones
|
||||
* are migratable they should not be movable because its not
|
||||
* feasible to migrate them from movable zone.
|
||||
*/
|
||||
static inline bool hugepage_movable_supported(struct hstate *h)
|
||||
{
|
||||
if (!hugepage_migration_supported(h))
|
||||
return false;
|
||||
|
||||
if (hstate_is_gigantic(h))
|
||||
return false;
|
||||
return true;
|
||||
}
|
||||
|
||||
static inline spinlock_t *huge_pte_lockptr(struct hstate *h,
|
||||
|
|
@ -543,6 +582,26 @@ static inline void set_huge_swap_pte_at(struct mm_struct *mm, unsigned long addr
|
|||
set_huge_pte_at(mm, addr, ptep, pte);
|
||||
}
|
||||
#endif
|
||||
|
||||
#ifndef huge_ptep_modify_prot_start
|
||||
#define huge_ptep_modify_prot_start huge_ptep_modify_prot_start
|
||||
static inline pte_t huge_ptep_modify_prot_start(struct vm_area_struct *vma,
|
||||
unsigned long addr, pte_t *ptep)
|
||||
{
|
||||
return huge_ptep_get_and_clear(vma->vm_mm, addr, ptep);
|
||||
}
|
||||
#endif
|
||||
|
||||
#ifndef huge_ptep_modify_prot_commit
|
||||
#define huge_ptep_modify_prot_commit huge_ptep_modify_prot_commit
|
||||
static inline void huge_ptep_modify_prot_commit(struct vm_area_struct *vma,
|
||||
unsigned long addr, pte_t *ptep,
|
||||
pte_t old_pte, pte_t pte)
|
||||
{
|
||||
set_huge_pte_at(vma->vm_mm, addr, ptep, pte);
|
||||
}
|
||||
#endif
|
||||
|
||||
#else /* CONFIG_HUGETLB_PAGE */
|
||||
struct hstate {};
|
||||
#define alloc_huge_page(v, a, r) NULL
|
||||
|
|
@ -602,6 +661,11 @@ static inline bool hugepage_migration_supported(struct hstate *h)
|
|||
return false;
|
||||
}
|
||||
|
||||
static inline bool hugepage_movable_supported(struct hstate *h)
|
||||
{
|
||||
return false;
|
||||
}
|
||||
|
||||
static inline spinlock_t *huge_pte_lockptr(struct hstate *h,
|
||||
struct mm_struct *mm, pte_t *pte)
|
||||
{
|
||||
|
|
|
|||
|
|
@ -2,7 +2,7 @@
|
|||
#ifndef _LINUX_KASAN_CHECKS_H
|
||||
#define _LINUX_KASAN_CHECKS_H
|
||||
|
||||
#ifdef CONFIG_KASAN
|
||||
#if defined(__SANITIZE_ADDRESS__) || defined(__KASAN_INTERNAL)
|
||||
void kasan_check_read(const volatile void *p, unsigned int size);
|
||||
void kasan_check_write(const volatile void *p, unsigned int size);
|
||||
#else
|
||||
|
|
|
|||
|
|
@ -25,6 +25,7 @@ struct seq_file;
|
|||
struct vm_area_struct;
|
||||
struct super_block;
|
||||
struct file_system_type;
|
||||
struct poll_table_struct;
|
||||
|
||||
struct kernfs_open_node;
|
||||
struct kernfs_iattrs;
|
||||
|
|
@ -261,6 +262,9 @@ struct kernfs_ops {
|
|||
ssize_t (*write)(struct kernfs_open_file *of, char *buf, size_t bytes,
|
||||
loff_t off);
|
||||
|
||||
__poll_t (*poll)(struct kernfs_open_file *of,
|
||||
struct poll_table_struct *pt);
|
||||
|
||||
int (*mmap)(struct kernfs_open_file *of, struct vm_area_struct *vma);
|
||||
|
||||
#ifdef CONFIG_DEBUG_LOCK_ALLOC
|
||||
|
|
@ -350,6 +354,8 @@ int kernfs_remove_by_name_ns(struct kernfs_node *parent, const char *name,
|
|||
int kernfs_rename_ns(struct kernfs_node *kn, struct kernfs_node *new_parent,
|
||||
const char *new_name, const void *new_ns);
|
||||
int kernfs_setattr(struct kernfs_node *kn, const struct iattr *iattr);
|
||||
__poll_t kernfs_generic_poll(struct kernfs_open_file *of,
|
||||
struct poll_table_struct *pt);
|
||||
void kernfs_notify(struct kernfs_node *kn);
|
||||
|
||||
const void *kernfs_super_ns(struct super_block *sb);
|
||||
|
|
|
|||
|
|
@ -53,6 +53,8 @@ struct page *ksm_might_need_to_copy(struct page *page,
|
|||
|
||||
void rmap_walk_ksm(struct page *page, struct rmap_walk_control *rwc);
|
||||
void ksm_migrate_page(struct page *newpage, struct page *oldpage);
|
||||
bool reuse_ksm_page(struct page *page,
|
||||
struct vm_area_struct *vma, unsigned long address);
|
||||
|
||||
#else /* !CONFIG_KSM */
|
||||
|
||||
|
|
@ -86,6 +88,11 @@ static inline void rmap_walk_ksm(struct page *page,
|
|||
static inline void ksm_migrate_page(struct page *newpage, struct page *oldpage)
|
||||
{
|
||||
}
|
||||
static inline bool reuse_ksm_page(struct page *page,
|
||||
struct vm_area_struct *vma, unsigned long address)
|
||||
{
|
||||
return false;
|
||||
}
|
||||
#endif /* CONFIG_MMU */
|
||||
#endif /* !CONFIG_KSM */
|
||||
|
||||
|
|
|
|||
|
|
@ -206,6 +206,17 @@ static inline void list_bulk_move_tail(struct list_head *head,
|
|||
head->prev = last;
|
||||
}
|
||||
|
||||
/**
|
||||
* list_is_first -- tests whether @ list is the first entry in list @head
|
||||
* @list: the entry to test
|
||||
* @head: the head of the list
|
||||
*/
|
||||
static inline int list_is_first(const struct list_head *list,
|
||||
const struct list_head *head)
|
||||
{
|
||||
return list->prev == head;
|
||||
}
|
||||
|
||||
/**
|
||||
* list_is_last - tests whether @list is the last entry in list @head
|
||||
* @list: the entry to test
|
||||
|
|
|
|||
|
|
@ -429,6 +429,11 @@ static inline unsigned short mem_cgroup_id(struct mem_cgroup *memcg)
|
|||
}
|
||||
struct mem_cgroup *mem_cgroup_from_id(unsigned short id);
|
||||
|
||||
static inline struct mem_cgroup *mem_cgroup_from_seq(struct seq_file *m)
|
||||
{
|
||||
return mem_cgroup_from_css(seq_css(m));
|
||||
}
|
||||
|
||||
static inline struct mem_cgroup *lruvec_memcg(struct lruvec *lruvec)
|
||||
{
|
||||
struct mem_cgroup_per_node *mz;
|
||||
|
|
@ -937,6 +942,11 @@ static inline struct mem_cgroup *mem_cgroup_from_id(unsigned short id)
|
|||
return NULL;
|
||||
}
|
||||
|
||||
static inline struct mem_cgroup *mem_cgroup_from_seq(struct seq_file *m)
|
||||
{
|
||||
return NULL;
|
||||
}
|
||||
|
||||
static inline struct mem_cgroup *lruvec_memcg(struct lruvec *lruvec)
|
||||
{
|
||||
return NULL;
|
||||
|
|
@ -1273,12 +1283,12 @@ static inline bool mem_cgroup_under_socket_pressure(struct mem_cgroup *memcg)
|
|||
|
||||
struct kmem_cache *memcg_kmem_get_cache(struct kmem_cache *cachep);
|
||||
void memcg_kmem_put_cache(struct kmem_cache *cachep);
|
||||
int memcg_kmem_charge_memcg(struct page *page, gfp_t gfp, int order,
|
||||
struct mem_cgroup *memcg);
|
||||
|
||||
#ifdef CONFIG_MEMCG_KMEM
|
||||
int memcg_kmem_charge(struct page *page, gfp_t gfp, int order);
|
||||
void memcg_kmem_uncharge(struct page *page, int order);
|
||||
int __memcg_kmem_charge(struct page *page, gfp_t gfp, int order);
|
||||
void __memcg_kmem_uncharge(struct page *page, int order);
|
||||
int __memcg_kmem_charge_memcg(struct page *page, gfp_t gfp, int order,
|
||||
struct mem_cgroup *memcg);
|
||||
|
||||
extern struct static_key_false memcg_kmem_enabled_key;
|
||||
extern struct workqueue_struct *memcg_kmem_cache_wq;
|
||||
|
|
@ -1300,6 +1310,26 @@ static inline bool memcg_kmem_enabled(void)
|
|||
return static_branch_unlikely(&memcg_kmem_enabled_key);
|
||||
}
|
||||
|
||||
static inline int memcg_kmem_charge(struct page *page, gfp_t gfp, int order)
|
||||
{
|
||||
if (memcg_kmem_enabled())
|
||||
return __memcg_kmem_charge(page, gfp, order);
|
||||
return 0;
|
||||
}
|
||||
|
||||
static inline void memcg_kmem_uncharge(struct page *page, int order)
|
||||
{
|
||||
if (memcg_kmem_enabled())
|
||||
__memcg_kmem_uncharge(page, order);
|
||||
}
|
||||
|
||||
static inline int memcg_kmem_charge_memcg(struct page *page, gfp_t gfp,
|
||||
int order, struct mem_cgroup *memcg)
|
||||
{
|
||||
if (memcg_kmem_enabled())
|
||||
return __memcg_kmem_charge_memcg(page, gfp, order, memcg);
|
||||
return 0;
|
||||
}
|
||||
/*
|
||||
* helper for accessing a memcg's index. It will be used as an index in the
|
||||
* child cache array in kmem_cache, and also to derive its name. This function
|
||||
|
|
@ -1325,6 +1355,15 @@ static inline void memcg_kmem_uncharge(struct page *page, int order)
|
|||
{
|
||||
}
|
||||
|
||||
static inline int __memcg_kmem_charge(struct page *page, gfp_t gfp, int order)
|
||||
{
|
||||
return 0;
|
||||
}
|
||||
|
||||
static inline void __memcg_kmem_uncharge(struct page *page, int order)
|
||||
{
|
||||
}
|
||||
|
||||
#define for_each_memcg_cache_index(_idx) \
|
||||
for (; NULL; )
|
||||
|
||||
|
|
|
|||
|
|
@ -89,7 +89,7 @@ extern int test_pages_in_a_zone(unsigned long start_pfn, unsigned long end_pfn,
|
|||
unsigned long *valid_start, unsigned long *valid_end);
|
||||
extern void __offline_isolated_pages(unsigned long, unsigned long);
|
||||
|
||||
typedef void (*online_page_callback_t)(struct page *page);
|
||||
typedef void (*online_page_callback_t)(struct page *page, unsigned int order);
|
||||
|
||||
extern int set_online_page_callback(online_page_callback_t callback);
|
||||
extern int restore_online_page_callback(online_page_callback_t callback);
|
||||
|
|
|
|||
|
|
@ -1536,7 +1536,8 @@ long get_user_pages_locked(unsigned long start, unsigned long nr_pages,
|
|||
unsigned int gup_flags, struct page **pages, int *locked);
|
||||
long get_user_pages_unlocked(unsigned long start, unsigned long nr_pages,
|
||||
struct page **pages, unsigned int gup_flags);
|
||||
#ifdef CONFIG_FS_DAX
|
||||
|
||||
#if defined(CONFIG_FS_DAX) || defined(CONFIG_CMA)
|
||||
long get_user_pages_longterm(unsigned long start, unsigned long nr_pages,
|
||||
unsigned int gup_flags, struct page **pages,
|
||||
struct vm_area_struct **vmas);
|
||||
|
|
|
|||
|
|
@ -80,7 +80,7 @@ struct page {
|
|||
struct { /* Page cache and anonymous pages */
|
||||
/**
|
||||
* @lru: Pageout list, eg. active_list protected by
|
||||
* zone_lru_lock. Sometimes used as a generic list
|
||||
* pgdat->lru_lock. Sometimes used as a generic list
|
||||
* by the page owner.
|
||||
*/
|
||||
struct list_head lru;
|
||||
|
|
|
|||
|
|
@ -480,6 +480,8 @@ struct zone {
|
|||
unsigned long compact_cached_free_pfn;
|
||||
/* pfn where async and sync compaction migration scanner should start */
|
||||
unsigned long compact_cached_migrate_pfn[2];
|
||||
unsigned long compact_init_migrate_pfn;
|
||||
unsigned long compact_init_free_pfn;
|
||||
#endif
|
||||
|
||||
#ifdef CONFIG_COMPACTION
|
||||
|
|
@ -728,10 +730,6 @@ typedef struct pglist_data {
|
|||
|
||||
#define node_start_pfn(nid) (NODE_DATA(nid)->node_start_pfn)
|
||||
#define node_end_pfn(nid) pgdat_end_pfn(NODE_DATA(nid))
|
||||
static inline spinlock_t *zone_lru_lock(struct zone *zone)
|
||||
{
|
||||
return &zone->zone_pgdat->lru_lock;
|
||||
}
|
||||
|
||||
static inline struct lruvec *node_lruvec(struct pglist_data *pgdat)
|
||||
{
|
||||
|
|
@ -1299,7 +1297,7 @@ void memory_present(int nid, unsigned long start, unsigned long end);
|
|||
|
||||
/*
|
||||
* If it is possible to have holes within a MAX_ORDER_NR_PAGES, then we
|
||||
* need to check pfn validility within that MAX_ORDER_NR_PAGES block.
|
||||
* need to check pfn validity within that MAX_ORDER_NR_PAGES block.
|
||||
* pfn_valid_within() should be used in this case; we optimise this away
|
||||
* when we have no holes within a MAX_ORDER_NR_PAGES block.
|
||||
*/
|
||||
|
|
|
|||
|
|
@ -444,8 +444,8 @@ static inline int next_memory_node(int nid)
|
|||
return next_node(nid, node_states[N_MEMORY]);
|
||||
}
|
||||
|
||||
extern int nr_node_ids;
|
||||
extern int nr_online_nodes;
|
||||
extern unsigned int nr_node_ids;
|
||||
extern unsigned int nr_online_nodes;
|
||||
|
||||
static inline void node_set_online(int nid)
|
||||
{
|
||||
|
|
@ -485,8 +485,8 @@ static inline int num_node_state(enum node_states state)
|
|||
#define first_online_node 0
|
||||
#define first_memory_node 0
|
||||
#define next_online_node(nid) (MAX_NUMNODES)
|
||||
#define nr_node_ids 1
|
||||
#define nr_online_nodes 1
|
||||
#define nr_node_ids 1U
|
||||
#define nr_online_nodes 1U
|
||||
|
||||
#define node_set_online(node) node_set_state((node), N_ONLINE)
|
||||
#define node_set_offline(node) node_clear_state((node), N_ONLINE)
|
||||
|
|
|
|||
|
|
@ -17,8 +17,37 @@
|
|||
/*
|
||||
* Various page->flags bits:
|
||||
*
|
||||
* PG_reserved is set for special pages, which can never be swapped out. Some
|
||||
* of them might not even exist...
|
||||
* PG_reserved is set for special pages. The "struct page" of such a page
|
||||
* should in general not be touched (e.g. set dirty) except by its owner.
|
||||
* Pages marked as PG_reserved include:
|
||||
* - Pages part of the kernel image (including vDSO) and similar (e.g. BIOS,
|
||||
* initrd, HW tables)
|
||||
* - Pages reserved or allocated early during boot (before the page allocator
|
||||
* was initialized). This includes (depending on the architecture) the
|
||||
* initial vmemmap, initial page tables, crashkernel, elfcorehdr, and much
|
||||
* much more. Once (if ever) freed, PG_reserved is cleared and they will
|
||||
* be given to the page allocator.
|
||||
* - Pages falling into physical memory gaps - not IORESOURCE_SYSRAM. Trying
|
||||
* to read/write these pages might end badly. Don't touch!
|
||||
* - The zero page(s)
|
||||
* - Pages not added to the page allocator when onlining a section because
|
||||
* they were excluded via the online_page_callback() or because they are
|
||||
* PG_hwpoison.
|
||||
* - Pages allocated in the context of kexec/kdump (loaded kernel image,
|
||||
* control pages, vmcoreinfo)
|
||||
* - MMIO/DMA pages. Some architectures don't allow to ioremap pages that are
|
||||
* not marked PG_reserved (as they might be in use by somebody else who does
|
||||
* not respect the caching strategy).
|
||||
* - Pages part of an offline section (struct pages of offline sections should
|
||||
* not be trusted as they will be initialized when first onlined).
|
||||
* - MCA pages on ia64
|
||||
* - Pages holding CPU notes for POWER Firmware Assisted Dump
|
||||
* - Device memory (e.g. PMEM, DAX, HMM)
|
||||
* Some PG_reserved pages will be excluded from the hibernation image.
|
||||
* PG_reserved does in general not hinder anybody from dumping or swapping
|
||||
* and is no longer required for remap_pfn_range(). ioremap might require it.
|
||||
* Consequently, PG_reserved for a page mapped into user space can indicate
|
||||
* the zero page, the vDSO, MMIO pages or device memory.
|
||||
*
|
||||
* The PG_private bitflag is set on pagecache pages if they contain filesystem
|
||||
* specific data (which is normally at page->private). It can be used by
|
||||
|
|
@ -671,7 +700,7 @@ PAGEFLAG_FALSE(DoubleMap)
|
|||
/* Reserve 0x0000007f to catch underflows of page_mapcount */
|
||||
#define PAGE_MAPCOUNT_RESERVE -128
|
||||
#define PG_buddy 0x00000080
|
||||
#define PG_balloon 0x00000100
|
||||
#define PG_offline 0x00000100
|
||||
#define PG_kmemcg 0x00000200
|
||||
#define PG_table 0x00000400
|
||||
|
||||
|
|
@ -706,10 +735,13 @@ static __always_inline void __ClearPage##uname(struct page *page) \
|
|||
PAGE_TYPE_OPS(Buddy, buddy)
|
||||
|
||||
/*
|
||||
* PageBalloon() is true for pages that are on the balloon page list
|
||||
* (see mm/balloon_compaction.c).
|
||||
* PageOffline() indicates that the page is logically offline although the
|
||||
* containing section is online. (e.g. inflated in a balloon driver or
|
||||
* not onlined when onlining the section).
|
||||
* The content of these pages is effectively stale. Such pages should not
|
||||
* be touched (read/write/dump/save) except by their owner.
|
||||
*/
|
||||
PAGE_TYPE_OPS(Balloon, balloon)
|
||||
PAGE_TYPE_OPS(Offline, offline)
|
||||
|
||||
/*
|
||||
* If kmemcg is enabled, the buddy allocator will set PageKmemcg() on
|
||||
|
|
|
|||
|
|
@ -164,7 +164,7 @@ void release_pages(struct page **pages, int nr);
|
|||
* will find the page or it will not. Likewise, the old find_get_page could run
|
||||
* either before the insertion or afterwards, depending on timing.
|
||||
*/
|
||||
static inline int page_cache_get_speculative(struct page *page)
|
||||
static inline int __page_cache_add_speculative(struct page *page, int count)
|
||||
{
|
||||
#ifdef CONFIG_TINY_RCU
|
||||
# ifdef CONFIG_PREEMPT_COUNT
|
||||
|
|
@ -180,10 +180,10 @@ static inline int page_cache_get_speculative(struct page *page)
|
|||
* SMP requires.
|
||||
*/
|
||||
VM_BUG_ON_PAGE(page_count(page) == 0, page);
|
||||
page_ref_inc(page);
|
||||
page_ref_add(page, count);
|
||||
|
||||
#else
|
||||
if (unlikely(!get_page_unless_zero(page))) {
|
||||
if (unlikely(!page_ref_add_unless(page, count, 0))) {
|
||||
/*
|
||||
* Either the page has been freed, or will be freed.
|
||||
* In either case, retry here and the caller should
|
||||
|
|
@ -197,27 +197,14 @@ static inline int page_cache_get_speculative(struct page *page)
|
|||
return 1;
|
||||
}
|
||||
|
||||
/*
|
||||
* Same as above, but add instead of inc (could just be merged)
|
||||
*/
|
||||
static inline int page_cache_get_speculative(struct page *page)
|
||||
{
|
||||
return __page_cache_add_speculative(page, 1);
|
||||
}
|
||||
|
||||
static inline int page_cache_add_speculative(struct page *page, int count)
|
||||
{
|
||||
VM_BUG_ON(in_interrupt());
|
||||
|
||||
#if !defined(CONFIG_SMP) && defined(CONFIG_TREE_RCU)
|
||||
# ifdef CONFIG_PREEMPT_COUNT
|
||||
VM_BUG_ON(!in_atomic() && !irqs_disabled());
|
||||
# endif
|
||||
VM_BUG_ON_PAGE(page_count(page) == 0, page);
|
||||
page_ref_add(page, count);
|
||||
|
||||
#else
|
||||
if (unlikely(!page_ref_add_unless(page, count, 0)))
|
||||
return 0;
|
||||
#endif
|
||||
VM_BUG_ON_PAGE(PageCompound(page) && page != compound_head(page), page);
|
||||
|
||||
return 1;
|
||||
return __page_cache_add_speculative(page, count);
|
||||
}
|
||||
|
||||
#ifdef CONFIG_NUMA
|
||||
|
|
|
|||
|
|
@ -30,7 +30,7 @@
|
|||
*/
|
||||
#define TIMER_ENTRY_STATIC ((void *) 0x300 + POISON_POINTER_DELTA)
|
||||
|
||||
/********** mm/debug-pagealloc.c **********/
|
||||
/********** mm/page_poison.c **********/
|
||||
#ifdef CONFIG_PAGE_POISONING_ZERO
|
||||
#define PAGE_POISON 0x00
|
||||
#else
|
||||
|
|
|
|||
|
|
@ -48,6 +48,7 @@ struct pid_namespace;
|
|||
struct pipe_inode_info;
|
||||
struct rcu_node;
|
||||
struct reclaim_state;
|
||||
struct capture_control;
|
||||
struct robust_list_head;
|
||||
struct sched_attr;
|
||||
struct sched_param;
|
||||
|
|
@ -950,6 +951,9 @@ struct task_struct {
|
|||
|
||||
struct io_context *io_context;
|
||||
|
||||
#ifdef CONFIG_COMPACTION
|
||||
struct capture_control *capture_control;
|
||||
#endif
|
||||
/* Ptrace state: */
|
||||
unsigned long ptrace_message;
|
||||
kernel_siginfo_t *last_siginfo;
|
||||
|
|
@ -1395,6 +1399,7 @@ extern struct pid *cad_pid;
|
|||
#define PF_UMH 0x02000000 /* I'm an Usermodehelper process */
|
||||
#define PF_NO_SETAFFINITY 0x04000000 /* Userland is not allowed to meddle with cpus_allowed */
|
||||
#define PF_MCE_EARLY 0x08000000 /* Early kill for mce process policy */
|
||||
#define PF_MEMALLOC_NOCMA 0x10000000 /* All allocation request will have _GFP_MOVABLE cleared */
|
||||
#define PF_FREEZER_SKIP 0x40000000 /* Freezer should not count it as freezable */
|
||||
#define PF_SUSPEND_TASK 0x80000000 /* This thread called freeze_processes() and should not be frozen */
|
||||
|
||||
|
|
|
|||
|
|
@ -148,17 +148,25 @@ static inline bool in_vfork(struct task_struct *tsk)
|
|||
* Applies per-task gfp context to the given allocation flags.
|
||||
* PF_MEMALLOC_NOIO implies GFP_NOIO
|
||||
* PF_MEMALLOC_NOFS implies GFP_NOFS
|
||||
* PF_MEMALLOC_NOCMA implies no allocation from CMA region.
|
||||
*/
|
||||
static inline gfp_t current_gfp_context(gfp_t flags)
|
||||
{
|
||||
/*
|
||||
* NOIO implies both NOIO and NOFS and it is a weaker context
|
||||
* so always make sure it makes precedence
|
||||
*/
|
||||
if (unlikely(current->flags & PF_MEMALLOC_NOIO))
|
||||
flags &= ~(__GFP_IO | __GFP_FS);
|
||||
else if (unlikely(current->flags & PF_MEMALLOC_NOFS))
|
||||
flags &= ~__GFP_FS;
|
||||
if (unlikely(current->flags &
|
||||
(PF_MEMALLOC_NOIO | PF_MEMALLOC_NOFS | PF_MEMALLOC_NOCMA))) {
|
||||
/*
|
||||
* NOIO implies both NOIO and NOFS and it is a weaker context
|
||||
* so always make sure it makes precedence
|
||||
*/
|
||||
if (current->flags & PF_MEMALLOC_NOIO)
|
||||
flags &= ~(__GFP_IO | __GFP_FS);
|
||||
else if (current->flags & PF_MEMALLOC_NOFS)
|
||||
flags &= ~__GFP_FS;
|
||||
#ifdef CONFIG_CMA
|
||||
if (current->flags & PF_MEMALLOC_NOCMA)
|
||||
flags &= ~__GFP_MOVABLE;
|
||||
#endif
|
||||
}
|
||||
return flags;
|
||||
}
|
||||
|
||||
|
|
@ -248,6 +256,30 @@ static inline void memalloc_noreclaim_restore(unsigned int flags)
|
|||
current->flags = (current->flags & ~PF_MEMALLOC) | flags;
|
||||
}
|
||||
|
||||
#ifdef CONFIG_CMA
|
||||
static inline unsigned int memalloc_nocma_save(void)
|
||||
{
|
||||
unsigned int flags = current->flags & PF_MEMALLOC_NOCMA;
|
||||
|
||||
current->flags |= PF_MEMALLOC_NOCMA;
|
||||
return flags;
|
||||
}
|
||||
|
||||
static inline void memalloc_nocma_restore(unsigned int flags)
|
||||
{
|
||||
current->flags = (current->flags & ~PF_MEMALLOC_NOCMA) | flags;
|
||||
}
|
||||
#else
|
||||
static inline unsigned int memalloc_nocma_save(void)
|
||||
{
|
||||
return 0;
|
||||
}
|
||||
|
||||
static inline void memalloc_nocma_restore(unsigned int flags)
|
||||
{
|
||||
}
|
||||
#endif
|
||||
|
||||
#ifdef CONFIG_MEMCG
|
||||
/**
|
||||
* memalloc_use_memcg - Starts the remote memcg charging scope.
|
||||
|
|
|
|||
|
|
@ -72,7 +72,8 @@ extern void shmem_unlock_mapping(struct address_space *mapping);
|
|||
extern struct page *shmem_read_mapping_page_gfp(struct address_space *mapping,
|
||||
pgoff_t index, gfp_t gfp_mask);
|
||||
extern void shmem_truncate_range(struct inode *inode, loff_t start, loff_t end);
|
||||
extern int shmem_unuse(swp_entry_t entry, struct page *page);
|
||||
extern int shmem_unuse(unsigned int type, bool frontswap,
|
||||
unsigned long *fs_pages_to_unuse);
|
||||
|
||||
extern unsigned long shmem_swap_usage(struct vm_area_struct *vma);
|
||||
extern unsigned long shmem_partial_swap_usage(struct address_space *mapping,
|
||||
|
|
|
|||
|
|
@ -81,12 +81,12 @@ struct kmem_cache_order_objects {
|
|||
*/
|
||||
struct kmem_cache {
|
||||
struct kmem_cache_cpu __percpu *cpu_slab;
|
||||
/* Used for retriving partial slabs etc */
|
||||
/* Used for retrieving partial slabs, etc. */
|
||||
slab_flags_t flags;
|
||||
unsigned long min_partial;
|
||||
unsigned int size; /* The size of an object including meta data */
|
||||
unsigned int object_size;/* The size of an object without meta data */
|
||||
unsigned int offset; /* Free pointer offset. */
|
||||
unsigned int size; /* The size of an object including metadata */
|
||||
unsigned int object_size;/* The size of an object without metadata */
|
||||
unsigned int offset; /* Free pointer offset */
|
||||
#ifdef CONFIG_SLUB_CPU_PARTIAL
|
||||
/* Number of per cpu partial objects to keep around */
|
||||
unsigned int cpu_partial;
|
||||
|
|
@ -110,7 +110,7 @@ struct kmem_cache {
|
|||
#endif
|
||||
#ifdef CONFIG_MEMCG
|
||||
struct memcg_cache_params memcg_params;
|
||||
/* for propagation, maximum size of a stored attr */
|
||||
/* For propagation, maximum size of a stored attr */
|
||||
unsigned int max_attr_size;
|
||||
#ifdef CONFIG_SYSFS
|
||||
struct kset *memcg_kset;
|
||||
|
|
@ -151,7 +151,7 @@ struct kmem_cache {
|
|||
#else
|
||||
#define slub_cpu_partial(s) (0)
|
||||
#define slub_set_cpu_partial(s, n)
|
||||
#endif // CONFIG_SLUB_CPU_PARTIAL
|
||||
#endif /* CONFIG_SLUB_CPU_PARTIAL */
|
||||
|
||||
#ifdef CONFIG_SYSFS
|
||||
#define SLAB_SUPPORTS_SYSFS
|
||||
|
|
|
|||
|
|
@ -307,7 +307,7 @@ struct vma_swap_readahead {
|
|||
};
|
||||
|
||||
/* linux/mm/workingset.c */
|
||||
void *workingset_eviction(struct address_space *mapping, struct page *page);
|
||||
void *workingset_eviction(struct page *page);
|
||||
void workingset_refault(struct page *page, void *shadow);
|
||||
void workingset_activation(struct page *page);
|
||||
|
||||
|
|
@ -625,7 +625,7 @@ static inline int mem_cgroup_swappiness(struct mem_cgroup *memcg)
|
|||
return vm_swappiness;
|
||||
|
||||
/* root ? */
|
||||
if (mem_cgroup_disabled() || !memcg->css.parent)
|
||||
if (mem_cgroup_disabled() || mem_cgroup_is_root(memcg))
|
||||
return vm_swappiness;
|
||||
|
||||
return memcg->swappiness;
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue