mm: convert totalram_pages and totalhigh_pages variables to atomic

totalram_pages and totalhigh_pages are made static inline function.

Main motivation was that managed_page_count_lock handling was complicating
things.  It was discussed in length here,
https://lore.kernel.org/patchwork/patch/995739/#1181785 So it seemes
better to remove the lock and convert variables to atomic, with preventing
poteintial store-to-read tearing as a bonus.

[akpm@linux-foundation.org: coding style fixes]
Link: http://lkml.kernel.org/r/1542090790-21750-4-git-send-email-arunks@codeaurora.org
Signed-off-by: Arun KS <arunks@codeaurora.org>
Suggested-by: Michal Hocko <mhocko@suse.com>
Suggested-by: Vlastimil Babka <vbabka@suse.cz>
Reviewed-by: Konstantin Khlebnikov <khlebnikov@yandex-team.ru>
Reviewed-by: Pavel Tatashin <pasha.tatashin@soleen.com>
Acked-by: Michal Hocko <mhocko@suse.com>
Acked-by: Vlastimil Babka <vbabka@suse.cz>
Cc: David Hildenbrand <david@redhat.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
This commit is contained in:
Arun KS 2018-12-28 00:34:29 -08:00 committed by Linus Torvalds
parent 9705bea5f8
commit ca79b0c211
53 changed files with 131 additions and 81 deletions

View File

@ -71,7 +71,7 @@ void free_initrd_mem(unsigned long start, unsigned long end)
ClearPageReserved(virt_to_page(start)); ClearPageReserved(virt_to_page(start));
init_page_count(virt_to_page(start)); init_page_count(virt_to_page(start));
free_page(start); free_page(start);
totalram_pages++; totalram_pages_inc();
} }
} }
#endif #endif
@ -88,7 +88,7 @@ void free_initmem(void)
ClearPageReserved(virt_to_page(addr)); ClearPageReserved(virt_to_page(addr));
init_page_count(virt_to_page(addr)); init_page_count(virt_to_page(addr));
free_page(addr); free_page(addr);
totalram_pages++; totalram_pages_inc();
addr += PAGE_SIZE; addr += PAGE_SIZE;
} }

View File

@ -208,7 +208,7 @@ static long cmm_alloc_pages(long nr)
pa->page[pa->index++] = addr; pa->page[pa->index++] = addr;
loaned_pages++; loaned_pages++;
totalram_pages--; totalram_pages_dec();
spin_unlock(&cmm_lock); spin_unlock(&cmm_lock);
nr--; nr--;
} }
@ -247,7 +247,7 @@ static long cmm_free_pages(long nr)
free_page(addr); free_page(addr);
loaned_pages--; loaned_pages--;
nr--; nr--;
totalram_pages++; totalram_pages_inc();
} }
spin_unlock(&cmm_lock); spin_unlock(&cmm_lock);
cmm_dbg("End request with %ld pages unfulfilled\n", nr); cmm_dbg("End request with %ld pages unfulfilled\n", nr);
@ -291,7 +291,7 @@ static void cmm_get_mpp(void)
int rc; int rc;
struct hvcall_mpp_data mpp_data; struct hvcall_mpp_data mpp_data;
signed long active_pages_target, page_loan_request, target; signed long active_pages_target, page_loan_request, target;
signed long total_pages = totalram_pages + loaned_pages; signed long total_pages = totalram_pages() + loaned_pages;
signed long min_mem_pages = (min_mem_mb * 1024 * 1024) / PAGE_SIZE; signed long min_mem_pages = (min_mem_mb * 1024 * 1024) / PAGE_SIZE;
rc = h_get_mpp(&mpp_data); rc = h_get_mpp(&mpp_data);
@ -322,7 +322,7 @@ static void cmm_get_mpp(void)
cmm_dbg("delta = %ld, loaned = %lu, target = %lu, oom = %lu, totalram = %lu\n", cmm_dbg("delta = %ld, loaned = %lu, target = %lu, oom = %lu, totalram = %lu\n",
page_loan_request, loaned_pages, loaned_pages_target, page_loan_request, loaned_pages, loaned_pages_target,
oom_freed_pages, totalram_pages); oom_freed_pages, totalram_pages());
} }
static struct notifier_block cmm_oom_nb = { static struct notifier_block cmm_oom_nb = {
@ -581,7 +581,7 @@ static int cmm_mem_going_offline(void *arg)
free_page(pa_curr->page[idx]); free_page(pa_curr->page[idx]);
freed++; freed++;
loaned_pages--; loaned_pages--;
totalram_pages++; totalram_pages_inc();
pa_curr->page[idx] = pa_last->page[--pa_last->index]; pa_curr->page[idx] = pa_last->page[--pa_last->index];
if (pa_last->index == 0) { if (pa_last->index == 0) {
if (pa_curr == pa_last) if (pa_curr == pa_last)

View File

@ -59,7 +59,7 @@ static void __init setup_zero_pages(void)
order = 7; order = 7;
/* Limit number of empty zero pages for small memory sizes */ /* Limit number of empty zero pages for small memory sizes */
while (order > 2 && (totalram_pages >> 10) < (1UL << order)) while (order > 2 && (totalram_pages() >> 10) < (1UL << order))
order--; order--;
empty_zero_page = __get_free_pages(GFP_KERNEL | __GFP_ZERO, order); empty_zero_page = __get_free_pages(GFP_KERNEL | __GFP_ZERO, order);

View File

@ -51,7 +51,7 @@ void __init mem_init(void)
/* this will put all low memory onto the freelists */ /* this will put all low memory onto the freelists */
memblock_free_all(); memblock_free_all();
max_low_pfn = totalram_pages; max_low_pfn = totalram_pages();
max_pfn = max_low_pfn; max_pfn = max_low_pfn;
mem_init_print_info(NULL); mem_init_print_info(NULL);
kmalloc_ok = 1; kmalloc_ok = 1;

View File

@ -434,7 +434,7 @@ static ssize_t microcode_write(struct file *file, const char __user *buf,
size_t len, loff_t *ppos) size_t len, loff_t *ppos)
{ {
ssize_t ret = -EINVAL; ssize_t ret = -EINVAL;
unsigned long nr_pages = totalram_pages; unsigned long nr_pages = totalram_pages();
if ((len >> PAGE_SHIFT) > nr_pages) { if ((len >> PAGE_SHIFT) > nr_pages) {
pr_err("too much data (max %ld pages)\n", nr_pages); pr_err("too much data (max %ld pages)\n", nr_pages);

View File

@ -115,9 +115,9 @@ static int agp_find_max(void)
long memory, index, result; long memory, index, result;
#if PAGE_SHIFT < 20 #if PAGE_SHIFT < 20
memory = totalram_pages >> (20 - PAGE_SHIFT); memory = totalram_pages() >> (20 - PAGE_SHIFT);
#else #else
memory = totalram_pages << (PAGE_SHIFT - 20); memory = totalram_pages() << (PAGE_SHIFT - 20);
#endif #endif
index = 1; index = 1;

View File

@ -2559,7 +2559,7 @@ static int i915_gem_object_get_pages_gtt(struct drm_i915_gem_object *obj)
* If there's no chance of allocating enough pages for the whole * If there's no chance of allocating enough pages for the whole
* object, bail early. * object, bail early.
*/ */
if (page_count > totalram_pages) if (page_count > totalram_pages())
return -ENOMEM; return -ENOMEM;
st = kmalloc(sizeof(*st), GFP_KERNEL); st = kmalloc(sizeof(*st), GFP_KERNEL);

View File

@ -170,7 +170,7 @@ static int igt_ppgtt_alloc(void *arg)
* This should ensure that we do not run into the oomkiller during * This should ensure that we do not run into the oomkiller during
* the test and take down the machine wilfully. * the test and take down the machine wilfully.
*/ */
limit = totalram_pages << PAGE_SHIFT; limit = totalram_pages() << PAGE_SHIFT;
limit = min(ppgtt->vm.total, limit); limit = min(ppgtt->vm.total, limit);
/* Check we can allocate the entire range */ /* Check we can allocate the entire range */
@ -1244,7 +1244,7 @@ static int exercise_mock(struct drm_i915_private *i915,
u64 hole_start, u64 hole_end, u64 hole_start, u64 hole_end,
unsigned long end_time)) unsigned long end_time))
{ {
const u64 limit = totalram_pages << PAGE_SHIFT; const u64 limit = totalram_pages() << PAGE_SHIFT;
struct i915_gem_context *ctx; struct i915_gem_context *ctx;
struct i915_hw_ppgtt *ppgtt; struct i915_hw_ppgtt *ppgtt;
IGT_TIMEOUT(end_time); IGT_TIMEOUT(end_time);

View File

@ -1090,7 +1090,7 @@ static void process_info(struct hv_dynmem_device *dm, struct dm_info_msg *msg)
static unsigned long compute_balloon_floor(void) static unsigned long compute_balloon_floor(void)
{ {
unsigned long min_pages; unsigned long min_pages;
unsigned long nr_pages = totalram_pages; unsigned long nr_pages = totalram_pages();
#define MB2PAGES(mb) ((mb) << (20 - PAGE_SHIFT)) #define MB2PAGES(mb) ((mb) << (20 - PAGE_SHIFT))
/* Simple continuous piecewiese linear function: /* Simple continuous piecewiese linear function:
* max MiB -> min MiB gradient * max MiB -> min MiB gradient

View File

@ -1887,7 +1887,7 @@ static int __init dm_bufio_init(void)
dm_bufio_allocated_vmalloc = 0; dm_bufio_allocated_vmalloc = 0;
dm_bufio_current_allocated = 0; dm_bufio_current_allocated = 0;
mem = (__u64)mult_frac(totalram_pages - totalhigh_pages, mem = (__u64)mult_frac(totalram_pages() - totalhigh_pages(),
DM_BUFIO_MEMORY_PERCENT, 100) << PAGE_SHIFT; DM_BUFIO_MEMORY_PERCENT, 100) << PAGE_SHIFT;
if (mem > ULONG_MAX) if (mem > ULONG_MAX)

View File

@ -2158,7 +2158,7 @@ static int crypt_wipe_key(struct crypt_config *cc)
static void crypt_calculate_pages_per_client(void) static void crypt_calculate_pages_per_client(void)
{ {
unsigned long pages = (totalram_pages - totalhigh_pages) * DM_CRYPT_MEMORY_PERCENT / 100; unsigned long pages = (totalram_pages() - totalhigh_pages()) * DM_CRYPT_MEMORY_PERCENT / 100;
if (!dm_crypt_clients_n) if (!dm_crypt_clients_n)
return; return;

View File

@ -2843,7 +2843,7 @@ static int create_journal(struct dm_integrity_c *ic, char **error)
journal_pages = roundup((__u64)ic->journal_sections * ic->journal_section_sectors, journal_pages = roundup((__u64)ic->journal_sections * ic->journal_section_sectors,
PAGE_SIZE >> SECTOR_SHIFT) >> (PAGE_SHIFT - SECTOR_SHIFT); PAGE_SIZE >> SECTOR_SHIFT) >> (PAGE_SHIFT - SECTOR_SHIFT);
journal_desc_size = journal_pages * sizeof(struct page_list); journal_desc_size = journal_pages * sizeof(struct page_list);
if (journal_pages >= totalram_pages - totalhigh_pages || journal_desc_size > ULONG_MAX) { if (journal_pages >= totalram_pages() - totalhigh_pages() || journal_desc_size > ULONG_MAX) {
*error = "Journal doesn't fit into memory"; *error = "Journal doesn't fit into memory";
r = -ENOMEM; r = -ENOMEM;
goto bad; goto bad;

View File

@ -85,7 +85,7 @@ static bool __check_shared_memory(size_t alloc_size)
a = shared_memory_amount + alloc_size; a = shared_memory_amount + alloc_size;
if (a < shared_memory_amount) if (a < shared_memory_amount)
return false; return false;
if (a >> PAGE_SHIFT > totalram_pages / DM_STATS_MEMORY_FACTOR) if (a >> PAGE_SHIFT > totalram_pages() / DM_STATS_MEMORY_FACTOR)
return false; return false;
#ifdef CONFIG_MMU #ifdef CONFIG_MMU
if (a > (VMALLOC_END - VMALLOC_START) / DM_STATS_VMALLOC_FACTOR) if (a > (VMALLOC_END - VMALLOC_START) / DM_STATS_VMALLOC_FACTOR)

View File

@ -855,7 +855,7 @@ static int mtk_vpu_probe(struct platform_device *pdev)
/* Set PTCM to 96K and DTCM to 32K */ /* Set PTCM to 96K and DTCM to 32K */
vpu_cfg_writel(vpu, 0x2, VPU_TCM_CFG); vpu_cfg_writel(vpu, 0x2, VPU_TCM_CFG);
vpu->enable_4GB = !!(totalram_pages > (SZ_2G >> PAGE_SHIFT)); vpu->enable_4GB = !!(totalram_pages() > (SZ_2G >> PAGE_SHIFT));
dev_info(dev, "4GB mode %u\n", vpu->enable_4GB); dev_info(dev, "4GB mode %u\n", vpu->enable_4GB);
if (vpu->enable_4GB) { if (vpu->enable_4GB) {

View File

@ -570,7 +570,7 @@ static int vmballoon_send_get_target(struct vmballoon *b)
unsigned long status; unsigned long status;
unsigned long limit; unsigned long limit;
limit = totalram_pages; limit = totalram_pages();
/* Ensure limit fits in 32-bits */ /* Ensure limit fits in 32-bits */
if (limit != (u32)limit) if (limit != (u32)limit)

View File

@ -1251,7 +1251,7 @@ ccio_ioc_init(struct ioc *ioc)
** Hot-Plug/Removal of PCI cards. (aka PCI OLARD). ** Hot-Plug/Removal of PCI cards. (aka PCI OLARD).
*/ */
iova_space_size = (u32) (totalram_pages / count_parisc_driver(&ccio_driver)); iova_space_size = (u32) (totalram_pages() / count_parisc_driver(&ccio_driver));
/* limit IOVA space size to 1MB-1GB */ /* limit IOVA space size to 1MB-1GB */
@ -1290,7 +1290,7 @@ ccio_ioc_init(struct ioc *ioc)
DBG_INIT("%s() hpa 0x%p mem %luMB IOV %dMB (%d bits)\n", DBG_INIT("%s() hpa 0x%p mem %luMB IOV %dMB (%d bits)\n",
__func__, ioc->ioc_regs, __func__, ioc->ioc_regs,
(unsigned long) totalram_pages >> (20 - PAGE_SHIFT), (unsigned long) totalram_pages() >> (20 - PAGE_SHIFT),
iova_space_size>>20, iova_space_size>>20,
iov_order + PAGE_SHIFT); iov_order + PAGE_SHIFT);

View File

@ -1414,7 +1414,7 @@ sba_ioc_init(struct parisc_device *sba, struct ioc *ioc, int ioc_num)
** for DMA hints - ergo only 30 bits max. ** for DMA hints - ergo only 30 bits max.
*/ */
iova_space_size = (u32) (totalram_pages/global_ioc_cnt); iova_space_size = (u32) (totalram_pages()/global_ioc_cnt);
/* limit IOVA space size to 1MB-1GB */ /* limit IOVA space size to 1MB-1GB */
if (iova_space_size < (1 << (20 - PAGE_SHIFT))) { if (iova_space_size < (1 << (20 - PAGE_SHIFT))) {
@ -1439,7 +1439,7 @@ sba_ioc_init(struct parisc_device *sba, struct ioc *ioc, int ioc_num)
DBG_INIT("%s() hpa 0x%lx mem %ldMB IOV %dMB (%d bits)\n", DBG_INIT("%s() hpa 0x%lx mem %ldMB IOV %dMB (%d bits)\n",
__func__, __func__,
ioc->ioc_hpa, ioc->ioc_hpa,
(unsigned long) totalram_pages >> (20 - PAGE_SHIFT), (unsigned long) totalram_pages() >> (20 - PAGE_SHIFT),
iova_space_size>>20, iova_space_size>>20,
iov_order + PAGE_SHIFT); iov_order + PAGE_SHIFT);

View File

@ -110,7 +110,7 @@ static int ion_system_heap_allocate(struct ion_heap *heap,
unsigned long size_remaining = PAGE_ALIGN(size); unsigned long size_remaining = PAGE_ALIGN(size);
unsigned int max_order = orders[0]; unsigned int max_order = orders[0];
if (size / PAGE_SIZE > totalram_pages / 2) if (size / PAGE_SIZE > totalram_pages() / 2)
return -ENOMEM; return -ENOMEM;
INIT_LIST_HEAD(&pages); INIT_LIST_HEAD(&pages);

View File

@ -189,7 +189,7 @@ static void selfballoon_process(struct work_struct *work)
bool reset_timer = false; bool reset_timer = false;
if (xen_selfballooning_enabled) { if (xen_selfballooning_enabled) {
cur_pages = totalram_pages; cur_pages = totalram_pages();
tgt_pages = cur_pages; /* default is no change */ tgt_pages = cur_pages; /* default is no change */
goal_pages = vm_memory_committed() + goal_pages = vm_memory_committed() +
totalreserve_pages + totalreserve_pages +
@ -227,7 +227,7 @@ static void selfballoon_process(struct work_struct *work)
if (tgt_pages < floor_pages) if (tgt_pages < floor_pages)
tgt_pages = floor_pages; tgt_pages = floor_pages;
balloon_set_new_target(tgt_pages + balloon_set_new_target(tgt_pages +
balloon_stats.current_pages - totalram_pages); balloon_stats.current_pages - totalram_pages());
reset_timer = true; reset_timer = true;
} }
#ifdef CONFIG_FRONTSWAP #ifdef CONFIG_FRONTSWAP
@ -569,7 +569,7 @@ int xen_selfballoon_init(bool use_selfballooning, bool use_frontswap_selfshrink)
* much more reliably and response faster in some cases. * much more reliably and response faster in some cases.
*/ */
if (!selfballoon_reserved_mb) { if (!selfballoon_reserved_mb) {
reserve_pages = totalram_pages / 10; reserve_pages = totalram_pages() / 10;
selfballoon_reserved_mb = PAGES2MB(reserve_pages); selfballoon_reserved_mb = PAGES2MB(reserve_pages);
} }
schedule_delayed_work(&selfballoon_worker, selfballoon_interval * HZ); schedule_delayed_work(&selfballoon_worker, selfballoon_interval * HZ);

View File

@ -810,7 +810,7 @@ static inline int default_congestion_kb(void)
* This allows larger machines to have larger/more transfers. * This allows larger machines to have larger/more transfers.
* Limit the default to 256M * Limit the default to 256M
*/ */
congestion_kb = (16*int_sqrt(totalram_pages)) << (PAGE_SHIFT-10); congestion_kb = (16*int_sqrt(totalram_pages())) << (PAGE_SHIFT-10);
if (congestion_kb > 256*1024) if (congestion_kb > 256*1024)
congestion_kb = 256*1024; congestion_kb = 256*1024;

View File

@ -380,7 +380,7 @@ void __init files_init(void)
void __init files_maxfiles_init(void) void __init files_maxfiles_init(void)
{ {
unsigned long n; unsigned long n;
unsigned long nr_pages = totalram_pages; unsigned long nr_pages = totalram_pages();
unsigned long memreserve = (nr_pages - nr_free_pages()) * 3/2; unsigned long memreserve = (nr_pages - nr_free_pages()) * 3/2;
memreserve = min(memreserve, nr_pages - 1); memreserve = min(memreserve, nr_pages - 1);

View File

@ -824,7 +824,7 @@ static const struct super_operations fuse_super_operations = {
static void sanitize_global_limit(unsigned *limit) static void sanitize_global_limit(unsigned *limit)
{ {
if (*limit == 0) if (*limit == 0)
*limit = ((totalram_pages << PAGE_SHIFT) >> 13) / *limit = ((totalram_pages() << PAGE_SHIFT) >> 13) /
sizeof(struct fuse_req); sizeof(struct fuse_req);
if (*limit >= 1 << 16) if (*limit >= 1 << 16)

View File

@ -2121,7 +2121,7 @@ int __init nfs_init_writepagecache(void)
* This allows larger machines to have larger/more transfers. * This allows larger machines to have larger/more transfers.
* Limit the default to 256M * Limit the default to 256M
*/ */
nfs_congestion_kb = (16*int_sqrt(totalram_pages)) << (PAGE_SHIFT-10); nfs_congestion_kb = (16*int_sqrt(totalram_pages())) << (PAGE_SHIFT-10);
if (nfs_congestion_kb > 256*1024) if (nfs_congestion_kb > 256*1024)
nfs_congestion_kb = 256*1024; nfs_congestion_kb = 256*1024;

View File

@ -99,7 +99,7 @@ static unsigned int
nfsd_cache_size_limit(void) nfsd_cache_size_limit(void)
{ {
unsigned int limit; unsigned int limit;
unsigned long low_pages = totalram_pages - totalhigh_pages; unsigned long low_pages = totalram_pages() - totalhigh_pages();
limit = (16 * int_sqrt(low_pages)) << (PAGE_SHIFT-10); limit = (16 * int_sqrt(low_pages)) << (PAGE_SHIFT-10);
return min_t(unsigned int, limit, 256*1024); return min_t(unsigned int, limit, 256*1024);

View File

@ -47,7 +47,7 @@ static inline void *__ntfs_malloc(unsigned long size, gfp_t gfp_mask)
return kmalloc(PAGE_SIZE, gfp_mask & ~__GFP_HIGHMEM); return kmalloc(PAGE_SIZE, gfp_mask & ~__GFP_HIGHMEM);
/* return (void *)__get_free_page(gfp_mask); */ /* return (void *)__get_free_page(gfp_mask); */
} }
if (likely((size >> PAGE_SHIFT) < totalram_pages)) if (likely((size >> PAGE_SHIFT) < totalram_pages()))
return __vmalloc(size, gfp_mask, PAGE_KERNEL); return __vmalloc(size, gfp_mask, PAGE_KERNEL);
return NULL; return NULL;
} }

View File

@ -530,7 +530,7 @@ static const struct file_operations proc_lstats_operations = {
static int proc_oom_score(struct seq_file *m, struct pid_namespace *ns, static int proc_oom_score(struct seq_file *m, struct pid_namespace *ns,
struct pid *pid, struct task_struct *task) struct pid *pid, struct task_struct *task)
{ {
unsigned long totalpages = totalram_pages + total_swap_pages; unsigned long totalpages = totalram_pages() + total_swap_pages;
unsigned long points = 0; unsigned long points = 0;
points = oom_badness(task, NULL, NULL, totalpages) * points = oom_badness(task, NULL, NULL, totalpages) *

View File

@ -36,7 +36,31 @@ static inline void invalidate_kernel_vmap_range(void *vaddr, int size)
/* declarations for linux/mm/highmem.c */ /* declarations for linux/mm/highmem.c */
unsigned int nr_free_highpages(void); unsigned int nr_free_highpages(void);
extern unsigned long totalhigh_pages; extern atomic_long_t _totalhigh_pages;
static inline unsigned long totalhigh_pages(void)
{
return (unsigned long)atomic_long_read(&_totalhigh_pages);
}
static inline void totalhigh_pages_inc(void)
{
atomic_long_inc(&_totalhigh_pages);
}
static inline void totalhigh_pages_dec(void)
{
atomic_long_dec(&_totalhigh_pages);
}
static inline void totalhigh_pages_add(long count)
{
atomic_long_add(count, &_totalhigh_pages);
}
static inline void totalhigh_pages_set(long val)
{
atomic_long_set(&_totalhigh_pages, val);
}
void kmap_flush_unused(void); void kmap_flush_unused(void);
@ -51,7 +75,7 @@ static inline struct page *kmap_to_page(void *addr)
return virt_to_page(addr); return virt_to_page(addr);
} }
#define totalhigh_pages 0UL static inline unsigned long totalhigh_pages(void) { return 0UL; }
#ifndef ARCH_HAS_KMAP #ifndef ARCH_HAS_KMAP
static inline void *kmap(struct page *page) static inline void *kmap(struct page *page)

View File

@ -48,7 +48,32 @@ static inline void set_max_mapnr(unsigned long limit)
static inline void set_max_mapnr(unsigned long limit) { } static inline void set_max_mapnr(unsigned long limit) { }
#endif #endif
extern unsigned long totalram_pages; extern atomic_long_t _totalram_pages;
static inline unsigned long totalram_pages(void)
{
return (unsigned long)atomic_long_read(&_totalram_pages);
}
static inline void totalram_pages_inc(void)
{
atomic_long_inc(&_totalram_pages);
}
static inline void totalram_pages_dec(void)
{
atomic_long_dec(&_totalram_pages);
}
static inline void totalram_pages_add(long count)
{
atomic_long_add(count, &_totalram_pages);
}
static inline void totalram_pages_set(long val)
{
atomic_long_set(&_totalram_pages, val);
}
extern void * high_memory; extern void * high_memory;
extern int page_cluster; extern int page_cluster;

View File

@ -310,7 +310,6 @@ void workingset_update_node(struct xa_node *node);
} while (0) } while (0)
/* linux/mm/page_alloc.c */ /* linux/mm/page_alloc.c */
extern unsigned long totalram_pages;
extern unsigned long totalreserve_pages; extern unsigned long totalreserve_pages;
extern unsigned long nr_free_buffer_pages(void); extern unsigned long nr_free_buffer_pages(void);
extern unsigned long nr_free_pagecache_pages(void); extern unsigned long nr_free_pagecache_pages(void);

View File

@ -744,7 +744,7 @@ void __init __weak arch_task_cache_init(void) { }
static void set_max_threads(unsigned int max_threads_suggested) static void set_max_threads(unsigned int max_threads_suggested)
{ {
u64 threads; u64 threads;
unsigned long nr_pages = totalram_pages; unsigned long nr_pages = totalram_pages();
/* /*
* The number of threads shall be limited such that the thread * The number of threads shall be limited such that the thread

View File

@ -152,7 +152,7 @@ int sanity_check_segment_list(struct kimage *image)
int i; int i;
unsigned long nr_segments = image->nr_segments; unsigned long nr_segments = image->nr_segments;
unsigned long total_pages = 0; unsigned long total_pages = 0;
unsigned long nr_pages = totalram_pages; unsigned long nr_pages = totalram_pages();
/* /*
* Verify we have good destination addresses. The caller is * Verify we have good destination addresses. The caller is

View File

@ -105,7 +105,7 @@ unsigned long image_size;
void __init hibernate_image_size_init(void) void __init hibernate_image_size_init(void)
{ {
image_size = ((totalram_pages * 2) / 5) * PAGE_SIZE; image_size = ((totalram_pages() * 2) / 5) * PAGE_SIZE;
} }
/* /*

View File

@ -105,9 +105,8 @@ static inline wait_queue_head_t *get_pkmap_wait_queue_head(unsigned int color)
} }
#endif #endif
unsigned long totalhigh_pages __read_mostly; atomic_long_t _totalhigh_pages __read_mostly;
EXPORT_SYMBOL(totalhigh_pages); EXPORT_SYMBOL(_totalhigh_pages);
EXPORT_PER_CPU_SYMBOL(__kmap_atomic_idx); EXPORT_PER_CPU_SYMBOL(__kmap_atomic_idx);

View File

@ -420,7 +420,7 @@ static int __init hugepage_init(void)
* where the extra memory used could hurt more than TLB overhead * where the extra memory used could hurt more than TLB overhead
* is likely to save. The admin can still enable it through /sys. * is likely to save. The admin can still enable it through /sys.
*/ */
if (totalram_pages < (512 << (20 - PAGE_SHIFT))) { if (totalram_pages() < (512 << (20 - PAGE_SHIFT))) {
transparent_hugepage_flags = 0; transparent_hugepage_flags = 0;
return 0; return 0;
} }

View File

@ -237,7 +237,7 @@ void quarantine_reduce(void)
* Update quarantine size in case of hotplug. Allocate a fraction of * Update quarantine size in case of hotplug. Allocate a fraction of
* the installed memory to quarantine minus per-cpu queue limits. * the installed memory to quarantine minus per-cpu queue limits.
*/ */
total_size = (READ_ONCE(totalram_pages) << PAGE_SHIFT) / total_size = (totalram_pages() << PAGE_SHIFT) /
QUARANTINE_FRACTION; QUARANTINE_FRACTION;
percpu_quarantines = QUARANTINE_PERCPU_SIZE * num_online_cpus(); percpu_quarantines = QUARANTINE_PERCPU_SIZE * num_online_cpus();
new_quarantine_size = (total_size < percpu_quarantines) ? new_quarantine_size = (total_size < percpu_quarantines) ?

View File

@ -1576,7 +1576,7 @@ void __init __memblock_free_late(phys_addr_t base, phys_addr_t size)
for (; cursor < end; cursor++) { for (; cursor < end; cursor++) {
memblock_free_pages(pfn_to_page(cursor), cursor, 0); memblock_free_pages(pfn_to_page(cursor), cursor, 0);
totalram_pages++; totalram_pages_inc();
} }
} }
@ -1978,7 +1978,7 @@ unsigned long __init memblock_free_all(void)
reset_all_zones_managed_pages(); reset_all_zones_managed_pages();
pages = free_low_memory_core_early(); pages = free_low_memory_core_early();
totalram_pages += pages; totalram_pages_add(pages);
return pages; return pages;
} }

View File

@ -146,7 +146,7 @@ static void __meminit mm_compute_batch(void)
s32 batch = max_t(s32, nr*2, 32); s32 batch = max_t(s32, nr*2, 32);
/* batch size set to 0.4% of (total memory/#cpus), or max int32 */ /* batch size set to 0.4% of (total memory/#cpus), or max int32 */
memsized_batch = min_t(u64, (totalram_pages/nr)/256, 0x7fffffff); memsized_batch = min_t(u64, (totalram_pages()/nr)/256, 0x7fffffff);
vm_committed_as_batch = max_t(s32, memsized_batch, batch); vm_committed_as_batch = max_t(s32, memsized_batch, batch);
} }

View File

@ -269,7 +269,7 @@ static enum oom_constraint constrained_alloc(struct oom_control *oc)
} }
/* Default to all available memory */ /* Default to all available memory */
oc->totalpages = totalram_pages + total_swap_pages; oc->totalpages = totalram_pages() + total_swap_pages;
if (!IS_ENABLED(CONFIG_NUMA)) if (!IS_ENABLED(CONFIG_NUMA))
return CONSTRAINT_NONE; return CONSTRAINT_NONE;

View File

@ -16,6 +16,7 @@
#include <linux/stddef.h> #include <linux/stddef.h>
#include <linux/mm.h> #include <linux/mm.h>
#include <linux/highmem.h>
#include <linux/swap.h> #include <linux/swap.h>
#include <linux/interrupt.h> #include <linux/interrupt.h>
#include <linux/pagemap.h> #include <linux/pagemap.h>
@ -124,7 +125,8 @@ EXPORT_SYMBOL(node_states);
/* Protect totalram_pages and zone->managed_pages */ /* Protect totalram_pages and zone->managed_pages */
static DEFINE_SPINLOCK(managed_page_count_lock); static DEFINE_SPINLOCK(managed_page_count_lock);
unsigned long totalram_pages __read_mostly; atomic_long_t _totalram_pages __read_mostly;
EXPORT_SYMBOL(_totalram_pages);
unsigned long totalreserve_pages __read_mostly; unsigned long totalreserve_pages __read_mostly;
unsigned long totalcma_pages __read_mostly; unsigned long totalcma_pages __read_mostly;
@ -4747,11 +4749,11 @@ EXPORT_SYMBOL_GPL(si_mem_available);
void si_meminfo(struct sysinfo *val) void si_meminfo(struct sysinfo *val)
{ {
val->totalram = totalram_pages; val->totalram = totalram_pages();
val->sharedram = global_node_page_state(NR_SHMEM); val->sharedram = global_node_page_state(NR_SHMEM);
val->freeram = global_zone_page_state(NR_FREE_PAGES); val->freeram = global_zone_page_state(NR_FREE_PAGES);
val->bufferram = nr_blockdev_pages(); val->bufferram = nr_blockdev_pages();
val->totalhigh = totalhigh_pages; val->totalhigh = totalhigh_pages();
val->freehigh = nr_free_highpages(); val->freehigh = nr_free_highpages();
val->mem_unit = PAGE_SIZE; val->mem_unit = PAGE_SIZE;
} }
@ -7077,10 +7079,10 @@ void adjust_managed_page_count(struct page *page, long count)
{ {
spin_lock(&managed_page_count_lock); spin_lock(&managed_page_count_lock);
atomic_long_add(count, &page_zone(page)->managed_pages); atomic_long_add(count, &page_zone(page)->managed_pages);
totalram_pages += count; totalram_pages_add(count);
#ifdef CONFIG_HIGHMEM #ifdef CONFIG_HIGHMEM
if (PageHighMem(page)) if (PageHighMem(page))
totalhigh_pages += count; totalhigh_pages_add(count);
#endif #endif
spin_unlock(&managed_page_count_lock); spin_unlock(&managed_page_count_lock);
} }
@ -7123,9 +7125,9 @@ EXPORT_SYMBOL(free_reserved_area);
void free_highmem_page(struct page *page) void free_highmem_page(struct page *page)
{ {
__free_reserved_page(page); __free_reserved_page(page);
totalram_pages++; totalram_pages_inc();
atomic_long_inc(&page_zone(page)->managed_pages); atomic_long_inc(&page_zone(page)->managed_pages);
totalhigh_pages++; totalhigh_pages_inc();
} }
#endif #endif
@ -7174,10 +7176,10 @@ void __init mem_init_print_info(const char *str)
physpages << (PAGE_SHIFT - 10), physpages << (PAGE_SHIFT - 10),
codesize >> 10, datasize >> 10, rosize >> 10, codesize >> 10, datasize >> 10, rosize >> 10,
(init_data_size + init_code_size) >> 10, bss_size >> 10, (init_data_size + init_code_size) >> 10, bss_size >> 10,
(physpages - totalram_pages - totalcma_pages) << (PAGE_SHIFT - 10), (physpages - totalram_pages() - totalcma_pages) << (PAGE_SHIFT - 10),
totalcma_pages << (PAGE_SHIFT - 10), totalcma_pages << (PAGE_SHIFT - 10),
#ifdef CONFIG_HIGHMEM #ifdef CONFIG_HIGHMEM
totalhigh_pages << (PAGE_SHIFT - 10), totalhigh_pages() << (PAGE_SHIFT - 10),
#endif #endif
str ? ", " : "", str ? str : ""); str ? ", " : "", str ? str : "");
} }

View File

@ -109,13 +109,14 @@ struct shmem_falloc {
#ifdef CONFIG_TMPFS #ifdef CONFIG_TMPFS
static unsigned long shmem_default_max_blocks(void) static unsigned long shmem_default_max_blocks(void)
{ {
return totalram_pages / 2; return totalram_pages() / 2;
} }
static unsigned long shmem_default_max_inodes(void) static unsigned long shmem_default_max_inodes(void)
{ {
unsigned long nr_pages = totalram_pages; unsigned long nr_pages = totalram_pages();
return min(nr_pages - totalhigh_pages, nr_pages / 2);
return min(nr_pages - totalhigh_pages(), nr_pages / 2);
} }
#endif #endif
@ -3302,7 +3303,7 @@ static int shmem_parse_options(char *options, struct shmem_sb_info *sbinfo,
size = memparse(value,&rest); size = memparse(value,&rest);
if (*rest == '%') { if (*rest == '%') {
size <<= PAGE_SHIFT; size <<= PAGE_SHIFT;
size *= totalram_pages; size *= totalram_pages();
do_div(size, 100); do_div(size, 100);
rest++; rest++;
} }

View File

@ -1235,7 +1235,7 @@ void __init kmem_cache_init(void)
* page orders on machines with more than 32MB of memory if * page orders on machines with more than 32MB of memory if
* not overridden on the command line. * not overridden on the command line.
*/ */
if (!slab_max_order_set && totalram_pages > (32 << 20) >> PAGE_SHIFT) if (!slab_max_order_set && totalram_pages() > (32 << 20) >> PAGE_SHIFT)
slab_max_order = SLAB_MAX_ORDER_HI; slab_max_order = SLAB_MAX_ORDER_HI;
/* Bootstrap is tricky, because several objects are allocated /* Bootstrap is tricky, because several objects are allocated

View File

@ -1022,7 +1022,7 @@ EXPORT_SYMBOL(pagevec_lookup_range_nr_tag);
*/ */
void __init swap_setup(void) void __init swap_setup(void)
{ {
unsigned long megs = totalram_pages >> (20 - PAGE_SHIFT); unsigned long megs = totalram_pages() >> (20 - PAGE_SHIFT);
/* Use a smaller cluster for small-memory machines */ /* Use a smaller cluster for small-memory machines */
if (megs < 16) if (megs < 16)

View File

@ -593,7 +593,7 @@ unsigned long vm_commit_limit(void)
if (sysctl_overcommit_kbytes) if (sysctl_overcommit_kbytes)
allowed = sysctl_overcommit_kbytes >> (PAGE_SHIFT - 10); allowed = sysctl_overcommit_kbytes >> (PAGE_SHIFT - 10);
else else
allowed = ((totalram_pages - hugetlb_total_pages()) allowed = ((totalram_pages() - hugetlb_total_pages())
* sysctl_overcommit_ratio / 100); * sysctl_overcommit_ratio / 100);
allowed += total_swap_pages; allowed += total_swap_pages;

View File

@ -1634,7 +1634,7 @@ void *vmap(struct page **pages, unsigned int count,
might_sleep(); might_sleep();
if (count > totalram_pages) if (count > totalram_pages())
return NULL; return NULL;
size = (unsigned long)count << PAGE_SHIFT; size = (unsigned long)count << PAGE_SHIFT;
@ -1739,7 +1739,7 @@ void *__vmalloc_node_range(unsigned long size, unsigned long align,
unsigned long real_size = size; unsigned long real_size = size;
size = PAGE_ALIGN(size); size = PAGE_ALIGN(size);
if (!size || (size >> PAGE_SHIFT) > totalram_pages) if (!size || (size >> PAGE_SHIFT) > totalram_pages())
goto fail; goto fail;
area = __get_vm_area_node(size, align, VM_ALLOC | VM_UNINITIALIZED | area = __get_vm_area_node(size, align, VM_ALLOC | VM_UNINITIALIZED |

View File

@ -549,7 +549,7 @@ static int __init workingset_init(void)
* double the initial memory by using totalram_pages as-is. * double the initial memory by using totalram_pages as-is.
*/ */
timestamp_bits = BITS_PER_LONG - EVICTION_SHIFT; timestamp_bits = BITS_PER_LONG - EVICTION_SHIFT;
max_order = fls_long(totalram_pages - 1); max_order = fls_long(totalram_pages() - 1);
if (max_order > timestamp_bits) if (max_order > timestamp_bits)
bucket_order = max_order - timestamp_bits; bucket_order = max_order - timestamp_bits;
pr_info("workingset: timestamp_bits=%d max_order=%d bucket_order=%u\n", pr_info("workingset: timestamp_bits=%d max_order=%d bucket_order=%u\n",

View File

@ -219,8 +219,8 @@ static const struct zpool_ops zswap_zpool_ops = {
static bool zswap_is_full(void) static bool zswap_is_full(void)
{ {
return totalram_pages * zswap_max_pool_percent / 100 < return totalram_pages() * zswap_max_pool_percent / 100 <
DIV_ROUND_UP(zswap_pool_total_size, PAGE_SIZE); DIV_ROUND_UP(zswap_pool_total_size, PAGE_SIZE);
} }
static void zswap_update_total_size(void) static void zswap_update_total_size(void)

View File

@ -1131,7 +1131,7 @@ EXPORT_SYMBOL_GPL(dccp_debug);
static int __init dccp_init(void) static int __init dccp_init(void)
{ {
unsigned long goal; unsigned long goal;
unsigned long nr_pages = totalram_pages; unsigned long nr_pages = totalram_pages();
int ehash_order, bhash_order, i; int ehash_order, bhash_order, i;
int rc; int rc;

View File

@ -1866,7 +1866,7 @@ void __init dn_route_init(void)
dn_route_timer.expires = jiffies + decnet_dst_gc_interval * HZ; dn_route_timer.expires = jiffies + decnet_dst_gc_interval * HZ;
add_timer(&dn_route_timer); add_timer(&dn_route_timer);
goal = totalram_pages >> (26 - PAGE_SHIFT); goal = totalram_pages() >> (26 - PAGE_SHIFT);
for(order = 0; (1UL << order) < goal; order++) for(order = 0; (1UL << order) < goal; order++)
/* NOTHING */; /* NOTHING */;

View File

@ -1000,7 +1000,7 @@ static int __net_init tcp_net_metrics_init(struct net *net)
slots = tcpmhash_entries; slots = tcpmhash_entries;
if (!slots) { if (!slots) {
if (totalram_pages >= 128 * 1024) if (totalram_pages() >= 128 * 1024)
slots = 16 * 1024; slots = 16 * 1024;
else else
slots = 8 * 1024; slots = 8 * 1024;

View File

@ -2248,7 +2248,7 @@ static __always_inline unsigned int total_extension_size(void)
int nf_conntrack_init_start(void) int nf_conntrack_init_start(void)
{ {
unsigned long nr_pages = totalram_pages; unsigned long nr_pages = totalram_pages();
int max_factor = 8; int max_factor = 8;
int ret = -ENOMEM; int ret = -ENOMEM;
int i; int i;

View File

@ -274,7 +274,7 @@ static int htable_create(struct net *net, struct hashlimit_cfg3 *cfg,
struct xt_hashlimit_htable *hinfo; struct xt_hashlimit_htable *hinfo;
const struct seq_operations *ops; const struct seq_operations *ops;
unsigned int size, i; unsigned int size, i;
unsigned long nr_pages = totalram_pages; unsigned long nr_pages = totalram_pages();
int ret; int ret;
if (cfg->size) { if (cfg->size) {

View File

@ -1368,7 +1368,7 @@ static __init int sctp_init(void)
int status = -EINVAL; int status = -EINVAL;
unsigned long goal; unsigned long goal;
unsigned long limit; unsigned long limit;
unsigned long nr_pages = totalram_pages; unsigned long nr_pages = totalram_pages();
int max_share; int max_share;
int order; int order;
int num_entries; int num_entries;

View File

@ -106,7 +106,7 @@ void ima_add_kexec_buffer(struct kimage *image)
kexec_segment_size = ALIGN(ima_get_binary_runtime_size() + kexec_segment_size = ALIGN(ima_get_binary_runtime_size() +
PAGE_SIZE / 2, PAGE_SIZE); PAGE_SIZE / 2, PAGE_SIZE);
if ((kexec_segment_size == ULONG_MAX) || if ((kexec_segment_size == ULONG_MAX) ||
((kexec_segment_size >> PAGE_SHIFT) > totalram_pages / 2)) { ((kexec_segment_size >> PAGE_SHIFT) > totalram_pages() / 2)) {
pr_err("Binary measurement list too large.\n"); pr_err("Binary measurement list too large.\n");
return; return;
} }