From: Hugh Dickins First of a sequence of six patches, extending Dave McCracken's objrmap to handle anonymous memory too, eliminating pte_chains. Based upon 2.5.65-mm2, the aggregate has anobjrmap 1/6 create include/linux/rmap.h anobjrmap 2/6 free page->mapping for use by anon anobjrmap 3/6 remove pte-pointer-based rmap anobjrmap 4/6 add anonmm to track anonymous pages anonjrmap 5/6 virtual address chains for odd cases anonjrmap 6/6 updates to arches other than i386 I've not done any timings, hope others can do that better than I would. My guess is that Dave has already covered the worst cases, but this should cut the rmap overhead when forking. anobjrmap 1/6 create include/linux/rmap.h Start small: linux/rmap-locking.h has already gathered some declarations unrelated to locking, then the rest of the rmap declarations were over in linux/swap.h: gather them all together in linux/rmap.h. Omit SWAP_ERROR (unused), page_over_rsslimit (non-existent). Fix a couple of missed unlocks in rmap.c page_convert_anon, before the whole function is removed in the next patch. fs/exec.c | 2 - include/linux/rmap-locking.h | 49 ---------------------------- include/linux/rmap.h | 73 +++++++++++++++++++++++++++++++++++++++++++ include/linux/swap.h | 19 ----------- mm/fremap.c | 2 - mm/memory.c | 2 - mm/mremap.c | 2 - mm/rmap.c | 9 +---- mm/swapfile.c | 2 - mm/vmscan.c | 3 - (forwarded by akpm@digeo.com) /dev/null | 49 ---------------------------- 25-akpm/fs/exec.c | 2 - 25-akpm/include/linux/rmap.h | 73 +++++++++++++++++++++++++++++++++++++++++++ 25-akpm/include/linux/swap.h | 19 ----------- 25-akpm/mm/fremap.c | 2 - 25-akpm/mm/memory.c | 2 - 25-akpm/mm/mremap.c | 2 - 25-akpm/mm/rmap.c | 9 +---- 25-akpm/mm/swapfile.c | 2 - 25-akpm/mm/vmscan.c | 3 - 10 files changed, 82 insertions(+), 81 deletions(-) diff -puN fs/exec.c~anobjrmap-1-rmap_h fs/exec.c --- 25/fs/exec.c~anobjrmap-1-rmap_h Thu Mar 20 17:45:59 2003 +++ 25-akpm/fs/exec.c Thu Mar 20 17:45:59 2003 @@ -45,7 +45,7 @@ #include #include #include -#include +#include #include #include diff -puN /dev/null include/linux/rmap.h --- /dev/null Thu Apr 11 07:25:15 2002 +++ 25-akpm/include/linux/rmap.h Thu Mar 20 17:45:59 2003 @@ -0,0 +1,73 @@ +#ifndef _LINUX_RMAP_H +#define _LINUX_RMAP_H +/* + * Declarations for Reverse Mapping functions in mm/rmap.c + * Its structures are declared within that file. + */ +#include +#include + +#ifdef CONFIG_MMU + +struct pte_chain; +struct pte_chain *pte_chain_alloc(int gfp_flags); +void __pte_chain_free(struct pte_chain *pte_chain); + +static inline void pte_chain_free(struct pte_chain *pte_chain) +{ + if (pte_chain) + __pte_chain_free(pte_chain); +} + +struct pte_chain *FASTCALL( + page_add_rmap(struct page *, pte_t *, struct pte_chain *)); +void FASTCALL(page_remove_rmap(struct page *, pte_t *)); +void page_convert_anon(struct page *page); + +/* + * Called from mm/vmscan.c to handle paging out + */ +int FASTCALL(page_referenced(struct page *)); +int FASTCALL(try_to_unmap(struct page *)); + +/* + * Return values of try_to_unmap + */ +#define SWAP_SUCCESS 0 +#define SWAP_AGAIN 1 +#define SWAP_FAIL 2 + +#else /* !CONFIG_MMU */ + +#define page_referenced(page) TestClearPageReferenced(page) + +#endif /* CONFIG_MMU */ + +static inline void pte_chain_lock(struct page *page) +{ + /* + * Assuming the lock is uncontended, this never enters + * the body of the outer loop. If it is contended, then + * within the inner loop a non-atomic test is used to + * busywait with less bus contention for a good time to + * attempt to acquire the lock bit. + */ + preempt_disable(); +#ifdef CONFIG_SMP + while (test_and_set_bit(PG_chainlock, &page->flags)) { + while (test_bit(PG_chainlock, &page->flags)) + cpu_relax(); + } +#endif +} + +static inline void pte_chain_unlock(struct page *page) +{ +#ifdef CONFIG_SMP + smp_mb__before_clear_bit(); + clear_bit(PG_chainlock, &page->flags); +#endif + preempt_enable(); +} + +#endif /* _LINUX_RMAP_H */ diff -puN -L include/linux/rmap-locking.h include/linux/rmap-locking.h~anobjrmap-1-rmap_h /dev/null --- 25/include/linux/rmap-locking.h +++ /dev/null Thu Apr 11 07:25:15 2002 @@ -1,49 +0,0 @@ -/* - * include/linux/rmap-locking.h - * - * Locking primitives for exclusive access to a page's reverse-mapping - * pte chain. - */ - -#include - -struct pte_chain; -extern kmem_cache_t *pte_chain_cache; - -static inline void pte_chain_lock(struct page *page) -{ - /* - * Assuming the lock is uncontended, this never enters - * the body of the outer loop. If it is contended, then - * within the inner loop a non-atomic test is used to - * busywait with less bus contention for a good time to - * attempt to acquire the lock bit. - */ - preempt_disable(); -#ifdef CONFIG_SMP - while (test_and_set_bit(PG_chainlock, &page->flags)) { - while (test_bit(PG_chainlock, &page->flags)) - cpu_relax(); - } -#endif -} - -static inline void pte_chain_unlock(struct page *page) -{ -#ifdef CONFIG_SMP - smp_mb__before_clear_bit(); - clear_bit(PG_chainlock, &page->flags); -#endif - preempt_enable(); -} - -struct pte_chain *pte_chain_alloc(int gfp_flags); -void __pte_chain_free(struct pte_chain *pte_chain); - -static inline void pte_chain_free(struct pte_chain *pte_chain) -{ - if (pte_chain) - __pte_chain_free(pte_chain); -} - -void page_convert_anon(struct page *page); diff -puN include/linux/swap.h~anobjrmap-1-rmap_h include/linux/swap.h --- 25/include/linux/swap.h~anobjrmap-1-rmap_h Thu Mar 20 17:45:59 2003 +++ 25-akpm/include/linux/swap.h Thu Mar 20 17:45:59 2003 @@ -69,7 +69,6 @@ typedef struct { #ifdef __KERNEL__ struct address_space; -struct pte_chain; struct sysinfo; struct writeback_control; struct zone; @@ -167,27 +166,9 @@ extern int try_to_free_pages(struct zone extern int shrink_all_memory(int); extern int vm_swappiness; -/* linux/mm/rmap.c */ #ifdef CONFIG_MMU -int FASTCALL(page_referenced(struct page *)); -struct pte_chain *FASTCALL(page_add_rmap(struct page *, pte_t *, - struct pte_chain *)); -void FASTCALL(page_remove_rmap(struct page *, pte_t *)); -int FASTCALL(try_to_unmap(struct page *)); -int FASTCALL(page_over_rsslimit(struct page *)); - -/* return values of try_to_unmap */ -#define SWAP_SUCCESS 0 -#define SWAP_AGAIN 1 -#define SWAP_FAIL 2 -#define SWAP_ERROR 3 - /* linux/mm/shmem.c */ extern int shmem_unuse(swp_entry_t entry, struct page *page); - -#else -#define page_referenced(page) \ - TestClearPageReferenced(page) #endif /* CONFIG_MMU */ #ifdef CONFIG_SWAP diff -puN mm/fremap.c~anobjrmap-1-rmap_h mm/fremap.c --- 25/mm/fremap.c~anobjrmap-1-rmap_h Thu Mar 20 17:45:59 2003 +++ 25-akpm/mm/fremap.c Thu Mar 20 17:45:59 2003 @@ -11,7 +11,7 @@ #include #include #include -#include +#include #include #include #include diff -puN mm/memory.c~anobjrmap-1-rmap_h mm/memory.c --- 25/mm/memory.c~anobjrmap-1-rmap_h Thu Mar 20 17:45:59 2003 +++ 25-akpm/mm/memory.c Thu Mar 20 17:45:59 2003 @@ -44,7 +44,7 @@ #include #include #include -#include +#include #include #include diff -puN mm/mremap.c~anobjrmap-1-rmap_h mm/mremap.c --- 25/mm/mremap.c~anobjrmap-1-rmap_h Thu Mar 20 17:45:59 2003 +++ 25-akpm/mm/mremap.c Thu Mar 20 17:45:59 2003 @@ -15,7 +15,7 @@ #include #include #include -#include +#include #include #include diff -puN mm/rmap.c~anobjrmap-1-rmap_h mm/rmap.c --- 25/mm/rmap.c~anobjrmap-1-rmap_h Thu Mar 20 17:45:59 2003 +++ 25-akpm/mm/rmap.c Thu Mar 20 17:45:59 2003 @@ -25,7 +25,7 @@ #include #include #include -#include +#include #include #include @@ -677,7 +677,6 @@ out_unlock: * SWAP_SUCCESS - we succeeded in removing all mappings * SWAP_AGAIN - we missed a trylock, try again later * SWAP_FAIL - the page is unswappable - * SWAP_ERROR - an error occurred */ int try_to_unmap(struct page * page) { @@ -754,9 +753,6 @@ int try_to_unmap(struct page * page) case SWAP_FAIL: ret = SWAP_FAIL; goto out; - case SWAP_ERROR: - ret = SWAP_ERROR; - goto out; } } } @@ -812,6 +808,7 @@ retry: */ if (mapcount < page->pte.mapcount) { pte_chain_unlock(page); + up(&mapping->i_shared_sem); goto retry; } else if ((mapcount > page->pte.mapcount) && (mapcount > 1)) { mapcount = page->pte.mapcount; @@ -827,7 +824,7 @@ retry: SetPageAnon(page); if (mapcount == 0) - goto out; + goto out_unlock; else if (mapcount == 1) { SetPageDirect(page); page->pte.direct = 0; diff -puN mm/swapfile.c~anobjrmap-1-rmap_h mm/swapfile.c --- 25/mm/swapfile.c~anobjrmap-1-rmap_h Thu Mar 20 17:45:59 2003 +++ 25-akpm/mm/swapfile.c Thu Mar 20 17:45:59 2003 @@ -20,7 +20,7 @@ #include #include #include -#include +#include #include #include diff -puN mm/vmscan.c~anobjrmap-1-rmap_h mm/vmscan.c --- 25/mm/vmscan.c~anobjrmap-1-rmap_h Thu Mar 20 17:45:59 2003 +++ 25-akpm/mm/vmscan.c Thu Mar 20 17:45:59 2003 @@ -26,7 +26,7 @@ #include #include #include -#include +#include #include #include @@ -284,7 +284,6 @@ shrink_list(struct list_head *page_list, */ if (page_mapped(page) && mapping) { switch (try_to_unmap(page)) { - case SWAP_ERROR: case SWAP_FAIL: pte_chain_unlock(page); goto activate_locked; _