diff --git a/mm/rmap.c b/mm/rmap.c
index 2540a39eea4a..9701574bb67a 100644
--- a/mm/rmap.c
+++ b/mm/rmap.c
@@ -869,11 +869,11 @@ int page_referenced(struct page *page,
 								vm_flags);
 		if (we_locked)
 			unlock_page(page);
+
+		if (page_test_and_clear_young(page_to_pfn(page)))
+			referenced++;
 	}
 out:
-	if (page_test_and_clear_young(page_to_pfn(page)))
-		referenced++;
-
 	return referenced;
 }