pagination: Spring Data JPA Page vs Slice vs keyset Window scrolling, with the SQL and REST shapes

Co-Authored-By: Claude Sonnet 5.5 <[email protected]>
Claude-Session: https://claude.ai/code/session_017ic4b4bLvQVbZweAqqf54x
This commit is contained in:
Claude
2026-11-16 11:13:56 +05:30
parent e3d1b550ae
commit 91938727f2
42 changed files with 1446 additions and 0 deletions
@@ -0,0 +1,11 @@
package com.ankurm.pagination;
import org.springframework.boot.SpringApplication;
import org.springframework.boot.autoconfigure.SpringBootApplication;
@SpringBootApplication
public class PaginationApplication {
public static void main(String[] args) {
SpringApplication.run(PaginationApplication.class, args);
}
}
@@ -0,0 +1,40 @@
package com.ankurm.pagination;
import jakarta.persistence.Entity;
import jakarta.persistence.GeneratedValue;
import jakarta.persistence.GenerationType;
import jakarta.persistence.Id;
import jakarta.persistence.OneToMany;
import java.math.BigDecimal;
import java.util.ArrayList;
import java.util.List;
/** A product in a catalogue. Prices repeat on purpose: ties are what break naive keyset scrolling. */
@Entity
public class Product {
@Id
@GeneratedValue(strategy = GenerationType.IDENTITY)
private Long id;
private String name;
private String category;
private BigDecimal price;
@OneToMany(mappedBy = "product")
private List<Review> reviews = new ArrayList<>();
protected Product() {}
public Product(String name, String category, BigDecimal price) {
this.name = name;
this.category = category;
this.price = price;
}
public Long getId() { return id; }
public String getName() { return name; }
public String getCategory() { return category; }
public BigDecimal getPrice() { return price; }
public List<Review> getReviews() { return reviews; }
}
@@ -0,0 +1,88 @@
package com.ankurm.pagination;
import java.math.BigDecimal;
import java.nio.charset.StandardCharsets;
import java.util.Base64;
import java.util.List;
import java.util.Map;
import org.springframework.dao.InvalidDataAccessApiUsageException;
import org.springframework.data.core.PropertyReferenceException;
import org.springframework.data.domain.Page;
import org.springframework.data.domain.Pageable;
import org.springframework.data.domain.ScrollPosition;
import org.springframework.data.domain.Window;
import org.springframework.data.web.PagedModel;
import org.springframework.http.HttpStatus;
import org.springframework.http.ProblemDetail;
import org.springframework.web.bind.annotation.ExceptionHandler;
import org.springframework.web.bind.annotation.GetMapping;
import org.springframework.web.bind.annotation.RequestMapping;
import org.springframework.web.bind.annotation.RequestParam;
import org.springframework.web.bind.annotation.RestController;
/** The REST shapes the post compares. Entities are never returned directly: they carry lazy collections. */
@RestController
@RequestMapping("/products")
public class ProductController {
/** What the API exposes about a product. */
public record ProductView(Long id, String name, BigDecimal price) {
static ProductView of(Product p) { return new ProductView(p.getId(), p.getName(), p.getPrice()); }
}
/** Keyset response: the items plus an opaque cursor, and no total. */
public record Scrolled(List<ProductView> items, String nextCursor) {}
private final ProductRepository repo;
public ProductController(ProductRepository repo) { this.repo = repo; }
/** 1. Returning a Page directly: Jackson serialises the PageImpl class itself. */
@GetMapping("/raw")
public Page<ProductView> raw(@RequestParam(defaultValue = "books") String category, Pageable pageable) {
return repo.findByCategory(category, pageable).map(ProductView::of);
}
/** 2. Wrapping the Page in PagedModel: a documented, stable JSON shape. */
@GetMapping("/model")
public PagedModel<ProductView> model(@RequestParam(defaultValue = "books") String category, Pageable pageable) {
return new PagedModel<>(repo.findByCategory(category, pageable).map(ProductView::of));
}
/** 3. Keyset over HTTP: the client sends back the cursor it was given, never a page number. */
@GetMapping("/scroll")
public Scrolled scroll(@RequestParam(defaultValue = "books") String category,
@RequestParam(required = false) String after) {
ScrollPosition position = after == null ? ScrollPosition.keyset() : decode(after);
Window<Product> window = repo.findFirst10ByCategoryOrderByIdAsc(category, position);
String next = window.hasNext() ? encode(window.positionAt(window.size() - 1)) : null;
return new Scrolled(window.getContent().stream().map(ProductView::of).toList(), next);
}
/** Shows what Spring MVC resolved from the query string, before any repository is involved. */
@GetMapping("/echo")
public Map<String, Object> echo(Pageable pageable) {
return Map.of("page", pageable.getPageNumber(), "size", pageable.getPageSize(), "sort", pageable.getSort().toString());
}
/**
* A bad sort value becomes a 400 instead of a 500. Two different exceptions can arrive here:
* an unknown property (PropertyReferenceException) and a value that is not a property reference
* at all (InvalidDataAccessApiUsageException). The second is broad, so a real application would
* validate the sort against an allow-list before calling the repository.
*/
@ExceptionHandler({PropertyReferenceException.class, InvalidDataAccessApiUsageException.class})
public ProblemDetail badSort(RuntimeException e) {
return ProblemDetail.forStatusAndDetail(HttpStatus.BAD_REQUEST, e.getMessage());
}
private static String encode(ScrollPosition position) {
Object id = ((org.springframework.data.domain.KeysetScrollPosition) position).getKeys().get("id");
return Base64.getUrlEncoder().withoutPadding().encodeToString(String.valueOf(id).getBytes(StandardCharsets.UTF_8));
}
private static ScrollPosition decode(String cursor) {
long id = Long.parseLong(new String(Base64.getUrlDecoder().decode(cursor), StandardCharsets.UTF_8));
return ScrollPosition.forward(Map.of("id", id));
}
}
@@ -0,0 +1,62 @@
package com.ankurm.pagination;
import java.util.Collection;
import java.util.List;
import org.springframework.data.domain.Page;
import org.springframework.data.domain.Pageable;
import org.springframework.data.domain.ScrollPosition;
import org.springframework.data.domain.Slice;
import org.springframework.data.domain.Window;
import org.springframework.data.jpa.repository.EntityGraph;
import org.springframework.data.jpa.repository.JpaRepository;
import org.springframework.data.jpa.repository.Query;
import org.springframework.data.repository.query.Param;
/** Every pagination form the post talks about, numbered so the post can point at them. */
public interface ProductRepository extends JpaRepository<Product, Long> {
// 1. Page: the data query plus (usually) a count query.
Page<Product> findByCategory(String category, Pageable pageable);
// 2. Slice: the data query only, fetching one extra row to learn whether a next slice exists.
Slice<Product> findSliceByCategory(String category, Pageable pageable);
// 3. List with a Pageable: just the rows, no total, no "has next".
List<Product> findListByCategory(String category, Pageable pageable);
// 4. Keyset (or offset) scrolling: the method name carries the page size, the argument carries the position.
Window<Product> findFirst10ByCategoryOrderByIdAsc(String category, ScrollPosition position);
// 5. Keyset scrolling on a NON-unique column, WITH an id tie-breaker.
Window<Product> findFirst5ByCategoryOrderByPriceAscIdAsc(String category, ScrollPosition position);
// 6. Keyset scrolling on a NON-unique column, WITHOUT a tie-breaker (the broken one).
Window<Product> findFirst5ByCategoryOrderByPriceAsc(String category, ScrollPosition position);
// 7. @Query with a Pageable: sorting is applied to the alias you chose.
@Query("select p from Product p where p.category = :category")
Page<Product> queryByCategory(@Param("category") String category, Pageable pageable);
// 8. The trap: a collection join fetch combined with paging.
@Query("select p from Product p join fetch p.reviews where p.category = :category")
Page<Product> withReviewsInMemoryPaging(@Param("category") String category, Pageable pageable);
// 8b. Same query, but with an explicit count query that counts parents, not joined rows.
@Query(value = "select p from Product p join fetch p.reviews where p.category = :category",
countQuery = "select count(distinct p) from Product p join p.reviews where p.category = :category")
Page<Product> withReviewsCorrectCount(@Param("category") String category, Pageable pageable);
// 8c. An entity graph does the same fetch without a hand-written query.
@EntityGraph(attributePaths = "reviews")
Page<Product> findGraphByCategory(String category, Pageable pageable);
// 9a/9b. The fix for old Hibernate versions: page the ids first, then fetch the collection for exactly those ids.
@Query("select p.id from Product p where p.category = :category")
Page<Long> idsByCategory(@Param("category") String category, Pageable pageable);
@Query("select distinct p from Product p join fetch p.reviews where p.id in :ids order by p.id")
List<Product> fetchWithReviews(@Param("ids") Collection<Long> ids);
// 10. Newest-first feed, keyset style (used by the drift demo).
Window<Product> findFirst5ByCategoryOrderByIdDesc(String category, ScrollPosition position);
}
@@ -0,0 +1,33 @@
package com.ankurm.pagination;
import jakarta.persistence.Entity;
import jakarta.persistence.FetchType;
import jakarta.persistence.GeneratedValue;
import jakarta.persistence.GenerationType;
import jakarta.persistence.Id;
import jakarta.persistence.ManyToOne;
/** A review of a product. Exists to demonstrate what a collection fetch does to paging. */
@Entity
public class Review {
@Id
@GeneratedValue(strategy = GenerationType.IDENTITY)
private Long id;
@ManyToOne(fetch = FetchType.LAZY)
private Product product;
private int rating;
protected Review() {}
public Review(Product product, int rating) {
this.product = product;
this.rating = rating;
}
public Long getId() { return id; }
public Product getProduct() { return product; }
public int getRating() { return rating; }
}
@@ -0,0 +1,24 @@
package com.ankurm.pagination;
import java.util.List;
import java.util.concurrent.CopyOnWriteArrayList;
import org.hibernate.resource.jdbc.spi.StatementInspector;
/**
* Records every SQL string Hibernate is about to send to the database.
* Registered through hibernate.session_factory.statement_inspector in application.properties.
* Hibernate instantiates it itself, so the captured list has to be static.
*/
public class SqlRecorder implements StatementInspector {
private static final List<String> CAPTURED = new CopyOnWriteArrayList<>();
@Override
public String inspect(String sql) {
CAPTURED.add(sql);
return sql;
}
public static void clear() { CAPTURED.clear(); }
public static List<String> captured() { return List.copyOf(CAPTURED); }
}
@@ -0,0 +1,5 @@
spring.datasource.url=jdbc:h2:mem:pagination;DB_CLOSE_DELAY=-1
spring.jpa.hibernate.ddl-auto=create-drop
spring.jpa.open-in-view=false
# Every SQL statement is captured by SqlRecorder (a Hibernate StatementInspector), not by show_sql.
spring.jpa.properties.hibernate.session_factory.statement_inspector=com.ankurm.pagination.SqlRecorder
@@ -0,0 +1,19 @@
package com.ankurm.brokenrepo;
import com.ankurm.pagination.Product;
import org.springframework.data.domain.ScrollPosition;
import org.springframework.data.domain.Window;
import org.springframework.data.jpa.repository.Query;
import org.springframework.data.repository.Repository;
import org.springframework.data.repository.query.Param;
/**
* Deliberately broken: a String-based @Query that returns a Window.
* It lives outside the com.ankurm.pagination package so Spring Boot does not try to register it at startup;
* KeysetScrollingTest builds it by hand and records the real error.
*/
public interface BrokenScrollRepository extends Repository<Product, Long> {
@Query("select p from Product p where p.category = :category order by p.id")
Window<Product> scrollByQuery(@Param("category") String category, ScrollPosition position);
}
@@ -0,0 +1,109 @@
package com.ankurm.pagination;
import static org.assertj.core.api.Assertions.assertThat;
import jakarta.persistence.EntityManager;
import java.util.List;
import org.hibernate.Session;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.extension.ExtendWith;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.boot.data.jpa.test.autoconfigure.DataJpaTest;
import org.springframework.boot.test.system.CapturedOutput;
import org.springframework.boot.test.system.OutputCaptureExtension;
import org.springframework.data.domain.Page;
import org.springframework.data.domain.PageRequest;
import org.springframework.data.domain.Sort;
import org.springframework.test.annotation.DirtiesContext;
/**
* Paging over a collection fetch. The transcript file name carries the Hibernate version that ran, because the
* behaviour differs between 7.3 and 7.4: scripts/old-hibernate.sh re-runs this class on 7.3.13.Final.
*/
@DataJpaTest
@DirtiesContext(classMode = DirtiesContext.ClassMode.BEFORE_EACH_TEST_METHOD)
@ExtendWith(OutputCaptureExtension.class)
class CollectionFetchPagingTest {
@Autowired ProductRepository repo;
@Autowired EntityManager em;
@BeforeEach
void seed() {
TestData.catalogue(em);
TestData.reviews(em);
}
private long managed() { return em.unwrap(Session.class).getStatistics().getEntityCount(); }
private List<String> sqlThenReset() {
List<String> sql = SqlRecorder.captured();
SqlRecorder.clear();
em.clear();
return sql;
}
@Test
void pagingOverACollectionFetch(CapturedOutput out) {
String hibernate = org.hibernate.Version.getVersionString();
boolean pagedInSql = Integer.parseInt(hibernate.split("\\.")[1]) >= 4;
Transcript t = new Transcript("12-collection-fetch-paging-" + hibernate + ".txt");
t.line("Hibernate ORM: %s", hibernate);
t.line("# 25 books, 6 of them with 2 reviews each (12 joined rows). Page of 2 products, join fetch p.reviews.");
Page<Product> page = repo.withReviewsInMemoryPaging("books", PageRequest.of(0, 2, Sort.by("id")));
long entities = managed();
List<String> sql = sqlThenReset();
boolean warned = out.getAll().contains("HHH90003004");
t.line("");
t.line("## method 8: @Query(\"select p from Product p join fetch p.reviews where ...\")");
sql.forEach(t::line);
t.line("products on the page: %s, reviews loaded on them: %d",
page.getContent().stream().map(Product::getName).toList(),
page.getContent().stream().mapToInt(p -> p.getReviews().size()).sum());
t.line("entities held by the persistence context: %d", entities);
t.line("HHH90003004 (applying in memory) logged: %s", warned);
t.line("totalElements reported: %d (products that actually have reviews: 6)", page.getTotalElements());
assertThat(page.getContent()).hasSize(2);
assertThat(page.getTotalElements()).as("the derived count query counts joined rows").isEqualTo(12);
if (pagedInSql) {
assertThat(sql.get(0)).contains("offset ? rows fetch first ? rows only");
assertThat(entities).isEqualTo(6);
assertThat(warned).isFalse();
} else {
assertThat(sql.get(0)).doesNotContain("fetch first");
assertThat(entities).isEqualTo(18);
assertThat(warned).isTrue();
}
Page<Product> counted = repo.withReviewsCorrectCount("books", PageRequest.of(0, 2, Sort.by("id")));
List<String> countedSql = sqlThenReset();
t.line("");
t.line("## method 8b: the same query with countQuery = \"select count(distinct p) ...\"");
t.line(countedSql.get(1));
t.line("totalElements reported: %d, totalPages: %d", counted.getTotalElements(), counted.getTotalPages());
assertThat(counted.getTotalElements()).isEqualTo(6);
Page<Product> graph = repo.findGraphByCategory("books", PageRequest.of(0, 4, Sort.by("id")));
long graphEntities = managed();
List<String> graphSql = sqlThenReset();
t.line("");
t.line("## method 8c: @EntityGraph(attributePaths = \"reviews\") on a derived query, page of 4");
graphSql.forEach(t::line);
t.line("rows: %d, totalElements: %d, entities held by the persistence context: %d",
graph.getContent().size(), graph.getTotalElements(), graphEntities);
assertThat(graph.getTotalElements()).as("the count query ignores the fetch").isEqualTo(25);
Page<Long> ids = repo.idsByCategory("books", PageRequest.of(0, 2, Sort.by("id")));
List<Product> withReviews = repo.fetchWithReviews(ids.getContent());
List<String> twoStepSql = sqlThenReset();
t.line("");
t.line("## methods 9a + 9b: page the ids, then fetch the reviews for exactly those ids");
twoStepSql.forEach(t::line);
t.line("ids on the page: %s, products fetched: %d, reviews fetched: %d", ids.getContent(), withReviews.size(),
withReviews.stream().mapToInt(p -> p.getReviews().size()).sum());
assertThat(withReviews).hasSize(2);
t.write();
}
}
@@ -0,0 +1,329 @@
package com.ankurm.pagination;
import static org.assertj.core.api.Assertions.assertThat;
import com.ankurm.brokenrepo.BrokenScrollRepository;
import jakarta.persistence.EntityManager;
import java.math.BigDecimal;
import java.util.ArrayList;
import java.util.HashSet;
import java.util.List;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.boot.data.jpa.test.autoconfigure.DataJpaTest;
import org.springframework.data.domain.Example;
import org.springframework.data.domain.ExampleMatcher;
import org.springframework.data.domain.Page;
import org.springframework.data.domain.PageRequest;
import org.springframework.data.domain.ScrollPosition;
import org.springframework.data.domain.Sort;
import org.springframework.data.domain.Window;
import org.springframework.data.jpa.repository.support.JpaRepositoryFactory;
import org.springframework.data.support.WindowIterator;
import org.springframework.test.annotation.DirtiesContext;
/** Offset versus keyset scrolling. Every test writes output/NN-*.txt while asserting the same facts. */
@DataJpaTest
@DirtiesContext(classMode = DirtiesContext.ClassMode.BEFORE_EACH_TEST_METHOD)
class KeysetScrollingTest {
@Autowired ProductRepository repo;
@Autowired EntityManager em;
@BeforeEach
void seed() { TestData.catalogue(em); }
private List<String> sqlThenReset() {
List<String> sql = SqlRecorder.captured();
SqlRecorder.clear();
em.clear();
return sql;
}
private static List<String> names(Iterable<Product> products) {
List<String> out = new ArrayList<>();
products.forEach(p -> out.add(p.getName()));
return out;
}
// ---- 05: the SQL of offset paging versus keyset scrolling ----------------------------
@Test
void offsetVersusKeysetSql() {
Transcript t = new Transcript("05-offset-vs-keyset-sql.txt");
repo.findListByCategory("books", PageRequest.of(2, 10, Sort.by("id")));
t.line("# offset paging, third page of 10: PageRequest.of(2, 10, Sort.by(\"id\"))");
List<String> offsetSql = sqlThenReset();
offsetSql.forEach(t::line);
assertThat(offsetSql.get(0)).contains("offset ? rows fetch first ? rows only");
Window<Product> w1 = repo.findFirst10ByCategoryOrderByIdAsc("books", ScrollPosition.keyset());
List<String> sql1 = sqlThenReset();
t.line("");
t.line("# keyset scrolling, first window: ScrollPosition.keyset()");
sql1.forEach(t::line);
t.line("rows: %d, hasNext: %s, position of the last row: %s", w1.size(), w1.hasNext(), w1.positionAt(w1.size() - 1));
assertThat(sql1.get(0)).doesNotContain("offset").doesNotContain("id>?");
Window<Product> w2 = repo.findFirst10ByCategoryOrderByIdAsc("books", w1.positionAt(w1.size() - 1));
List<String> sql2 = sqlThenReset();
t.line("");
t.line("# keyset scrolling, second window: the position from the first window");
sql2.forEach(t::line);
t.line("rows: %d (%s .. %s), hasNext: %s", w2.size(), w2.getContent().get(0).getName(),
w2.getContent().get(w2.size() - 1).getName(), w2.hasNext());
assertThat(sql2.get(0)).contains("p1_0.id>?").doesNotContain("offset");
assertThat(w2.getContent().get(0).getName()).isEqualTo("book-11");
Window<Product> w3 = repo.findFirst10ByCategoryOrderByIdAsc("books", w2.positionAt(w2.size() - 1));
sqlThenReset();
t.line("");
t.line("# third window: %d rows (%s .. %s), hasNext: %s", w3.size(), w3.getContent().get(0).getName(),
w3.getContent().get(w3.size() - 1).getName(), w3.hasNext());
assertThat(w3.hasNext()).isFalse();
WindowIterator<Product> it = WindowIterator
.of(position -> repo.findFirst10ByCategoryOrderByIdAsc("books", position))
.startingAt(ScrollPosition.keyset());
int rows = 0;
while (it.hasNext()) { it.next(); rows++; }
List<String> itSql = sqlThenReset();
t.line("");
t.line("# WindowIterator over all 25 books, windows of 10");
t.line("rows visited: %d, statements: %d", rows, itSql.size());
assertThat(rows).isEqualTo(25);
assertThat(itSql).hasSize(3);
Product probe = new Product(null, "books", null);
Window<Product> fluent = repo.findBy(Example.of(probe, ExampleMatcher.matching().withIgnorePaths("price")),
q -> q.sortBy(Sort.by("id")).limit(5).scroll(ScrollPosition.keyset()));
List<String> fsql = sqlThenReset();
t.line("");
t.line("# Query by Example: findBy(example, q -> q.sortBy(Sort.by(\"id\")).limit(5).scroll(ScrollPosition.keyset()))");
fsql.forEach(t::line);
t.line("rows: %d, hasNext: %s", fluent.size(), fluent.hasNext());
assertThat(fluent.size()).isEqualTo(5);
// Last on purpose: after an offset call, keyset scrolling on the SAME method misbehaves (see test mixingOffsetAndKeysetOnOneMethod).
Window<Product> o = repo.findFirst10ByCategoryOrderByIdAsc("books", ScrollPosition.offset(0));
List<String> osql = sqlThenReset();
t.line("");
t.line("# offset scrolling: ScrollPosition.offset(0) is a position, so the result starts AFTER it");
osql.forEach(t::line);
t.line("first row returned: %s", o.getContent().get(0).getName());
assertThat(o.getContent().get(0).getName()).isEqualTo("book-2");
t.write();
}
// ---- 06: what a deep offset costs ----------------------------------------------------
@Test
void deepOffsetVersusKeysetScanCount() {
em.createQuery("delete from Product").executeUpdate(); // only bulk rows, so scanCount is not padded by the seed
em.createNativeQuery("insert into product(category, name, price) select 'bulk', 'p' || x, 10 from system_range(1, 100000)")
.executeUpdate();
em.flush();
em.clear();
Transcript t = new Transcript("06-deep-offset-scan-count.txt");
t.line("# 100,000 rows in category 'bulk'. H2 EXPLAIN ANALYZE reports scanCount: rows the database read.");
record Probe(String label, String sql) {}
List<Probe> probes = List.of(
new Probe("offset 0, fetch 10",
"explain analyze select id from product where category='bulk' order by id offset 0 rows fetch first 10 rows only"),
new Probe("offset 90000, fetch 10",
"explain analyze select id from product where category='bulk' order by id offset 90000 rows fetch first 10 rows only"),
new Probe("keyset id > 90000, fetch 10",
"explain analyze select id from product where category='bulk' and id>90000 order by id fetch first 10 rows only"));
Pattern scan = Pattern.compile("scanCount: (\\d+)");
int[] counts = new int[3];
for (int i = 0; i < probes.size(); i++) {
Object plan = em.createNativeQuery(probes.get(i).sql()).getSingleResult();
Matcher m = scan.matcher(String.valueOf(plan));
assertThat(m.find()).isTrue();
counts[i] = Integer.parseInt(m.group(1));
t.line("%-28s scanCount: %,d", probes.get(i).label(), counts[i]);
}
assertThat(counts[0]).isEqualTo(10);
assertThat(counts[1]).isEqualTo(90_010);
assertThat(counts[2]).isEqualTo(11);
t.line("");
t.line("These are row counts, not timings. Timings depend on your database, indexes and cache.");
t.write();
}
// ---- 07 / 08: page drift -------------------------------------------------------------
@Test
void driftWhenARowIsInserted() {
Transcript t = new Transcript("07-page-drift-insert.txt");
t.line("# newest-first feed of 25 books, 5 per page; a new book is inserted between the two requests");
Page<Product> o0 = repo.findByCategory("books", PageRequest.of(0, 5, Sort.by(Sort.Direction.DESC, "id")));
Window<Product> k0 = repo.findFirst5ByCategoryOrderByIdDesc("books", ScrollPosition.keyset());
t.line("offset page 0: %s", names(o0.getContent()));
t.line("keyset window 0: %s", names(k0.getContent()));
assertThat(names(o0.getContent())).isEqualTo(names(k0.getContent()));
em.persist(new Product("book-NEW", "books", BigDecimal.TEN));
em.flush();
em.clear();
t.line("-- book-NEW inserted --");
Page<Product> o1 = repo.findByCategory("books", PageRequest.of(1, 5, Sort.by(Sort.Direction.DESC, "id")));
Window<Product> k1 = repo.findFirst5ByCategoryOrderByIdDesc("books", k0.positionAt(k0.size() - 1));
t.line("offset page 1: %s", names(o1.getContent()));
t.line("keyset window 1: %s", names(k1.getContent()));
List<String> seen = new ArrayList<>(names(o0.getContent()));
seen.addAll(names(o1.getContent()));
t.line("offset: book-21 appears on both pages: %s", seen.stream().filter("book-21"::equals).count() == 2);
assertThat(names(o1.getContent()).get(0)).isEqualTo("book-21");
assertThat(names(k1.getContent()).get(0)).isEqualTo("book-20");
t.write();
}
@Test
void driftWhenARowIsDeleted() {
Transcript t = new Transcript("08-page-drift-delete.txt");
t.line("# newest-first feed of 25 books, 5 per page; book-24 is deleted between the two requests");
Page<Product> o0 = repo.findByCategory("books", PageRequest.of(0, 5, Sort.by(Sort.Direction.DESC, "id")));
Window<Product> k0 = repo.findFirst5ByCategoryOrderByIdDesc("books", ScrollPosition.keyset());
t.line("offset page 0: %s", names(o0.getContent()));
t.line("keyset window 0: %s", names(k0.getContent()));
em.createQuery("delete from Product p where p.name = 'book-24'").executeUpdate();
em.clear();
t.line("-- book-24 deleted --");
Page<Product> o1 = repo.findByCategory("books", PageRequest.of(1, 5, Sort.by(Sort.Direction.DESC, "id")));
Window<Product> k1 = repo.findFirst5ByCategoryOrderByIdDesc("books", k0.positionAt(k0.size() - 1));
t.line("offset page 1: %s", names(o1.getContent()));
t.line("keyset window 1: %s", names(k1.getContent()));
t.line("offset skipped book-20: %s", !names(o1.getContent()).contains("book-20"));
assertThat(names(o1.getContent())).doesNotContain("book-20");
assertThat(names(k1.getContent()).get(0)).isEqualTo("book-20");
t.write();
}
// ---- 09: ties and nulls --------------------------------------------------------------
@Test
void ties() {
Transcript t = new Transcript("09-keyset-ties-and-nulls.txt");
t.line("# sorting by price, which repeats: 25 books over 4 distinct prices, pages of 5");
WindowIterator<Product> it = WindowIterator
.of(position -> repo.findFirst5ByCategoryOrderByPriceAsc("books", position))
.startingAt(ScrollPosition.keyset());
List<String> all = new ArrayList<>();
while (it.hasNext()) all.add(it.next().getName());
List<String> sql = sqlThenReset();
t.line("rows visited: %d, distinct rows: %d, statements: %d", all.size(), new HashSet<>(all).size(), sql.size());
t.line("second statement:");
t.line(sql.get(1));
assertThat(all).hasSize(25);
assertThat(new HashSet<>(all)).hasSize(25);
assertThat(sql.get(1)).contains("p1_0.price>? or p1_0.price=? and p1_0.id>?");
t.line("Method 6 names only the price in OrderBy; the id in the statement above was added by Spring Data.");
t.line("");
t.line("# nullable sort column (H2 sorts NULLs first for ASC)");
em.persist(new Product("a", "mixed", new BigDecimal("1")));
em.persist(new Product("b", "mixed", null));
em.persist(new Product("c", "mixed", new BigDecimal("2")));
em.persist(new Product("d", "mixed", null));
em.persist(new Product("e", "mixed", new BigDecimal("3")));
em.flush();
em.clear();
SqlRecorder.clear();
Window<Product> all5 = repo.findFirst5ByCategoryOrderByPriceAsc("mixed", ScrollPosition.keyset());
sqlThenReset();
t.line("full window: %s", all5.getContent().stream().map(p -> p.getName() + ":" + p.getPrice()).toList());
Window<Product> afterB = repo.findFirst5ByCategoryOrderByPriceAsc("mixed", all5.positionAt(0));
List<String> afterSql = sqlThenReset();
t.line("after the first row (price null): %s", names(afterB.getContent()));
t.line(afterSql.get(0));
assertThat(names(afterB.getContent())).containsExactly("d", "a", "c", "e");
assertThat(afterSql.get(0)).contains("p1_0.price is not null or p1_0.price is null and p1_0.id>?");
t.line("That predicate is right where NULLs sort first, as on H2. It was not run on a database that sorts them last.");
t.write();
}
// ---- 10: @Query cannot scroll --------------------------------------------------------
@Test
void stringQueryCannotScroll() {
Transcript t = new Transcript("10-query-scroll-unsupported.txt");
t.line("# a String-based @Query method returning Window<Product>");
try {
new JpaRepositoryFactory(em).getRepository(BrokenScrollRepository.class);
throw new AssertionError("expected the repository to be rejected");
} catch (RuntimeException e) {
t.line(e.getClass().getName());
t.line(e.getMessage());
Throwable root = e;
while (root.getCause() != null) root = root.getCause();
t.line("root cause: %s", root.getMessage());
assertThat(root.getMessage()).contains("Scroll queries are not supported using String-based queries");
}
t.write();
}
// ---- 11: mixing offset and keyset on one method --------------------------------------
/**
* Observed on Spring Data JPA 4.1.2 (and on the 2025.1.8 and 2026.0.0 release trains, see scripts/other-spring-data.sh):
* once a query method has been called with an OFFSET scroll position, later calls with a non-initial KEYSET
* position on that same method are silently run without the keyset predicate. If this test starts failing after an
* upgrade, the behaviour has been fixed and the post should be updated.
*/
@Test
void mixingOffsetAndKeysetOnOneMethod() {
String jpa = org.springframework.data.jpa.repository.JpaRepository.class.getPackage().getImplementationVersion();
Transcript t = new Transcript("11-offset-then-keyset-same-method-" + jpa + ".txt");
t.line("Spring Data JPA: %s", jpa);
t.line("# one fresh context, one repository method: findFirst10ByCategoryOrderByIdAsc(\"books\", position)");
Window<Product> k1 = window(ScrollPosition.keyset());
t.line("1. keyset, initial -> %s", range(k1));
Window<Product> k2 = window(k1.positionAt(9));
t.line("2. keyset, after book-10 -> %s", range(k2));
Window<Product> o1 = window(ScrollPosition.offset());
t.line("3. offset, initial -> %s", range(o1));
Window<Product> o2 = window(o1.positionAt(9));
t.line("4. offset, after row 10 -> %s", range(o2));
SqlRecorder.clear();
Window<Product> k3 = window(k1.positionAt(9));
List<String> sql = sqlThenReset();
t.line("5. keyset, after book-10 again -> %s (step 2 returned %s)", range(k3), range(k2));
t.line(" statement for step 5: %s", sql.get(0));
Window<Product> d1 = repo.findFirst5ByCategoryOrderByIdDesc("books", ScrollPosition.keyset());
Window<Product> d2 = repo.findFirst5ByCategoryOrderByIdDesc("books", d1.positionAt(4));
t.line("");
t.line("# a different method, findFirst5ByCategoryOrderByIdDesc, after the offset calls above");
t.line("keyset initial -> %s, keyset after book-21 -> %s", range(d1), range(d2));
assertThat(range(k2)).isEqualTo("book-11..book-20");
assertThat(range(o2)).isEqualTo("book-11..book-20");
assertThat(range(k3)).isEqualTo("book-1..book-10");
assertThat(sql.get(0)).doesNotContain("p1_0.id>?");
assertThat(range(d2)).isEqualTo("book-20..book-16");
t.write();
}
private Window<Product> window(ScrollPosition position) {
Window<Product> w = repo.findFirst10ByCategoryOrderByIdAsc("books", position);
em.clear();
return w;
}
private static String range(Window<Product> w) {
return w.getContent().get(0).getName() + ".." + w.getContent().get(w.size() - 1).getName();
}
}
@@ -0,0 +1,174 @@
package com.ankurm.pagination;
import static org.assertj.core.api.Assertions.assertThat;
import jakarta.persistence.EntityManager;
import java.util.List;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.boot.data.jpa.test.autoconfigure.DataJpaTest;
import org.springframework.dao.InvalidDataAccessApiUsageException;
import org.springframework.data.core.PropertyReferenceException;
import org.springframework.data.domain.Page;
import org.springframework.data.domain.PageRequest;
import org.springframework.data.domain.Pageable;
import org.springframework.data.domain.Slice;
import org.springframework.data.domain.Sort;
import org.springframework.data.jpa.domain.JpaSort;
import org.springframework.test.annotation.DirtiesContext;
/** Page, Slice, List, the count query, and sorting. Every test writes output/NN-*.txt while asserting the same facts. */
@DataJpaTest
@DirtiesContext(classMode = DirtiesContext.ClassMode.BEFORE_EACH_TEST_METHOD)
class PaginationBasicsTest {
@Autowired ProductRepository repo;
@Autowired EntityManager em;
@BeforeEach
void seed() { TestData.catalogue(em); }
private List<String> sqlThenReset() {
List<String> sql = SqlRecorder.captured();
SqlRecorder.clear();
em.clear();
return sql;
}
private String tryQuerySort(Sort sort) {
try {
repo.queryByCategory("books", PageRequest.of(0, 3, sort));
String sql = SqlRecorder.captured().get(0);
sqlThenReset();
return "accepted: " + sql.substring(sql.indexOf(" order by "));
} catch (InvalidDataAccessApiUsageException e) {
sqlThenReset();
return "rejected (" + e.getClass().getSimpleName() + ")";
}
}
// ---- 01: the three return types ------------------------------------------------------
@Test
void pageSliceAndList() {
Transcript t = new Transcript("01-page-slice-list.txt");
Pageable second = PageRequest.of(1, 10, Sort.by("id"));
Page<Product> page = repo.findByCategory("books", second);
List<String> pageSql = sqlThenReset();
t.line("# Page<Product> findByCategory(\"books\", PageRequest.of(1, 10, Sort.by(\"id\")))");
pageSql.forEach(t::line);
t.line("statements: %d, rows: %d, totalElements: %d, totalPages: %d, hasNext: %s",
pageSql.size(), page.getContent().size(), page.getTotalElements(), page.getTotalPages(), page.hasNext());
assertThat(pageSql).hasSize(2);
assertThat(pageSql.get(1)).startsWith("select count(*)");
assertThat(page.getTotalElements()).isEqualTo(25);
assertThat(page.getTotalPages()).isEqualTo(3);
Slice<Product> slice = repo.findSliceByCategory("books", second);
List<String> sliceSql = sqlThenReset();
t.line("");
t.line("# Slice<Product> findSliceByCategory(\"books\", PageRequest.of(1, 10, Sort.by(\"id\")))");
sliceSql.forEach(t::line);
t.line("statements: %d, rows: %d, hasNext: %s (no totalElements exists on a Slice)",
sliceSql.size(), slice.getContent().size(), slice.hasNext());
assertThat(sliceSql).hasSize(1);
assertThat(slice.hasNext()).isTrue();
List<Product> list = repo.findListByCategory("books", second);
List<String> listSql = sqlThenReset();
t.line("");
t.line("# List<Product> findListByCategory(\"books\", PageRequest.of(1, 10, Sort.by(\"id\")))");
listSql.forEach(t::line);
t.line("statements: %d, rows: %d (no total, no hasNext)", listSql.size(), list.size());
assertThat(listSql).hasSize(1);
t.write();
}
@Test
void sliceKnowsHasNextByFetchingOneExtraRow() {
Transcript t = new Transcript("02-slice-boundary.txt");
t.line("# 5 toys in the table. Does the Slice know whether another slice follows?");
for (int size : new int[] {4, 5, 6}) {
Slice<Product> s = repo.findSliceByCategory("toys", PageRequest.of(0, size, Sort.by("id")));
t.line("slice size %d -> rows returned: %d, hasNext: %s, statements: %d",
size, s.getContent().size(), s.hasNext(), SqlRecorder.captured().size());
if (size == 4) { assertThat(s.hasNext()).isTrue(); assertThat(s.getContent()).hasSize(4); }
else { assertThat(s.hasNext()).isFalse(); }
sqlThenReset();
}
t.write();
}
// ---- 03: when the count query is skipped ---------------------------------------------
@Test
void countQueryIsSometimesSkipped() {
Transcript t = new Transcript("03-count-query-skipped.txt");
t.line("# Page<Product>: how many statements does each request issue? (25 books, 5 toys)");
record Case(String label, String category, int page, int size, int expectedStatements) {}
List<Case> cases = List.of(
new Case("middle page ", "books", 1, 10, 2),
new Case("first page, partial ", "toys", 0, 10, 1),
new Case("last page, partial ", "books", 2, 10, 1),
new Case("full page in the middle", "books", 1, 5, 2),
new Case("last page, exactly full", "books", 4, 5, 2),
new Case("page past the end ", "books", 5, 10, 2));
for (Case c : cases) {
Page<Product> p = repo.findByCategory(c.category(), PageRequest.of(c.page(), c.size(), Sort.by("id")));
int statements = SqlRecorder.captured().size();
t.line("%s page=%d size=%-2d rows=%-2d statements=%d totalElements=%d",
c.label(), c.page(), c.size(), p.getContent().size(), statements, p.getTotalElements());
assertThat(statements).as(c.label()).isEqualTo(c.expectedStatements());
sqlThenReset();
}
t.write();
}
// ---- 04: sorting ---------------------------------------------------------------------
@Test
void sortingAndSortSafety() {
Transcript t = new Transcript("04-sorting.txt");
repo.findByCategory("books", PageRequest.of(0, 3, Sort.by("price").descending().and(Sort.by("id"))));
t.line("# derived query, Sort.by(\"price\").descending().and(Sort.by(\"id\"))");
sqlThenReset().stream().limit(1).forEach(t::line);
t.line("");
t.line("# derived query, unknown property");
try {
repo.findByCategory("books", PageRequest.of(0, 3, Sort.by("nope")));
throw new AssertionError("expected an exception");
} catch (PropertyReferenceException e) {
t.line(e.getClass().getName());
t.line(e.getMessage());
}
repo.queryByCategory("books", PageRequest.of(0, 3, Sort.by("price").descending()));
t.line("");
t.line("# @Query with Sort.by(\"price\").descending() (alias p is applied for you)");
sqlThenReset().forEach(t::line);
t.line("");
t.line("# a value that is not a property reference at all: Sort.by(\"price) desc, (select 1\")");
for (String kind : new String[] {"derived", "@Query"}) {
try {
Pageable bad = PageRequest.of(0, 3, Sort.by("price) desc, (select 1"));
if (kind.equals("derived")) repo.findByCategory("books", bad); else repo.queryByCategory("books", bad);
throw new AssertionError("expected an exception");
} catch (InvalidDataAccessApiUsageException e) {
t.line("%s -> %s", kind, e.getClass().getSimpleName());
t.line(e.getMessage());
}
}
t.line("");
t.line("# @Query with a function-style sort value, three calls in one fresh context");
t.line("1. Sort.by(\"length(name)\") -> %s", tryQuerySort(Sort.by("length(name)")));
t.line("2. JpaSort.unsafe(\"length(name)\") -> %s", tryQuerySort(JpaSort.unsafe("length(name)")));
t.line("3. Sort.by(\"length(name)\") again -> %s", tryQuerySort(Sort.by("length(name)")));
t.write();
}
}
@@ -0,0 +1,108 @@
package com.ankurm.pagination;
import static org.assertj.core.api.Assertions.assertThat;
import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.get;
import tools.jackson.databind.JsonNode;
import tools.jackson.databind.json.JsonMapper;
import jakarta.persistence.EntityManager;
import java.util.ArrayList;
import java.util.List;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.boot.test.context.SpringBootTest;
import org.springframework.boot.webmvc.test.autoconfigure.AutoConfigureMockMvc;
import org.springframework.test.annotation.DirtiesContext;
import org.springframework.test.web.servlet.MockMvc;
import org.springframework.transaction.annotation.Transactional;
/** What a client sees over HTTP: Pageable binding, size caps, JSON shapes, bad sorts, keyset cursors. */
@SpringBootTest
@AutoConfigureMockMvc
@Transactional
@DirtiesContext(classMode = DirtiesContext.ClassMode.BEFORE_EACH_TEST_METHOD)
class RestPagingTest {
@Autowired MockMvc mvc;
@Autowired EntityManager em;
final JsonMapper json = JsonMapper.builder().build();
@BeforeEach
void seed() {
TestData.catalogue(em);
em.flush();
}
private String body(String url) throws Exception {
return mvc.perform(get(url)).andReturn().getResponse().getContentAsString();
}
private int status(String url) throws Exception {
return mvc.perform(get(url)).andReturn().getResponse().getStatus();
}
@Test
void pageableBindingAndCaps() throws Exception {
Transcript t = new Transcript("13-pageable-binding.txt");
for (String q : List.of("", "?page=2&size=5&sort=price,desc&sort=id", "?size=100000", "?page=-3&size=0", "?page=abc")) {
String url = "/products/echo" + q;
t.line("GET %s -> %d %s", url, status(url), body(url));
}
assertThat(body("/products/echo")).contains("\"size\":20").contains("\"page\":0");
assertThat(body("/products/echo?size=100000")).contains("\"size\":2000");
assertThat(body("/products/echo?page=-3&size=0")).contains("\"size\":20").contains("\"page\":0");
t.write();
}
@Test
void pageJsonShapes() throws Exception {
Transcript t = new Transcript("14-page-json-shapes.txt");
String raw = body("/products/raw?size=2");
String model = body("/products/model?size=2");
t.line("GET /products/raw?size=2 (return type Page<ProductView>)");
t.line(raw);
t.line("");
t.line("GET /products/model?size=2 (return type PagedModel<ProductView>)");
t.line(model);
JsonNode m = json.readTree(model);
assertThat(m.has("content")).isTrue();
assertThat(m.get("page").get("totalElements").asInt()).isEqualTo(25);
assertThat(m.get("page").get("totalPages").asInt()).isEqualTo(13);
t.write();
}
@Test
void sortValidation() throws Exception {
Transcript t = new Transcript("15-rest-bad-sort.txt");
for (String q : List.of("sort=price,desc", "sort=nope", "sort=price;drop", "sort=,desc")) {
String url = "/products/raw?size=1&" + q;
t.line("GET %s -> %d", url, status(url));
t.line(" %s", body(url).replaceAll("\\s+", " "));
}
assertThat(status("/products/raw?size=1&sort=nope")).isEqualTo(400);
assertThat(status("/products/raw?size=1&sort=price,desc")).isEqualTo(200);
t.write();
}
@Test
void keysetCursorWalk() throws Exception {
Transcript t = new Transcript("16-keyset-cursor-walk.txt");
List<String> names = new ArrayList<>();
String cursor = null;
int calls = 0;
do {
String url = "/products/scroll" + (cursor == null ? "" : "?after=" + cursor);
String b = body(url);
calls++;
JsonNode n = json.readTree(b);
n.get("items").forEach(i -> names.add(i.get("name").asString()));
t.line("GET %s -> %d items, nextCursor=%s", url, n.get("items").size(), n.get("nextCursor"));
cursor = n.get("nextCursor").isNull() ? null : n.get("nextCursor").asString();
} while (cursor != null);
t.line("calls: %d, items: %d, distinct: %d", calls, names.size(), names.stream().distinct().count());
assertThat(calls).isEqualTo(3);
assertThat(names).hasSize(25).doesNotHaveDuplicates();
t.write();
}
}
@@ -0,0 +1,41 @@
package com.ankurm.pagination;
import static org.assertj.core.api.Assertions.assertThat;
import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.get;
import tools.jackson.databind.JsonNode;
import tools.jackson.databind.json.JsonMapper;
import jakarta.persistence.EntityManager;
import org.junit.jupiter.api.Test;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.boot.test.context.SpringBootTest;
import org.springframework.boot.webmvc.test.autoconfigure.AutoConfigureMockMvc;
import org.springframework.test.annotation.DirtiesContext;
import org.springframework.test.web.servlet.MockMvc;
import org.springframework.transaction.annotation.Transactional;
/** Same endpoint as RestPagingTest, with spring.data.web.pageable.serialization-mode=via-dto. */
@SpringBootTest(properties = "spring.data.web.pageable.serialization-mode=via-dto")
@AutoConfigureMockMvc
@Transactional
@DirtiesContext(classMode = DirtiesContext.ClassMode.BEFORE_EACH_TEST_METHOD)
class RestViaDtoTest {
@Autowired MockMvc mvc;
@Autowired EntityManager em;
@Test
void rawPageBecomesPagedModelShape() throws Exception {
TestData.catalogue(em);
em.flush();
String body = mvc.perform(get("/products/raw?size=2")).andReturn().getResponse().getContentAsString();
Transcript t = new Transcript("17-page-json-via-dto.txt");
t.line("spring.data.web.pageable.serialization-mode=via-dto");
t.line("GET /products/raw?size=2 (return type Page<ProductView>)");
t.line(body);
JsonNode n = JsonMapper.builder().build().readTree(body);
assertThat(n.has("page")).isTrue();
assertThat(n.get("page").get("totalElements").asInt()).isEqualTo(25);
t.write();
}
}
@@ -0,0 +1,30 @@
package com.ankurm.pagination;
import jakarta.persistence.EntityManager;
import java.math.BigDecimal;
/** Deterministic catalogue used by every test. Ids are 1..30 because each test gets a fresh context. */
final class TestData {
private TestData() {}
/** 25 books (ids 1-25, price 10 + i % 4, so prices tie) and 5 toys (ids 26-30). */
static void catalogue(EntityManager em) {
for (int i = 1; i <= 25; i++) em.persist(new Product("book-" + i, "books", BigDecimal.valueOf(10 + (i % 4))));
for (int i = 1; i <= 5; i++) em.persist(new Product("toy-" + i, "toys", BigDecimal.valueOf(5 + i)));
em.flush();
em.clear();
SqlRecorder.clear();
}
/** Two reviews each for books 1-6, so 6 products have reviews and the join has 12 rows. */
static void reviews(EntityManager em) {
for (int i = 1; i <= 6; i++) {
Product p = em.find(Product.class, (long) i);
em.persist(new Review(p, 5));
em.persist(new Review(p, 3));
}
em.flush();
em.clear();
SqlRecorder.clear();
}
}
@@ -0,0 +1,29 @@
package com.ankurm.pagination;
import java.io.IOException;
import java.nio.file.Files;
import java.nio.file.Path;
import java.util.ArrayList;
import java.util.List;
/** Writes output/NN-*.txt while a test asserts the same facts. */
final class Transcript {
private final List<String> lines = new ArrayList<>();
private final Path file;
Transcript(String name) { this.file = Path.of("output/" + name); }
Transcript line(String fmt, Object... args) {
String s = args.length == 0 ? fmt : fmt.formatted(args);
lines.add(s);
System.out.println(s);
return this;
}
void write() {
try {
Files.createDirectories(file.getParent());
Files.writeString(file, String.join("\n", lines) + "\n");
} catch (IOException e) { throw new RuntimeException(e); }
}
}